From 605198854104e04f343630e2fcec8fe78103be3e Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 10 Aug 2017 20:18:59 +0300 Subject: [PATCH 1/9] IGNITE-6029 Record serializer refactoring and initial stuff for Record V2 serialization. --- .../wal/FileWriteAheadLogManager.java | 11 +- .../persistence/wal/RecordDataSerializer.java | 38 + .../serializer/RecordDataV1Serializer.java | 1528 +++++++++++++++ .../serializer/RecordDataV2Serializer.java | 38 + .../wal/serializer/RecordV1Serializer.java | 1647 ++--------------- .../wal/serializer/RecordV2Serializer.java | 112 ++ .../wal/serializer/io/RecordIO.java | 40 + 7 files changed, 1873 insertions(+), 1541 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordDataSerializer.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/io/RecordIO.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 17db8f8fd8825..170237c8e7ef5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -63,7 +63,10 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; +import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordDataV1Serializer; +import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordDataV2Serializer; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer; +import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer; import org.apache.ignite.internal.processors.timeout.GridTimeoutObject; import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor; import org.apache.ignite.internal.util.GridUnsafe; @@ -261,7 +264,7 @@ public FileWriteAheadLogManager(@NotNull final GridKernalContext ctx) { "write ahead log archive directory" ); - serializer = new RecordV1Serializer(cctx); + serializer = new RecordV1Serializer(new RecordDataV1Serializer(cctx)); GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)cctx.database(); @@ -1014,7 +1017,11 @@ static RecordSerializer forVersion(GridCacheSharedContext cctx, int ver) throws switch (ver) { case 1: - return new RecordV1Serializer(cctx); + return new RecordV1Serializer(new RecordDataV1Serializer(cctx)); + + case 2: + RecordDataV2Serializer dataV2Serializer = new RecordDataV2Serializer(new RecordDataV1Serializer(cctx)); + return new RecordV2Serializer(dataV2Serializer); default: throw new IgniteCheckedException("Failed to create a serializer with the given version " + diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordDataSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordDataSerializer.java new file mode 100644 index 0000000000000..cf04729524d44 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordDataSerializer.java @@ -0,0 +1,38 @@ +package org.apache.ignite.internal.processors.cache.persistence.wal; + +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.pagemem.wal.record.WALRecord; + +/** + * Interface to provide size, read and write operations with WAL records + * including only record without headers and other meta information + */ +public interface RecordDataSerializer { + /** + * + * @param record + * @return + * @throws IgniteCheckedException + */ + int size(WALRecord record) throws IgniteCheckedException; + + /** + * + * @param type + * @param in + * @return + * @throws IOException + * @throws IgniteCheckedException + */ + WALRecord readRecord(WALRecord.RecordType type, ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException; + + /** + * + * @param record + * @param buf + * @throws IgniteCheckedException + */ + void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException; +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java new file mode 100644 index 0000000000000..7a46a08a413f7 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java @@ -0,0 +1,1528 @@ +package org.apache.ignite.internal.processors.cache.persistence.wal.serializer; + +import java.io.DataInput; +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.wal.record.CacheState; +import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord; +import org.apache.ignite.internal.pagemem.wal.record.DataEntry; +import org.apache.ignite.internal.pagemem.wal.record.DataRecord; +import org.apache.ignite.internal.pagemem.wal.record.LazyDataEntry; +import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord; +import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot; +import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertFragmentRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageRemoveRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageSetFreeListPageRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.FixLeftmostChildRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.FixRemoveId; +import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.InnerReplaceRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.InsertRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.MergeRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageAddRootRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageCutRootRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootInlineRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastAllocatedIndex; +import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastSuccessfulFullSnapshotId; +import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastSuccessfulSnapshotId; +import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateNextSnapshotId; +import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.NewRootInitRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.PageListMetaResetCountRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListAddPageRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListInitNewPageRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListRemovePageRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetNextRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetPreviousRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.RecycleRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.RemoveRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.ReplaceRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.SplitExistingPageRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.SplitForwardPageRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.TrackingPageDeltaRecord; +import org.apache.ignite.internal.processors.cache.CacheObject; +import org.apache.ignite.internal.processors.cache.CacheObjectContext; +import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.GridCacheOperation; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.KeyCacheObject; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO; +import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; +import org.apache.ignite.internal.processors.cache.persistence.wal.RecordDataSerializer; +import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; +import org.apache.ignite.internal.util.typedef.internal.U; + +public class RecordDataV1Serializer implements RecordDataSerializer { + /** */ + static final int HEADER_RECORD_DATA_SIZE = /*Magic*/8 + /*Version*/4; + + /** Cache shared context */ + private final GridCacheSharedContext cctx; + + /** Size of page used for PageMemory regions */ + private final int pageSize; + + /** Cache object processor to reading {@link DataEntry DataEntries} */ + private final IgniteCacheObjectProcessor co; + + public RecordDataV1Serializer(GridCacheSharedContext cctx) { + this.cctx = cctx; + + this.co = cctx.kernalContext().cacheObjects(); + this.pageSize = cctx.database().pageSize(); + } + + @Override + public int size(WALRecord record) throws IgniteCheckedException { + switch (record.type()) { + case PAGE_RECORD: + assert record instanceof PageSnapshot; + + PageSnapshot pageRec = (PageSnapshot)record; + + return pageRec.pageData().length + 12; + + case CHECKPOINT_RECORD: + CheckpointRecord cpRec = (CheckpointRecord)record; + + assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer : + "Invalid WAL record: " + cpRec; + + int cacheStatesSize = cacheStatesSize(cpRec.cacheGroupStates()); + + FileWALPointer walPtr = (FileWALPointer)cpRec.checkpointMark(); + + return 18 + cacheStatesSize + (walPtr == null ? 0 : 16); + + case META_PAGE_INIT: + return /*cache ID*/4 + /*page ID*/8 + /*ioType*/2 + /*ioVer*/2 + /*tree root*/8 + /*reuse root*/8; + + case PARTITION_META_PAGE_UPDATE_COUNTERS: + return /*cache ID*/4 + /*page ID*/8 + /*upd cntr*/8 + /*rmv id*/8 + /*part size*/4 + /*counters page id*/8 + /*state*/ 1 + + /*allocatedIdxCandidate*/ 4; + + case MEMORY_RECOVERY: + return 8; + + case PARTITION_DESTROY: + return /*cacheId*/4 + /*partId*/4; + + case DATA_RECORD: + DataRecord dataRec = (DataRecord)record; + + return 4 + dataSize(dataRec); + + case HEADER_RECORD: + return HEADER_RECORD_DATA_SIZE; + + case DATA_PAGE_INSERT_RECORD: + DataPageInsertRecord diRec = (DataPageInsertRecord)record; + + return 4 + 8 + 2 + diRec.payload().length; + + case DATA_PAGE_UPDATE_RECORD: + DataPageUpdateRecord uRec = (DataPageUpdateRecord)record; + + return 4 + 8 + 2 + 4 + + uRec.payload().length; + + case DATA_PAGE_INSERT_FRAGMENT_RECORD: + final DataPageInsertFragmentRecord difRec = (DataPageInsertFragmentRecord)record; + + return 4 + 8 + 8 + 4 + difRec.payloadSize(); + + case DATA_PAGE_REMOVE_RECORD: + return 4 + 8 + 1; + + case DATA_PAGE_SET_FREE_LIST_PAGE: + return 4 + 8 + 8; + + case INIT_NEW_PAGE_RECORD: + return 4 + 8 + 2 + 2 + 8; + + case BTREE_META_PAGE_INIT_ROOT: + return 4 + 8 + 8; + + case BTREE_META_PAGE_INIT_ROOT2: + return 4 + 8 + 8 + 2; + + case BTREE_META_PAGE_ADD_ROOT: + return 4 + 8 + 8; + + case BTREE_META_PAGE_CUT_ROOT: + return 4 + 8; + + case BTREE_INIT_NEW_ROOT: + NewRootInitRecord riRec = (NewRootInitRecord)record; + + return 4 + 8 + 8 + 2 + 2 + 8 + 8 + riRec.io().getItemSize(); + + case BTREE_PAGE_RECYCLE: + return 4 + 8 + 8; + + case BTREE_PAGE_INSERT: + InsertRecord inRec = (InsertRecord)record; + + return 4 + 8 + 2 + 2 + 2 + 8 + inRec.io().getItemSize(); + + case BTREE_FIX_LEFTMOST_CHILD: + return 4 + 8 + 8; + + case BTREE_FIX_COUNT: + return 4 + 8 + 2; + + case BTREE_PAGE_REPLACE: + ReplaceRecord rRec = (ReplaceRecord)record; + + return 4 + 8 + 2 + 2 + 2 + rRec.io().getItemSize(); + + case BTREE_PAGE_REMOVE: + return 4 + 8 + 2 + 2; + + case BTREE_PAGE_INNER_REPLACE: + return 4 + 8 + 2 + 8 + 2 + 8; + + case BTREE_FORWARD_PAGE_SPLIT: + return 4 + 8 + 8 + 2 + 2 + 8 + 2 + 2; + + case BTREE_EXISTING_PAGE_SPLIT: + return 4 + 8 + 2 + 8; + + case BTREE_PAGE_MERGE: + return 4 + 8 + 8 + 2 + 8 + 1; + + case BTREE_FIX_REMOVE_ID: + return 4 + 8 + 8; + + case PAGES_LIST_SET_NEXT: + return 4 + 8 + 8; + + case PAGES_LIST_SET_PREVIOUS: + return 4 + 8 + 8; + + case PAGES_LIST_INIT_NEW_PAGE: + return 4 + 8 + 4 + 4 + 8 + 8 + 8; + + case PAGES_LIST_ADD_PAGE: + return 4 + 8 + 8; + + case PAGES_LIST_REMOVE_PAGE: + return 4 + 8 + 8; + + case TRACKING_PAGE_DELTA: + return 4 + 8 + 8 + 8 + 8; + + case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID: + return 4 + 8 + 8 + 8; + + case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID: + return 4 + 8 + 8; + + case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID: + return 4 + 8 + 8; + + case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX: + return 4 + 8 + 4; + + case PART_META_UPDATE_STATE: + return /*cacheId*/ 4 + /*partId*/ 4 + /*State*/1 + /*Update Counter*/ 8; + + case PAGE_LIST_META_RESET_COUNT_RECORD: + return /*cacheId*/ 4 + /*pageId*/ 8; + + case SWITCH_SEGMENT_RECORD: + return 0; + + default: + throw new UnsupportedOperationException("Type: " + record.type()); + } + } + + @Override + public WALRecord readRecord(WALRecord.RecordType type, ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException { + WALRecord res; + + switch (type) { + case PAGE_RECORD: + byte[] arr = new byte[pageSize]; + + int cacheId = in.readInt(); + long pageId = in.readLong(); + + in.readFully(arr); + + res = new PageSnapshot(new FullPageId(pageId, cacheId), arr); + + break; + + case CHECKPOINT_RECORD: + long msb = in.readLong(); + long lsb = in.readLong(); + boolean hasPtr = in.readByte() != 0; + int idx = hasPtr ? in.readInt() : 0; + int offset = hasPtr ? in.readInt() : 0; + int len = hasPtr ? in.readInt() : 0; + + Map states = readPartitionStates(in); + + boolean end = in.readByte() != 0; + + FileWALPointer walPtr = hasPtr ? new FileWALPointer(idx, offset, len) : null; + + CheckpointRecord cpRec = new CheckpointRecord(new UUID(msb, lsb), walPtr, end); + + cpRec.cacheGroupStates(states); + + res = cpRec; + + break; + + case META_PAGE_INIT: + cacheId = in.readInt(); + pageId = in.readLong(); + + int ioType = in.readUnsignedShort(); + int ioVer = in.readUnsignedShort(); + long treeRoot = in.readLong(); + long reuseListRoot = in.readLong(); + + res = new MetaPageInitRecord(cacheId, pageId, ioType, ioVer, treeRoot, reuseListRoot); + + break; + + case PARTITION_META_PAGE_UPDATE_COUNTERS: + cacheId = in.readInt(); + pageId = in.readLong(); + + long updCntr = in.readLong(); + long rmvId = in.readLong(); + int partSize = in.readInt(); + long countersPageId = in.readLong(); + byte state = in.readByte(); + int allocatedIdxCandidate = in.readInt(); + + res = new MetaPageUpdatePartitionDataRecord(cacheId, pageId, updCntr, rmvId, partSize, countersPageId, state, allocatedIdxCandidate); + + break; + + case MEMORY_RECOVERY: + long ts = in.readLong(); + + res = new MemoryRecoveryRecord(ts); + + break; + + case PARTITION_DESTROY: + cacheId = in.readInt(); + int partId = in.readInt(); + + res = new PartitionDestroyRecord(cacheId, partId); + + break; + + case DATA_RECORD: + int entryCnt = in.readInt(); + + List entries = new ArrayList<>(entryCnt); + + for (int i = 0; i < entryCnt; i++) + entries.add(readDataEntry(in)); + + res = new DataRecord(entries); + + break; + + case HEADER_RECORD: + long magic = in.readLong(); + + if (magic != HeaderRecord.MAGIC) + throw new EOFException("Magic is corrupted [exp=" + U.hexLong(HeaderRecord.MAGIC) + + ", actual=" + U.hexLong(magic) + ']'); + + int ver = in.readInt(); + + res = new HeaderRecord(ver); + + break; + + case DATA_PAGE_INSERT_RECORD: { + cacheId = in.readInt(); + pageId = in.readLong(); + + int size = in.readUnsignedShort(); + + in.ensure(size); + + byte[] payload = new byte[size]; + + in.readFully(payload); + + res = new DataPageInsertRecord(cacheId, pageId, payload); + + break; + } + + case DATA_PAGE_UPDATE_RECORD: { + cacheId = in.readInt(); + pageId = in.readLong(); + + int itemId = in.readInt(); + + int size = in.readUnsignedShort(); + + in.ensure(size); + + byte[] payload = new byte[size]; + + in.readFully(payload); + + res = new DataPageUpdateRecord(cacheId, pageId, itemId, payload); + + break; + } + + case DATA_PAGE_INSERT_FRAGMENT_RECORD: { + cacheId = in.readInt(); + pageId = in.readLong(); + + final long lastLink = in.readLong(); + final int payloadSize = in.readInt(); + + final byte[] payload = new byte[payloadSize]; + + in.readFully(payload); + + res = new DataPageInsertFragmentRecord(cacheId, pageId, payload, lastLink); + + break; + } + + case DATA_PAGE_REMOVE_RECORD: + cacheId = in.readInt(); + pageId = in.readLong(); + + int itemId = in.readUnsignedByte(); + + res = new DataPageRemoveRecord(cacheId, pageId, itemId); + + break; + + case DATA_PAGE_SET_FREE_LIST_PAGE: + cacheId = in.readInt(); + pageId = in.readLong(); + + long freeListPage = in.readLong(); + + res = new DataPageSetFreeListPageRecord(cacheId, pageId, freeListPage); + + break; + + case INIT_NEW_PAGE_RECORD: + cacheId = in.readInt(); + pageId = in.readLong(); + + ioType = in.readUnsignedShort(); + ioVer = in.readUnsignedShort(); + long virtualPageId = in.readLong(); + + res = new InitNewPageRecord(cacheId, pageId, ioType, ioVer, virtualPageId); + + break; + + case BTREE_META_PAGE_INIT_ROOT: + cacheId = in.readInt(); + pageId = in.readLong(); + + long rootId = in.readLong(); + + res = new MetaPageInitRootRecord(cacheId, pageId, rootId); + + break; + + case BTREE_META_PAGE_INIT_ROOT2: + cacheId = in.readInt(); + pageId = in.readLong(); + + long rootId2 = in.readLong(); + int inlineSize = in.readShort(); + + res = new MetaPageInitRootInlineRecord(cacheId, pageId, rootId2, inlineSize); + + break; + + case BTREE_META_PAGE_ADD_ROOT: + cacheId = in.readInt(); + pageId = in.readLong(); + + rootId = in.readLong(); + + res = new MetaPageAddRootRecord(cacheId, pageId, rootId); + + break; + + case BTREE_META_PAGE_CUT_ROOT: + cacheId = in.readInt(); + pageId = in.readLong(); + + res = new MetaPageCutRootRecord(cacheId, pageId); + + break; + + case BTREE_INIT_NEW_ROOT: + cacheId = in.readInt(); + pageId = in.readLong(); + + rootId = in.readLong(); + ioType = in.readUnsignedShort(); + ioVer = in.readUnsignedShort(); + long leftId = in.readLong(); + long rightId = in.readLong(); + + BPlusIO io = BPlusIO.getBPlusIO(ioType, ioVer); + + byte[] rowBytes = new byte[io.getItemSize()]; + + in.readFully(rowBytes); + + res = new NewRootInitRecord<>(cacheId, pageId, rootId, (BPlusInnerIO)io, leftId, rowBytes, rightId); + + break; + + case BTREE_PAGE_RECYCLE: + cacheId = in.readInt(); + pageId = in.readLong(); + + long newPageId = in.readLong(); + + res = new RecycleRecord(cacheId, pageId, newPageId); + + break; + + case BTREE_PAGE_INSERT: + cacheId = in.readInt(); + pageId = in.readLong(); + + ioType = in.readUnsignedShort(); + ioVer = in.readUnsignedShort(); + int itemIdx = in.readUnsignedShort(); + rightId = in.readLong(); + + io = BPlusIO.getBPlusIO(ioType, ioVer); + + rowBytes = new byte[io.getItemSize()]; + + in.readFully(rowBytes); + + res = new InsertRecord<>(cacheId, pageId, io, itemIdx, rowBytes, rightId); + + break; + + case BTREE_FIX_LEFTMOST_CHILD: + cacheId = in.readInt(); + pageId = in.readLong(); + + rightId = in.readLong(); + + res = new FixLeftmostChildRecord(cacheId, pageId, rightId); + + break; + + case BTREE_FIX_COUNT: + cacheId = in.readInt(); + pageId = in.readLong(); + + int cnt = in.readUnsignedShort(); + + res = new FixCountRecord(cacheId, pageId, cnt); + + break; + + case BTREE_PAGE_REPLACE: + cacheId = in.readInt(); + pageId = in.readLong(); + + ioType = in.readUnsignedShort(); + ioVer = in.readUnsignedShort(); + itemIdx = in.readUnsignedShort(); + + io = BPlusIO.getBPlusIO(ioType, ioVer); + + rowBytes = new byte[io.getItemSize()]; + + in.readFully(rowBytes); + + res = new ReplaceRecord<>(cacheId, pageId, io, rowBytes, itemIdx); + + break; + + case BTREE_PAGE_REMOVE: + cacheId = in.readInt(); + pageId = in.readLong(); + + itemIdx = in.readUnsignedShort(); + cnt = in.readUnsignedShort(); + + res = new RemoveRecord(cacheId, pageId, itemIdx, cnt); + + break; + + case BTREE_PAGE_INNER_REPLACE: + cacheId = in.readInt(); + pageId = in.readLong(); + + int dstIdx = in.readUnsignedShort(); + long srcPageId = in.readLong(); + int srcIdx = in.readUnsignedShort(); + rmvId = in.readLong(); + + res = new InnerReplaceRecord<>(cacheId, pageId, dstIdx, srcPageId, srcIdx, rmvId); + + break; + + case BTREE_FORWARD_PAGE_SPLIT: + cacheId = in.readInt(); + pageId = in.readLong(); + + long fwdId = in.readLong(); + ioType = in.readUnsignedShort(); + ioVer = in.readUnsignedShort(); + srcPageId = in.readLong(); + int mid = in.readUnsignedShort(); + cnt = in.readUnsignedShort(); + + res = new SplitForwardPageRecord(cacheId, pageId, fwdId, ioType, ioVer, srcPageId, mid, cnt); + + break; + + case BTREE_EXISTING_PAGE_SPLIT: + cacheId = in.readInt(); + pageId = in.readLong(); + + mid = in.readUnsignedShort(); + fwdId = in.readLong(); + + res = new SplitExistingPageRecord(cacheId, pageId, mid, fwdId); + + break; + + case BTREE_PAGE_MERGE: + cacheId = in.readInt(); + pageId = in.readLong(); + + long prntId = in.readLong(); + int prntIdx = in.readUnsignedShort(); + rightId = in.readLong(); + boolean emptyBranch = in.readBoolean(); + + res = new MergeRecord<>(cacheId, pageId, prntId, prntIdx, rightId, emptyBranch); + + break; + + case BTREE_FIX_REMOVE_ID: + cacheId = in.readInt(); + pageId = in.readLong(); + + rmvId = in.readLong(); + + res = new FixRemoveId(cacheId, pageId, rmvId); + + break; + + case PAGES_LIST_SET_NEXT: + cacheId = in.readInt(); + pageId = in.readLong(); + long nextPageId = in.readLong(); + + res = new PagesListSetNextRecord(cacheId, pageId, nextPageId); + + break; + + case PAGES_LIST_SET_PREVIOUS: + cacheId = in.readInt(); + pageId = in.readLong(); + long prevPageId = in.readLong(); + + res = new PagesListSetPreviousRecord(cacheId, pageId, prevPageId); + + break; + + case PAGES_LIST_INIT_NEW_PAGE: + cacheId = in.readInt(); + pageId = in.readLong(); + ioType = in.readInt(); + ioVer = in.readInt(); + newPageId = in.readLong(); + prevPageId = in.readLong(); + long addDataPageId = in.readLong(); + + res = new PagesListInitNewPageRecord(cacheId, pageId, ioType, ioVer, newPageId, prevPageId, addDataPageId); + + break; + + case PAGES_LIST_ADD_PAGE: + cacheId = in.readInt(); + pageId = in.readLong(); + long dataPageId = in.readLong(); + + res = new PagesListAddPageRecord(cacheId, pageId, dataPageId); + + break; + + case PAGES_LIST_REMOVE_PAGE: + cacheId = in.readInt(); + pageId = in.readLong(); + long rmvdPageId = in.readLong(); + + res = new PagesListRemovePageRecord(cacheId, pageId, rmvdPageId); + + break; + + case TRACKING_PAGE_DELTA: + cacheId = in.readInt(); + pageId = in.readLong(); + + long pageIdToMark = in.readLong(); + long nextSnapshotId0 = in.readLong(); + long lastSuccessfulSnapshotId0 = in.readLong(); + + res = new TrackingPageDeltaRecord(cacheId, pageId, pageIdToMark, nextSnapshotId0, lastSuccessfulSnapshotId0); + + break; + + case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID: + cacheId = in.readInt(); + pageId = in.readLong(); + + long nextSnapshotId = in.readLong(); + + res = new MetaPageUpdateNextSnapshotId(cacheId, pageId, nextSnapshotId); + + break; + + case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID: + cacheId = in.readInt(); + pageId = in.readLong(); + + long lastSuccessfulFullSnapshotId = in.readLong(); + + res = new MetaPageUpdateLastSuccessfulFullSnapshotId(cacheId, pageId, lastSuccessfulFullSnapshotId); + + break; + + case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID: + cacheId = in.readInt(); + pageId = in.readLong(); + + long lastSuccessfulSnapshotId = in.readLong(); + long lastSuccessfulSnapshotTag = in.readLong(); + + res = new MetaPageUpdateLastSuccessfulSnapshotId(cacheId, pageId, lastSuccessfulSnapshotId, lastSuccessfulSnapshotTag); + + break; + + case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX: + cacheId = in.readInt(); + pageId = in.readLong(); + + int lastAllocatedIdx = in.readInt(); + + res = new MetaPageUpdateLastAllocatedIndex(cacheId, pageId, lastAllocatedIdx); + + break; + + case PART_META_UPDATE_STATE: + cacheId = in.readInt(); + partId = in.readInt(); + + state = in.readByte(); + + long updateCounter = in.readLong(); + + res = new PartitionMetaStateRecord(cacheId, partId, GridDhtPartitionState.fromOrdinal(state), updateCounter); + + break; + + case PAGE_LIST_META_RESET_COUNT_RECORD: + cacheId = in.readInt(); + pageId = in.readLong(); + + res = new PageListMetaResetCountRecord(cacheId, pageId); + break; + + case SWITCH_SEGMENT_RECORD: + throw new EOFException("END OF SEGMENT"); + + default: + throw new UnsupportedOperationException("Type: " + type); + } + + return res; + } + + @Override + public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { + switch (record.type()) { + case PAGE_RECORD: + PageSnapshot snap = (PageSnapshot)record; + + buf.putInt(snap.fullPageId().groupId()); + buf.putLong(snap.fullPageId().pageId()); + buf.put(snap.pageData()); + + break; + + case MEMORY_RECOVERY: + MemoryRecoveryRecord memoryRecoveryRecord = (MemoryRecoveryRecord)record; + + buf.putLong(memoryRecoveryRecord.time()); + + break; + + case PARTITION_DESTROY: + PartitionDestroyRecord partDestroy = (PartitionDestroyRecord)record; + + buf.putInt(partDestroy.groupId()); + buf.putInt(partDestroy.partitionId()); + + break; + + case META_PAGE_INIT: + MetaPageInitRecord updRootsRec = (MetaPageInitRecord)record; + + buf.putInt(updRootsRec.groupId()); + buf.putLong(updRootsRec.pageId()); + + buf.putShort((short)updRootsRec.ioType()); + buf.putShort((short)updRootsRec.ioVersion()); + buf.putLong(updRootsRec.treeRoot()); + buf.putLong(updRootsRec.reuseListRoot()); + + break; + + case PARTITION_META_PAGE_UPDATE_COUNTERS: + MetaPageUpdatePartitionDataRecord partDataRec = (MetaPageUpdatePartitionDataRecord)record; + + buf.putInt(partDataRec.groupId()); + buf.putLong(partDataRec.pageId()); + + buf.putLong(partDataRec.updateCounter()); + buf.putLong(partDataRec.globalRemoveId()); + buf.putInt(partDataRec.partitionSize()); + buf.putLong(partDataRec.countersPageId()); + buf.put(partDataRec.state()); + buf.putInt(partDataRec.allocatedIndexCandidate()); + + break; + + case CHECKPOINT_RECORD: + CheckpointRecord cpRec = (CheckpointRecord)record; + + assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer : + "Invalid WAL record: " + cpRec; + + FileWALPointer walPtr = (FileWALPointer)cpRec.checkpointMark(); + UUID cpId = cpRec.checkpointId(); + + buf.putLong(cpId.getMostSignificantBits()); + buf.putLong(cpId.getLeastSignificantBits()); + + buf.put(walPtr == null ? (byte)0 : 1); + + if (walPtr != null) { + buf.putLong(walPtr.index()); + buf.putInt(walPtr.fileOffset()); + buf.putInt(walPtr.length()); + } + + putCacheStates(buf, cpRec.cacheGroupStates()); + + buf.put(cpRec.end() ? (byte)1 : 0); + + break; + + case DATA_RECORD: + DataRecord dataRec = (DataRecord)record; + + buf.putInt(dataRec.writeEntries().size()); + + for (DataEntry dataEntry : dataRec.writeEntries()) + putDataEntry(buf, dataEntry); + + break; + + case HEADER_RECORD: + buf.putLong(HeaderRecord.MAGIC); + + buf.putInt(((HeaderRecord)record).version()); + + break; + + case DATA_PAGE_INSERT_RECORD: + DataPageInsertRecord diRec = (DataPageInsertRecord)record; + + buf.putInt(diRec.groupId()); + buf.putLong(diRec.pageId()); + + buf.putShort((short)diRec.payload().length); + + buf.put(diRec.payload()); + + break; + + case DATA_PAGE_UPDATE_RECORD: + DataPageUpdateRecord uRec = (DataPageUpdateRecord)record; + + buf.putInt(uRec.groupId()); + buf.putLong(uRec.pageId()); + buf.putInt(uRec.itemId()); + + buf.putShort((short)uRec.payload().length); + + buf.put(uRec.payload()); + + break; + + case DATA_PAGE_INSERT_FRAGMENT_RECORD: + final DataPageInsertFragmentRecord difRec = (DataPageInsertFragmentRecord)record; + + buf.putInt(difRec.groupId()); + buf.putLong(difRec.pageId()); + + buf.putLong(difRec.lastLink()); + buf.putInt(difRec.payloadSize()); + buf.put(difRec.payload()); + + break; + + case DATA_PAGE_REMOVE_RECORD: + DataPageRemoveRecord drRec = (DataPageRemoveRecord)record; + + buf.putInt(drRec.groupId()); + buf.putLong(drRec.pageId()); + + buf.put((byte)drRec.itemId()); + + break; + + case DATA_PAGE_SET_FREE_LIST_PAGE: + DataPageSetFreeListPageRecord freeListRec = (DataPageSetFreeListPageRecord)record; + + buf.putInt(freeListRec.groupId()); + buf.putLong(freeListRec.pageId()); + + buf.putLong(freeListRec.freeListPage()); + + break; + + case INIT_NEW_PAGE_RECORD: + InitNewPageRecord inpRec = (InitNewPageRecord)record; + + buf.putInt(inpRec.groupId()); + buf.putLong(inpRec.pageId()); + + buf.putShort((short)inpRec.ioType()); + buf.putShort((short)inpRec.ioVersion()); + buf.putLong(inpRec.newPageId()); + + break; + + case BTREE_META_PAGE_INIT_ROOT: + MetaPageInitRootRecord imRec = (MetaPageInitRootRecord)record; + + buf.putInt(imRec.groupId()); + buf.putLong(imRec.pageId()); + + buf.putLong(imRec.rootId()); + + break; + + case BTREE_META_PAGE_INIT_ROOT2: + MetaPageInitRootInlineRecord imRec2 = (MetaPageInitRootInlineRecord)record; + + buf.putInt(imRec2.groupId()); + buf.putLong(imRec2.pageId()); + + buf.putLong(imRec2.rootId()); + + buf.putShort((short)imRec2.inlineSize()); + break; + + case BTREE_META_PAGE_ADD_ROOT: + MetaPageAddRootRecord arRec = (MetaPageAddRootRecord)record; + + buf.putInt(arRec.groupId()); + buf.putLong(arRec.pageId()); + + buf.putLong(arRec.rootId()); + + break; + + case BTREE_META_PAGE_CUT_ROOT: + MetaPageCutRootRecord crRec = (MetaPageCutRootRecord)record; + + buf.putInt(crRec.groupId()); + buf.putLong(crRec.pageId()); + + break; + + case BTREE_INIT_NEW_ROOT: + NewRootInitRecord riRec = (NewRootInitRecord)record; + + buf.putInt(riRec.groupId()); + buf.putLong(riRec.pageId()); + + buf.putLong(riRec.rootId()); + buf.putShort((short)riRec.io().getType()); + buf.putShort((short)riRec.io().getVersion()); + buf.putLong(riRec.leftId()); + buf.putLong(riRec.rightId()); + + putRow(buf, riRec.rowBytes()); + + break; + + case BTREE_PAGE_RECYCLE: + RecycleRecord recRec = (RecycleRecord)record; + + buf.putInt(recRec.groupId()); + buf.putLong(recRec.pageId()); + + buf.putLong(recRec.newPageId()); + + break; + + case BTREE_PAGE_INSERT: + InsertRecord inRec = (InsertRecord)record; + + buf.putInt(inRec.groupId()); + buf.putLong(inRec.pageId()); + + buf.putShort((short)inRec.io().getType()); + buf.putShort((short)inRec.io().getVersion()); + buf.putShort((short)inRec.index()); + buf.putLong(inRec.rightId()); + + putRow(buf, inRec.rowBytes()); + + break; + + case BTREE_FIX_LEFTMOST_CHILD: + FixLeftmostChildRecord flRec = (FixLeftmostChildRecord)record; + + buf.putInt(flRec.groupId()); + buf.putLong(flRec.pageId()); + + buf.putLong(flRec.rightId()); + + break; + + case BTREE_FIX_COUNT: + FixCountRecord fcRec = (FixCountRecord)record; + + buf.putInt(fcRec.groupId()); + buf.putLong(fcRec.pageId()); + + buf.putShort((short)fcRec.count()); + + break; + + case BTREE_PAGE_REPLACE: + ReplaceRecord rRec = (ReplaceRecord)record; + + buf.putInt(rRec.groupId()); + buf.putLong(rRec.pageId()); + + buf.putShort((short)rRec.io().getType()); + buf.putShort((short)rRec.io().getVersion()); + buf.putShort((short)rRec.index()); + + putRow(buf, rRec.rowBytes()); + + break; + + case BTREE_PAGE_REMOVE: + RemoveRecord rmRec = (RemoveRecord)record; + + buf.putInt(rmRec.groupId()); + buf.putLong(rmRec.pageId()); + + buf.putShort((short)rmRec.index()); + buf.putShort((short)rmRec.count()); + + break; + + case BTREE_PAGE_INNER_REPLACE: + InnerReplaceRecord irRec = (InnerReplaceRecord)record; + + buf.putInt(irRec.groupId()); + buf.putLong(irRec.pageId()); + + buf.putShort((short)irRec.destinationIndex()); + buf.putLong(irRec.sourcePageId()); + buf.putShort((short)irRec.sourceIndex()); + buf.putLong(irRec.removeId()); + + break; + + case BTREE_FORWARD_PAGE_SPLIT: + SplitForwardPageRecord sfRec = (SplitForwardPageRecord)record; + + buf.putInt(sfRec.groupId()); + buf.putLong(sfRec.pageId()); + + buf.putLong(sfRec.forwardId()); + buf.putShort((short)sfRec.ioType()); + buf.putShort((short)sfRec.ioVersion()); + buf.putLong(sfRec.sourcePageId()); + buf.putShort((short)sfRec.middleIndex()); + buf.putShort((short)sfRec.count()); + + break; + + case BTREE_EXISTING_PAGE_SPLIT: + SplitExistingPageRecord seRec = (SplitExistingPageRecord)record; + + buf.putInt(seRec.groupId()); + buf.putLong(seRec.pageId()); + + buf.putShort((short)seRec.middleIndex()); + buf.putLong(seRec.forwardId()); + + break; + + case BTREE_PAGE_MERGE: + MergeRecord mRec = (MergeRecord)record; + + buf.putInt(mRec.groupId()); + buf.putLong(mRec.pageId()); + + buf.putLong(mRec.parentId()); + buf.putShort((short)mRec.parentIndex()); + buf.putLong(mRec.rightId()); + buf.put((byte)(mRec.isEmptyBranch() ? 1 : 0)); + + break; + + case PAGES_LIST_SET_NEXT: + PagesListSetNextRecord plNextRec = (PagesListSetNextRecord)record; + + buf.putInt(plNextRec.groupId()); + buf.putLong(plNextRec.pageId()); + + buf.putLong(plNextRec.nextPageId()); + + break; + + case PAGES_LIST_SET_PREVIOUS: + PagesListSetPreviousRecord plPrevRec = (PagesListSetPreviousRecord)record; + + buf.putInt(plPrevRec.groupId()); + buf.putLong(plPrevRec.pageId()); + + buf.putLong(plPrevRec.previousPageId()); + + break; + + case PAGES_LIST_INIT_NEW_PAGE: + PagesListInitNewPageRecord plNewRec = (PagesListInitNewPageRecord)record; + + buf.putInt(plNewRec.groupId()); + buf.putLong(plNewRec.pageId()); + buf.putInt(plNewRec.ioType()); + buf.putInt(plNewRec.ioVersion()); + buf.putLong(plNewRec.newPageId()); + + buf.putLong(plNewRec.previousPageId()); + buf.putLong(plNewRec.dataPageId()); + + break; + + case PAGES_LIST_ADD_PAGE: + PagesListAddPageRecord plAddRec = (PagesListAddPageRecord)record; + + buf.putInt(plAddRec.groupId()); + buf.putLong(plAddRec.pageId()); + + buf.putLong(plAddRec.dataPageId()); + + break; + + case PAGES_LIST_REMOVE_PAGE: + PagesListRemovePageRecord plRmvRec = (PagesListRemovePageRecord)record; + + buf.putInt(plRmvRec.groupId()); + buf.putLong(plRmvRec.pageId()); + + buf.putLong(plRmvRec.removedPageId()); + + break; + + case BTREE_FIX_REMOVE_ID: + FixRemoveId frRec = (FixRemoveId)record; + + buf.putInt(frRec.groupId()); + buf.putLong(frRec.pageId()); + + buf.putLong(frRec.removeId()); + + break; + + case TRACKING_PAGE_DELTA: + TrackingPageDeltaRecord tpDelta = (TrackingPageDeltaRecord)record; + + buf.putInt(tpDelta.groupId()); + buf.putLong(tpDelta.pageId()); + + buf.putLong(tpDelta.pageIdToMark()); + buf.putLong(tpDelta.nextSnapshotId()); + buf.putLong(tpDelta.lastSuccessfulSnapshotId()); + + break; + + case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID: + MetaPageUpdateNextSnapshotId mpUpdateNextSnapshotId = (MetaPageUpdateNextSnapshotId)record; + + buf.putInt(mpUpdateNextSnapshotId.groupId()); + buf.putLong(mpUpdateNextSnapshotId.pageId()); + + buf.putLong(mpUpdateNextSnapshotId.nextSnapshotId()); + + break; + + case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID: + MetaPageUpdateLastSuccessfulFullSnapshotId mpUpdateLastSuccFullSnapshotId = + (MetaPageUpdateLastSuccessfulFullSnapshotId)record; + + buf.putInt(mpUpdateLastSuccFullSnapshotId.groupId()); + buf.putLong(mpUpdateLastSuccFullSnapshotId.pageId()); + + buf.putLong(mpUpdateLastSuccFullSnapshotId.lastSuccessfulFullSnapshotId()); + + break; + + case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID: + MetaPageUpdateLastSuccessfulSnapshotId mpUpdateLastSuccSnapshotId = + (MetaPageUpdateLastSuccessfulSnapshotId)record; + + buf.putInt(mpUpdateLastSuccSnapshotId.groupId()); + buf.putLong(mpUpdateLastSuccSnapshotId.pageId()); + + buf.putLong(mpUpdateLastSuccSnapshotId.lastSuccessfulSnapshotId()); + buf.putLong(mpUpdateLastSuccSnapshotId.lastSuccessfulSnapshotTag()); + + break; + + case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX: + MetaPageUpdateLastAllocatedIndex mpUpdateLastAllocatedIdx = + (MetaPageUpdateLastAllocatedIndex) record; + + buf.putInt(mpUpdateLastAllocatedIdx.groupId()); + buf.putLong(mpUpdateLastAllocatedIdx.pageId()); + + buf.putInt(mpUpdateLastAllocatedIdx.lastAllocatedIndex()); + + break; + + case PART_META_UPDATE_STATE: + PartitionMetaStateRecord partMetaStateRecord = (PartitionMetaStateRecord) record; + + buf.putInt(partMetaStateRecord.groupId()); + + buf.putInt(partMetaStateRecord.partitionId()); + + buf.put(partMetaStateRecord.state()); + + buf.putLong(partMetaStateRecord.updateCounter()); + + break; + + case PAGE_LIST_META_RESET_COUNT_RECORD: + PageListMetaResetCountRecord pageListMetaResetCntRecord = (PageListMetaResetCountRecord) record; + + buf.putInt(pageListMetaResetCntRecord.groupId()); + buf.putLong(pageListMetaResetCntRecord.pageId()); + + break; + + default: + throw new UnsupportedOperationException("Type: " + record.type()); + } + } + + /** + * @param buf Buffer to write to. + * @param entry Data entry. + */ + private static void putDataEntry(ByteBuffer buf, DataEntry entry) throws IgniteCheckedException { + buf.putInt(entry.cacheId()); + + if (!entry.key().putValue(buf)) + throw new AssertionError(); + + if (entry.value() == null) + buf.putInt(-1); + else if (!entry.value().putValue(buf)) + throw new AssertionError(); + + buf.put((byte)entry.op().ordinal()); + + putVersion(buf, entry.nearXidVersion(), true); + putVersion(buf, entry.writeVersion(), false); + + buf.putInt(entry.partitionId()); + buf.putLong(entry.partitionCounter()); + buf.putLong(entry.expireTime()); + } + + /** + * @param states Cache states. + */ + private static void putCacheStates(ByteBuffer buf, Map states) { + buf.putShort((short)states.size()); + + for (Map.Entry entry : states.entrySet()) { + buf.putInt(entry.getKey()); + + CacheState state = entry.getValue(); + + // Need 2 bytes for the number of partitions. + buf.putShort((short)state.size()); + + for (int i = 0; i < state.size(); i++) { + buf.putShort((short)state.partitionByIndex(i)); + + buf.putLong(state.partitionSizeByIndex(i)); + buf.putLong(state.partitionCounterByIndex(i)); + } + } + } + + /** + * @param buf Buffer. + * @param ver Version to write. + * @param allowNull Is {@code null}version allowed. + */ + private static void putVersion(ByteBuffer buf, GridCacheVersion ver, boolean allowNull) { + CacheVersionIO.write(buf, ver, allowNull); + } + + /** + * @param buf Buffer. + * @param rowBytes Row bytes. + */ + @SuppressWarnings("unchecked") + private static void putRow(ByteBuffer buf, byte[] rowBytes) { + assert rowBytes.length > 0; + + buf.put(rowBytes); + } + + /** + * @param in Input to read from. + * @return Read entry. + */ + private DataEntry readDataEntry(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException { + int cacheId = in.readInt(); + + int keySize = in.readInt(); + byte keyType = in.readByte(); + byte[] keyBytes = new byte[keySize]; + in.readFully(keyBytes); + + int valSize = in.readInt(); + + byte valType = 0; + byte[] valBytes = null; + + if (valSize >= 0) { + valType = in.readByte(); + valBytes = new byte[valSize]; + in.readFully(valBytes); + } + + byte ord = in.readByte(); + + GridCacheOperation op = GridCacheOperation.fromOrdinal(ord & 0xFF); + + GridCacheVersion nearXidVer = readVersion(in, true); + GridCacheVersion writeVer = readVersion(in, false); + + int partId = in.readInt(); + long partCntr = in.readLong(); + long expireTime = in.readLong(); + + GridCacheContext cacheCtx = cctx.cacheContext(cacheId); + + if (cacheCtx != null) { + CacheObjectContext coCtx = cacheCtx.cacheObjectContext(); + + KeyCacheObject key = co.toKeyCacheObject(coCtx, keyType, keyBytes); + CacheObject val = valBytes != null ? co.toCacheObject(coCtx, valType, valBytes) : null; + + return new DataEntry( + cacheId, + key, + val, + op, + nearXidVer, + writeVer, + expireTime, + partId, + partCntr + ); + } + else + return new LazyDataEntry( + cctx, + cacheId, + keyType, + keyBytes, + valType, + valBytes, + op, + nearXidVer, + writeVer, + expireTime, + partId, + partCntr); + } + + /** + * @param buf Buffer to read from. + * @return Read map. + */ + private Map readPartitionStates(DataInput buf) throws IOException { + int caches = buf.readShort() & 0xFFFF; + + if (caches == 0) + return Collections.emptyMap(); + + Map states = new HashMap<>(caches, 1.0f); + + for (int i = 0; i < caches; i++) { + int cacheId = buf.readInt(); + + int parts = buf.readShort() & 0xFFFF; + + CacheState state = new CacheState(parts); + + for (int p = 0; p < parts; p++) { + int partId = buf.readShort() & 0xFFFF; + long size = buf.readLong(); + long partCntr = buf.readLong(); + + state.addPartitionState(partId, size, partCntr); + } + + states.put(cacheId, state); + } + + return states; + } + + /** + * Changes the buffer position by the number of read bytes. + * + * @param in Data input to read from. + * @param allowNull Is {@code null}version allowed. + * @return Read cache version. + */ + private GridCacheVersion readVersion(ByteBufferBackedDataInput in, boolean allowNull) throws IOException { + // To be able to read serialization protocol version. + in.ensure(1); + + try { + int size = CacheVersionIO.readSize(in.buffer(), allowNull); + + in.ensure(size); + + return CacheVersionIO.read(in.buffer(), allowNull); + } + catch (IgniteCheckedException e) { + throw new IOException(e); + } + } + + /** + * @param dataRec Data record to serialize. + * @return Full data record size. + * @throws IgniteCheckedException If failed to obtain the length of one of the entries. + */ + private int dataSize(DataRecord dataRec) throws IgniteCheckedException { + int sz = 0; + + for (DataEntry entry : dataRec.writeEntries()) + sz += entrySize(entry); + + return sz; + } + + /** + * @param entry Entry to get size for. + * @return Entry size. + * @throws IgniteCheckedException If failed to get key or value bytes length. + */ + private int entrySize(DataEntry entry) throws IgniteCheckedException { + GridCacheContext cctx = this.cctx.cacheContext(entry.cacheId()); + CacheObjectContext coCtx = cctx.cacheObjectContext(); + + return + /*cache ID*/4 + + /*key*/entry.key().valueBytesLength(coCtx) + + /*value*/(entry.value() == null ? 4 : entry.value().valueBytesLength(coCtx)) + + /*op*/1 + + /*near xid ver*/CacheVersionIO.size(entry.nearXidVersion(), true) + + /*write ver*/CacheVersionIO.size(entry.writeVersion(), false) + + /*part ID*/4 + + /*expire Time*/8 + + /*part cnt*/8; + } + + /** + * @param states Partition states. + * @return Size required to write partition states. + */ + private int cacheStatesSize(Map states) { + // Need 4 bytes for the number of caches. + int size = 2; + + for (Map.Entry entry : states.entrySet()) { + // Cache ID. + size += 4; + + // Need 2 bytes for the number of partitions. + size += 2; + + CacheState state = entry.getValue(); + + // 2 bytes partition ID, size and counter per partition. + size += 18 * state.size(); + } + + return size; + } + +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java new file mode 100644 index 0000000000000..575baa86981b6 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java @@ -0,0 +1,38 @@ +package org.apache.ignite.internal.processors.cache.persistence.wal.serializer; + +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput; +import org.apache.ignite.internal.processors.cache.persistence.wal.RecordDataSerializer; + +/** + * Record data V2 serializer. + */ +public class RecordDataV2Serializer implements RecordDataSerializer { + + private final RecordDataV1Serializer delegateSerializer; + + public RecordDataV2Serializer(RecordDataV1Serializer delegateSerializer) { + this.delegateSerializer = delegateSerializer; + } + + @Override + public int size(WALRecord record) throws IgniteCheckedException { + //TODO: Implementation will be changed. + return delegateSerializer.size(record); + } + + @Override + public WALRecord readRecord(WALRecord.RecordType type, ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException { + //TODO: Implementation will be changed. + return delegateSerializer.readRecord(type, in); + } + + @Override + public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { + //TODO: Implementation will be changed. + delegateSerializer.writeRecord(record, buf); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java index 663fe0c0bb560..f834afd30472c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java @@ -21,83 +21,19 @@ import java.io.EOFException; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteSystemProperties; -import org.apache.ignite.internal.pagemem.FullPageId; import org.apache.ignite.internal.pagemem.wal.WALPointer; -import org.apache.ignite.internal.pagemem.wal.record.CacheState; -import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord; -import org.apache.ignite.internal.pagemem.wal.record.DataEntry; -import org.apache.ignite.internal.pagemem.wal.record.DataRecord; -import org.apache.ignite.internal.pagemem.wal.record.LazyDataEntry; -import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord; -import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType; -import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertFragmentRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageRemoveRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageSetFreeListPageRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.FixLeftmostChildRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.FixRemoveId; -import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.InnerReplaceRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.InsertRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.MergeRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageAddRootRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageCutRootRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootInlineRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastAllocatedIndex; -import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastSuccessfulFullSnapshotId; -import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastSuccessfulSnapshotId; -import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateNextSnapshotId; -import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.NewRootInitRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.PageListMetaResetCountRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListAddPageRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListInitNewPageRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListRemovePageRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetNextRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetPreviousRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.RecycleRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.RemoveRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.ReplaceRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.SplitExistingPageRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.SplitForwardPageRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.TrackingPageDeltaRecord; -import org.apache.ignite.internal.processors.cache.CacheObject; -import org.apache.ignite.internal.processors.cache.CacheObjectContext; -import org.apache.ignite.internal.processors.cache.GridCacheContext; -import org.apache.ignite.internal.processors.cache.GridCacheOperation; -import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; -import org.apache.ignite.internal.processors.cache.KeyCacheObject; -import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO; -import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO; -import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO; import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput; import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; import org.apache.ignite.internal.processors.cache.persistence.wal.RecordSerializer; import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; -import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState; -import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; -import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; +import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.io.RecordIO; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.U; import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC; @@ -122,1280 +58,87 @@ public class RecordV1Serializer implements RecordSerializer { private static final int CRC_SIZE = 4; /** */ - public static final int HEADER_RECORD_SIZE = REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + /*Magic*/8 + /*Version*/4 + CRC_SIZE; + public static final int HEADER_RECORD_SIZE = REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + CRC_SIZE + RecordDataV1Serializer.HEADER_RECORD_DATA_SIZE; - /** Cache shared context */ - private GridCacheSharedContext cctx; - - /** Size of page used for PageMemory regions */ - private int pageSize; - - /** Cache object processor to reading {@link DataEntry DataEntries} */ - private IgniteCacheObjectProcessor co; - - /** Skip CRC calculation/check flag */ - private boolean skipCrc = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CRC, false); - - /** - * @param cctx Cache shared context. - */ - public RecordV1Serializer(GridCacheSharedContext cctx) { - this.cctx = cctx; - - co = cctx.kernalContext().cacheObjects(); - pageSize = cctx.database().pageSize(); - } - - /** {@inheritDoc} */ - @Override public int version() { - return 1; - } - - /** {@inheritDoc} */ - @SuppressWarnings("CastConflictsWithInstanceof") - @Override public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { - assert record.size() > 0 && buf.remaining() >= record.size() : record.size(); - - int startPos = buf.position(); - - buf.put((byte)(record.type().ordinal() + 1)); - - putPosition(buf, (FileWALPointer)record.position()); - - switch (record.type()) { - case PAGE_RECORD: - PageSnapshot snap = (PageSnapshot)record; - - buf.putInt(snap.fullPageId().groupId()); - buf.putLong(snap.fullPageId().pageId()); - buf.put(snap.pageData()); - - break; - - case MEMORY_RECOVERY: - MemoryRecoveryRecord memoryRecoveryRecord = (MemoryRecoveryRecord)record; - - buf.putLong(memoryRecoveryRecord.time()); - - break; - - case PARTITION_DESTROY: - PartitionDestroyRecord partDestroy = (PartitionDestroyRecord)record; - - buf.putInt(partDestroy.groupId()); - buf.putInt(partDestroy.partitionId()); - - break; - - case META_PAGE_INIT: - MetaPageInitRecord updRootsRec = (MetaPageInitRecord)record; - - buf.putInt(updRootsRec.groupId()); - buf.putLong(updRootsRec.pageId()); - - buf.putShort((short)updRootsRec.ioType()); - buf.putShort((short)updRootsRec.ioVersion()); - buf.putLong(updRootsRec.treeRoot()); - buf.putLong(updRootsRec.reuseListRoot()); - - break; - - case PARTITION_META_PAGE_UPDATE_COUNTERS: - MetaPageUpdatePartitionDataRecord partDataRec = (MetaPageUpdatePartitionDataRecord)record; - - buf.putInt(partDataRec.groupId()); - buf.putLong(partDataRec.pageId()); - - buf.putLong(partDataRec.updateCounter()); - buf.putLong(partDataRec.globalRemoveId()); - buf.putInt(partDataRec.partitionSize()); - buf.putLong(partDataRec.countersPageId()); - buf.put(partDataRec.state()); - buf.putInt(partDataRec.allocatedIndexCandidate()); - - break; - - case CHECKPOINT_RECORD: - CheckpointRecord cpRec = (CheckpointRecord)record; - - assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer : - "Invalid WAL record: " + cpRec; - - FileWALPointer walPtr = (FileWALPointer)cpRec.checkpointMark(); - UUID cpId = cpRec.checkpointId(); - - buf.putLong(cpId.getMostSignificantBits()); - buf.putLong(cpId.getLeastSignificantBits()); - - buf.put(walPtr == null ? (byte)0 : 1); - - if (walPtr != null) { - buf.putLong(walPtr.index()); - buf.putInt(walPtr.fileOffset()); - buf.putInt(walPtr.length()); - } - - putCacheStates(buf, cpRec.cacheGroupStates()); - - buf.put(cpRec.end() ? (byte)1 : 0); - - break; - - case DATA_RECORD: - DataRecord dataRec = (DataRecord)record; - - buf.putInt(dataRec.writeEntries().size()); - - for (DataEntry dataEntry : dataRec.writeEntries()) - putDataEntry(buf, dataEntry); - - break; - - case HEADER_RECORD: - buf.putLong(HeaderRecord.MAGIC); - - buf.putInt(((HeaderRecord)record).version()); - - break; - - case DATA_PAGE_INSERT_RECORD: - DataPageInsertRecord diRec = (DataPageInsertRecord)record; - - buf.putInt(diRec.groupId()); - buf.putLong(diRec.pageId()); - - buf.putShort((short)diRec.payload().length); - - buf.put(diRec.payload()); - - break; - - case DATA_PAGE_UPDATE_RECORD: - DataPageUpdateRecord uRec = (DataPageUpdateRecord)record; - - buf.putInt(uRec.groupId()); - buf.putLong(uRec.pageId()); - buf.putInt(uRec.itemId()); - - buf.putShort((short)uRec.payload().length); - - buf.put(uRec.payload()); - - break; - - case DATA_PAGE_INSERT_FRAGMENT_RECORD: - final DataPageInsertFragmentRecord difRec = (DataPageInsertFragmentRecord)record; - - buf.putInt(difRec.groupId()); - buf.putLong(difRec.pageId()); - - buf.putLong(difRec.lastLink()); - buf.putInt(difRec.payloadSize()); - buf.put(difRec.payload()); - - break; - - case DATA_PAGE_REMOVE_RECORD: - DataPageRemoveRecord drRec = (DataPageRemoveRecord)record; - - buf.putInt(drRec.groupId()); - buf.putLong(drRec.pageId()); - - buf.put((byte)drRec.itemId()); - - break; - - case DATA_PAGE_SET_FREE_LIST_PAGE: - DataPageSetFreeListPageRecord freeListRec = (DataPageSetFreeListPageRecord)record; - - buf.putInt(freeListRec.groupId()); - buf.putLong(freeListRec.pageId()); - - buf.putLong(freeListRec.freeListPage()); - - break; - - case INIT_NEW_PAGE_RECORD: - InitNewPageRecord inpRec = (InitNewPageRecord)record; - - buf.putInt(inpRec.groupId()); - buf.putLong(inpRec.pageId()); - - buf.putShort((short)inpRec.ioType()); - buf.putShort((short)inpRec.ioVersion()); - buf.putLong(inpRec.newPageId()); - - break; - - case BTREE_META_PAGE_INIT_ROOT: - MetaPageInitRootRecord imRec = (MetaPageInitRootRecord)record; - - buf.putInt(imRec.groupId()); - buf.putLong(imRec.pageId()); - - buf.putLong(imRec.rootId()); - - break; - - case BTREE_META_PAGE_INIT_ROOT2: - MetaPageInitRootInlineRecord imRec2 = (MetaPageInitRootInlineRecord)record; - - buf.putInt(imRec2.groupId()); - buf.putLong(imRec2.pageId()); - - buf.putLong(imRec2.rootId()); - - buf.putShort((short)imRec2.inlineSize()); - break; - - case BTREE_META_PAGE_ADD_ROOT: - MetaPageAddRootRecord arRec = (MetaPageAddRootRecord)record; - - buf.putInt(arRec.groupId()); - buf.putLong(arRec.pageId()); - - buf.putLong(arRec.rootId()); - - break; - - case BTREE_META_PAGE_CUT_ROOT: - MetaPageCutRootRecord crRec = (MetaPageCutRootRecord)record; - - buf.putInt(crRec.groupId()); - buf.putLong(crRec.pageId()); - - break; - - case BTREE_INIT_NEW_ROOT: - NewRootInitRecord riRec = (NewRootInitRecord)record; - - buf.putInt(riRec.groupId()); - buf.putLong(riRec.pageId()); - - buf.putLong(riRec.rootId()); - buf.putShort((short)riRec.io().getType()); - buf.putShort((short)riRec.io().getVersion()); - buf.putLong(riRec.leftId()); - buf.putLong(riRec.rightId()); - - putRow(buf, riRec.rowBytes()); - - break; - - case BTREE_PAGE_RECYCLE: - RecycleRecord recRec = (RecycleRecord)record; - - buf.putInt(recRec.groupId()); - buf.putLong(recRec.pageId()); - - buf.putLong(recRec.newPageId()); - - break; - - case BTREE_PAGE_INSERT: - InsertRecord inRec = (InsertRecord)record; - - buf.putInt(inRec.groupId()); - buf.putLong(inRec.pageId()); - - buf.putShort((short)inRec.io().getType()); - buf.putShort((short)inRec.io().getVersion()); - buf.putShort((short)inRec.index()); - buf.putLong(inRec.rightId()); - - putRow(buf, inRec.rowBytes()); - - break; - - case BTREE_FIX_LEFTMOST_CHILD: - FixLeftmostChildRecord flRec = (FixLeftmostChildRecord)record; - - buf.putInt(flRec.groupId()); - buf.putLong(flRec.pageId()); - - buf.putLong(flRec.rightId()); - - break; - - case BTREE_FIX_COUNT: - FixCountRecord fcRec = (FixCountRecord)record; - - buf.putInt(fcRec.groupId()); - buf.putLong(fcRec.pageId()); - - buf.putShort((short)fcRec.count()); - - break; - - case BTREE_PAGE_REPLACE: - ReplaceRecord rRec = (ReplaceRecord)record; - - buf.putInt(rRec.groupId()); - buf.putLong(rRec.pageId()); - - buf.putShort((short)rRec.io().getType()); - buf.putShort((short)rRec.io().getVersion()); - buf.putShort((short)rRec.index()); - - putRow(buf, rRec.rowBytes()); - - break; - - case BTREE_PAGE_REMOVE: - RemoveRecord rmRec = (RemoveRecord)record; - - buf.putInt(rmRec.groupId()); - buf.putLong(rmRec.pageId()); - - buf.putShort((short)rmRec.index()); - buf.putShort((short)rmRec.count()); - - break; - - case BTREE_PAGE_INNER_REPLACE: - InnerReplaceRecord irRec = (InnerReplaceRecord)record; - - buf.putInt(irRec.groupId()); - buf.putLong(irRec.pageId()); - - buf.putShort((short)irRec.destinationIndex()); - buf.putLong(irRec.sourcePageId()); - buf.putShort((short)irRec.sourceIndex()); - buf.putLong(irRec.removeId()); - - break; - - case BTREE_FORWARD_PAGE_SPLIT: - SplitForwardPageRecord sfRec = (SplitForwardPageRecord)record; - - buf.putInt(sfRec.groupId()); - buf.putLong(sfRec.pageId()); - - buf.putLong(sfRec.forwardId()); - buf.putShort((short)sfRec.ioType()); - buf.putShort((short)sfRec.ioVersion()); - buf.putLong(sfRec.sourcePageId()); - buf.putShort((short)sfRec.middleIndex()); - buf.putShort((short)sfRec.count()); - - break; - - case BTREE_EXISTING_PAGE_SPLIT: - SplitExistingPageRecord seRec = (SplitExistingPageRecord)record; - - buf.putInt(seRec.groupId()); - buf.putLong(seRec.pageId()); - - buf.putShort((short)seRec.middleIndex()); - buf.putLong(seRec.forwardId()); - - break; - - case BTREE_PAGE_MERGE: - MergeRecord mRec = (MergeRecord)record; - - buf.putInt(mRec.groupId()); - buf.putLong(mRec.pageId()); - - buf.putLong(mRec.parentId()); - buf.putShort((short)mRec.parentIndex()); - buf.putLong(mRec.rightId()); - buf.put((byte)(mRec.isEmptyBranch() ? 1 : 0)); - - break; - - case PAGES_LIST_SET_NEXT: - PagesListSetNextRecord plNextRec = (PagesListSetNextRecord)record; - - buf.putInt(plNextRec.groupId()); - buf.putLong(plNextRec.pageId()); - - buf.putLong(plNextRec.nextPageId()); - - break; - - case PAGES_LIST_SET_PREVIOUS: - PagesListSetPreviousRecord plPrevRec = (PagesListSetPreviousRecord)record; - - buf.putInt(plPrevRec.groupId()); - buf.putLong(plPrevRec.pageId()); - - buf.putLong(plPrevRec.previousPageId()); - - break; - - case PAGES_LIST_INIT_NEW_PAGE: - PagesListInitNewPageRecord plNewRec = (PagesListInitNewPageRecord)record; - - buf.putInt(plNewRec.groupId()); - buf.putLong(plNewRec.pageId()); - buf.putInt(plNewRec.ioType()); - buf.putInt(plNewRec.ioVersion()); - buf.putLong(plNewRec.newPageId()); - - buf.putLong(plNewRec.previousPageId()); - buf.putLong(plNewRec.dataPageId()); - - break; - - case PAGES_LIST_ADD_PAGE: - PagesListAddPageRecord plAddRec = (PagesListAddPageRecord)record; - - buf.putInt(plAddRec.groupId()); - buf.putLong(plAddRec.pageId()); - - buf.putLong(plAddRec.dataPageId()); - - break; - - case PAGES_LIST_REMOVE_PAGE: - PagesListRemovePageRecord plRmvRec = (PagesListRemovePageRecord)record; - - buf.putInt(plRmvRec.groupId()); - buf.putLong(plRmvRec.pageId()); - - buf.putLong(plRmvRec.removedPageId()); - - break; - - case BTREE_FIX_REMOVE_ID: - FixRemoveId frRec = (FixRemoveId)record; - - buf.putInt(frRec.groupId()); - buf.putLong(frRec.pageId()); - - buf.putLong(frRec.removeId()); - - break; - - case TRACKING_PAGE_DELTA: - TrackingPageDeltaRecord tpDelta = (TrackingPageDeltaRecord)record; - - buf.putInt(tpDelta.groupId()); - buf.putLong(tpDelta.pageId()); - - buf.putLong(tpDelta.pageIdToMark()); - buf.putLong(tpDelta.nextSnapshotId()); - buf.putLong(tpDelta.lastSuccessfulSnapshotId()); - - break; - - case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID: - MetaPageUpdateNextSnapshotId mpUpdateNextSnapshotId = (MetaPageUpdateNextSnapshotId)record; - - buf.putInt(mpUpdateNextSnapshotId.groupId()); - buf.putLong(mpUpdateNextSnapshotId.pageId()); - - buf.putLong(mpUpdateNextSnapshotId.nextSnapshotId()); - - break; - - case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID: - MetaPageUpdateLastSuccessfulFullSnapshotId mpUpdateLastSuccFullSnapshotId = - (MetaPageUpdateLastSuccessfulFullSnapshotId)record; - - buf.putInt(mpUpdateLastSuccFullSnapshotId.groupId()); - buf.putLong(mpUpdateLastSuccFullSnapshotId.pageId()); - - buf.putLong(mpUpdateLastSuccFullSnapshotId.lastSuccessfulFullSnapshotId()); - - break; - - case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID: - MetaPageUpdateLastSuccessfulSnapshotId mpUpdateLastSuccSnapshotId = - (MetaPageUpdateLastSuccessfulSnapshotId)record; - - buf.putInt(mpUpdateLastSuccSnapshotId.groupId()); - buf.putLong(mpUpdateLastSuccSnapshotId.pageId()); - - buf.putLong(mpUpdateLastSuccSnapshotId.lastSuccessfulSnapshotId()); - buf.putLong(mpUpdateLastSuccSnapshotId.lastSuccessfulSnapshotTag()); - - break; - - case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX: - MetaPageUpdateLastAllocatedIndex mpUpdateLastAllocatedIdx = - (MetaPageUpdateLastAllocatedIndex) record; - - buf.putInt(mpUpdateLastAllocatedIdx.groupId()); - buf.putLong(mpUpdateLastAllocatedIdx.pageId()); - - buf.putInt(mpUpdateLastAllocatedIdx.lastAllocatedIndex()); - - break; - - case PART_META_UPDATE_STATE: - PartitionMetaStateRecord partMetaStateRecord = (PartitionMetaStateRecord) record; - - buf.putInt(partMetaStateRecord.groupId()); - - buf.putInt(partMetaStateRecord.partitionId()); - - buf.put(partMetaStateRecord.state()); - - buf.putLong(partMetaStateRecord.updateCounter()); - - break; - - case PAGE_LIST_META_RESET_COUNT_RECORD: - PageListMetaResetCountRecord pageListMetaResetCntRecord = (PageListMetaResetCountRecord) record; - - buf.putInt(pageListMetaResetCntRecord.groupId()); - buf.putLong(pageListMetaResetCntRecord.pageId()); - - break; - - default: - throw new UnsupportedOperationException("Type: " + record.type()); - } - - if (!skipCrc) { - int curPos = buf.position(); - - buf.position(startPos); - - // This call will move buffer position to the end of the record again. - int crcVal = PureJavaCrc32.calcCrc32(buf, curPos - startPos); - - buf.putInt(crcVal); - } - else - buf.putInt(0); - } - - /** {@inheritDoc} */ - @Override public WALRecord readRecord(FileInput in0, WALPointer expPtr) throws IOException, IgniteCheckedException { - long startPos = -1; - - try (FileInput.Crc32CheckingFileInput in = in0.startRead(skipCrc)) { - startPos = in0.position(); - - WALRecord res = readRecord(in, expPtr); - - assert res != null; - - res.size((int)(in0.position() - startPos + CRC_SIZE)); // Account for CRC which will be read afterwards. - - return res; - } - catch (EOFException | SegmentEofException e) { - throw e; - } - catch (Exception e) { - throw new IgniteCheckedException("Failed to read WAL record at position: " + startPos, e); - } - } - - /** - * Loads record from input, does not read CRC value - * - * @param in Input to read record from - * @param expPtr expected WAL pointer for record. Used to validate actual position against expected from the file - * @throws SegmentEofException if end of WAL segment reached - */ - private WALRecord readRecord(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException { - int type = in.readUnsignedByte(); - - if (type == WALRecord.RecordType.STOP_ITERATION_RECORD_TYPE) - throw new SegmentEofException("Reached logical end of the segment", null); - - FileWALPointer ptr = readPosition(in); - - if (!F.eq(ptr, expPtr)) - throw new SegmentEofException("WAL segment rollover detected (will end iteration) [expPtr=" + expPtr + - ", readPtr=" + ptr + ']', null); - - RecordType recType = RecordType.fromOrdinal(type - 1); - - if (recType == null) - throw new IOException("Unknown record type: " + type); - - WALRecord res; - - switch (recType) { - case PAGE_RECORD: - byte[] arr = new byte[pageSize]; - - int cacheId = in.readInt(); - long pageId = in.readLong(); - - in.readFully(arr); - - res = new PageSnapshot(new FullPageId(pageId, cacheId), arr); - - break; - - case CHECKPOINT_RECORD: - long msb = in.readLong(); - long lsb = in.readLong(); - boolean hasPtr = in.readByte() != 0; - int idx = hasPtr ? in.readInt() : 0; - int offset = hasPtr ? in.readInt() : 0; - int len = hasPtr ? in.readInt() : 0; - - Map states = readPartitionStates(in); - - boolean end = in.readByte() != 0; - - FileWALPointer walPtr = hasPtr ? new FileWALPointer(idx, offset, len) : null; - - CheckpointRecord cpRec = new CheckpointRecord(new UUID(msb, lsb), walPtr, end); - - cpRec.cacheGroupStates(states); - - res = cpRec; - - break; - - case META_PAGE_INIT: - cacheId = in.readInt(); - pageId = in.readLong(); - - int ioType = in.readUnsignedShort(); - int ioVer = in.readUnsignedShort(); - long treeRoot = in.readLong(); - long reuseListRoot = in.readLong(); - - res = new MetaPageInitRecord(cacheId, pageId, ioType, ioVer, treeRoot, reuseListRoot); - - break; - - case PARTITION_META_PAGE_UPDATE_COUNTERS: - cacheId = in.readInt(); - pageId = in.readLong(); - - long updCntr = in.readLong(); - long rmvId = in.readLong(); - int partSize = in.readInt(); - long countersPageId = in.readLong(); - byte state = in.readByte(); - int allocatedIdxCandidate = in.readInt(); - - res = new MetaPageUpdatePartitionDataRecord(cacheId, pageId, updCntr, rmvId, partSize, countersPageId, state, allocatedIdxCandidate); - - break; - - case MEMORY_RECOVERY: - long ts = in.readLong(); - - res = new MemoryRecoveryRecord(ts); - - break; - - case PARTITION_DESTROY: - cacheId = in.readInt(); - int partId = in.readInt(); - - res = new PartitionDestroyRecord(cacheId, partId); - - break; - - case DATA_RECORD: - int entryCnt = in.readInt(); - - List entries = new ArrayList<>(entryCnt); - - for (int i = 0; i < entryCnt; i++) - entries.add(readDataEntry(in)); - - res = new DataRecord(entries); - - break; - - case HEADER_RECORD: - long magic = in.readLong(); - - if (magic != HeaderRecord.MAGIC) - throw new EOFException("Magic is corrupted [exp=" + U.hexLong(HeaderRecord.MAGIC) + - ", actual=" + U.hexLong(magic) + ']'); - - int ver = in.readInt(); - - res = new HeaderRecord(ver); - - break; - - case DATA_PAGE_INSERT_RECORD: { - cacheId = in.readInt(); - pageId = in.readLong(); - - int size = in.readUnsignedShort(); - - in.ensure(size); - - byte[] payload = new byte[size]; - - in.readFully(payload); - - res = new DataPageInsertRecord(cacheId, pageId, payload); - - break; - } - - case DATA_PAGE_UPDATE_RECORD: { - cacheId = in.readInt(); - pageId = in.readLong(); - - int itemId = in.readInt(); - - int size = in.readUnsignedShort(); - - in.ensure(size); - - byte[] payload = new byte[size]; - - in.readFully(payload); - - res = new DataPageUpdateRecord(cacheId, pageId, itemId, payload); - - break; - } - - case DATA_PAGE_INSERT_FRAGMENT_RECORD: { - cacheId = in.readInt(); - pageId = in.readLong(); - - final long lastLink = in.readLong(); - final int payloadSize = in.readInt(); - - final byte[] payload = new byte[payloadSize]; - - in.readFully(payload); - - res = new DataPageInsertFragmentRecord(cacheId, pageId, payload, lastLink); - - break; - } - - case DATA_PAGE_REMOVE_RECORD: - cacheId = in.readInt(); - pageId = in.readLong(); - - int itemId = in.readUnsignedByte(); - - res = new DataPageRemoveRecord(cacheId, pageId, itemId); - - break; - - case DATA_PAGE_SET_FREE_LIST_PAGE: - cacheId = in.readInt(); - pageId = in.readLong(); - - long freeListPage = in.readLong(); - - res = new DataPageSetFreeListPageRecord(cacheId, pageId, freeListPage); - - break; - - case INIT_NEW_PAGE_RECORD: - cacheId = in.readInt(); - pageId = in.readLong(); - - ioType = in.readUnsignedShort(); - ioVer = in.readUnsignedShort(); - long virtualPageId = in.readLong(); - - res = new InitNewPageRecord(cacheId, pageId, ioType, ioVer, virtualPageId); - - break; - - case BTREE_META_PAGE_INIT_ROOT: - cacheId = in.readInt(); - pageId = in.readLong(); - - long rootId = in.readLong(); - - res = new MetaPageInitRootRecord(cacheId, pageId, rootId); - - break; - - case BTREE_META_PAGE_INIT_ROOT2: - cacheId = in.readInt(); - pageId = in.readLong(); - - long rootId2 = in.readLong(); - int inlineSize = in.readShort(); - - res = new MetaPageInitRootInlineRecord(cacheId, pageId, rootId2, inlineSize); - - break; - - case BTREE_META_PAGE_ADD_ROOT: - cacheId = in.readInt(); - pageId = in.readLong(); - - rootId = in.readLong(); - - res = new MetaPageAddRootRecord(cacheId, pageId, rootId); - - break; - - case BTREE_META_PAGE_CUT_ROOT: - cacheId = in.readInt(); - pageId = in.readLong(); - - res = new MetaPageCutRootRecord(cacheId, pageId); - - break; - - case BTREE_INIT_NEW_ROOT: - cacheId = in.readInt(); - pageId = in.readLong(); - - rootId = in.readLong(); - ioType = in.readUnsignedShort(); - ioVer = in.readUnsignedShort(); - long leftId = in.readLong(); - long rightId = in.readLong(); - - BPlusIO io = BPlusIO.getBPlusIO(ioType, ioVer); - - byte[] rowBytes = new byte[io.getItemSize()]; - - in.readFully(rowBytes); - - res = new NewRootInitRecord<>(cacheId, pageId, rootId, (BPlusInnerIO)io, leftId, rowBytes, rightId); - - break; - - case BTREE_PAGE_RECYCLE: - cacheId = in.readInt(); - pageId = in.readLong(); - - long newPageId = in.readLong(); - - res = new RecycleRecord(cacheId, pageId, newPageId); - - break; - - case BTREE_PAGE_INSERT: - cacheId = in.readInt(); - pageId = in.readLong(); - - ioType = in.readUnsignedShort(); - ioVer = in.readUnsignedShort(); - int itemIdx = in.readUnsignedShort(); - rightId = in.readLong(); - - io = BPlusIO.getBPlusIO(ioType, ioVer); - - rowBytes = new byte[io.getItemSize()]; - - in.readFully(rowBytes); - - res = new InsertRecord<>(cacheId, pageId, io, itemIdx, rowBytes, rightId); - - break; - - case BTREE_FIX_LEFTMOST_CHILD: - cacheId = in.readInt(); - pageId = in.readLong(); - - rightId = in.readLong(); - - res = new FixLeftmostChildRecord(cacheId, pageId, rightId); - - break; - - case BTREE_FIX_COUNT: - cacheId = in.readInt(); - pageId = in.readLong(); - - int cnt = in.readUnsignedShort(); - - res = new FixCountRecord(cacheId, pageId, cnt); - - break; - - case BTREE_PAGE_REPLACE: - cacheId = in.readInt(); - pageId = in.readLong(); - - ioType = in.readUnsignedShort(); - ioVer = in.readUnsignedShort(); - itemIdx = in.readUnsignedShort(); - - io = BPlusIO.getBPlusIO(ioType, ioVer); - - rowBytes = new byte[io.getItemSize()]; - - in.readFully(rowBytes); - - res = new ReplaceRecord<>(cacheId, pageId, io, rowBytes, itemIdx); - - break; - - case BTREE_PAGE_REMOVE: - cacheId = in.readInt(); - pageId = in.readLong(); - - itemIdx = in.readUnsignedShort(); - cnt = in.readUnsignedShort(); - - res = new RemoveRecord(cacheId, pageId, itemIdx, cnt); - - break; - - case BTREE_PAGE_INNER_REPLACE: - cacheId = in.readInt(); - pageId = in.readLong(); - - int dstIdx = in.readUnsignedShort(); - long srcPageId = in.readLong(); - int srcIdx = in.readUnsignedShort(); - rmvId = in.readLong(); - - res = new InnerReplaceRecord<>(cacheId, pageId, dstIdx, srcPageId, srcIdx, rmvId); - - break; - - case BTREE_FORWARD_PAGE_SPLIT: - cacheId = in.readInt(); - pageId = in.readLong(); - - long fwdId = in.readLong(); - ioType = in.readUnsignedShort(); - ioVer = in.readUnsignedShort(); - srcPageId = in.readLong(); - int mid = in.readUnsignedShort(); - cnt = in.readUnsignedShort(); - - res = new SplitForwardPageRecord(cacheId, pageId, fwdId, ioType, ioVer, srcPageId, mid, cnt); - - break; - - case BTREE_EXISTING_PAGE_SPLIT: - cacheId = in.readInt(); - pageId = in.readLong(); - - mid = in.readUnsignedShort(); - fwdId = in.readLong(); - - res = new SplitExistingPageRecord(cacheId, pageId, mid, fwdId); - - break; - - case BTREE_PAGE_MERGE: - cacheId = in.readInt(); - pageId = in.readLong(); - - long prntId = in.readLong(); - int prntIdx = in.readUnsignedShort(); - rightId = in.readLong(); - boolean emptyBranch = in.readBoolean(); - - res = new MergeRecord<>(cacheId, pageId, prntId, prntIdx, rightId, emptyBranch); - - break; - - case BTREE_FIX_REMOVE_ID: - cacheId = in.readInt(); - pageId = in.readLong(); - - rmvId = in.readLong(); - - res = new FixRemoveId(cacheId, pageId, rmvId); - - break; - - case PAGES_LIST_SET_NEXT: - cacheId = in.readInt(); - pageId = in.readLong(); - long nextPageId = in.readLong(); - - res = new PagesListSetNextRecord(cacheId, pageId, nextPageId); - - break; - - case PAGES_LIST_SET_PREVIOUS: - cacheId = in.readInt(); - pageId = in.readLong(); - long prevPageId = in.readLong(); - - res = new PagesListSetPreviousRecord(cacheId, pageId, prevPageId); - - break; - - case PAGES_LIST_INIT_NEW_PAGE: - cacheId = in.readInt(); - pageId = in.readLong(); - ioType = in.readInt(); - ioVer = in.readInt(); - newPageId = in.readLong(); - prevPageId = in.readLong(); - long addDataPageId = in.readLong(); - - res = new PagesListInitNewPageRecord(cacheId, pageId, ioType, ioVer, newPageId, prevPageId, addDataPageId); - - break; - - case PAGES_LIST_ADD_PAGE: - cacheId = in.readInt(); - pageId = in.readLong(); - long dataPageId = in.readLong(); - - res = new PagesListAddPageRecord(cacheId, pageId, dataPageId); - - break; - - case PAGES_LIST_REMOVE_PAGE: - cacheId = in.readInt(); - pageId = in.readLong(); - long rmvdPageId = in.readLong(); - - res = new PagesListRemovePageRecord(cacheId, pageId, rmvdPageId); - - break; - - case TRACKING_PAGE_DELTA: - cacheId = in.readInt(); - pageId = in.readLong(); - - long pageIdToMark = in.readLong(); - long nextSnapshotId0 = in.readLong(); - long lastSuccessfulSnapshotId0 = in.readLong(); - - res = new TrackingPageDeltaRecord(cacheId, pageId, pageIdToMark, nextSnapshotId0, lastSuccessfulSnapshotId0); - - break; - - case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID: - cacheId = in.readInt(); - pageId = in.readLong(); - - long nextSnapshotId = in.readLong(); - - res = new MetaPageUpdateNextSnapshotId(cacheId, pageId, nextSnapshotId); - - break; - - case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID: - cacheId = in.readInt(); - pageId = in.readLong(); - - long lastSuccessfulFullSnapshotId = in.readLong(); - - res = new MetaPageUpdateLastSuccessfulFullSnapshotId(cacheId, pageId, lastSuccessfulFullSnapshotId); - - break; - - case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID: - cacheId = in.readInt(); - pageId = in.readLong(); - - long lastSuccessfulSnapshotId = in.readLong(); - long lastSuccessfulSnapshotTag = in.readLong(); - - res = new MetaPageUpdateLastSuccessfulSnapshotId(cacheId, pageId, lastSuccessfulSnapshotId, lastSuccessfulSnapshotTag); - - break; - - case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX: - cacheId = in.readInt(); - pageId = in.readLong(); - - int lastAllocatedIdx = in.readInt(); + /** Skip CRC calculation/check flag */ + public static boolean SKIP_CRC = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CRC, false); - res = new MetaPageUpdateLastAllocatedIndex(cacheId, pageId, lastAllocatedIdx); + /** */ + private final RecordDataV1Serializer dataSerializer; - break; + /** Record read/write functional interface. */ + private final RecordIO recordIO = new RecordIO() { - case PART_META_UPDATE_STATE: - cacheId = in.readInt(); - partId = in.readInt(); + @Override public int size(WALRecord record) throws IgniteCheckedException { + return dataSerializer.size(record); + } - state = in.readByte(); + /** + * Reads record from input, does not read CRC value. + * + * @param in Input to read record from + * @param expPtr expected WAL pointer for record. Used to validate actual position against expected from the file + * @throws SegmentEofException if end of WAL segment reached + */ + @Override public WALRecord read(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException { + RecordType recType = readRecordType(in); - long updateCounter = in.readLong(); + FileWALPointer ptr = readPosition(in); - res = new PartitionMetaStateRecord(cacheId, partId, GridDhtPartitionState.fromOrdinal(state), updateCounter); + if (!F.eq(ptr, expPtr)) + throw new SegmentEofException("WAL segment rollover detected (will end iteration) [expPtr=" + expPtr + + ", readPtr=" + ptr + ']', null); - break; + return dataSerializer.readRecord(recType, in); + } - case PAGE_LIST_META_RESET_COUNT_RECORD: - cacheId = in.readInt(); - pageId = in.readLong(); + /** + * Writes record to output, does not write CRC value. + * + * @param record + * @param buf + * @throws IgniteCheckedException + */ + @Override public void write(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { + // Write record type. + putRecordType(buf, record); + + // Write record file position. + putPositionOfRecord(buf, record); + + // Write record data. + dataSerializer.writeRecord(record, buf); + } + }; - res = new PageListMetaResetCountRecord(cacheId, pageId); - break; + /** + * + * @param dataSerializer + */ + public RecordV1Serializer(RecordDataV1Serializer dataSerializer) { + this.dataSerializer = dataSerializer; + } - case SWITCH_SEGMENT_RECORD: - throw new EOFException("END OF SEGMENT"); + /** {@inheritDoc} */ + @Override public int version() { + return 1; + } - default: - throw new UnsupportedOperationException("Type: " + recType); - } + /** {@inheritDoc} */ + @SuppressWarnings("CastConflictsWithInstanceof") + @Override public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { + writeWithCrc(record, buf, recordIO); + } - return res; + /** {@inheritDoc} */ + @Override public WALRecord readRecord(FileInput in0, WALPointer expPtr) throws IOException, IgniteCheckedException { + return readWithCrc(in0, expPtr, recordIO); } /** {@inheritDoc} */ @SuppressWarnings("CastConflictsWithInstanceof") @Override public int size(WALRecord record) throws IgniteCheckedException { - int commonFields = REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + CRC_SIZE; - - switch (record.type()) { - case PAGE_RECORD: - assert record instanceof PageSnapshot; - - PageSnapshot pageRec = (PageSnapshot)record; - - return commonFields + pageRec.pageData().length + 12; - - case CHECKPOINT_RECORD: - CheckpointRecord cpRec = (CheckpointRecord)record; - - assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer : - "Invalid WAL record: " + cpRec; - - int cacheStatesSize = cacheStatesSize(cpRec.cacheGroupStates()); - - FileWALPointer walPtr = (FileWALPointer)cpRec.checkpointMark(); - - return commonFields + 18 + cacheStatesSize + (walPtr == null ? 0 : 16); - - case META_PAGE_INIT: - return commonFields + /*cache ID*/4 + /*page ID*/8 + /*ioType*/2 + /*ioVer*/2 + /*tree root*/8 + /*reuse root*/8; - - case PARTITION_META_PAGE_UPDATE_COUNTERS: - return commonFields + /*cache ID*/4 + /*page ID*/8 + /*upd cntr*/8 + /*rmv id*/8 + /*part size*/4 + /*counters page id*/8 + /*state*/ 1 - + /*allocatedIdxCandidate*/ 4; - - case MEMORY_RECOVERY: - return commonFields + 8; - - case PARTITION_DESTROY: - return commonFields + /*cacheId*/4 + /*partId*/4; - - case DATA_RECORD: - DataRecord dataRec = (DataRecord)record; - - return commonFields + 4 + dataSize(dataRec); - - case HEADER_RECORD: - return HEADER_RECORD_SIZE; - - case DATA_PAGE_INSERT_RECORD: - DataPageInsertRecord diRec = (DataPageInsertRecord)record; - - return commonFields + 4 + 8 + 2 + diRec.payload().length; - - case DATA_PAGE_UPDATE_RECORD: - DataPageUpdateRecord uRec = (DataPageUpdateRecord)record; - - return commonFields + 4 + 8 + 2 + 4 + - uRec.payload().length; - - case DATA_PAGE_INSERT_FRAGMENT_RECORD: - final DataPageInsertFragmentRecord difRec = (DataPageInsertFragmentRecord)record; - - return commonFields + 4 + 8 + 8 + 4 + difRec.payloadSize(); - - case DATA_PAGE_REMOVE_RECORD: - return commonFields + 4 + 8 + 1; - - case DATA_PAGE_SET_FREE_LIST_PAGE: - return commonFields + 4 + 8 + 8; - - case INIT_NEW_PAGE_RECORD: - return commonFields + 4 + 8 + 2 + 2 + 8; - - case BTREE_META_PAGE_INIT_ROOT: - return commonFields + 4 + 8 + 8; - - case BTREE_META_PAGE_INIT_ROOT2: - return commonFields + 4 + 8 + 8 + 2; - - case BTREE_META_PAGE_ADD_ROOT: - return commonFields + 4 + 8 + 8; - - case BTREE_META_PAGE_CUT_ROOT: - return commonFields + 4 + 8; - - case BTREE_INIT_NEW_ROOT: - NewRootInitRecord riRec = (NewRootInitRecord)record; - - return commonFields + 4 + 8 + 8 + 2 + 2 + 8 + 8 + riRec.io().getItemSize(); - - case BTREE_PAGE_RECYCLE: - return commonFields + 4 + 8 + 8; - - case BTREE_PAGE_INSERT: - InsertRecord inRec = (InsertRecord)record; - - return commonFields + 4 + 8 + 2 + 2 + 2 + 8 + inRec.io().getItemSize(); - - case BTREE_FIX_LEFTMOST_CHILD: - return commonFields + 4 + 8 + 8; - - case BTREE_FIX_COUNT: - return commonFields + 4 + 8 + 2; - - case BTREE_PAGE_REPLACE: - ReplaceRecord rRec = (ReplaceRecord)record; - - return commonFields + 4 + 8 + 2 + 2 + 2 + rRec.io().getItemSize(); - - case BTREE_PAGE_REMOVE: - return commonFields + 4 + 8 + 2 + 2; - - case BTREE_PAGE_INNER_REPLACE: - return commonFields + 4 + 8 + 2 + 8 + 2 + 8; - - case BTREE_FORWARD_PAGE_SPLIT: - return commonFields + 4 + 8 + 8 + 2 + 2 + 8 + 2 + 2; - - case BTREE_EXISTING_PAGE_SPLIT: - return commonFields + 4 + 8 + 2 + 8; - - case BTREE_PAGE_MERGE: - return commonFields + 4 + 8 + 8 + 2 + 8 + 1; - - case BTREE_FIX_REMOVE_ID: - return commonFields + 4 + 8 + 8; - - case PAGES_LIST_SET_NEXT: - return commonFields + 4 + 8 + 8; - - case PAGES_LIST_SET_PREVIOUS: - return commonFields + 4 + 8 + 8; - - case PAGES_LIST_INIT_NEW_PAGE: - return commonFields + 4 + 8 + 4 + 4 + 8 + 8 + 8; - - case PAGES_LIST_ADD_PAGE: - return commonFields + 4 + 8 + 8; - - case PAGES_LIST_REMOVE_PAGE: - return commonFields + 4 + 8 + 8; - - case TRACKING_PAGE_DELTA: - return commonFields + 4 + 8 + 8 + 8 + 8; - - case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID: - return commonFields + 4 + 8 + 8 + 8; - - case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID: - return commonFields + 4 + 8 + 8; - - case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID: - return commonFields + 4 + 8 + 8; - - case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX: - return commonFields + 4 + 8 + 4; - - case PART_META_UPDATE_STATE: - return commonFields + /*cacheId*/ 4 + /*partId*/ 4 + /*State*/1 + /*Update Counter*/ 8; - - case PAGE_LIST_META_RESET_COUNT_RECORD: - return commonFields + /*cacheId*/ 4 + /*pageId*/ 8; - - case SWITCH_SEGMENT_RECORD: - return commonFields - CRC_SIZE; //CRC is not loaded for switch segment, exception is thrown instead - - default: - throw new UnsupportedOperationException("Type: " + record.type()); - } + return sizeWithHeadersAndCrc(record, recordIO); } /** @@ -1420,256 +163,82 @@ public static FileWALPointer readPosition(DataInput in) throws IOException { return new FileWALPointer(idx, fileOffset, 0); } - /** - * @param dataRec Data record to serialize. - * @return Full data record size. - * @throws IgniteCheckedException If failed to obtain the length of one of the entries. - */ - private int dataSize(DataRecord dataRec) throws IgniteCheckedException { - int sz = 0; - - for (DataEntry entry : dataRec.writeEntries()) - sz += entrySize(entry); - - return sz; - } - - /** - * @param entry Entry to get size for. - * @return Entry size. - * @throws IgniteCheckedException If failed to get key or value bytes length. - */ - private int entrySize(DataEntry entry) throws IgniteCheckedException { - GridCacheContext cctx = this.cctx.cacheContext(entry.cacheId()); - CacheObjectContext coCtx = cctx.cacheObjectContext(); - - return - /*cache ID*/4 + - /*key*/entry.key().valueBytesLength(coCtx) + - /*value*/(entry.value() == null ? 4 : entry.value().valueBytesLength(coCtx)) + - /*op*/1 + - /*near xid ver*/CacheVersionIO.size(entry.nearXidVersion(), true) + - /*write ver*/CacheVersionIO.size(entry.writeVersion(), false) + - /*part ID*/4 + - /*expire Time*/8 + - /*part cnt*/8; + public static void putPositionOfRecord(ByteBuffer buf, WALRecord record) { + putPosition(buf, (FileWALPointer) record.position()); } - /** - * @param states Partition states. - * @return Size required to write partition states. - */ - private int cacheStatesSize(Map states) { - // Need 4 bytes for the number of caches. - int size = 2; - - for (Map.Entry entry : states.entrySet()) { - // Cache ID. - size += 4; - - // Need 2 bytes for the number of partitions. - size += 2; - - CacheState state = entry.getValue(); - - // 2 bytes partition ID, size and counter per partition. - size += 18 * state.size(); - } - - return size; + public static void putRecordType(ByteBuffer buf, WALRecord record) { + buf.put((byte)(record.type().ordinal() + 1)); } - /** - * @param buf Buffer to write to. - * @param entry Data entry. - */ - private void putDataEntry(ByteBuffer buf, DataEntry entry) throws IgniteCheckedException { - buf.putInt(entry.cacheId()); - - if (!entry.key().putValue(buf)) - throw new AssertionError(); + public static RecordType readRecordType(DataInput in) throws IgniteCheckedException, IOException { + int type = in.readUnsignedByte(); - if (entry.value() == null) - buf.putInt(-1); - else if (!entry.value().putValue(buf)) - throw new AssertionError(); + if (type == WALRecord.RecordType.STOP_ITERATION_RECORD_TYPE) + throw new SegmentEofException("Reached logical end of the segment", null); - buf.put((byte)entry.op().ordinal()); + RecordType recType = RecordType.fromOrdinal(type - 1); - putVersion(buf, entry.nearXidVersion(), true); - putVersion(buf, entry.writeVersion(), false); + if (recType == null) + throw new IOException("Unknown record type: " + type); - buf.putInt(entry.partitionId()); - buf.putLong(entry.partitionCounter()); - buf.putLong(entry.expireTime()); + return recType; } - /** - * @param states Cache states. - */ - private void putCacheStates(ByteBuffer buf, Map states) { - buf.putShort((short)states.size()); + public static WALRecord readWithCrc(FileInput in0, WALPointer expPtr, RecordIO reader) throws EOFException, IgniteCheckedException { + long startPos = -1; - for (Map.Entry entry : states.entrySet()) { - buf.putInt(entry.getKey()); + try (FileInput.Crc32CheckingFileInput in = in0.startRead(SKIP_CRC)) { + startPos = in0.position(); - CacheState state = entry.getValue(); + WALRecord res = reader.read(in, expPtr); - // Need 2 bytes for the number of partitions. - buf.putShort((short)state.size()); + assert res != null; - for (int i = 0; i < state.size(); i++) { - buf.putShort((short)state.partitionByIndex(i)); + res.size((int)(in0.position() - startPos + CRC_SIZE)); // Account for CRC which will be read afterwards. - buf.putLong(state.partitionSizeByIndex(i)); - buf.putLong(state.partitionCounterByIndex(i)); - } + return res; } - } - - /** - * @param in Input to read from. - * @return Read entry. - */ - private DataEntry readDataEntry(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException { - int cacheId = in.readInt(); - - int keySize = in.readInt(); - byte keyType = in.readByte(); - byte[] keyBytes = new byte[keySize]; - in.readFully(keyBytes); - - int valSize = in.readInt(); - - byte valType = 0; - byte[] valBytes = null; - - if (valSize >= 0) { - valType = in.readByte(); - valBytes = new byte[valSize]; - in.readFully(valBytes); + catch (EOFException | SegmentEofException e) { + throw e; } - - byte ord = in.readByte(); - - GridCacheOperation op = GridCacheOperation.fromOrdinal(ord & 0xFF); - - GridCacheVersion nearXidVer = readVersion(in, true); - GridCacheVersion writeVer = readVersion(in, false); - - int partId = in.readInt(); - long partCntr = in.readLong(); - long expireTime = in.readLong(); - - GridCacheContext cacheCtx = cctx.cacheContext(cacheId); - - if (cacheCtx != null) { - CacheObjectContext coCtx = cacheCtx.cacheObjectContext(); - - KeyCacheObject key = co.toKeyCacheObject(coCtx, keyType, keyBytes); - CacheObject val = valBytes != null ? co.toCacheObject(coCtx, valType, valBytes) : null; - - return new DataEntry( - cacheId, - key, - val, - op, - nearXidVer, - writeVer, - expireTime, - partId, - partCntr - ); + catch (Exception e) { + throw new IgniteCheckedException("Failed to read WAL record at position: " + startPos, e); } - else - return new LazyDataEntry( - cctx, - cacheId, - keyType, - keyBytes, - valType, - valBytes, - op, - nearXidVer, - writeVer, - expireTime, - partId, - partCntr); } - /** - * @param buf Buffer to read from. - * @return Read map. - */ - private Map readPartitionStates(DataInput buf) throws IOException { - int caches = buf.readShort() & 0xFFFF; - - if (caches == 0) - return Collections.emptyMap(); - - Map states = new HashMap<>(caches, 1.0f); + public static void writeWithCrc(WALRecord record, ByteBuffer buf, RecordIO writer) throws IgniteCheckedException { + assert record.size() > 0 && buf.remaining() >= record.size() : record.size(); - for (int i = 0; i < caches; i++) { - int cacheId = buf.readInt(); + int startPos = buf.position(); - int parts = buf.readShort() & 0xFFFF; + writer.write(record, buf); - CacheState state = new CacheState(parts); + if (!SKIP_CRC) { + int curPos = buf.position(); - for (int p = 0; p < parts; p++) { - int partId = buf.readShort() & 0xFFFF; - long size = buf.readLong(); - long partCntr = buf.readLong(); + buf.position(startPos); - state.addPartitionState(partId, size, partCntr); - } + // This call will move buffer position to the end of the record again. + int crcVal = PureJavaCrc32.calcCrc32(buf, curPos - startPos); - states.put(cacheId, state); + buf.putInt(crcVal); } - - return states; - } - - /** - * @param buf Buffer. - * @param ver Version to write. - * @param allowNull Is {@code null}version allowed. - */ - private void putVersion(ByteBuffer buf, GridCacheVersion ver, boolean allowNull) { - CacheVersionIO.write(buf, ver, allowNull); + else + buf.putInt(0); } - /** - * Changes the buffer position by the number of read bytes. - * - * @param in Data input to read from. - * @param allowNull Is {@code null}version allowed. - * @return Read cache version. - */ - private GridCacheVersion readVersion(ByteBufferBackedDataInput in, boolean allowNull) throws IOException { - // To be able to read serialization protocol version. - in.ensure(1); + public static int sizeWithHeadersAndCrc(WALRecord record, RecordIO io) throws IgniteCheckedException { + int commonFields = REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + CRC_SIZE; - try { - int size = CacheVersionIO.readSize(in.buffer(), allowNull); + int recordSize = io.size(record); - in.ensure(size); + switch (record.type()) { + case SWITCH_SEGMENT_RECORD: + return commonFields + recordSize - CRC_SIZE; //CRC is not loaded for switch segment, exception is thrown instead - return CacheVersionIO.read(in.buffer(), allowNull); - } - catch (IgniteCheckedException e) { - throw new IOException(e); + default: + return commonFields + recordSize; } } - - /** - * @param buf Buffer. - * @param rowBytes Row bytes. - */ - @SuppressWarnings("unchecked") - private static void putRow(ByteBuffer buf, byte[] rowBytes) { - assert rowBytes.length > 0; - - buf.put(rowBytes); - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java new file mode 100644 index 0000000000000..343785e9e2503 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java @@ -0,0 +1,112 @@ +package org.apache.ignite.internal.processors.cache.persistence.wal.serializer; + +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.pagemem.wal.WALPointer; +import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; +import org.apache.ignite.internal.processors.cache.persistence.wal.RecordSerializer; +import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException; +import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.io.RecordIO; +import org.apache.ignite.internal.util.typedef.F; + +/** + * Record V2 serializer. + * Stores records in following format: + *
    + *
  • Record type from {@link WALRecord.RecordType#ordinal()} incremented by 1
    • + *
  • WAL pointer to double check consistency
    • + * TODO: record data length + *
  • Data
    • + *
  • CRC or zero padding
    • + *
+ */ +public class RecordV2Serializer implements RecordSerializer { + // TODO: Define additional header size here + // public static final int RECORD_LENGTH_HEADER_SIZE = 4; + + /** */ + private final RecordDataV2Serializer dataSerializer; + + /** Record read/write functional interface. */ + private final RecordIO recordIO = new RecordIO() { + + @Override public int size(WALRecord record) throws IgniteCheckedException { + // TODO: Add to size the record length header size + return dataSerializer.size(record); + } + + /** + * Reads record from input, does not read CRC value. + * + * @param in Input to read record from + * @param expPtr expected WAL pointer for record. Used to validate actual position against expected from the file + * @throws SegmentEofException if end of WAL segment reached + */ + @Override public WALRecord read(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException { + WALRecord.RecordType recType = RecordV1Serializer.readRecordType(in); + + FileWALPointer ptr = RecordV1Serializer.readPosition(in); + + if (!F.eq(ptr, expPtr)) + throw new SegmentEofException("WAL segment rollover detected (will end iteration) [expPtr=" + expPtr + + ", readPtr=" + ptr + ']', null); + + //TODO read record length here + + return dataSerializer.readRecord(recType, in); + } + + /** + * Writes record to output, does not write CRC value. + * + * @param record + * @param buf + * @throws IgniteCheckedException + */ + @Override public void write(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { + // Write record type. + RecordV1Serializer.putRecordType(buf, record); + + // Write record file position. + RecordV1Serializer.putPositionOfRecord(buf, record); + + // TODO write record length here. + + // Write record data. + dataSerializer.writeRecord(record, buf); + } + }; + + /** + * Create an instance of Record V2 serializer. + * + * @param dataSerializer Record data V2 serializer. + */ + public RecordV2Serializer(RecordDataV2Serializer dataSerializer) { + this.dataSerializer = dataSerializer; + } + + /** {@inheritDoc} */ + @Override public int version() { + return 2; + } + + /** {@inheritDoc} */ + @Override public int size(WALRecord record) throws IgniteCheckedException { + return RecordV1Serializer.sizeWithHeadersAndCrc(record, recordIO); + } + + /** {@inheritDoc} */ + @Override public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { + RecordV1Serializer.writeWithCrc(record, buf, recordIO); + } + + /** {@inheritDoc} */ + @Override public WALRecord readRecord(FileInput in, WALPointer expPtr) throws IOException, IgniteCheckedException { + return RecordV1Serializer.readWithCrc(in, expPtr, recordIO); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/io/RecordIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/io/RecordIO.java new file mode 100644 index 0000000000000..d695167e869ca --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/io/RecordIO.java @@ -0,0 +1,40 @@ +package org.apache.ignite.internal.processors.cache.persistence.wal.serializer.io; + +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.pagemem.wal.WALPointer; +import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput; + +/** + * Internal interface to provide size, read and write operations of WAL records + * including record header and data. + */ +public interface RecordIO { + /** + * + * @param record + * @return + * @throws IgniteCheckedException + */ + int size(WALRecord record) throws IgniteCheckedException; + + /** + * + * @param in + * @param expPtr + * @return + * @throws IOException + * @throws IgniteCheckedException + */ + WALRecord read(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException; + + /** + * + * @param record + * @param buf + * @throws IgniteCheckedException + */ + void write(WALRecord record, ByteBuffer buf) throws IgniteCheckedException; +} From eba4f3aea03509e16a10a33f9635d94c8f805889 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Mon, 14 Aug 2017 13:36:41 +0300 Subject: [PATCH 2/9] IGNITE-6029 Code cleaning and documentation. --- .../persistence/wal/RecordDataSerializer.java | 27 +++--- .../serializer/RecordDataV1Serializer.java | 5 +- .../serializer/RecordDataV2Serializer.java | 21 +++-- .../wal/serializer/RecordV1Serializer.java | 82 +++++++++++++------ .../wal/serializer/RecordV2Serializer.java | 27 ++---- .../wal/serializer/io/RecordIO.java | 31 +++---- 6 files changed, 115 insertions(+), 78 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordDataSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordDataSerializer.java index cf04729524d44..242641df6c9db 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordDataSerializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordDataSerializer.java @@ -7,32 +7,35 @@ /** * Interface to provide size, read and write operations with WAL records - * including only record without headers and other meta information + * without any headers and meta information. */ public interface RecordDataSerializer { /** + * Calculates size of record data. * - * @param record - * @return - * @throws IgniteCheckedException + * @param record WAL record. + * @return Size of record in bytes. + * @throws IgniteCheckedException If it's unable to calculate record data size. */ int size(WALRecord record) throws IgniteCheckedException; /** + * Reads record data of {@code type} from buffer {@code in}. * - * @param type - * @param in - * @return - * @throws IOException - * @throws IgniteCheckedException + * @param type Record type. + * @param in Buffer to read. + * @return WAL record. + * @throws IOException In case of I/O problems. + * @throws IgniteCheckedException If it's unable to read record. */ WALRecord readRecord(WALRecord.RecordType type, ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException; /** + * Writes record data to buffer {@code buf}. * - * @param record - * @param buf - * @throws IgniteCheckedException + * @param record WAL record. + * @param buf Buffer to write. + * @throws IgniteCheckedException If it's unable to write record. */ void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java index 7a46a08a413f7..4b12338692414 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java @@ -75,8 +75,11 @@ import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; import org.apache.ignite.internal.util.typedef.internal.U; +/** + * Record data V1 serializer. + */ public class RecordDataV1Serializer implements RecordDataSerializer { - /** */ + /** Length of HEADER record data. */ static final int HEADER_RECORD_DATA_SIZE = /*Magic*/8 + /*Version*/4; /** Cache shared context */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java index 575baa86981b6..89a136308ee12 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java @@ -12,27 +12,30 @@ */ public class RecordDataV2Serializer implements RecordDataSerializer { + /** V1 data serializer delegate. */ private final RecordDataV1Serializer delegateSerializer; + /** + * Create an instance of V2 data serializer. + * + * @param delegateSerializer V1 data serializer. + */ public RecordDataV2Serializer(RecordDataV1Serializer delegateSerializer) { this.delegateSerializer = delegateSerializer; } - @Override - public int size(WALRecord record) throws IgniteCheckedException { - //TODO: Implementation will be changed. + /** {@inheritDoc} */ + @Override public int size(WALRecord record) throws IgniteCheckedException { return delegateSerializer.size(record); } - @Override - public WALRecord readRecord(WALRecord.RecordType type, ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException { - //TODO: Implementation will be changed. + /** {@inheritDoc} */ + @Override public WALRecord readRecord(WALRecord.RecordType type, ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException { return delegateSerializer.readRecord(type, in); } - @Override - public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { - //TODO: Implementation will be changed. + /** {@inheritDoc} */ + @Override public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { delegateSerializer.writeRecord(record, buf); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java index f834afd30472c..3a4348317a142 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java @@ -57,30 +57,25 @@ public class RecordV1Serializer implements RecordSerializer { /** Length of CRC value */ private static final int CRC_SIZE = 4; - /** */ + /** Total length of HEADER record. */ public static final int HEADER_RECORD_SIZE = REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + CRC_SIZE + RecordDataV1Serializer.HEADER_RECORD_DATA_SIZE; /** Skip CRC calculation/check flag */ public static boolean SKIP_CRC = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CRC, false); - /** */ + /** V1 data serializer. */ private final RecordDataV1Serializer dataSerializer; /** Record read/write functional interface. */ private final RecordIO recordIO = new RecordIO() { - @Override public int size(WALRecord record) throws IgniteCheckedException { + /** {@inheritDoc} */ + @Override public int sizeWithHeaders(WALRecord record) throws IgniteCheckedException { return dataSerializer.size(record); } - /** - * Reads record from input, does not read CRC value. - * - * @param in Input to read record from - * @param expPtr expected WAL pointer for record. Used to validate actual position against expected from the file - * @throws SegmentEofException if end of WAL segment reached - */ - @Override public WALRecord read(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException { + /** {@inheritDoc} */ + @Override public WALRecord readWithHeaders(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException { RecordType recType = readRecordType(in); FileWALPointer ptr = readPosition(in); @@ -92,14 +87,8 @@ public class RecordV1Serializer implements RecordSerializer { return dataSerializer.readRecord(recType, in); } - /** - * Writes record to output, does not write CRC value. - * - * @param record - * @param buf - * @throws IgniteCheckedException - */ - @Override public void write(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { + /** {@inheritDoc} */ + @Override public void writeWithHeaders(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { // Write record type. putRecordType(buf, record); @@ -112,8 +101,9 @@ public class RecordV1Serializer implements RecordSerializer { }; /** + * Create an instance of V1 serializer. * - * @param dataSerializer + * @param dataSerializer V1 data serializer. */ public RecordV1Serializer(RecordDataV1Serializer dataSerializer) { this.dataSerializer = dataSerializer; @@ -163,14 +153,34 @@ public static FileWALPointer readPosition(DataInput in) throws IOException { return new FileWALPointer(idx, fileOffset, 0); } + /** + * Writes record file position to given {@code buf}. + * + * @param buf Buffer to write record file position. + * @param record WAL record. + */ public static void putPositionOfRecord(ByteBuffer buf, WALRecord record) { putPosition(buf, (FileWALPointer) record.position()); } + /** + * Writes record type to given {@code buf}. + * + * @param buf Buffer to write record type. + * @param record WAL record. + */ public static void putRecordType(ByteBuffer buf, WALRecord record) { buf.put((byte)(record.type().ordinal() + 1)); } + /** + * Reads record type from given {@code in}. + * + * @param in Buffer to read record type. + * @return Record type. + * @throws IgniteCheckedException If logical end of segment is reached. + * @throws IOException In case of I/O problems. + */ public static RecordType readRecordType(DataInput in) throws IgniteCheckedException, IOException { int type = in.readUnsignedByte(); @@ -185,13 +195,23 @@ public static RecordType readRecordType(DataInput in) throws IgniteCheckedExcept return recType; } + /** + * Reads record from file {@code in0} and validates CRC of record. + * + * @param in0 File input. + * @param expPtr Expected WAL pointer for record. Used to validate actual position against expected from the file. + * @param reader Record reader I/O interface. + * @return WAL record. + * @throws EOFException In case of end of file. + * @throws IgniteCheckedException If it's unable to read record. + */ public static WALRecord readWithCrc(FileInput in0, WALPointer expPtr, RecordIO reader) throws EOFException, IgniteCheckedException { long startPos = -1; try (FileInput.Crc32CheckingFileInput in = in0.startRead(SKIP_CRC)) { startPos = in0.position(); - WALRecord res = reader.read(in, expPtr); + WALRecord res = reader.readWithHeaders(in, expPtr); assert res != null; @@ -207,12 +227,20 @@ public static WALRecord readWithCrc(FileInput in0, WALPointer expPtr, RecordIO r } } + /** + * Writes record with calculated CRC to buffer {@code buf}. + * + * @param record WAL record. + * @param buf Buffer to write. + * @param writer Record write I/O interface. + * @throws IgniteCheckedException If it's unable to write record. + */ public static void writeWithCrc(WALRecord record, ByteBuffer buf, RecordIO writer) throws IgniteCheckedException { assert record.size() > 0 && buf.remaining() >= record.size() : record.size(); int startPos = buf.position(); - writer.write(record, buf); + writer.writeWithHeaders(record, buf); if (!SKIP_CRC) { int curPos = buf.position(); @@ -228,10 +256,18 @@ public static void writeWithCrc(WALRecord record, ByteBuffer buf, RecordIO write buf.putInt(0); } + /** + * Calculates size of record with headers and CRC. + * + * @param record WAL record to calculate size. + * @param io Record size I/O interface. + * @return Size of record, header and CRC in bytes. + * @throws IgniteCheckedException If it's unable to calculate size of record. + */ public static int sizeWithHeadersAndCrc(WALRecord record, RecordIO io) throws IgniteCheckedException { int commonFields = REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + CRC_SIZE; - int recordSize = io.size(record); + int recordSize = io.sizeWithHeaders(record); switch (record.type()) { case SWITCH_SEGMENT_RECORD: diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java index 343785e9e2503..0688355d29d84 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java @@ -28,25 +28,20 @@ public class RecordV2Serializer implements RecordSerializer { // TODO: Define additional header size here // public static final int RECORD_LENGTH_HEADER_SIZE = 4; - /** */ + /** V2 data serializer. */ private final RecordDataV2Serializer dataSerializer; /** Record read/write functional interface. */ private final RecordIO recordIO = new RecordIO() { - @Override public int size(WALRecord record) throws IgniteCheckedException { + /** {@inheritDoc} */ + @Override public int sizeWithHeaders(WALRecord record) throws IgniteCheckedException { // TODO: Add to size the record length header size return dataSerializer.size(record); } - /** - * Reads record from input, does not read CRC value. - * - * @param in Input to read record from - * @param expPtr expected WAL pointer for record. Used to validate actual position against expected from the file - * @throws SegmentEofException if end of WAL segment reached - */ - @Override public WALRecord read(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException { + /** {@inheritDoc} */ + @Override public WALRecord readWithHeaders(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException { WALRecord.RecordType recType = RecordV1Serializer.readRecordType(in); FileWALPointer ptr = RecordV1Serializer.readPosition(in); @@ -60,14 +55,8 @@ public class RecordV2Serializer implements RecordSerializer { return dataSerializer.readRecord(recType, in); } - /** - * Writes record to output, does not write CRC value. - * - * @param record - * @param buf - * @throws IgniteCheckedException - */ - @Override public void write(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { + /** {@inheritDoc} */ + @Override public void writeWithHeaders(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { // Write record type. RecordV1Serializer.putRecordType(buf, record); @@ -84,7 +73,7 @@ public class RecordV2Serializer implements RecordSerializer { /** * Create an instance of Record V2 serializer. * - * @param dataSerializer Record data V2 serializer. + * @param dataSerializer V2 data serializer. */ public RecordV2Serializer(RecordDataV2Serializer dataSerializer) { this.dataSerializer = dataSerializer; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/io/RecordIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/io/RecordIO.java index d695167e869ca..625371df3edff 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/io/RecordIO.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/io/RecordIO.java @@ -13,28 +13,31 @@ */ public interface RecordIO { /** + * Calculates and returns size of record data and headers. * - * @param record - * @return - * @throws IgniteCheckedException + * @param record WAL record. + * @return Size in bytes. + * @throws IgniteCheckedException If it's unable to calculate size of record. */ - int size(WALRecord record) throws IgniteCheckedException; + int sizeWithHeaders(WALRecord record) throws IgniteCheckedException; /** + * Reads record data with headers from {@code in}. * - * @param in - * @param expPtr - * @return - * @throws IOException - * @throws IgniteCheckedException + * @param in Buffer to read. + * @param expPtr Expected WAL pointer for record. Used to validate actual position against expected from the file. + * @return WAL record. + * @throws IOException In case of I/O problems. + * @throws IgniteCheckedException If it's unable to read record. */ - WALRecord read(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException; + WALRecord readWithHeaders(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException; /** + * Writes record data with headers to {@code buf}. * - * @param record - * @param buf - * @throws IgniteCheckedException + * @param record WAL record. + * @param buf Buffer to write. + * @throws IgniteCheckedException If it's unable to write record. */ - void write(WALRecord record, ByteBuffer buf) throws IgniteCheckedException; + void writeWithHeaders(WALRecord record, ByteBuffer buf) throws IgniteCheckedException; } From d9fe91c86b550e01b6bb5def076f7ca2443f40d7 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Mon, 14 Aug 2017 14:25:45 +0300 Subject: [PATCH 3/9] IGNITE-6029 Fixed compilation. --- .../persistence/wal/reader/StandaloneWalRecordsIterator.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java index 85022ad7fd6e2..40459bfd1db55 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java @@ -38,6 +38,7 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException; +import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordDataV1Serializer; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer; import org.apache.ignite.internal.util.typedef.F; import org.jetbrains.annotations.NotNull; @@ -92,7 +93,7 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator { @NotNull FileIOFactory ioFactory) throws IgniteCheckedException { super(log, sharedCtx, - new RecordV1Serializer(sharedCtx), + new RecordV1Serializer(new RecordDataV1Serializer(sharedCtx)), ioFactory, BUF_SIZE); init(walFilesDir, false, null); @@ -116,7 +117,7 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator { @NotNull File... walFiles) throws IgniteCheckedException { super(log, sharedCtx, - new RecordV1Serializer(sharedCtx), + new RecordV1Serializer(new RecordDataV1Serializer(sharedCtx)), ioFactory, BUF_SIZE); this.workDir = workDir; From 21f96b5372234295ac9b0dab11ea2608f4adbc8b Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Mon, 14 Aug 2017 17:53:08 +0300 Subject: [PATCH 4/9] IGNITE-6029 introduced length in wal point --- .../wal/AbstractWalRecordsIterator.java | 32 ++++-- .../wal/FileWriteAheadLogManager.java | 1 + .../wal/WalSegmentTailReachedException.java | 37 +++++++ .../serializer/RecordDataV1Serializer.java | 34 ++++-- .../serializer/RecordDataV2Serializer.java | 18 +++- .../wal/serializer/RecordV1Serializer.java | 30 +----- .../wal/serializer/RecordV2Serializer.java | 101 ++++++++++++++---- .../wal/serializer/io/RecordIO.java | 17 +++ 8 files changed, 205 insertions(+), 65 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/WalSegmentTailReachedException.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java index beed90b1c4996..2ab8141865d57 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java @@ -141,21 +141,32 @@ protected static FileWriteAheadLogManager.FileDescriptor[] loadFileDescriptors(@ */ protected void advance() throws IgniteCheckedException { while (true) { - curRec = advanceRecord(currWalSegment); - - if (curRec != null) - return; - else { - currWalSegment = advanceSegment(currWalSegment); + try { + curRec = advanceRecord(currWalSegment); - if (currWalSegment == null) + if (curRec != null) return; + else { + currWalSegment = advanceSegment(currWalSegment); + + if (currWalSegment == null) + return; + } + } + catch (WalSegmentTailReachedException e) { + if (log != null) + log.warning(e.getMessage()); + + curRec = null; + + return; } } } /** * Closes and returns WAL segment (if any) + * * @return closed handle * @throws IgniteCheckedException if IO failed */ @@ -186,7 +197,8 @@ protected abstract FileWriteAheadLogManager.ReadFileHandle advanceSegment( * @return next advanced record */ private IgniteBiTuple advanceRecord( - @Nullable final FileWriteAheadLogManager.ReadFileHandle hnd) { + @Nullable final FileWriteAheadLogManager.ReadFileHandle hnd + ) throws IgniteCheckedException { if (hnd == null) return null; @@ -204,8 +216,12 @@ private IgniteBiTuple advanceRecord( return new IgniteBiTuple<>((WALPointer)ptr, rec); } catch (IOException | IgniteCheckedException e) { + if (e instanceof WalSegmentTailReachedException) + throw (WalSegmentTailReachedException)e; + if (!(e instanceof SegmentEofException)) handleRecordException(e, ptr); + return null; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 170237c8e7ef5..d5509556592f3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -1021,6 +1021,7 @@ static RecordSerializer forVersion(GridCacheSharedContext cctx, int ver) throws case 2: RecordDataV2Serializer dataV2Serializer = new RecordDataV2Serializer(new RecordDataV1Serializer(cctx)); + return new RecordV2Serializer(dataV2Serializer); default: diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/WalSegmentTailReachedException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/WalSegmentTailReachedException.java new file mode 100644 index 0000000000000..5a282a0eca691 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/WalSegmentTailReachedException.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.wal; + +import org.apache.ignite.IgniteCheckedException; +import org.jetbrains.annotations.Nullable; + +//todo class description +/** + * + */ +public class WalSegmentTailReachedException extends IgniteCheckedException { + /** */ + private static final long serialVersionUID = 0L; + + /** + * + */ + public WalSegmentTailReachedException(String msg, @Nullable Throwable cause) { + super(msg, cause); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java index 4b12338692414..25bef0b860def 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.processors.cache.persistence.wal.serializer; import java.io.DataInput; @@ -75,6 +92,8 @@ import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; import org.apache.ignite.internal.util.typedef.internal.U; +import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.CRC_SIZE; + /** * Record data V1 serializer. */ @@ -98,8 +117,8 @@ public RecordDataV1Serializer(GridCacheSharedContext cctx) { this.pageSize = cctx.database().pageSize(); } - @Override - public int size(WALRecord record) throws IgniteCheckedException { + /** {@inheritDoc} */ + @Override public int size(WALRecord record) throws IgniteCheckedException { switch (record.type()) { case PAGE_RECORD: assert record instanceof PageSnapshot; @@ -257,15 +276,16 @@ assert record instanceof PageSnapshot; return /*cacheId*/ 4 + /*pageId*/ 8; case SWITCH_SEGMENT_RECORD: - return 0; + // CRC is not loaded for switch segment. + return -CRC_SIZE; default: throw new UnsupportedOperationException("Type: " + record.type()); } } - @Override - public WALRecord readRecord(WALRecord.RecordType type, ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException { + /** {@inheritDoc} */ + @Override public WALRecord readRecord(WALRecord.RecordType type, ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException { WALRecord res; switch (type) { @@ -785,8 +805,8 @@ public WALRecord readRecord(WALRecord.RecordType type, ByteBufferBackedDataInput return res; } - @Override - public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { + /** {@inheritDoc} */ + @Override public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { switch (record.type()) { case PAGE_RECORD: PageSnapshot snap = (PageSnapshot)record; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java index 89a136308ee12..bd23ab3c68641 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.processors.cache.persistence.wal.serializer; import java.io.IOException; @@ -11,7 +28,6 @@ * Record data V2 serializer. */ public class RecordDataV2Serializer implements RecordDataSerializer { - /** V1 data serializer delegate. */ private final RecordDataV1Serializer delegateSerializer; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java index 3a4348317a142..eae717cba8a31 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java @@ -52,10 +52,10 @@ public class RecordV1Serializer implements RecordSerializer { public static final int REC_TYPE_SIZE = 1; /** Length of WAL Pointer */ - public static final int FILE_WAL_POINTER_SIZE = 12; + public static final int FILE_WAL_POINTER_SIZE = 8 + 4; /** Length of CRC value */ - private static final int CRC_SIZE = 4; + public static final int CRC_SIZE = 4; /** Total length of HEADER record. */ public static final int HEADER_RECORD_SIZE = REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + CRC_SIZE + RecordDataV1Serializer.HEADER_RECORD_DATA_SIZE; @@ -71,7 +71,7 @@ public class RecordV1Serializer implements RecordSerializer { /** {@inheritDoc} */ @Override public int sizeWithHeaders(WALRecord record) throws IgniteCheckedException { - return dataSerializer.size(record); + return dataSerializer.size(record) + REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + CRC_SIZE; } /** {@inheritDoc} */ @@ -128,7 +128,7 @@ public RecordV1Serializer(RecordDataV1Serializer dataSerializer) { /** {@inheritDoc} */ @SuppressWarnings("CastConflictsWithInstanceof") @Override public int size(WALRecord record) throws IgniteCheckedException { - return sizeWithHeadersAndCrc(record, recordIO); + return recordIO.sizeWithHeaders(record); } /** @@ -255,26 +255,4 @@ public static void writeWithCrc(WALRecord record, ByteBuffer buf, RecordIO write else buf.putInt(0); } - - /** - * Calculates size of record with headers and CRC. - * - * @param record WAL record to calculate size. - * @param io Record size I/O interface. - * @return Size of record, header and CRC in bytes. - * @throws IgniteCheckedException If it's unable to calculate size of record. - */ - public static int sizeWithHeadersAndCrc(WALRecord record, RecordIO io) throws IgniteCheckedException { - int commonFields = REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + CRC_SIZE; - - int recordSize = io.sizeWithHeaders(record); - - switch (record.type()) { - case SWITCH_SEGMENT_RECORD: - return commonFields + recordSize - CRC_SIZE; //CRC is not loaded for switch segment, exception is thrown instead - - default: - return commonFields + recordSize; - } - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java index 0688355d29d84..2be1c78572bfb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.processors.cache.persistence.wal.serializer; +import java.io.DataInput; import java.io.IOException; import java.nio.ByteBuffer; import org.apache.ignite.IgniteCheckedException; @@ -9,24 +27,27 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; import org.apache.ignite.internal.processors.cache.persistence.wal.RecordSerializer; -import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException; +import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.io.RecordIO; import org.apache.ignite.internal.util.typedef.F; +import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.CRC_SIZE; +import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.REC_TYPE_SIZE; + /** * Record V2 serializer. * Stores records in following format: *
    - *
  • Record type from {@link WALRecord.RecordType#ordinal()} incremented by 1
    • - *
  • WAL pointer to double check consistency
    • - * TODO: record data length - *
  • Data
    • - *
  • CRC or zero padding
    • + *
  • Record type from {@link WALRecord.RecordType#ordinal()} incremented by 1
    • + *
  • WAL pointer to double check consistency
    • + *
  • Record length
    • + *
  • Data
    • + *
  • CRC or zero padding
    • *
*/ public class RecordV2Serializer implements RecordSerializer { - // TODO: Define additional header size here - // public static final int RECORD_LENGTH_HEADER_SIZE = 4; + /** */ + public static final int FILE_WAL_POINTER_SIZE = 8 + 4 + 4; /** V2 data serializer. */ private final RecordDataV2Serializer dataSerializer; @@ -36,34 +57,31 @@ public class RecordV2Serializer implements RecordSerializer { /** {@inheritDoc} */ @Override public int sizeWithHeaders(WALRecord record) throws IgniteCheckedException { - // TODO: Add to size the record length header size - return dataSerializer.size(record); + return dataSerializer.size(record) + REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + CRC_SIZE; } /** {@inheritDoc} */ - @Override public WALRecord readWithHeaders(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException { + @Override public WALRecord readWithHeaders( + ByteBufferBackedDataInput in, + WALPointer expPtr + ) throws IOException, IgniteCheckedException { WALRecord.RecordType recType = RecordV1Serializer.readRecordType(in); - FileWALPointer ptr = RecordV1Serializer.readPosition(in); - - if (!F.eq(ptr, expPtr)) - throw new SegmentEofException("WAL segment rollover detected (will end iteration) [expPtr=" + expPtr + - ", readPtr=" + ptr + ']', null); - - //TODO read record length here + FileWALPointer ptr = readPositionAndCheckPoint(in, expPtr); return dataSerializer.readRecord(recType, in); } /** {@inheritDoc} */ - @Override public void writeWithHeaders(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { + @Override public void writeWithHeaders( + WALRecord record, + ByteBuffer buf + ) throws IgniteCheckedException { // Write record type. RecordV1Serializer.putRecordType(buf, record); // Write record file position. - RecordV1Serializer.putPositionOfRecord(buf, record); - - // TODO write record length here. + putPositionOfRecord(buf, record); // Write record data. dataSerializer.writeRecord(record, buf); @@ -86,7 +104,7 @@ public RecordV2Serializer(RecordDataV2Serializer dataSerializer) { /** {@inheritDoc} */ @Override public int size(WALRecord record) throws IgniteCheckedException { - return RecordV1Serializer.sizeWithHeadersAndCrc(record, recordIO); + return recordIO.sizeWithHeaders(record); } /** {@inheritDoc} */ @@ -98,4 +116,41 @@ public RecordV2Serializer(RecordDataV2Serializer dataSerializer) { @Override public WALRecord readRecord(FileInput in, WALPointer expPtr) throws IOException, IgniteCheckedException { return RecordV1Serializer.readWithCrc(in, expPtr, recordIO); } + + /** + * @param in Data input to read pointer from. + * @return Read file WAL pointer. + * @throws IOException If failed to write. + */ + public static FileWALPointer readPositionAndCheckPoint( + DataInput in, + WALPointer expPtr + ) throws IgniteCheckedException, IOException { + long idx = in.readLong(); + int fileOffset = in.readInt(); + int length = in.readInt(); + + FileWALPointer p = (FileWALPointer)expPtr; + + //todo msg rework + if (!F.eq(idx, p.index()) || !F.eq(fileOffset, p.fileOffset())) + throw new WalSegmentTailReachedException( + "WAL segment rollover detected (will end iteration) [expPtr=" + expPtr + ", readPtr=" + null + ']', null); + + return new FileWALPointer(idx, fileOffset, length); + } + + /** + * Writes record file position to given {@code buf}. + * + * @param buf Buffer to write record file position. + * @param record WAL record. + */ + public static void putPositionOfRecord(ByteBuffer buf, WALRecord record) { + FileWALPointer p = (FileWALPointer)record.position(); + + buf.putLong(p.index()); + buf.putInt(p.fileOffset()); + buf.putInt(record.size()); + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/io/RecordIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/io/RecordIO.java index 625371df3edff..d609e612d74ce 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/io/RecordIO.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/io/RecordIO.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.processors.cache.persistence.wal.serializer.io; import java.io.IOException; From 6cca330e7f30fc62285914f0a3ee1b98ca699aa1 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Tue, 15 Aug 2017 13:12:36 +0300 Subject: [PATCH 5/9] IGNITE-6029 Extract read/write header version record. --- .../wal/AbstractWalRecordsIterator.java | 23 +-- .../wal/FileWriteAheadLogManager.java | 144 ++++++++++++++---- .../wal/WalSegmentTailReachedException.java | 4 +- .../serializer/RecordDataV2Serializer.java | 7 + .../wal/serializer/RecordV1Serializer.java | 2 +- .../wal/serializer/RecordV2Serializer.java | 7 +- 6 files changed, 133 insertions(+), 54 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java index 2ab8141865d57..9bc1373372006 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java @@ -30,7 +30,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; -import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; import org.apache.ignite.internal.util.GridCloseableIteratorAdapter; import org.apache.ignite.lang.IgniteBiTuple; import org.jetbrains.annotations.NotNull; @@ -253,24 +252,18 @@ protected FileWriteAheadLogManager.ReadFileHandle initReadHandle( FileIO fileIO = ioFactory.create(desc.file, "r"); try { - FileInput in = new FileInput(fileIO, buf); - - // Header record must be agnostic to the serializer version. - WALRecord rec = serializer.readRecord(in, - new FileWALPointer(desc.idx, (int)fileIO.position(), 0)); - - if (rec == null) - return null; + int serVer = FileWriteAheadLogManager.readSerializerVersion(fileIO, desc.file); - if (rec.type() != WALRecord.RecordType.HEADER_RECORD) - throw new IOException("Missing file header record: " + desc.file.getAbsoluteFile()); + RecordSerializer ser = FileWriteAheadLogManager.forVersion(sharedCtx, serVer); - int ver = ((HeaderRecord)rec).version(); + FileInput in = new FileInput(fileIO, buf); - RecordSerializer ser = FileWriteAheadLogManager.forVersion(sharedCtx, ver); + if (start != null && desc.idx == start.index()) { + // Make sure we skip header with serializer version. + long startOffset = Math.max(start.fileOffset(), fileIO.position()); - if (start != null && desc.idx == start.index()) - in.seek(start.fileOffset()); + in.seek(startOffset); + } return new FileWriteAheadLogManager.ReadFileHandle(fileIO, desc.idx, sharedCtx.igniteInstanceName(), ser, in); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index d5509556592f3..32f68e1ea4b57 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.cache.persistence.wal; -import java.io.EOFException; import java.io.File; import java.io.FileFilter; import java.io.FileNotFoundException; @@ -62,6 +61,7 @@ import org.apache.ignite.internal.processors.cache.persistence.PersistenceMetricsImpl; import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordDataV1Serializer; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordDataV2Serializer; @@ -150,9 +150,12 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl /** WAL archive directory (including consistent ID as subfolder) */ private File walArchiveDir; - /** Serializer of current version, used to read header record and for write records */ + /** Serializer of latest version. */ private RecordSerializer serializer; + /** Serializer latest version to use. */ + private int serializerVersion = 2; + /** */ private volatile long oldestArchiveSegmentIdx; @@ -264,7 +267,7 @@ public FileWriteAheadLogManager(@NotNull final GridKernalContext ctx) { "write ahead log archive directory" ); - serializer = new RecordV1Serializer(new RecordDataV1Serializer(cctx)); + serializer = forVersion(cctx, serializerVersion); GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)cctx.database(); @@ -816,10 +819,11 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig FileIO fileIO = ioFactory.create(curFile); try { - // readSerializerVersion will change the channel position. - // This is fine because the FileWriteHandle consitructor will move it - // to offset + len anyways. - int serVer = readSerializerVersion(fileIO, curFile, absIdx); + int serVer = serializerVersion; + + // If we have existing segment, try to read version from it. + if (lastReadPtr != null) + serVer = readSerializerVersion(fileIO, curFile, serializerVersion); RecordSerializer ser = forVersion(cctx, serVer); @@ -835,13 +839,9 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig maxWalSegmentSize, ser); - if (lastReadPtr == null) { - HeaderRecord header = new HeaderRecord(serializer.version()); - - header.size(serializer.size(header)); - - hnd.addRecord(header); - } + // For new handle write serializer version to it. + if (lastReadPtr == null) + hnd.writeSerializerVersion(); archiver.currentWalIndex(absIdx); @@ -885,11 +885,7 @@ private FileWriteHandle initNextWriteHandle(long curIdx) throws StorageException maxWalSegmentSize, serializer); - HeaderRecord header = new HeaderRecord(serializer.version()); - - header.size(serializer.size(header)); - - hnd.addRecord(header); + hnd.writeSerializerVersion(); return hnd; } @@ -1429,35 +1425,97 @@ else if (create) } /** + * Reads record serializer version from provided {@code io}. + * NOTE: Method mutates position of {@code io}. + * * @param io I/O interface for file. * @param file File object. - * @param idx File index to read. * @return Serializer version stored in the file. - * @throws IOException If failed to read serializer version. * @throws IgniteCheckedException If failed to read serializer version. */ - private int readSerializerVersion(FileIO io, File file, long idx) - throws IOException, IgniteCheckedException { + public static int readSerializerVersion(FileIO io, File file) + throws IgniteCheckedException, IOException { try { - ByteBuffer buf = ByteBuffer.allocate(RecordV1Serializer.HEADER_RECORD_SIZE); - buf.order(ByteOrder.nativeOrder()); - FileInput in = new FileInput(io, new ByteBufferExpander(RecordV1Serializer.HEADER_RECORD_SIZE, ByteOrder.nativeOrder())); - // Header record must be agnostic to the serializer version. - WALRecord rec = serializer.readRecord(in, new FileWALPointer(idx, 0, 0)); + // Reserve Header record bytes. + in.ensure(RecordV1Serializer.HEADER_RECORD_SIZE); + + int recordType = in.readByte(); + + WALRecord.RecordType type = WALRecord.RecordType.fromOrdinal(recordType); + + if (type != WALRecord.RecordType.HEADER_RECORD) + throw new IOException("Can't read serializer version", null); - if (rec.type() != WALRecord.RecordType.HEADER_RECORD) - throw new IOException("Missing file header record: " + file.getAbsoluteFile()); + // Skip record type and file pointer fields. + in.seek(RecordV1Serializer.REC_TYPE_SIZE + RecordV1Serializer.FILE_WAL_POINTER_SIZE); - return ((HeaderRecord)rec).version(); + long headerMagicNumber = in.readLong(); + + if (headerMagicNumber != HeaderRecord.MAGIC) + throw new IOException("Magic is corrupted [exp=" + U.hexLong(HeaderRecord.MAGIC) + + ", actual=" + U.hexLong(headerMagicNumber) + ']'); + + // Read serializer version. + int version = in.readInt(); + + // Read and skip CRC. + int crc = in.readInt(); + + return version; } - catch (SegmentEofException | EOFException ignore) { - return serializer.version(); + catch (IOException e) { + throw new IgniteCheckedException("Unable to read serializer version: " + file.getAbsoluteFile(), e); } } + /** + * Writes record serializer version to provided {@code io}. + * NOTE: Method mutates position of {@code io}. + * + * @param io I/O interface for file. + * @param version Serializer version. + * @throws IOException If failed to write serializer version. + */ + public static long writeSerializerVersion(FileIO io, int version) throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(RecordV1Serializer.HEADER_RECORD_SIZE); + buffer.order(ByteOrder.nativeOrder()); + + // Write record type. + buffer.put((byte) WALRecord.RecordType.HEADER_RECORD.ordinal()); + + // Skip file pointer field. + buffer.position(buffer.position() + RecordV1Serializer.FILE_WAL_POINTER_SIZE); + + // Place magic number. + buffer.putLong(HeaderRecord.MAGIC); + + // Place serializer version. + buffer.putInt(version); + + // Place CRC if needed. + if (!RecordV1Serializer.SKIP_CRC) { + int curPos = buffer.position(); + + buffer.position(0); + + // This call will move buffer position to the end of the record again. + int crcVal = PureJavaCrc32.calcCrc32(buffer, curPos); + + buffer.putInt(crcVal); + } + else + buffer.putInt(0); + + io.write(buffer); + + io.force(); + + return io.position(); + } + /** * WAL file descriptor. */ @@ -1713,6 +1771,26 @@ private FileWriteHandle( lastFsyncPos = pos; } + /** + * Write serializer version to current handle. + * NOTE: Method mutates {@code fileIO} position, written and lastFsyncPos fields. + * + * @throws IgniteCheckedException If fail to write serializer version. + */ + public void writeSerializerVersion() throws IgniteCheckedException { + try { + assert fileIO.position() == 0 : "Serializer version can be written only at the begin of file " + fileIO.position(); + + long updatedPosition = FileWriteAheadLogManager.writeSerializerVersion(fileIO, serializer.version()); + + written = updatedPosition; + lastFsyncPos = updatedPosition; + } + catch (IOException e) { + throw new IgniteCheckedException("Unable to write serializer version for segment " + idx, e); + } + } + /** * Checks if current head is a close fake record and returns {@code true} if so. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/WalSegmentTailReachedException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/WalSegmentTailReachedException.java index 5a282a0eca691..36298dcf97756 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/WalSegmentTailReachedException.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/WalSegmentTailReachedException.java @@ -20,9 +20,9 @@ import org.apache.ignite.IgniteCheckedException; import org.jetbrains.annotations.Nullable; -//todo class description /** - * + * An exception is thrown when we reached tail of WAL segment cyclic buffer + * during reading from WAL. */ public class WalSegmentTailReachedException extends IgniteCheckedException { /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java index bd23ab3c68641..2b55c5f3599e8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java @@ -23,6 +23,7 @@ import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput; import org.apache.ignite.internal.processors.cache.persistence.wal.RecordDataSerializer; +import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; /** * Record data V2 serializer. @@ -42,6 +43,9 @@ public RecordDataV2Serializer(RecordDataV1Serializer delegateSerializer) { /** {@inheritDoc} */ @Override public int size(WALRecord record) throws IgniteCheckedException { + if (record instanceof HeaderRecord) + throw new UnsupportedOperationException("Getting size of header records is forbidden since version 2 of serializer"); + return delegateSerializer.size(record); } @@ -52,6 +56,9 @@ public RecordDataV2Serializer(RecordDataV1Serializer delegateSerializer) { /** {@inheritDoc} */ @Override public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException { + if (record instanceof HeaderRecord) + throw new UnsupportedOperationException("Writing header records is forbidden since version 2 of serializer"); + delegateSerializer.writeRecord(record, buf); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java index eae717cba8a31..9b083943a8470 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java @@ -51,7 +51,7 @@ public class RecordV1Serializer implements RecordSerializer { /** Length of Type */ public static final int REC_TYPE_SIZE = 1; - /** Length of WAL Pointer */ + /** Length of WAL Pointer: Index (8) + File offset (4). */ public static final int FILE_WAL_POINTER_SIZE = 8 + 4; /** Length of CRC value */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java index 2be1c78572bfb..6b9394bb6cc1c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java @@ -46,7 +46,7 @@ * */ public class RecordV2Serializer implements RecordSerializer { - /** */ + /** Length of WAL Pointer: Index (8) + File offset (4) + Record length (4) */ public static final int FILE_WAL_POINTER_SIZE = 8 + 4 + 4; /** V2 data serializer. */ @@ -132,10 +132,11 @@ public static FileWALPointer readPositionAndCheckPoint( FileWALPointer p = (FileWALPointer)expPtr; - //todo msg rework if (!F.eq(idx, p.index()) || !F.eq(fileOffset, p.fileOffset())) throw new WalSegmentTailReachedException( - "WAL segment rollover detected (will end iteration) [expPtr=" + expPtr + ", readPtr=" + null + ']', null); + "WAL segment tail is reached. [ " + + "Expected next state: {Index=" + p.index() + ",Offset=" + p.fileOffset() + "}, " + + "Actual state : {Index=" + idx + ",Offset=" + fileOffset + "} ]", null); return new FileWALPointer(idx, fileOffset, length); } From e20a7b66512f4bc3833cc3137aff3e4bffe3e783 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 16 Aug 2017 13:27:51 +0300 Subject: [PATCH 6/9] IGNITE-6029 Reworked writing serializer version and Switch segment record. --- .../wal/record/SwitchSegmentRecord.java | 28 ++++++ .../wal/AbstractWalRecordsIterator.java | 4 +- .../wal/FileWriteAheadLogManager.java | 86 +++++++++++-------- .../serializer/RecordDataV1Serializer.java | 3 + .../wal/serializer/RecordV1Serializer.java | 8 +- .../wal/serializer/RecordV2Serializer.java | 4 + .../db/wal/IgniteWalRecoveryTest.java | 5 +- 7 files changed, 98 insertions(+), 40 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SwitchSegmentRecord.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SwitchSegmentRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SwitchSegmentRecord.java new file mode 100644 index 0000000000000..948ec7e27fa5b --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SwitchSegmentRecord.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.pagemem.wal.record; + +/** + * Record is needed to mark end of segment. + */ +public class SwitchSegmentRecord extends WALRecord { + /** {@inheritDoc} */ + @Override public RecordType type() { + return RecordType.SWITCH_SEGMENT_RECORD; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java index 9bc1373372006..8e88eca89dd8f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java @@ -252,7 +252,7 @@ protected FileWriteAheadLogManager.ReadFileHandle initReadHandle( FileIO fileIO = ioFactory.create(desc.file, "r"); try { - int serVer = FileWriteAheadLogManager.readSerializerVersion(fileIO, desc.file); + int serVer = FileWriteAheadLogManager.readSerializerVersion(fileIO); RecordSerializer ser = FileWriteAheadLogManager.forVersion(sharedCtx, serVer); @@ -267,7 +267,7 @@ protected FileWriteAheadLogManager.ReadFileHandle initReadHandle( return new FileWriteAheadLogManager.ReadFileHandle(fileIO, desc.idx, sharedCtx.igniteInstanceName(), ser, in); } - catch (SegmentEofException | EOFException ignore) { + catch (SegmentEofException | EOFException | WalSegmentTailReachedException ignore) { try { fileIO.close(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 32f68e1ea4b57..431a5107e10cf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache.persistence.wal; +import java.io.EOFException; import java.io.File; import java.io.FileFilter; import java.io.FileNotFoundException; @@ -54,6 +55,7 @@ import org.apache.ignite.internal.pagemem.wal.WALIterator; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord; +import org.apache.ignite.internal.pagemem.wal.record.SwitchSegmentRecord; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; @@ -822,8 +824,14 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig int serVer = serializerVersion; // If we have existing segment, try to read version from it. - if (lastReadPtr != null) - serVer = readSerializerVersion(fileIO, curFile, serializerVersion); + if (lastReadPtr != null) { + try { + serVer = readSerializerVersion(fileIO); + } + catch (SegmentEofException | EOFException | WalSegmentTailReachedException ignore) { + serVer = serializerVersion; + } + } RecordSerializer ser = forVersion(cctx, serVer); @@ -1429,46 +1437,43 @@ else if (create) * NOTE: Method mutates position of {@code io}. * * @param io I/O interface for file. - * @param file File object. * @return Serializer version stored in the file. * @throws IgniteCheckedException If failed to read serializer version. */ - public static int readSerializerVersion(FileIO io, File file) + public static int readSerializerVersion(FileIO io) throws IgniteCheckedException, IOException { - try { - FileInput in = new FileInput(io, - new ByteBufferExpander(RecordV1Serializer.HEADER_RECORD_SIZE, ByteOrder.nativeOrder())); + FileInput in = new FileInput(io, + new ByteBufferExpander(RecordV1Serializer.HEADER_RECORD_SIZE, ByteOrder.nativeOrder())); - // Reserve Header record bytes. - in.ensure(RecordV1Serializer.HEADER_RECORD_SIZE); + in.ensure(RecordV1Serializer.HEADER_RECORD_SIZE); - int recordType = in.readByte(); + int recordType = in.readUnsignedByte(); - WALRecord.RecordType type = WALRecord.RecordType.fromOrdinal(recordType); + if (recordType == WALRecord.RecordType.STOP_ITERATION_RECORD_TYPE) + throw new SegmentEofException("Reached logical end of the segment", null); - if (type != WALRecord.RecordType.HEADER_RECORD) - throw new IOException("Can't read serializer version", null); + WALRecord.RecordType type = WALRecord.RecordType.fromOrdinal(recordType - 1); - // Skip record type and file pointer fields. - in.seek(RecordV1Serializer.REC_TYPE_SIZE + RecordV1Serializer.FILE_WAL_POINTER_SIZE); + if (type != WALRecord.RecordType.HEADER_RECORD) + throw new IOException("Can't read serializer version", null); - long headerMagicNumber = in.readLong(); + // Read and skip file pointer. + in.readLong(); + in.readInt(); - if (headerMagicNumber != HeaderRecord.MAGIC) - throw new IOException("Magic is corrupted [exp=" + U.hexLong(HeaderRecord.MAGIC) + - ", actual=" + U.hexLong(headerMagicNumber) + ']'); + long headerMagicNumber = in.readLong(); - // Read serializer version. - int version = in.readInt(); + if (headerMagicNumber != HeaderRecord.MAGIC) + throw new IOException("Magic is corrupted [exp=" + U.hexLong(HeaderRecord.MAGIC) + + ", actual=" + U.hexLong(headerMagicNumber) + ']'); - // Read and skip CRC. - int crc = in.readInt(); + // Read serializer version. + int version = in.readInt(); - return version; - } - catch (IOException e) { - throw new IgniteCheckedException("Unable to read serializer version: " + file.getAbsoluteFile(), e); - } + // Read and skip CRC. + in.readInt(); + + return version; } /** @@ -1477,6 +1482,7 @@ public static int readSerializerVersion(FileIO io, File file) * * @param io I/O interface for file. * @param version Serializer version. + * @return I/O position after write version. * @throws IOException If failed to write serializer version. */ public static long writeSerializerVersion(FileIO io, int version) throws IOException { @@ -1484,7 +1490,7 @@ public static long writeSerializerVersion(FileIO io, int version) throws IOExcep buffer.order(ByteOrder.nativeOrder()); // Write record type. - buffer.put((byte) WALRecord.RecordType.HEADER_RECORD.ordinal()); + buffer.put((byte) (WALRecord.RecordType.HEADER_RECORD.ordinal() + 1)); // Skip file pointer field. buffer.position(buffer.position() + RecordV1Serializer.FILE_WAL_POINTER_SIZE); @@ -1509,8 +1515,15 @@ public static long writeSerializerVersion(FileIO io, int version) throws IOExcep else buffer.putInt(0); - io.write(buffer); + // Write header record through io. + buffer.position(0); + + do { + io.write(buffer); + } + while (buffer.hasRemaining()); + // Flush io.force(); return io.position(); @@ -1785,6 +1798,7 @@ public void writeSerializerVersion() throws IgniteCheckedException { written = updatedPosition; lastFsyncPos = updatedPosition; + head.set(new FakeRecord(new FileWALPointer(idx, (int)updatedPosition, 0), false)); } catch (IOException e) { throw new IgniteCheckedException("Unable to write serializer version for segment " + idx, e); @@ -2151,15 +2165,17 @@ private boolean close(boolean rollOver) throws IgniteCheckedException, StorageEx assert stopped() : "Segment is not closed after close flush: " + head.get(); try { - int switchSegmentRecSize = RecordV1Serializer.REC_TYPE_SIZE + RecordV1Serializer.FILE_WAL_POINTER_SIZE; + int switchSegmentRecSize = RecordV1Serializer.REC_TYPE_SIZE + RecordV1Serializer.FILE_WAL_POINTER_SIZE + RecordV1Serializer.CRC_SIZE; if (rollOver && written < (maxSegmentSize - switchSegmentRecSize)) { - //it is expected there is sufficient space for this record because rollover should run early + RecordV1Serializer backwardSerializer = new RecordV1Serializer(new RecordDataV1Serializer(cctx)); + final ByteBuffer buf = ByteBuffer.allocate(switchSegmentRecSize); - buf.put((byte)(WALRecord.RecordType.SWITCH_SEGMENT_RECORD.ordinal() + 1)); - final FileWALPointer pointer = new FileWALPointer(idx, (int)fileIO.position(), -1); - RecordV1Serializer.putPosition(buf, pointer); + SwitchSegmentRecord segmentRecord = new SwitchSegmentRecord(); + segmentRecord.position(new FileWALPointer(idx, (int) written, -1)); + + backwardSerializer.writeRecord(segmentRecord, buf); buf.rewind(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java index 25bef0b860def..8f9c89c7b0e19 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java @@ -1290,6 +1290,9 @@ assert record instanceof PageSnapshot; break; + case SWITCH_SEGMENT_RECORD: + break; + default: throw new UnsupportedOperationException("Type: " + record.type()); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java index 9b083943a8470..5336034c241fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java @@ -31,6 +31,7 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; import org.apache.ignite.internal.processors.cache.persistence.wal.RecordSerializer; import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException; +import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.io.RecordIO; import org.apache.ignite.internal.util.typedef.F; @@ -78,6 +79,9 @@ public class RecordV1Serializer implements RecordSerializer { @Override public WALRecord readWithHeaders(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException { RecordType recType = readRecordType(in); + if (recType == RecordType.SWITCH_SEGMENT_RECORD) + throw new SegmentEofException("Reached end of segment", null); + FileWALPointer ptr = readPosition(in); if (!F.eq(ptr, expPtr)) @@ -219,7 +223,7 @@ public static WALRecord readWithCrc(FileInput in0, WALPointer expPtr, RecordIO r return res; } - catch (EOFException | SegmentEofException e) { + catch (EOFException | SegmentEofException | WalSegmentTailReachedException e) { throw e; } catch (Exception e) { @@ -236,7 +240,7 @@ public static WALRecord readWithCrc(FileInput in0, WALPointer expPtr, RecordIO r * @throws IgniteCheckedException If it's unable to write record. */ public static void writeWithCrc(WALRecord record, ByteBuffer buf, RecordIO writer) throws IgniteCheckedException { - assert record.size() > 0 && buf.remaining() >= record.size() : record.size(); + assert record.size() >= 0 && buf.remaining() >= record.size() : record.size(); int startPos = buf.position(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java index 6b9394bb6cc1c..833e8d5ab37be 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java @@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; import org.apache.ignite.internal.processors.cache.persistence.wal.RecordSerializer; +import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException; import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.io.RecordIO; import org.apache.ignite.internal.util.typedef.F; @@ -67,6 +68,9 @@ public class RecordV2Serializer implements RecordSerializer { ) throws IOException, IgniteCheckedException { WALRecord.RecordType recType = RecordV1Serializer.readRecordType(in); + if (recType == WALRecord.RecordType.SWITCH_SEGMENT_RECORD) + throw new SegmentEofException("Reached end of segment", null); + FileWALPointer ptr = readPositionAndCheckPoint(in, expPtr); return dataSerializer.readRecord(recType, in); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java index c5d6a8b16d7c3..925709dd8f641 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java @@ -736,7 +736,7 @@ public void testRandomCrash() throws Exception { rmt.run(new AsyncLoadRunnable()); - Thread.sleep(20_000); + Thread.sleep(30_000); info(">>> Killing remote process..."); @@ -1147,6 +1147,9 @@ private static class VerifyCallable implements IgniteCallable { } }, 10_000); + if (successfulWaiting == false) { + int k = 2; + } assertTrue(successfulWaiting); } catch (IgniteInterruptedCheckedException e) { From 5f9b82e0b74129f672972012e510e07dcad1971c Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 16 Aug 2017 13:49:19 +0300 Subject: [PATCH 7/9] IGNITE-6029 Remove trash from code. --- .../cache/persistence/db/wal/IgniteWalRecoveryTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java index 925709dd8f641..c5d6a8b16d7c3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java @@ -736,7 +736,7 @@ public void testRandomCrash() throws Exception { rmt.run(new AsyncLoadRunnable()); - Thread.sleep(30_000); + Thread.sleep(20_000); info(">>> Killing remote process..."); @@ -1147,9 +1147,6 @@ private static class VerifyCallable implements IgniteCallable { } }, 10_000); - if (successfulWaiting == false) { - int k = 2; - } assertTrue(successfulWaiting); } catch (IgniteInterruptedCheckedException e) { From bc7aab16b51f883f5afc55e5dd2fbae4b1906b00 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 16 Aug 2017 16:43:29 +0300 Subject: [PATCH 8/9] IGNITE-6029 Remove unnecessary exceptions. --- .../cache/persistence/wal/AbstractWalRecordsIterator.java | 2 +- .../cache/persistence/wal/FileWriteAheadLogManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java index 8e88eca89dd8f..1a881ff85c274 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java @@ -267,7 +267,7 @@ protected FileWriteAheadLogManager.ReadFileHandle initReadHandle( return new FileWriteAheadLogManager.ReadFileHandle(fileIO, desc.idx, sharedCtx.igniteInstanceName(), ser, in); } - catch (SegmentEofException | EOFException | WalSegmentTailReachedException ignore) { + catch (SegmentEofException | EOFException ignore) { try { fileIO.close(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 431a5107e10cf..a18bfbcce1ccd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -828,7 +828,7 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig try { serVer = readSerializerVersion(fileIO); } - catch (SegmentEofException | EOFException | WalSegmentTailReachedException ignore) { + catch (SegmentEofException | EOFException ignore) { serVer = serializerVersion; } } From 4e952b4d649031c04ad58a8505f69a1f0cc4649f Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 16 Aug 2017 17:32:20 +0300 Subject: [PATCH 9/9] IGNITE-6029 Place actual file pointer for Header record. Supported StandaloneWalRecords with possibility to change serializer version. --- .../wal/AbstractWalRecordsIterator.java | 3 +-- .../wal/FileWriteAheadLogManager.java | 23 +++++++++++-------- .../wal/reader/IgniteWalIteratorFactory.java | 11 +++++---- .../reader/StandaloneWalRecordsIterator.java | 10 ++++---- .../db/wal/reader/IgniteWalReaderTest.java | 2 +- 5 files changed, 29 insertions(+), 20 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java index 585573681c0c3..7e17575346a70 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java @@ -162,8 +162,7 @@ protected void advance() throws IgniteCheckedException { } } catch (WalSegmentTailReachedException e) { - if (log != null) - log.warning(e.getMessage()); + log.warning(e.getMessage()); curRec = null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index b841f890acf3d..1775186efca00 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -116,6 +116,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl } }; + /** Latest serializer version to use. */ + public static final int LATEST_SERIALIZER_VERSION = 2; + /** */ private final boolean alwaysWriteFullPages; @@ -156,7 +159,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl private RecordSerializer serializer; /** Serializer latest version to use. */ - private int serializerVersion = 2; + private int serializerVersion = LATEST_SERIALIZER_VERSION; /** */ private volatile long oldestArchiveSegmentIdx; @@ -1015,7 +1018,7 @@ private File pollNextFile(long curIdx) throws IgniteCheckedException { * @param ver Serializer version. * @return Entry serializer. */ - static RecordSerializer forVersion(GridCacheSharedContext cctx, int ver) throws IgniteCheckedException { + public static RecordSerializer forVersion(GridCacheSharedContext cctx, int ver) throws IgniteCheckedException { if (ver <= 0) throw new IgniteCheckedException("Failed to create a serializer (corrupted WAL file)."); @@ -1457,9 +1460,10 @@ public static int readSerializerVersion(FileIO io) if (type != WALRecord.RecordType.HEADER_RECORD) throw new IOException("Can't read serializer version", null); - // Read and skip file pointer. - in.readLong(); - in.readInt(); + // Read file pointer. + FileWALPointer ptr = RecordV1Serializer.readPosition(in); + + assert ptr.fileOffset() == 0 : "Header record should be placed at the beginning of file " + ptr; long headerMagicNumber = in.readLong(); @@ -1482,19 +1486,20 @@ public static int readSerializerVersion(FileIO io) * NOTE: Method mutates position of {@code io}. * * @param io I/O interface for file. + * @param idx Segment index. * @param version Serializer version. * @return I/O position after write version. * @throws IOException If failed to write serializer version. */ - public static long writeSerializerVersion(FileIO io, int version) throws IOException { + public static long writeSerializerVersion(FileIO io, long idx, int version) throws IOException { ByteBuffer buffer = ByteBuffer.allocate(RecordV1Serializer.HEADER_RECORD_SIZE); buffer.order(ByteOrder.nativeOrder()); // Write record type. buffer.put((byte) (WALRecord.RecordType.HEADER_RECORD.ordinal() + 1)); - // Skip file pointer field. - buffer.position(buffer.position() + RecordV1Serializer.FILE_WAL_POINTER_SIZE); + // Write position. + RecordV1Serializer.putPosition(buffer, new FileWALPointer(idx, 0, 0)); // Place magic number. buffer.putLong(HeaderRecord.MAGIC); @@ -1795,7 +1800,7 @@ public void writeSerializerVersion() throws IgniteCheckedException { try { assert fileIO.position() == 0 : "Serializer version can be written only at the begin of file " + fileIO.position(); - long updatedPosition = FileWriteAheadLogManager.writeSerializerVersion(fileIO, serializer.version()); + long updatedPosition = FileWriteAheadLogManager.writeSerializerVersion(fileIO, idx, serializer.version()); written = updatedPosition; lastFsyncPos = updatedPosition; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java index 4e3998bf70853..ceb63953ba759 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java @@ -35,6 +35,8 @@ public class IgniteWalIteratorFactory { private final IgniteLogger log; /** Page size, in standalone iterator mode this value can't be taken from memory configuration */ private final int pageSize; + /** Version of serializer to read WAL. */ + private final int serializerVersion; /** Factory to provide I/O interfaces for read/write operations with files */ private final FileIOFactory ioFactory; @@ -43,10 +45,11 @@ public class IgniteWalIteratorFactory { * @param log Logger. * @param pageSize Page size, size is validated */ - public IgniteWalIteratorFactory(@NotNull IgniteLogger log, @NotNull FileIOFactory ioFactory, int pageSize) { + public IgniteWalIteratorFactory(@NotNull IgniteLogger log, @NotNull FileIOFactory ioFactory, int serializerVersion, int pageSize) { this.log = log; this.pageSize = pageSize; this.ioFactory = ioFactory; + this.serializerVersion = serializerVersion; new MemoryConfiguration().setPageSize(pageSize); // just for validate } @@ -61,7 +64,7 @@ public IgniteWalIteratorFactory(@NotNull IgniteLogger log, @NotNull FileIOFactor * @throws IgniteCheckedException if failed to read folder */ public WALIterator iteratorArchiveDirectory(@NotNull final File walDirWithConsistentId) throws IgniteCheckedException { - return new StandaloneWalRecordsIterator(walDirWithConsistentId, log, prepareSharedCtx(), ioFactory); + return new StandaloneWalRecordsIterator(walDirWithConsistentId, log, prepareSharedCtx(), ioFactory, serializerVersion); } /** @@ -73,7 +76,7 @@ public WALIterator iteratorArchiveDirectory(@NotNull final File walDirWithConsis * @throws IgniteCheckedException if failed to read files */ public WALIterator iteratorArchiveFiles(@NotNull final File ...files) throws IgniteCheckedException { - return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, false, files); + return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, serializerVersion, false, files); } /** @@ -85,7 +88,7 @@ public WALIterator iteratorArchiveFiles(@NotNull final File ...files) throws Ign * @throws IgniteCheckedException if failed to read files */ public WALIterator iteratorWorkFiles(@NotNull final File ...files) throws IgniteCheckedException { - return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, true, files); + return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, serializerVersion, true, files); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java index 13cdfb511b5ee..9558c24ef199b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java @@ -38,7 +38,6 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException; -import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordDataV1Serializer; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer; import org.apache.ignite.internal.util.typedef.F; import org.jetbrains.annotations.NotNull; @@ -90,10 +89,12 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator { @NotNull File walFilesDir, @NotNull IgniteLogger log, @NotNull GridCacheSharedContext sharedCtx, - @NotNull FileIOFactory ioFactory) throws IgniteCheckedException { + @NotNull FileIOFactory ioFactory, + int serializerVersion + ) throws IgniteCheckedException { super(log, sharedCtx, - new RecordV1Serializer(new RecordDataV1Serializer(sharedCtx)), + FileWriteAheadLogManager.forVersion(sharedCtx, serializerVersion), ioFactory, BUF_SIZE); init(walFilesDir, false, null); @@ -113,11 +114,12 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator { @NotNull IgniteLogger log, @NotNull GridCacheSharedContext sharedCtx, @NotNull FileIOFactory ioFactory, + int serializerVersion, boolean workDir, @NotNull File... walFiles) throws IgniteCheckedException { super(log, sharedCtx, - new RecordV1Serializer(new RecordDataV1Serializer(sharedCtx)), + FileWriteAheadLogManager.forVersion(sharedCtx, serializerVersion), ioFactory, BUF_SIZE); this.workDir = workDir; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java index ca54e6fdd0e0a..6e3eb2f64fa18 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java @@ -191,7 +191,7 @@ public void testFillWalAndReadRecords() throws Exception { final File walArchiveDirWithConsistentId = new File(walArchive, consistentId); final File walWorkDirWithConsistentId = new File(wal, consistentId); - final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, fileIOFactory, PAGE_SIZE); + final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, fileIOFactory, FileWriteAheadLogManager.LATEST_SERIALIZER_VERSION, PAGE_SIZE); final int cntArchiveDir = iterateAndCount(factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId)); log.info("Total records loaded using directory : " + cntArchiveDir);