From cbf3905c9c798c20025e0a7155b238792f48c913 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 28 Jan 2020 13:42:38 +0100 Subject: [PATCH 01/47] bck --- .../snapshots/SourceOnlySnapshotRepository.java | 1 + .../snapshots/SourceOnlySnapshotShardTests.java | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index af9d95a519a34..2d9a77ef7d446 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -138,6 +138,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s try { FSDirectory directory = new SimpleFSDirectory(snapPath); toClose.add(directory); + toClose.add(() -> IOUtils.rm(snapPath)); Store tempStore = new Store(store.shardId(), store.indexSettings(), directory, new ShardLock(store.shardId()) { @Override protected void closeInternal() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java index bf6512cc531e2..e9a03ccf531de 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java @@ -153,10 +153,25 @@ public void testIncrementalSnapshot() throws IOException { assertEquals(totalFileCount+4, copy.getTotalFileCount()); assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); } - deleteDoc(shard, Integer.toString(10)); try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { SnapshotId snapshotId = new SnapshotId("test_2", "test_2"); + IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); + final PlainActionFuture future = PlainActionFuture.newFuture(); + runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, + snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, future)); + shardGeneration = future.actionGet(); + IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); + // we processed the segments_N file plus _1.si, _1.fdx, _1.fnm, _1.fdt + assertEquals(0, copy.getIncrementalFileCount()); + // in total we have 4 more files than the previous snap since we don't count the segments_N twice + assertEquals(totalFileCount, copy.getTotalFileCount()); + assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); + } + deleteDoc(shard, Integer.toString(10)); + try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { + SnapshotId snapshotId = new SnapshotId("test_3", "test_3"); + IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, From 11ac7115df152ffaaacc1b9b1ab4449b77ca3574 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 28 Jan 2020 15:23:48 +0100 Subject: [PATCH 02/47] bck --- .../blobstore/BlobStoreRepository.java | 6 +-- .../SourceOnlySnapshotRepository.java | 10 ++--- .../SourceOnlySnapshotShardTests.java | 40 +++++++++++++------ 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 9787faab98510..68ad6a025eb87 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1494,7 +1494,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s int indexIncrementalFileCount = 0; int indexTotalNumberOfFiles = 0; long indexIncrementalSize = 0; - long indexTotalFileCount = 0; + long indexTotalFileSize = 0; for (String fileName : fileNames) { if (snapshotStatus.isAborted()) { logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName); @@ -1516,7 +1516,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } } - indexTotalFileCount += md.length(); + indexTotalFileSize += md.length(); indexTotalNumberOfFiles++; if (existingFileInfo == null) { @@ -1533,7 +1533,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } snapshotStatus.moveToStarted(startTime, indexIncrementalFileCount, - indexTotalNumberOfFiles, indexIncrementalSize, indexTotalFileCount); + indexTotalNumberOfFiles, indexIncrementalSize, indexTotalFileSize); assert indexIncrementalFileCount == filesToSnapshot.size(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 2d9a77ef7d446..4e8e5acaccc45 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -134,11 +134,11 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s Path dataPath = ((FSDirectory) unwrap).getDirectory().getParent(); // TODO should we have a snapshot tmp directory per shard that is maintained by the system? Path snapPath = dataPath.resolve(SNAPSHOT_DIR_NAME); - final List toClose = new ArrayList<>(3); + final List toClose = new ArrayList<>(4); + toClose.add(() -> IOUtils.rm(snapPath)); try { FSDirectory directory = new SimpleFSDirectory(snapPath); - toClose.add(directory); - toClose.add(() -> IOUtils.rm(snapPath)); + toClose.add(0, directory); Store tempStore = new Store(store.shardId(), store.indexSettings(), directory, new ShardLock(store.shardId()) { @Override protected void closeInternal() { @@ -154,10 +154,10 @@ protected void closeInternal() { final long maxDoc = segmentInfos.totalMaxDoc(); tempStore.bootstrapNewHistory(maxDoc, maxDoc); store.incRef(); - toClose.add(store::decRef); + toClose.add(1, store::decRef); DirectoryReader reader = DirectoryReader.open(tempStore.directory(), Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name())); - toClose.add(reader); + toClose.add(2, reader); IndexCommit indexCommit = reader.getIndexCommit(); super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, snapshotStatus, writeShardGens, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java index e9a03ccf531de..43acd1cd81c7e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java @@ -121,7 +121,7 @@ public void testIncrementalSnapshot() throws IOException { IndexId indexId = new IndexId(shard.shardId().getIndexName(), shard.shardId().getIndex().getUUID()); SourceOnlySnapshotRepository repository = new SourceOnlySnapshotRepository(createRepository()); repository.start(); - int totalFileCount; + final int cleanSegmentFileCount; String shardGeneration; try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(null); @@ -132,7 +132,7 @@ public void testIncrementalSnapshot() throws IOException { shardGeneration = future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount()); - totalFileCount = copy.getTotalFileCount(); + cleanSegmentFileCount = copy.getTotalFileCount(); assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); } @@ -149,8 +149,8 @@ public void testIncrementalSnapshot() throws IOException { IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); // we processed the segments_N file plus _1.si, _1.fdx, _1.fnm, _1.fdt assertEquals(5, copy.getIncrementalFileCount()); - // in total we have 4 more files than the previous snap since we don't count the segments_N twice - assertEquals(totalFileCount+4, copy.getTotalFileCount()); + // in total we have 4 more files than the previous snap since we don't reuse the segments_N + assertEquals(cleanSegmentFileCount + 4, copy.getTotalFileCount()); assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); } try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { @@ -162,26 +162,42 @@ public void testIncrementalSnapshot() throws IOException { snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, future)); shardGeneration = future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); - // we processed the segments_N file plus _1.si, _1.fdx, _1.fnm, _1.fdt - assertEquals(0, copy.getIncrementalFileCount()); - // in total we have 4 more files than the previous snap since we don't count the segments_N twice - assertEquals(totalFileCount, copy.getTotalFileCount()); + // we processed a new segments_N file but nothing else changed + assertEquals(1, copy.getIncrementalFileCount()); + // Same total file count as in the previous step since nothing about the shard changed + assertEquals(cleanSegmentFileCount + 4, copy.getTotalFileCount()); assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); } deleteDoc(shard, Integer.toString(10)); try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { SnapshotId snapshotId = new SnapshotId("test_3", "test_3"); + IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); + final PlainActionFuture future = PlainActionFuture.newFuture(); + runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, + snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, future)); + shardGeneration = future.actionGet(); + IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); + // we processed the segments_N file plus _1.si, _1.fdx, _1.fnm, _1.fdt + assertEquals(4, copy.getIncrementalFileCount()); + // in total we have 5 more files than in the first clean snap, the 4 new segment blobs above plus a .liv file tracking the + // delete + assertEquals(cleanSegmentFileCount + 5, copy.getTotalFileCount()); + assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); + } + try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { + SnapshotId snapshotId = new SnapshotId("test_4", "test_4"); + IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, future)); future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); - // we processed the segments_N file plus _1_1.liv - assertEquals(2, copy.getIncrementalFileCount()); - // in total we have 5 more files than the previous snap since we don't count the segments_N twice - assertEquals(totalFileCount+5, copy.getTotalFileCount()); + // we processed a new segments_N file but nothing else changed + assertEquals(1, copy.getIncrementalFileCount()); + // Same total file count as in the previous step since nothing about the shard changed + assertEquals(cleanSegmentFileCount + 5, copy.getTotalFileCount()); assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); } closeShards(shard); From a47de87626eb6121e45a3b9fc3cb999973b83de6 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 30 Jan 2020 12:45:03 +0100 Subject: [PATCH 03/47] bck --- .../SourceOnlySnapshotShardTests.java | 49 ++++--------------- 1 file changed, 9 insertions(+), 40 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java index ac9e174531f58..65d30267e2bfb 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java @@ -121,7 +121,7 @@ public void testIncrementalSnapshot() throws IOException { IndexId indexId = new IndexId(shard.shardId().getIndexName(), shard.shardId().getIndex().getUUID()); SourceOnlySnapshotRepository repository = new SourceOnlySnapshotRepository(createRepository()); repository.start(); - final int cleanSegmentFileCount; + int totalFileCount; String shardGeneration; try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(null); @@ -132,7 +132,7 @@ public void testIncrementalSnapshot() throws IOException { shardGeneration = future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount()); - cleanSegmentFileCount = copy.getTotalFileCount(); + totalFileCount = copy.getTotalFileCount(); assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); } @@ -149,44 +149,13 @@ public void testIncrementalSnapshot() throws IOException { IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); // we processed the segments_N file plus _1.si, _1.fdx, _1.fnm, _1.fdt assertEquals(5, copy.getIncrementalFileCount()); - // in total we have 4 more files than the previous snap since we don't reuse the segments_N - assertEquals(cleanSegmentFileCount + 4, copy.getTotalFileCount()); - assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); - } - try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { - SnapshotId snapshotId = new SnapshotId("test_2", "test_2"); - - IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); - final PlainActionFuture future = PlainActionFuture.newFuture(); - runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, Collections.emptyMap(), future)); - shardGeneration = future.actionGet(); - IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); - // we processed a new segments_N file but nothing else changed - assertEquals(1, copy.getIncrementalFileCount()); - // Same total file count as in the previous step since nothing about the shard changed - assertEquals(cleanSegmentFileCount + 4, copy.getTotalFileCount()); + // in total we have 4 more files than the previous snap since we don't count the segments_N twice + assertEquals(totalFileCount+4, copy.getTotalFileCount()); assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); } deleteDoc(shard, Integer.toString(10)); try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { - SnapshotId snapshotId = new SnapshotId("test_3", "test_3"); - - IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); - final PlainActionFuture future = PlainActionFuture.newFuture(); - runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, Collections.emptyMap(), future)); - shardGeneration = future.actionGet(); - IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); - // we processed the segments_N file plus _1.si, _1.fdx, _1.fnm, _1.fdt - assertEquals(4, copy.getIncrementalFileCount()); - // in total we have 5 more files than in the first clean snap, the 4 new segment blobs above plus a .liv file tracking the - // delete - assertEquals(cleanSegmentFileCount + 5, copy.getTotalFileCount()); - assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); - } - try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { - SnapshotId snapshotId = new SnapshotId("test_4", "test_4"); + SnapshotId snapshotId = new SnapshotId("test_2", "test_2"); IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); @@ -194,10 +163,10 @@ public void testIncrementalSnapshot() throws IOException { snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, Collections.emptyMap(), future)); future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); - // we processed a new segments_N file but nothing else changed - assertEquals(1, copy.getIncrementalFileCount()); - // Same total file count as in the previous step since nothing about the shard changed - assertEquals(cleanSegmentFileCount + 5, copy.getTotalFileCount()); + // we processed the segments_N file plus _1_1.liv + assertEquals(2, copy.getIncrementalFileCount()); + // in total we have 5 more files than the previous snap since we don't count the segments_N twice + assertEquals(totalFileCount+5, copy.getTotalFileCount()); assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); } closeShards(shard); From 4e54a400cc53a86b111c84f701727a36ceb1ba89 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 30 Jan 2020 12:47:02 +0100 Subject: [PATCH 04/47] bck --- .../elasticsearch/snapshots/SourceOnlySnapshotRepository.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 01eb1d91ec995..5aa4729f8998b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -132,7 +132,6 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s throw new AssertionError("expected FSDirectory but got " + unwrap.toString()); } Path dataPath = ((FSDirectory) unwrap).getDirectory().getParent(); - // TODO should we have a snapshot tmp directory per shard that is maintained by the system? Path snapPath = dataPath.resolve(SNAPSHOT_DIR_NAME); final List toClose = new ArrayList<>(4); toClose.add(() -> IOUtils.rm(snapPath)); From 4dcea768648c15bbb3cd54b5df1291fa059d1d9a Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 30 Jan 2020 13:29:17 +0100 Subject: [PATCH 05/47] bck --- .../snapshots/SourceOnlySnapshotRepository.java | 2 +- .../snapshots/SourceOnlySnapshotShardTests.java | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 5aa4729f8998b..b7d0c3f751d3c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -134,7 +134,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s Path dataPath = ((FSDirectory) unwrap).getDirectory().getParent(); Path snapPath = dataPath.resolve(SNAPSHOT_DIR_NAME); final List toClose = new ArrayList<>(4); - toClose.add(() -> IOUtils.rm(snapPath)); + //toClose.add(() -> IOUtils.rm(snapPath)); try { FSDirectory directory = new SimpleFSDirectory(snapPath); toClose.add(0, directory); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java index 65d30267e2bfb..6df3adae79b5e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java @@ -135,11 +135,23 @@ public void testIncrementalSnapshot() throws IOException { totalFileCount = copy.getTotalFileCount(); assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); } + try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { + IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); + SnapshotId snapshotId = new SnapshotId("test_1", "test"); + final PlainActionFuture future = PlainActionFuture.newFuture(); + runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, + snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, Collections.emptyMap(), future)); + shardGeneration = future.actionGet(); + IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); + assertEquals(0, copy.getIncrementalFileCount()); + assertEquals(totalFileCount, copy.getTotalFileCount()); + assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); + } indexDoc(shard, "_doc", Integer.toString(10)); indexDoc(shard, "_doc", Integer.toString(11)); try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { - SnapshotId snapshotId = new SnapshotId("test_1", "test_1"); + SnapshotId snapshotId = new SnapshotId("test_2", "test_1"); IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); @@ -155,7 +167,7 @@ public void testIncrementalSnapshot() throws IOException { } deleteDoc(shard, Integer.toString(10)); try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { - SnapshotId snapshotId = new SnapshotId("test_2", "test_2"); + SnapshotId snapshotId = new SnapshotId("test_3", "test_2"); IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); From 1ab25bcb880318c3d1ab31389edc1bb6c06b4f71 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sat, 1 Feb 2020 19:38:18 +0100 Subject: [PATCH 06/47] Stop creating new segments_N needlessly --- .../elasticsearch/snapshots/SourceOnlySnapshot.java | 11 ++++++++--- .../snapshots/SourceOnlySnapshotRepository.java | 8 +++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java index c0cae8c5426e3..775b3c70d7e6f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java @@ -69,13 +69,15 @@ public SourceOnlySnapshot(Directory targetDirectory) { public synchronized List syncSnapshot(IndexCommit commit) throws IOException { long generation; Map existingSegments = new HashMap<>(); + final SegmentInfos existingSegmentInfos; if (Lucene.indexExists(targetDirectory)) { - SegmentInfos existingsSegmentInfos = Lucene.readSegmentInfos(targetDirectory); - for (SegmentCommitInfo info : existingsSegmentInfos) { + existingSegmentInfos = Lucene.readSegmentInfos(targetDirectory); + for (SegmentCommitInfo info : existingSegmentInfos) { existingSegments.put(new BytesRef(info.info.getId()), info); } - generation = existingsSegmentInfos.getGeneration(); + generation = existingSegmentInfos.getGeneration(); } else { + existingSegmentInfos = null; generation = 1; } List createdFiles = new ArrayList<>(); @@ -97,6 +99,9 @@ public synchronized List syncSnapshot(IndexCommit commit) throws IOExcep } segmentInfos.clear(); segmentInfos.addAll(newInfos); + if (existingSegmentInfos != null && segmentInfos.asList().equals(existingSegmentInfos.asList())) { + return Collections.emptyList(); + } segmentInfos.setNextWriteGeneration(Math.max(segmentInfos.getGeneration(), generation) + 1); String pendingSegmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS, "", segmentInfos.getGeneration()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index b7d0c3f751d3c..31a3b9ba57739 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -147,11 +147,13 @@ protected void closeInternal() { Supplier querySupplier = mapperService.hasNested() ? Queries::newNestedFilter : null; // SourceOnlySnapshot will take care of soft- and hard-deletes no special casing needed here SourceOnlySnapshot snapshot = new SourceOnlySnapshot(tempStore.directory(), querySupplier); - snapshot.syncSnapshot(snapshotIndexCommit); + final List newFiles = snapshot.syncSnapshot(snapshotIndexCommit); // we will use the lucene doc ID as the seq ID so we set the local checkpoint to maxDoc with a new index UUID SegmentInfos segmentInfos = tempStore.readLastCommittedSegmentsInfo(); - final long maxDoc = segmentInfos.totalMaxDoc(); - tempStore.bootstrapNewHistory(maxDoc, maxDoc); + if (newFiles.isEmpty() == false) { + final long maxDoc = segmentInfos.totalMaxDoc(); + tempStore.bootstrapNewHistory(maxDoc, maxDoc); + } store.incRef(); toClose.add(1, store::decRef); DirectoryReader reader = DirectoryReader.open(tempStore.directory(), From fead3137e5149322baa858ef994f4f37cf2df7a9 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 2 Feb 2020 10:45:57 +0100 Subject: [PATCH 07/47] reorg --- .../elasticsearch/snapshots/SourceOnlySnapshot.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java index 775b3c70d7e6f..50a8b32e120bc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java @@ -67,7 +67,6 @@ public SourceOnlySnapshot(Directory targetDirectory) { } public synchronized List syncSnapshot(IndexCommit commit) throws IOException { - long generation; Map existingSegments = new HashMap<>(); final SegmentInfos existingSegmentInfos; if (Lucene.indexExists(targetDirectory)) { @@ -75,10 +74,8 @@ public synchronized List syncSnapshot(IndexCommit commit) throws IOExcep for (SegmentCommitInfo info : existingSegmentInfos) { existingSegments.put(new BytesRef(info.info.getId()), info); } - generation = existingSegmentInfos.getGeneration(); } else { existingSegmentInfos = null; - generation = 1; } List createdFiles = new ArrayList<>(); String segmentFileName; @@ -97,10 +94,16 @@ public synchronized List syncSnapshot(IndexCommit commit) throws IOExcep newInfos.add(newInfo); } } + if (existingSegmentInfos != null && newInfos.equals(existingSegmentInfos.asList())) { + return Collections.emptyList(); + } segmentInfos.clear(); segmentInfos.addAll(newInfos); - if (existingSegmentInfos != null && segmentInfos.asList().equals(existingSegmentInfos.asList())) { - return Collections.emptyList(); + final long generation; + if (existingSegmentInfos == null) { + generation = 1; + } else { + generation = existingSegmentInfos.getGeneration(); } segmentInfos.setNextWriteGeneration(Math.max(segmentInfos.getGeneration(), generation) + 1); String pendingSegmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS, From fcd23dbd587b48ff8541bdca16f07e5d394b6f7b Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 4 Feb 2020 17:31:45 +0100 Subject: [PATCH 08/47] bck --- .../BlobStoreIndexShardSnapshots.java | 5 +++ .../repositories/Repository.java | 6 ++++ .../blobstore/BlobStoreRepository.java | 13 ++++++++ .../snapshots/SourceOnlySnapshot.java | 32 ++++++++----------- .../SourceOnlySnapshotRepository.java | 5 +-- .../snapshots/SourceOnlySnapshotTests.java | 13 ++++---- 6 files changed, 47 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java index 309a3c3a38694..4b85e94f8676d 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -94,6 +95,10 @@ private BlobStoreIndexShardSnapshots(Map files, List files() { + return files.values(); + } + /** * Returns list of snapshots * diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index 95c4da6666a1b..3dcf21b66fcd5 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -19,6 +19,7 @@ package org.elasticsearch.repositories; import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.SegmentCommitInfo; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.SnapshotsInProgress; @@ -37,6 +38,7 @@ import org.elasticsearch.snapshots.SnapshotShardFailure; import java.io.IOException; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -238,6 +240,10 @@ void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, ShardId s */ void updateState(ClusterState state); + default List segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { + throw new UnsupportedOperationException("segmentsInShard not implemented"); + } + /** * Hook that allows a repository to filter the user supplied snapshot metadata in {@link SnapshotsInProgress.Entry#userMetadata()} * during snapshot initialization. diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 6ffa1a74e37b9..9480d1698d826 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -22,10 +22,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -1443,6 +1447,15 @@ private void writeAtomic(final String blobName, final BytesReference bytesRef, b } } + @Override + public List segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { + final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots(Collections.emptySet(), + shardContainer(indexId, shardId), generation).v1(); + return blobStoreIndexShardSnapshots.files().stream().filter(fileInfo -> fileInfo.name().endsWith(".si") + && fileInfo.metadata().hash().length == fileInfo.length()) + .map(fileInfo -> ) + } + @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java index 50a8b32e120bc..ba3d152dcad19 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java @@ -37,6 +37,7 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.core.internal.io.IOUtils; @@ -56,26 +57,24 @@ public class SourceOnlySnapshot { private final Directory targetDirectory; private final Supplier deleteByQuerySupplier; + private final CheckedSupplier, IOException> knownSegmentCommitsProvider; - public SourceOnlySnapshot(Directory targetDirectory, Supplier deleteByQuerySupplier) { + public SourceOnlySnapshot(Directory targetDirectory, Supplier deleteByQuerySupplier, + CheckedSupplier, IOException> knownSegmentCommitsProvider) { this.targetDirectory = targetDirectory; this.deleteByQuerySupplier = deleteByQuerySupplier; + this.knownSegmentCommitsProvider = knownSegmentCommitsProvider; } - public SourceOnlySnapshot(Directory targetDirectory) { - this(targetDirectory, null); + public SourceOnlySnapshot(Directory targetDirectory, CheckedSupplier, IOException> knownSegmentCommitsProvider) { + this(targetDirectory, null, knownSegmentCommitsProvider); } public synchronized List syncSnapshot(IndexCommit commit) throws IOException { - Map existingSegments = new HashMap<>(); - final SegmentInfos existingSegmentInfos; - if (Lucene.indexExists(targetDirectory)) { - existingSegmentInfos = Lucene.readSegmentInfos(targetDirectory); - for (SegmentCommitInfo info : existingSegmentInfos) { - existingSegments.put(new BytesRef(info.info.getId()), info); - } - } else { - existingSegmentInfos = null; + Map existingSegments; + existingSegments = new HashMap<>(); + for (SegmentCommitInfo info : knownSegmentCommitsProvider.get()) { + existingSegments.put(new BytesRef(info.info.getId()), info); } List createdFiles = new ArrayList<>(); String segmentFileName; @@ -94,17 +93,12 @@ public synchronized List syncSnapshot(IndexCommit commit) throws IOExcep newInfos.add(newInfo); } } - if (existingSegmentInfos != null && newInfos.equals(existingSegmentInfos.asList())) { + if (existingSegments.values().containsAll(newInfos)) { return Collections.emptyList(); } segmentInfos.clear(); segmentInfos.addAll(newInfos); - final long generation; - if (existingSegmentInfos == null) { - generation = 1; - } else { - generation = existingSegmentInfos.getGeneration(); - } + final long generation = 1; segmentInfos.setNextWriteGeneration(Math.max(segmentInfos.getGeneration(), generation) + 1); String pendingSegmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS, "", segmentInfos.getGeneration()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 31a3b9ba57739..4be0ac34ee826 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -134,7 +134,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s Path dataPath = ((FSDirectory) unwrap).getDirectory().getParent(); Path snapPath = dataPath.resolve(SNAPSHOT_DIR_NAME); final List toClose = new ArrayList<>(4); - //toClose.add(() -> IOUtils.rm(snapPath)); + toClose.add(() -> IOUtils.rm(snapPath)); try { FSDirectory directory = new SimpleFSDirectory(snapPath); toClose.add(0, directory); @@ -146,7 +146,8 @@ protected void closeInternal() { }, Store.OnClose.EMPTY); Supplier querySupplier = mapperService.hasNested() ? Queries::newNestedFilter : null; // SourceOnlySnapshot will take care of soft- and hard-deletes no special casing needed here - SourceOnlySnapshot snapshot = new SourceOnlySnapshot(tempStore.directory(), querySupplier); + SourceOnlySnapshot snapshot = new SourceOnlySnapshot(tempStore.directory(), querySupplier, + () -> segmentsInShard(indexId, store.shardId().id(), snapshotStatus.generation())); final List newFiles = snapshot.syncSnapshot(snapshotIndexCommit); // we will use the lucene doc ID as the seq ID so we set the local checkpoint to maxDoc with a new index UUID SegmentInfos segmentInfos = tempStore.readLastCommittedSegmentsInfo(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java index 48db059b2178e..c1974074fb21a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java @@ -57,7 +57,8 @@ public void testSourceOnlyRandom() throws IOException { // we either use the soft deletes directly or manually delete them to test the additional delete functionality boolean modifyDeletedDocs = softDeletesField != null && randomBoolean(); SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, - modifyDeletedDocs ? () -> new DocValuesFieldExistsQuery(softDeletesField) : null) { + modifyDeletedDocs ? () -> new DocValuesFieldExistsQuery(softDeletesField) : null, + () -> SegmentInfos.readLatestCommit(dir).asList()) { @Override DirectoryReader wrapReader(DirectoryReader reader) throws IOException { return modifyDeletedDocs ? reader : super.wrapReader(reader); @@ -167,7 +168,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, writer.commit(); Directory targetDir = newDirectory(); IndexCommit snapshot = deletionPolicy.snapshot(); - SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir); + SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, () -> SegmentInfos.readLatestCommit(dir).asList()); snapshoter.syncSnapshot(snapshot); StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(snapshot); @@ -182,7 +183,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, assertEquals(0, id.totalHits.value); } - snapshoter = new SourceOnlySnapshot(targetDir); + snapshoter = new SourceOnlySnapshot(targetDir, () -> SegmentInfos.readLatestCommit(dir).asList()); List createdFiles = snapshoter.syncSnapshot(snapshot); assertEquals(0, createdFiles.size()); deletionPolicy.release(snapshot); @@ -202,7 +203,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, writer.commit(); { snapshot = deletionPolicy.snapshot(); - snapshoter = new SourceOnlySnapshot(targetDir); + snapshoter = new SourceOnlySnapshot(targetDir, () -> SegmentInfos.readLatestCommit(dir).asList()); createdFiles = snapshoter.syncSnapshot(snapshot); assertEquals(4, createdFiles.size()); for (String file : createdFiles) { @@ -227,7 +228,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, writer.commit(); { snapshot = deletionPolicy.snapshot(); - snapshoter = new SourceOnlySnapshot(targetDir); + snapshoter = new SourceOnlySnapshot(targetDir, () -> SegmentInfos.readLatestCommit(dir).asList()); createdFiles = snapshoter.syncSnapshot(snapshot); assertEquals(1, createdFiles.size()); for (String file : createdFiles) { @@ -285,7 +286,7 @@ public boolean keepFullyDeletedSegment(IOSupplier readerIOSupplier) writer.commit(); try (Directory targetDir = newDirectory()) { IndexCommit snapshot = deletionPolicy.snapshot(); - SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir); + SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, () -> SegmentInfos.readLatestCommit(dir).asList()); snapshoter.syncSnapshot(snapshot); try (DirectoryReader snapReader = DirectoryReader.open(targetDir)) { From 28e3ae0ae7a76a245aa9ca60c8176e0be5c56769 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 4 Feb 2020 18:11:29 +0100 Subject: [PATCH 09/47] bck --- .../BlobStoreIndexShardSnapshots.java | 4 -- .../repositories/FilterRepository.java | 6 +++ .../repositories/Repository.java | 3 +- .../blobstore/BlobStoreRepository.java | 28 ++++++++++-- .../snapshots/SourceOnlySnapshot.java | 10 +++-- .../SourceOnlySnapshotRepository.java | 45 ++++++++++++++++++- .../snapshots/SourceOnlySnapshotTests.java | 12 ++--- 7 files changed, 87 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java index 4b85e94f8676d..1ffa4fe94507d 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java @@ -95,10 +95,6 @@ private BlobStoreIndexShardSnapshots(Map files, List files() { - return files.values(); - } - /** * Returns list of snapshots * diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index 2b5304cf05f8c..15b1ef019295b 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -19,6 +19,7 @@ package org.elasticsearch.repositories; import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.SegmentInfos; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -169,4 +170,9 @@ public void stop() { public void close() { in.close(); } + + @Override + public List segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { + return in.segmentsInShard(indexId, shardId, generation); + } } diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index 3dcf21b66fcd5..63678e334b3c7 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -20,6 +20,7 @@ import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfos; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.SnapshotsInProgress; @@ -240,7 +241,7 @@ void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, ShardId s */ void updateState(ClusterState state); - default List segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { + default List segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { throw new UnsupportedOperationException("segmentsInShard not implemented"); } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 9480d1698d826..7a17656f76e2d 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -30,10 +30,13 @@ import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RateLimiter; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.SetOnce; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; @@ -1448,12 +1451,29 @@ private void writeAtomic(final String blobName, final BytesReference bytesRef, b } @Override - public List segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { + public List segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots(Collections.emptySet(), shardContainer(indexId, shardId), generation).v1(); - return blobStoreIndexShardSnapshots.files().stream().filter(fileInfo -> fileInfo.name().endsWith(".si") - && fileInfo.metadata().hash().length == fileInfo.length()) - .map(fileInfo -> ) + return blobStoreIndexShardSnapshots.snapshots().stream().map( + snapshotFiles -> { + final Directory dir = new ByteBuffersDirectory(); + snapshotFiles.indexFiles().stream().filter(fileInfo -> fileInfo.metadata().length() == fileInfo.length()).forEach( + fileInfo -> { + try(IndexOutput indexOutput = dir.createOutput(fileInfo.physicalName(), IOContext.DEFAULT)) { + final BytesRef fileContent = fileInfo.metadata().hash(); + indexOutput.writeBytes(fileContent.bytes, fileContent.offset, fileContent.length); + } catch (IOException e) { + throw new AssertionError(e); + } + } + ); + try { + return SegmentInfos.readLatestCommit(dir); + } catch (IOException e) { + throw new AssertionError(e); + } + } + ).collect(Collectors.toList()); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java index ba3d152dcad19..ed389c4480c9f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java @@ -50,6 +50,7 @@ import java.util.List; import java.util.Map; import java.util.function.Supplier; +import java.util.stream.Collectors; import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.FIELDS_EXTENSION; import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.FIELDS_INDEX_EXTENSION; @@ -57,23 +58,24 @@ public class SourceOnlySnapshot { private final Directory targetDirectory; private final Supplier deleteByQuerySupplier; - private final CheckedSupplier, IOException> knownSegmentCommitsProvider; + private final CheckedSupplier, IOException> knownSegmentCommitsProvider; public SourceOnlySnapshot(Directory targetDirectory, Supplier deleteByQuerySupplier, - CheckedSupplier, IOException> knownSegmentCommitsProvider) { + CheckedSupplier, IOException> knownSegmentCommitsProvider) { this.targetDirectory = targetDirectory; this.deleteByQuerySupplier = deleteByQuerySupplier; this.knownSegmentCommitsProvider = knownSegmentCommitsProvider; } - public SourceOnlySnapshot(Directory targetDirectory, CheckedSupplier, IOException> knownSegmentCommitsProvider) { + public SourceOnlySnapshot(Directory targetDirectory, CheckedSupplier, IOException> knownSegmentCommitsProvider) { this(targetDirectory, null, knownSegmentCommitsProvider); } public synchronized List syncSnapshot(IndexCommit commit) throws IOException { Map existingSegments; existingSegments = new HashMap<>(); - for (SegmentCommitInfo info : knownSegmentCommitsProvider.get()) { + for (SegmentCommitInfo info : knownSegmentCommitsProvider.get().stream() + .flatMap(infos -> infos.asList().stream()).collect(Collectors.toList())) { existingSegments.put(new BytesRef(info.info.getId()), info); } List createdFiles = new ArrayList<>(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 4be0ac34ee826..bee9d2dd4ffa3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -150,8 +150,8 @@ protected void closeInternal() { () -> segmentsInShard(indexId, store.shardId().id(), snapshotStatus.generation())); final List newFiles = snapshot.syncSnapshot(snapshotIndexCommit); // we will use the lucene doc ID as the seq ID so we set the local checkpoint to maxDoc with a new index UUID - SegmentInfos segmentInfos = tempStore.readLastCommittedSegmentsInfo(); if (newFiles.isEmpty() == false) { + SegmentInfos segmentInfos = tempStore.readLastCommittedSegmentsInfo(); final long maxDoc = segmentInfos.totalMaxDoc(); tempStore.bootstrapNewHistory(maxDoc, maxDoc); } @@ -161,7 +161,48 @@ protected void closeInternal() { Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name())); toClose.add(2, reader); IndexCommit indexCommit = reader.getIndexCommit(); - super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, snapshotStatus, writeShardGens, + super.snapshotShard(tempStore, mapperService, snapshotId, indexId, + new IndexCommit() { + @Override + public String getSegmentsFileName() { + return null; + } + + @Override + public Collection getFileNames() throws IOException { + return null; + } + + @Override + public Directory getDirectory() { + return tempStore.directory(); + } + + @Override + public void delete() { + throw new UnsupportedOperationException("not supported"); + } + + @Override + public boolean isDeleted() { + return false; + } + + @Override + public int getSegmentCount() { + return 0; + } + + @Override + public long getGeneration() { + return 0; + } + + @Override + public Map getUserData() throws IOException { + throw new UnsupportedOperationException("not supported"); + } + }, snapshotStatus, writeShardGens, userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); } catch (IOException e) { try { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java index c1974074fb21a..522080fbad729 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java @@ -58,7 +58,7 @@ public void testSourceOnlyRandom() throws IOException { boolean modifyDeletedDocs = softDeletesField != null && randomBoolean(); SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, modifyDeletedDocs ? () -> new DocValuesFieldExistsQuery(softDeletesField) : null, - () -> SegmentInfos.readLatestCommit(dir).asList()) { + () -> List.of(SegmentInfos.readLatestCommit(dir))) { @Override DirectoryReader wrapReader(DirectoryReader reader) throws IOException { return modifyDeletedDocs ? reader : super.wrapReader(reader); @@ -168,7 +168,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, writer.commit(); Directory targetDir = newDirectory(); IndexCommit snapshot = deletionPolicy.snapshot(); - SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, () -> SegmentInfos.readLatestCommit(dir).asList()); + SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, () -> List.of(SegmentInfos.readLatestCommit(dir))); snapshoter.syncSnapshot(snapshot); StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(snapshot); @@ -183,7 +183,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, assertEquals(0, id.totalHits.value); } - snapshoter = new SourceOnlySnapshot(targetDir, () -> SegmentInfos.readLatestCommit(dir).asList()); + snapshoter = new SourceOnlySnapshot(targetDir, () -> List.of(SegmentInfos.readLatestCommit(dir))); List createdFiles = snapshoter.syncSnapshot(snapshot); assertEquals(0, createdFiles.size()); deletionPolicy.release(snapshot); @@ -203,7 +203,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, writer.commit(); { snapshot = deletionPolicy.snapshot(); - snapshoter = new SourceOnlySnapshot(targetDir, () -> SegmentInfos.readLatestCommit(dir).asList()); + snapshoter = new SourceOnlySnapshot(targetDir, () -> List.of(SegmentInfos.readLatestCommit(dir))); createdFiles = snapshoter.syncSnapshot(snapshot); assertEquals(4, createdFiles.size()); for (String file : createdFiles) { @@ -228,7 +228,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, writer.commit(); { snapshot = deletionPolicy.snapshot(); - snapshoter = new SourceOnlySnapshot(targetDir, () -> SegmentInfos.readLatestCommit(dir).asList()); + snapshoter = new SourceOnlySnapshot(targetDir, () -> List.of(SegmentInfos.readLatestCommit(dir))); createdFiles = snapshoter.syncSnapshot(snapshot); assertEquals(1, createdFiles.size()); for (String file : createdFiles) { @@ -286,7 +286,7 @@ public boolean keepFullyDeletedSegment(IOSupplier readerIOSupplier) writer.commit(); try (Directory targetDir = newDirectory()) { IndexCommit snapshot = deletionPolicy.snapshot(); - SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, () -> SegmentInfos.readLatestCommit(dir).asList()); + SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, () -> List.of(SegmentInfos.readLatestCommit(dir))); snapshoter.syncSnapshot(snapshot); try (DirectoryReader snapReader = DirectoryReader.open(targetDir)) { From c44644d759d450c546343f9e1bdd6bbf144c6081 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 4 Feb 2020 18:31:12 +0100 Subject: [PATCH 10/47] bck --- .../snapshots/SourceOnlySnapshot.java | 3 +-- .../snapshots/SourceOnlySnapshotRepository.java | 14 +++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java index ed389c4480c9f..856015af1dd02 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java @@ -100,8 +100,7 @@ public synchronized List syncSnapshot(IndexCommit commit) throws IOExcep } segmentInfos.clear(); segmentInfos.addAll(newInfos); - final long generation = 1; - segmentInfos.setNextWriteGeneration(Math.max(segmentInfos.getGeneration(), generation) + 1); + segmentInfos.setNextWriteGeneration(segmentInfos.getGeneration()); String pendingSegmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS, "", segmentInfos.getGeneration()); try (IndexOutput segnOutput = targetDirectory.createOutput(pendingSegmentFileName, IOContext.DEFAULT)) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index bee9d2dd4ffa3..170f0cafdd89a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.ShardLock; import org.elasticsearch.index.engine.EngineFactory; @@ -43,6 +44,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -160,22 +162,24 @@ protected void closeInternal() { DirectoryReader reader = DirectoryReader.open(tempStore.directory(), Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name())); toClose.add(2, reader); - IndexCommit indexCommit = reader.getIndexCommit(); super.snapshotShard(tempStore, mapperService, snapshotId, indexId, new IndexCommit() { @Override public String getSegmentsFileName() { - return null; + return "segments_" + getGeneration(); // TODO: gotta implement crafting a segments info into the store } @Override public Collection getFileNames() throws IOException { - return null; + final Collection result = new HashSet<>(); + result.addAll(newFiles); + result.addAll(List.of(getDirectory().listAll())); + return result; } @Override public Directory getDirectory() { - return tempStore.directory(); + return reader.directory(); // Maybe not } @Override @@ -195,7 +199,7 @@ public int getSegmentCount() { @Override public long getGeneration() { - return 0; + return snapshotIndexCommit.getGeneration() + 1; } @Override From 04ee734466e7bfe5a538c6d013847dae7b9d04d8 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 4 Feb 2020 19:43:09 +0100 Subject: [PATCH 11/47] bck --- .../snapshots/SourceOnlySnapshot.java | 10 ++++++---- .../SourceOnlySnapshotRepository.java | 19 ++++++++++--------- .../snapshots/SourceOnlySnapshotTests.java | 7 ++++--- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java index 856015af1dd02..0ee5184b18a5c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java @@ -71,7 +71,7 @@ public SourceOnlySnapshot(Directory targetDirectory, CheckedSupplier syncSnapshot(IndexCommit commit) throws IOException { + public synchronized SegmentInfos syncSnapshot(IndexCommit commit) throws IOException { Map existingSegments; existingSegments = new HashMap<>(); for (SegmentCommitInfo info : knownSegmentCommitsProvider.get().stream() @@ -80,10 +80,11 @@ public synchronized List syncSnapshot(IndexCommit commit) throws IOExcep } List createdFiles = new ArrayList<>(); String segmentFileName; + SegmentInfos segmentInfos; try (Lock writeLock = targetDirectory.obtainLock(IndexWriter.WRITE_LOCK_NAME); StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(commit, Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name()))) { - SegmentInfos segmentInfos = reader.getSegmentInfos().clone(); + segmentInfos = reader.getSegmentInfos().clone(); DirectoryReader wrappedReader = wrapReader(reader); List newInfos = new ArrayList<>(); for (LeafReaderContext ctx : wrappedReader.leaves()) { @@ -96,7 +97,8 @@ public synchronized List syncSnapshot(IndexCommit commit) throws IOExcep } } if (existingSegments.values().containsAll(newInfos)) { - return Collections.emptyList(); + return knownSegmentCommitsProvider.get().stream().filter(segmentCommitInfos -> + segmentCommitInfos.asList().containsAll(newInfos)).findFirst().get(); } segmentInfos.clear(); segmentInfos.addAll(newInfos); @@ -113,7 +115,7 @@ public synchronized List syncSnapshot(IndexCommit commit) throws IOExcep } Lucene.pruneUnreferencedFiles(segmentFileName, targetDirectory); assert assertCheckIndex(); - return Collections.unmodifiableList(createdFiles); + return SegmentInfos.readLatestCommit(targetDirectory); } private LiveDocs getLiveDocs(LeafReader reader) throws IOException { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 170f0cafdd89a..f4ef289897a4d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -150,15 +150,20 @@ protected void closeInternal() { // SourceOnlySnapshot will take care of soft- and hard-deletes no special casing needed here SourceOnlySnapshot snapshot = new SourceOnlySnapshot(tempStore.directory(), querySupplier, () -> segmentsInShard(indexId, store.shardId().id(), snapshotStatus.generation())); - final List newFiles = snapshot.syncSnapshot(snapshotIndexCommit); + final SegmentInfos newFiles = snapshot.syncSnapshot(snapshotIndexCommit); + final boolean changed; // we will use the lucene doc ID as the seq ID so we set the local checkpoint to maxDoc with a new index UUID - if (newFiles.isEmpty() == false) { + if (segmentsInShard(indexId, store.shardId().id(), snapshotStatus.generation()).contains(newFiles) == false) { SegmentInfos segmentInfos = tempStore.readLastCommittedSegmentsInfo(); final long maxDoc = segmentInfos.totalMaxDoc(); tempStore.bootstrapNewHistory(maxDoc, maxDoc); + changed = true; + } else { + changed = false; } store.incRef(); toClose.add(1, store::decRef); + // TODO: need fake directory that properly answers for segments DirectoryReader reader = DirectoryReader.open(tempStore.directory(), Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name())); toClose.add(2, reader); @@ -171,10 +176,7 @@ public String getSegmentsFileName() { @Override public Collection getFileNames() throws IOException { - final Collection result = new HashSet<>(); - result.addAll(newFiles); - result.addAll(List.of(getDirectory().listAll())); - return result; + return new HashSet<>(newFiles.files(true)); } @Override @@ -199,15 +201,14 @@ public int getSegmentCount() { @Override public long getGeneration() { - return snapshotIndexCommit.getGeneration() + 1; + return newFiles.getLastGeneration() + (changed ? 1 : 0); } @Override public Map getUserData() throws IOException { throw new UnsupportedOperationException("not supported"); } - }, snapshotStatus, writeShardGens, - userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); + }, snapshotStatus, writeShardGens, userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); } catch (IOException e) { try { IOUtils.close(toClose); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java index 522080fbad729..95fdbdec90999 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java @@ -44,6 +44,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; public class SourceOnlySnapshotTests extends ESTestCase { @@ -184,7 +185,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, } snapshoter = new SourceOnlySnapshot(targetDir, () -> List.of(SegmentInfos.readLatestCommit(dir))); - List createdFiles = snapshoter.syncSnapshot(snapshot); + Collection createdFiles = snapshoter.syncSnapshot(snapshot).files(false); assertEquals(0, createdFiles.size()); deletionPolicy.release(snapshot); // now add another doc @@ -204,7 +205,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, { snapshot = deletionPolicy.snapshot(); snapshoter = new SourceOnlySnapshot(targetDir, () -> List.of(SegmentInfos.readLatestCommit(dir))); - createdFiles = snapshoter.syncSnapshot(snapshot); + createdFiles = snapshoter.syncSnapshot(snapshot).files(false); assertEquals(4, createdFiles.size()); for (String file : createdFiles) { String extension = IndexFileNames.getExtension(file); @@ -229,7 +230,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, { snapshot = deletionPolicy.snapshot(); snapshoter = new SourceOnlySnapshot(targetDir, () -> List.of(SegmentInfos.readLatestCommit(dir))); - createdFiles = snapshoter.syncSnapshot(snapshot); + createdFiles = snapshoter.syncSnapshot(snapshot).files(false); assertEquals(1, createdFiles.size()); for (String file : createdFiles) { String extension = IndexFileNames.getExtension(file); From 3ced3f0c3ab42c177e2f29bfa641e8dc3208d1b8 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 5 Feb 2020 09:44:37 +0100 Subject: [PATCH 12/47] better --- .../SourceOnlySnapshotRepository.java | 116 ++++++++++-------- 1 file changed, 66 insertions(+), 50 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index f4ef289897a4d..7815441a4e3fe 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -138,6 +138,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s final List toClose = new ArrayList<>(4); toClose.add(() -> IOUtils.rm(snapPath)); try { + final List segmentInfosInRepo = segmentsInShard(indexId, store.shardId().id(), snapshotStatus.generation()); FSDirectory directory = new SimpleFSDirectory(snapPath); toClose.add(0, directory); Store tempStore = new Store(store.shardId(), store.indexSettings(), directory, new ShardLock(store.shardId()) { @@ -145,21 +146,25 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s protected void closeInternal() { // do nothing; } - }, Store.OnClose.EMPTY); + }, Store.OnClose.EMPTY) { + + @Override + public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException { + final MetadataSnapshot metadataSnapshot = super.getMetadata(commit); + return new MetadataSnapshot(metadataSnapshot.asMap(), + metadataSnapshot.getCommitUserData(), metadataSnapshot.getNumDocs()); + } + }; Supplier querySupplier = mapperService.hasNested() ? Queries::newNestedFilter : null; // SourceOnlySnapshot will take care of soft- and hard-deletes no special casing needed here SourceOnlySnapshot snapshot = new SourceOnlySnapshot(tempStore.directory(), querySupplier, - () -> segmentsInShard(indexId, store.shardId().id(), snapshotStatus.generation())); - final SegmentInfos newFiles = snapshot.syncSnapshot(snapshotIndexCommit); - final boolean changed; + () -> segmentInfosInRepo); + SegmentInfos newFiles = snapshot.syncSnapshot(snapshotIndexCommit); // we will use the lucene doc ID as the seq ID so we set the local checkpoint to maxDoc with a new index UUID if (segmentsInShard(indexId, store.shardId().id(), snapshotStatus.generation()).contains(newFiles) == false) { - SegmentInfos segmentInfos = tempStore.readLastCommittedSegmentsInfo(); - final long maxDoc = segmentInfos.totalMaxDoc(); + final long maxDoc = newFiles.totalMaxDoc(); tempStore.bootstrapNewHistory(maxDoc, maxDoc); - changed = true; - } else { - changed = false; + newFiles = tempStore.readLastCommittedSegmentsInfo(); } store.incRef(); toClose.add(1, store::decRef); @@ -168,47 +173,8 @@ protected void closeInternal() { Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name())); toClose.add(2, reader); super.snapshotShard(tempStore, mapperService, snapshotId, indexId, - new IndexCommit() { - @Override - public String getSegmentsFileName() { - return "segments_" + getGeneration(); // TODO: gotta implement crafting a segments info into the store - } - - @Override - public Collection getFileNames() throws IOException { - return new HashSet<>(newFiles.files(true)); - } - - @Override - public Directory getDirectory() { - return reader.directory(); // Maybe not - } - - @Override - public void delete() { - throw new UnsupportedOperationException("not supported"); - } - - @Override - public boolean isDeleted() { - return false; - } - - @Override - public int getSegmentCount() { - return 0; - } - - @Override - public long getGeneration() { - return newFiles.getLastGeneration() + (changed ? 1 : 0); - } - - @Override - public Map getUserData() throws IOException { - throw new UnsupportedOperationException("not supported"); - } - }, snapshotStatus, writeShardGens, userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); + new SourceOnlyIndexCommit(newFiles, reader), snapshotStatus, writeShardGens, userMetadata, + ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); } catch (IOException e) { try { IOUtils.close(toClose); @@ -256,4 +222,54 @@ public Repository create(RepositoryMetaData metaData, Function getFileNames() throws IOException { + return new HashSet<>(newFiles.files(true)); + } + + @Override + public Directory getDirectory() { + return reader.directory(); // Maybe not + } + + @Override + public void delete() { + throw new UnsupportedOperationException("not supported"); + } + + @Override + public boolean isDeleted() { + return false; + } + + @Override + public int getSegmentCount() { + return 0; + } + + @Override + public long getGeneration() { + return newFiles.getLastGeneration(); + } + + @Override + public Map getUserData() { + throw new UnsupportedOperationException("not supported"); + } + } } From 6a6270b85f8bc420e0337be40bb5fc046489d7f6 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 5 Feb 2020 10:08:58 +0100 Subject: [PATCH 13/47] bck --- .../SourceOnlySnapshotRepository.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 7815441a4e3fe..bd3eba583e352 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -135,7 +135,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } Path dataPath = ((FSDirectory) unwrap).getDirectory().getParent(); Path snapPath = dataPath.resolve(SNAPSHOT_DIR_NAME); - final List toClose = new ArrayList<>(4); + final List toClose = new ArrayList<>(3); toClose.add(() -> IOUtils.rm(snapPath)); try { final List segmentInfosInRepo = segmentsInShard(indexId, store.shardId().id(), snapshotStatus.generation()); @@ -150,6 +150,7 @@ protected void closeInternal() { @Override public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException { + // TODO: craft metdata ourselves here instead of physically reading it from the store final MetadataSnapshot metadataSnapshot = super.getMetadata(commit); return new MetadataSnapshot(metadataSnapshot.asMap(), metadataSnapshot.getCommitUserData(), metadataSnapshot.getNumDocs()); @@ -161,19 +162,15 @@ public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException { () -> segmentInfosInRepo); SegmentInfos newFiles = snapshot.syncSnapshot(snapshotIndexCommit); // we will use the lucene doc ID as the seq ID so we set the local checkpoint to maxDoc with a new index UUID - if (segmentsInShard(indexId, store.shardId().id(), snapshotStatus.generation()).contains(newFiles) == false) { + if (segmentInfosInRepo.contains(newFiles) == false) { final long maxDoc = newFiles.totalMaxDoc(); tempStore.bootstrapNewHistory(maxDoc, maxDoc); newFiles = tempStore.readLastCommittedSegmentsInfo(); } store.incRef(); toClose.add(1, store::decRef); - // TODO: need fake directory that properly answers for segments - DirectoryReader reader = DirectoryReader.open(tempStore.directory(), - Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name())); - toClose.add(2, reader); super.snapshotShard(tempStore, mapperService, snapshotId, indexId, - new SourceOnlyIndexCommit(newFiles, reader), snapshotStatus, writeShardGens, userMetadata, + new SourceOnlyIndexCommit(newFiles, tempStore.directory()), snapshotStatus, writeShardGens, userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); } catch (IOException e) { try { @@ -225,11 +222,11 @@ public Repository create(RepositoryMetaData metaData, Function getFileNames() throws IOException { @Override public Directory getDirectory() { - return reader.directory(); // Maybe not + return directory; } @Override From d4ab9fbf2e546f25e5042577f50fee92ca89a7e0 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 5 Feb 2020 10:09:36 +0100 Subject: [PATCH 14/47] bck --- .../elasticsearch/snapshots/SourceOnlySnapshotRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index bd3eba583e352..5b611771761e6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -150,7 +150,7 @@ protected void closeInternal() { @Override public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException { - // TODO: craft metdata ourselves here instead of physically reading it from the store + // TODO: craft metadata ourselves here instead of physically reading it from the store final MetadataSnapshot metadataSnapshot = super.getMetadata(commit); return new MetadataSnapshot(metadataSnapshot.asMap(), metadataSnapshot.getCommitUserData(), metadataSnapshot.getNumDocs()); From d607c94d060dad1ef1ad01729d7b91cbd920e7ee Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 6 Feb 2020 04:47:08 +0100 Subject: [PATCH 15/47] bck --- .../snapshots/SourceOnlySnapshotRepository.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 5b611771761e6..144cb5fe11916 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -141,6 +141,11 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s final List segmentInfosInRepo = segmentsInShard(indexId, store.shardId().id(), snapshotStatus.generation()); FSDirectory directory = new SimpleFSDirectory(snapPath); toClose.add(0, directory); + Supplier querySupplier = mapperService.hasNested() ? Queries::newNestedFilter : null; + // SourceOnlySnapshot will take care of soft- and hard-deletes no special casing needed here + SourceOnlySnapshot snapshot = new SourceOnlySnapshot(directory, querySupplier, + () -> segmentInfosInRepo); + SegmentInfos newFiles = snapshot.syncSnapshot(snapshotIndexCommit); Store tempStore = new Store(store.shardId(), store.indexSettings(), directory, new ShardLock(store.shardId()) { @Override protected void closeInternal() { @@ -156,11 +161,6 @@ public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException { metadataSnapshot.getCommitUserData(), metadataSnapshot.getNumDocs()); } }; - Supplier querySupplier = mapperService.hasNested() ? Queries::newNestedFilter : null; - // SourceOnlySnapshot will take care of soft- and hard-deletes no special casing needed here - SourceOnlySnapshot snapshot = new SourceOnlySnapshot(tempStore.directory(), querySupplier, - () -> segmentInfosInRepo); - SegmentInfos newFiles = snapshot.syncSnapshot(snapshotIndexCommit); // we will use the lucene doc ID as the seq ID so we set the local checkpoint to maxDoc with a new index UUID if (segmentInfosInRepo.contains(newFiles) == false) { final long maxDoc = newFiles.totalMaxDoc(); @@ -170,7 +170,7 @@ public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException { store.incRef(); toClose.add(1, store::decRef); super.snapshotShard(tempStore, mapperService, snapshotId, indexId, - new SourceOnlyIndexCommit(newFiles, tempStore.directory()), snapshotStatus, writeShardGens, userMetadata, + new SourceOnlyIndexCommit(newFiles, directory), snapshotStatus, writeShardGens, userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); } catch (IOException e) { try { From fec8d131bc00744d6e12625f542f60a0deabb68f Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 6 Feb 2020 13:36:28 +0100 Subject: [PATCH 16/47] bck --- .../repositories/FilterRepository.java | 4 +-- .../repositories/Repository.java | 3 +- .../blobstore/BlobStoreRepository.java | 27 +++------------ .../snapshots/SourceOnlySnapshot.java | 8 +++-- .../SourceOnlySnapshotRepository.java | 34 +++++++++++++++---- 5 files changed, 41 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index 15b1ef019295b..98ef02bb1359a 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -19,7 +19,6 @@ package org.elasticsearch.repositories; import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.index.SegmentInfos; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -32,6 +31,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; @@ -172,7 +172,7 @@ public void close() { } @Override - public List segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { + public List> segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { return in.segmentsInShard(indexId, shardId, generation); } } diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index 63678e334b3c7..1a470a5d5a7a3 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; @@ -241,7 +242,7 @@ void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, ShardId s */ void updateState(ClusterState state); - default List segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { + default List> segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { throw new UnsupportedOperationException("segmentsInShard not implemented"); } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 7a17656f76e2d..8a23848e906a0 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -22,13 +22,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; @@ -1451,28 +1448,12 @@ private void writeAtomic(final String blobName, final BytesReference bytesRef, b } @Override - public List segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { + public List> segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots(Collections.emptySet(), - shardContainer(indexId, shardId), generation).v1(); + shardContainer(indexId, shardId), generation).v1(); return blobStoreIndexShardSnapshots.snapshots().stream().map( - snapshotFiles -> { - final Directory dir = new ByteBuffersDirectory(); - snapshotFiles.indexFiles().stream().filter(fileInfo -> fileInfo.metadata().length() == fileInfo.length()).forEach( - fileInfo -> { - try(IndexOutput indexOutput = dir.createOutput(fileInfo.physicalName(), IOContext.DEFAULT)) { - final BytesRef fileContent = fileInfo.metadata().hash(); - indexOutput.writeBytes(fileContent.bytes, fileContent.offset, fileContent.length); - } catch (IOException e) { - throw new AssertionError(e); - } - } - ); - try { - return SegmentInfos.readLatestCommit(dir); - } catch (IOException e) { - throw new AssertionError(e); - } - } + snapshotFiles -> snapshotFiles.indexFiles().stream() + .map(BlobStoreIndexShardSnapshot.FileInfo::metadata).collect(Collectors.toList()) ).collect(Collectors.toList()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java index 0ee5184b18a5c..952db1cb89cb7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.index.store.StoreFileMetaData; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -58,21 +59,22 @@ public class SourceOnlySnapshot { private final Directory targetDirectory; private final Supplier deleteByQuerySupplier; - private final CheckedSupplier, IOException> knownSegmentCommitsProvider; + private final CheckedSupplier>, IOException> knownSegmentCommitsProvider; public SourceOnlySnapshot(Directory targetDirectory, Supplier deleteByQuerySupplier, - CheckedSupplier, IOException> knownSegmentCommitsProvider) { + CheckedSupplier>, IOException> knownSegmentCommitsProvider) { this.targetDirectory = targetDirectory; this.deleteByQuerySupplier = deleteByQuerySupplier; this.knownSegmentCommitsProvider = knownSegmentCommitsProvider; } - public SourceOnlySnapshot(Directory targetDirectory, CheckedSupplier, IOException> knownSegmentCommitsProvider) { + public SourceOnlySnapshot(Directory targetDirectory, CheckedSupplier>, IOException> knownSegmentCommitsProvider) { this(targetDirectory, null, knownSegmentCommitsProvider); } public synchronized SegmentInfos syncSnapshot(IndexCommit commit) throws IOException { Map existingSegments; + Map> existingFileSets; existingSegments = new HashMap<>(); for (SegmentCommitInfo info : knownSegmentCommitsProvider.get().stream() .flatMap(infos -> infos.asList().stream()).collect(Collectors.toList())) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 144cb5fe11916..ad632f97f3c64 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -5,15 +5,17 @@ */ package org.elasticsearch.snapshots; -import org.apache.lucene.codecs.blocktree.BlockTreeTermsReader; -import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.search.Query; +import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FilterDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; @@ -23,7 +25,6 @@ import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.ShardLock; import org.elasticsearch.index.engine.EngineFactory; @@ -31,6 +32,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.repositories.FilterRepository; import org.elasticsearch.repositories.IndexId; @@ -43,7 +45,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -138,7 +139,8 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s final List toClose = new ArrayList<>(3); toClose.add(() -> IOUtils.rm(snapPath)); try { - final List segmentInfosInRepo = segmentsInShard(indexId, store.shardId().id(), snapshotStatus.generation()); + final List> segmentInfosInRepo = + segmentsInShard(indexId, store.shardId().id(), snapshotStatus.generation()); FSDirectory directory = new SimpleFSDirectory(snapPath); toClose.add(0, directory); Supplier querySupplier = mapperService.hasNested() ? Queries::newNestedFilter : null; @@ -170,7 +172,7 @@ public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException { store.incRef(); toClose.add(1, store::decRef); super.snapshotShard(tempStore, mapperService, snapshotId, indexId, - new SourceOnlyIndexCommit(newFiles, directory), snapshotStatus, writeShardGens, userMetadata, + new SourceOnlyIndexCommit(newFiles, tempStore.directory()), snapshotStatus, writeShardGens, userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); } catch (IOException e) { try { @@ -182,6 +184,26 @@ public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException { } } + private static SegmentInfos segmentInfosFromMeta(Iterable files) { + final Directory dir = new ByteBuffersDirectory(); + for (StoreFileMetaData m : files) { + if (m.length() != m.hash().length) { + continue; + } + try (IndexOutput indexOutput = dir.createOutput(m.name(), IOContext.DEFAULT)) { + final BytesRef fileContent = m.hash(); + indexOutput.writeBytes(fileContent.bytes, fileContent.offset, fileContent.length); + } catch (IOException e) { + throw new AssertionError(e); + } + } + try { + return SegmentInfos.readLatestCommit(dir); + } catch (IOException e) { + throw new AssertionError(e); + } + } + /** * Returns an {@link EngineFactory} for the source only snapshots. */ From 2a87a71a078b3ba8596af1cc7debd38899fd7541 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sat, 8 Feb 2020 14:30:04 +0100 Subject: [PATCH 17/47] bck --- .../repositories/blobstore/BlobStoreRepository.java | 4 ---- .../java/org/elasticsearch/snapshots/SourceOnlySnapshot.java | 2 +- .../elasticsearch/snapshots/SourceOnlySnapshotRepository.java | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 8a23848e906a0..fd6c95515bb60 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -26,14 +26,10 @@ import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.store.ByteBuffersDirectory; -import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RateLimiter; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.SetOnce; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java index 952db1cb89cb7..a6fecdf0d0e5d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java @@ -77,7 +77,7 @@ public synchronized SegmentInfos syncSnapshot(IndexCommit commit) throws IOExcep Map> existingFileSets; existingSegments = new HashMap<>(); for (SegmentCommitInfo info : knownSegmentCommitsProvider.get().stream() - .flatMap(infos -> infos.asList().stream()).collect(Collectors.toList())) { + .flatMap(infos -> SourceOnlySnapshotRepository.segmentInfosFromMeta(infos).asList().stream()).collect(Collectors.toList())) { existingSegments.put(new BytesRef(info.info.getId()), info); } List createdFiles = new ArrayList<>(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index ad632f97f3c64..2fe38bf1ebed8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -184,7 +184,7 @@ public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException { } } - private static SegmentInfos segmentInfosFromMeta(Iterable files) { + public static SegmentInfos segmentInfosFromMeta(Iterable files) { final Directory dir = new ByteBuffersDirectory(); for (StoreFileMetaData m : files) { if (m.length() != m.hash().length) { From e54d34ddb78071f6d775e84b195e37987a6888eb Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sat, 8 Feb 2020 16:55:57 +0100 Subject: [PATCH 18/47] sorta --- .../elasticsearch/snapshots/SourceOnlySnapshot.java | 10 ++++++---- .../snapshots/SourceOnlySnapshotRepository.java | 1 + .../snapshots/SourceOnlySnapshotTests.java | 13 +++++++------ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java index a6fecdf0d0e5d..cc14a9f693032 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java @@ -75,9 +75,11 @@ public SourceOnlySnapshot(Directory targetDirectory, CheckedSupplier existingSegments; Map> existingFileSets; + final List knownSegmentInfos = knownSegmentCommitsProvider.get().stream() + .map(SourceOnlySnapshotRepository::segmentInfosFromMeta).collect(Collectors.toList()); existingSegments = new HashMap<>(); - for (SegmentCommitInfo info : knownSegmentCommitsProvider.get().stream() - .flatMap(infos -> SourceOnlySnapshotRepository.segmentInfosFromMeta(infos).asList().stream()).collect(Collectors.toList())) { + for (SegmentCommitInfo info : knownSegmentInfos.stream() + .flatMap(infos -> infos.asList().stream()).collect(Collectors.toList())) { existingSegments.put(new BytesRef(info.info.getId()), info); } List createdFiles = new ArrayList<>(); @@ -99,8 +101,8 @@ public synchronized SegmentInfos syncSnapshot(IndexCommit commit) throws IOExcep } } if (existingSegments.values().containsAll(newInfos)) { - return knownSegmentCommitsProvider.get().stream().filter(segmentCommitInfos -> - segmentCommitInfos.asList().containsAll(newInfos)).findFirst().get(); + return knownSegmentInfos + .stream().filter(segmentCommitInfos -> segmentCommitInfos.asList().containsAll(newInfos)).findFirst().get(); } segmentInfos.clear(); segmentInfos.addAll(newInfos); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 2fe38bf1ebed8..8715af817b5ea 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -164,6 +164,7 @@ public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException { } }; // we will use the lucene doc ID as the seq ID so we set the local checkpoint to maxDoc with a new index UUID + // TODO: Fix contains call obviously if (segmentInfosInRepo.contains(newFiles) == false) { final long maxDoc = newFiles.totalMaxDoc(); tempStore.bootstrapNewHistory(maxDoc, maxDoc); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java index 95fdbdec90999..96c08162e480b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java @@ -45,6 +45,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; public class SourceOnlySnapshotTests extends ESTestCase { @@ -59,7 +60,7 @@ public void testSourceOnlyRandom() throws IOException { boolean modifyDeletedDocs = softDeletesField != null && randomBoolean(); SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, modifyDeletedDocs ? () -> new DocValuesFieldExistsQuery(softDeletesField) : null, - () -> List.of(SegmentInfos.readLatestCommit(dir))) { + () -> Collections.emptyList()) { //TODO: Fix @Override DirectoryReader wrapReader(DirectoryReader reader) throws IOException { return modifyDeletedDocs ? reader : super.wrapReader(reader); @@ -169,7 +170,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, writer.commit(); Directory targetDir = newDirectory(); IndexCommit snapshot = deletionPolicy.snapshot(); - SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, () -> List.of(SegmentInfos.readLatestCommit(dir))); + SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, Collections::emptyList); snapshoter.syncSnapshot(snapshot); StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(snapshot); @@ -184,7 +185,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, assertEquals(0, id.totalHits.value); } - snapshoter = new SourceOnlySnapshot(targetDir, () -> List.of(SegmentInfos.readLatestCommit(dir))); + snapshoter = new SourceOnlySnapshot(targetDir, Collections::emptyList); Collection createdFiles = snapshoter.syncSnapshot(snapshot).files(false); assertEquals(0, createdFiles.size()); deletionPolicy.release(snapshot); @@ -204,7 +205,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, writer.commit(); { snapshot = deletionPolicy.snapshot(); - snapshoter = new SourceOnlySnapshot(targetDir, () -> List.of(SegmentInfos.readLatestCommit(dir))); + snapshoter = new SourceOnlySnapshot(targetDir, Collections::emptyList); createdFiles = snapshoter.syncSnapshot(snapshot).files(false); assertEquals(4, createdFiles.size()); for (String file : createdFiles) { @@ -229,7 +230,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, writer.commit(); { snapshot = deletionPolicy.snapshot(); - snapshoter = new SourceOnlySnapshot(targetDir, () -> List.of(SegmentInfos.readLatestCommit(dir))); + snapshoter = new SourceOnlySnapshot(targetDir, Collections::emptyList); createdFiles = snapshoter.syncSnapshot(snapshot).files(false); assertEquals(1, createdFiles.size()); for (String file : createdFiles) { @@ -287,7 +288,7 @@ public boolean keepFullyDeletedSegment(IOSupplier readerIOSupplier) writer.commit(); try (Directory targetDir = newDirectory()) { IndexCommit snapshot = deletionPolicy.snapshot(); - SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, () -> List.of(SegmentInfos.readLatestCommit(dir))); + SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, Collections::emptyList); snapshoter.syncSnapshot(snapshot); try (DirectoryReader snapReader = DirectoryReader.open(targetDir)) { From 3040e081420cabdf04b63d3cc2ef2963a6cb2714 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 10 Feb 2020 21:23:43 +0100 Subject: [PATCH 19/47] reverts --- .../repositories/FilterRepository.java | 6 - .../blobstore/BlobStoreRepository.java | 23 ++++ .../snapshots/SourceOnlySnapshot.java | 43 +++--- .../SourceOnlySnapshotRepository.java | 125 +++--------------- .../SourceOnlySnapshotShardTests.java | 16 +-- .../snapshots/SourceOnlySnapshotTests.java | 15 +-- 6 files changed, 68 insertions(+), 160 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index 98ef02bb1359a..2b5304cf05f8c 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -31,7 +31,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.store.Store; -import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; @@ -170,9 +169,4 @@ public void stop() { public void close() { in.close(); } - - @Override - public List> segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { - return in.segmentsInShard(indexId, shardId, generation); - } } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 2b04400361931..ecb12305b2ad3 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -26,6 +26,9 @@ import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -296,6 +299,26 @@ protected BlobStoreRepository( SnapshotInfo::fromXContentInternal, namedXContentRegistry, compress); } + public static SegmentInfos segmentInfosFromMeta(Iterable files) { + final Directory dir = new ByteBuffersDirectory(); + for (StoreFileMetaData m : files) { + if (m.length() != m.hash().length) { + continue; + } + try (IndexOutput indexOutput = dir.createOutput(m.name(), IOContext.DEFAULT)) { + final BytesRef fileContent = m.hash(); + indexOutput.writeBytes(fileContent.bytes, fileContent.offset, fileContent.length); + } catch (IOException e) { + throw new AssertionError(e); + } + } + try { + return SegmentInfos.readLatestCommit(dir); + } catch (IOException e) { + throw new AssertionError(e); + } + } + @Override protected void doStart() { uncleanStart = metadata.pendingGeneration() > RepositoryData.EMPTY_REPO_GEN && diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java index 8c989c34a3d9d..ef9a8e8d5f063 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java @@ -37,10 +37,8 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; -import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.core.internal.io.IOUtils; -import org.elasticsearch.index.store.StoreFileMetaData; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -52,7 +50,6 @@ import java.util.List; import java.util.Map; import java.util.function.Supplier; -import java.util.stream.Collectors; import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.FIELDS_EXTENSION; import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.INDEX_EXTENSION_PREFIX; @@ -65,36 +62,34 @@ public class SourceOnlySnapshot { private static final String FIELDS_META_EXTENSION = INDEX_EXTENSION_PREFIX + FIELDS_META_EXTENSION_SUFFIX; private final Directory targetDirectory; private final Supplier deleteByQuerySupplier; - private final CheckedSupplier>, IOException> knownSegmentCommitsProvider; - public SourceOnlySnapshot(Directory targetDirectory, Supplier deleteByQuerySupplier, - CheckedSupplier>, IOException> knownSegmentCommitsProvider) { + public SourceOnlySnapshot(Directory targetDirectory, Supplier deleteByQuerySupplier) { this.targetDirectory = targetDirectory; this.deleteByQuerySupplier = deleteByQuerySupplier; - this.knownSegmentCommitsProvider = knownSegmentCommitsProvider; } - public SourceOnlySnapshot(Directory targetDirectory, CheckedSupplier>, IOException> knownSegmentCommitsProvider) { - this(targetDirectory, null, knownSegmentCommitsProvider); + public SourceOnlySnapshot(Directory targetDirectory) { + this(targetDirectory, null); } - public synchronized SegmentInfos syncSnapshot(IndexCommit commit) throws IOException { - Map existingSegments; - Map> existingFileSets; - final List knownSegmentInfos = knownSegmentCommitsProvider.get().stream() - .map(SourceOnlySnapshotRepository::segmentInfosFromMeta).collect(Collectors.toList()); - existingSegments = new HashMap<>(); - for (SegmentCommitInfo info : knownSegmentInfos.stream() - .flatMap(infos -> infos.asList().stream()).collect(Collectors.toList())) { - existingSegments.put(new BytesRef(info.info.getId()), info); + public synchronized List syncSnapshot(IndexCommit commit) throws IOException { + long generation; + Map existingSegments = new HashMap<>(); + if (Lucene.indexExists(targetDirectory)) { + SegmentInfos existingsSegmentInfos = Lucene.readSegmentInfos(targetDirectory); + for (SegmentCommitInfo info : existingsSegmentInfos) { + existingSegments.put(new BytesRef(info.info.getId()), info); + } + generation = existingsSegmentInfos.getGeneration(); + } else { + generation = 1; } List createdFiles = new ArrayList<>(); String segmentFileName; - SegmentInfos segmentInfos; try (Lock writeLock = targetDirectory.obtainLock(IndexWriter.WRITE_LOCK_NAME); StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(commit, Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name()))) { - segmentInfos = reader.getSegmentInfos().clone(); + SegmentInfos segmentInfos = reader.getSegmentInfos().clone(); DirectoryReader wrappedReader = wrapReader(reader); List newInfos = new ArrayList<>(); for (LeafReaderContext ctx : wrappedReader.leaves()) { @@ -106,13 +101,9 @@ public synchronized SegmentInfos syncSnapshot(IndexCommit commit) throws IOExcep newInfos.add(newInfo); } } - if (existingSegments.values().containsAll(newInfos)) { - return knownSegmentInfos - .stream().filter(segmentCommitInfos -> segmentCommitInfos.asList().containsAll(newInfos)).findFirst().get(); - } segmentInfos.clear(); segmentInfos.addAll(newInfos); - segmentInfos.setNextWriteGeneration(segmentInfos.getGeneration()); + segmentInfos.setNextWriteGeneration(Math.max(segmentInfos.getGeneration(), generation) + 1); String pendingSegmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS, "", segmentInfos.getGeneration()); try (IndexOutput segnOutput = targetDirectory.createOutput(pendingSegmentFileName, IOContext.DEFAULT)) { @@ -125,7 +116,7 @@ public synchronized SegmentInfos syncSnapshot(IndexCommit commit) throws IOExcep } Lucene.pruneUnreferencedFiles(segmentFileName, targetDirectory); assert assertCheckIndex(); - return SegmentInfos.readLatestCommit(targetDirectory); + return Collections.unmodifiableList(createdFiles); } private LiveDocs getLiveDocs(LeafReader reader) throws IOException { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 8715af817b5ea..5291819df4663 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -5,17 +5,15 @@ */ package org.elasticsearch.snapshots; +import org.apache.lucene.codecs.blocktree.BlockTreeTermsReader; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.search.Query; -import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FilterDirectory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.SimpleFSDirectory; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; @@ -32,7 +30,6 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.store.Store; -import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.repositories.FilterRepository; import org.elasticsearch.repositories.IndexId; @@ -45,7 +42,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; -import java.util.HashSet; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -135,46 +132,34 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s throw new AssertionError("expected FSDirectory but got " + unwrap.toString()); } Path dataPath = ((FSDirectory) unwrap).getDirectory().getParent(); + // TODO should we have a snapshot tmp directory per shard that is maintained by the system? Path snapPath = dataPath.resolve(SNAPSHOT_DIR_NAME); final List toClose = new ArrayList<>(3); - toClose.add(() -> IOUtils.rm(snapPath)); try { - final List> segmentInfosInRepo = - segmentsInShard(indexId, store.shardId().id(), snapshotStatus.generation()); FSDirectory directory = new SimpleFSDirectory(snapPath); - toClose.add(0, directory); - Supplier querySupplier = mapperService.hasNested() ? Queries::newNestedFilter : null; - // SourceOnlySnapshot will take care of soft- and hard-deletes no special casing needed here - SourceOnlySnapshot snapshot = new SourceOnlySnapshot(directory, querySupplier, - () -> segmentInfosInRepo); - SegmentInfos newFiles = snapshot.syncSnapshot(snapshotIndexCommit); + toClose.add(directory); Store tempStore = new Store(store.shardId(), store.indexSettings(), directory, new ShardLock(store.shardId()) { @Override protected void closeInternal() { // do nothing; } - }, Store.OnClose.EMPTY) { - - @Override - public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException { - // TODO: craft metadata ourselves here instead of physically reading it from the store - final MetadataSnapshot metadataSnapshot = super.getMetadata(commit); - return new MetadataSnapshot(metadataSnapshot.asMap(), - metadataSnapshot.getCommitUserData(), metadataSnapshot.getNumDocs()); - } - }; + }, Store.OnClose.EMPTY); + Supplier querySupplier = mapperService.hasNested() ? Queries::newNestedFilter : null; + // SourceOnlySnapshot will take care of soft- and hard-deletes no special casing needed here + SourceOnlySnapshot snapshot = new SourceOnlySnapshot(tempStore.directory(), querySupplier); + snapshot.syncSnapshot(snapshotIndexCommit); // we will use the lucene doc ID as the seq ID so we set the local checkpoint to maxDoc with a new index UUID - // TODO: Fix contains call obviously - if (segmentInfosInRepo.contains(newFiles) == false) { - final long maxDoc = newFiles.totalMaxDoc(); - tempStore.bootstrapNewHistory(maxDoc, maxDoc); - newFiles = tempStore.readLastCommittedSegmentsInfo(); - } + SegmentInfos segmentInfos = tempStore.readLastCommittedSegmentsInfo(); + final long maxDoc = segmentInfos.totalMaxDoc(); + tempStore.bootstrapNewHistory(maxDoc, maxDoc); store.incRef(); - toClose.add(1, store::decRef); - super.snapshotShard(tempStore, mapperService, snapshotId, indexId, - new SourceOnlyIndexCommit(newFiles, tempStore.directory()), snapshotStatus, writeShardGens, userMetadata, - ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); + toClose.add(store::decRef); + DirectoryReader reader = DirectoryReader.open(tempStore.directory(), + Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name())); + toClose.add(reader); + IndexCommit indexCommit = reader.getIndexCommit(); + super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, snapshotStatus, writeShardGens, + userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); } catch (IOException e) { try { IOUtils.close(toClose); @@ -185,26 +170,6 @@ public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException { } } - public static SegmentInfos segmentInfosFromMeta(Iterable files) { - final Directory dir = new ByteBuffersDirectory(); - for (StoreFileMetaData m : files) { - if (m.length() != m.hash().length) { - continue; - } - try (IndexOutput indexOutput = dir.createOutput(m.name(), IOContext.DEFAULT)) { - final BytesRef fileContent = m.hash(); - indexOutput.writeBytes(fileContent.bytes, fileContent.offset, fileContent.length); - } catch (IOException e) { - throw new AssertionError(e); - } - } - try { - return SegmentInfos.readLatestCommit(dir); - } catch (IOException e) { - throw new AssertionError(e); - } - } - /** * Returns an {@link EngineFactory} for the source only snapshots. */ @@ -242,54 +207,4 @@ public Repository create(RepositoryMetaData metaData, Function getFileNames() throws IOException { - return new HashSet<>(newFiles.files(true)); - } - - @Override - public Directory getDirectory() { - return directory; - } - - @Override - public void delete() { - throw new UnsupportedOperationException("not supported"); - } - - @Override - public boolean isDeleted() { - return false; - } - - @Override - public int getSegmentCount() { - return 0; - } - - @Override - public long getGeneration() { - return newFiles.getLastGeneration(); - } - - @Override - public Map getUserData() { - throw new UnsupportedOperationException("not supported"); - } - } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java index 93198df82cd05..654df5b039843 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java @@ -135,23 +135,11 @@ public void testIncrementalSnapshot() throws IOException { totalFileCount = copy.getTotalFileCount(); assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); } - try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { - IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); - SnapshotId snapshotId = new SnapshotId("test_1", "test"); - final PlainActionFuture future = PlainActionFuture.newFuture(); - runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, Collections.emptyMap(), future)); - shardGeneration = future.actionGet(); - IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); - assertEquals(0, copy.getIncrementalFileCount()); - assertEquals(totalFileCount, copy.getTotalFileCount()); - assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE); - } indexDoc(shard, "_doc", Integer.toString(10)); indexDoc(shard, "_doc", Integer.toString(11)); try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { - SnapshotId snapshotId = new SnapshotId("test_2", "test_1"); + SnapshotId snapshotId = new SnapshotId("test_1", "test_1"); IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); @@ -167,7 +155,7 @@ public void testIncrementalSnapshot() throws IOException { } deleteDoc(shard, Integer.toString(10)); try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { - SnapshotId snapshotId = new SnapshotId("test_3", "test_2"); + SnapshotId snapshotId = new SnapshotId("test_2", "test_2"); IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java index 0620269caaf7b..014a5847d5137 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java @@ -44,8 +44,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.List; public class SourceOnlySnapshotTests extends ESTestCase { @@ -59,8 +57,7 @@ public void testSourceOnlyRandom() throws IOException { // we either use the soft deletes directly or manually delete them to test the additional delete functionality boolean modifyDeletedDocs = softDeletesField != null && randomBoolean(); SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, - modifyDeletedDocs ? () -> new DocValuesFieldExistsQuery(softDeletesField) : null, - () -> Collections.emptyList()) { //TODO: Fix + modifyDeletedDocs ? () -> new DocValuesFieldExistsQuery(softDeletesField) : null) { @Override DirectoryReader wrapReader(DirectoryReader reader) throws IOException { return modifyDeletedDocs ? reader : super.wrapReader(reader); @@ -170,7 +167,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, writer.commit(); Directory targetDir = newDirectory(); IndexCommit snapshot = deletionPolicy.snapshot(); - SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, Collections::emptyList); + SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir); snapshoter.syncSnapshot(snapshot); StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(snapshot); @@ -185,8 +182,8 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, assertEquals(0, id.totalHits.value); } - snapshoter = new SourceOnlySnapshot(targetDir, Collections::emptyList); - Collection createdFiles = snapshoter.syncSnapshot(snapshot).files(false); + snapshoter = new SourceOnlySnapshot(targetDir); + List createdFiles = snapshoter.syncSnapshot(snapshot); assertEquals(0, createdFiles.size()); deletionPolicy.release(snapshot); // now add another doc @@ -231,8 +228,8 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, writer.commit(); { snapshot = deletionPolicy.snapshot(); - snapshoter = new SourceOnlySnapshot(targetDir, Collections::emptyList); - createdFiles = snapshoter.syncSnapshot(snapshot).files(false); + snapshoter = new SourceOnlySnapshot(targetDir); + createdFiles = snapshoter.syncSnapshot(snapshot); assertEquals(1, createdFiles.size()); for (String file : createdFiles) { String extension = IndexFileNames.getExtension(file); From 97ccb91e26508eebae2cab9db9ddfb1933cedb3f Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 10 Feb 2020 21:58:40 +0100 Subject: [PATCH 20/47] shorter --- .../elasticsearch/index/engine/Engine.java | 1 + .../index/engine/InternalEngine.java | 3 +- .../BlobStoreIndexShardSnapshots.java | 1 - .../repositories/Repository.java | 8 - .../blobstore/BlobStoreRepository.java | 150 ++++++++++-------- .../snapshots/SourceOnlySnapshotTests.java | 8 +- 6 files changed, 93 insertions(+), 78 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 4a7eb59469aac..6a9eff7bddf1c 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -111,6 +111,7 @@ public abstract class Engine implements Closeable { public static final String SYNC_COMMIT_ID = "sync_id"; // TODO: Remove sync_id in 9.0 public static final String HISTORY_UUID_KEY = "history_uuid"; + public static final String MAX_PRIMARY_TERM = "max_primary_term"; public static final String MIN_RETAINED_SEQNO = "min_retained_seq_no"; public static final String MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID = "max_unsafe_auto_id_timestamp"; diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 9656b091c2c6f..ae9b0c94c3f8a 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -2283,13 +2283,14 @@ protected void commitIndexWriter(final IndexWriter writer, final Translog transl * {@link IndexWriter#commit()} call flushes all documents, we defer computation of the maximum sequence number to the time * of invocation of the commit data iterator (which occurs after all documents have been flushed to Lucene). */ - final Map commitData = new HashMap<>(6); + final Map commitData = new HashMap<>(7); commitData.put(Translog.TRANSLOG_UUID_KEY, translog.getTranslogUUID()); commitData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(localCheckpoint)); commitData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(localCheckpointTracker.getMaxSeqNo())); commitData.put(MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, Long.toString(maxUnsafeAutoIdTimestamp.get())); commitData.put(HISTORY_UUID_KEY, historyUUID); commitData.put(Engine.MIN_RETAINED_SEQNO, Long.toString(softDeletesPolicy.getMinRetainedSeqNo())); + commitData.put(MAX_PRIMARY_TERM, Long.toString(1)); // TODO: Track real primary term obviously logger.trace("committing writer with commit data [{}]", commitData); return commitData.entrySet().iterator(); }); diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java index 1ffa4fe94507d..309a3c3a38694 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java @@ -28,7 +28,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index 1a470a5d5a7a3..95c4da6666a1b 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -19,8 +19,6 @@ package org.elasticsearch.repositories; import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.index.SegmentCommitInfo; -import org.apache.lucene.index.SegmentInfos; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.SnapshotsInProgress; @@ -33,14 +31,12 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.store.Store; -import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotShardFailure; import java.io.IOException; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -242,10 +238,6 @@ void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, ShardId s */ void updateState(ClusterState state); - default List> segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { - throw new UnsupportedOperationException("segmentsInShard not implemented"); - } - /** * Hook that allows a repository to filter the user supplied snapshot metadata in {@link SnapshotsInProgress.Entry#userMetadata()} * during snapshot initialization. diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index ecb12305b2ad3..f9d1226c28d52 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -84,7 +84,9 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; +import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException; import org.elasticsearch.index.snapshots.IndexShardSnapshotException; @@ -299,9 +301,10 @@ protected BlobStoreRepository( SnapshotInfo::fromXContentInternal, namedXContentRegistry, compress); } - public static SegmentInfos segmentInfosFromMeta(Iterable files) { + public static SegmentInfos segmentInfosFromMeta(Iterable files) { final Directory dir = new ByteBuffersDirectory(); - for (StoreFileMetaData m : files) { + for (BlobStoreIndexShardSnapshot.FileInfo f : files) { + final StoreFileMetaData m = f.metadata(); if (m.length() != m.hash().length) { continue; } @@ -1475,16 +1478,6 @@ private void writeAtomic(final String blobName, final BytesReference bytesRef, b } } - @Override - public List> segmentsInShard(IndexId indexId, int shardId, String generation) throws IOException { - final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots(Collections.emptySet(), - shardContainer(indexId, shardId), generation).v1(); - return blobStoreIndexShardSnapshots.snapshots().stream().map( - snapshotFiles -> snapshotFiles.indexFiles().stream() - .map(BlobStoreIndexShardSnapshot.FileInfo::metadata).collect(Collectors.toList()) - ).collect(Collectors.toList()); - } - @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, @@ -1515,71 +1508,100 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s "Duplicate snapshot name [" + snapshotId.getName() + "] detected, aborting"); } - final List indexCommitPointFiles = new ArrayList<>(); - final BlockingQueue filesToSnapshot = new LinkedBlockingQueue<>(); - store.incRef(); - final Collection fileNames; - final Store.MetadataSnapshot metadataFromStore; - try { - // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should - try { - logger.trace( - "[{}] [{}] Loading store metadata using index commit [{}]", shardId, snapshotId, snapshotIndexCommit); - metadataFromStore = store.getMetadata(snapshotIndexCommit); - fileNames = snapshotIndexCommit.getFileNames(); - } catch (IOException e) { - throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e); + final List> snapshotFileSets = snapshots.snapshots().stream().map( + snapshotFiles -> new ArrayList<>(snapshotFiles.indexFiles()) + ).collect(Collectors.toList()); + + List filesFromSegmentInfos = null; + + final Map userCommitData = snapshotIndexCommit.getUserData(); + final long sequenceNum = Long.parseLong(userCommitData.get(SequenceNumbers.MAX_SEQ_NO)); + final long localCheckpoint = Long.parseLong(userCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)); + final String historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); + assert historyUUID != null; + + for (List snapshotFileSet : snapshotFileSets) { + final SegmentInfos segmentInfos = segmentInfosFromMeta(snapshotFileSet); + final Map snapshotUserCommitData = segmentInfos.getUserData(); + final long snapshotSequenceNum = Long.parseLong(snapshotUserCommitData.get(SequenceNumbers.MAX_SEQ_NO)); + final long snapshotLocalCheckpoint = Long.parseLong(snapshotUserCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)); + if (snapshotSequenceNum == sequenceNum && snapshotLocalCheckpoint == localCheckpoint + && historyUUID.equals(snapshotUserCommitData.get(Engine.HISTORY_UUID_KEY))) { + filesFromSegmentInfos = snapshotFileSet; + break; } - } finally { - store.decRef(); } + + final List indexCommitPointFiles; int indexIncrementalFileCount = 0; int indexTotalNumberOfFiles = 0; long indexIncrementalSize = 0; long indexTotalFileSize = 0; - for (String fileName : fileNames) { - if (snapshotStatus.isAborted()) { - logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName); - throw new IndexShardSnapshotFailedException(shardId, "Aborted"); + final BlockingQueue filesToSnapshot = new LinkedBlockingQueue<>(); + if (filesFromSegmentInfos == null) { + indexCommitPointFiles = new ArrayList<>(); + store.incRef(); + final Collection fileNames; + final Store.MetadataSnapshot metadataFromStore; + try { + // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should + try { + logger.trace( + "[{}] [{}] Loading store metadata using index commit [{}]", shardId, snapshotId, snapshotIndexCommit); + metadataFromStore = store.getMetadata(snapshotIndexCommit); + fileNames = snapshotIndexCommit.getFileNames(); + } catch (IOException e) { + throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e); + } + } finally { + store.decRef(); } + for (String fileName : fileNames) { + if (snapshotStatus.isAborted()) { + logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName); + throw new IndexShardSnapshotFailedException(shardId, "Aborted"); + } - logger.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName); - final StoreFileMetaData md = metadataFromStore.get(fileName); - BlobStoreIndexShardSnapshot.FileInfo existingFileInfo = null; - List filesInfo = snapshots.findPhysicalIndexFiles(fileName); - if (filesInfo != null) { - for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesInfo) { - if (fileInfo.isSame(md)) { - // a commit point file with the same name, size and checksum was already copied to repository - // we will reuse it for this snapshot - existingFileInfo = fileInfo; - break; + logger.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName); + final StoreFileMetaData md = metadataFromStore.get(fileName); + BlobStoreIndexShardSnapshot.FileInfo existingFileInfo = null; + List filesInfo = snapshots.findPhysicalIndexFiles(fileName); + if (filesInfo != null) { + for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesInfo) { + if (fileInfo.isSame(md)) { + // a commit point file with the same name, size and checksum was already copied to repository + // we will reuse it for this snapshot + existingFileInfo = fileInfo; + break; + } } } - } - // We can skip writing blobs where the metadata hash is equal to the blob's contents because we store the hash/contents - // directly in the shard level metadata in this case - final boolean needsWrite = md.hashEqualsContents() == false; - indexTotalFileSize += md.length(); - indexTotalNumberOfFiles++; - - if (existingFileInfo == null) { - indexIncrementalFileCount++; - indexIncrementalSize += md.length(); - // create a new FileInfo - BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo = - new BlobStoreIndexShardSnapshot.FileInfo( - (needsWrite ? UPLOADED_DATA_BLOB_PREFIX : VIRTUAL_DATA_BLOB_PREFIX) + UUIDs.randomBase64UUID(), - md, chunkSize()); - indexCommitPointFiles.add(snapshotFileInfo); - if (needsWrite) { - filesToSnapshot.add(snapshotFileInfo); + // We can skip writing blobs where the metadata hash is equal to the blob's contents because we store the hash/contents + // directly in the shard level metadata in this case + final boolean needsWrite = md.hashEqualsContents() == false; + indexTotalFileSize += md.length(); + indexTotalNumberOfFiles++; + + if (existingFileInfo == null) { + indexIncrementalFileCount++; + indexIncrementalSize += md.length(); + // create a new FileInfo + BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo = + new BlobStoreIndexShardSnapshot.FileInfo( + (needsWrite ? UPLOADED_DATA_BLOB_PREFIX : VIRTUAL_DATA_BLOB_PREFIX) + UUIDs.randomBase64UUID(), + md, chunkSize()); + indexCommitPointFiles.add(snapshotFileInfo); + if (needsWrite) { + filesToSnapshot.add(snapshotFileInfo); + } + assert needsWrite || assertFileContentsMatchHash(snapshotFileInfo, store); + } else { + indexCommitPointFiles.add(existingFileInfo); } - assert needsWrite || assertFileContentsMatchHash(snapshotFileInfo, store); - } else { - indexCommitPointFiles.add(existingFileInfo); } + } else { + indexCommitPointFiles = filesFromSegmentInfos; } snapshotStatus.moveToStarted(startTime, indexIncrementalFileCount, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java index 014a5847d5137..29854002a68b9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java @@ -202,9 +202,9 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, writer.commit(); { snapshot = deletionPolicy.snapshot(); - snapshoter = new SourceOnlySnapshot(targetDir, Collections::emptyList); - createdFiles = snapshoter.syncSnapshot(snapshot).files(false); - assertEquals(4, createdFiles.size()); + snapshoter = new SourceOnlySnapshot(targetDir); + createdFiles = snapshoter.syncSnapshot(snapshot); + assertEquals(5, createdFiles.size()); for (String file : createdFiles) { String extension = IndexFileNames.getExtension(file); switch (extension) { @@ -286,7 +286,7 @@ public boolean keepFullyDeletedSegment(IOSupplier readerIOSupplier) writer.commit(); try (Directory targetDir = newDirectory()) { IndexCommit snapshot = deletionPolicy.snapshot(); - SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir, Collections::emptyList); + SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir); snapshoter.syncSnapshot(snapshot); try (DirectoryReader snapReader = DirectoryReader.open(targetDir)) { From 0defec84aa334866d6f387e990f21af669775266 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 10 Feb 2020 22:24:16 +0100 Subject: [PATCH 21/47] works so far :) --- .../snapshots/BlobStoreIncrementalityIT.java | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java diff --git a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java new file mode 100644 index 0000000000000..22e4066838cb5 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.snapshots; + +import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESIntegTestCase; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +import static org.hamcrest.Matchers.is; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +public class BlobStoreIncrementalityIT extends AbstractSnapshotIntegTestCase { + + public void testIncrementalBehaviorOnPrimaryFailover() throws InterruptedException, ExecutionException, IOException { + internalCluster().startMasterOnlyNode(); + final String primaryNode = internalCluster().startDataOnlyNode(); + final String indexName = "test-index"; + createIndex(indexName, Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build()); + ensureYellow(indexName); + final String replicaNode = internalCluster().startDataOnlyNode(); + ensureGreen(indexName); + for (int j = 0; j < randomIntBetween(1, 10); ++j) { + final BulkRequest bulkRequest = new BulkRequest(); + for (int i = 0; i < scaledRandomIntBetween(1, 100); ++i) { + bulkRequest.add(new IndexRequest(indexName).source("foo" + j, "bar" + i)); + } + client().bulk(bulkRequest).get(); + } + final String snapshot1 = "snap-1"; + final String snapshot2 = "snap-2"; + final String repo = "test-repo"; + assertThat(client().admin().cluster().preparePutRepository(repo) + .setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())).execute().actionGet().isAcknowledged(), + is(true)); + + client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setIndices(indexName).setWaitForCompletion(true).get(); + stopNode(primaryNode); + ensureYellow(indexName); + client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setIndices(indexName).setWaitForCompletion(true).get(); + final SnapshotsStatusResponse response = + client().admin().cluster().prepareSnapshotStatus(repo).setSnapshots(snapshot1, snapshot2).get(); + assertThat(response.getSnapshots().get(1).getIndices().get(indexName) + .getShards().get(0).getStats().getIncrementalFileCount(), is(0)); + } +} From 392945222f015b72932a818aa171f421111ea30f Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 10 Feb 2020 22:24:57 +0100 Subject: [PATCH 22/47] nicer --- .../org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java index 22e4066838cb5..7b84cb37097c6 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java @@ -41,7 +41,7 @@ public void testIncrementalBehaviorOnPrimaryFailover() throws InterruptedExcepti .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build()); ensureYellow(indexName); - final String replicaNode = internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNode(); ensureGreen(indexName); for (int j = 0; j < randomIntBetween(1, 10); ++j) { final BulkRequest bulkRequest = new BulkRequest(); From 15d186a8753fb6a670d759a5db4e2fe51ab505e3 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 11 Feb 2020 01:50:57 +0100 Subject: [PATCH 23/47] makes sense --- .../index/engine/InternalEngine.java | 11 ++++++- .../blobstore/BlobStoreRepository.java | 32 +++++++++++-------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index ae9b0c94c3f8a..929c590c0d0c2 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -180,6 +180,8 @@ public class InternalEngine extends Engine { @Nullable private final String historyUUID; + private volatile long lastOpPrimaryTerm; + public InternalEngine(EngineConfig engineConfig) { this(engineConfig, LocalCheckpointTracker::new); } @@ -220,6 +222,12 @@ public InternalEngine(EngineConfig engineConfig) { bootstrapAppendOnlyInfoFromWriter(writer); historyUUID = loadHistoryUUID(writer); indexWriter = writer; + final String maxPrimaryTerm = store.readLastCommittedSegmentsInfo().getUserData().get(MAX_PRIMARY_TERM); + if (maxPrimaryTerm == null) { + lastOpPrimaryTerm = engineConfig.getPrimaryTermSupplier().getAsLong(); + } else { + lastOpPrimaryTerm = Long.parseLong(maxPrimaryTerm); + } } catch (IOException | TranslogCorruptedException e) { throw new EngineCreationFailureException(shardId, "failed to create engine", e); } catch (AssertionError e) { @@ -2290,7 +2298,7 @@ protected void commitIndexWriter(final IndexWriter writer, final Translog transl commitData.put(MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, Long.toString(maxUnsafeAutoIdTimestamp.get())); commitData.put(HISTORY_UUID_KEY, historyUUID); commitData.put(Engine.MIN_RETAINED_SEQNO, Long.toString(softDeletesPolicy.getMinRetainedSeqNo())); - commitData.put(MAX_PRIMARY_TERM, Long.toString(1)); // TODO: Track real primary term obviously + commitData.put(MAX_PRIMARY_TERM, Long.toString(lastOpPrimaryTerm)); logger.trace("committing writer with commit data [{}]", commitData); return commitData.entrySet().iterator(); }); @@ -2367,6 +2375,7 @@ public long getPersistedLocalCheckpoint() { */ protected final void markSeqNoAsSeen(long seqNo) { localCheckpointTracker.advanceMaxSeqNo(seqNo); + lastOpPrimaryTerm = engineConfig.getPrimaryTermSupplier().getAsLong(); } /** diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index f9d1226c28d52..616042f1c25c7 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1515,20 +1515,24 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s List filesFromSegmentInfos = null; final Map userCommitData = snapshotIndexCommit.getUserData(); - final long sequenceNum = Long.parseLong(userCommitData.get(SequenceNumbers.MAX_SEQ_NO)); - final long localCheckpoint = Long.parseLong(userCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)); - final String historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); - assert historyUUID != null; - - for (List snapshotFileSet : snapshotFileSets) { - final SegmentInfos segmentInfos = segmentInfosFromMeta(snapshotFileSet); - final Map snapshotUserCommitData = segmentInfos.getUserData(); - final long snapshotSequenceNum = Long.parseLong(snapshotUserCommitData.get(SequenceNumbers.MAX_SEQ_NO)); - final long snapshotLocalCheckpoint = Long.parseLong(snapshotUserCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)); - if (snapshotSequenceNum == sequenceNum && snapshotLocalCheckpoint == localCheckpoint - && historyUUID.equals(snapshotUserCommitData.get(Engine.HISTORY_UUID_KEY))) { - filesFromSegmentInfos = snapshotFileSet; - break; + final String sequenceNumString = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); + final String localCheckpointString = userCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY); + if (sequenceNumString != null && localCheckpointString != null) { + final long sequenceNum = Long.parseLong(sequenceNumString); + final long localCheckpoint = Long.parseLong(localCheckpointString); + final String historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); + assert historyUUID != null; + + for (List snapshotFileSet : snapshotFileSets) { + final SegmentInfos segmentInfos = segmentInfosFromMeta(snapshotFileSet); + final Map snapshotUserCommitData = segmentInfos.getUserData(); + final long snapshotSequenceNum = Long.parseLong(snapshotUserCommitData.get(SequenceNumbers.MAX_SEQ_NO)); + final long snapshotLocalCheckpoint = Long.parseLong(snapshotUserCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)); + if (snapshotSequenceNum == sequenceNum && snapshotLocalCheckpoint == localCheckpoint + && historyUUID.equals(snapshotUserCommitData.get(Engine.HISTORY_UUID_KEY))) { + filesFromSegmentInfos = snapshotFileSet; + break; + } } } From e110f836e2414406b777dcb0dff20ba6f41ec630 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 11 Feb 2020 04:22:47 +0100 Subject: [PATCH 24/47] nicer --- .../blobstore/BlobStoreRepository.java | 75 +++++++++---------- 1 file changed, 36 insertions(+), 39 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 616042f1c25c7..1bcb3ac6c41ae 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -301,27 +301,6 @@ protected BlobStoreRepository( SnapshotInfo::fromXContentInternal, namedXContentRegistry, compress); } - public static SegmentInfos segmentInfosFromMeta(Iterable files) { - final Directory dir = new ByteBuffersDirectory(); - for (BlobStoreIndexShardSnapshot.FileInfo f : files) { - final StoreFileMetaData m = f.metadata(); - if (m.length() != m.hash().length) { - continue; - } - try (IndexOutput indexOutput = dir.createOutput(m.name(), IOContext.DEFAULT)) { - final BytesRef fileContent = m.hash(); - indexOutput.writeBytes(fileContent.bytes, fileContent.offset, fileContent.length); - } catch (IOException e) { - throw new AssertionError(e); - } - } - try { - return SegmentInfos.readLatestCommit(dir); - } catch (IOException e) { - throw new AssertionError(e); - } - } - @Override protected void doStart() { uncleanStart = metadata.pendingGeneration() > RepositoryData.EMPTY_REPO_GEN && @@ -1508,29 +1487,30 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s "Duplicate snapshot name [" + snapshotId.getName() + "] detected, aborting"); } - final List> snapshotFileSets = snapshots.snapshots().stream().map( - snapshotFiles -> new ArrayList<>(snapshotFiles.indexFiles()) - ).collect(Collectors.toList()); - List filesFromSegmentInfos = null; - + // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository that has the + // same sequence number, primary term id and history uuid as our current commit. final Map userCommitData = snapshotIndexCommit.getUserData(); final String sequenceNumString = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); final String localCheckpointString = userCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY); - if (sequenceNumString != null && localCheckpointString != null) { - final long sequenceNum = Long.parseLong(sequenceNumString); - final long localCheckpoint = Long.parseLong(localCheckpointString); - final String historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); - assert historyUUID != null; - - for (List snapshotFileSet : snapshotFileSets) { - final SegmentInfos segmentInfos = segmentInfosFromMeta(snapshotFileSet); + final String maxTermString = userCommitData.get(Engine.MAX_PRIMARY_TERM); + final String historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); + if (sequenceNumString != null && localCheckpointString != null && maxTermString != null && historyUUID != null) { + for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) { + final List files = snapshotFileSet.indexFiles(); + final SegmentInfos segmentInfos; + try { + segmentInfos = segmentInfosFromMeta(files); + } catch (IOException e) { + logger.debug("Failed to read SegmentInfos from files {}", files); + continue; + } final Map snapshotUserCommitData = segmentInfos.getUserData(); - final long snapshotSequenceNum = Long.parseLong(snapshotUserCommitData.get(SequenceNumbers.MAX_SEQ_NO)); - final long snapshotLocalCheckpoint = Long.parseLong(snapshotUserCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)); - if (snapshotSequenceNum == sequenceNum && snapshotLocalCheckpoint == localCheckpoint - && historyUUID.equals(snapshotUserCommitData.get(Engine.HISTORY_UUID_KEY))) { - filesFromSegmentInfos = snapshotFileSet; + if (sequenceNumString.equals(snapshotUserCommitData.get(SequenceNumbers.MAX_SEQ_NO)) && + localCheckpointString.equals(snapshotUserCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) && + maxTermString.equals(snapshotUserCommitData.get(Engine.MAX_PRIMARY_TERM)) && + historyUUID.equals(snapshotUserCommitData.get(Engine.HISTORY_UUID_KEY))) { + filesFromSegmentInfos = files; break; } } @@ -1542,6 +1522,8 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s long indexIncrementalSize = 0; long indexTotalFileSize = 0; final BlockingQueue filesToSnapshot = new LinkedBlockingQueue<>(); + // If we did not find a set of files that is equal to the current commit we determine the files to upload by comparing files + // in the commit with files already in the repository if (filesFromSegmentInfos == null) { indexCommitPointFiles = new ArrayList<>(); store.incRef(); @@ -1699,6 +1681,21 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } } + private static SegmentInfos segmentInfosFromMeta(Iterable files) throws IOException { + final Directory dir = new ByteBuffersDirectory(); + for (BlobStoreIndexShardSnapshot.FileInfo f : files) { + final StoreFileMetaData m = f.metadata(); + if (m.length() != m.hash().length) { + continue; + } + try (IndexOutput indexOutput = dir.createOutput(m.name(), IOContext.DEFAULT)) { + final BytesRef fileContent = m.hash(); + indexOutput.writeBytes(fileContent.bytes, fileContent.offset, fileContent.length); + } + } + return SegmentInfos.readLatestCommit(dir); + } + private static boolean assertFileContentsMatchHash(BlobStoreIndexShardSnapshot.FileInfo fileInfo, Store store) { try (IndexInput indexInput = store.openVerifyingInput(fileInfo.physicalName(), IOContext.READONCE, fileInfo.metadata())) { final byte[] tmp = new byte[Math.toIntExact(fileInfo.metadata().length())]; From 72f83923d7a2ab35d62d7a2d514474afd1e6e95f Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 11 Feb 2020 04:36:44 +0100 Subject: [PATCH 25/47] nicer --- .../blobstore/BlobStoreRepository.java | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 1bcb3ac6c41ae..78202c66f56b1 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1487,34 +1487,9 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s "Duplicate snapshot name [" + snapshotId.getName() + "] detected, aborting"); } - List filesFromSegmentInfos = null; // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository that has the // same sequence number, primary term id and history uuid as our current commit. - final Map userCommitData = snapshotIndexCommit.getUserData(); - final String sequenceNumString = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); - final String localCheckpointString = userCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY); - final String maxTermString = userCommitData.get(Engine.MAX_PRIMARY_TERM); - final String historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); - if (sequenceNumString != null && localCheckpointString != null && maxTermString != null && historyUUID != null) { - for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) { - final List files = snapshotFileSet.indexFiles(); - final SegmentInfos segmentInfos; - try { - segmentInfos = segmentInfosFromMeta(files); - } catch (IOException e) { - logger.debug("Failed to read SegmentInfos from files {}", files); - continue; - } - final Map snapshotUserCommitData = segmentInfos.getUserData(); - if (sequenceNumString.equals(snapshotUserCommitData.get(SequenceNumbers.MAX_SEQ_NO)) && - localCheckpointString.equals(snapshotUserCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) && - maxTermString.equals(snapshotUserCommitData.get(Engine.MAX_PRIMARY_TERM)) && - historyUUID.equals(snapshotUserCommitData.get(Engine.HISTORY_UUID_KEY))) { - filesFromSegmentInfos = files; - break; - } - } - } + final List filesFromSegmentInfos = findMatchingShardSnapshot(snapshotIndexCommit, snapshots); final List indexCommitPointFiles; int indexIncrementalFileCount = 0; @@ -1681,6 +1656,35 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } } + private static List findMatchingShardSnapshot( + IndexCommit snapshotIndexCommit, BlobStoreIndexShardSnapshots snapshots) throws IOException { + final Map userCommitData = snapshotIndexCommit.getUserData(); + final String sequenceNumString = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); + final String localCheckpointString = userCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY); + final String maxTermString = userCommitData.get(Engine.MAX_PRIMARY_TERM); + final String historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); + if (sequenceNumString != null && localCheckpointString != null && maxTermString != null && historyUUID != null) { + for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) { + final List files = snapshotFileSet.indexFiles(); + final SegmentInfos segmentInfos; + try { + segmentInfos = segmentInfosFromMeta(files); + } catch (IOException e) { + logger.debug("Failed to read SegmentInfos from files {}", files); + continue; + } + final Map snapshotUserCommitData = segmentInfos.getUserData(); + if (sequenceNumString.equals(snapshotUserCommitData.get(SequenceNumbers.MAX_SEQ_NO)) && + localCheckpointString.equals(snapshotUserCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) && + maxTermString.equals(snapshotUserCommitData.get(Engine.MAX_PRIMARY_TERM)) && + historyUUID.equals(snapshotUserCommitData.get(Engine.HISTORY_UUID_KEY))) { + return files; + } + } + } + return null; + } + private static SegmentInfos segmentInfosFromMeta(Iterable files) throws IOException { final Directory dir = new ByteBuffersDirectory(); for (BlobStoreIndexShardSnapshot.FileInfo f : files) { From a57ca4318713f5ff5cdc7f07bb1f79d9fd2acd2b Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 11 Feb 2020 04:44:45 +0100 Subject: [PATCH 26/47] nicer --- .../blobstore/BlobStoreRepository.java | 45 +++++++++---------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 78202c66f56b1..e31f070560697 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1489,7 +1489,8 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository that has the // same sequence number, primary term id and history uuid as our current commit. - final List filesFromSegmentInfos = findMatchingShardSnapshot(snapshotIndexCommit, snapshots); + final List filesFromSegmentInfos = + findMatchingShardSnapshot(snapshotIndexCommit, snapshots); final List indexCommitPointFiles; int indexIncrementalFileCount = 0; @@ -1659,24 +1660,35 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s private static List findMatchingShardSnapshot( IndexCommit snapshotIndexCommit, BlobStoreIndexShardSnapshots snapshots) throws IOException { final Map userCommitData = snapshotIndexCommit.getUserData(); - final String sequenceNumString = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); - final String localCheckpointString = userCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY); - final String maxTermString = userCommitData.get(Engine.MAX_PRIMARY_TERM); + final String sequenceNum = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); + final String localCheckpoint = userCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY); + final String primaryTerm = userCommitData.get(Engine.MAX_PRIMARY_TERM); final String historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); - if (sequenceNumString != null && localCheckpointString != null && maxTermString != null && historyUUID != null) { + if (sequenceNum != null && localCheckpoint != null && primaryTerm != null && historyUUID != null) { for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) { final List files = snapshotFileSet.indexFiles(); final SegmentInfos segmentInfos; try { - segmentInfos = segmentInfosFromMeta(files); + final Directory dir = new ByteBuffersDirectory(); + for (BlobStoreIndexShardSnapshot.FileInfo f : files) { + final StoreFileMetaData m = f.metadata(); + if (m.length() != m.hash().length) { + continue; + } + try (IndexOutput indexOutput = dir.createOutput(m.name(), IOContext.DEFAULT)) { + final BytesRef fileContent = m.hash(); + indexOutput.writeBytes(fileContent.bytes, fileContent.offset, fileContent.length); + } + } + segmentInfos = SegmentInfos.readLatestCommit(dir); } catch (IOException e) { logger.debug("Failed to read SegmentInfos from files {}", files); continue; } final Map snapshotUserCommitData = segmentInfos.getUserData(); - if (sequenceNumString.equals(snapshotUserCommitData.get(SequenceNumbers.MAX_SEQ_NO)) && - localCheckpointString.equals(snapshotUserCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) && - maxTermString.equals(snapshotUserCommitData.get(Engine.MAX_PRIMARY_TERM)) && + if (sequenceNum.equals(snapshotUserCommitData.get(SequenceNumbers.MAX_SEQ_NO)) && + localCheckpoint.equals(snapshotUserCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) && + primaryTerm.equals(snapshotUserCommitData.get(Engine.MAX_PRIMARY_TERM)) && historyUUID.equals(snapshotUserCommitData.get(Engine.HISTORY_UUID_KEY))) { return files; } @@ -1685,21 +1697,6 @@ private static List findMatchingShardSnaps return null; } - private static SegmentInfos segmentInfosFromMeta(Iterable files) throws IOException { - final Directory dir = new ByteBuffersDirectory(); - for (BlobStoreIndexShardSnapshot.FileInfo f : files) { - final StoreFileMetaData m = f.metadata(); - if (m.length() != m.hash().length) { - continue; - } - try (IndexOutput indexOutput = dir.createOutput(m.name(), IOContext.DEFAULT)) { - final BytesRef fileContent = m.hash(); - indexOutput.writeBytes(fileContent.bytes, fileContent.offset, fileContent.length); - } - } - return SegmentInfos.readLatestCommit(dir); - } - private static boolean assertFileContentsMatchHash(BlobStoreIndexShardSnapshot.FileInfo fileInfo, Store store) { try (IndexInput indexInput = store.openVerifyingInput(fileInfo.physicalName(), IOContext.READONCE, fileInfo.metadata())) { final byte[] tmp = new byte[Math.toIntExact(fileInfo.metadata().length())]; From 82d5bd4296fce816eeeb29ae2f26d3574b44d959 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 11 Feb 2020 05:02:57 +0100 Subject: [PATCH 27/47] doccs --- .../blobstore/BlobStoreRepository.java | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index e31f070560697..64780d2172fc9 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1487,8 +1487,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s "Duplicate snapshot name [" + snapshotId.getName() + "] detected, aborting"); } - // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository that has the - // same sequence number, primary term id and history uuid as our current commit. + // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository final List filesFromSegmentInfos = findMatchingShardSnapshot(snapshotIndexCommit, snapshots); @@ -1657,9 +1656,35 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } } + /** + * Extracts an instance of {@link SegmentInfos} for each snapshot tracked in the given {@link BlobStoreIndexShardSnapshots} and + * compares it against the given {@link IndexCommit}. + * If the sequence number, local checkpoint, primary term and history UUID of the given commit and the found shard snapshot are equal + * then we will not snapshot the files in the index commit but rather assign the files in the existing matching snapshot to the new + * shard snapshot. + * Compared to merely comparing files in the index commit and repository, the logic here will be able to identify shards with equal + * content but different segment structure (as may be the case after replica promotion) and avoid redundant snapshot creation in this + * situation. + * Note: This method does not load any blobs from the repository. It instead exploits the fact that segment info files are stored + * alongside their content as hash in the {@link BlobStoreIndexShardSnapshots} metadata to quickly load all {@link SegmentInfos} for + * each shard snapshot. + * Also see {@link StoreFileMetaData#hashEqualsContents()}. + * + * @param snapshotIndexCommit Index commit to snapshot + * @param snapshots shard snapshots already in the repository + * @return List of files already in the repository to use as the given shard's snapshot or {@code null} if no such set of files could + * be found in the current shard snapshots + */ + @Nullable private static List findMatchingShardSnapshot( - IndexCommit snapshotIndexCommit, BlobStoreIndexShardSnapshots snapshots) throws IOException { - final Map userCommitData = snapshotIndexCommit.getUserData(); + IndexCommit snapshotIndexCommit, BlobStoreIndexShardSnapshots snapshots) { + final Map userCommitData; + try { + userCommitData = snapshotIndexCommit.getUserData(); + } catch (IOException e) { + assert false : new AssertionError("Did not expect #getUserData to throw but saw exception", e); + return null; + } final String sequenceNum = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); final String localCheckpoint = userCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY); final String primaryTerm = userCommitData.get(Engine.MAX_PRIMARY_TERM); @@ -1672,7 +1697,7 @@ private static List findMatchingShardSnaps final Directory dir = new ByteBuffersDirectory(); for (BlobStoreIndexShardSnapshot.FileInfo f : files) { final StoreFileMetaData m = f.metadata(); - if (m.length() != m.hash().length) { + if (m.hashEqualsContents() == false) { continue; } try (IndexOutput indexOutput = dir.createOutput(m.name(), IOContext.DEFAULT)) { From 31c46f8261a5e1570398eaf6b6602ff86766f3a4 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 11 Feb 2020 07:07:19 +0100 Subject: [PATCH 28/47] test and fix delete case --- .../index/engine/InternalEngine.java | 1 + .../snapshots/BlobStoreIncrementalityIT.java | 114 +++++++++++++++++- 2 files changed, 109 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 929c590c0d0c2..3ed08d2b6cf19 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -2612,6 +2612,7 @@ public void advanceMaxSeqNoOfUpdatesOrDeletes(long maxSeqNoOfUpdatesOnPrimary) { throw new IllegalArgumentException("max_seq_no_of_updates on primary is unassigned"); } this.maxSeqNoOfUpdatesOrDeletes.updateAndGet(curr -> Math.max(curr, maxSeqNoOfUpdatesOnPrimary)); + lastOpPrimaryTerm = engineConfig.getPrimaryTermSupplier().getAsLong(); } private boolean assertMaxSeqNoOfUpdatesIsAdvanced(Term id, long seqNo, boolean allowDeleted, boolean relaxIfGapInSeqNo) { diff --git a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java index 7b84cb37097c6..f7bc18f7bd351 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java @@ -18,16 +18,27 @@ */ package org.elasticsearch.snapshots; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStats; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; +import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; import java.util.concurrent.ExecutionException; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) @@ -39,31 +50,122 @@ public void testIncrementalBehaviorOnPrimaryFailover() throws InterruptedExcepti final String indexName = "test-index"; createIndex(indexName, Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build()); + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0).build()); ensureYellow(indexName); - internalCluster().startDataOnlyNode(); + final String newPrimary = internalCluster().startDataOnlyNode(); + final Collection toDelete = new ArrayList<>(); ensureGreen(indexName); + + logger.info("--> adding some documents to test index"); for (int j = 0; j < randomIntBetween(1, 10); ++j) { final BulkRequest bulkRequest = new BulkRequest(); for (int i = 0; i < scaledRandomIntBetween(1, 100); ++i) { bulkRequest.add(new IndexRequest(indexName).source("foo" + j, "bar" + i)); } - client().bulk(bulkRequest).get(); + final BulkResponse bulkResponse = client().bulk(bulkRequest).get(); + for (BulkItemResponse item : bulkResponse.getItems()) { + if (randomBoolean()) { + toDelete.add(item.getId()); + } + } } + refresh(indexName); + + final long documentCountOriginal = getCountForIndex(indexName); + final String snapshot1 = "snap-1"; - final String snapshot2 = "snap-2"; final String repo = "test-repo"; + logger.info("--> creating repository"); assertThat(client().admin().cluster().preparePutRepository(repo) .setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())).execute().actionGet().isAcknowledged(), is(true)); + logger.info("--> creating snapshot 1"); client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setIndices(indexName).setWaitForCompletion(true).get(); + + logger.info("--> Shutting down initial primary node [{}]", primaryNode); stopNode(primaryNode); + ensureYellow(indexName); + final String snapshot2 = "snap-2"; + logger.info("--> creating snapshot 2"); client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setIndices(indexName).setWaitForCompletion(true).get(); + + assertTwoIdenticalShardSnapshots(repo, indexName, snapshot1, snapshot2); + + ensureRestoreSingleShardSuccessfully(repo, indexName, snapshot1, "-copy-1"); + assertCountInIndexThenDelete(indexName + "-copy-1", documentCountOriginal); + + ensureRestoreSingleShardSuccessfully(repo, indexName, snapshot2, "-copy-2"); + assertCountInIndexThenDelete(indexName + "-copy-2", documentCountOriginal); + + internalCluster().startDataOnlyNode(); + ensureGreen(indexName); + + logger.info("--> delete some documents from test index"); + for (final String id : toDelete) { + assertThat(client().prepareDelete(indexName, id).get().getResult(), is(DocWriteResponse.Result.DELETED)); + } + + refresh(indexName); + + final String snapshot3 = "snap-3"; + logger.info("--> creating snapshot 3"); + client().admin().cluster().prepareCreateSnapshot(repo, snapshot3).setIndices(indexName).setWaitForCompletion(true).get(); + + logger.info("--> Shutting down new primary node [{}]", newPrimary); + stopNode(newPrimary); + ensureYellow(indexName); + + final String snapshot4 = "snap-4"; + logger.info("--> creating snapshot 4"); + client().admin().cluster().prepareCreateSnapshot(repo, snapshot4).setIndices(indexName).setWaitForCompletion(true).get(); + + assertTwoIdenticalShardSnapshots(repo, indexName, snapshot3, snapshot4); + + final long countAfterDelete = documentCountOriginal - toDelete.size(); + ensureRestoreSingleShardSuccessfully(repo, indexName, snapshot3, "-copy-3"); + assertCountInIndexThenDelete(indexName + "-copy-3", countAfterDelete); + + ensureRestoreSingleShardSuccessfully(repo, indexName, snapshot4, "-copy-4"); + assertCountInIndexThenDelete(indexName + "-copy-4", countAfterDelete); + } + + private void assertCountInIndexThenDelete(String index, long expectedCount) throws ExecutionException, InterruptedException { + logger.info("--> asserting that index [{}] contains [{}] documents", index, expectedCount); + assertThat(getCountForIndex(index), is(expectedCount)); + logger.info("--> deleting index [{}]", index); + assertThat(client().admin().indices().prepareDelete(index).get().isAcknowledged(), is(true)); + } + + private void assertTwoIdenticalShardSnapshots(String repo, String indexName, String snapshot1, String snapshot2) { + logger.info( + "--> asserting that snapshots [{}] and [{}] are referring to the same files in the repository", snapshot1, snapshot2); final SnapshotsStatusResponse response = client().admin().cluster().prepareSnapshotStatus(repo).setSnapshots(snapshot1, snapshot2).get(); - assertThat(response.getSnapshots().get(1).getIndices().get(indexName) - .getShards().get(0).getStats().getIncrementalFileCount(), is(0)); + final SnapshotStats firstSnapshotShardStatus = + response.getSnapshots().get(0).getIndices().get(indexName).getShards().get(0).getStats(); + final int totalFilesInShard = firstSnapshotShardStatus.getTotalFileCount(); + assertThat(totalFilesInShard, greaterThan(0)); + assertThat(firstSnapshotShardStatus.getIncrementalFileCount(), is(totalFilesInShard)); + final SnapshotStats secondSnapshotShardStatus = + response.getSnapshots().get(1).getIndices().get(indexName).getShards().get(0).getStats(); + assertThat(secondSnapshotShardStatus.getTotalFileCount(), is(totalFilesInShard)); + assertThat(secondSnapshotShardStatus.getIncrementalFileCount(), is(0)); + } + + private void ensureRestoreSingleShardSuccessfully(String repo, String indexName, String snapshot, String indexSuffix) { + logger.info("--> restoring [{}]", snapshot); + final RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot(repo, snapshot) + .setIndices(indexName).setRenamePattern("(.+)").setRenameReplacement("$1" + indexSuffix).setWaitForCompletion(true).get(); + final RestoreInfo restoreInfo = restoreSnapshotResponse.getRestoreInfo(); + assertThat(restoreInfo.totalShards(), is(1)); + assertThat(restoreInfo.failedShards(), is(0)); + } + + private long getCountForIndex(String indexName) throws ExecutionException, InterruptedException { + return client().search(new SearchRequest(new SearchRequest(indexName).source( + new SearchSourceBuilder().size(0).trackTotalHits(true)))).get().getHits().getTotalHits().value; } } From 02733bf9675512348a6c60c439e5aaadb088bb10 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 17 Feb 2020 18:41:07 +0100 Subject: [PATCH 29/47] bck --- .../snapshots/blobstore/SnapshotFiles.java | 16 ++++++++- .../blobstore/BlobStoreRepository.java | 36 +++---------------- .../xpack/ccr/repository/CcrRepository.java | 3 +- 3 files changed, 22 insertions(+), 33 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java index 5da7b2b1f8c6b..2c8aba2ac050f 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java @@ -33,6 +33,10 @@ public class SnapshotFiles { private final List indexFiles; + private final long sequenceNo; + + private final String historyUUID; + private Map physicalFiles = null; /** @@ -48,9 +52,19 @@ public String snapshot() { * @param snapshot snapshot name * @param indexFiles index files */ - public SnapshotFiles(String snapshot, List indexFiles ) { + public SnapshotFiles(String snapshot, List indexFiles, long sequenceNo, String historyUUID) { this.snapshot = snapshot; this.indexFiles = indexFiles; + this.sequenceNo = sequenceNo; + this.historyUUID = historyUUID; + } + + public long sequenceNo() { + return sequenceNo; + } + + public String historyUUID() { + return historyUUID; } /** diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index dd7fd3f37722e..f014b4a64830a 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1689,39 +1689,12 @@ private static List findMatchingShardSnaps assert false : new AssertionError("Did not expect #getUserData to throw but saw exception", e); return null; } - final String sequenceNum = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); - final String localCheckpoint = userCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY); - final String primaryTerm = userCommitData.get(Engine.MAX_PRIMARY_TERM); + final long sequenceNum = Long.parseLong(userCommitData.get(SequenceNumbers.MAX_SEQ_NO)); final String historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); - if (sequenceNum != null && localCheckpoint != null && primaryTerm != null && historyUUID != null) { for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) { - final List files = snapshotFileSet.indexFiles(); - final SegmentInfos segmentInfos; - try { - final Directory dir = new ByteBuffersDirectory(); - for (BlobStoreIndexShardSnapshot.FileInfo f : files) { - final StoreFileMetaData m = f.metadata(); - if (m.hashEqualsContents() == false) { - continue; - } - try (IndexOutput indexOutput = dir.createOutput(m.name(), IOContext.DEFAULT)) { - final BytesRef fileContent = m.hash(); - indexOutput.writeBytes(fileContent.bytes, fileContent.offset, fileContent.length); - } - } - segmentInfos = SegmentInfos.readLatestCommit(dir); - } catch (IOException e) { - logger.debug("Failed to read SegmentInfos from files {}", files); - continue; + if(snapshotFileSet.historyUUID().equals(historyUUID) && snapshotFileSet.sequenceNo() == sequenceNum) { + return snapshotFileSet.indexFiles(); } - final Map snapshotUserCommitData = segmentInfos.getUserData(); - if (sequenceNum.equals(snapshotUserCommitData.get(SequenceNumbers.MAX_SEQ_NO)) && - localCheckpoint.equals(snapshotUserCommitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) && - primaryTerm.equals(snapshotUserCommitData.get(Engine.MAX_PRIMARY_TERM)) && - historyUUID.equals(snapshotUserCommitData.get(Engine.HISTORY_UUID_KEY))) { - return files; - } - } } return null; } @@ -1747,7 +1720,8 @@ public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, Sh final BlobContainer container = shardContainer(indexId, snapshotShardId); executor.execute(ActionRunnable.wrap(restoreListener, l -> { final BlobStoreIndexShardSnapshot snapshot = loadShardSnapshot(container, snapshotId); - final SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()); + final SnapshotFiles snapshotFiles = + new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), SequenceNumbers.UNASSIGNED_SEQ_NO, ""); new FileRestoreContext(metadata.name(), shardId, snapshotId, recoveryState) { @Override protected void restoreFiles(List filesToRecover, Store store, diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 7f4c045f22643..d75b47879c42d 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -46,6 +46,7 @@ import org.elasticsearch.index.seqno.RetentionLeaseAlreadyExistsException; import org.elasticsearch.index.seqno.RetentionLeaseInvalidRetainingSeqNoException; import org.elasticsearch.index.seqno.RetentionLeaseNotFoundException; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexShardRecoveryException; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException; @@ -487,7 +488,7 @@ void restoreFiles(Store store, ActionListener listener) { ByteSizeValue fileSize = new ByteSizeValue(fileMetaData.length()); fileInfos.add(new FileInfo(fileMetaData.name(), fileMetaData, fileSize)); } - SnapshotFiles snapshotFiles = new SnapshotFiles(LATEST, fileInfos); + SnapshotFiles snapshotFiles = new SnapshotFiles(LATEST, fileInfos, SequenceNumbers.UNASSIGNED_SEQ_NO, ""); restore(snapshotFiles, store, listener); } From ed0477cf36f74195d6e92dc79eeaa1014966b58e Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 18 Feb 2020 18:50:01 +0100 Subject: [PATCH 30/47] works --- .../BlobStoreIndexShardSnapshots.java | 25 ++++++-- .../blobstore/BlobStoreRepository.java | 58 +++++++------------ .../snapshots/SnapshotsService.java | 2 + 3 files changed, 42 insertions(+), 43 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java index 309a3c3a38694..dd11ea1952f34 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import java.io.IOException; @@ -135,6 +136,8 @@ static final class Fields { static final class ParseFields { static final ParseField FILES = new ParseField("files"); + static final ParseField SEQUENCE_NUM = new ParseField("sequence_num"); + static final ParseField HISTORY_UUID = new ParseField("history_uuid"); static final ParseField SNAPSHOTS = new ParseField("snapshots"); } @@ -207,6 +210,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.value(fileInfo.name()); } builder.endArray(); + if (snapshot.sequenceNo() > SequenceNumbers.UNASSIGNED_SEQ_NO) { + builder.field(ParseFields.SEQUENCE_NUM.getPreferredName(), snapshot.sequenceNo()); + builder.field(ParseFields.HISTORY_UUID.getPreferredName(), snapshot.historyUUID()); + } builder.endObject(); } builder.endObject(); @@ -219,6 +226,8 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t token = parser.nextToken(); } Map> snapshotsMap = new HashMap<>(); + Map historyUUIDs = new HashMap<>(); + Map sequenceNumbers = new HashMap<>(); Map files = new HashMap<>(); if (token == XContentParser.Token.START_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { @@ -250,15 +259,19 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); - if (parser.nextToken() == XContentParser.Token.START_ARRAY) { - if (ParseFields.FILES.match(currentFieldName, parser.getDeprecationHandler()) == false) { - throw new ElasticsearchParseException("unknown array [{}]", currentFieldName); - } + if (ParseFields.FILES.match(currentFieldName, parser.getDeprecationHandler()) && + parser.nextToken() == XContentParser.Token.START_ARRAY) { List fileNames = new ArrayList<>(); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { fileNames.add(parser.text()); } snapshotsMap.put(snapshot, fileNames); + } else if (ParseFields.HISTORY_UUID.match(currentFieldName, parser.getDeprecationHandler())) { + parser.nextToken(); + historyUUIDs.put(snapshot, parser.text()); + } else if (ParseFields.SEQUENCE_NUM.match(currentFieldName, parser.getDeprecationHandler())) { + parser.nextToken(); + sequenceNumbers.put(snapshot, parser.longValue()); } } } @@ -277,7 +290,9 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t assert fileInfo != null; fileInfosBuilder.add(fileInfo); } - snapshots.add(new SnapshotFiles(entry.getKey(), Collections.unmodifiableList(fileInfosBuilder))); + snapshots.add(new SnapshotFiles(entry.getKey(), Collections.unmodifiableList(fileInfosBuilder), + sequenceNumbers.getOrDefault(entry.getKey(), SequenceNumbers.UNASSIGNED_SEQ_NO), + historyUUIDs.getOrDefault(entry.getKey(), ""))); } return new BlobStoreIndexShardSnapshots(files, Collections.unmodifiableList(snapshots)); } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index f014b4a64830a..597733c93a2e3 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -26,9 +26,6 @@ import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.store.ByteBuffersDirectory; -import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -1489,10 +1486,20 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s throw new IndexShardSnapshotFailedException(shardId, "Duplicate snapshot name [" + snapshotId.getName() + "] detected, aborting"); } - + final Map userCommitData = snapshotIndexCommit.getUserData(); + final String seqNumString = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); + final long sequenceNum; + final String historyUUID; + if (seqNumString == null) { + sequenceNum = SequenceNumbers.UNASSIGNED_SEQ_NO; + historyUUID = ""; + } else { + sequenceNum = Long.parseLong(userCommitData.get(SequenceNumbers.MAX_SEQ_NO)); + historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); + } // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository final List filesFromSegmentInfos = - findMatchingShardSnapshot(snapshotIndexCommit, snapshots); + findMatchingShardSnapshot(sequenceNum, historyUUID, snapshots); final List indexCommitPointFiles; int indexIncrementalFileCount = 0; @@ -1594,7 +1601,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones List newSnapshotsList = new ArrayList<>(); - newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles())); + newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), sequenceNum, historyUUID)); for (SnapshotFiles point : snapshots) { newSnapshotsList.add(point); } @@ -1660,41 +1667,16 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } } - /** - * Extracts an instance of {@link SegmentInfos} for each snapshot tracked in the given {@link BlobStoreIndexShardSnapshots} and - * compares it against the given {@link IndexCommit}. - * If the sequence number, local checkpoint, primary term and history UUID of the given commit and the found shard snapshot are equal - * then we will not snapshot the files in the index commit but rather assign the files in the existing matching snapshot to the new - * shard snapshot. - * Compared to merely comparing files in the index commit and repository, the logic here will be able to identify shards with equal - * content but different segment structure (as may be the case after replica promotion) and avoid redundant snapshot creation in this - * situation. - * Note: This method does not load any blobs from the repository. It instead exploits the fact that segment info files are stored - * alongside their content as hash in the {@link BlobStoreIndexShardSnapshots} metadata to quickly load all {@link SegmentInfos} for - * each shard snapshot. - * Also see {@link StoreFileMetaData#hashEqualsContents()}. - * - * @param snapshotIndexCommit Index commit to snapshot - * @param snapshots shard snapshots already in the repository - * @return List of files already in the repository to use as the given shard's snapshot or {@code null} if no such set of files could - * be found in the current shard snapshots - */ @Nullable - private static List findMatchingShardSnapshot( - IndexCommit snapshotIndexCommit, BlobStoreIndexShardSnapshots snapshots) { - final Map userCommitData; - try { - userCommitData = snapshotIndexCommit.getUserData(); - } catch (IOException e) { - assert false : new AssertionError("Did not expect #getUserData to throw but saw exception", e); + private static List findMatchingShardSnapshot(long sequenceNum, String historyUUID, + BlobStoreIndexShardSnapshots snapshots) { + if (sequenceNum == SequenceNumbers.UNASSIGNED_SEQ_NO) { return null; } - final long sequenceNum = Long.parseLong(userCommitData.get(SequenceNumbers.MAX_SEQ_NO)); - final String historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); - for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) { - if(snapshotFileSet.historyUUID().equals(historyUUID) && snapshotFileSet.sequenceNo() == sequenceNum) { - return snapshotFileSet.indexFiles(); - } + for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) { + if (snapshotFileSet.historyUUID().equals(historyUUID) && snapshotFileSet.sequenceNo() == sequenceNum) { + return snapshotFileSet.indexFiles(); + } } return null; } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 040e574f67505..caab6d201a425 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -122,6 +122,8 @@ */ public class SnapshotsService extends AbstractLifecycleComponent implements ClusterStateApplier { + public static final Version SEQUENCE_NUM_TRACKING_VERSION = Version.V_8_0_0; + public static final Version SHARD_GEN_IN_REPO_DATA_VERSION = Version.V_7_6_0; public static final Version OLD_SNAPSHOT_FORMAT = Version.V_7_5_0; From 2aefef323928f1131efbeeddce4d92147d7dd9d2 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 18 Feb 2020 22:26:08 +0100 Subject: [PATCH 31/47] cleanup useless changes --- .../java/org/elasticsearch/index/engine/Engine.java | 1 - .../elasticsearch/index/engine/InternalEngine.java | 13 +------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 6a9eff7bddf1c..4a7eb59469aac 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -111,7 +111,6 @@ public abstract class Engine implements Closeable { public static final String SYNC_COMMIT_ID = "sync_id"; // TODO: Remove sync_id in 9.0 public static final String HISTORY_UUID_KEY = "history_uuid"; - public static final String MAX_PRIMARY_TERM = "max_primary_term"; public static final String MIN_RETAINED_SEQNO = "min_retained_seq_no"; public static final String MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID = "max_unsafe_auto_id_timestamp"; diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 3ed08d2b6cf19..9656b091c2c6f 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -180,8 +180,6 @@ public class InternalEngine extends Engine { @Nullable private final String historyUUID; - private volatile long lastOpPrimaryTerm; - public InternalEngine(EngineConfig engineConfig) { this(engineConfig, LocalCheckpointTracker::new); } @@ -222,12 +220,6 @@ public InternalEngine(EngineConfig engineConfig) { bootstrapAppendOnlyInfoFromWriter(writer); historyUUID = loadHistoryUUID(writer); indexWriter = writer; - final String maxPrimaryTerm = store.readLastCommittedSegmentsInfo().getUserData().get(MAX_PRIMARY_TERM); - if (maxPrimaryTerm == null) { - lastOpPrimaryTerm = engineConfig.getPrimaryTermSupplier().getAsLong(); - } else { - lastOpPrimaryTerm = Long.parseLong(maxPrimaryTerm); - } } catch (IOException | TranslogCorruptedException e) { throw new EngineCreationFailureException(shardId, "failed to create engine", e); } catch (AssertionError e) { @@ -2291,14 +2283,13 @@ protected void commitIndexWriter(final IndexWriter writer, final Translog transl * {@link IndexWriter#commit()} call flushes all documents, we defer computation of the maximum sequence number to the time * of invocation of the commit data iterator (which occurs after all documents have been flushed to Lucene). */ - final Map commitData = new HashMap<>(7); + final Map commitData = new HashMap<>(6); commitData.put(Translog.TRANSLOG_UUID_KEY, translog.getTranslogUUID()); commitData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(localCheckpoint)); commitData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(localCheckpointTracker.getMaxSeqNo())); commitData.put(MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, Long.toString(maxUnsafeAutoIdTimestamp.get())); commitData.put(HISTORY_UUID_KEY, historyUUID); commitData.put(Engine.MIN_RETAINED_SEQNO, Long.toString(softDeletesPolicy.getMinRetainedSeqNo())); - commitData.put(MAX_PRIMARY_TERM, Long.toString(lastOpPrimaryTerm)); logger.trace("committing writer with commit data [{}]", commitData); return commitData.entrySet().iterator(); }); @@ -2375,7 +2366,6 @@ public long getPersistedLocalCheckpoint() { */ protected final void markSeqNoAsSeen(long seqNo) { localCheckpointTracker.advanceMaxSeqNo(seqNo); - lastOpPrimaryTerm = engineConfig.getPrimaryTermSupplier().getAsLong(); } /** @@ -2612,7 +2602,6 @@ public void advanceMaxSeqNoOfUpdatesOrDeletes(long maxSeqNoOfUpdatesOnPrimary) { throw new IllegalArgumentException("max_seq_no_of_updates on primary is unassigned"); } this.maxSeqNoOfUpdatesOrDeletes.updateAndGet(curr -> Math.max(curr, maxSeqNoOfUpdatesOnPrimary)); - lastOpPrimaryTerm = engineConfig.getPrimaryTermSupplier().getAsLong(); } private boolean assertMaxSeqNoOfUpdatesIsAdvanced(Term id, long seqNo, boolean allowDeleted, boolean relaxIfGapInSeqNo) { From 10a14c47e36572c664135c7d43c85e8d5f64ce3e Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 18 Feb 2020 22:26:33 +0100 Subject: [PATCH 32/47] cleanup useless changes --- .../main/java/org/elasticsearch/snapshots/SnapshotsService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index caab6d201a425..040e574f67505 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -122,8 +122,6 @@ */ public class SnapshotsService extends AbstractLifecycleComponent implements ClusterStateApplier { - public static final Version SEQUENCE_NUM_TRACKING_VERSION = Version.V_8_0_0; - public static final Version SHARD_GEN_IN_REPO_DATA_VERSION = Version.V_7_6_0; public static final Version OLD_SNAPSHOT_FORMAT = Version.V_7_5_0; From 2b7a19fa6ef8e80e6dbda1e52742f797e9a5629d Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 20 Feb 2020 11:11:00 +0100 Subject: [PATCH 33/47] determine whether commit is safe in snapshot shards service --- .../repositories/FilterRepository.java | 8 ++++---- .../elasticsearch/repositories/Repository.java | 6 ++++-- .../blobstore/BlobStoreRepository.java | 8 +++++--- .../snapshots/SnapshotShardsService.java | 18 +++++++++++++++++- .../repositories/RepositoriesServiceTests.java | 4 ++-- .../repositories/fs/FsRepositoryTests.java | 8 ++++---- .../RepositoryFilterUserMetadataIT.java | 8 ++++---- .../index/shard/IndexShardTestCase.java | 2 +- .../index/shard/RestoreOnlyRepository.java | 4 ++-- .../xpack/ccr/repository/CcrRepository.java | 4 ++-- .../SourceOnlySnapshotRepository.java | 8 ++++---- .../SourceOnlySnapshotShardTests.java | 10 +++++----- 12 files changed, 54 insertions(+), 34 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index f41137c763289..fe0c1533a2721 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -121,10 +121,10 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, - Map userMetadata, ActionListener listener) { - in.snapshotShard( - store, mapperService, snapshotId, indexId, snapshotIndexCommit, snapshotStatus, repositoryMetaVersion, userMetadata, listener); + IndexCommit snapshotIndexCommit, boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, + Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { + in.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, isSafeCommit, snapshotStatus, + repositoryMetaVersion, userMetadata, listener); } @Override public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState, diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index bca034564bdd9..d99cb889f1781 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -199,14 +199,16 @@ void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, * @param snapshotId snapshot id * @param indexId id for the index being snapshotted * @param snapshotIndexCommit commit point + * @param isSafeCommit true if the passed index commit was taken when the shards local checkpoint was equal to its + * global checkpoint * @param snapshotStatus snapshot status * @param repositoryMetaVersion version of the updated repository metadata to write * @param userMetadata user metadata of the snapshot found in {@link SnapshotsInProgress.Entry#userMetadata()} * @param listener listener invoked on completion */ void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, - IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, - ActionListener listener); + boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, + Map userMetadata, ActionListener listener); /** * Restores snapshot of the shard. diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 597733c93a2e3..cb37b120a3ee0 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1459,8 +1459,8 @@ private void writeAtomic(final String blobName, final BytesReference bytesRef, b @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, - Map userMetadata, ActionListener listener) { + IndexCommit snapshotIndexCommit, boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, + Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { final ShardId shardId = store.shardId(); final long startTime = threadPool.absoluteTimeInMillis(); try { @@ -1487,7 +1487,9 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s "Duplicate snapshot name [" + snapshotId.getName() + "] detected, aborting"); } final Map userCommitData = snapshotIndexCommit.getUserData(); - final String seqNumString = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); + // We only check the sequence number to see if the shard has changed if we know that the commit is safe, + // otherwise we short-circuit things here by not reading the sequence number from the commit + final String seqNumString = isSafeCommit ? userCommitData.get(SequenceNumbers.MAX_SEQ_NO) : null; final long sequenceNum; final String historyUUID; if (seqNumString == null) { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 33ec18da99660..02031e5fd6300 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -23,6 +23,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.index.IndexCommit; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; @@ -55,6 +56,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; @@ -340,8 +342,9 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde try { // we flush first to make sure we get the latest writes snapshotted snapshotRef = indexShard.acquireLastIndexCommit(true); + final IndexCommit indexCommit = snapshotRef.getIndexCommit(); repository.snapshotShard(indexShard.store(), indexShard.mapperService(), snapshot.getSnapshotId(), indexId, - snapshotRef.getIndexCommit(), snapshotStatus, version, userMetadata, + indexCommit, isSafeIndexCommit(indexShard, indexCommit), snapshotStatus, version, userMetadata, ActionListener.runBefore(listener, snapshotRef::close)); } catch (Exception e) { IOUtils.close(snapshotRef); @@ -352,6 +355,19 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde } } + /** + * Checks whether the index commit is from a sequence number that is equal to the shards global and local checkpoint. + * + * @param indexShard shard + * @param indexCommit index commit + * @return true if max sequence number in the index commit is equal to the shard's global and local checkpoint + */ + private static boolean isSafeIndexCommit(IndexShard indexShard, IndexCommit indexCommit) throws IOException { + final long localCheckPoint = indexShard.getLocalCheckpoint(); + return localCheckPoint == indexShard.getLastKnownGlobalCheckpoint() + && localCheckPoint == Long.parseLong(indexCommit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)); + } + /** * Checks if any shards were processed that the new master doesn't know about */ diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index 8778cbdc8689f..cb430b564e233 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -201,8 +201,8 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, - Map userMetadata, ActionListener listener) { + IndexCommit snapshotIndexCommit, boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, + Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { } diff --git a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java index 49ca709118074..c2e31fc32ca99 100644 --- a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java @@ -105,8 +105,8 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException { final PlainActionFuture future1 = PlainActionFuture.newFuture(); runGeneric(threadPool, () -> { IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(null); - repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, snapshotStatus, Version.CURRENT, - Collections.emptyMap(), future1); + repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, randomBoolean(), snapshotStatus, + Version.CURRENT, Collections.emptyMap(), future1); future1.actionGet(); IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy(); assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount()); @@ -134,8 +134,8 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException { final PlainActionFuture future2 = PlainActionFuture.newFuture(); runGeneric(threadPool, () -> { IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); - repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit, snapshotStatus, Version.CURRENT, - Collections.emptyMap(), future2); + repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit, randomBoolean(), snapshotStatus, + Version.CURRENT, Collections.emptyMap(), future2); future2.actionGet(); IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy(); assertEquals(2, copy.getIncrementalFileCount()); diff --git a/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java b/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java index 1f09afdc76908..7902c39325c2a 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java @@ -95,11 +95,11 @@ public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenera @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, - Version repositoryMetaVersion, Map userMetadata, - ActionListener listener) { + IndexCommit snapshotIndexCommit, boolean isSafeCommit, + IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, + Map userMetadata, ActionListener listener) { assertThat(userMetadata, is(Collections.singletonMap(MOCK_FILTERED_META, initialMetaValue))); - super.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, snapshotStatus, + super.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, isSafeCommit, snapshotStatus, repositoryMetaVersion, userMetadata, listener); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index e73c145c3b5b8..635a5e7d1cb37 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -838,7 +838,7 @@ protected String snapshotShard(final IndexShard shard, final String shardGen; try (Engine.IndexCommitRef indexCommitRef = shard.acquireLastIndexCommit(true)) { repository.snapshotShard(shard.store(), shard.mapperService(), snapshot.getSnapshotId(), indexId, - indexCommitRef.getIndexCommit(), snapshotStatus, Version.CURRENT, Collections.emptyMap(), future); + indexCommitRef.getIndexCommit(), randomBoolean(), snapshotStatus, Version.CURRENT, Collections.emptyMap(), future); shardGen = future.actionGet(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java index 4fae0fc14952a..f800580d2824b 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java @@ -133,8 +133,8 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, - Map userMetadata, ActionListener listener) { + IndexCommit snapshotIndexCommit, boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, + Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { } @Override diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index d75b47879c42d..226513c134d1b 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -295,8 +295,8 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, - Map userMetadata, ActionListener listener) { + IndexCommit snapshotIndexCommit, boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, + Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 2cd00bc3301f0..0830f9a4ac845 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -120,8 +120,8 @@ private static MetaData metadataToSnapshot(Collection indices, MetaData @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, - Map userMetadata, ActionListener listener) { + IndexCommit snapshotIndexCommit, boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, + Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { if (mapperService.documentMapper() != null // if there is no mapping this is null && mapperService.documentMapper().sourceMapper().isComplete() == false) { listener.onFailure( @@ -160,8 +160,8 @@ protected void closeInternal() { Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name())); toClose.add(reader); IndexCommit indexCommit = reader.getIndexCommit(); - super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, snapshotStatus, repositoryMetaVersion, - userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); + super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, isSafeCommit, snapshotStatus, + repositoryMetaVersion, userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); } catch (IOException e) { try { IOUtils.close(toClose); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java index 62b1a43e4ef95..526ec4965e0f5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java @@ -102,7 +102,7 @@ public void testSourceIncomplete() throws IOException { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing("-1"); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); + snapshotRef.getIndexCommit(), randomBoolean(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, future::actionGet); assertEquals( "Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source", @@ -128,7 +128,7 @@ public void testIncrementalSnapshot() throws IOException { SnapshotId snapshotId = new SnapshotId("test", "test"); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); + snapshotRef.getIndexCommit(), randomBoolean(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); shardGeneration = future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount()); @@ -144,7 +144,7 @@ public void testIncrementalSnapshot() throws IOException { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); + snapshotRef.getIndexCommit(), randomBoolean(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); shardGeneration = future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); // we processed the segments_N file plus _1.si, _1.fnm, _1.fdx, _1.fdt, _1.fdm @@ -160,7 +160,7 @@ public void testIncrementalSnapshot() throws IOException { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); + snapshotRef.getIndexCommit(), randomBoolean(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); // we processed the segments_N file plus _1_1.liv @@ -208,7 +208,7 @@ public void testRestoreMinmal() throws IOException { final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> { repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, snapshotRef.getIndexCommit(), - indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future); + randomBoolean(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future); future.actionGet(); final PlainActionFuture finFuture = PlainActionFuture.newFuture(); repository.finalizeSnapshot(snapshotId, From 91048f4cef828db33c5bb4cb254d549857e3b3b1 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 21 Feb 2020 09:14:45 +0100 Subject: [PATCH 34/47] bck --- .../org/elasticsearch/repositories/Repository.java | 5 ++--- .../repositories/blobstore/BlobStoreRepository.java | 11 ++++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index d99cb889f1781..a8789b9cb26d9 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -199,15 +199,14 @@ void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, * @param snapshotId snapshot id * @param indexId id for the index being snapshotted * @param snapshotIndexCommit commit point - * @param isSafeCommit true if the passed index commit was taken when the shards local checkpoint was equal to its - * global checkpoint + * @param globalCheckpoint the current global checkpoint of the shard * @param snapshotStatus snapshot status * @param repositoryMetaVersion version of the updated repository metadata to write * @param userMetadata user metadata of the snapshot found in {@link SnapshotsInProgress.Entry#userMetadata()} * @param listener listener invoked on completion */ void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, - boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, + long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener); /** diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 99e67808a97da..d957b1b9fd13c 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1544,7 +1544,7 @@ private void writeAtomic(final String blobName, final BytesReference bytesRef, b @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, + IndexCommit snapshotIndexCommit, long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { final ShardId shardId = store.shardId(); final long startTime = threadPool.absoluteTimeInMillis(); @@ -1574,7 +1574,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s final Map userCommitData = snapshotIndexCommit.getUserData(); // We only check the sequence number to see if the shard has changed if we know that the commit is safe, // otherwise we short-circuit things here by not reading the sequence number from the commit - final String seqNumString = isSafeCommit ? userCommitData.get(SequenceNumbers.MAX_SEQ_NO) : null; + final String seqNumString = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); final long sequenceNum; final String historyUUID; if (seqNumString == null) { @@ -1755,9 +1755,10 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } @Nullable - private static List findMatchingShardSnapshot(long sequenceNum, String historyUUID, - BlobStoreIndexShardSnapshots snapshots) { - if (sequenceNum == SequenceNumbers.UNASSIGNED_SEQ_NO) { + private static List findMatchingShardSnapshot(long globalCheckpoint, long sequenceNum, + String historyUUID, + BlobStoreIndexShardSnapshots snapshots) { + if (sequenceNum == SequenceNumbers.UNASSIGNED_SEQ_NO || globalCheckpoint != sequenceNum) { return null; } for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) { From d0b19acb155a464beaebc49512db814bb327d68d Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 21 Feb 2020 10:13:59 +0100 Subject: [PATCH 35/47] pass down global checkpoint --- .../repositories/FilterRepository.java | 4 ++-- .../blobstore/BlobStoreRepository.java | 2 +- .../snapshots/SnapshotShardsService.java | 16 +--------------- .../repositories/RepositoriesServiceTests.java | 2 +- .../repositories/fs/FsRepositoryTests.java | 9 +++++---- .../RepositoryFilterUserMetadataIT.java | 6 +++--- .../index/shard/IndexShardTestCase.java | 3 ++- .../index/shard/RestoreOnlyRepository.java | 2 +- .../xpack/ccr/repository/CcrRepository.java | 2 +- .../snapshots/SourceOnlySnapshotRepository.java | 4 ++-- .../snapshots/SourceOnlySnapshotShardTests.java | 14 +++++++++----- 11 files changed, 28 insertions(+), 36 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index fe0c1533a2721..ec1cf31faf206 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -121,9 +121,9 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, + IndexCommit snapshotIndexCommit, long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { - in.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, isSafeCommit, snapshotStatus, + in.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, globalCheckpoint, snapshotStatus, repositoryMetaVersion, userMetadata, listener); } @Override diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index d957b1b9fd13c..7c45777fa74b0 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1586,7 +1586,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository final List filesFromSegmentInfos = - findMatchingShardSnapshot(sequenceNum, historyUUID, snapshots); + findMatchingShardSnapshot(globalCheckpoint, sequenceNum, historyUUID, snapshots); final List indexCommitPointFiles; int indexIncrementalFileCount = 0; diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 02031e5fd6300..529be4addb23e 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -56,7 +56,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; @@ -344,7 +343,7 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde snapshotRef = indexShard.acquireLastIndexCommit(true); final IndexCommit indexCommit = snapshotRef.getIndexCommit(); repository.snapshotShard(indexShard.store(), indexShard.mapperService(), snapshot.getSnapshotId(), indexId, - indexCommit, isSafeIndexCommit(indexShard, indexCommit), snapshotStatus, version, userMetadata, + indexCommit, indexShard.getLastSyncedGlobalCheckpoint(), snapshotStatus, version, userMetadata, ActionListener.runBefore(listener, snapshotRef::close)); } catch (Exception e) { IOUtils.close(snapshotRef); @@ -355,19 +354,6 @@ indexCommit, isSafeIndexCommit(indexShard, indexCommit), snapshotStatus, version } } - /** - * Checks whether the index commit is from a sequence number that is equal to the shards global and local checkpoint. - * - * @param indexShard shard - * @param indexCommit index commit - * @return true if max sequence number in the index commit is equal to the shard's global and local checkpoint - */ - private static boolean isSafeIndexCommit(IndexShard indexShard, IndexCommit indexCommit) throws IOException { - final long localCheckPoint = indexShard.getLocalCheckpoint(); - return localCheckPoint == indexShard.getLastKnownGlobalCheckpoint() - && localCheckPoint == Long.parseLong(indexCommit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)); - } - /** * Checks if any shards were processed that the new master doesn't know about */ diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index cb430b564e233..04159fe6080e9 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -201,7 +201,7 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, + IndexCommit snapshotIndexCommit, long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { } diff --git a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java index c2e31fc32ca99..97db9c257fc51 100644 --- a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java @@ -49,6 +49,7 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.store.Store; @@ -105,8 +106,8 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException { final PlainActionFuture future1 = PlainActionFuture.newFuture(); runGeneric(threadPool, () -> { IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(null); - repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, randomBoolean(), snapshotStatus, - Version.CURRENT, Collections.emptyMap(), future1); + repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, SequenceNumbers.UNASSIGNED_SEQ_NO, + snapshotStatus, Version.CURRENT, Collections.emptyMap(), future1); future1.actionGet(); IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy(); assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount()); @@ -134,8 +135,8 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException { final PlainActionFuture future2 = PlainActionFuture.newFuture(); runGeneric(threadPool, () -> { IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); - repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit, randomBoolean(), snapshotStatus, - Version.CURRENT, Collections.emptyMap(), future2); + repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit, + SequenceNumbers.UNASSIGNED_SEQ_NO, snapshotStatus, Version.CURRENT, Collections.emptyMap(), future2); future2.actionGet(); IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy(); assertEquals(2, copy.getIncrementalFileCount()); diff --git a/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java b/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java index 7902c39325c2a..9f59548163974 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java @@ -95,12 +95,12 @@ public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenera @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, boolean isSafeCommit, + IndexCommit snapshotIndexCommit, long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { assertThat(userMetadata, is(Collections.singletonMap(MOCK_FILTERED_META, initialMetaValue))); - super.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, isSafeCommit, snapshotStatus, - repositoryMetaVersion, userMetadata, listener); + super.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, globalCheckpoint, + snapshotStatus, repositoryMetaVersion, userMetadata, listener); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index 635a5e7d1cb37..c2db6072789ef 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -838,7 +838,8 @@ protected String snapshotShard(final IndexShard shard, final String shardGen; try (Engine.IndexCommitRef indexCommitRef = shard.acquireLastIndexCommit(true)) { repository.snapshotShard(shard.store(), shard.mapperService(), snapshot.getSnapshotId(), indexId, - indexCommitRef.getIndexCommit(), randomBoolean(), snapshotStatus, Version.CURRENT, Collections.emptyMap(), future); + indexCommitRef.getIndexCommit(), shard.getLastSyncedGlobalCheckpoint(), snapshotStatus, Version.CURRENT, + Collections.emptyMap(), future); shardGen = future.actionGet(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java index f800580d2824b..71c50852003a8 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java @@ -133,7 +133,7 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, + IndexCommit snapshotIndexCommit, long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 226513c134d1b..6c7bd2556f701 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -295,7 +295,7 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, + IndexCommit snapshotIndexCommit, long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 0830f9a4ac845..19622a730eab0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -120,7 +120,7 @@ private static MetaData metadataToSnapshot(Collection indices, MetaData @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, boolean isSafeCommit, IndexShardSnapshotStatus snapshotStatus, + IndexCommit snapshotIndexCommit, long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { if (mapperService.documentMapper() != null // if there is no mapping this is null && mapperService.documentMapper().sourceMapper().isComplete() == false) { @@ -160,7 +160,7 @@ protected void closeInternal() { Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name())); toClose.add(reader); IndexCommit indexCommit = reader.getIndexCommit(); - super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, isSafeCommit, snapshotStatus, + super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, globalCheckpoint, snapshotStatus, repositoryMetaVersion, userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); } catch (IOException e) { try { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java index 526ec4965e0f5..eece78867e276 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java @@ -102,7 +102,8 @@ public void testSourceIncomplete() throws IOException { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing("-1"); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), randomBoolean(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); + snapshotRef.getIndexCommit(), shard.getLastSyncedGlobalCheckpoint(), indexShardSnapshotStatus, Version.CURRENT, + Collections.emptyMap(), future)); IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, future::actionGet); assertEquals( "Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source", @@ -128,7 +129,8 @@ public void testIncrementalSnapshot() throws IOException { SnapshotId snapshotId = new SnapshotId("test", "test"); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), randomBoolean(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); + snapshotRef.getIndexCommit(), shard.getLastSyncedGlobalCheckpoint(), indexShardSnapshotStatus, Version.CURRENT, + Collections.emptyMap(), future)); shardGeneration = future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount()); @@ -144,7 +146,8 @@ public void testIncrementalSnapshot() throws IOException { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), randomBoolean(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); + snapshotRef.getIndexCommit(), shard.getLastSyncedGlobalCheckpoint(), indexShardSnapshotStatus, Version.CURRENT, + Collections.emptyMap(), future)); shardGeneration = future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); // we processed the segments_N file plus _1.si, _1.fnm, _1.fdx, _1.fdt, _1.fdm @@ -160,7 +163,8 @@ public void testIncrementalSnapshot() throws IOException { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), randomBoolean(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); + snapshotRef.getIndexCommit(), shard.getLastSyncedGlobalCheckpoint(), indexShardSnapshotStatus, Version.CURRENT, + Collections.emptyMap(), future)); future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); // we processed the segments_N file plus _1_1.liv @@ -208,7 +212,7 @@ public void testRestoreMinmal() throws IOException { final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> { repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, snapshotRef.getIndexCommit(), - randomBoolean(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future); + shard.getLastSyncedGlobalCheckpoint(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future); future.actionGet(); final PlainActionFuture finFuture = PlainActionFuture.newFuture(); repository.finalizeSnapshot(snapshotId, From 8284af2417f35b1dfc24d028e64eb73dd6fa8e35 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 21 Feb 2020 11:08:08 +0100 Subject: [PATCH 36/47] review comments --- .../BlobStoreIndexShardSnapshots.java | 8 ++++---- .../snapshots/blobstore/SnapshotFiles.java | 16 ++++++++++----- .../blobstore/BlobStoreRepository.java | 20 +++++++++---------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java index dd11ea1952f34..a504c260df21e 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java @@ -136,7 +136,7 @@ static final class Fields { static final class ParseFields { static final ParseField FILES = new ParseField("files"); - static final ParseField SEQUENCE_NUM = new ParseField("sequence_num"); + static final ParseField GLOBAL_CHECKPOINT = new ParseField("global_checkpoint"); static final ParseField HISTORY_UUID = new ParseField("history_uuid"); static final ParseField SNAPSHOTS = new ParseField("snapshots"); } @@ -210,8 +210,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.value(fileInfo.name()); } builder.endArray(); - if (snapshot.sequenceNo() > SequenceNumbers.UNASSIGNED_SEQ_NO) { - builder.field(ParseFields.SEQUENCE_NUM.getPreferredName(), snapshot.sequenceNo()); + if (snapshot.globalCheckpoint() > SequenceNumbers.UNASSIGNED_SEQ_NO) { + builder.field(ParseFields.GLOBAL_CHECKPOINT.getPreferredName(), snapshot.globalCheckpoint()); builder.field(ParseFields.HISTORY_UUID.getPreferredName(), snapshot.historyUUID()); } builder.endObject(); @@ -269,7 +269,7 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t } else if (ParseFields.HISTORY_UUID.match(currentFieldName, parser.getDeprecationHandler())) { parser.nextToken(); historyUUIDs.put(snapshot, parser.text()); - } else if (ParseFields.SEQUENCE_NUM.match(currentFieldName, parser.getDeprecationHandler())) { + } else if (ParseFields.GLOBAL_CHECKPOINT.match(currentFieldName, parser.getDeprecationHandler())) { parser.nextToken(); sequenceNumbers.put(snapshot, parser.longValue()); } diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java index 2c8aba2ac050f..f2b06c210d0f9 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java @@ -33,7 +33,7 @@ public class SnapshotFiles { private final List indexFiles; - private final long sequenceNo; + private final long globalCheckpoint; private final String historyUUID; @@ -52,17 +52,23 @@ public String snapshot() { * @param snapshot snapshot name * @param indexFiles index files */ - public SnapshotFiles(String snapshot, List indexFiles, long sequenceNo, String historyUUID) { + public SnapshotFiles(String snapshot, List indexFiles, long globalCheckpoint, String historyUUID) { this.snapshot = snapshot; this.indexFiles = indexFiles; - this.sequenceNo = sequenceNo; + this.globalCheckpoint = globalCheckpoint; this.historyUUID = historyUUID; } - public long sequenceNo() { - return sequenceNo; + /** + * Returns the shard's global checkpoint at the time the snapshot was taken + */ + public long globalCheckpoint() { + return globalCheckpoint; } + /** + * Returns the shard's history uuid at the time the snapshot was taken. + */ public String historyUUID() { return historyUUID; } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 7c45777fa74b0..7c88454e17c14 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1574,19 +1574,19 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s final Map userCommitData = snapshotIndexCommit.getUserData(); // We only check the sequence number to see if the shard has changed if we know that the commit is safe, // otherwise we short-circuit things here by not reading the sequence number from the commit - final String seqNumString = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); - final long sequenceNum; + final String localCheckpointString = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); + final long localCheckpoint; final String historyUUID; - if (seqNumString == null) { - sequenceNum = SequenceNumbers.UNASSIGNED_SEQ_NO; + if (localCheckpointString == null) { + localCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO; historyUUID = ""; } else { - sequenceNum = Long.parseLong(userCommitData.get(SequenceNumbers.MAX_SEQ_NO)); + localCheckpoint = Long.parseLong(userCommitData.get(SequenceNumbers.MAX_SEQ_NO)); historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); } // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository final List filesFromSegmentInfos = - findMatchingShardSnapshot(globalCheckpoint, sequenceNum, historyUUID, snapshots); + findMatchingShardSnapshot(globalCheckpoint, localCheckpoint, historyUUID, snapshots); final List indexCommitPointFiles; int indexIncrementalFileCount = 0; @@ -1688,7 +1688,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones List newSnapshotsList = new ArrayList<>(); - newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), sequenceNum, historyUUID)); + newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), globalCheckpoint, historyUUID)); for (SnapshotFiles point : snapshots) { newSnapshotsList.add(point); } @@ -1755,14 +1755,14 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } @Nullable - private static List findMatchingShardSnapshot(long globalCheckpoint, long sequenceNum, + private static List findMatchingShardSnapshot(long globalCheckpoint, long localCheckpoint, String historyUUID, BlobStoreIndexShardSnapshots snapshots) { - if (sequenceNum == SequenceNumbers.UNASSIGNED_SEQ_NO || globalCheckpoint != sequenceNum) { + if (localCheckpoint == SequenceNumbers.UNASSIGNED_SEQ_NO || globalCheckpoint != localCheckpoint) { return null; } for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) { - if (snapshotFileSet.historyUUID().equals(historyUUID) && snapshotFileSet.sequenceNo() == sequenceNum) { + if (snapshotFileSet.historyUUID().equals(historyUUID) && snapshotFileSet.globalCheckpoint() == localCheckpoint) { return snapshotFileSet.indexFiles(); } } From db5859777189ae4411028dad019fbb25212822c0 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 21 Feb 2020 11:09:06 +0100 Subject: [PATCH 37/47] renaming --- .../blobstore/BlobStoreRepository.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 7c88454e17c14..712f02b0d70f3 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1574,19 +1574,19 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s final Map userCommitData = snapshotIndexCommit.getUserData(); // We only check the sequence number to see if the shard has changed if we know that the commit is safe, // otherwise we short-circuit things here by not reading the sequence number from the commit - final String localCheckpointString = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); - final long localCheckpoint; + final String maxSequenceNumString = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); + final long maxSequenceNum; final String historyUUID; - if (localCheckpointString == null) { - localCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO; + if (maxSequenceNumString == null) { + maxSequenceNum = SequenceNumbers.UNASSIGNED_SEQ_NO; historyUUID = ""; } else { - localCheckpoint = Long.parseLong(userCommitData.get(SequenceNumbers.MAX_SEQ_NO)); + maxSequenceNum = Long.parseLong(userCommitData.get(SequenceNumbers.MAX_SEQ_NO)); historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); } // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository final List filesFromSegmentInfos = - findMatchingShardSnapshot(globalCheckpoint, localCheckpoint, historyUUID, snapshots); + findMatchingShardSnapshot(globalCheckpoint, maxSequenceNum, historyUUID, snapshots); final List indexCommitPointFiles; int indexIncrementalFileCount = 0; @@ -1755,14 +1755,14 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } @Nullable - private static List findMatchingShardSnapshot(long globalCheckpoint, long localCheckpoint, + private static List findMatchingShardSnapshot(long globalCheckpoint, long maxSequenceNum, String historyUUID, BlobStoreIndexShardSnapshots snapshots) { - if (localCheckpoint == SequenceNumbers.UNASSIGNED_SEQ_NO || globalCheckpoint != localCheckpoint) { + if (maxSequenceNum == SequenceNumbers.UNASSIGNED_SEQ_NO || globalCheckpoint != maxSequenceNum) { return null; } for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) { - if (snapshotFileSet.historyUUID().equals(historyUUID) && snapshotFileSet.globalCheckpoint() == localCheckpoint) { + if (snapshotFileSet.historyUUID().equals(historyUUID) && snapshotFileSet.globalCheckpoint() == maxSequenceNum) { return snapshotFileSet.indexFiles(); } } From 624b1fbd846a85770be9cd1c68cb337a91d87db1 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 21 Feb 2020 12:52:36 +0100 Subject: [PATCH 38/47] shorter --- .../org/elasticsearch/snapshots/SnapshotShardsService.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 529be4addb23e..2fdb6eaab4df7 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -23,7 +23,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.lucene.index.IndexCommit; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; @@ -341,9 +340,8 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde try { // we flush first to make sure we get the latest writes snapshotted snapshotRef = indexShard.acquireLastIndexCommit(true); - final IndexCommit indexCommit = snapshotRef.getIndexCommit(); repository.snapshotShard(indexShard.store(), indexShard.mapperService(), snapshot.getSnapshotId(), indexId, - indexCommit, indexShard.getLastSyncedGlobalCheckpoint(), snapshotStatus, version, userMetadata, + snapshotRef.getIndexCommit(), indexShard.getLastSyncedGlobalCheckpoint(), snapshotStatus, version, userMetadata, ActionListener.runBefore(listener, snapshotRef::close)); } catch (Exception e) { IOUtils.close(snapshotRef); From d90216a96f13e84426d734cae3f98d1d3047abb6 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 21 Feb 2020 17:30:00 +0100 Subject: [PATCH 39/47] CR comments --- .../blobstore/BlobStoreIndexShardSnapshots.java | 6 +++--- .../blobstore/BlobStoreRepository.java | 16 ++++++---------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java index a504c260df21e..8590e598c92fc 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java @@ -227,7 +227,7 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t } Map> snapshotsMap = new HashMap<>(); Map historyUUIDs = new HashMap<>(); - Map sequenceNumbers = new HashMap<>(); + Map globalCheckpoints = new HashMap<>(); Map files = new HashMap<>(); if (token == XContentParser.Token.START_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { @@ -271,7 +271,7 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t historyUUIDs.put(snapshot, parser.text()); } else if (ParseFields.GLOBAL_CHECKPOINT.match(currentFieldName, parser.getDeprecationHandler())) { parser.nextToken(); - sequenceNumbers.put(snapshot, parser.longValue()); + globalCheckpoints.put(snapshot, parser.longValue()); } } } @@ -291,7 +291,7 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t fileInfosBuilder.add(fileInfo); } snapshots.add(new SnapshotFiles(entry.getKey(), Collections.unmodifiableList(fileInfosBuilder), - sequenceNumbers.getOrDefault(entry.getKey(), SequenceNumbers.UNASSIGNED_SEQ_NO), + globalCheckpoints.getOrDefault(entry.getKey(), SequenceNumbers.UNASSIGNED_SEQ_NO), historyUUIDs.getOrDefault(entry.getKey(), ""))); } return new BlobStoreIndexShardSnapshots(files, Collections.unmodifiableList(snapshots)); diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 0c6b25deb9e12..8685b29b3f811 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1566,19 +1566,15 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s final Map userCommitData = snapshotIndexCommit.getUserData(); // We only check the sequence number to see if the shard has changed if we know that the commit is safe, // otherwise we short-circuit things here by not reading the sequence number from the commit - final String maxSequenceNumString = userCommitData.get(SequenceNumbers.MAX_SEQ_NO); - final long maxSequenceNum; + final SequenceNumbers.CommitInfo seqNumInfo = + SequenceNumbers.loadSeqNoInfoFromLuceneCommit(snapshotIndexCommit.getUserData().entrySet()); + final long maxSeqNo; final String historyUUID; - if (maxSequenceNumString == null) { - maxSequenceNum = SequenceNumbers.UNASSIGNED_SEQ_NO; - historyUUID = ""; - } else { - maxSequenceNum = Long.parseLong(userCommitData.get(SequenceNumbers.MAX_SEQ_NO)); - historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); - } + maxSeqNo = seqNumInfo.maxSeqNo; + historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository final List filesFromSegmentInfos = - findMatchingShardSnapshot(globalCheckpoint, maxSequenceNum, historyUUID, snapshots); + findMatchingShardSnapshot(globalCheckpoint, maxSeqNo, historyUUID, snapshots); final List indexCommitPointFiles; int indexIncrementalFileCount = 0; From c17c7c6e29dc44e197c0a6ddf46eb6ed48b088bf Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 22 Mar 2020 12:46:50 +0100 Subject: [PATCH 40/47] CR: use single identifier and pass it to repos --- .../BlobStoreIndexShardSnapshots.java | 17 +++----- .../snapshots/blobstore/SnapshotFiles.java | 31 ++++++------- .../repositories/FilterRepository.java | 4 +- .../repositories/Repository.java | 7 ++- .../blobstore/BlobStoreRepository.java | 43 ++++++------------- .../snapshots/SnapshotShardsService.java | 18 +++++++- .../RepositoriesServiceTests.java | 2 +- .../repositories/fs/FsRepositoryTests.java | 5 +-- .../RepositoryFilterUserMetadataIT.java | 4 +- .../index/shard/IndexShardTestCase.java | 2 +- .../index/shard/RestoreOnlyRepository.java | 2 +- .../xpack/ccr/repository/CcrRepository.java | 5 +-- .../SourceOnlySnapshotRepository.java | 4 +- .../SourceOnlySnapshotShardTests.java | 10 ++--- 14 files changed, 70 insertions(+), 84 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java index 8590e598c92fc..c34a449ad7f79 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import java.io.IOException; @@ -136,8 +135,7 @@ static final class Fields { static final class ParseFields { static final ParseField FILES = new ParseField("files"); - static final ParseField GLOBAL_CHECKPOINT = new ParseField("global_checkpoint"); - static final ParseField HISTORY_UUID = new ParseField("history_uuid"); + static final ParseField SHARD_STATE_ID = new ParseField("shard_state_id"); static final ParseField SNAPSHOTS = new ParseField("snapshots"); } @@ -210,9 +208,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.value(fileInfo.name()); } builder.endArray(); - if (snapshot.globalCheckpoint() > SequenceNumbers.UNASSIGNED_SEQ_NO) { - builder.field(ParseFields.GLOBAL_CHECKPOINT.getPreferredName(), snapshot.globalCheckpoint()); - builder.field(ParseFields.HISTORY_UUID.getPreferredName(), snapshot.historyUUID()); + if (snapshot.shardStateIdentifier() != null) { + builder.field(ParseFields.SHARD_STATE_ID.getPreferredName(), snapshot.shardStateIdentifier()); } builder.endObject(); } @@ -266,12 +263,9 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t fileNames.add(parser.text()); } snapshotsMap.put(snapshot, fileNames); - } else if (ParseFields.HISTORY_UUID.match(currentFieldName, parser.getDeprecationHandler())) { + } else if (ParseFields.SHARD_STATE_ID.match(currentFieldName, parser.getDeprecationHandler())) { parser.nextToken(); historyUUIDs.put(snapshot, parser.text()); - } else if (ParseFields.GLOBAL_CHECKPOINT.match(currentFieldName, parser.getDeprecationHandler())) { - parser.nextToken(); - globalCheckpoints.put(snapshot, parser.longValue()); } } } @@ -291,8 +285,7 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t fileInfosBuilder.add(fileInfo); } snapshots.add(new SnapshotFiles(entry.getKey(), Collections.unmodifiableList(fileInfosBuilder), - globalCheckpoints.getOrDefault(entry.getKey(), SequenceNumbers.UNASSIGNED_SEQ_NO), - historyUUIDs.getOrDefault(entry.getKey(), ""))); + historyUUIDs.get(entry.getKey()))); } return new BlobStoreIndexShardSnapshots(files, Collections.unmodifiableList(snapshots)); } diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java index f2b06c210d0f9..039ea9405a158 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index.snapshots.blobstore; +import org.elasticsearch.common.Nullable; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import java.util.HashMap; @@ -33,9 +34,8 @@ public class SnapshotFiles { private final List indexFiles; - private final long globalCheckpoint; - - private final String historyUUID; + @Nullable + private final String shardStateIdentifier; private Map physicalFiles = null; @@ -49,28 +49,23 @@ public String snapshot() { } /** - * @param snapshot snapshot name - * @param indexFiles index files + * @param snapshot snapshot name + * @param indexFiles index files + * @param shardStateIdentifier unique identifier for the state of the shard that this snapshot was taken from */ - public SnapshotFiles(String snapshot, List indexFiles, long globalCheckpoint, String historyUUID) { + public SnapshotFiles(String snapshot, List indexFiles, @Nullable String shardStateIdentifier) { this.snapshot = snapshot; this.indexFiles = indexFiles; - this.globalCheckpoint = globalCheckpoint; - this.historyUUID = historyUUID; - } - - /** - * Returns the shard's global checkpoint at the time the snapshot was taken - */ - public long globalCheckpoint() { - return globalCheckpoint; + this.shardStateIdentifier = shardStateIdentifier; } /** - * Returns the shard's history uuid at the time the snapshot was taken. + * Returns an identifier for the shard state that can be used to check whether a shard has changed between + * snapshots or not. */ - public String historyUUID() { - return historyUUID; + @Nullable + public String shardStateIdentifier() { + return shardStateIdentifier; } /** diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index ec1cf31faf206..ec708976f4e75 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -121,9 +121,9 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, + IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { - in.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, globalCheckpoint, snapshotStatus, + in.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, shardStateIdentifier, snapshotStatus, repositoryMetaVersion, userMetadata, listener); } @Override diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index a8789b9cb26d9..c2e93c172eb94 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.ShardId; @@ -199,14 +200,16 @@ void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, * @param snapshotId snapshot id * @param indexId id for the index being snapshotted * @param snapshotIndexCommit commit point - * @param globalCheckpoint the current global checkpoint of the shard + * @param shardStateIdentifier a unique identifier of the state of the shard that is stored with the shard's snapshot and used + * to detect if the shard has changed between snapshots. If {@code null} is passed as the identifier + * snapshotting will be done by inspecting the physical files referenced by {@code snapshotIndexCommit} * @param snapshotStatus snapshot status * @param repositoryMetaVersion version of the updated repository metadata to write * @param userMetadata user metadata of the snapshot found in {@link SnapshotsInProgress.Entry#userMetadata()} * @param listener listener invoked on completion */ void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, - long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, + @Nullable String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener); /** diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 8685b29b3f811..80f24625daabe 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -84,9 +84,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; -import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException; import org.elasticsearch.index.snapshots.IndexShardSnapshotException; @@ -127,6 +125,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -1536,7 +1535,7 @@ private void writeAtomic(final String blobName, final BytesReference bytesRef, b @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, + IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { final ShardId shardId = store.shardId(); final long startTime = threadPool.absoluteTimeInMillis(); @@ -1563,18 +1562,15 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s throw new IndexShardSnapshotFailedException(shardId, "Duplicate snapshot name [" + snapshotId.getName() + "] detected, aborting"); } - final Map userCommitData = snapshotIndexCommit.getUserData(); - // We only check the sequence number to see if the shard has changed if we know that the commit is safe, - // otherwise we short-circuit things here by not reading the sequence number from the commit - final SequenceNumbers.CommitInfo seqNumInfo = - SequenceNumbers.loadSeqNoInfoFromLuceneCommit(snapshotIndexCommit.getUserData().entrySet()); - final long maxSeqNo; - final String historyUUID; - maxSeqNo = seqNumInfo.maxSeqNo; - historyUUID = userCommitData.get(Engine.HISTORY_UUID_KEY); // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository - final List filesFromSegmentInfos = - findMatchingShardSnapshot(globalCheckpoint, maxSeqNo, historyUUID, snapshots); + final List filesFromSegmentInfos = Optional.ofNullable(shardStateIdentifier).map(id -> { + for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) { + if (id.equals(snapshotFileSet.shardStateIdentifier())) { + return snapshotFileSet.indexFiles(); + } + } + return null; + }).orElse(null); final List indexCommitPointFiles; int indexIncrementalFileCount = 0; @@ -1676,7 +1672,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones List newSnapshotsList = new ArrayList<>(); - newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), globalCheckpoint, historyUUID)); + newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), shardStateIdentifier)); for (SnapshotFiles point : snapshots) { newSnapshotsList.add(point); } @@ -1742,21 +1738,6 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s } } - @Nullable - private static List findMatchingShardSnapshot(long globalCheckpoint, long maxSequenceNum, - String historyUUID, - BlobStoreIndexShardSnapshots snapshots) { - if (maxSequenceNum == SequenceNumbers.UNASSIGNED_SEQ_NO || globalCheckpoint != maxSequenceNum) { - return null; - } - for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) { - if (snapshotFileSet.historyUUID().equals(historyUUID) && snapshotFileSet.globalCheckpoint() == maxSequenceNum) { - return snapshotFileSet.indexFiles(); - } - } - return null; - } - private static boolean assertFileContentsMatchHash(BlobStoreIndexShardSnapshot.FileInfo fileInfo, Store store) { try (IndexInput indexInput = store.openVerifyingInput(fileInfo.physicalName(), IOContext.READONCE, fileInfo.metadata())) { final byte[] tmp = new byte[Math.toIntExact(fileInfo.metadata().length())]; @@ -1779,7 +1760,7 @@ public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, Sh executor.execute(ActionRunnable.wrap(restoreListener, l -> { final BlobStoreIndexShardSnapshot snapshot = loadShardSnapshot(container, snapshotId); final SnapshotFiles snapshotFiles = - new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), SequenceNumbers.UNASSIGNED_SEQ_NO, ""); + new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), ""); new FileRestoreContext(metadata.name(), shardId, snapshotId, recoveryState) { @Override protected void restoreFiles(List filesToRecover, Store store, diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 2fdb6eaab4df7..dd1adcb00e11e 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -23,6 +23,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.index.IndexCommit; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; @@ -55,6 +56,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; @@ -340,8 +342,22 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde try { // we flush first to make sure we get the latest writes snapshotted snapshotRef = indexShard.acquireLastIndexCommit(true); + final IndexCommit snapshotIndexCommit = snapshotRef.getIndexCommit(); + final Map userCommitData = snapshotIndexCommit.getUserData(); + // We only check the sequence number to see if the shard has changed if we know that the commit is safe, + // otherwise we short-circuit things here by not reading the sequence number from the commit + final SequenceNumbers.CommitInfo seqNumInfo = + SequenceNumbers.loadSeqNoInfoFromLuceneCommit(snapshotIndexCommit.getUserData().entrySet()); + final String shardStateId; + final long maxSeqNo = seqNumInfo.maxSeqNo; + if (maxSeqNo == indexShard.getLastSyncedGlobalCheckpoint()) { + shardStateId = userCommitData.get(Engine.HISTORY_UUID_KEY) + "-" + + userCommitData.getOrDefault(Engine.FORCE_MERGE_UUID_KEY, "na") + "-" + maxSeqNo; + } else { + shardStateId = null; + } repository.snapshotShard(indexShard.store(), indexShard.mapperService(), snapshot.getSnapshotId(), indexId, - snapshotRef.getIndexCommit(), indexShard.getLastSyncedGlobalCheckpoint(), snapshotStatus, version, userMetadata, + snapshotRef.getIndexCommit(), shardStateId, snapshotStatus, version, userMetadata, ActionListener.runBefore(listener, snapshotRef::close)); } catch (Exception e) { IOUtils.close(snapshotRef); diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index 04159fe6080e9..083872ce83768 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -201,7 +201,7 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, + IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { } diff --git a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java index 97db9c257fc51..22b704389aa26 100644 --- a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java @@ -49,7 +49,6 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.store.Store; @@ -106,7 +105,7 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException { final PlainActionFuture future1 = PlainActionFuture.newFuture(); runGeneric(threadPool, () -> { IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(null); - repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, SequenceNumbers.UNASSIGNED_SEQ_NO, + repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, null, snapshotStatus, Version.CURRENT, Collections.emptyMap(), future1); future1.actionGet(); IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy(); @@ -136,7 +135,7 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException { runGeneric(threadPool, () -> { IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit, - SequenceNumbers.UNASSIGNED_SEQ_NO, snapshotStatus, Version.CURRENT, Collections.emptyMap(), future2); + null, snapshotStatus, Version.CURRENT, Collections.emptyMap(), future2); future2.actionGet(); IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy(); assertEquals(2, copy.getIncrementalFileCount()); diff --git a/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java b/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java index 9f59548163974..62eeea8b24f34 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java @@ -95,11 +95,11 @@ public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenera @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, long globalCheckpoint, + IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { assertThat(userMetadata, is(Collections.singletonMap(MOCK_FILTERED_META, initialMetaValue))); - super.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, globalCheckpoint, + super.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, shardStateIdentifier, snapshotStatus, repositoryMetaVersion, userMetadata, listener); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index c2db6072789ef..b95281530d844 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -838,7 +838,7 @@ protected String snapshotShard(final IndexShard shard, final String shardGen; try (Engine.IndexCommitRef indexCommitRef = shard.acquireLastIndexCommit(true)) { repository.snapshotShard(shard.store(), shard.mapperService(), snapshot.getSnapshotId(), indexId, - indexCommitRef.getIndexCommit(), shard.getLastSyncedGlobalCheckpoint(), snapshotStatus, Version.CURRENT, + indexCommitRef.getIndexCommit(), null, snapshotStatus, Version.CURRENT, Collections.emptyMap(), future); shardGen = future.actionGet(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java index 71c50852003a8..9d697d04dc4ed 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java @@ -133,7 +133,7 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, + IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 6c7bd2556f701..d4203e335994c 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -46,7 +46,6 @@ import org.elasticsearch.index.seqno.RetentionLeaseAlreadyExistsException; import org.elasticsearch.index.seqno.RetentionLeaseInvalidRetainingSeqNoException; import org.elasticsearch.index.seqno.RetentionLeaseNotFoundException; -import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexShardRecoveryException; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException; @@ -295,7 +294,7 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, + IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE); } @@ -488,7 +487,7 @@ void restoreFiles(Store store, ActionListener listener) { ByteSizeValue fileSize = new ByteSizeValue(fileMetaData.length()); fileInfos.add(new FileInfo(fileMetaData.name(), fileMetaData, fileSize)); } - SnapshotFiles snapshotFiles = new SnapshotFiles(LATEST, fileInfos, SequenceNumbers.UNASSIGNED_SEQ_NO, ""); + SnapshotFiles snapshotFiles = new SnapshotFiles(LATEST, fileInfos, null); restore(snapshotFiles, store, listener); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 19622a730eab0..420a85bd4c06d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -120,7 +120,7 @@ private static MetaData metadataToSnapshot(Collection indices, MetaData @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, long globalCheckpoint, IndexShardSnapshotStatus snapshotStatus, + IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map userMetadata, ActionListener listener) { if (mapperService.documentMapper() != null // if there is no mapping this is null && mapperService.documentMapper().sourceMapper().isComplete() == false) { @@ -160,7 +160,7 @@ protected void closeInternal() { Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name())); toClose.add(reader); IndexCommit indexCommit = reader.getIndexCommit(); - super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, globalCheckpoint, snapshotStatus, + super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, shardStateIdentifier, snapshotStatus, repositoryMetaVersion, userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); } catch (IOException e) { try { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java index eece78867e276..443b4630740a4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java @@ -102,7 +102,7 @@ public void testSourceIncomplete() throws IOException { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing("-1"); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), shard.getLastSyncedGlobalCheckpoint(), indexShardSnapshotStatus, Version.CURRENT, + snapshotRef.getIndexCommit(), null, indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, future::actionGet); assertEquals( @@ -129,7 +129,7 @@ public void testIncrementalSnapshot() throws IOException { SnapshotId snapshotId = new SnapshotId("test", "test"); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), shard.getLastSyncedGlobalCheckpoint(), indexShardSnapshotStatus, Version.CURRENT, + snapshotRef.getIndexCommit(), null, indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); shardGeneration = future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); @@ -146,7 +146,7 @@ public void testIncrementalSnapshot() throws IOException { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), shard.getLastSyncedGlobalCheckpoint(), indexShardSnapshotStatus, Version.CURRENT, + snapshotRef.getIndexCommit(), null, indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); shardGeneration = future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); @@ -163,7 +163,7 @@ public void testIncrementalSnapshot() throws IOException { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), shard.getLastSyncedGlobalCheckpoint(), indexShardSnapshotStatus, Version.CURRENT, + snapshotRef.getIndexCommit(), null, indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future)); future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); @@ -212,7 +212,7 @@ public void testRestoreMinmal() throws IOException { final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> { repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, snapshotRef.getIndexCommit(), - shard.getLastSyncedGlobalCheckpoint(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future); + null, indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future); future.actionGet(); final PlainActionFuture finFuture = PlainActionFuture.newFuture(); repository.finalizeSnapshot(snapshotId, From 280287667f1922c34e61277d1d313f55e7ae2822 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 22 Mar 2020 12:53:03 +0100 Subject: [PATCH 41/47] remove pointless commit --- .../org/elasticsearch/snapshots/SnapshotShardsService.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index dd1adcb00e11e..2465248cf23e1 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -344,10 +344,8 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde snapshotRef = indexShard.acquireLastIndexCommit(true); final IndexCommit snapshotIndexCommit = snapshotRef.getIndexCommit(); final Map userCommitData = snapshotIndexCommit.getUserData(); - // We only check the sequence number to see if the shard has changed if we know that the commit is safe, - // otherwise we short-circuit things here by not reading the sequence number from the commit final SequenceNumbers.CommitInfo seqNumInfo = - SequenceNumbers.loadSeqNoInfoFromLuceneCommit(snapshotIndexCommit.getUserData().entrySet()); + SequenceNumbers.loadSeqNoInfoFromLuceneCommit(userCommitData.entrySet()); final String shardStateId; final long maxSeqNo = seqNumInfo.maxSeqNo; if (maxSeqNo == indexShard.getLastSyncedGlobalCheckpoint()) { From 4c59524e4dea5ee8204c7ac3575ddceaaee07b27 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 22 Mar 2020 13:03:10 +0100 Subject: [PATCH 42/47] better docs --- .../snapshots/SnapshotShardsService.java | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 2465248cf23e1..d88afa7bfc7df 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -343,17 +343,7 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde // we flush first to make sure we get the latest writes snapshotted snapshotRef = indexShard.acquireLastIndexCommit(true); final IndexCommit snapshotIndexCommit = snapshotRef.getIndexCommit(); - final Map userCommitData = snapshotIndexCommit.getUserData(); - final SequenceNumbers.CommitInfo seqNumInfo = - SequenceNumbers.loadSeqNoInfoFromLuceneCommit(userCommitData.entrySet()); - final String shardStateId; - final long maxSeqNo = seqNumInfo.maxSeqNo; - if (maxSeqNo == indexShard.getLastSyncedGlobalCheckpoint()) { - shardStateId = userCommitData.get(Engine.HISTORY_UUID_KEY) + "-" + - userCommitData.getOrDefault(Engine.FORCE_MERGE_UUID_KEY, "na") + "-" + maxSeqNo; - } else { - shardStateId = null; - } + final String shardStateId = getShardStateId(indexShard, snapshotIndexCommit); repository.snapshotShard(indexShard.store(), indexShard.mapperService(), snapshot.getSnapshotId(), indexId, snapshotRef.getIndexCommit(), shardStateId, snapshotStatus, version, userMetadata, ActionListener.runBefore(listener, snapshotRef::close)); @@ -366,6 +356,30 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde } } + /** + * Generates an identifier that identifies the current contents of the shard that can be used to identify whether + * a shard's contents have been changed between two snapshots. + * A shard is assumed to have unchanged contents if its global- and local checkpoint are equal, its maximum + * sequence number has not changed and its history and force merge uuid have not changed. + * The method returns {@code null} if global and local checkpoint are different for a shard since no safe unique + * shard state id can be used in this case because of the possibility of a primary failover. + * + * @param indexShard Shard + * @param snapshotIndexCommit IndexCommit for shard + * @return shard state id or {@code null} if non is to be used + */ + @Nullable + private static String getShardStateId(IndexShard indexShard, IndexCommit snapshotIndexCommit) throws IOException { + final Map userCommitData = snapshotIndexCommit.getUserData(); + final SequenceNumbers.CommitInfo seqNumInfo = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(userCommitData.entrySet()); + final long maxSeqNo = seqNumInfo.maxSeqNo; + if (maxSeqNo != indexShard.getLastSyncedGlobalCheckpoint()) { + return null; + } + return userCommitData.get(Engine.HISTORY_UUID_KEY) + "-" + + userCommitData.getOrDefault(Engine.FORCE_MERGE_UUID_KEY, "na") + "-" + maxSeqNo; + } + /** * Checks if any shards were processed that the new master doesn't know about */ From 0bfb717ea73b0093289004369ba962128c464179 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 22 Mar 2020 14:46:25 +0100 Subject: [PATCH 43/47] safety --- .../java/org/elasticsearch/snapshots/SnapshotShardsService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index d88afa7bfc7df..d1c4ef3886b69 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -373,7 +373,7 @@ private static String getShardStateId(IndexShard indexShard, IndexCommit snapsho final Map userCommitData = snapshotIndexCommit.getUserData(); final SequenceNumbers.CommitInfo seqNumInfo = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(userCommitData.entrySet()); final long maxSeqNo = seqNumInfo.maxSeqNo; - if (maxSeqNo != indexShard.getLastSyncedGlobalCheckpoint()) { + if (maxSeqNo != seqNumInfo.localCheckpoint || maxSeqNo != indexShard.getLastSyncedGlobalCheckpoint()) { return null; } return userCommitData.get(Engine.HISTORY_UUID_KEY) + "-" + From f8ca09d9d584f6f0adb7f1479b3ece7d59b480d0 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 22 Mar 2020 14:55:36 +0100 Subject: [PATCH 44/47] simpler --- .../repositories/blobstore/BlobStoreRepository.java | 3 +-- .../org/elasticsearch/snapshots/SnapshotShardsService.java | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 80f24625daabe..7102bbaa943f9 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1759,8 +1759,7 @@ public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, Sh final BlobContainer container = shardContainer(indexId, snapshotShardId); executor.execute(ActionRunnable.wrap(restoreListener, l -> { final BlobStoreIndexShardSnapshot snapshot = loadShardSnapshot(container, snapshotId); - final SnapshotFiles snapshotFiles = - new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), ""); + final SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), null); new FileRestoreContext(metadata.name(), shardId, snapshotId, recoveryState) { @Override protected void restoreFiles(List filesToRecover, Store store, diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index d1c4ef3886b69..d2ee00749863d 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -343,9 +343,8 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde // we flush first to make sure we get the latest writes snapshotted snapshotRef = indexShard.acquireLastIndexCommit(true); final IndexCommit snapshotIndexCommit = snapshotRef.getIndexCommit(); - final String shardStateId = getShardStateId(indexShard, snapshotIndexCommit); repository.snapshotShard(indexShard.store(), indexShard.mapperService(), snapshot.getSnapshotId(), indexId, - snapshotRef.getIndexCommit(), shardStateId, snapshotStatus, version, userMetadata, + snapshotRef.getIndexCommit(), getShardStateId(indexShard, snapshotIndexCommit), snapshotStatus, version, userMetadata, ActionListener.runBefore(listener, snapshotRef::close)); } catch (Exception e) { IOUtils.close(snapshotRef); From 2b12eb801be2006a047f758e4b012d3ba0dab76a Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 22 Mar 2020 14:58:18 +0100 Subject: [PATCH 45/47] better wording --- .../elasticsearch/snapshots/SnapshotShardsService.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index d2ee00749863d..4dd8dc8dde6d6 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -356,12 +356,13 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde } /** - * Generates an identifier that identifies the current contents of the shard that can be used to identify whether - * a shard's contents have been changed between two snapshots. + * Generates an identifier from the current state of a shard that can be used to detect whether a shard's contents + * have changed between two snapshots. * A shard is assumed to have unchanged contents if its global- and local checkpoint are equal, its maximum - * sequence number has not changed and its history and force merge uuid have not changed. + * sequence number has not changed and its history- and force-merge-uuid have not changed. * The method returns {@code null} if global and local checkpoint are different for a shard since no safe unique - * shard state id can be used in this case because of the possibility of a primary failover. + * shard state id can be used in this case because of the possibility of a primary failover leading to different + * shard content for the same sequence number on a subsequent snapshot. * * @param indexShard Shard * @param snapshotIndexCommit IndexCommit for shard From 983eb58aa5b2ba59c2dcaa4548587473e02b5f23 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 22 Mar 2020 14:59:15 +0100 Subject: [PATCH 46/47] better wording --- .../java/org/elasticsearch/snapshots/SnapshotShardsService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 4dd8dc8dde6d6..a39c2691a9db4 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -366,7 +366,7 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde * * @param indexShard Shard * @param snapshotIndexCommit IndexCommit for shard - * @return shard state id or {@code null} if non is to be used + * @return shard state id or {@code null} if none can be used */ @Nullable private static String getShardStateId(IndexShard indexShard, IndexCommit snapshotIndexCommit) throws IOException { From b508fb3f1a64f0e10f988eef78fc405368f51aa8 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 23 Mar 2020 13:10:05 +0100 Subject: [PATCH 47/47] CR: add test for force merge scenario --- .../snapshots/BlobStoreIncrementalityIT.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java index f7bc18f7bd351..897e75a2e0d9f 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java @@ -22,6 +22,8 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStats; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; +import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -132,6 +134,52 @@ public void testIncrementalBehaviorOnPrimaryFailover() throws InterruptedExcepti assertCountInIndexThenDelete(indexName + "-copy-4", countAfterDelete); } + public void testForceMergeCausesFullSnapshot() throws Exception { + internalCluster().startMasterOnlyNode(); + internalCluster().ensureAtLeastNumDataNodes(2); + final String indexName = "test-index"; + createIndex(indexName, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).build()); + ensureGreen(indexName); + + logger.info("--> adding some documents to test index and flush in between to get at least two segments"); + for (int j = 0; j < 2; j++) { + final BulkRequest bulkRequest = new BulkRequest(); + for (int i = 0; i < scaledRandomIntBetween(1, 100); ++i) { + bulkRequest.add(new IndexRequest(indexName).source("foo" + j, "bar" + i)); + } + client().bulk(bulkRequest).get(); + flushAndRefresh(indexName); + } + final IndexStats indexStats = client().admin().indices().prepareStats(indexName).get().getIndex(indexName); + assertThat(indexStats.getIndexShards().get(0).getPrimary().getSegments().getCount(), greaterThan(1L)); + + final String snapshot1 = "snap-1"; + final String repo = "test-repo"; + logger.info("--> creating repository"); + assertThat(client().admin().cluster().preparePutRepository(repo) + .setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())).execute().actionGet().isAcknowledged(), + is(true)); + + logger.info("--> creating snapshot 1"); + client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setIndices(indexName).setWaitForCompletion(true).get(); + + logger.info("--> force merging down to a single segment"); + final ForceMergeResponse forceMergeResponse = + client().admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).setFlush(true).get(); + assertThat(forceMergeResponse.getFailedShards(), is(0)); + + final String snapshot2 = "snap-2"; + logger.info("--> creating snapshot 2"); + client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setIndices(indexName).setWaitForCompletion(true).get(); + + logger.info("--> asserting that the two snapshots refer to different files in the repository"); + final SnapshotsStatusResponse response = + client().admin().cluster().prepareSnapshotStatus(repo).setSnapshots(snapshot2).get(); + final SnapshotStats secondSnapshotShardStatus = + response.getSnapshots().get(0).getIndices().get(indexName).getShards().get(0).getStats(); + assertThat(secondSnapshotShardStatus.getIncrementalFileCount(), greaterThan(0)); + } + private void assertCountInIndexThenDelete(String index, long expectedCount) throws ExecutionException, InterruptedException { logger.info("--> asserting that index [{}] contains [{}] documents", index, expectedCount); assertThat(getCountForIndex(index), is(expectedCount));