-
Notifications
You must be signed in to change notification settings - Fork 24.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Untangle Engine Constructor logic #28245
Changes from 17 commits
2826b41
922bdb4
d9c236b
e1ed1dc
5be23dc
88bd738
7816210
e13f0f2
963e86e
aba67a4
8d1fe79
4bc7f9b
d6d3f6c
0bbd50b
152aca3
f52ffd6
2c36b88
e78d93d
5b9b8cb
ac96ede
3b11801
0b942b0
8fa3106
3a81d8f
7251980
aad5774
7086f92
0e98b3c
627c7d6
5e58fb2
d5f5a0c
5a4e957
f1114bd
1bc2a64
b8ecd0e
f8ff0aa
37853c9
72b9cb0
7139116
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,117 @@ | ||
/* | ||
* Licensed to Elasticsearch under one or more contributor | ||
* license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright | ||
* ownership. Elasticsearch licenses this file to you under | ||
* the Apache License, Version 2.0 (the "License"); you may | ||
* not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, | ||
* software distributed under the License is distributed on an | ||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
* KIND, either express or implied. See the License for the | ||
* specific language governing permissions and limitations | ||
* under the License. | ||
*/ | ||
|
||
package org.elasticsearch.index.engine; | ||
|
||
import org.apache.lucene.index.IndexWriter; | ||
import org.apache.lucene.index.IndexWriterConfig; | ||
import org.apache.lucene.index.NoMergePolicy; | ||
import org.apache.lucene.store.Directory; | ||
import org.elasticsearch.common.UUIDs; | ||
import org.elasticsearch.index.seqno.SequenceNumbers; | ||
import org.elasticsearch.index.shard.ShardId; | ||
import org.elasticsearch.index.translog.Translog; | ||
|
||
import java.io.IOException; | ||
import java.nio.file.Path; | ||
import java.util.Collections; | ||
import java.util.HashMap; | ||
import java.util.Map; | ||
|
||
|
||
public final class EngineDiskUtils { | ||
|
||
private EngineDiskUtils() { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. as an alternative you can mark it as final abstract then you don't need the private ctor There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't care. I've seen requests to go both ways. I just want people to be happy. I'll go with abstract. I |
||
} | ||
|
||
public static void createEmpty(final Directory dir, final Path translogPath, final ShardId shardId) throws IOException { | ||
try (IndexWriter writer = newIndexWriter(true, dir)) { | ||
final String translogUuid = Translog.createEmptyTranslog(translogPath, SequenceNumbers.NO_OPS_PERFORMED, shardId); | ||
final Map<String, String> map = new HashMap<>(); | ||
map.put(Translog.TRANSLOG_GENERATION_KEY, "1"); | ||
map.put(Translog.TRANSLOG_UUID_KEY, translogUuid); | ||
map.put(Engine.HISTORY_UUID_KEY, UUIDs.randomBase64UUID()); | ||
map.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(SequenceNumbers.NO_OPS_PERFORMED)); | ||
map.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(SequenceNumbers.NO_OPS_PERFORMED)); | ||
map.put(InternalEngine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, "-1"); | ||
updateCommitData(writer, map); | ||
} | ||
} | ||
|
||
|
||
public static void bootstrapNewHistoryFromLuceneIndex(final Directory dir, final Path translogPath, final ShardId shardId) | ||
throws IOException { | ||
try (IndexWriter writer = newIndexWriter(false, dir)) { | ||
final Map<String, String> userData = getUserData(writer); | ||
final long maxSeqNo = Long.parseLong(userData.get(SequenceNumbers.MAX_SEQ_NO)); | ||
final String translogUuid = Translog.createEmptyTranslog(translogPath, maxSeqNo, shardId); | ||
final Map<String, String> map = new HashMap<>(); | ||
map.put(Translog.TRANSLOG_GENERATION_KEY, "1"); | ||
map.put(Translog.TRANSLOG_UUID_KEY, translogUuid); | ||
map.put(Engine.HISTORY_UUID_KEY, UUIDs.randomBase64UUID()); | ||
map.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(maxSeqNo)); | ||
updateCommitData(writer, map); | ||
} | ||
} | ||
|
||
public static void createNewTranslog(final Directory dir, final Path translogPath, long initialGlobalCheckpoint, final ShardId shardId) | ||
throws IOException { | ||
try (IndexWriter writer = newIndexWriter(false, dir)) { | ||
final String translogUuid = Translog.createEmptyTranslog(translogPath, initialGlobalCheckpoint, shardId); | ||
final Map<String, String> map = new HashMap<>(); | ||
map.put(Translog.TRANSLOG_GENERATION_KEY, "1"); | ||
map.put(Translog.TRANSLOG_UUID_KEY, translogUuid); | ||
updateCommitData(writer, map); | ||
} | ||
} | ||
|
||
|
||
public static void verifyHasHistoryUUID(final Directory dir) throws IOException { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I though this is an assertion but not. Should we give it a better name? |
||
try (IndexWriter writer = newIndexWriter(false, dir)) { | ||
final Map<String, String> userData = getUserData(writer); | ||
if (userData.containsKey(Engine.HISTORY_UUID_KEY) == false) { | ||
updateCommitData(writer, Collections.singletonMap(Engine.HISTORY_UUID_KEY, UUIDs.randomBase64UUID())); | ||
} | ||
} | ||
} | ||
|
||
private static void updateCommitData(IndexWriter writer, Map<String, String> keysToUpdate) throws IOException { | ||
final Map<String, String> userData = getUserData(writer); | ||
userData.putAll(keysToUpdate); | ||
writer.setLiveCommitData(userData.entrySet()); | ||
writer.commit(); | ||
} | ||
|
||
private static Map<String, String> getUserData(IndexWriter writer) { | ||
final Map<String, String> userData = new HashMap<>(); | ||
writer.getLiveCommitData().forEach(e -> userData.put(e.getKey(), e.getValue())); | ||
return userData; | ||
} | ||
|
||
private static IndexWriter newIndexWriter(final boolean create, final Directory dir) throws IOException { | ||
IndexWriterConfig iwc = new IndexWriterConfig(null) | ||
.setCommitOnClose(false) | ||
// we don't want merges to happen here - we call maybe merge on the engine | ||
// later once we stared it up otherwise we would need to wait for it here | ||
// we also don't specify a codec here and merges should use the engines for this index | ||
.setMergePolicy(NoMergePolicy.INSTANCE) | ||
.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND); | ||
return new IndexWriter(dir, iwc); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this needs javadocs
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
will add.