diff --git a/build.gradle b/build.gradle index f7c085bb7b..2b98b8a1d3 100644 --- a/build.gradle +++ b/build.gradle @@ -186,6 +186,7 @@ project(":samza-core_$scalaSuffix") { compile "org.apache.commons:commons-lang3:$commonsLang3Version" compile "commons-io:commons-io:$commonsIoVersion" compile "com.fasterxml.jackson.core:jackson-databind:$jacksonVersion" + compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$jacksonVersion" compile "org.eclipse.jetty:jetty-webapp:$jettyVersion" compile "org.scala-lang:scala-library:$scalaVersion" compile "org.slf4j:slf4j-api:$slf4jVersion" diff --git a/gradle/dependency-versions.gradle b/gradle/dependency-versions.gradle index 738e2d0c35..019cde3feb 100644 --- a/gradle/dependency-versions.gradle +++ b/gradle/dependency-versions.gradle @@ -48,7 +48,7 @@ yarnVersion = "2.7.1" zkClientVersion = "0.11" zookeeperVersion = "3.4.13" - failsafeVersion = "1.1.0" + failsafeVersion = "2.4.0" jlineVersion = "3.8.2" jnaVersion = "4.5.1" couchbaseClientVersion = "2.7.2" diff --git a/samza-api/src/main/java/org/apache/samza/checkpoint/Checkpoint.java b/samza-api/src/main/java/org/apache/samza/checkpoint/Checkpoint.java index 593d118724..5f9276f4e9 100644 --- a/samza-api/src/main/java/org/apache/samza/checkpoint/Checkpoint.java +++ b/samza-api/src/main/java/org/apache/samza/checkpoint/Checkpoint.java @@ -19,54 +19,27 @@ package org.apache.samza.checkpoint; +import java.util.Map; import org.apache.samza.system.SystemStreamPartition; -import java.util.Collections; -import java.util.Map; - -/** - * A checkpoint is a mapping of all the streams a job is consuming and the most recent current offset for each. - * It is used to restore a {@link org.apache.samza.task.StreamTask}, either as part of a job restart or as part - * of restarting a failed container within a running job. - */ -public class Checkpoint { - private final Map offsets; +public interface Checkpoint { /** - * Constructs a new checkpoint based off a map of Samza stream offsets. - * @param offsets Map of Samza streams to their current offset. + * Gets the version number of the Checkpoint + * @return Short indicating the version number */ - public Checkpoint(Map offsets) { - this.offsets = offsets; - } + short getVersion(); /** - * Gets a unmodifiable view of the current Samza stream offsets. - * @return A unmodifiable view of a Map of Samza streams to their recorded offsets. + * Gets a unmodifiable view of the last processed offsets for {@link SystemStreamPartition}s. + * The returned value differs based on the Checkpoint version: + *
    + *
  1. For {@link CheckpointV1}, returns the input {@link SystemStreamPartition} offsets, as well + * as the latest KafkaStateChangelogOffset for any store changelog {@link SystemStreamPartition}
    2. + *
  2. For {@link CheckpointV2} returns the input offsets only.
    3. + *
+ * + * @return a unmodifiable view of last processed offsets for {@link SystemStreamPartition}s. */ - public Map getOffsets() { - return Collections.unmodifiableMap(offsets); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof Checkpoint)) return false; - - Checkpoint that = (Checkpoint) o; - - if (offsets != null ? !offsets.equals(that.offsets) : that.offsets != null) return false; - - return true; - } - - @Override - public int hashCode() { - return offsets != null ? offsets.hashCode() : 0; - } - - @Override - public String toString() { - return "Checkpoint [offsets=" + offsets + "]"; - } + Map getOffsets(); } diff --git a/samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointId.java b/samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointId.java index 95dfd2422f..a4d4ca33bb 100644 --- a/samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointId.java +++ b/samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointId.java @@ -16,54 +16,64 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.samza.checkpoint; import java.util.Objects; import org.apache.commons.lang3.StringUtils; import org.apache.samza.annotation.InterfaceStability; + /** * Checkpoint ID has the format: [currentTimeMillis, last 6 digits of nanotime], separated by a dash. * This is to avoid conflicts, e.g when requesting frequent manual commits. * - * It is expected that persistent stores use the {@link #toString()} representation of the checkpoint id + * It is expected that persistent stores use the {@link #serialize()} representation of the checkpoint id * as the store checkpoint directory name. */ @InterfaceStability.Unstable -public class CheckpointId { +public class CheckpointId implements Comparable { public static final String SEPARATOR = "-"; private final long millis; - private final long nanos; + private final long nanoId; - public CheckpointId(long millis, long nanos) { + private CheckpointId(long millis, long nanoId) { this.millis = millis; - this.nanos = nanos; + this.nanoId = nanoId; } public static CheckpointId create() { return new CheckpointId(System.currentTimeMillis(), System.nanoTime() % 1000000); } - public static CheckpointId fromString(String checkpointId) { + public static CheckpointId deserialize(String checkpointId) { if (StringUtils.isBlank(checkpointId)) { throw new IllegalArgumentException("Invalid checkpoint id: " + checkpointId); } - String[] parts = checkpointId.split(SEPARATOR); - return new CheckpointId(Long.parseLong(parts[0]), Long.parseLong(parts[1])); + try { + String[] parts = checkpointId.split(SEPARATOR); + return new CheckpointId(Long.parseLong(parts[0]), Long.parseLong(parts[1])); + } catch (NumberFormatException ex) { + throw new IllegalArgumentException(String.format( + "Could not deserialize CheckpointId: %s", checkpointId), ex); + } } public long getMillis() { return millis; } - public long getNanos() { - return nanos; + public long getNanoId() { + return nanoId; } - @Override - public String toString() { - return String.format("%s%s%s", millis, SEPARATOR, nanos); + /** + * Serialization of {@link CheckpointId} as part of task checkpoints, in conjunction with {@link #deserialize(String)}. + * @return the String representation of this {@link CheckpointId}. + */ + public String serialize() { + return String.format("%s%s%s", getMillis(), SEPARATOR, getNanoId()); } @Override @@ -72,11 +82,22 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; CheckpointId that = (CheckpointId) o; return millis == that.millis && - nanos == that.nanos; + nanoId == that.nanoId; } @Override public int hashCode() { - return Objects.hash(millis, nanos); + return Objects.hash(millis, nanoId); + } + + @Override + public int compareTo(CheckpointId that) { + if (this.millis != that.millis) return Long.compare(this.millis, that.millis); + else return Long.compare(this.nanoId, that.nanoId); + } + + @Override + public String toString() { + return String.format("%s%s%s", millis, SEPARATOR, nanoId); } } \ No newline at end of file diff --git a/samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointV1.java b/samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointV1.java new file mode 100644 index 0000000000..3e2cce3a32 --- /dev/null +++ b/samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointV1.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.checkpoint; + +import org.apache.samza.system.SystemStreamPartition; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; + +/** + * A checkpoint is a mapping of all the streams a job is consuming and the most recent current offset for each. + * It is used to restore a {@link org.apache.samza.task.StreamTask}, either as part of a job restart or as part + * of restarting a failed container within a running job. + */ +public class CheckpointV1 implements Checkpoint { + public static final short CHECKPOINT_VERSION = 1; + + private final Map offsets; + + /** + * Constructs a new checkpoint based off a map of Samza stream offsets. + * + * @param offsets Map of Samza streams to their current offset. + */ + public CheckpointV1(Map offsets) { + this.offsets = offsets; + } + + public short getVersion() { + return CHECKPOINT_VERSION; + } + + /** + * Gets a unmodifiable view of the current Samza input stream offsets. + * + * @return a unmodifiable view of a Map of Samza streams to their recorded offsets. + */ + @Override + public Map getOffsets() { + return Collections.unmodifiableMap(offsets); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CheckpointV1 that = (CheckpointV1) o; + + return Objects.equals(offsets, that.offsets); + } + + @Override + public int hashCode() { + return Objects.hash(offsets); + } + + @Override + public String toString() { + return "CheckpointV1 [CHECKPOINT_VERSION=" + CHECKPOINT_VERSION + ", offsets=" + offsets + "]"; + } +} diff --git a/samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointV2.java b/samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointV2.java new file mode 100644 index 0000000000..ac553f44ec --- /dev/null +++ b/samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointV2.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.checkpoint; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Objects; +import org.apache.samza.system.SystemStreamPartition; + +import java.util.Map; + +/** + * A checkpoint is a mapping of all the streams a job is consuming and the most recent current offset for each. + * It is used to restore a {@link org.apache.samza.task.StreamTask}, either as part of a job restart or as part + * of restarting a failed container within a running job. + */ + +public class CheckpointV2 implements Checkpoint { + public static final short CHECKPOINT_VERSION = 2; + + private final CheckpointId checkpointId; + private final Map inputOffsets; + private final Map> stateCheckpointMarkers; + + /** + * Constructs the checkpoint with separated input and state offsets + * + * @param checkpointId {@link CheckpointId} associated with this checkpoint + * @param inputOffsets Map of Samza system stream partition to offset of the checkpoint + * @param stateCheckpoints Map of state backend factory name to map of local state store names + * to state checkpoints + */ + public CheckpointV2(CheckpointId checkpointId, + Map inputOffsets, + Map> stateCheckpoints) { + Preconditions.checkArgument(inputOffsets != null, + "inputOffsets for CheckpointV2 must not be null"); + Preconditions.checkArgument(stateCheckpoints != null, + "stateCheckpoints for CheckpointV2 must not be null"); + this.checkpointId = checkpointId; + this.inputOffsets = ImmutableMap.copyOf(inputOffsets); + this.stateCheckpointMarkers = ImmutableMap.copyOf(stateCheckpoints); + } + + public short getVersion() { + return CHECKPOINT_VERSION; + } + + /** + * Gets the checkpoint id for the checkpoint + * @return The timestamp based checkpoint identifier associated with the checkpoint + */ + public CheckpointId getCheckpointId() { + return checkpointId; + } + + /** + * Gets a unmodifiable view of the current input {@link SystemStreamPartition} offsets. + * @return An unmodifiable map of input {@link SystemStreamPartition}s to their recorded offsets. + */ + @Override + public Map getOffsets() { + return inputOffsets; + } + + /** + * Gets the state checkpoint markers for all stores for each configured state backend. + * + * Note: We don't add this method to the {@link Checkpoint} interface since it is difficult + * to implement it for {@link CheckpointV1} without changing the underlying serialization format - + * the changelog SSP offsets are serialized in the same way as input offsets, and at + * deserialization time we don't have enough information (e.g. configs) to decide whether a + * particular entry is for an input SSP or a changelog SSP. + * + * @return Map of state backend factory name to map of local state store names to state checkpoint markers + */ + public Map> getStateCheckpointMarkers() { + return stateCheckpointMarkers; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CheckpointV2 that = (CheckpointV2) o; + + return checkpointId.equals(that.checkpointId) && + Objects.equals(inputOffsets, that.inputOffsets) && + Objects.equals(stateCheckpointMarkers, that.stateCheckpointMarkers); + } + + @Override + public int hashCode() { + return Objects.hash(checkpointId, inputOffsets, stateCheckpointMarkers); + } + + @Override + public String toString() { + return "CheckpointV2 [CHECKPOINT_VERSION=" + CHECKPOINT_VERSION + ", checkpointId=" + checkpointId + + ", inputOffsets=" + inputOffsets + ", stateCheckpointMarkers=" + stateCheckpointMarkers + "]"; + } +} \ No newline at end of file diff --git a/samza-api/src/main/java/org/apache/samza/job/model/JobModel.java b/samza-api/src/main/java/org/apache/samza/job/model/JobModel.java index d1f5e724fa..4bd995b27d 100644 --- a/samza-api/src/main/java/org/apache/samza/job/model/JobModel.java +++ b/samza-api/src/main/java/org/apache/samza/job/model/JobModel.java @@ -57,6 +57,10 @@ public JobModel(Config config, Map containers) { } } + public int getMaxChangeLogStreamPartitions() { + return maxChangeLogStreamPartitions; + } + public Config getConfig() { return config; } diff --git a/samza-api/src/main/java/org/apache/samza/storage/BlobStoreAdminFactory.java b/samza-api/src/main/java/org/apache/samza/storage/BlobStoreAdminFactory.java new file mode 100644 index 0000000000..6070ef1be6 --- /dev/null +++ b/samza-api/src/main/java/org/apache/samza/storage/BlobStoreAdminFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage; + +import org.apache.samza.config.Config; +import org.apache.samza.job.model.JobModel; + + +/** + * Factory to create instance of {@link StateBackendAdmin}s that needs to be implemented for every + * state backend + */ +public interface BlobStoreAdminFactory { + /** + * Returns an instance of {@link StateBackendAdmin} + * @param config job configuration + * @param jobModel Job Model + */ + StateBackendAdmin getStateBackendAdmin(Config config, JobModel jobModel); +} diff --git a/samza-api/src/main/java/org/apache/samza/storage/KafkaChangelogRestoreParams.java b/samza-api/src/main/java/org/apache/samza/storage/KafkaChangelogRestoreParams.java new file mode 100644 index 0000000000..ed4601a5f5 --- /dev/null +++ b/samza-api/src/main/java/org/apache/samza/storage/KafkaChangelogRestoreParams.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage; + +import java.util.Map; +import org.apache.samza.serializers.Serde; +import org.apache.samza.system.SystemAdmin; +import org.apache.samza.system.SystemConsumer; +import org.apache.samza.task.MessageCollector; + +/** + * Provides the required for Kafka Changelog restore managers + */ +public class KafkaChangelogRestoreParams { + private final Map storeConsumers; + private final Map inMemoryStores; + private final Map systemAdmins; + private final Map> storageEngineFactories; + private final Map> serdes; + private final MessageCollector collector; + + public KafkaChangelogRestoreParams( + Map storeConsumers, + Map inMemoryStores, + Map systemAdmins, + Map> storageEngineFactories, + Map> serdes, + MessageCollector collector) { + this.storeConsumers = storeConsumers; + this.inMemoryStores = inMemoryStores; + this.systemAdmins = systemAdmins; + this.storageEngineFactories = storageEngineFactories; + this.serdes = serdes; + this.collector = collector; + } + + public Map getStoreConsumers() { + return storeConsumers; + } + + public Map getInMemoryStores() { + return inMemoryStores; + } + + public Map getSystemAdmins() { + return systemAdmins; + } + + public Map> getStorageEngineFactories() { + return storageEngineFactories; + } + + public Map> getSerdes() { + return serdes; + } + + public MessageCollector getCollector() { + return collector; + } +} diff --git a/samza-api/src/main/java/org/apache/samza/storage/StateBackendAdmin.java b/samza-api/src/main/java/org/apache/samza/storage/StateBackendAdmin.java new file mode 100644 index 0000000000..2cdb86f4e4 --- /dev/null +++ b/samza-api/src/main/java/org/apache/samza/storage/StateBackendAdmin.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage; + +/** + * Admin responsible for loading any resources related to state backend + */ +public interface StateBackendAdmin { + /** + * Create all the resources required per job per store state backend + */ + void createResources(); + + /** + * Validate all resources required per job per state for state backend + */ + void validateResources(); +} diff --git a/samza-api/src/main/java/org/apache/samza/storage/StateBackendFactory.java b/samza-api/src/main/java/org/apache/samza/storage/StateBackendFactory.java new file mode 100644 index 0000000000..91f3df3d04 --- /dev/null +++ b/samza-api/src/main/java/org/apache/samza/storage/StateBackendFactory.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage; + +import java.io.File; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import org.apache.samza.config.Config; +import org.apache.samza.context.ContainerContext; +import org.apache.samza.context.JobContext; +import org.apache.samza.job.model.ContainerModel; +import org.apache.samza.job.model.JobModel; +import org.apache.samza.job.model.TaskModel; +import org.apache.samza.metrics.MetricsRegistry; +import org.apache.samza.util.Clock; + + +/** + * Factory to build the Samza {@link TaskBackupManager}, {@link TaskRestoreManager} and {@link StateBackendAdmin} + * for a particular state storage backend, which are used to durably backup the Samza task state. + */ +public interface StateBackendFactory { + TaskBackupManager getBackupManager(JobContext jobContext, + ContainerModel containerModel, + TaskModel taskModel, + ExecutorService backupExecutor, + MetricsRegistry taskInstanceMetricsRegistry, + Config config, + Clock clock, + File loggedStoreBaseDir, + File nonLoggedStoreBaseDir); + + TaskRestoreManager getRestoreManager(JobContext jobContext, + ContainerContext containerContext, + TaskModel taskModel, + ExecutorService restoreExecutor, + MetricsRegistry metricsRegistry, + Set storesToRestore, + Config config, + Clock clock, + File loggedStoreBaseDir, + File nonLoggedStoreBaseDir, + KafkaChangelogRestoreParams kafkaChangelogRestoreParams); + + StateBackendAdmin getAdmin(JobModel jobModel, Config config); +} diff --git a/samza-api/src/main/java/org/apache/samza/storage/StorageEngine.java b/samza-api/src/main/java/org/apache/samza/storage/StorageEngine.java index 7b12c85eee..0d62ca4c10 100644 --- a/samza-api/src/main/java/org/apache/samza/storage/StorageEngine.java +++ b/samza-api/src/main/java/org/apache/samza/storage/StorageEngine.java @@ -24,6 +24,9 @@ import java.util.Optional; import org.apache.samza.annotation.InterfaceStability; import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.context.ContainerContext; +import org.apache.samza.context.ExternalContext; +import org.apache.samza.context.JobContext; import org.apache.samza.system.ChangelogSSPIterator; /** @@ -38,6 +41,11 @@ */ public interface StorageEngine { + /** + * Initialize the storage engine + */ + default void init(ExternalContext externalContext, JobContext jobContext, ContainerContext containerContext) { }; + /** * Restore the content of this StorageEngine from the changelog. Messages are * provided in one {@link java.util.Iterator} and not deserialized for diff --git a/samza-api/src/main/java/org/apache/samza/storage/StoreProperties.java b/samza-api/src/main/java/org/apache/samza/storage/StoreProperties.java index a398271e4c..1244adfe17 100644 --- a/samza-api/src/main/java/org/apache/samza/storage/StoreProperties.java +++ b/samza-api/src/main/java/org/apache/samza/storage/StoreProperties.java @@ -24,12 +24,15 @@ public class StoreProperties { private final boolean persistedToDisk; private final boolean loggedStore; + private final boolean durable; private StoreProperties( final boolean persistedToDisk, - final boolean loggedStore) { + final boolean loggedStore, + final boolean durable) { this.persistedToDisk = persistedToDisk; this.loggedStore = loggedStore; + this.durable = durable; } /** @@ -50,9 +53,20 @@ public boolean isLoggedStore() { return loggedStore; } + /** + * Flag to indicate whether a store is durable, that is, it's contents are available cross container restarts + * or host reallocation. + * + * @return True, if the store is durable. False by default. + */ + public boolean isDurableStore() { + return durable; + } + public static class StorePropertiesBuilder { private boolean persistedToDisk = false; private boolean loggedStore = false; + private boolean durable = false; public StorePropertiesBuilder setPersistedToDisk(boolean persistedToDisk) { this.persistedToDisk = persistedToDisk; @@ -64,8 +78,13 @@ public StorePropertiesBuilder setLoggedStore(boolean loggedStore) { return this; } + public StorePropertiesBuilder setIsDurable(boolean durable) { + this.durable = durable; + return this; + } + public StoreProperties build() { - return new StoreProperties(persistedToDisk, loggedStore); + return new StoreProperties(persistedToDisk, loggedStore, durable); } } } diff --git a/samza-api/src/main/java/org/apache/samza/storage/TaskBackupManager.java b/samza-api/src/main/java/org/apache/samza/storage/TaskBackupManager.java new file mode 100644 index 0000000000..a00d7159dc --- /dev/null +++ b/samza-api/src/main/java/org/apache/samza/storage/TaskBackupManager.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import javax.annotation.Nullable; +import org.apache.samza.checkpoint.Checkpoint; +import org.apache.samza.checkpoint.CheckpointId; + + +/** + *

+ * TaskBackupManager is the interface that must be implemented for any remote system that Samza persists its state to + * during the task commit operation. + * {@link #snapshot(CheckpointId)} will be evoked synchronous to task processing and get a snapshot of the stores + * state to be persisted for the commit. {@link #upload(CheckpointId, Map)} will then use the snapshotted state + * to persist to the underlying backup system and will be asynchronous to task processing. + *

+ * The interface will be evoked in the following way: + *
    + *
  • Snapshot will be called before Upload.
    • + *
  • persistToFilesystem will be called after Upload is completed
    • + *
  • Cleanup is only called after Upload and persistToFilesystem has successfully completed
    • + *
+ */ +public interface TaskBackupManager { + + /** + * Initializes the TaskBackupManager instance. + * + * @param checkpoint last recorded checkpoint from the CheckpointManager or null if no last checkpoint was found + */ + void init(@Nullable Checkpoint checkpoint); + + /** + * Snapshot is used to capture the current state of the stores in order to persist it to the backup manager in the + * {@link #upload(CheckpointId, Map)} (CheckpointId, Map)} phase. Performs the commit operation that is + * synchronous to processing. Returns the per store name state checkpoint markers to be used in upload. + * + * @param checkpointId {@link CheckpointId} of the current commit + * @return a map of store name to state checkpoint markers for stores managed by this state backend + */ + Map snapshot(CheckpointId checkpointId); + + /** + * Upload is used to persist the state provided by the {@link #snapshot(CheckpointId)} to the + * underlying backup system. Commit operation that is asynchronous to message processing and returns a + * {@link CompletableFuture} containing the successfully uploaded state checkpoint markers . + * + * @param checkpointId {@link CheckpointId} of the current commit + * @param stateCheckpointMarkers the map of storename to state checkpoint markers returned by + * {@link #snapshot(CheckpointId)} + * @return a {@link CompletableFuture} containing a map of store name to state checkpoint markers + * after the upload is complete + */ + CompletableFuture> upload(CheckpointId checkpointId, Map stateCheckpointMarkers); + + /** + * Cleanup any local or remote state for checkpoint information that is older than the provided checkpointId + * This operation is required to be idempotent. + * + * @param checkpointId the {@link CheckpointId} of the last successfully committed checkpoint + * @param stateCheckpointMarkers a map of store name to state checkpoint markers returned by + * {@link #upload(CheckpointId, Map)} (CheckpointId, Map)} upload} + */ + CompletableFuture cleanUp(CheckpointId checkpointId, Map stateCheckpointMarkers); + + /** + * Shutdown hook the backup manager to cleanup any allocated resources + */ + void close(); + +} \ No newline at end of file diff --git a/samza-core/src/main/java/org/apache/samza/storage/TaskRestoreManager.java b/samza-api/src/main/java/org/apache/samza/storage/TaskRestoreManager.java similarity index 86% rename from samza-core/src/main/java/org/apache/samza/storage/TaskRestoreManager.java rename to samza-api/src/main/java/org/apache/samza/storage/TaskRestoreManager.java index f60e14876b..1f5a5ce4d0 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/TaskRestoreManager.java +++ b/samza-api/src/main/java/org/apache/samza/storage/TaskRestoreManager.java @@ -19,8 +19,7 @@ package org.apache.samza.storage; -import java.util.Map; -import org.apache.samza.system.SystemStreamPartition; +import org.apache.samza.checkpoint.Checkpoint; /** @@ -29,12 +28,12 @@ public interface TaskRestoreManager { /** - * Init state resources such as file directories. + * Initialize state resources such as store directories. */ - void init(Map checkpointedChangelogSSPOffsets); + void init(Checkpoint checkpoint); /** - * Restore state from checkpoints, state snapshots and changelog. + * Restore state from checkpoints, state snapshots and changelogs. * Currently, store restoration happens on a separate thread pool within {@code ContainerStorageManager}. In case of * interrupt/shutdown signals from {@code SamzaContainer}, {@code ContainerStorageManager} may interrupt the restore * thread. @@ -48,8 +47,7 @@ public interface TaskRestoreManager { void restore() throws InterruptedException; /** - * Stop all persistent stores after restoring. + * Closes all initiated resources include storage engines */ - void stopPersistentStores(); - + void close(); } diff --git a/samza-api/src/main/java/org/apache/samza/storage/blobstore/BlobStoreManager.java b/samza-api/src/main/java/org/apache/samza/storage/blobstore/BlobStoreManager.java new file mode 100644 index 0000000000..ef3f0c6961 --- /dev/null +++ b/samza-api/src/main/java/org/apache/samza/storage/blobstore/BlobStoreManager.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore; + +import java.io.InputStream; +import java.io.OutputStream; +import java.util.concurrent.CompletionStage; +import org.apache.samza.annotation.InterfaceStability; + + +/** + * Provides interface for common blob store operations: GET, PUT and DELETE + */ +@InterfaceStability.Unstable +public interface BlobStoreManager { + /** + * Initialize underlying blob store client, if necessary. + * + */ + void init(); + + /** + * Non-blocking PUT call to remote blob store with supplied metadata + * @param inputStream InputStream to read the file + * @param metadata user supplied {@link Metadata} of the request + * @return a future containing the blob ID of the uploaded blob if the upload is successful. + */ + CompletionStage put(InputStream inputStream, Metadata metadata); + + /** + * Non-blocking GET call to remote blob store + * @param id Blob ID of the blob to get + * @param outputStream OutputStream to write the downloaded blob + * @param metadata User supplied {@link Metadata} of the request + * @return A future that completes when all the chunks are downloaded and written successfully to the OutputStream + * @throws org.apache.samza.storage.blobstore.exceptions.DeletedException returned future should complete + * exceptionally with DeletedException on failure with the blob already deleted error. + */ + CompletionStage get(String id, OutputStream outputStream, Metadata metadata); + + /** + * Non-blocking call to mark a blob for deletion in the remote blob store + * @param id Blob ID of the blob to delete + * @param metadata User supplied {@link Metadata} of the request + * @return A future that completes when the blob is successfully deleted from the blob store. + * @throws org.apache.samza.storage.blobstore.exceptions.DeletedException returned future should complete + * exceptionally with DeletedException on failure with the blob already deleted error. This exception is + * caught and ignored by the caller of the delete method during initial cleanup and SnapshotIndex read. + */ + CompletionStage delete(String id, Metadata metadata); + + /** + * Non-blocking call to remove the Time-To-Live (TTL) for a blob and make it permanent. + * @param blobId Blob ID of blob to remove TTL for. + * @param metadata User supplied {@link Metadata} of the request + * @return a future that completes when the TTL for the blob is removed. + * @throws org.apache.samza.storage.blobstore.exceptions.DeletedException returned future should complete + * exceptionally with DeletedException on failure with the blob already deleted error. + */ + CompletionStage removeTTL(String blobId, Metadata metadata); + + /** + * Cleanly close resources like blob store client + */ + void close(); +} diff --git a/samza-api/src/main/java/org/apache/samza/storage/blobstore/BlobStoreManagerFactory.java b/samza-api/src/main/java/org/apache/samza/storage/blobstore/BlobStoreManagerFactory.java new file mode 100644 index 0000000000..5a913930a3 --- /dev/null +++ b/samza-api/src/main/java/org/apache/samza/storage/blobstore/BlobStoreManagerFactory.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore; + +import java.util.concurrent.ExecutorService; +import org.apache.samza.annotation.InterfaceStability; +import org.apache.samza.config.Config; + +@InterfaceStability.Unstable +public interface BlobStoreManagerFactory { + BlobStoreManager getBackupBlobStoreManager(Config config, ExecutorService backupExecutor); + + BlobStoreManager getRestoreBlobStoreManager(Config config, ExecutorService restoreExecutor); +} diff --git a/samza-api/src/main/java/org/apache/samza/storage/blobstore/Metadata.java b/samza-api/src/main/java/org/apache/samza/storage/blobstore/Metadata.java new file mode 100644 index 0000000000..3423f0bed5 --- /dev/null +++ b/samza-api/src/main/java/org/apache/samza/storage/blobstore/Metadata.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore; + +import java.util.Optional; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + + +/** + * Metadata associated with every BlobStore request. This class is used to trace a request and to determine the + * bucket/container informationof the blob. + */ +public class Metadata { + public static final String SNAPSHOT_INDEX_PAYLOAD_PATH = "snapshot-index"; + + private final String payloadPath; + private final long payloadSize; + private final String jobName; + private final String jobId; + private final String taskName; + private final String storeName; + + public Metadata(String payloadPath, Optional payloadSize, + String jobName, String jobId, String taskName, String storeName) { + this.payloadPath = payloadPath; + // Payload size may not be known in advance for requests like getSnapshotIndex, where only blob ID is known. Set -1. + this.payloadSize = payloadSize.orElse(-1L); + this.jobName = jobName; + this.jobId = jobId; + this.taskName = taskName; + this.storeName = storeName; + } + + public String getPayloadPath() { + return payloadPath; + } + + public long getPayloadSize() { + return payloadSize; + } + + public String getJobName() { + return jobName; + } + + public String getJobId() { + return jobId; + } + + public String getTaskName() { + return taskName; + } + + public String getStoreName() { + return storeName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (!(o instanceof Metadata)) { + return false; + } + + Metadata that = (Metadata) o; + + return new EqualsBuilder().append(getPayloadPath(), that.getPayloadPath()) + .append(getPayloadSize(), that.getPayloadSize()) + .append(getJobName(), that.getJobName()) + .append(getJobId(), that.getJobId()) + .append(getTaskName(), that.getTaskName()) + .append(getStoreName(), that.getStoreName()) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37).append(getPayloadPath()) + .append(getPayloadSize()) + .append(getJobName()) + .append(getJobId()) + .append(getTaskName()) + .append(getStoreName()) + .toHashCode(); + } + + @Override + public String toString() { + return "Metadata{" + "payloadPath='" + payloadPath + '\'' + ", payloadSize='" + payloadSize + '\'' + + ", jobName='" + jobName + '\'' + ", jobId='" + jobId + '\'' + ", taskName='" + taskName + '\'' + + ", storeName='" + storeName + '\'' + '}'; + } +} diff --git a/samza-api/src/main/java/org/apache/samza/storage/blobstore/exceptions/DeletedException.java b/samza-api/src/main/java/org/apache/samza/storage/blobstore/exceptions/DeletedException.java new file mode 100644 index 0000000000..8fcd2309b0 --- /dev/null +++ b/samza-api/src/main/java/org/apache/samza/storage/blobstore/exceptions/DeletedException.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.exceptions; + +/** + * Future should complete with this exception to indicate that the exception occurred due to the request for an + * already deleted blob. This exception is caught and ignored by caller of the DELETE request during initial cleanup + * and snapshot index read. + * + */ +public class DeletedException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public DeletedException(String message, Throwable cause) { + super(message, cause); + } + + public DeletedException(String message) { + super(message); + } + + public DeletedException(Throwable cause) { + super(cause); + } + + public DeletedException() { + } +} diff --git a/samza-core/src/main/scala/org/apache/samza/storage/TaskStorageManager.scala b/samza-api/src/main/java/org/apache/samza/storage/blobstore/exceptions/RetriableException.java similarity index 63% rename from samza-core/src/main/scala/org/apache/samza/storage/TaskStorageManager.scala rename to samza-api/src/main/java/org/apache/samza/storage/blobstore/exceptions/RetriableException.java index 50d64185f9..e4ff789957 100644 --- a/samza-core/src/main/scala/org/apache/samza/storage/TaskStorageManager.scala +++ b/samza-api/src/main/java/org/apache/samza/storage/blobstore/exceptions/RetriableException.java @@ -17,21 +17,26 @@ * under the License. */ -package org.apache.samza.storage +package org.apache.samza.storage.blobstore.exceptions; -import org.apache.samza.checkpoint.CheckpointId -import org.apache.samza.system.SystemStreamPartition -trait TaskStorageManager { +public class RetriableException extends RuntimeException { - def getStore(storeName: String): Option[StorageEngine] + private static final long serialVersionUID = 1L; - def flush(): Map[SystemStreamPartition, Option[String]] + public RetriableException(String message, Throwable cause) { + super(message, cause); + } - def checkpoint(checkpointId: CheckpointId, newestChangelogOffsets: Map[SystemStreamPartition, Option[String]]): Unit + public RetriableException(String message) { + super(message); + } - def removeOldCheckpoints(checkpointId: CheckpointId): Unit + public RetriableException(Throwable cause) { + super(cause); + } - def stop(): Unit + public RetriableException() { + } +} -} \ No newline at end of file diff --git a/samza-api/src/main/java/org/apache/samza/storage/kv/KeyValueStore.java b/samza-api/src/main/java/org/apache/samza/storage/kv/KeyValueStore.java index 41faac352e..a3552f08b9 100644 --- a/samza-api/src/main/java/org/apache/samza/storage/kv/KeyValueStore.java +++ b/samza-api/src/main/java/org/apache/samza/storage/kv/KeyValueStore.java @@ -26,6 +26,9 @@ import java.util.Optional; import org.apache.samza.annotation.InterfaceStability; import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.context.ContainerContext; +import org.apache.samza.context.ExternalContext; +import org.apache.samza.context.JobContext; /** @@ -35,6 +38,16 @@ * @param the type of values maintained by this key-value store. */ public interface KeyValueStore { + + /** + * Initializes the KeyValueStore + * + * @param externalContext any external store required for initialization + * @param jobContext context of the job the KeyValueStore is in + * @param containerContext context of the KeyValueStore's container + */ + default void init(ExternalContext externalContext, JobContext jobContext, ContainerContext containerContext) { } + /** * Gets the value associated with the specified {@code key}. * diff --git a/samza-api/src/main/java/org/apache/samza/system/ChangelogSSPIterator.java b/samza-api/src/main/java/org/apache/samza/system/ChangelogSSPIterator.java index ea44b9d118..8e5bc93834 100644 --- a/samza-api/src/main/java/org/apache/samza/system/ChangelogSSPIterator.java +++ b/samza-api/src/main/java/org/apache/samza/system/ChangelogSSPIterator.java @@ -61,7 +61,7 @@ public ChangelogSSPIterator(SystemConsumer systemConsumer, SystemStreamPartition public IncomingMessageEnvelope next() { IncomingMessageEnvelope envelope = super.next(); - // if trimming changelog is enabled, then switch to trim mode if if we've consumed past the restore offset + // if trimming changelog is enabled, then switch to trim mode if we've consumed past the restore offset // (i.e., restoreOffset was null or current offset is > restoreOffset) if (this.trimEnabled && (restoreOffset == null || admin.offsetComparator(envelope.getOffset(), restoreOffset) > 0)) { mode = Mode.TRIM; diff --git a/samza-api/src/main/java/org/apache/samza/system/SystemFactory.java b/samza-api/src/main/java/org/apache/samza/system/SystemFactory.java index 08c1b49534..2841bb1446 100644 --- a/samza-api/src/main/java/org/apache/samza/system/SystemFactory.java +++ b/samza-api/src/main/java/org/apache/samza/system/SystemFactory.java @@ -73,7 +73,7 @@ default SystemProducer getProducer(String systemName, Config config, MetricsRegi * * @param systemName The name of the system to create admin for. * @param config The config to create admin with. - * @param adminLabel a string to provide info the admin instance. + * @param adminLabel a string to provide info for the admin instance. * @return A SystemAdmin */ default SystemAdmin getAdmin(String systemName, Config config, String adminLabel) { diff --git a/samza-api/src/test/java/org/apache/samza/checkpoint/TestCheckpointId.java b/samza-api/src/test/java/org/apache/samza/checkpoint/TestCheckpointId.java new file mode 100644 index 0000000000..2ac1b88244 --- /dev/null +++ b/samza-api/src/test/java/org/apache/samza/checkpoint/TestCheckpointId.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.checkpoint; + +import org.junit.Test; + +import static org.apache.samza.checkpoint.CheckpointId.*; +import static org.junit.Assert.assertEquals; + + +public class TestCheckpointId { + @Test + public void testSerializationDeserialization() { + CheckpointId checkpointId = CheckpointId.create(); + CheckpointId deserializedCheckpointId = CheckpointId.deserialize(checkpointId.serialize()); + + assertEquals(checkpointId.getMillis(), deserializedCheckpointId.getMillis()); + assertEquals(checkpointId.getNanoId(), deserializedCheckpointId.getNanoId()); + assertEquals(checkpointId, deserializedCheckpointId); + } + + @Test + public void testSerializationFormatForBackwardsCompatibility() { + CheckpointId checkpointId = CheckpointId.create(); + String serializedCheckpointId = checkpointId.serialize(); + + // WARNING: This format is written to persisted remotes stores and local files, making a change in the format + // would be backwards incompatible + String legacySerializedFormat = serializeLegacy(checkpointId); + assertEquals(checkpointId, CheckpointId.deserialize(legacySerializedFormat)); + } + + public String serializeLegacy(CheckpointId id) { + return String.format("%s%s%s", id.getMillis(), SEPARATOR, id.getNanoId()); + } +} diff --git a/samza-azure/src/main/java/org/apache/samza/checkpoint/azure/AzureCheckpointManager.java b/samza-azure/src/main/java/org/apache/samza/checkpoint/azure/AzureCheckpointManager.java index 901319634e..1d49c2c852 100644 --- a/samza-azure/src/main/java/org/apache/samza/checkpoint/azure/AzureCheckpointManager.java +++ b/samza-azure/src/main/java/org/apache/samza/checkpoint/azure/AzureCheckpointManager.java @@ -19,6 +19,7 @@ package org.apache.samza.checkpoint.azure; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.table.*; @@ -28,6 +29,7 @@ import org.apache.samza.SamzaException; import org.apache.samza.checkpoint.Checkpoint; import org.apache.samza.checkpoint.CheckpointManager; +import org.apache.samza.checkpoint.CheckpointV1; import org.apache.samza.config.AzureConfig; import org.apache.samza.container.TaskName; import org.apache.samza.serializers.JsonSerdeV2; @@ -113,13 +115,17 @@ public void register(TaskName taskName) { @Override public void writeCheckpoint(TaskName taskName, Checkpoint checkpoint) { + Preconditions.checkArgument(checkpoint instanceof CheckpointV1, + "Only CheckpointV1 could be written to Azure"); + if (!taskNames.contains(taskName)) { throw new SamzaException("writing checkpoint of unregistered task"); } TableBatchOperation batchOperation = new TableBatchOperation(); - Iterator> iterator = checkpoint.getOffsets().entrySet().iterator(); + Iterator> iterator = + checkpoint.getOffsets().entrySet().iterator(); while (iterator.hasNext()) { Map.Entry entry = iterator.next(); SystemStreamPartition ssp = entry.getKey(); @@ -205,7 +211,7 @@ public Checkpoint readLastCheckpoint(TaskName taskName) { return null; } LOG.debug("Received checkpoint state for taskName=%s", taskName); - return new Checkpoint(builder.build()); + return new CheckpointV1(builder.build()); } @Override diff --git a/samza-azure/src/test/java/org/apache/samza/checkpoint/azure/ITestAzureCheckpointManager.java b/samza-azure/src/test/java/org/apache/samza/checkpoint/azure/ITestAzureCheckpointManager.java index 7cf01c0baf..2240e65434 100644 --- a/samza-azure/src/test/java/org/apache/samza/checkpoint/azure/ITestAzureCheckpointManager.java +++ b/samza-azure/src/test/java/org/apache/samza/checkpoint/azure/ITestAzureCheckpointManager.java @@ -22,6 +22,7 @@ import org.apache.samza.Partition; import org.apache.samza.checkpoint.Checkpoint; import org.apache.samza.checkpoint.CheckpointManager; +import org.apache.samza.checkpoint.CheckpointV1; import org.apache.samza.config.AzureConfig; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; @@ -69,10 +70,10 @@ public void testStoringAndReadingCheckpointsSamePartition() { Map sspMap = new HashMap<>(); sspMap.put(ssp, "12345"); - Checkpoint cp0 = new Checkpoint(sspMap); + Checkpoint cp0 = new CheckpointV1(sspMap); sspMap.put(ssp, "54321"); - Checkpoint cp1 = new Checkpoint(sspMap); + Checkpoint cp1 = new CheckpointV1(sspMap); checkpointManager.register(taskName); @@ -96,12 +97,12 @@ public void testStoringAndReadingCheckpointsMultiPartitions() { Map sspMap = new HashMap<>(); sspMap.put(ssp, "12345"); sspMap.put(ssp1, "54321"); - Checkpoint cp1 = new Checkpoint(sspMap); + Checkpoint cp1 = new CheckpointV1(sspMap); Map sspMap2 = new HashMap<>(); sspMap2.put(ssp, "12347"); sspMap2.put(ssp1, "54323"); - Checkpoint cp2 = new Checkpoint(sspMap2); + Checkpoint cp2 = new CheckpointV1(sspMap2); checkpointManager.register(taskName); @@ -126,12 +127,12 @@ public void testStoringAndReadingCheckpointsMultiTasks() { Map sspMap = new HashMap<>(); sspMap.put(ssp, "12345"); sspMap.put(ssp1, "54321"); - Checkpoint cp1 = new Checkpoint(sspMap); + Checkpoint cp1 = new CheckpointV1(sspMap); Map sspMap2 = new HashMap<>(); sspMap2.put(ssp, "12347"); sspMap2.put(ssp1, "54323"); - Checkpoint cp2 = new Checkpoint(sspMap2); + Checkpoint cp2 = new CheckpointV1(sspMap2); checkpointManager.register(taskName); checkpointManager.register(taskName1); @@ -171,7 +172,7 @@ public void testMultipleBatchWrites() { sspMap.put(ssp, String.valueOf(i)); } - Checkpoint cp0 = new Checkpoint(sspMap); + Checkpoint cp0 = new CheckpointV1(sspMap); checkpointManager.register(taskName); checkpointManager.writeCheckpoint(taskName, cp0); Checkpoint readCp = checkpointManager.readLastCheckpoint(taskName); diff --git a/samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointedChangelogOffset.java b/samza-core/src/main/java/org/apache/samza/checkpoint/kafka/KafkaChangelogSSPOffset.java similarity index 57% rename from samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointedChangelogOffset.java rename to samza-core/src/main/java/org/apache/samza/checkpoint/kafka/KafkaChangelogSSPOffset.java index 407ce7a444..09ea4975f2 100644 --- a/samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointedChangelogOffset.java +++ b/samza-core/src/main/java/org/apache/samza/checkpoint/kafka/KafkaChangelogSSPOffset.java @@ -16,67 +16,75 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.samza.checkpoint; +package org.apache.samza.checkpoint.kafka; import java.util.Objects; import org.apache.commons.lang3.StringUtils; import org.apache.samza.annotation.InterfaceStability; +import org.apache.samza.checkpoint.CheckpointId; /** - * Checkpointed changelog offset has the format: [checkpointId, offset], separated by a colon. + * Used in {@link org.apache.samza.checkpoint.CheckpointV1} for tracking the latest offset for store changelogs at + * the time of commit. Checkpointed changelog offset has the format: [checkpointId, offset], separated by a colon. */ @InterfaceStability.Unstable -public class CheckpointedChangelogOffset { +public class KafkaChangelogSSPOffset { public static final String SEPARATOR = ":"; private final CheckpointId checkpointId; - private final String offset; + private final String changelogOffset; - public CheckpointedChangelogOffset(CheckpointId checkpointId, String offset) { + public KafkaChangelogSSPOffset(CheckpointId checkpointId, String changelogOffset) { this.checkpointId = checkpointId; - this.offset = offset; + this.changelogOffset = changelogOffset; } - public static CheckpointedChangelogOffset fromString(String message) { + public static KafkaChangelogSSPOffset fromString(String message) { if (StringUtils.isBlank(message)) { throw new IllegalArgumentException("Invalid checkpointed changelog message: " + message); } - String[] checkpointIdAndOffset = message.split(":"); + String[] checkpointIdAndOffset = message.split(SEPARATOR); if (checkpointIdAndOffset.length != 2) { throw new IllegalArgumentException("Invalid checkpointed changelog offset: " + message); } - CheckpointId checkpointId = CheckpointId.fromString(checkpointIdAndOffset[0]); + CheckpointId checkpointId = CheckpointId.deserialize(checkpointIdAndOffset[0]); String offset = null; if (!"null".equals(checkpointIdAndOffset[1])) { offset = checkpointIdAndOffset[1]; } - return new CheckpointedChangelogOffset(checkpointId, offset); + + return new KafkaChangelogSSPOffset(checkpointId, offset); } public CheckpointId getCheckpointId() { return checkpointId; } - public String getOffset() { - return offset; + public String getChangelogOffset() { + return changelogOffset; } + /** + * WARNING: Do not change the toString() representation. It is used for serde'ing the store changelog offsets + * as part of task checkpoints, in conjunction with {@link #fromString(String)}. + * @return the String representation of this {@link KafkaChangelogSSPOffset} + */ @Override public String toString() { - return String.format("%s%s%s", checkpointId, SEPARATOR, offset); + return String.format("%s%s%s", checkpointId, SEPARATOR, changelogOffset); } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - CheckpointedChangelogOffset that = (CheckpointedChangelogOffset) o; + KafkaChangelogSSPOffset that = (KafkaChangelogSSPOffset) o; return Objects.equals(checkpointId, that.checkpointId) && - Objects.equals(offset, that.offset); + Objects.equals(changelogOffset, that.changelogOffset); } @Override public int hashCode() { - return Objects.hash(checkpointId, offset); + return Objects.hash(checkpointId, changelogOffset); } } diff --git a/samza-core/src/main/java/org/apache/samza/checkpoint/kafka/KafkaStateCheckpointMarker.java b/samza-core/src/main/java/org/apache/samza/checkpoint/kafka/KafkaStateCheckpointMarker.java new file mode 100644 index 0000000000..ccd06d1df1 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/checkpoint/kafka/KafkaStateCheckpointMarker.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.checkpoint.kafka; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import org.apache.samza.SamzaException; +import org.apache.samza.annotation.InterfaceStability; +import org.apache.samza.serializers.model.SamzaObjectMapper; +import org.apache.samza.storage.KafkaChangelogStateBackendFactory; +import org.apache.samza.system.SystemStreamPartition; +import scala.Option; + + +/** + * Used as the serialization format for the state checkpoints of {@link org.apache.samza.checkpoint.CheckpointV2} + * for a store using {@link org.apache.samza.storage.KafkaTransactionalStateTaskBackupManager} or + * {@link org.apache.samza.storage.KafkaNonTransactionalStateTaskBackupManager} for tracking the latest committed + * store changelog offsets. + * + * Kafka state checkpoint marker has the format: [system, stream, partition, offset], separated by a semi-colon. + */ +@InterfaceStability.Unstable +public class KafkaStateCheckpointMarker { + public static final String KAFKA_STATE_BACKEND_FACTORY_NAME = KafkaChangelogStateBackendFactory.class.getName(); + public static final short MARKER_VERSION = 1; + private static final ObjectMapper MAPPER = SamzaObjectMapper.getObjectMapper(); + + // Required for Jackson Serde + private final short version; + private final SystemStreamPartition changelogSSP; + private final String changelogOffset; + + public KafkaStateCheckpointMarker(SystemStreamPartition changelogSSP, String changelogOffset) { + this(MARKER_VERSION, changelogSSP, changelogOffset); + } + + public KafkaStateCheckpointMarker(short version, SystemStreamPartition changelogSSP, String changelogOffset) { + this.version = version; + this.changelogSSP = changelogSSP; + this.changelogOffset = changelogOffset; + } + + public static KafkaStateCheckpointMarker deserialize(String stateCheckpointMarker) { + try { + return MAPPER.readValue(stateCheckpointMarker, KafkaStateCheckpointMarker.class); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("Could not deserialize KafkaStateCheckpointMarker: " + stateCheckpointMarker); + } + } + + public short getVersion() { + return version; + } + + public SystemStreamPartition getChangelogSSP() { + return changelogSSP; + } + + public String getChangelogOffset() { + return changelogOffset; + } + + /** + * Builds a map of store changelog SSPs to their offset for Kafka changelog backed stores from the provided + * map of state backend factory name to map of store name to serialized state checkpoint markers. + * + * @param stateBackendToStoreSCMs Map of state backend factory name to map of store name to serialized + * state checkpoint markers + * @return Map of store changelog SSPss to their optional offset, or an empty map if there is no mapping for + * {@link #KAFKA_STATE_BACKEND_FACTORY_NAME} in the input map. Optional offset may be empty if the + * changelog SSP was empty. + */ + public static Map> scmsToSSPOffsetMap( + Map> stateBackendToStoreSCMs) { + Map> sspToOffsetOptions = new HashMap<>(); + if (stateBackendToStoreSCMs.containsKey(KAFKA_STATE_BACKEND_FACTORY_NAME)) { + Map storeToKafkaSCMs = stateBackendToStoreSCMs.get(KAFKA_STATE_BACKEND_FACTORY_NAME); + storeToKafkaSCMs.forEach((key, value) -> { + KafkaStateCheckpointMarker stateMarker = KafkaStateCheckpointMarker.deserialize(value); + Option offsetOption = Option.apply(stateMarker.getChangelogOffset()); + sspToOffsetOptions.put(new SystemStreamPartition(stateMarker.getChangelogSSP()), offsetOption); + }); + } + return sspToOffsetOptions; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + KafkaStateCheckpointMarker that = (KafkaStateCheckpointMarker) o; + return Objects.equals(changelogSSP, that.changelogSSP) && + Objects.equals(changelogOffset, that.changelogOffset); + } + + @Override + public int hashCode() { + return Objects.hash(changelogSSP, changelogOffset); + } + + /** + * It is used for serde'ing {@link KafkaStateCheckpointMarker}s, in conjunction with {@link #deserialize(String)}. + * @return the String representation of this {@link KafkaStateCheckpointMarker} + */ + public static String serialize(KafkaStateCheckpointMarker marker) { + try { + return MAPPER.writeValueAsString(marker); + } catch (JsonProcessingException e) { + throw new SamzaException(String.format("Error serializing KafkaCheckpointMarker %s", marker), e); + } + } + + @Override + public String toString() { + String separator = ","; + return String.format("%s%s%s%s%s%s%s", + changelogSSP.getSystem(), separator, changelogSSP.getStream(), separator, + changelogSSP.getPartition().getPartitionId(), separator, changelogOffset); + } +} diff --git a/samza-core/src/main/java/org/apache/samza/clustermanager/ClusterBasedJobCoordinator.java b/samza-core/src/main/java/org/apache/samza/clustermanager/ClusterBasedJobCoordinator.java index 08bcfda8fd..e956413884 100644 --- a/samza-core/src/main/java/org/apache/samza/clustermanager/ClusterBasedJobCoordinator.java +++ b/samza-core/src/main/java/org/apache/samza/clustermanager/ClusterBasedJobCoordinator.java @@ -59,10 +59,13 @@ import org.apache.samza.metrics.MetricsRegistryMap; import org.apache.samza.startpoint.StartpointManager; import org.apache.samza.storage.ChangelogStreamManager; +import org.apache.samza.storage.StateBackendAdmin; +import org.apache.samza.storage.StateBackendFactory; import org.apache.samza.system.StreamMetadataCache; import org.apache.samza.system.SystemAdmins; import org.apache.samza.system.SystemStream; import org.apache.samza.util.DiagnosticsUtil; +import org.apache.samza.util.ReflectionUtil; import org.apache.samza.util.SystemClock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -267,6 +270,18 @@ public void run() { MetadataResourceUtil metadataResourceUtil = new MetadataResourceUtil(jobModel, this.metrics, config); metadataResourceUtil.createResources(); + // create all the resources required for state backend factories + StorageConfig storageConfig = new StorageConfig(config); + storageConfig.getBackupFactories().forEach(stateStorageBackendBackupFactory -> { + StateBackendFactory stateBackendFactory = + ReflectionUtil.getObj(stateStorageBackendBackupFactory, StateBackendFactory.class); + StateBackendAdmin stateBackendAdmin = stateBackendFactory.getAdmin(jobModel, config); + // Create resources required for state backend admin + stateBackendAdmin.createResources(); + // Validate resources required for state backend admin + stateBackendAdmin.validateResources(); + }); + /* * We fanout startpoint if and only if * 1. Startpoint is enabled in configuration diff --git a/samza-core/src/main/java/org/apache/samza/config/BlobStoreConfig.java b/samza-core/src/main/java/org/apache/samza/config/BlobStoreConfig.java new file mode 100644 index 0000000000..769ba29ca6 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/config/BlobStoreConfig.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.config; + +/** + * Config related helper methods for BlobStore. + */ +public class BlobStoreConfig extends MapConfig { + + public static final String BLOB_STORE_MANAGER_FACTORY = "blob.store.manager.factory"; + public static final String BLOB_STORE_ADMIN_FACTORY = "blob.store.admin.factory"; + public BlobStoreConfig(Config config) { + super(config); + } + + + public String getBlobStoreManagerFactory() { + return get(BLOB_STORE_MANAGER_FACTORY); + } + + public String getBlobStoreAdminFactory() { + return get(BLOB_STORE_ADMIN_FACTORY); + } +} diff --git a/samza-core/src/main/java/org/apache/samza/config/JobConfig.java b/samza-core/src/main/java/org/apache/samza/config/JobConfig.java index 2aa5c2ad34..48220670a7 100644 --- a/samza-core/src/main/java/org/apache/samza/config/JobConfig.java +++ b/samza-core/src/main/java/org/apache/samza/config/JobConfig.java @@ -63,6 +63,18 @@ public class JobConfig extends MapConfig { public static final String JOB_CONTAINER_COUNT = "job.container.count"; static final int DEFAULT_JOB_CONTAINER_COUNT = 1; public static final String JOB_CONTAINER_THREAD_POOL_SIZE = "job.container.thread.pool.size"; + // num commit threads == min(max(2 * num tasks in container, thread pool size), max thread pool size) + public static final String COMMIT_THREAD_POOL_SIZE = "job.container.commit.thread.pool.size"; + static final int DEFAULT_COMMIT_THREAD_POOL_SIZE = 2; + public static final String COMMIT_THREAD_POOL_MAX_SIZE = "job.container.commit.thread.pool.max.size"; + static final int DEFAULT_COMMIT_THREAD_POOL_MAX_SIZE = 64; + + // num commit threads == min(max(2 * num tasks in container, thread pool size), max thread pool size) + public static final String RESTORE_THREAD_POOL_SIZE = "job.container.restore.thread.pool.size"; + static final int DEFAULT_RESTORE_THREAD_POOL_SIZE = 2; + public static final String RESTORE_THREAD_POOL_MAX_SIZE = "job.container.restore.thread.pool.max.size"; + static final int DEFAULT_RESTORE_THREAD_POOL_MAX_SIZE = 64; + public static final String JOB_INTERMEDIATE_STREAM_PARTITIONS = "job.intermediate.stream.partitions"; public static final String JOB_DEBOUNCE_TIME_MS = "job.debounce.time.ms"; @@ -334,6 +346,22 @@ public int getThreadPoolSize() { } } + public int getCommitThreadPoolSize() { + return getInt(COMMIT_THREAD_POOL_SIZE, DEFAULT_COMMIT_THREAD_POOL_SIZE); + } + + public int getCommitThreadPoolMaxSize() { + return getInt(COMMIT_THREAD_POOL_MAX_SIZE, DEFAULT_COMMIT_THREAD_POOL_MAX_SIZE); + } + + public int getRestoreThreadPoolSize() { + return getInt(RESTORE_THREAD_POOL_SIZE, DEFAULT_RESTORE_THREAD_POOL_SIZE); + } + + public int getRestoreThreadPoolMaxSize() { + return getInt(RESTORE_THREAD_POOL_MAX_SIZE, DEFAULT_RESTORE_THREAD_POOL_MAX_SIZE); + } + public int getDebounceTimeMs() { return getInt(JOB_DEBOUNCE_TIME_MS, DEFAULT_DEBOUNCE_TIME_MS); } diff --git a/samza-core/src/main/java/org/apache/samza/config/StorageConfig.java b/samza-core/src/main/java/org/apache/samza/config/StorageConfig.java index 548cc27676..b170bacdee 100644 --- a/samza-core/src/main/java/org/apache/samza/config/StorageConfig.java +++ b/samza-core/src/main/java/org/apache/samza/config/StorageConfig.java @@ -19,16 +19,22 @@ package org.apache.samza.config; +import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.samza.SamzaException; import org.apache.samza.execution.StreamManager; +import org.apache.samza.system.SystemStream; +import org.apache.samza.util.StreamUtil; import static com.google.common.base.Preconditions.*; @@ -60,6 +66,19 @@ public class StorageConfig extends MapConfig { public static final String CHANGELOG_MIN_COMPACTION_LAG_MS = STORE_PREFIX + "%s.changelog." + MIN_COMPACTION_LAG_MS; public static final long DEFAULT_CHANGELOG_MIN_COMPACTION_LAG_MS = TimeUnit.HOURS.toMillis(4); + public static final String INMEMORY_KV_STORAGE_ENGINE_FACTORY = + "org.apache.samza.storage.kv.inmemory.InMemoryKeyValueStorageEngineFactory"; + public static final String KAFKA_STATE_BACKEND_FACTORY = + "org.apache.samza.storage.KafkaChangelogStateBackendFactory"; + public static final List DEFAULT_BACKUP_FACTORIES = ImmutableList.of( + KAFKA_STATE_BACKEND_FACTORY); + public static final String JOB_BACKUP_FACTORIES = STORE_PREFIX + "backup.factories"; + public static final String STORE_BACKUP_FACTORIES = STORE_PREFIX + "%s.backup.factories"; + public static final String RESTORE_FACTORIES_SUFFIX = "restore.factories"; + public static final String STORE_RESTORE_FACTORIES = STORE_PREFIX + "%s." + RESTORE_FACTORIES_SUFFIX; + public static final String JOB_RESTORE_FACTORIES = STORE_PREFIX + RESTORE_FACTORIES_SUFFIX; + public static final List DEFAULT_RESTORE_FACTORIES = ImmutableList.of(KAFKA_STATE_BACKEND_FACTORY); + static final String CHANGELOG_SYSTEM = "job.changelog.system"; static final String CHANGELOG_DELETE_RETENTION_MS = STORE_PREFIX + "%s.changelog.delete.retention.ms"; static final long DEFAULT_CHANGELOG_DELETE_RETENTION_MS = TimeUnit.DAYS.toMillis(1); @@ -70,8 +89,6 @@ public class StorageConfig extends MapConfig { static final String SIDE_INPUTS_PROCESSOR_FACTORY = STORE_PREFIX + "%s" + SIDE_INPUT_PROCESSOR_FACTORY_SUFFIX; static final String SIDE_INPUTS_PROCESSOR_SERIALIZED_INSTANCE = STORE_PREFIX + "%s.side.inputs.processor.serialized.instance"; - static final String INMEMORY_KV_STORAGE_ENGINE_FACTORY = - "org.apache.samza.storage.kv.inmemory.InMemoryKeyValueStorageEngineFactory"; // Internal config to clean storeDirs of a store on container start. This is used to benchmark bootstrap performance. static final String CLEAN_LOGGED_STOREDIRS_ON_START = STORE_PREFIX + "%s.clean.on.container.start"; @@ -93,6 +110,11 @@ public List getStoreNames() { return storeNames; } + public Map getStoreChangelogs() { + return getStoreNames().stream().filter(store -> getChangelogStream(store).isPresent()) + .collect(Collectors.toMap(Function.identity(), n -> StreamUtil.getSystemStreamFromNames(getChangelogStream(n).get()))); + } + /** * If the config specifies 'stores.<storename>.changelog' as '<system>.<stream>' combination - it will take * precedence. @@ -258,11 +280,105 @@ public int getNumPersistentStores() { .count(); } + private List getJobStoreBackupFactories() { + return getList(JOB_BACKUP_FACTORIES, new ArrayList<>()); + } + + /** + * Backup state backend factory follows the precedence: + * + * 1. If stores.store-name.backup.factories config key exists the store-name, that value is used + * 2. If stores.backup.factories is set for the job, that value is used + * 3. If stores.store-name.changelog is set for store-name, the default Kafka changelog state backend factory + * 4. Otherwise no backup factories will be configured for the store + * + * Note: that 2 takes precedence over 3 enables job based migration off of Changelog restores + * @return List of backup factories for the store in order of backup precedence + */ + public List getStoreBackupFactories(String storeName) { + List storeBackupManagers; + if (containsKey(String.format(STORE_BACKUP_FACTORIES, storeName))) { + storeBackupManagers = getList(String.format(STORE_BACKUP_FACTORIES, storeName), new ArrayList<>()); + } else { + storeBackupManagers = getJobStoreBackupFactories(); + // For backwards compatibility if the changelog is enabled, we use default kafka backup factory + if (storeBackupManagers.isEmpty() && getChangelogStream(storeName).isPresent()) { + storeBackupManagers = DEFAULT_BACKUP_FACTORIES; + } + } + return storeBackupManagers; + } + + public Set getBackupFactories() { + return getStoreNames().stream() + .flatMap((storeName) -> getStoreBackupFactories(storeName).stream()) + .collect(Collectors.toSet()); + } + + public List getStoresWithBackupFactory(String backendFactoryName) { + return getStoreNames().stream() + .filter((storeName) -> getStoreBackupFactories(storeName) + .contains(backendFactoryName)) + .collect(Collectors.toList()); + } + + public List getPersistentStoresWithBackupFactory(String backendFactoryName) { + return getStoreNames().stream() + .filter(storeName -> { + Optional storeFactory = getStorageFactoryClassName(storeName); + return storeFactory.isPresent() && + !storeFactory.get().equals(StorageConfig.INMEMORY_KV_STORAGE_ENGINE_FACTORY); + }) + .filter((storeName) -> getStoreBackupFactories(storeName) + .contains(backendFactoryName)) + .collect(Collectors.toList()); + } + + private List getJobStoreRestoreFactories() { + return getList(JOB_RESTORE_FACTORIES, new ArrayList<>()); + } + + /** + * Restore state backend factory follows the precedence: + * + * 1. If stores.store-name.restore.factories config key exists for the store-name, that value is used + * 2. If stores.restore.factories is set for the job, that value is used + * 3. If stores.store-name.changelog is set for store-name, the default Kafka changelog state backend factory + * 4. Otherwise no restore factories will be configured for the store + * + * Note that 2 takes precedence over 3 enables job based migration off of Changelog restores + * @return List of restore factories for the store in order of restoration precedence + */ + public List getStoreRestoreFactories(String storeName) { + List storeRestoreManagers; + if (containsKey(String.format(STORE_RESTORE_FACTORIES, storeName))) { + storeRestoreManagers = getList(String.format(STORE_RESTORE_FACTORIES, storeName), new ArrayList<>()); + } else { + storeRestoreManagers = getJobStoreRestoreFactories(); + // for backwards compatibility if changelog is enabled, we use default Kafka backup factory + if (storeRestoreManagers.isEmpty() && getChangelogStream(storeName).isPresent()) { + storeRestoreManagers = DEFAULT_RESTORE_FACTORIES; + } + } + return storeRestoreManagers; + } + + public Set getRestoreFactories() { + return getStoreNames().stream() + .flatMap((storesName) -> getStoreRestoreFactories(storesName).stream()) + .collect(Collectors.toSet()); + } + + public List getStoresWithRestoreFactory(String backendFactoryName) { + return getStoreNames().stream() + .filter((storeName) -> getStoreRestoreFactories(storeName).contains(backendFactoryName)) + .collect(Collectors.toList()); + } + /** * Helper method to get if logged store dirs should be deleted regardless of their contents. - * @return */ - public boolean getCleanLoggedStoreDirsOnStart(String storeName) { + public boolean cleanLoggedStoreDirsOnStart(String storeName) { return getBoolean(String.format(CLEAN_LOGGED_STOREDIRS_ON_START, storeName), false); } } diff --git a/samza-core/src/main/java/org/apache/samza/config/TaskConfig.java b/samza-core/src/main/java/org/apache/samza/config/TaskConfig.java index 461b647062..06a8727480 100644 --- a/samza-core/src/main/java/org/apache/samza/config/TaskConfig.java +++ b/samza-core/src/main/java/org/apache/samza/config/TaskConfig.java @@ -19,6 +19,9 @@ package org.apache.samza.config; +import com.google.common.collect.ImmutableList; + +import java.time.Duration; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -53,6 +56,15 @@ public class TaskConfig extends MapConfig { // commit period in milliseconds public static final String COMMIT_MS = "task.commit.ms"; static final long DEFAULT_COMMIT_MS = 60000L; + // maximum amount of time a task may continue processing while a previous commit is pending + public static final String COMMIT_MAX_DELAY_MS = "task.commit.max.delay.ms"; + static final long DEFAULT_COMMIT_MAX_DELAY_MS = Duration.ofMinutes(1).toMillis(); + // maximum amount of time to block for a pending task commit to complete *after* + // COMMIT_MAX_DELAY_MS have passed since the pending commit start. if the pending commit + // does not complete within this timeout, the container will shut down. + public static final String COMMIT_TIMEOUT_MS = "task.commit.timeout.ms"; + static final long DEFAULT_COMMIT_TIMEOUT_MS = Duration.ofMinutes(30).toMillis(); + // how long to wait for a clean shutdown public static final String TASK_SHUTDOWN_MS = "task.shutdown.ms"; static final long DEFAULT_TASK_SHUTDOWN_MS = 30000L; @@ -108,6 +120,14 @@ public class TaskConfig extends MapConfig { // standby containers use this flag to indicate that checkpoints will be polled continually, rather than only once at startup like in an active container public static final String INTERNAL_CHECKPOINT_MANAGER_CONSUMER_STOP_AFTER_FIRST_READ = "samza.internal.task.checkpoint.consumer.stop.after.first.read"; + // list of checkpoint versions to write during processing + public static final String CHECKPOINT_WRITE_VERSIONS = "task.checkpoint.write.versions"; + public static final List DEFAULT_CHECKPOINT_WRITE_VERSIONS = ImmutableList.of("1", "2"); + + // checkpoint version to read during container startup + public static final String CHECKPOINT_READ_VERSIONS = "task.checkpoint.read.versions"; + public static final List DEFAULT_CHECKPOINT_READ_VERSIONS = ImmutableList.of("1"); + public static final String TRANSACTIONAL_STATE_CHECKPOINT_ENABLED = "task.transactional.state.checkpoint.enabled"; private static final boolean DEFAULT_TRANSACTIONAL_STATE_CHECKPOINT_ENABLED = true; public static final String TRANSACTIONAL_STATE_RESTORE_ENABLED = "task.transactional.state.restore.enabled"; @@ -143,6 +163,14 @@ public long getCommitMs() { return getLong(COMMIT_MS, DEFAULT_COMMIT_MS); } + public long getCommitMaxDelayMs() { + return getLong(COMMIT_MAX_DELAY_MS, DEFAULT_COMMIT_MAX_DELAY_MS); + } + + public long getCommitTimeoutMs() { + return getLong(COMMIT_TIMEOUT_MS, DEFAULT_COMMIT_TIMEOUT_MS); + } + public Optional getTaskClass() { return Optional.ofNullable(get(TASK_CLASS)); } @@ -315,6 +343,23 @@ public long getShutdownMs() { } } + public List getCheckpointWriteVersions() { + return getList(CHECKPOINT_WRITE_VERSIONS, DEFAULT_CHECKPOINT_WRITE_VERSIONS) + .stream().map(Short::valueOf).collect(Collectors.toList()); + } + + public List getCheckpointReadVersions() { + List checkpointReadPriorityList = getList(CHECKPOINT_READ_VERSIONS, DEFAULT_CHECKPOINT_READ_VERSIONS) + .stream().map(Short::valueOf).collect(Collectors.toList()); + if (checkpointReadPriorityList.isEmpty()) { + // if the user explicitly defines the checkpoint read list to be empty + throw new IllegalArgumentException("No checkpoint read versions defined for job. " + + "Please remove the task.checkpoint.read.versions or define valid checkpoint versions"); + } else { + return checkpointReadPriorityList; + } + } + public boolean getTransactionalStateCheckpointEnabled() { return getBoolean(TRANSACTIONAL_STATE_CHECKPOINT_ENABLED, DEFAULT_TRANSACTIONAL_STATE_CHECKPOINT_ENABLED); } diff --git a/samza-core/src/main/java/org/apache/samza/coordinator/MetadataResourceUtil.java b/samza-core/src/main/java/org/apache/samza/coordinator/MetadataResourceUtil.java index 1050662920..9e366ea3c6 100644 --- a/samza-core/src/main/java/org/apache/samza/coordinator/MetadataResourceUtil.java +++ b/samza-core/src/main/java/org/apache/samza/coordinator/MetadataResourceUtil.java @@ -61,7 +61,7 @@ public void createResources() { @VisibleForTesting void createChangelogStreams() { - ChangelogStreamManager.createChangelogStreams(config, jobModel.maxChangeLogStreamPartitions); + ChangelogStreamManager.createChangelogStreams(config, jobModel.getMaxChangeLogStreamPartitions()); } @VisibleForTesting diff --git a/samza-core/src/main/java/org/apache/samza/serializers/CheckpointV2Serde.java b/samza-core/src/main/java/org/apache/samza/serializers/CheckpointV2Serde.java new file mode 100644 index 0000000000..48aa564e32 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/serializers/CheckpointV2Serde.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.serializers; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.samza.SamzaException; +import org.apache.samza.checkpoint.CheckpointV2; +import org.apache.samza.serializers.model.SamzaObjectMapper; + + +/** + * The {@link Serde} for {@link CheckpointV2} which includes {@link org.apache.samza.checkpoint.CheckpointId}s, + * state checkpoint markers and the input {@link org.apache.samza.system.SystemStreamPartition} offsets. + * + * The overall payload is serde'd as JSON using {@link SamzaObjectMapper}. Since the Samza classes cannot be directly + * serialized by Jackson using {@link org.apache.samza.serializers.model.JsonCheckpointV2Mixin}. + */ +public class CheckpointV2Serde implements Serde { + private static final ObjectMapper OBJECT_MAPPER = SamzaObjectMapper.getObjectMapper(); + + public CheckpointV2Serde() { } + + @Override + public CheckpointV2 fromBytes(byte[] bytes) { + try { + return OBJECT_MAPPER.readValue(bytes, CheckpointV2.class); + } catch (Exception e) { + throw new SamzaException(String.format("Exception while deserializing checkpoint: %s", new String(bytes)), e); + } + } + + @Override + public byte[] toBytes(CheckpointV2 checkpoint) { + try { + return OBJECT_MAPPER.writeValueAsBytes(checkpoint); + } catch (Exception e) { + throw new SamzaException(String.format("Exception while serializing checkpoint: %s", checkpoint.toString()), e); + } + } +} diff --git a/samza-core/src/main/java/org/apache/samza/serializers/JsonCheckpoint.java b/samza-core/src/main/java/org/apache/samza/serializers/JsonCheckpoint.java new file mode 100644 index 0000000000..47a8f7309d --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/serializers/JsonCheckpoint.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.serializers; + +import java.util.Map; + +/** + * Used for Json serialization of the {@link org.apache.samza.checkpoint.Checkpoint} class by the + * {@link CheckpointV2Serde} + * This cannot be an internal class as required by Jackson Object mapper + */ +public class JsonCheckpoint { + private String checkpointId; + private Map> inputOffsets; + // Map> + private Map> stateCheckpointMarkers; + + // Default constructor required for Jackson ObjectMapper + public JsonCheckpoint() {} + + public JsonCheckpoint(String checkpointId, + Map> inputOffsets, + Map> stateCheckpointMakers) { + this.checkpointId = checkpointId; + this.inputOffsets = inputOffsets; + this.stateCheckpointMarkers = stateCheckpointMakers; + } + + public String getCheckpointId() { + return checkpointId; + } + + public Map> getInputOffsets() { + return inputOffsets; + } + + public Map> getStateCheckpointMarkers() { + return stateCheckpointMarkers; + } +} diff --git a/samza-core/src/main/java/org/apache/samza/serializers/model/JsonCheckpointV2Mixin.java b/samza-core/src/main/java/org/apache/samza/serializers/model/JsonCheckpointV2Mixin.java new file mode 100644 index 0000000000..9edd234319 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/serializers/model/JsonCheckpointV2Mixin.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.serializers.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; +import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.system.SystemStreamPartition; + +@JsonIgnoreProperties(ignoreUnknown = true) +public abstract class JsonCheckpointV2Mixin { + @JsonCreator + public JsonCheckpointV2Mixin( + @JsonProperty("checkpoint-id") CheckpointId checkpointId, + @JsonProperty("input-offsets") Map inputOffsets, + @JsonProperty("state-checkpoint-markers") Map> stateCheckpointMarkers) { + } + + @JsonProperty("checkpoint-id") + abstract CheckpointId getCheckpointId(); + + @JsonProperty("input-offsets") + abstract Map getOffsets(); + + @JsonProperty("state-checkpoint-markers") + abstract Map> getStateCheckpointMarkers(); +} diff --git a/samza-core/src/main/java/org/apache/samza/serializers/model/KafkaStateCheckpointMarkerMixin.java b/samza-core/src/main/java/org/apache/samza/serializers/model/KafkaStateCheckpointMarkerMixin.java new file mode 100644 index 0000000000..6194877995 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/serializers/model/KafkaStateCheckpointMarkerMixin.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.serializers.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.samza.system.SystemStreamPartition; + + +/** + * A mix-in Jackson class to convert {@link org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker} to/from JSON + */ +@JsonIgnoreProperties(ignoreUnknown = true) +abstract public class KafkaStateCheckpointMarkerMixin { + @JsonCreator + public KafkaStateCheckpointMarkerMixin( + @JsonProperty("version") short version, + @JsonProperty("changelog-ssp") SystemStreamPartition changelogSSP, + @JsonProperty("changelog-offset") String changelogOffset) { + } + + @JsonProperty("version") + abstract short getVersion(); + + @JsonProperty("changelog-ssp") + abstract SystemStreamPartition getChangelogSSP(); + + @JsonProperty("changelog-offset") + abstract String getChangelogOffset(); +} diff --git a/samza-core/src/main/java/org/apache/samza/serializers/model/SamzaObjectMapper.java b/samza-core/src/main/java/org/apache/samza/serializers/model/SamzaObjectMapper.java index 5c5078f47b..7888f1fbef 100644 --- a/samza-core/src/main/java/org/apache/samza/serializers/model/SamzaObjectMapper.java +++ b/samza-core/src/main/java/org/apache/samza/serializers/model/SamzaObjectMapper.java @@ -41,6 +41,9 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import org.apache.samza.Partition; import org.apache.samza.SamzaException; +import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.checkpoint.CheckpointV2; +import org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; import org.apache.samza.container.TaskName; @@ -95,11 +98,15 @@ public static ObjectMapper getObjectMapper() { module.addKeyDeserializer(SystemStreamPartition.class, new SystemStreamPartitionKeyDeserializer()); module.addDeserializer(Config.class, new ConfigDeserializer()); module.addDeserializer(TaskMode.class, new TaskModeDeserializer()); + module.addSerializer(CheckpointId.class, new CheckpointIdSerializer()); + module.addDeserializer(CheckpointId.class, new CheckpointIdDeserializer()); // Setup mixins for data models. mapper.addMixIn(TaskModel.class, JsonTaskModelMixIn.class); mapper.addMixIn(ContainerModel.class, JsonContainerModelMixIn.class); mapper.addMixIn(JobModel.class, JsonJobModelMixIn.class); + mapper.addMixIn(CheckpointV2.class, JsonCheckpointV2Mixin.class); + mapper.addMixIn(KafkaStateCheckpointMarker.class, KafkaStateCheckpointMarkerMixin.class); module.addDeserializer(ContainerModel.class, new JsonDeserializer() { @Override @@ -255,6 +262,22 @@ public SystemStreamPartition deserialize(JsonParser jsonParser, DeserializationC } } + public static class CheckpointIdSerializer extends JsonSerializer { + @Override + public void serialize(CheckpointId checkpointId, JsonGenerator gen, SerializerProvider serializers) throws IOException { + gen.writeString(checkpointId.serialize()); + } + } + + public static class CheckpointIdDeserializer extends JsonDeserializer { + @Override + public CheckpointId deserialize(JsonParser jsonParser, DeserializationContext context) throws IOException { + ObjectCodec oc = jsonParser.getCodec(); + JsonNode node = oc.readTree(jsonParser); + return CheckpointId.deserialize(node.textValue()); + } + } + /** * A Jackson property naming strategy that converts camel case JSON fields to * hyphenated names. For example, myVariableName would be converted to diff --git a/samza-core/src/main/java/org/apache/samza/standalone/PassthroughJobCoordinator.java b/samza-core/src/main/java/org/apache/samza/standalone/PassthroughJobCoordinator.java index cc51732fb9..7bf29fe6af 100644 --- a/samza-core/src/main/java/org/apache/samza/standalone/PassthroughJobCoordinator.java +++ b/samza-core/src/main/java/org/apache/samza/standalone/PassthroughJobCoordinator.java @@ -19,25 +19,27 @@ package org.apache.samza.standalone; import com.google.common.collect.ImmutableMap; +import java.util.Collections; import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.container.grouper.task.GrouperMetadata; import org.apache.samza.container.grouper.task.GrouperMetadataImpl; import org.apache.samza.coordinator.JobCoordinator; +import org.apache.samza.coordinator.JobCoordinatorListener; import org.apache.samza.coordinator.JobModelManager; import org.apache.samza.coordinator.MetadataResourceUtil; import org.apache.samza.job.model.JobModel; -import org.apache.samza.coordinator.JobCoordinatorListener; +import org.apache.samza.metrics.MetricsRegistry; import org.apache.samza.runtime.LocationId; import org.apache.samza.runtime.LocationIdProvider; import org.apache.samza.runtime.LocationIdProviderFactory; -import org.apache.samza.metrics.MetricsRegistry; import org.apache.samza.system.StreamMetadataCache; import org.apache.samza.system.SystemAdmins; -import org.apache.samza.util.*; +import org.apache.samza.util.ReflectionUtil; +import org.apache.samza.util.SystemClock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; + /** * Standalone Job Coordinator does not implement any leader elector module or cluster manager @@ -125,7 +127,9 @@ public JobModel getJobModel() { systemAdmins.start(); try { String containerId = Integer.toString(config.getInt(JobConfig.PROCESSOR_ID)); - GrouperMetadata grouperMetadata = new GrouperMetadataImpl(ImmutableMap.of(String.valueOf(containerId), locationId), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); + GrouperMetadata grouperMetadata = + new GrouperMetadataImpl(ImmutableMap.of(String.valueOf(containerId), locationId), Collections.emptyMap(), + Collections.emptyMap(), Collections.emptyMap()); return JobModelManager.readJobModel(this.config, Collections.emptyMap(), streamMetadataCache, grouperMetadata); } finally { systemAdmins.stop(); diff --git a/samza-core/src/main/java/org/apache/samza/startpoint/StartpointManager.java b/samza-core/src/main/java/org/apache/samza/startpoint/StartpointManager.java index c1722889ae..083b483d6b 100644 --- a/samza-core/src/main/java/org/apache/samza/startpoint/StartpointManager.java +++ b/samza-core/src/main/java/org/apache/samza/startpoint/StartpointManager.java @@ -244,7 +244,7 @@ public void deleteAllStartpoints() { * The Startpoints that are written to with {@link #writeStartpoint(SystemStreamPartition, Startpoint)} and with * {@link #writeStartpoint(SystemStreamPartition, TaskName, Startpoint)} are moved from a "read-write" namespace * to a "fan out" namespace. - * This method is not atomic or thread-safe. The intent is for the Samza Processor's coordinator to use this + * This method is not atomic nor thread-safe. The intent is for the Samza Processor's coordinator to use this * method to assign the Startpoints to the appropriate tasks. * @param taskToSSPs Determines which {@link TaskName} each {@link SystemStreamPartition} maps to. * @return The set of active {@link TaskName}s that were fanned out to. diff --git a/samza-core/src/main/java/org/apache/samza/storage/KafkaChangelogStateBackendFactory.java b/samza-core/src/main/java/org/apache/samza/storage/KafkaChangelogStateBackendFactory.java new file mode 100644 index 0000000000..84339968e0 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/KafkaChangelogStateBackendFactory.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage; + +import com.google.common.annotations.VisibleForTesting; +import java.io.File; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; +import org.apache.commons.collections4.MapUtils; +import org.apache.samza.config.Config; +import org.apache.samza.config.StorageConfig; +import org.apache.samza.config.TaskConfig; +import org.apache.samza.context.ContainerContext; +import org.apache.samza.context.JobContext; +import org.apache.samza.job.model.ContainerModel; +import org.apache.samza.job.model.JobModel; +import org.apache.samza.job.model.TaskMode; +import org.apache.samza.job.model.TaskModel; +import org.apache.samza.metrics.MetricsRegistry; +import org.apache.samza.system.SSPMetadataCache; +import org.apache.samza.system.StreamMetadataCache; +import org.apache.samza.system.SystemAdmins; +import org.apache.samza.system.SystemStream; +import org.apache.samza.system.SystemStreamPartition; +import org.apache.samza.util.Clock; + + +public class KafkaChangelogStateBackendFactory implements StateBackendFactory { + private StreamMetadataCache streamCache; + /* + * This keeps track of the changelog SSPs that are associated with the whole container. This is used so that we can + * prefetch the metadata about the all of the changelog SSPs associated with the container whenever we need the + * metadata about some of the changelog SSPs. + * An example use case is when Samza writes offset files for stores ({@link TaskStorageManager}). Each task is + * responsible for its own offset file, but if we can do prefetching, then most tasks will already have cached + * metadata by the time they need the offset metadata. + * Note: By using all changelog streams to build the sspsToPrefetch, any fetches done for persisted stores will + * include the ssps for non-persisted stores, so this is slightly suboptimal. However, this does not increase the + * actual number of calls to the {@link SystemAdmin}, and we can decouple this logic from the per-task objects (e.g. + * {@link TaskStorageManager}). + */ + private SSPMetadataCache sspCache; + + @Override + public TaskBackupManager getBackupManager(JobContext jobContext, + ContainerModel containerModel, + TaskModel taskModel, + ExecutorService backupExecutor, + MetricsRegistry metricsRegistry, + Config config, + Clock clock, + File loggedStoreBaseDir, + File nonLoggedStoreBaseDir) { + SystemAdmins systemAdmins = new SystemAdmins(config); + StorageConfig storageConfig = new StorageConfig(config); + Map storeChangelogs = storageConfig.getStoreChangelogs(); + + if (new TaskConfig(config).getTransactionalStateCheckpointEnabled()) { + return new KafkaTransactionalStateTaskBackupManager(taskModel.getTaskName(), storeChangelogs, + systemAdmins, taskModel.getChangelogPartition()); + } else { + return new KafkaNonTransactionalStateTaskBackupManager(taskModel.getTaskName(), storeChangelogs, + systemAdmins, taskModel.getChangelogPartition()); + } + } + + @Override + public TaskRestoreManager getRestoreManager(JobContext jobContext, + ContainerContext containerContext, + TaskModel taskModel, + ExecutorService restoreExecutor, + MetricsRegistry metricsRegistry, + Set storesToRestore, + Config config, + Clock clock, + File loggedStoreBaseDir, + File nonLoggedStoreBaseDir, + KafkaChangelogRestoreParams kafkaChangelogRestoreParams) { + Map storeChangelogs = new StorageConfig(config).getStoreChangelogs(); + Set changelogSSPs = storeChangelogs.values().stream() + .flatMap(ss -> containerContext.getContainerModel().getTasks().values().stream() + .map(tm -> new SystemStreamPartition(ss, tm.getChangelogPartition()))) + .collect(Collectors.toSet()); + // filter out standby store-ssp pairs + Map filteredStoreChangelogs = + filterStandbySystemStreams(storeChangelogs, containerContext.getContainerModel()); + SystemAdmins systemAdmins = new SystemAdmins(kafkaChangelogRestoreParams.getSystemAdmins()); + + if (new TaskConfig(config).getTransactionalStateRestoreEnabled()) { + return new TransactionalStateTaskRestoreManager( + storesToRestore, + jobContext, + containerContext, + taskModel, + filteredStoreChangelogs, + kafkaChangelogRestoreParams.getInMemoryStores(), + kafkaChangelogRestoreParams.getStorageEngineFactories(), + kafkaChangelogRestoreParams.getSerdes(), + systemAdmins, + kafkaChangelogRestoreParams.getStoreConsumers(), + metricsRegistry, + kafkaChangelogRestoreParams.getCollector(), + getSspCache(systemAdmins, clock, changelogSSPs), + loggedStoreBaseDir, + nonLoggedStoreBaseDir, + config, + clock + ); + } else { + return new NonTransactionalStateTaskRestoreManager( + storesToRestore, + jobContext, + containerContext, + taskModel, + filteredStoreChangelogs, + kafkaChangelogRestoreParams.getInMemoryStores(), + kafkaChangelogRestoreParams.getStorageEngineFactories(), + kafkaChangelogRestoreParams.getSerdes(), + systemAdmins, + getStreamCache(systemAdmins, clock), + kafkaChangelogRestoreParams.getStoreConsumers(), + metricsRegistry, + kafkaChangelogRestoreParams.getCollector(), + jobContext.getJobModel().getMaxChangeLogStreamPartitions(), + loggedStoreBaseDir, + nonLoggedStoreBaseDir, + config, + clock + ); + } + } + + @Override + //TODO HIGH snjain implement this + public StateBackendAdmin getAdmin(JobModel jobModel, Config config) { + return new NoOpKafkaChangelogStateBackendAdmin(); + } + + public Set getChangelogSSPForContainer(Map storeChangelogs, + ContainerContext containerContext) { + return storeChangelogs.values().stream() + .flatMap(ss -> containerContext.getContainerModel().getTasks().values().stream() + .map(tm -> new SystemStreamPartition(ss, tm.getChangelogPartition()))) + .collect(Collectors.toSet()); + } + + /** + * Shared cache across all KafkaRestoreManagers for the Kafka topic + * + * @param admins system admins used the fetch the stream metadata + * @param clock for cache invalidation + * @return StreamMetadataCache containing the stream metadata + */ + @VisibleForTesting + StreamMetadataCache getStreamCache(SystemAdmins admins, Clock clock) { + if (streamCache == null) { + streamCache = new StreamMetadataCache(admins, 5000, clock); + } + return streamCache; + } + + /** + * Shared cache across KafkaRestoreManagers for the Kafka partition + * + * @param admins system admins used the fetch the stream metadata + * @param clock for cache invalidation + * @param ssps SSPs to prefetch + * @return SSPMetadataCache containing the partition metadata + */ + private SSPMetadataCache getSspCache(SystemAdmins admins, Clock clock, Set ssps) { + if (sspCache == null) { + sspCache = new SSPMetadataCache(admins, Duration.ofSeconds(5), clock, ssps); + } + return sspCache; + } + + @VisibleForTesting + Map filterStandbySystemStreams(Map changelogSystemStreams, + ContainerModel containerModel) { + Map changelogSSPToStore = new HashMap<>(); + changelogSystemStreams.forEach((storeName, systemStream) -> + containerModel.getTasks().forEach((taskName, taskModel) -> + changelogSSPToStore.put(new SystemStreamPartition(systemStream, taskModel.getChangelogPartition()), storeName)) + ); + + Set standbyTaskModels = containerModel.getTasks().values().stream() + .filter(taskModel -> taskModel.getTaskMode().equals(TaskMode.Standby)) + .collect(Collectors.toSet()); + + // remove all standby task changelog ssps + standbyTaskModels.forEach((taskModel) -> { + changelogSystemStreams.forEach((storeName, systemStream) -> { + SystemStreamPartition ssp = new SystemStreamPartition(systemStream, taskModel.getChangelogPartition()); + changelogSSPToStore.remove(ssp); + }); + }); + + // changelogSystemStreams correspond only to active tasks (since those of standby-tasks moved to sideInputs above) + return MapUtils.invertMap(changelogSSPToStore).entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, x -> x.getValue().getSystemStream())); + } + + public class NoOpKafkaChangelogStateBackendAdmin implements StateBackendAdmin { + + @Override + public void createResources() { + // all the changelog creations are handled by {@link ChangelogStreamManager} + } + + @Override + public void validateResources() { + // all the changelog validations are handled by {@link ChangelogStreamManager} + } + } +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/NonTransactionalStateTaskRestoreManager.java b/samza-core/src/main/java/org/apache/samza/storage/NonTransactionalStateTaskRestoreManager.java index 44dd59a93f..33af3677bc 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/NonTransactionalStateTaskRestoreManager.java +++ b/samza-core/src/main/java/org/apache/samza/storage/NonTransactionalStateTaskRestoreManager.java @@ -28,9 +28,14 @@ import java.util.stream.Collectors; import org.apache.samza.Partition; import org.apache.samza.SamzaException; +import org.apache.samza.checkpoint.Checkpoint; import org.apache.samza.config.Config; import org.apache.samza.config.StorageConfig; +import org.apache.samza.context.ContainerContext; +import org.apache.samza.context.JobContext; import org.apache.samza.job.model.TaskModel; +import org.apache.samza.metrics.MetricsRegistry; +import org.apache.samza.serializers.Serde; import org.apache.samza.system.ChangelogSSPIterator; import org.apache.samza.system.StreamMetadataCache; import org.apache.samza.system.StreamSpec; @@ -40,6 +45,7 @@ import org.apache.samza.system.SystemStream; import org.apache.samza.system.SystemStreamMetadata; import org.apache.samza.system.SystemStreamPartition; +import org.apache.samza.task.MessageCollector; import org.apache.samza.util.Clock; import org.apache.samza.util.FileUtil; import org.slf4j.Logger; @@ -64,44 +70,52 @@ class NonTransactionalStateTaskRestoreManager implements TaskRestoreManager { private final Clock clock; // Clock value used to validate base-directories for staleness. See isLoggedStoreValid. private Map changeLogOldestOffsets; // Map of changelog oldest known offsets private final Map fileOffsets; // Map of offsets read from offset file indexed by changelog SSP - private final Map changelogSystemStreams; // Map of change log system-streams indexed by store name + private final Map storeChangelogs; // Map of change log system-streams indexed by store name private final SystemAdmins systemAdmins; private final File loggedStoreBaseDirectory; private final File nonLoggedStoreBaseDirectory; private final StreamMetadataCache streamMetadataCache; private final Map storeConsumers; private final int maxChangeLogStreamPartitions; - private final StorageConfig storageConfig; + private final Config config; private final StorageManagerUtil storageManagerUtil; NonTransactionalStateTaskRestoreManager( + Set storeNames, + JobContext jobContext, + ContainerContext containerContext, TaskModel taskModel, - Map changelogSystemStreams, - Map taskStores, + Map storeChangelogs, + Map inMemoryStores, + Map> storageEngineFactories, + Map> serdes, SystemAdmins systemAdmins, StreamMetadataCache streamMetadataCache, Map storeConsumers, + MetricsRegistry metricsRegistry, + MessageCollector messageCollector, int maxChangeLogStreamPartitions, File loggedStoreBaseDirectory, File nonLoggedStoreBaseDirectory, Config config, Clock clock) { - this.taskStores = taskStores; this.taskModel = taskModel; this.clock = clock; - this.changelogSystemStreams = changelogSystemStreams; + this.storeChangelogs = storeChangelogs; this.systemAdmins = systemAdmins; this.fileOffsets = new HashMap<>(); - this.taskStoresToRestore = this.taskStores.entrySet().stream() - .filter(x -> x.getValue().getStoreProperties().isLoggedStore()) - .map(x -> x.getKey()).collect(Collectors.toSet()); this.loggedStoreBaseDirectory = loggedStoreBaseDirectory; this.nonLoggedStoreBaseDirectory = nonLoggedStoreBaseDirectory; this.streamMetadataCache = streamMetadataCache; this.storeConsumers = storeConsumers; this.maxChangeLogStreamPartitions = maxChangeLogStreamPartitions; - this.storageConfig = new StorageConfig(config); + this.config = config; this.storageManagerUtil = new StorageManagerUtil(); + this.taskStores = createStoreEngines(storeNames, jobContext, containerContext, + storageEngineFactories, serdes, metricsRegistry, messageCollector, inMemoryStores); + this.taskStoresToRestore = this.taskStores.entrySet().stream() + .filter(x -> x.getValue().getStoreProperties().isLoggedStore()) + .map(x -> x.getKey()).collect(Collectors.toSet()); } /** @@ -109,7 +123,7 @@ class NonTransactionalStateTaskRestoreManager implements TaskRestoreManager { * and registers SSPs with the respective consumers. */ @Override - public void init(Map checkpointedChangelogSSPOffsets) { + public void init(Checkpoint checkpoint) { cleanBaseDirsAndReadOffsetFiles(); setupBaseDirs(); validateChangelogStreams(); @@ -124,7 +138,7 @@ public void init(Map checkpointedChangelogSSPOffs */ private void cleanBaseDirsAndReadOffsetFiles() { LOG.debug("Cleaning base directories for stores."); - + StorageConfig storageConfig = new StorageConfig(config); FileUtil fileUtil = new FileUtil(); taskStores.forEach((storeName, storageEngine) -> { if (!storageEngine.getStoreProperties().isLoggedStore()) { @@ -142,12 +156,12 @@ private void cleanBaseDirsAndReadOffsetFiles() { LOG.info("Got logged storage partition directory as " + loggedStorePartitionDir.toPath().toString()); // Delete the logged store if it is not valid. - if (!isLoggedStoreValid(storeName, loggedStorePartitionDir) || storageConfig.getCleanLoggedStoreDirsOnStart(storeName)) { + if (!isLoggedStoreValid(storeName, loggedStorePartitionDir) || storageConfig.cleanLoggedStoreDirsOnStart(storeName)) { LOG.info("Deleting logged storage partition directory " + loggedStorePartitionDir.toPath().toString()); fileUtil.rm(loggedStorePartitionDir); } else { - SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStreams.get(storeName), taskModel.getChangelogPartition()); + SystemStreamPartition changelogSSP = new SystemStreamPartition(storeChangelogs.get(storeName), taskModel.getChangelogPartition()); Map offset = storageManagerUtil.readOffsetFile(loggedStorePartitionDir, Collections.singleton(changelogSSP), false); LOG.info("Read offset {} for the store {} from logged storage partition directory {}", offset, storeName, loggedStorePartitionDir); @@ -170,10 +184,10 @@ private void cleanBaseDirsAndReadOffsetFiles() { * @return true if the logged store is valid, false otherwise. */ private boolean isLoggedStoreValid(String storeName, File loggedStoreDir) { - long changeLogDeleteRetentionInMs = storageConfig.getChangeLogDeleteRetentionInMs(storeName); + long changeLogDeleteRetentionInMs = new StorageConfig(config).getChangeLogDeleteRetentionInMs(storeName); - if (changelogSystemStreams.containsKey(storeName)) { - SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStreams.get(storeName), taskModel.getChangelogPartition()); + if (storeChangelogs.containsKey(storeName)) { + SystemStreamPartition changelogSSP = new SystemStreamPartition(storeChangelogs.get(storeName), taskModel.getChangelogPartition()); return this.taskStores.get(storeName).getStoreProperties().isPersistedToDisk() && storageManagerUtil.isOffsetFileValid(loggedStoreDir, Collections.singleton(changelogSSP), false) && !storageManagerUtil.isStaleStore(loggedStoreDir, changeLogDeleteRetentionInMs, clock.currentTimeMillis(), false); @@ -213,9 +227,9 @@ private void setupBaseDirs() { * Validates each changelog system-stream with its respective SystemAdmin. */ private void validateChangelogStreams() { - LOG.info("Validating change log streams: " + changelogSystemStreams); + LOG.info("Validating change log streams: " + storeChangelogs); - for (SystemStream changelogSystemStream : changelogSystemStreams.values()) { + for (SystemStream changelogSystemStream : storeChangelogs.values()) { SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(changelogSystemStream.getSystem()); StreamSpec changelogSpec = StreamSpec.createChangeLogStreamSpec(changelogSystemStream.getStream(), changelogSystemStream.getSystem(), @@ -232,7 +246,7 @@ private void getOldestChangeLogOffsets() { Map changeLogMetadata = JavaConverters.mapAsJavaMapConverter( streamMetadataCache.getStreamMetadata( - JavaConverters.asScalaSetConverter(new HashSet<>(changelogSystemStreams.values())).asScala().toSet(), + JavaConverters.asScalaSetConverter(new HashSet<>(storeChangelogs.values())).asScala().toSet(), false)).asJava(); LOG.info("Got change log stream metadata: {}", changeLogMetadata); @@ -267,7 +281,7 @@ private Map getChangeLogOldestOffsetsForPartition(Partitio */ private void registerStartingOffsets() { - for (Map.Entry changelogSystemStreamEntry : changelogSystemStreams.entrySet()) { + for (Map.Entry changelogSystemStreamEntry : storeChangelogs.entrySet()) { SystemStreamPartition systemStreamPartition = new SystemStreamPartition(changelogSystemStreamEntry.getValue(), taskModel.getChangelogPartition()); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(changelogSystemStreamEntry.getValue().getSystem()); @@ -312,6 +326,27 @@ private String getStartingOffset(SystemStreamPartition systemStreamPartition, Sy return storageManagerUtil.getStartingOffset(systemStreamPartition, systemAdmin, fileOffset, oldestOffset); } + private Map createStoreEngines(Set storeNames, JobContext jobContext, + ContainerContext containerContext, Map> storageEngineFactories, + Map> serdes, MetricsRegistry metricsRegistry, + MessageCollector messageCollector, Map nonPersistedStores) { + Map storageEngines = new HashMap<>(); + // Put non persisted stores + nonPersistedStores.forEach(storageEngines::put); + // Create persisted stores + storeNames.forEach(storeName -> { + boolean isLogged = this.storeChangelogs.containsKey(storeName); + File storeBaseDir = isLogged ? this.loggedStoreBaseDirectory : this.nonLoggedStoreBaseDirectory; + File storeDirectory = storageManagerUtil.getTaskStoreDir(storeBaseDir, storeName, taskModel.getTaskName(), + taskModel.getTaskMode()); + StorageEngine engine = ContainerStorageManager.createStore(storeName, storeDirectory, taskModel, jobContext, containerContext, + storageEngineFactories, serdes, metricsRegistry, messageCollector, + StorageEngineFactory.StoreMode.BulkLoad, this.storeChangelogs, this.config); + storageEngines.put(storeName, engine); + }); + return storageEngines; + } + /** * Restore each store in taskStoresToRestore sequentially */ @@ -320,7 +355,7 @@ public void restore() throws InterruptedException { for (String storeName : taskStoresToRestore) { LOG.info("Restoring store: {} for task: {}", storeName, taskModel.getTaskName()); SystemConsumer systemConsumer = storeConsumers.get(storeName); - SystemStream systemStream = changelogSystemStreams.get(storeName); + SystemStream systemStream = storeChangelogs.get(storeName); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemStream.getSystem()); ChangelogSSPIterator changelogSSPIterator = new ChangelogSSPIterator(systemConsumer, new SystemStreamPartition(systemStream, taskModel.getChangelogPartition()), null, systemAdmin, false); @@ -333,7 +368,7 @@ public void restore() throws InterruptedException { * Stop only persistent stores. In case of certain stores and store mode (such as RocksDB), this * can invoke compaction. */ - public void stopPersistentStores() { + public void close() { Map persistentStores = this.taskStores.entrySet().stream().filter(e -> { return e.getValue().getStoreProperties().isPersistedToDisk(); diff --git a/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java b/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java index cfd24d94fd..badeb28c00 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java +++ b/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java @@ -20,12 +20,11 @@ package org.apache.samza.storage; import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.collect.ImmutableMap; - import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -38,12 +37,15 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.samza.SamzaException; +import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.checkpoint.CheckpointV2; import org.apache.samza.clustermanager.StandbyTaskUtil; import org.apache.samza.config.Config; import org.apache.samza.config.StorageConfig; import org.apache.samza.container.TaskName; import org.apache.samza.job.model.TaskMode; import org.apache.samza.job.model.TaskModel; +import org.apache.samza.serializers.CheckpointV2Serde; import org.apache.samza.serializers.model.SamzaObjectMapper; import org.apache.samza.system.SystemAdmin; import org.apache.samza.system.SystemStream; @@ -56,14 +58,16 @@ public class StorageManagerUtil { private static final Logger LOG = LoggerFactory.getLogger(StorageManagerUtil.class); + public static final String CHECKPOINT_FILE_NAME = "CHECKPOINT-V2"; public static final String OFFSET_FILE_NAME_NEW = "OFFSET-v2"; public static final String OFFSET_FILE_NAME_LEGACY = "OFFSET"; public static final String SIDE_INPUT_OFFSET_FILE_NAME_LEGACY = "SIDE-INPUT-OFFSETS"; private static final ObjectMapper OBJECT_MAPPER = SamzaObjectMapper.getObjectMapper(); private static final TypeReference> OFFSETS_TYPE_REFERENCE = new TypeReference>() { }; - private static final ObjectWriter OBJECT_WRITER = OBJECT_MAPPER.writerWithType(OFFSETS_TYPE_REFERENCE); + private static final ObjectWriter SSP_OFFSET_OBJECT_WRITER = OBJECT_MAPPER.writerFor(OFFSETS_TYPE_REFERENCE); private static final String SST_FILE_SUFFIX = ".sst"; + private static final CheckpointV2Serde CHECKPOINT_V2_SERDE = new CheckpointV2Serde(); /** * Fetch the starting offset for the input {@link SystemStreamPartition} @@ -118,12 +122,15 @@ public boolean isStaleStore(File storeDir, long storeDeleteRetentionInMs, long c // We check if the new offset-file exists, if so we use its last-modified time, if it doesn't we use the legacy file // depending on if it is a side-input or not, - // if neither exists, we use 0L (the defauilt return value of lastModified() when file does not exist + // if neither exists, we use 0L (the default return value of lastModified() when file does not exist File offsetFileRefNew = new File(storeDir, OFFSET_FILE_NAME_NEW); File offsetFileRefLegacy = new File(storeDir, OFFSET_FILE_NAME_LEGACY); File sideInputOffsetFileRefLegacy = new File(storeDir, SIDE_INPUT_OFFSET_FILE_NAME_LEGACY); + File checkpointV2File = new File(storeDir, CHECKPOINT_FILE_NAME); - if (offsetFileRefNew.exists()) { + if (checkpointV2File.exists()) { + offsetFileLastModifiedTime = checkpointV2File.lastModified(); + } else if (offsetFileRefNew.exists()) { offsetFileLastModifiedTime = offsetFileRefNew.lastModified(); } else if (!isSideInput && offsetFileRefLegacy.exists()) { offsetFileLastModifiedTime = offsetFileRefLegacy.lastModified(); @@ -210,14 +217,14 @@ public void writeOffsetFile(File storeDir, Map of // First, we write the new-format offset file File offsetFile = new File(storeDir, OFFSET_FILE_NAME_NEW); - String fileContents = OBJECT_WRITER.writeValueAsString(offsets); + String fileContents = SSP_OFFSET_OBJECT_WRITER.writeValueAsString(offsets); FileUtil fileUtil = new FileUtil(); fileUtil.writeWithChecksum(offsetFile, fileContents); // Now we write the old format offset file, which are different for store-offset and side-inputs if (isSideInput) { offsetFile = new File(storeDir, SIDE_INPUT_OFFSET_FILE_NAME_LEGACY); - fileContents = OBJECT_WRITER.writeValueAsString(offsets); + fileContents = SSP_OFFSET_OBJECT_WRITER.writeValueAsString(offsets); fileUtil.writeWithChecksum(offsetFile, fileContents); } else { offsetFile = new File(storeDir, OFFSET_FILE_NAME_LEGACY); @@ -225,6 +232,19 @@ public void writeOffsetFile(File storeDir, Map of } } + /** + * Writes the checkpoint to the store checkpoint directory based on the checkpointId. + * + * @param storeDir store or store checkpoint directory to write the checkpoint to + * @param checkpoint checkpoint v2 containing the checkpoint Id + */ + public void writeCheckpointV2File(File storeDir, CheckpointV2 checkpoint) { + File offsetFile = new File(storeDir, CHECKPOINT_FILE_NAME); + byte[] fileContents = CHECKPOINT_V2_SERDE.toBytes(checkpoint); + FileUtil fileUtil = new FileUtil(); + fileUtil.writeWithChecksum(offsetFile, new String(fileContents)); + } + /** * Delete the offset file for this store, if one exists. * @param storeDir the directory of the store @@ -283,6 +303,24 @@ public Map readOffsetFile(File storagePartitionDi } } + /** + * Read and return the {@link CheckpointV2} from the directory's {@link #CHECKPOINT_FILE_NAME} file. + * If the file does not exist, returns null. + * + * @param storagePartitionDir store directory to read the checkpoint file from + * @return the {@link CheckpointV2} object retrieved from the checkpoint file if found, otherwise return null + */ + // TODO dchen use checkpoint v2 file before migrating off of dual checkpoints + public CheckpointV2 readCheckpointV2File(File storagePartitionDir) { + File checkpointFile = new File(storagePartitionDir, CHECKPOINT_FILE_NAME); + if (checkpointFile.exists()) { + String serializedCheckpointV2 = new FileUtil().readWithChecksum(checkpointFile); + return new CheckpointV2Serde().fromBytes(serializedCheckpointV2.getBytes()); + } else { + return null; + } + } + /** * Read and return the contents of the offset file. * @@ -356,6 +394,17 @@ public List getTaskStoreCheckpointDirs(File storeBaseDir, String storeName } } + /** + * Returns the path for a storage engine to create its checkpoint based on the current checkpoint id. + * + * @param taskStoreDir directory of the store as returned by {@link #getTaskStoreDir} + * @param checkpointId current checkpoint id + * @return String denoting the file path of the store with the given checkpoint id + */ + public String getStoreCheckpointDir(File taskStoreDir, CheckpointId checkpointId) { + return taskStoreDir.getPath() + "-" + checkpointId.serialize(); + } + public void restoreCheckpointFiles(File checkpointDir, File storeDir) { // the current task store dir should already be deleted for restore assert !storeDir.exists(); diff --git a/samza-core/src/main/java/org/apache/samza/storage/StorageRecovery.java b/samza-core/src/main/java/org/apache/samza/storage/StorageRecovery.java index 9d1896e3b2..6b657d59d9 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/StorageRecovery.java +++ b/samza-core/src/main/java/org/apache/samza/storage/StorageRecovery.java @@ -20,7 +20,6 @@ package org.apache.samza.storage; import java.io.File; -import java.time.Duration; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,12 +45,10 @@ import org.apache.samza.metrics.MetricsRegistryMap; import org.apache.samza.serializers.Serde; import org.apache.samza.serializers.SerdeFactory; -import org.apache.samza.system.SSPMetadataCache; import org.apache.samza.system.StreamMetadataCache; import org.apache.samza.system.SystemAdmins; import org.apache.samza.system.SystemFactory; import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamPartition; import org.apache.samza.util.Clock; import org.apache.samza.util.CoordinatorStreamUtil; import org.apache.samza.util.ReflectionUtil; @@ -220,6 +217,10 @@ private Map> getSerdes() { */ @SuppressWarnings("rawtypes") private void getContainerStorageManagers() { + Set factoryClasses = new StorageConfig(jobConfig).getRestoreFactories(); + Map stateBackendFactories = factoryClasses.stream().collect( + Collectors.toMap(factoryClass -> factoryClass, + factoryClass -> ReflectionUtil.getObj(factoryClass, StateBackendFactory.class))); Clock clock = SystemClock.instance(); StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 5000, clock); // don't worry about prefetching for this; looks like the tool doesn't flush to offset files anyways @@ -230,18 +231,11 @@ private void getContainerStorageManagers() { for (ContainerModel containerModel : containers.values()) { ContainerContext containerContext = new ContainerContextImpl(containerModel, new MetricsRegistryMap()); - Set changelogSSPs = changeLogSystemStreams.values().stream() - .flatMap(ss -> containerModel.getTasks().values().stream() - .map(tm -> new SystemStreamPartition(ss, tm.getChangelogPartition()))) - .collect(Collectors.toSet()); - SSPMetadataCache sspMetadataCache = new SSPMetadataCache(systemAdmins, Duration.ofMillis(5000), clock, changelogSSPs); - ContainerStorageManager containerStorageManager = new ContainerStorageManager( checkpointManager, containerModel, streamMetadataCache, - sspMetadataCache, systemAdmins, changeLogSystemStreams, new HashMap<>(), @@ -253,10 +247,10 @@ private void getContainerStorageManagers() { new SamzaContainerMetrics(containerModel.getId(), new MetricsRegistryMap(), ""), JobContextImpl.fromConfigWithDefaults(jobConfig, jobModel), containerContext, + stateBackendFactories, new HashMap<>(), storeBaseDir, storeBaseDir, - maxPartitionNumber, null, new SystemClock()); this.containerStorageManagers.put(containerModel.getId(), containerStorageManager); diff --git a/samza-core/src/main/java/org/apache/samza/storage/TaskRestoreManagerFactory.java b/samza-core/src/main/java/org/apache/samza/storage/TaskRestoreManagerFactory.java deleted file mode 100644 index 9da9bc0ad6..0000000000 --- a/samza-core/src/main/java/org/apache/samza/storage/TaskRestoreManagerFactory.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.samza.storage; - -import java.io.File; -import java.util.Map; -import org.apache.samza.config.Config; -import org.apache.samza.config.TaskConfig; -import org.apache.samza.job.model.TaskModel; -import org.apache.samza.system.SSPMetadataCache; -import org.apache.samza.system.StreamMetadataCache; -import org.apache.samza.system.SystemAdmins; -import org.apache.samza.system.SystemConsumer; -import org.apache.samza.system.SystemStream; -import org.apache.samza.util.Clock; - -/** - * Factory class to create {@link TaskRestoreManager}. - */ -class TaskRestoreManagerFactory { - - public static TaskRestoreManager create( - TaskModel taskModel, - Map changelogSystemStreams, - Map taskStores, - SystemAdmins systemAdmins, - StreamMetadataCache streamMetadataCache, - SSPMetadataCache sspMetadataCache, - Map storeConsumers, - int maxChangeLogStreamPartitions, - File loggedStoreBaseDirectory, - File nonLoggedStoreBaseDirectory, - Config config, - Clock clock) { - - if (new TaskConfig(config).getTransactionalStateRestoreEnabled()) { - // Create checkpoint-snapshot based state restoration which is transactional. - return new TransactionalStateTaskRestoreManager( - taskModel, - taskStores, - changelogSystemStreams, - systemAdmins, - storeConsumers, - sspMetadataCache, - loggedStoreBaseDirectory, - nonLoggedStoreBaseDirectory, - config, - clock - ); - } else { - // Create legacy offset-file based state restoration which is NOT transactional. - return new NonTransactionalStateTaskRestoreManager( - taskModel, - changelogSystemStreams, - taskStores, - systemAdmins, - streamMetadataCache, - storeConsumers, - maxChangeLogStreamPartitions, - loggedStoreBaseDirectory, - nonLoggedStoreBaseDirectory, - config, - clock); - } - } -} diff --git a/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java b/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java index c93e0b3ea1..61d30c3fd0 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java +++ b/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java @@ -20,15 +20,6 @@ package org.apache.samza.storage; import com.google.common.annotations.VisibleForTesting; -import org.apache.samza.SamzaException; -import org.apache.samza.container.TaskName; -import org.apache.samza.job.model.TaskMode; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.util.Clock; -import org.apache.samza.util.FileUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.util.HashMap; import java.util.Map; @@ -37,6 +28,14 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.samza.SamzaException; +import org.apache.samza.container.TaskName; +import org.apache.samza.job.model.TaskMode; +import org.apache.samza.system.SystemStreamPartition; +import org.apache.samza.util.Clock; +import org.apache.samza.util.FileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** @@ -211,6 +210,7 @@ private boolean isPersistedStore(String storeName) { private void validateStoreConfiguration(Map stores) { stores.forEach((storeName, storageEngine) -> { + // Ensure that the side inputs store is NOT logged (they are durable) if (storageEngine.getStoreProperties().isLoggedStore()) { throw new SamzaException( String.format("Cannot configure both side inputs and a changelog for store: %s.", storeName)); diff --git a/samza-core/src/main/java/org/apache/samza/storage/TaskStorageCommitManager.java b/samza-core/src/main/java/org/apache/samza/storage/TaskStorageCommitManager.java new file mode 100644 index 0000000000..09a8807104 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/TaskStorageCommitManager.java @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage; + +import com.google.common.annotations.VisibleForTesting; +import java.io.File; +import java.io.FileFilter; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.WildcardFileFilter; +import org.apache.samza.Partition; +import org.apache.samza.SamzaException; +import org.apache.samza.checkpoint.Checkpoint; +import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.checkpoint.CheckpointManager; +import org.apache.samza.checkpoint.CheckpointV1; +import org.apache.samza.checkpoint.CheckpointV2; +import org.apache.samza.checkpoint.kafka.KafkaChangelogSSPOffset; +import org.apache.samza.config.Config; +import org.apache.samza.container.TaskInstanceMetrics; +import org.apache.samza.container.TaskName; +import org.apache.samza.job.model.TaskMode; +import org.apache.samza.system.SystemStream; +import org.apache.samza.system.SystemStreamPartition; +import org.apache.samza.util.FutureUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Handles the commit of the state stores of the task. + */ +public class TaskStorageCommitManager { + + private static final Logger LOG = LoggerFactory.getLogger(TaskStorageCommitManager.class); + + private final TaskName taskName; + private final CheckpointManager checkpointManager; + private final ContainerStorageManager containerStorageManager; + private final Map stateBackendToBackupManager; + private final Partition taskChangelogPartition; + private final StorageManagerUtil storageManagerUtil; + private final ExecutorService backupExecutor; + private final File durableStoreBaseDir; + private final Map storeChangelogs; + private final TaskInstanceMetrics metrics; + + // Available after init(), since stores are created by ContainerStorageManager#start() + private Map storageEngines; + + public TaskStorageCommitManager(TaskName taskName, Map stateBackendToBackupManager, + ContainerStorageManager containerStorageManager, Map storeChangelogs, Partition changelogPartition, + CheckpointManager checkpointManager, Config config, ExecutorService backupExecutor, + StorageManagerUtil storageManagerUtil, File durableStoreBaseDir, TaskInstanceMetrics metrics) { + this.taskName = taskName; + this.containerStorageManager = containerStorageManager; + this.stateBackendToBackupManager = stateBackendToBackupManager; + this.taskChangelogPartition = changelogPartition; + this.checkpointManager = checkpointManager; + this.backupExecutor = backupExecutor; + this.durableStoreBaseDir = durableStoreBaseDir; + this.storeChangelogs = storeChangelogs; + this.storageManagerUtil = storageManagerUtil; + this.metrics = metrics; + } + + public void init() { + // Assuming that container storage manager has already started and created to stores + storageEngines = containerStorageManager.getAllStores(taskName); + if (checkpointManager != null) { + Checkpoint checkpoint = checkpointManager.readLastCheckpoint(taskName); + LOG.debug("Last checkpoint on start for task: {} is: {}", taskName, checkpoint); + stateBackendToBackupManager.values() + .forEach(storageBackupManager -> storageBackupManager.init(checkpoint)); + } else { + stateBackendToBackupManager.values() + .forEach(storageBackupManager -> storageBackupManager.init(null)); + } + } + + /** + * Synchronously captures the current state of the stores in order to persist it to the backup manager + * in the async {@link #upload(CheckpointId, Map)} phase. Returns a map of state backend factory name to + * a map of store name to state checkpoint markers for all configured state backends and stores. + * + * @param checkpointId {@link CheckpointId} of the current commit + * @return a map of state backend factory name to a map of store name to state checkpoint markers + */ + public Map> snapshot(CheckpointId checkpointId) { + // Flush all stores + storageEngines.values().forEach(StorageEngine::flush); + LOG.debug("Flushed all storage engines for taskName: {}, checkpoint id: {}", + taskName, checkpointId); + + + // Checkpoint all persisted and durable stores + long checkpointStartNs = System.nanoTime(); + storageEngines.forEach((storeName, storageEngine) -> { + if (storageEngine.getStoreProperties().isPersistedToDisk() && + storageEngine.getStoreProperties().isDurableStore()) { + storageEngine.checkpoint(checkpointId); + } + }); + long checkpointNs = System.nanoTime() - checkpointStartNs; + metrics.storeCheckpointNs().update(checkpointNs); + LOG.debug("Checkpointed all storage engines for taskName: {}, checkpoint id: {} in {} ns", + taskName, checkpointId, checkpointNs); + + // state backend factory -> store Name -> state checkpoint marker + Map> stateBackendToStoreSCMs = new HashMap<>(); + + // for each configured state backend factory, backup the state for all stores in this task. + stateBackendToBackupManager.forEach((stateBackendFactoryName, backupManager) -> { + Map snapshotSCMs = backupManager.snapshot(checkpointId); + LOG.debug("Created snapshot for taskName: {}, checkpoint id: {}, state backend: {}. Snapshot SCMs: {}", + taskName, checkpointId, stateBackendFactoryName, snapshotSCMs); + stateBackendToStoreSCMs.put(stateBackendFactoryName, snapshotSCMs); + }); + + return stateBackendToStoreSCMs; + } + + /** + * Asynchronously backs up the local state to the remote storage and returns a future containing the committed + * map of state backend factory name to the map of store name to state checkpoint marker. + * + * @param checkpointId the {@link CheckpointId} associated with this commit + * @return a future containing the Map of FactoryName to (Map of StoreName to StateCheckpointMarker). + */ + public CompletableFuture>> upload( + CheckpointId checkpointId, Map> snapshotSCMs) { + // state backend factory -> store Name -> state checkpoint marker + Map>> stateBackendToStoreSCMs = new HashMap<>(); + + // for each configured state backend factory, backup the state for all stores in this task. + stateBackendToBackupManager.forEach((stateBackendFactoryName, backupManager) -> { + try { + Map factorySnapshotSCMs = + snapshotSCMs.getOrDefault(stateBackendFactoryName, Collections.emptyMap()); + LOG.debug("Starting upload for taskName: {}, checkpoint id: {}, state backend snapshot SCM: {}", + taskName, checkpointId, factorySnapshotSCMs); + + CompletableFuture> uploadFuture = + backupManager.upload(checkpointId, factorySnapshotSCMs); + uploadFuture.thenAccept(uploadSCMs -> + LOG.debug("Finished upload for taskName: {}, checkpoint id: {}, state backend: {}. Upload SCMs: {}", + taskName, checkpointId, stateBackendFactoryName, uploadSCMs)); + + stateBackendToStoreSCMs.put(stateBackendFactoryName, uploadFuture); + } catch (Exception e) { + throw new SamzaException( + String.format("Error backing up local state for taskName: %s, checkpoint id: %s, state backend: %s", + taskName, checkpointId, stateBackendFactoryName), e); + } + }); + + return FutureUtil.toFutureOfMap(stateBackendToStoreSCMs); + } + + /** + * Writes the {@link Checkpoint} information returned by {@link #upload(CheckpointId, Map)} + * in each store directory and store checkpoint directory. Written content depends on the type of {@code checkpoint}. + * For {@link CheckpointV2}, writes the entire task {@link CheckpointV2}. + * For {@link CheckpointV1}, only writes the changelog ssp offsets in the OFFSET* files. + * + * Note: The assumption is that this method will be invoked once for each {@link Checkpoint} version that the + * task needs to write as determined by {@link org.apache.samza.config.TaskConfig#getCheckpointWriteVersions()}. + * This is required for upgrade and rollback compatibility. + * + * @param checkpoint the latest checkpoint to be persisted to local file system + */ + public void writeCheckpointToStoreDirectories(Checkpoint checkpoint) { + if (checkpoint instanceof CheckpointV1) { + LOG.debug("Writing CheckpointV1 to store and checkpoint directories for taskName: {} with checkpoint: {}", + taskName, checkpoint); + // Write CheckpointV1 changelog offsets to store and checkpoint directories + writeChangelogOffsetFiles(checkpoint.getOffsets()); + } else if (checkpoint instanceof CheckpointV2) { + LOG.debug("Writing CheckpointV2 to store and checkpoint directories for taskName: {} with checkpoint: {}", + taskName, checkpoint); + storageEngines.forEach((storeName, storageEngine) -> { + // Only write the checkpoint file if the store is durable and persisted to disk + if (storageEngine.getStoreProperties().isDurableStore() && + storageEngine.getStoreProperties().isPersistedToDisk()) { + CheckpointV2 checkpointV2 = (CheckpointV2) checkpoint; + + try { + File storeDir = storageManagerUtil.getTaskStoreDir(durableStoreBaseDir, storeName, taskName, TaskMode.Active); + storageManagerUtil.writeCheckpointV2File(storeDir, checkpointV2); + + CheckpointId checkpointId = checkpointV2.getCheckpointId(); + File checkpointDir = Paths.get(storageManagerUtil.getStoreCheckpointDir(storeDir, checkpointId)).toFile(); + storageManagerUtil.writeCheckpointV2File(checkpointDir, checkpointV2); + } catch (Exception e) { + throw new SamzaException( + String.format("Write checkpoint file failed for task: %s, storeName: %s, checkpointId: %s", + taskName, storeName, ((CheckpointV2) checkpoint).getCheckpointId()), e); + } + } + }); + } else { + throw new SamzaException("Unsupported checkpoint version: " + checkpoint.getVersion()); + } + } + + /** + * Performs any post-commit and cleanup actions after the {@link Checkpoint} is successfully written to the + * checkpoint topic. Invokes {@link TaskBackupManager#cleanUp(CheckpointId, Map)} on each of the configured task + * backup managers. Deletes all local store checkpoint directories older than the {@code latestCheckpointId}. + * + * @param latestCheckpointId CheckpointId of the most recent successful commit + * @param stateCheckpointMarkers map of map(stateBackendFactoryName to map(storeName to state checkpoint markers) from + * the latest commit + */ + public CompletableFuture cleanUp(CheckpointId latestCheckpointId, + Map> stateCheckpointMarkers) { + List> cleanUpFutures = new ArrayList<>(); + + // Call cleanup on each backup manager + stateCheckpointMarkers.forEach((factoryName, storeSCMs) -> { + if (stateBackendToBackupManager.containsKey(factoryName)) { + LOG.debug("Cleaning up commit for factory: {} for task: {}", factoryName, taskName); + TaskBackupManager backupManager = stateBackendToBackupManager.get(factoryName); + cleanUpFutures.add(backupManager.cleanUp(latestCheckpointId, storeSCMs)); + } else { + // This may happen during migration from one state backend to another, where the latest commit contains + // a state backend that is no longer supported for the current commit manager + LOG.warn("Ignored cleanup for scm: {} due to unknown factory: {} ", storeSCMs, factoryName); + } + }); + + return FutureUtil.allOf(cleanUpFutures) + .thenAcceptAsync(aVoid -> deleteOldCheckpointDirs(latestCheckpointId), backupExecutor); + } + + private void deleteOldCheckpointDirs(CheckpointId latestCheckpointId) { + // Delete directories for checkpoints older than latestCheckpointId + if (latestCheckpointId != null) { + LOG.debug("Deleting checkpoints older than checkpoint id: {}", latestCheckpointId); + File[] files = durableStoreBaseDir.listFiles(); + if (files != null) { + for (File storeDir : files) { + String storeName = storeDir.getName(); + String taskStoreName = storageManagerUtil + .getTaskStoreDir(durableStoreBaseDir, storeName, taskName, TaskMode.Active).getName(); + FileFilter fileFilter = new WildcardFileFilter(taskStoreName + "-*"); + File[] checkpointDirs = storeDir.listFiles(fileFilter); + if (checkpointDirs != null) { + for (File checkpointDir : checkpointDirs) { + if (!checkpointDir.getName().contains(latestCheckpointId.serialize())) { + try { + FileUtils.deleteDirectory(checkpointDir); + } catch (IOException e) { + throw new SamzaException( + String.format("Unable to delete checkpoint directory: %s", checkpointDir.getName()), e); + } + } + } + } + } + } + } + } + + /** + * Close all the state backup managers + */ + public void close() { + LOG.debug("Stopping backup managers for task {}.", taskName); + stateBackendToBackupManager.values().forEach(storageBackupManager -> { + if (storageBackupManager != null) { + storageBackupManager.close(); + } + }); + } + + /** + * Writes the newest changelog ssp offset for each logged and persistent store to the OFFSET file in the current + * store directory (for allowing rollbacks). If the Kafka transactional backup manager is enabled, also writes to + * the store checkpoint directory. + * + * These files are used during container startup to ensure transactional state, and to determine whether the + * there is any new information in the changelog that is not reflected in the on-disk copy of the store. + * If there is any delta, it is replayed from the changelog. E.g. this can happen if the job was run on this host, + * then another host, and then back to this host. + */ + @VisibleForTesting + void writeChangelogOffsetFiles(Map checkpointOffsets) { + if (storageEngines == null) { + throw new SamzaException(String.format( + "Storage engines are not initialized and writeChangelogOffsetFiles not be written for task %s", taskName)); + } + storeChangelogs.forEach((storeName, systemStream) -> { + SystemStreamPartition changelogSSP = new SystemStreamPartition( + systemStream.getSystem(), systemStream.getStream(), taskChangelogPartition); + + // Only write if the store is durable and persisted to disk + if (checkpointOffsets.containsKey(changelogSSP) && + storageEngines.containsKey(storeName) && + storageEngines.get(storeName).getStoreProperties().isDurableStore() && + storageEngines.get(storeName).getStoreProperties().isPersistedToDisk()) { + LOG.debug("Writing changelog offset for taskName {} store {} changelog {}.", taskName, storeName, systemStream); + File currentStoreDir = storageManagerUtil.getTaskStoreDir(durableStoreBaseDir, storeName, taskName, TaskMode.Active); + try { + KafkaChangelogSSPOffset kafkaChangelogSSPOffset = KafkaChangelogSSPOffset + .fromString(checkpointOffsets.get(changelogSSP)); + // Write offsets to file system if it is non-null + String newestOffset = kafkaChangelogSSPOffset.getChangelogOffset(); + if (newestOffset != null) { + // Write changelog SSP offset to the OFFSET files in the task store directory + writeChangelogOffsetFile(storeName, changelogSSP, newestOffset, currentStoreDir); + + // Write changelog SSP offset to the OFFSET files in the store checkpoint directory + File checkpointDir = Paths.get(storageManagerUtil.getStoreCheckpointDir( + currentStoreDir, kafkaChangelogSSPOffset.getCheckpointId())).toFile(); + writeChangelogOffsetFile(storeName, changelogSSP, newestOffset, checkpointDir); + } else { + // If newestOffset is null, then it means the changelog ssp is (or has become) empty. This could be + // either because the changelog topic was newly added, repartitioned, or manually deleted and recreated. + // No need to persist the offset file. + LOG.debug("Deleting OFFSET file for taskName {} store {} changelog ssp {} since the newestOffset is null.", + taskName, storeName, changelogSSP); + storageManagerUtil.deleteOffsetFile(currentStoreDir); + } + } catch (IOException e) { + throw new SamzaException( + String.format("Error storing offset for taskName %s store %s changelog %s.", taskName, storeName, + systemStream), e); + } + } + }); + LOG.debug("Done writing OFFSET files for logged persistent key value stores for task {}", taskName); + } + + @VisibleForTesting + void writeChangelogOffsetFile(String storeName, SystemStreamPartition ssp, String newestOffset, + File writeDirectory) throws IOException { + LOG.debug("Storing newest offset {} for taskName {} store {} changelog ssp {} in OFFSET file at path: {}.", + newestOffset, taskName, storeName, ssp, writeDirectory); + storageManagerUtil.writeOffsetFile(writeDirectory, Collections.singletonMap(ssp, newestOffset), false); + } +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/TransactionalStateTaskRestoreManager.java b/samza-core/src/main/java/org/apache/samza/storage/TransactionalStateTaskRestoreManager.java index 4b6ac1f2a1..f33bb5b34f 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/TransactionalStateTaskRestoreManager.java +++ b/samza-core/src/main/java/org/apache/samza/storage/TransactionalStateTaskRestoreManager.java @@ -24,31 +24,41 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ListMultimap; - import java.io.File; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.apache.samza.Partition; import org.apache.samza.SamzaException; -import org.apache.samza.checkpoint.CheckpointedChangelogOffset; +import org.apache.samza.checkpoint.Checkpoint; +import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.checkpoint.CheckpointV1; +import org.apache.samza.checkpoint.CheckpointV2; +import org.apache.samza.checkpoint.kafka.KafkaChangelogSSPOffset; +import org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker; import org.apache.samza.config.Config; import org.apache.samza.config.StorageConfig; import org.apache.samza.config.TaskConfig; import org.apache.samza.container.TaskName; +import org.apache.samza.context.ContainerContext; +import org.apache.samza.context.JobContext; import org.apache.samza.job.model.TaskMode; import org.apache.samza.job.model.TaskModel; +import org.apache.samza.metrics.MetricsRegistry; +import org.apache.samza.serializers.Serde; import org.apache.samza.system.ChangelogSSPIterator; import org.apache.samza.system.SSPMetadataCache; import org.apache.samza.system.SystemAdmin; import org.apache.samza.system.SystemAdmins; import org.apache.samza.system.SystemConsumer; import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamPartition; import org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata; +import org.apache.samza.system.SystemStreamPartition; +import org.apache.samza.task.MessageCollector; import org.apache.samza.util.Clock; import org.apache.samza.util.FileUtil; import org.slf4j.Logger; @@ -64,7 +74,7 @@ public class TransactionalStateTaskRestoreManager implements TaskRestoreManager private final Map storeEngines; // store name to storage engines private final Map storeChangelogs; // store name to changelog system stream private final SystemAdmins systemAdmins; - private final Map storeConsumers; + private final Map storeConsumers; // store name to system consumer private final SSPMetadataCache sspMetadataCache; private final File loggedStoreBaseDirectory; private final File nonLoggedStoreBaseDirectory; @@ -77,18 +87,24 @@ public class TransactionalStateTaskRestoreManager implements TaskRestoreManager private Map currentChangelogOffsets; public TransactionalStateTaskRestoreManager( + Set storeNames, // non-side input stores + JobContext jobContext, + ContainerContext containerContext, TaskModel taskModel, - Map storeEngines, Map storeChangelogs, + Map inMemoryStores, // in memory stores to be mutated during restore + Map> storageEngineFactories, + Map> serdes, SystemAdmins systemAdmins, Map storeConsumers, + MetricsRegistry metricsRegistry, + MessageCollector messageCollector, SSPMetadataCache sspMetadataCache, File loggedStoreBaseDirectory, File nonLoggedStoreBaseDirectory, Config config, Clock clock) { this.taskModel = taskModel; - this.storeEngines = storeEngines; this.storeChangelogs = storeChangelogs; this.systemAdmins = systemAdmins; this.storeConsumers = storeConsumers; @@ -101,14 +117,17 @@ public TransactionalStateTaskRestoreManager( this.clock = clock; this.storageManagerUtil = new StorageManagerUtil(); this.fileUtil = new FileUtil(); + this.storeEngines = createStoreEngines(storeNames, jobContext, containerContext, + storageEngineFactories, serdes, metricsRegistry, messageCollector, inMemoryStores); } @Override - public void init(Map checkpointedChangelogOffsets) { + public void init(Checkpoint checkpoint) { + Map storeStateCheckpointMarkers = getCheckpointedChangelogOffsets(checkpoint); currentChangelogOffsets = getCurrentChangelogOffsets(taskModel, storeChangelogs, sspMetadataCache); this.storeActions = getStoreActions(taskModel, storeEngines, storeChangelogs, - checkpointedChangelogOffsets, currentChangelogOffsets, systemAdmins, storageManagerUtil, + storeStateCheckpointMarkers, getCheckpointId(checkpoint), currentChangelogOffsets, systemAdmins, storageManagerUtil, loggedStoreBaseDirectory, nonLoggedStoreBaseDirectory, config, clock); setupStoreDirs(taskModel, storeEngines, storeActions, storageManagerUtil, fileUtil, @@ -143,7 +162,7 @@ public void restore() throws InterruptedException { * Stop only persistent stores. In case of certain stores and store mode (such as RocksDB), this * can invoke compaction. Persisted stores are recreated in read-write mode in {@link ContainerStorageManager}. */ - public void stopPersistentStores() { + public void close() { TaskName taskName = taskModel.getTaskName(); storeEngines.forEach((storeName, storeEngine) -> { if (storeEngine.getStoreProperties().isPersistedToDisk()) @@ -152,6 +171,27 @@ public void stopPersistentStores() { }); } + private Map createStoreEngines(Set storeNames, JobContext jobContext, + ContainerContext containerContext, Map> storageEngineFactories, + Map> serdes, MetricsRegistry metricsRegistry, + MessageCollector messageCollector, Map nonPersistedStores) { + Map storageEngines = new HashMap<>(); + // Put non persisted stores + nonPersistedStores.forEach(storageEngines::put); + // Create persisted stores + storeNames.forEach(storeName -> { + boolean isLogged = this.storeChangelogs.containsKey(storeName); + File storeBaseDir = isLogged ? this.loggedStoreBaseDirectory : this.nonLoggedStoreBaseDirectory; + File storeDirectory = storageManagerUtil.getTaskStoreDir(storeBaseDir, storeName, taskModel.getTaskName(), + taskModel.getTaskMode()); + StorageEngine engine = ContainerStorageManager.createStore(storeName, storeDirectory, taskModel, jobContext, containerContext, + storageEngineFactories, serdes, metricsRegistry, messageCollector, + StorageEngineFactory.StoreMode.BulkLoad, this.storeChangelogs, this.config); + storageEngines.put(storeName, engine); + }); + return storageEngines; + } + /** * Get offset metadata for each changelog SSP for this task. A task may have multiple changelog streams * (e.g., for different stores), but will have the same partition for all of them. @@ -194,7 +234,8 @@ static StoreActions getStoreActions( TaskModel taskModel, Map storeEngines, Map storeChangelogs, - Map checkpointedChangelogOffsets, + Map kafkaStateCheckpointMarkers, + CheckpointId checkpointId, Map currentChangelogOffsets, SystemAdmins systemAdmins, StorageManagerUtil storageManagerUtil, @@ -236,19 +277,16 @@ static StoreActions getStoreActions( String oldestOffset = changelogSSPMetadata.getOldestOffset(); String newestOffset = changelogSSPMetadata.getNewestOffset(); - String checkpointMessage = checkpointedChangelogOffsets.get(changelogSSP); - String checkpointedOffset = null; // can be null if no message, or message has null offset - long timeSinceLastCheckpointInMs = Long.MAX_VALUE; - if (StringUtils.isNotBlank(checkpointMessage)) { - CheckpointedChangelogOffset checkpointedChangelogOffset = CheckpointedChangelogOffset.fromString(checkpointMessage); - checkpointedOffset = checkpointedChangelogOffset.getOffset(); - timeSinceLastCheckpointInMs = System.currentTimeMillis() - - checkpointedChangelogOffset.getCheckpointId().getMillis(); + String checkpointedOffset = null; // can be null if no message, or message has null offset + if (kafkaStateCheckpointMarkers.containsKey(storeName) && + StringUtils.isNotBlank(kafkaStateCheckpointMarkers.get(storeName).getChangelogOffset())) { + checkpointedOffset = kafkaStateCheckpointMarkers.get(storeName).getChangelogOffset(); } + long timeSinceLastCheckpointInMs = checkpointId == null ? Long.MAX_VALUE : System.currentTimeMillis() - checkpointId.getMillis(); // if the clean.store.start config is set, delete current and checkpoint dirs, restore from oldest offset to checkpointed if (storageEngine.getStoreProperties().isPersistedToDisk() && new StorageConfig( - config).getCleanLoggedStoreDirsOnStart(storeName)) { + config).cleanLoggedStoreDirsOnStart(storeName)) { File currentDir = storageManagerUtil.getTaskStoreDir(loggedStoreBaseDirectory, storeName, taskName, taskMode); LOG.info("Marking current directory: {} for store: {} in task: {} for deletion due to clean.on.container.start config.", currentDir, storeName, taskName); @@ -557,6 +595,60 @@ private static void validateRestoreOffsets(RestoreOffsets restoreOffsets, System } } + private Map getCheckpointedChangelogOffsets(Checkpoint checkpoint) { + Map checkpointedChangelogOffsets = new HashMap<>(); + if (checkpoint == null) return checkpointedChangelogOffsets; + + if (checkpoint instanceof CheckpointV2) { + Map> factoryStoreSCMs = ((CheckpointV2) checkpoint).getStateCheckpointMarkers(); + if (factoryStoreSCMs.containsKey(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME)) { + factoryStoreSCMs.get(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME) + .forEach((storeName, scmString) -> { + KafkaStateCheckpointMarker kafkaSCM = KafkaStateCheckpointMarker.deserialize(scmString); + checkpointedChangelogOffsets.put(storeName, kafkaSCM); + }); + } // skip the non-KafkaStateCheckpointMarkers + } else if (checkpoint instanceof CheckpointV1) { + // If the checkpoint v1 is used, we need to fetch the changelog SSPs in the inputOffsets in order to get the + // store offset. + Map checkpointedOffsets = checkpoint.getOffsets(); + storeChangelogs.forEach((storeName, systemStream) -> { + Partition changelogPartition = taskModel.getChangelogPartition(); + SystemStreamPartition storeChangelogSSP = new SystemStreamPartition(systemStream, changelogPartition); + String checkpointedOffset = checkpointedOffsets.get(storeChangelogSSP); + if (StringUtils.isNotBlank(checkpointedOffset)) { + KafkaChangelogSSPOffset kafkaChangelogSSPOffset = KafkaChangelogSSPOffset.fromString(checkpointedOffset); + KafkaStateCheckpointMarker marker = new KafkaStateCheckpointMarker( + storeChangelogSSP, kafkaChangelogSSPOffset.getChangelogOffset()); + checkpointedChangelogOffsets.put(storeName, marker); + } + }); + } else { + throw new SamzaException("Unsupported checkpoint version: " + checkpoint.getVersion()); + } + + return checkpointedChangelogOffsets; + } + + private CheckpointId getCheckpointId(Checkpoint checkpoint) { + if (checkpoint == null) return null; + if (checkpoint instanceof CheckpointV1) { + for (Map.Entry storeNameSystemStream : storeChangelogs.entrySet()) { + SystemStreamPartition storeChangelogSSP = new SystemStreamPartition(storeNameSystemStream.getValue(), taskModel.getChangelogPartition()); + String checkpointMessage = checkpoint.getOffsets().get(storeChangelogSSP); + if (StringUtils.isNotBlank(checkpointMessage)) { + KafkaChangelogSSPOffset kafkaStateChanglogOffset = KafkaChangelogSSPOffset.fromString(checkpointMessage); + return kafkaStateChanglogOffset.getCheckpointId(); + } + } + } else if (checkpoint instanceof CheckpointV2) { + return ((CheckpointV2) checkpoint).getCheckpointId(); + } else { + throw new SamzaException("Unsupported checkpoint version: " + checkpoint.getVersion()); + } + return null; + } + @VisibleForTesting static class StoreActions { final Map storeDirsToRetain; diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreBackupManager.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreBackupManager.java new file mode 100644 index 0000000000..fb04ce5d67 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreBackupManager.java @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.samza.SamzaException; +import org.apache.samza.checkpoint.Checkpoint; +import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.config.Config; +import org.apache.samza.config.JobConfig; +import org.apache.samza.config.StorageConfig; +import org.apache.samza.job.model.ContainerModel; +import org.apache.samza.job.model.JobModel; +import org.apache.samza.job.model.TaskModel; +import org.apache.samza.storage.StorageManagerUtil; +import org.apache.samza.storage.TaskBackupManager; +import org.apache.samza.storage.blobstore.diff.DirDiff; +import org.apache.samza.storage.blobstore.index.DirIndex; +import org.apache.samza.storage.blobstore.index.SnapshotIndex; +import org.apache.samza.storage.blobstore.index.SnapshotMetadata; +import org.apache.samza.storage.blobstore.metrics.BlobStoreBackupManagerMetrics; +import org.apache.samza.storage.blobstore.util.BlobStoreUtil; +import org.apache.samza.storage.blobstore.util.DirDiffUtil; +import org.apache.samza.util.Clock; +import org.apache.samza.util.FutureUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class BlobStoreBackupManager implements TaskBackupManager { + private static final Logger LOG = LoggerFactory.getLogger(BlobStoreBackupManager.class); + + private final JobModel jobModel; + private final ExecutorService executor; + private final String jobName; + private final String jobId; + private final ContainerModel containerModel; + private final TaskModel taskModel; + private final String taskName; + private final Config config; + private final Clock clock; + private final StorageManagerUtil storageManagerUtil; + private final List storesToBackup; + private final File loggedStoreBaseDir; + private final BlobStoreManager blobStoreManager; + private final BlobStoreUtil blobStoreUtil; + private final BlobStoreBackupManagerMetrics metrics; + + /** + * Map of store name to a Pair of blob id of {@link SnapshotIndex} and the corresponding {@link SnapshotIndex} from + * last successful task checkpoint or {@link #upload}. + * + * After {@link #init}, the map reflects the contents of the last completed checkpoint for the task from the previous + * deployment, if any. + * + * During regular processing, this map is updated after each successful {@link #upload} with the blob id of + * {@link SnapshotIndex} and the corresponding {@link SnapshotIndex} of the upload. + * + * The contents of this map are used to calculate the diff for local state between the last and the current checkpoint + * during {@link #upload}. + * + * Since the task commit process guarantees that the async stage of the previous commit is complete before another + * commit can start, this future is guaranteed to be complete in the call to {@link #upload} during the next commit. + * + * This field is non-final, since the future itself is replaced in its entirety after init/upload. + * The internal map contents are never directly modified (e.g. using puts). It's volatile to ensure visibility + * across threads since the map assignment may happen on a different thread than the one reading the contents. + */ + private volatile CompletableFuture>> + prevStoreSnapshotIndexesFuture; + + public BlobStoreBackupManager(JobModel jobModel, ContainerModel containerModel, TaskModel taskModel, + ExecutorService backupExecutor, BlobStoreBackupManagerMetrics blobStoreTaskBackupMetrics, Config config, + Clock clock, File loggedStoreBaseDir, StorageManagerUtil storageManagerUtil, BlobStoreManager blobStoreManager) { + this.jobModel = jobModel; + this.jobName = new JobConfig(config).getName().get(); + this.jobId = new JobConfig(config).getJobId(); + this.containerModel = containerModel; + this.taskModel = taskModel; + this.taskName = taskModel.getTaskName().getTaskName(); + this.executor = backupExecutor; + this.config = config; + this.clock = clock; + this.storageManagerUtil = storageManagerUtil; + StorageConfig storageConfig = new StorageConfig(config); + this.storesToBackup = + storageConfig.getPersistentStoresWithBackupFactory(BlobStoreStateBackendFactory.class.getName()); + this.loggedStoreBaseDir = loggedStoreBaseDir; + this.blobStoreManager = blobStoreManager; + this.blobStoreUtil = createBlobStoreUtil(blobStoreManager, executor, blobStoreTaskBackupMetrics); + this.prevStoreSnapshotIndexesFuture = CompletableFuture.completedFuture(ImmutableMap.of()); + this.metrics = blobStoreTaskBackupMetrics; + metrics.initStoreMetrics(storesToBackup); + } + + @Override + public void init(Checkpoint checkpoint) { + long startTime = System.nanoTime(); + LOG.debug("Initializing blob store backup manager for task: {}", taskName); + + blobStoreManager.init(); + + // Note: blocks the caller thread. + // TODO LOW shesharma exclude stores that are no longer configured during init + Map> prevStoreSnapshotIndexes = + blobStoreUtil.getStoreSnapshotIndexes(jobName, jobId, taskName, checkpoint); + this.prevStoreSnapshotIndexesFuture = + CompletableFuture.completedFuture(ImmutableMap.copyOf(prevStoreSnapshotIndexes)); + metrics.initNs.set(System.nanoTime() - startTime); + } + + @Override + public Map snapshot(CheckpointId checkpointId) { + // No-op. Stores are flushed and checkpoints are created by commit manager + return Collections.emptyMap(); + } + + @Override + public CompletableFuture> upload(CheckpointId checkpointId, Map storeSCMs) { + long uploadStartTime = System.nanoTime(); + + // reset gauges for each upload + metrics.filesToUpload.getValue().set(0L); + metrics.bytesToUpload.getValue().set(0L); + metrics.filesUploaded.getValue().set(0L); + metrics.bytesUploaded.getValue().set(0L); + metrics.filesRemaining.getValue().set(0L); + metrics.bytesRemaining.getValue().set(0L); + metrics.filesToRetain.getValue().set(0L); + metrics.bytesToRetain.getValue().set(0L); + + // This map is used to atomically replace the prevStoreSnapshotIndexesFuture map at the end of the task commit + Map>> + storeToSCMAndSnapshotIndexPairFutures = new HashMap<>(); + // This map is used to return serialized State Checkpoint Markers to the caller + Map> storeToSerializedSCMFuture = new HashMap<>(); + + storesToBackup.forEach((storeName) -> { + long storeUploadStartTime = System.nanoTime(); + try { + // metadata for the current store snapshot to upload + SnapshotMetadata snapshotMetadata = new SnapshotMetadata(checkpointId, jobName, jobId, taskName, storeName); + + // get the local store dir corresponding to the current checkpointId + File storeDir = storageManagerUtil.getTaskStoreDir(loggedStoreBaseDir, storeName, + taskModel.getTaskName(), taskModel.getTaskMode()); + String checkpointDirPath = storageManagerUtil.getStoreCheckpointDir(storeDir, checkpointId); + File checkpointDir = new File(checkpointDirPath); + + LOG.debug("Got task: {} store: {} storeDir: {} and checkpointDir: {}", + taskName, storeName, storeDir, checkpointDir); + + // guaranteed to be available since a new task commit may not start until the previous one is complete + Map> prevStoreSnapshotIndexes = + prevStoreSnapshotIndexesFuture.get(0, TimeUnit.MILLISECONDS); + + // get the previous store directory contents + DirIndex prevDirIndex; + + if (prevStoreSnapshotIndexes.containsKey(storeName)) { + prevDirIndex = prevStoreSnapshotIndexes.get(storeName).getRight().getDirIndex(); + } else { + // no previous SnapshotIndex means that this is the first commit for this store. Create an empty DirIndex. + prevDirIndex = new DirIndex(checkpointDir.getName(), Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), Collections.emptyList()); + } + + long dirDiffStartTime = System.nanoTime(); + // get the diff between previous and current store directories + DirDiff dirDiff = DirDiffUtil.getDirDiff(checkpointDir, prevDirIndex, DirDiffUtil.areSameFile(false)); + metrics.storeDirDiffNs.get(storeName).update(System.nanoTime() - dirDiffStartTime); + + DirDiff.Stats stats = DirDiff.getStats(dirDiff); + updateStoreDiffMetrics(storeName, stats); + metrics.filesToUpload.getValue().addAndGet(stats.filesAdded); + metrics.bytesToUpload.getValue().addAndGet(stats.bytesAdded); + // Note: FilesRemaining metric is set to FilesAdded in the beginning of the current upload and then counted down + // for each upload. + metrics.filesRemaining.getValue().addAndGet(stats.filesAdded); + metrics.bytesRemaining.getValue().addAndGet(stats.bytesAdded); + metrics.filesToRetain.getValue().addAndGet(stats.filesRetained); + metrics.bytesToRetain.getValue().addAndGet(stats.bytesRetained); + + // upload the diff to the blob store and get the new directory index + CompletionStage dirIndexFuture = blobStoreUtil.putDir(dirDiff, snapshotMetadata); + + CompletionStage snapshotIndexFuture = + dirIndexFuture.thenApplyAsync(dirIndex -> { + LOG.trace("Dir upload complete. Returning new SnapshotIndex for task: {} store: {}.", taskName, storeName); + Optional prevSnapshotIndexBlobId = + Optional.ofNullable(prevStoreSnapshotIndexes.get(storeName)).map(Pair::getLeft); + return new SnapshotIndex(clock.currentTimeMillis(), snapshotMetadata, dirIndex, prevSnapshotIndexBlobId); + }, executor); + + // upload the new snapshot index to the blob store and get its blob id + CompletionStage snapshotIndexBlobIdFuture = + snapshotIndexFuture + .thenComposeAsync(si -> { + LOG.trace("Uploading Snapshot index for task: {} store: {}", taskName, storeName); + return blobStoreUtil.putSnapshotIndex(si); + }, executor); + + // save store name and it's SnapshotIndex blob id and SnapshotIndex pair. At the end of the upload, atomically + // update previous snapshot index map with this. + CompletableFuture> scmAndSnapshotIndexPairFuture = + FutureUtil.toFutureOfPair( + Pair.of(snapshotIndexBlobIdFuture.toCompletableFuture(), snapshotIndexFuture.toCompletableFuture())); + + scmAndSnapshotIndexPairFuture.whenComplete((res, ex) -> { + long uploadTimeNs = System.nanoTime() - storeUploadStartTime; + metrics.storeUploadNs.get(storeName).update(uploadTimeNs); + }); + + storeToSCMAndSnapshotIndexPairFutures.put(storeName, scmAndSnapshotIndexPairFuture); + storeToSerializedSCMFuture.put(storeName, snapshotIndexBlobIdFuture.toCompletableFuture()); + } catch (Exception e) { + throw new SamzaException( + String.format("Error uploading store snapshot to blob store for task: %s, store: %s, checkpointId: %s", + taskName, storeName, checkpointId), e); + } + }); + + // replace the previous storeName to snapshot index mapping with the new mapping. + this.prevStoreSnapshotIndexesFuture = + FutureUtil.toFutureOfMap(storeToSCMAndSnapshotIndexPairFutures); + + return FutureUtil.toFutureOfMap(storeToSerializedSCMFuture) + .whenComplete((res, ex) -> metrics.uploadNs.update(System.nanoTime() - uploadStartTime)); + } + + /** + * Clean up would be called at the end of every commit as well as on a container start/restart. + * Clean up involves the following steps: + * 1. Remove TTL of the snapshot index blob and for any associated files and sub-dirs marked for retention. + * 2. Delete the files/subdirs marked for deletion in the snapshot index. + * 3. Delete the remote {@link SnapshotIndex} blob for the previous checkpoint. + * @param checkpointId the {@link CheckpointId} of the last successfully committed checkpoint. + * @param storeSCMs store name to state checkpoint markers for the last successfully committed checkpoint + */ + @Override + public CompletableFuture cleanUp(CheckpointId checkpointId, Map storeSCMs) { + long startTime = System.nanoTime(); + List> removeTTLFutures = new ArrayList<>(); + List> cleanupRemoteSnapshotFutures = new ArrayList<>(); + List> removePrevRemoteSnapshotFutures = new ArrayList<>(); + + // SCM, in case of blob store backup and restore, is just the blob id of SnapshotIndex representing the remote snapshot + storeSCMs.forEach((storeName, snapshotIndexBlobId) -> { + // Only perform cleanup for stores configured with BlobStore State Backend Factory + if (storesToBackup.contains(storeName)) { + Metadata requestMetadata = + new Metadata(Metadata.SNAPSHOT_INDEX_PAYLOAD_PATH, Optional.empty(), jobName, jobId, taskName, storeName); + CompletionStage snapshotIndexFuture = + blobStoreUtil.getSnapshotIndex(snapshotIndexBlobId, requestMetadata); + + // 1. remove TTL of index blob and all of its files and sub-dirs marked for retention + CompletionStage removeTTLFuture = + snapshotIndexFuture.thenComposeAsync(snapshotIndex -> { + LOG.debug("Removing TTL for index blob: {} and all of its files and sub-dirs for task: {} store :{}", + snapshotIndexBlobId, taskName, storeName); + return blobStoreUtil.removeTTL(snapshotIndexBlobId, snapshotIndex, requestMetadata); + }, executor); + removeTTLFutures.add(removeTTLFuture); + + // 2. delete the files/subdirs marked for deletion in the snapshot index. + CompletionStage cleanupRemoteSnapshotFuture = + snapshotIndexFuture.thenComposeAsync(snapshotIndex -> { + LOG.debug("Deleting files and dirs to remove for current index blob: {} for task: {} store: {}", + snapshotIndexBlobId, taskName, storeName); + return blobStoreUtil.cleanUpDir(snapshotIndex.getDirIndex(), requestMetadata); + }, executor); + + cleanupRemoteSnapshotFutures.add(cleanupRemoteSnapshotFuture); + + // 3. delete the remote {@link SnapshotIndex} blob for the previous checkpoint. + CompletionStage removePrevRemoteSnapshotFuture = + snapshotIndexFuture.thenComposeAsync(snapshotIndex -> { + if (snapshotIndex.getPrevSnapshotIndexBlobId().isPresent()) { + String blobId = snapshotIndex.getPrevSnapshotIndexBlobId().get(); + LOG.debug("Removing previous snapshot index blob: {} from blob store for task: {} store: {}.", + blobId, taskName, storeName); + return blobStoreUtil.deleteSnapshotIndexBlob(blobId, requestMetadata); + } else { + // complete future immediately. There are no previous snapshots index blobs to delete. + return CompletableFuture.completedFuture(null); + } + }, executor); + removePrevRemoteSnapshotFutures.add(removePrevRemoteSnapshotFuture); + } + }); + + return FutureUtil.allOf(removeTTLFutures, cleanupRemoteSnapshotFutures, removePrevRemoteSnapshotFutures) + .whenComplete((res, ex) -> metrics.cleanupNs.update(System.nanoTime() - startTime)); + } + + @Override + public void close() { + blobStoreManager.close(); + } + + @VisibleForTesting + protected BlobStoreUtil createBlobStoreUtil(BlobStoreManager blobStoreManager, ExecutorService executor, + BlobStoreBackupManagerMetrics metrics) { + return new BlobStoreUtil(blobStoreManager, executor, metrics, null); + } + + private void updateStoreDiffMetrics(String storeName, DirDiff.Stats stats) { + metrics.storeFilesToUpload.get(storeName).set((long) stats.filesAdded); + metrics.storeFilesToRetain.get(storeName).set((long) stats.filesRetained); + metrics.storeFilesToRemove.get(storeName).set((long) stats.filesRemoved); + + metrics.storeSubDirsToUpload.get(storeName).set((long) stats.subDirsAdded); + metrics.storeSubDirsToRetain.get(storeName).set((long) stats.subDirsRetained); + metrics.storeSubDirsToRemove.get(storeName).set((long) stats.subDirsRemoved); + + metrics.storeBytesToUpload.get(storeName).set(stats.bytesAdded); + metrics.storeBytesToRetain.get(storeName).set(stats.bytesRetained); + metrics.storeBytesToRemove.get(storeName).set(stats.bytesRemoved); + } +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreRestoreManager.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreRestoreManager.java new file mode 100644 index 0000000000..64bc74e873 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreRestoreManager.java @@ -0,0 +1,367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutorService; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.samza.SamzaException; +import org.apache.samza.checkpoint.Checkpoint; +import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.config.BlobStoreConfig; +import org.apache.samza.config.Config; +import org.apache.samza.config.JobConfig; +import org.apache.samza.config.StorageConfig; +import org.apache.samza.container.TaskName; +import org.apache.samza.job.model.TaskMode; +import org.apache.samza.job.model.TaskModel; +import org.apache.samza.storage.StorageManagerUtil; +import org.apache.samza.storage.TaskRestoreManager; +import org.apache.samza.storage.blobstore.index.DirIndex; +import org.apache.samza.storage.blobstore.index.SnapshotIndex; +import org.apache.samza.storage.blobstore.metrics.BlobStoreRestoreManagerMetrics; +import org.apache.samza.storage.blobstore.util.BlobStoreUtil; +import org.apache.samza.storage.blobstore.util.DirDiffUtil; +import org.apache.samza.util.FileUtil; +import org.apache.samza.util.FutureUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class BlobStoreRestoreManager implements TaskRestoreManager { + private static final Logger LOG = LoggerFactory.getLogger(BlobStoreRestoreManager.class); + // when checking if checkpoint dir is the same as remote snapshot, exclude the "OFFSET" family of files files + // that are written to the checkpoint dir after the remote upload is complete as part of + // TaskStorageCommitManager#writeCheckpointToStoreDirectories. + private static final Set FILES_TO_IGNORE = ImmutableSet.of( + StorageManagerUtil.OFFSET_FILE_NAME_LEGACY, + StorageManagerUtil.OFFSET_FILE_NAME_NEW, + StorageManagerUtil.SIDE_INPUT_OFFSET_FILE_NAME_LEGACY, + StorageManagerUtil.CHECKPOINT_FILE_NAME); + + private final TaskModel taskModel; + private final String jobName; + private final String jobId; + private final ExecutorService executor; + private final Config config; + private final StorageConfig storageConfig; + private final BlobStoreConfig blobStoreConfig; + private final StorageManagerUtil storageManagerUtil; + private final BlobStoreUtil blobStoreUtil; + private final DirDiffUtil dirDiffUtil; + private final File loggedBaseDir; + private final File nonLoggedBaseDir; + private final String taskName; + private final Set storesToRestore; + private final BlobStoreRestoreManagerMetrics metrics; + + private BlobStoreManager blobStoreManager; + + /** + * Map of store name and Pair of blob id of SnapshotIndex and the corresponding SnapshotIndex from last successful + * task checkpoint + */ + private Map> prevStoreSnapshotIndexes; + + public BlobStoreRestoreManager(TaskModel taskModel, ExecutorService restoreExecutor, Set storesToRestore, + BlobStoreRestoreManagerMetrics metrics, Config config, File loggedBaseDir, File nonLoggedBaseDir, + StorageManagerUtil storageManagerUtil, BlobStoreManager blobStoreManager) { + this.taskModel = taskModel; + this.jobName = new JobConfig(config).getName().get(); + this.jobId = new JobConfig(config).getJobId(); + this.executor = restoreExecutor; + this.config = config; + this.storageConfig = new StorageConfig(config); + this.blobStoreConfig = new BlobStoreConfig(config); + this.storageManagerUtil = storageManagerUtil; + this.blobStoreManager = blobStoreManager; + this.blobStoreUtil = createBlobStoreUtil(blobStoreManager, executor, metrics); + this.dirDiffUtil = new DirDiffUtil(); + this.prevStoreSnapshotIndexes = new HashMap<>(); + this.loggedBaseDir = loggedBaseDir; + this.nonLoggedBaseDir = nonLoggedBaseDir; + this.taskName = taskModel.getTaskName().getTaskName(); + this.storesToRestore = storesToRestore; + this.metrics = metrics; + } + + @Override + public void init(Checkpoint checkpoint) { + long startTime = System.nanoTime(); + LOG.debug("Initializing blob store restore manager for task: {}", taskName); + + blobStoreManager.init(); + + // get previous SCMs from checkpoint + prevStoreSnapshotIndexes = blobStoreUtil.getStoreSnapshotIndexes(jobName, jobId, taskName, checkpoint); + metrics.getSnapshotIndexNs.set(System.nanoTime() - startTime); + LOG.trace("Found previous snapshot index during blob store restore manager init for task: {} to be: {}", + taskName, prevStoreSnapshotIndexes); + + metrics.initStoreMetrics(storesToRestore); + + // Note: blocks the caller thread. + deleteUnusedStoresFromBlobStore(jobName, jobId, taskName, storageConfig, blobStoreConfig, prevStoreSnapshotIndexes, + blobStoreUtil, executor); + metrics.initNs.set(System.nanoTime() - startTime); + } + + /** + * Restore state from checkpoints and state snapshots. + * State restore is performed by first retrieving the SnapshotIndex of the previous commit for every store from the + * prevStoreSnapshotIndexes map. Local store is deleted to perform a restore from local checkpoint directory or remote + * directory. If no local state checkpoint directory is found, or if the local checkpoint directory is different from + * the remote snapshot, local checkpoint directory is deleted and a restore from the remote store is done by + * downloading the state asynchronously and in parallel. + * + */ + @Override + public void restore() { + restoreStores(jobName, jobId, taskModel.getTaskName(), storesToRestore, prevStoreSnapshotIndexes, loggedBaseDir, + storageConfig, metrics, storageManagerUtil, blobStoreUtil, dirDiffUtil, executor); + } + + @Override + public void close() { + blobStoreManager.close(); + } + + @VisibleForTesting + protected BlobStoreUtil createBlobStoreUtil(BlobStoreManager blobStoreManager, ExecutorService executor, + BlobStoreRestoreManagerMetrics metrics) { + return new BlobStoreUtil(blobStoreManager, executor, null, metrics); + } + + /** + * Deletes blob store contents for stores that were present in the last checkpoint but are either no longer + * present in job configs (removed by user since last deployment) or are no longer configured to be backed + * up using blob stores. + * + * This method blocks until all the necessary store contents and snapshot index blobs have been marked for deletion. + */ + @VisibleForTesting + static void deleteUnusedStoresFromBlobStore(String jobName, String jobId, String taskName, StorageConfig storageConfig, + BlobStoreConfig blobStoreConfig, Map> initialStoreSnapshotIndexes, + BlobStoreUtil blobStoreUtil, ExecutorService executor) { + + List storesToBackup = + storageConfig.getStoresWithBackupFactory(BlobStoreStateBackendFactory.class.getName()); + List storesToRestore = + storageConfig.getStoresWithRestoreFactory(BlobStoreStateBackendFactory.class.getName()); + + List> storeDeletionFutures = new ArrayList<>(); + initialStoreSnapshotIndexes.forEach((storeName, scmAndSnapshotIndex) -> { + if (!storesToBackup.contains(storeName) && !storesToRestore.contains(storeName)) { + LOG.debug("Removing task: {} store: {} from blob store. It is either no longer used, " + + "or is no longer configured to be backed up or restored with blob store.", taskName, storeName); + DirIndex dirIndex = scmAndSnapshotIndex.getRight().getDirIndex(); + Metadata requestMetadata = + new Metadata(Metadata.SNAPSHOT_INDEX_PAYLOAD_PATH, Optional.empty(), jobName, jobId, taskName, storeName); + CompletionStage storeDeletionFuture = + blobStoreUtil.cleanUpDir(dirIndex, requestMetadata) // delete files and sub-dirs previously marked for removal + .thenComposeAsync(v -> + blobStoreUtil.deleteDir(dirIndex, requestMetadata), executor) // deleted files and dirs still present + .thenComposeAsync(v -> blobStoreUtil.deleteSnapshotIndexBlob( + scmAndSnapshotIndex.getLeft(), requestMetadata), + executor); // delete the snapshot index blob + storeDeletionFutures.add(storeDeletionFuture); + } + }); + + FutureUtil.allOf(storeDeletionFutures).join(); + } + + /** + * Restores all eligible stores in the task. + */ + @VisibleForTesting + static void restoreStores(String jobName, String jobId, TaskName taskName, Set storesToRestore, + Map> prevStoreSnapshotIndexes, + File loggedBaseDir, StorageConfig storageConfig, BlobStoreRestoreManagerMetrics metrics, + StorageManagerUtil storageManagerUtil, BlobStoreUtil blobStoreUtil, DirDiffUtil dirDiffUtil, + ExecutorService executor) { + long restoreStartTime = System.nanoTime(); + List> restoreFutures = new ArrayList<>(); + + LOG.debug("Starting restore for task: {} stores: {}", taskName, storesToRestore); + storesToRestore.forEach(storeName -> { + if (!prevStoreSnapshotIndexes.containsKey(storeName)) { + LOG.info("No checkpointed snapshot index found for task: {} store: {}. Skipping restore.", taskName, storeName); + // TODO HIGH shesharm what should we do with the local state already present on disk, if any? + // E.g. this will be the case if user changes a store from changelog based backup and restore to + // blob store based backup and restore, both at the same time. + return; + } + + Pair scmAndSnapshotIndex = prevStoreSnapshotIndexes.get(storeName); + + long storeRestoreStartTime = System.nanoTime(); + SnapshotIndex snapshotIndex = scmAndSnapshotIndex.getRight(); + DirIndex dirIndex = snapshotIndex.getDirIndex(); + + DirIndex.Stats stats = DirIndex.getStats(dirIndex); + metrics.filesToRestore.getValue().addAndGet(stats.filesPresent); + metrics.bytesToRestore.getValue().addAndGet(stats.bytesPresent); + metrics.filesRemaining.getValue().addAndGet(stats.filesPresent); + metrics.bytesRemaining.getValue().addAndGet(stats.bytesPresent); + + CheckpointId checkpointId = snapshotIndex.getSnapshotMetadata().getCheckpointId(); + File storeDir = storageManagerUtil.getTaskStoreDir(loggedBaseDir, storeName, taskName, TaskMode.Active); + Path storeCheckpointDir = Paths.get(storageManagerUtil.getStoreCheckpointDir(storeDir, checkpointId)); + LOG.trace("Got task: {} store: {} local store directory: {} and local store checkpoint directory: {}", + taskName, storeName, storeDir, storeCheckpointDir); + + // we always delete the store dir to preserve transactional state guarantees. + try { + LOG.debug("Deleting local store directory: {}. Will be restored from local store checkpoint directory " + + "or remote snapshot.", storeDir); + FileUtils.deleteDirectory(storeDir); + } catch (IOException e) { + throw new SamzaException(String.format("Error deleting store directory: %s", storeDir), e); + } + + boolean shouldRestore = shouldRestore(taskName.getTaskName(), storeName, dirIndex, + storeCheckpointDir, storageConfig, dirDiffUtil); + + if (shouldRestore) { // restore the store from the remote blob store + // delete all store checkpoint directories. if we only delete the store directory and don't + // delete the checkpoint directories, the store size on disk will grow to 2x after restore + // until the first commit is completed and older checkpoint dirs are deleted. This is + // because the hard-linked checkpoint dir files will no longer be de-duped with the + // now-deleted main store directory contents and will take up additional space of their + // own during the restore. + deleteCheckpointDirs(taskName, storeName, loggedBaseDir, storageManagerUtil); + + metrics.storePreRestoreNs.get(storeName).set(System.nanoTime() - storeRestoreStartTime); + enqueueRestore(jobName, jobId, taskName.toString(), storeName, storeDir, dirIndex, storeRestoreStartTime, + restoreFutures, blobStoreUtil, dirDiffUtil, metrics, executor); + } else { + LOG.debug("Renaming store checkpoint directory: {} to store directory: {} since its contents are identical " + + "to the remote snapshot.", storeCheckpointDir, storeDir); + // atomically rename the checkpoint dir to the store dir + new FileUtil().move(storeCheckpointDir.toFile(), storeDir); + + // delete any other checkpoint dirs. + deleteCheckpointDirs(taskName, storeName, loggedBaseDir, storageManagerUtil); + } + }); + + // wait for all restores to finish + FutureUtil.allOf(restoreFutures).whenComplete((res, ex) -> { + LOG.info("Restore completed for task: {} stores", taskName); + metrics.restoreNs.set(System.nanoTime() - restoreStartTime); + }).join(); // TODO dchen make non-blocking for the restore executor + } + + /** + * Determines if the store needs to be restored from remote snapshot based on local and remote state. + */ + @VisibleForTesting + static boolean shouldRestore(String taskName, String storeName, DirIndex dirIndex, + Path storeCheckpointDir, StorageConfig storageConfig, DirDiffUtil dirDiffUtil) { + // if a store checkpoint directory exists for the last successful task checkpoint, try to use it. + boolean restoreStore; + if (Files.exists(storeCheckpointDir)) { + if (storageConfig.cleanLoggedStoreDirsOnStart(storeName)) { + LOG.debug("Restoring task: {} store: {} from remote snapshot since the store is configured to be " + + "restored on each restart.", taskName, storeName); + restoreStore = true; + } else if (dirDiffUtil.areSameDir(FILES_TO_IGNORE, false).test(storeCheckpointDir.toFile(), dirIndex)) { + restoreStore = false; // no restore required for this store. + } else { + // we don't optimize for the case when the local host doesn't contain the most recent store checkpoint + // directory but contains an older checkpoint directory which could have partial overlap with the remote + // snapshot. we also don't try to optimize for any edge cases where the most recent checkpoint directory + // contents could be partially different than the remote store (afaik, there is no known valid scenario + // where this could happen right now, except for the offset file handling above). + // it's simpler and fast enough for now to restore the entire store instead. + + LOG.error("Local store checkpoint directory: {} contents are not the same as the remote snapshot. " + + "Queuing for restore from remote snapshot.", storeCheckpointDir); + restoreStore = true; + } + } else { // did not find last checkpoint dir, restore the store from the remote blob store + LOG.debug("No local store checkpoint directory found at: {}. " + + "Queuing for restore from remote snapshot.", storeCheckpointDir); + restoreStore = true; + } + + return restoreStore; + } + + /** + * Starts the restore for the store, enqueuing all restore-completion futures into {@param restoreFutures}. + */ + @VisibleForTesting + static void enqueueRestore(String jobName, String jobId, String taskName, String storeName, File storeDir, DirIndex dirIndex, + long storeRestoreStartTime, List> restoreFutures, BlobStoreUtil blobStoreUtil, + DirDiffUtil dirDiffUtil, BlobStoreRestoreManagerMetrics metrics, ExecutorService executor) { + + Metadata requestMetadata = new Metadata(storeDir.getAbsolutePath(), Optional.empty(), jobName, jobId, taskName, storeName); + CompletableFuture restoreFuture = + blobStoreUtil.restoreDir(storeDir, dirIndex, requestMetadata).thenRunAsync(() -> { + metrics.storeRestoreNs.get(storeName).set(System.nanoTime() - storeRestoreStartTime); + + long postRestoreStartTime = System.nanoTime(); + LOG.trace("Comparing restored store directory: {} and remote directory to verify restore.", storeDir); + if (!dirDiffUtil.areSameDir(FILES_TO_IGNORE, true).test(storeDir, dirIndex)) { + metrics.storePostRestoreNs.get(storeName).set(System.nanoTime() - postRestoreStartTime); + throw new SamzaException( + String.format("Restored store directory: %s contents " + "are not the same as the remote snapshot.", + storeDir.getAbsolutePath())); + } else { + metrics.storePostRestoreNs.get(storeName).set(System.nanoTime() - postRestoreStartTime); + LOG.info("Restore from remote snapshot completed for store: {}", storeDir); + } + }, executor); + + restoreFutures.add(restoreFuture); + } + + private static void deleteCheckpointDirs(TaskName taskName, String storeName, File loggedBaseDir, StorageManagerUtil storageManagerUtil) { + try { + List checkpointDirs = storageManagerUtil.getTaskStoreCheckpointDirs( + loggedBaseDir, storeName, taskName, TaskMode.Active); + for (File checkpointDir: checkpointDirs) { + LOG.debug("Deleting local store checkpoint directory: {} before restore.", checkpointDir); + FileUtils.deleteDirectory(checkpointDir); + } + } catch (Exception e) { + throw new SamzaException( + String.format("Error deleting checkpoint directory for task: %s store: %s.", + taskName, storeName), e); + } + } +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreStateBackendFactory.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreStateBackendFactory.java new file mode 100644 index 0000000000..e2512b4890 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreStateBackendFactory.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore; + +import com.google.common.base.Preconditions; +import java.io.File; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import org.apache.commons.lang3.StringUtils; +import org.apache.samza.config.BlobStoreConfig; +import org.apache.samza.config.Config; +import org.apache.samza.context.ContainerContext; +import org.apache.samza.context.JobContext; +import org.apache.samza.job.model.ContainerModel; +import org.apache.samza.job.model.JobModel; +import org.apache.samza.job.model.TaskModel; +import org.apache.samza.metrics.MetricsRegistry; +import org.apache.samza.storage.BlobStoreAdminFactory; +import org.apache.samza.storage.KafkaChangelogRestoreParams; +import org.apache.samza.storage.StateBackendAdmin; +import org.apache.samza.storage.StateBackendFactory; +import org.apache.samza.storage.StorageManagerUtil; +import org.apache.samza.storage.TaskBackupManager; +import org.apache.samza.storage.TaskRestoreManager; +import org.apache.samza.storage.blobstore.metrics.BlobStoreBackupManagerMetrics; +import org.apache.samza.storage.blobstore.metrics.BlobStoreRestoreManagerMetrics; +import org.apache.samza.util.Clock; +import org.apache.samza.util.ReflectionUtil; + + +public class BlobStoreStateBackendFactory implements StateBackendFactory { + @Override + public TaskBackupManager getBackupManager( + JobContext jobContext, + ContainerModel containerModel, + TaskModel taskModel, + ExecutorService backupExecutor, + MetricsRegistry metricsRegistry, + Config config, + Clock clock, + File loggedStoreBaseDir, + File nonLoggedStoreBaseDir) { + BlobStoreConfig blobStoreConfig = new BlobStoreConfig(config); + String blobStoreManagerFactory = blobStoreConfig.getBlobStoreManagerFactory(); + Preconditions.checkState(StringUtils.isNotBlank(blobStoreManagerFactory)); + BlobStoreManagerFactory factory = ReflectionUtil.getObj(blobStoreManagerFactory, BlobStoreManagerFactory.class); + BlobStoreManager blobStoreManager = factory.getBackupBlobStoreManager(config, backupExecutor); + BlobStoreBackupManagerMetrics metrics = new BlobStoreBackupManagerMetrics(metricsRegistry); + return new BlobStoreBackupManager(jobContext.getJobModel(), containerModel, taskModel, backupExecutor, + metrics, config, clock, loggedStoreBaseDir, new StorageManagerUtil(), blobStoreManager); + } + + @Override + public TaskRestoreManager getRestoreManager( + JobContext jobContext, + ContainerContext containerContext, + TaskModel taskModel, + ExecutorService restoreExecutor, + MetricsRegistry metricsRegistry, + Set storesToRestore, + Config config, + Clock clock, + File loggedStoreBaseDir, + File nonLoggedStoreBaseDir, + KafkaChangelogRestoreParams kafkaChangelogRestoreParams) { + BlobStoreConfig blobStoreConfig = new BlobStoreConfig(config); + String blobStoreManagerFactory = blobStoreConfig.getBlobStoreManagerFactory(); + Preconditions.checkState(StringUtils.isNotBlank(blobStoreManagerFactory)); + BlobStoreManagerFactory factory = ReflectionUtil.getObj(blobStoreManagerFactory, BlobStoreManagerFactory.class); + BlobStoreManager blobStoreManager = factory.getRestoreBlobStoreManager(config, restoreExecutor); + BlobStoreRestoreManagerMetrics metrics = new BlobStoreRestoreManagerMetrics(metricsRegistry); + return new BlobStoreRestoreManager(taskModel, restoreExecutor, storesToRestore, metrics, config, loggedStoreBaseDir, + nonLoggedStoreBaseDir, new StorageManagerUtil(), blobStoreManager); + } + + @Override + public StateBackendAdmin getAdmin(JobModel jobModel, Config config) { + BlobStoreConfig blobStoreConfig = new BlobStoreConfig(config); + String stateBackendAdminFactory = blobStoreConfig.getBlobStoreAdminFactory(); + BlobStoreAdminFactory factory = ReflectionUtil.getObj(stateBackendAdminFactory, BlobStoreAdminFactory.class); + return factory.getStateBackendAdmin(config, jobModel); + } +} \ No newline at end of file diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/diff/DirDiff.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/diff/DirDiff.java new file mode 100644 index 0000000000..76fdb08986 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/diff/DirDiff.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.diff; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; +import java.io.File; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.samza.storage.blobstore.index.DirIndex; +import org.apache.samza.storage.blobstore.index.FileIndex; + +/** + * Representation of the diff between a local directory and a remote directory contents. + */ +public class DirDiff { + + private final String dirName; + + /** + * New files in this directory that needs to be uploaded to the blob store. + */ + private final List filesAdded; + + /** + * Files that have already been uploaded to the blob store in a previous snapshot and haven't changed. + */ + private final List filesRetained; + + /** + * Files that have already been uploaded to the blob store in a previous snapshot and need to be removed. + */ + private final List filesRemoved; + + /** + * Subdirectories of this directory that are not already present in the previous snapshot and all of their contents + * need to be recursively added. + */ + private final List subDirsAdded; + + /** + * Subdirectories of this directory that are already present in the previous snapshot, but whose contents + * may have changed and may need to be recursively added or removed. + */ + private final List subDirsRetained; + + /** + * Subdirectories that are already present in the previous snapshot, but don't exist in the local snapshot, + * and hence all of their contents need to be recursively removed. + */ + private final List subDirsRemoved; + + public DirDiff(String dirName, + List filesAdded, List filesRetained, List filesRemoved, + List subDirsAdded, List subDirsRetained, List subDirsRemoved) { + Preconditions.checkNotNull(dirName); // may be empty for root dirs + Preconditions.checkNotNull(filesAdded); + Preconditions.checkNotNull(filesRetained); + Preconditions.checkNotNull(filesRemoved); + Preconditions.checkNotNull(subDirsAdded); + Preconditions.checkNotNull(subDirsRetained); + Preconditions.checkNotNull(subDirsRemoved); + + // validate that a file is not present in multiple lists + Set addedFilesSet = filesAdded.stream().map(File::getName).collect(Collectors.toSet()); + Set retainedFilesSet = filesRetained.stream().map(FileIndex::getFileName).collect(Collectors.toSet()); + Set removedFilesSet = filesRemoved.stream().map(FileIndex::getFileName).collect(Collectors.toSet()); + Sets.SetView addedAndRetainedFilesSet = Sets.intersection(addedFilesSet, retainedFilesSet); + Preconditions.checkState(addedAndRetainedFilesSet.isEmpty(), + String.format("Files present in both added and retained sets: %s", addedAndRetainedFilesSet.toString())); + Sets.SetView retainedAndRemovedFilesSet = Sets.intersection(retainedFilesSet, removedFilesSet); + Preconditions.checkState(retainedAndRemovedFilesSet.isEmpty(), + String.format("Files present in both retained and removed sets: %s", retainedAndRemovedFilesSet.toString())); + + // validate that a subDir is not present in multiple lists + Set addedSubDirsSet = subDirsAdded.stream().map(DirDiff::getDirName).collect(Collectors.toSet()); + Set retainedSubDirsSet = subDirsRetained.stream().map(DirDiff::getDirName).collect(Collectors.toSet()); + Set removedSubDirsSet = subDirsRemoved.stream().map(DirIndex::getDirName).collect(Collectors.toSet()); + Sets.SetView addedAndRetainedSubDirsSet = Sets.intersection(addedSubDirsSet, retainedSubDirsSet); + Preconditions.checkState(addedAndRetainedSubDirsSet.isEmpty(), + String.format("Sub-dirs present in both added and retained sets: %s", addedAndRetainedSubDirsSet.toString())); + Sets.SetView retainedAndRemovedSubDirsSet = Sets.intersection(retainedSubDirsSet, removedSubDirsSet); + Preconditions.checkState(retainedAndRemovedSubDirsSet.isEmpty(), + String.format("Sub-dirs present in both retained and removed sets: %s", retainedAndRemovedSubDirsSet.toString())); + + this.dirName = dirName; + this.filesAdded = filesAdded; + this.filesRetained = filesRetained; + this.filesRemoved = filesRemoved; + this.subDirsAdded = subDirsAdded; + this.subDirsRetained = subDirsRetained; + this.subDirsRemoved = subDirsRemoved; + } + + public String getDirName() { + return dirName; + } + + public List getFilesAdded() { + return filesAdded; + } + + public List getFilesRetained() { + return filesRetained; + } + + public List getFilesRemoved() { + return filesRemoved; + } + + public List getSubDirsAdded() { + return subDirsAdded; + } + + public List getSubDirsRetained() { + return subDirsRetained; + } + + public List getSubDirsRemoved() { + return subDirsRemoved; + } + + public static Stats getStats(DirDiff dirDiff) { + Stats stats = new Stats(); + updateStats(dirDiff, stats); + return stats; + } + + private static void updateStats(DirDiff dirDiff, Stats stats) { + stats.filesAdded += dirDiff.getFilesAdded().size(); + stats.filesRetained += dirDiff.getFilesRetained().size(); + stats.filesRemoved += dirDiff.getFilesRemoved().size(); + + stats.bytesAdded += dirDiff.getFilesAdded().stream().mapToLong(File::length).sum(); + stats.bytesRetained += dirDiff.getFilesRetained().stream().mapToLong(f -> f.getFileMetadata().getSize()).sum(); + stats.bytesRemoved += dirDiff.getFilesRemoved().stream().mapToLong(f -> f.getFileMetadata().getSize()).sum(); + + for (DirDiff subDirAdded: dirDiff.getSubDirsAdded()) { + stats.subDirsAdded += 1; + updateStats(subDirAdded, stats); + } + for (DirDiff subDirRetained: dirDiff.getSubDirsRetained()) { + stats.subDirsRetained += 1; + updateStats(subDirRetained, stats); + } + for (DirIndex subDirRemoved: dirDiff.getSubDirsRemoved()) { + stats.subDirsRemoved += 1; + updateStatsForDirRemoved(subDirRemoved, stats); + } + } + + private static void updateStatsForDirRemoved(DirIndex dirIndex, Stats stats) { + // every file and sub-dir present in a removed parent dir are to be removed as well + // files and sub-dirs to be removed don't matter since they would have already been + // cleaned up after the previous commit + stats.filesRemoved += dirIndex.getFilesRemoved().size(); + stats.bytesRemoved += dirIndex.getFilesPresent().stream().mapToLong(f -> f.getFileMetadata().getSize()).sum(); + for (DirIndex subDirRemoved: dirIndex.getSubDirsPresent()) { + stats.subDirsRemoved += 1; + updateStatsForDirRemoved(subDirRemoved, stats); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + + if (o == null || getClass() != o.getClass()) return false; + + DirDiff dirDiff = (DirDiff) o; + + return new EqualsBuilder() + .append(getDirName(), dirDiff.getDirName()) + .append(getFilesAdded(), dirDiff.getFilesAdded()) + .append(getFilesRetained(), dirDiff.getFilesRetained()) + .append(getFilesRemoved(), dirDiff.getFilesRemoved()) + .append(getSubDirsAdded(), dirDiff.getSubDirsAdded()) + .append(getSubDirsRetained(), dirDiff.getSubDirsRetained()) + .append(getSubDirsRemoved(), dirDiff.getSubDirsRemoved()) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(getDirName()) + .append(getFilesAdded()) + .append(getFilesRetained()) + .append(getFilesRemoved()) + .append(getSubDirsAdded()) + .append(getSubDirsRetained()) + .append(getSubDirsRemoved()) + .toHashCode(); + } + + public static class Stats { + public int filesAdded; + public int filesRetained; + public int filesRemoved; + + public int subDirsAdded; + public int subDirsRetained; + public int subDirsRemoved; + + public long bytesAdded; + public long bytesRetained; + public long bytesRemoved; + + @Override + public String toString() { + return "Stats{" + + "filesAdded=" + filesAdded + + ", filesRetained=" + filesRetained + + ", filesRemoved=" + filesRemoved + + ", subDirsAdded=" + subDirsAdded + + ", subDirsRetained=" + subDirsRetained + + ", subDirsRemoved=" + subDirsRemoved + + ", bytesAdded=" + bytesAdded + + ", bytesRetained=" + bytesRetained + + ", bytesRemoved=" + bytesRemoved + + '}'; + } + } +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/DirIndex.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/DirIndex.java new file mode 100644 index 0000000000..31819be54e --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/DirIndex.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.index; + +import com.google.common.base.Preconditions; +import java.util.List; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + +/** + * Representation of a directory in the blob store + */ +public class DirIndex { + public static final String ROOT_DIR_NAME = ""; + private static final short SCHEMA_VERSION = 1; + + private final String dirName; + + private final List filesPresent; + private final List filesRemoved; + + // Note: subDirsPresent can also have filesRemoved and subDirsRemoved within them. + private final List subDirsPresent; + private final List subDirsRemoved; + + public DirIndex(String dirName, + List filesPresent, List filesRemoved, + List subDirsPresent, List subDirsRemoved) { + Preconditions.checkNotNull(dirName); // may be empty for root dirs + Preconditions.checkNotNull(filesPresent); + Preconditions.checkNotNull(filesRemoved); + Preconditions.checkNotNull(subDirsPresent); + Preconditions.checkNotNull(subDirsRemoved); + this.dirName = dirName; + this.filesPresent = filesPresent; + this.filesRemoved = filesRemoved; + this.subDirsPresent = subDirsPresent; + this.subDirsRemoved = subDirsRemoved; + } + + public static short getSchemaVersion() { + return SCHEMA_VERSION; + } + + public String getDirName() { + return dirName; + } + + public List getFilesPresent() { + return filesPresent; + } + + public List getFilesRemoved() { + return filesRemoved; + } + + public List getSubDirsPresent() { + return subDirsPresent; + } + + public List getSubDirsRemoved() { + return subDirsRemoved; + } + + public static Stats getStats(DirIndex dirIndex) { + Stats stats = new Stats(); + updateStats(dirIndex, stats); + return stats; + } + + private static void updateStats(DirIndex dirIndex, Stats stats) { + stats.filesPresent += dirIndex.getFilesPresent().size(); + stats.filesRemoved += dirIndex.getFilesRemoved().size(); + + stats.subDirsPresent += dirIndex.getSubDirsPresent().size(); + stats.subDirsRemoved += dirIndex.getSubDirsRemoved().size(); + + stats.bytesPresent += dirIndex.getFilesPresent().stream().mapToLong(fi -> fi.getFileMetadata().getSize()).sum(); + stats.bytesRemoved += dirIndex.getFilesRemoved().stream().mapToLong(fi -> fi.getFileMetadata().getSize()).sum(); + + for (DirIndex subDirPresent : dirIndex.getSubDirsPresent()) { + updateStats(subDirPresent, stats); + } + + for (DirIndex subDirsRemoved : dirIndex.getSubDirsRemoved()) { + updateStatsForSubDirsRemoved(subDirsRemoved, stats); + } + } + + private static void updateStatsForSubDirsRemoved(DirIndex dirIndex, Stats stats) { + stats.filesRemoved += dirIndex.getFilesPresent().size(); + stats.bytesRemoved += dirIndex.getFilesPresent().stream().mapToLong(fi -> fi.getFileMetadata().getSize()).sum(); + for (DirIndex subDirToRemove : dirIndex.getSubDirsPresent()) { + updateStatsForSubDirsRemoved(subDirToRemove, stats); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + + if (o == null || getClass() != o.getClass()) return false; + + DirIndex that = (DirIndex) o; + + return new EqualsBuilder() + .append(getDirName(), that.getDirName()) + .append(getFilesPresent(), that.getFilesPresent()) + .append(getFilesRemoved(), that.getFilesRemoved()) + .append(getSubDirsPresent(), that.getSubDirsPresent()) + .append(getSubDirsRemoved(), that.getSubDirsRemoved()) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(getDirName()).append(getFilesPresent()) + .append(getFilesRemoved()) + .append(getSubDirsPresent()) + .append(getSubDirsRemoved()) + .toHashCode(); + } + + @Override + public String toString() { + return "DirIndex{" + + "dirName='" + + dirName + '\'' + + ", filesPresent=" + filesPresent + + ", filesRemoved=" + filesRemoved + + ", subDirsPresent=" + subDirsPresent + + ", subDirsRemoved=" + subDirsRemoved + + '}'; + } + + public static class Stats { + public int filesPresent; + public int filesRemoved; + + public int subDirsPresent; + public int subDirsRemoved; + + public long bytesPresent; + public long bytesRemoved; + + @Override + public String toString() { + return "Stats{" + + "filesAdded=" + filesPresent + + ", filesRemoved=" + filesRemoved + + ", subDirsAdded=" + subDirsPresent + + ", subDirsRemoved=" + subDirsRemoved + + ", bytesAdded=" + bytesPresent + + ", bytesRemoved=" + bytesRemoved + + '}'; + } + } +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileBlob.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileBlob.java new file mode 100644 index 0000000000..8a289aaf59 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileBlob.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.index; + +import com.google.common.base.Preconditions; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + + +/** + * Representation of a File in a Blob store + */ +public class FileBlob { + + private final String blobId; + /** + * Offset of this blob in the file. A file can be uploaded multiple chunks, and can have + * multiple blobs associated with it. Each blob then has its own ID and an offset in the file. + */ + private final int offset; + + public FileBlob(String blobId, int offset) { + Preconditions.checkState(StringUtils.isNotBlank(blobId)); + Preconditions.checkState(offset >= 0); + this.blobId = blobId; + this.offset = offset; + } + + public String getBlobId() { + return blobId; + } + + public int getOffset() { + return offset; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (!(o instanceof FileBlob)) { + return false; + } + + FileBlob fileBlob = (FileBlob) o; + + return new EqualsBuilder() + .append(blobId, fileBlob.blobId) + .append(offset, fileBlob.offset) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(blobId) + .append(offset) + .toHashCode(); + } + + @Override + public String toString() { + return "FileBlob{" + "blobId='" + blobId + '\'' + ", offset=" + offset + '}'; + } +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileIndex.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileIndex.java new file mode 100644 index 0000000000..b0666cb1ee --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileIndex.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.index; + +import com.google.common.base.Preconditions; +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + + +/** + * Representation of a file in blob store + */ +public class FileIndex { + private final String fileName; + /** + * Chunks of file uploaded to blob store as {@link FileBlob}s + */ + private final List fileBlobs; + /** + * Metadata (e.g. POSIX file attributes) associated with the file. + */ + private final FileMetadata fileMetadata; + /** + * Checksum of the file for verifying integrity. + */ + private final long checksum; + + + public FileIndex(String fileName, List fileBlobs, FileMetadata fileMetadata, long checksum) { + Preconditions.checkState(StringUtils.isNotBlank(fileName)); + Preconditions.checkNotNull(fileBlobs); + // fileBlobs can be empty list for a file of size 0 bytes. + Preconditions.checkNotNull(fileMetadata); + this.fileName = fileName; + this.fileBlobs = fileBlobs; + this.fileMetadata = fileMetadata; + this.checksum = checksum; + } + + public String getFileName() { + return fileName; + } + + public List getBlobs() { + return fileBlobs; + } + + public FileMetadata getFileMetadata() { + return fileMetadata; + } + + public long getChecksum() { + return checksum; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (!(o instanceof FileIndex)) { + return false; + } + + FileIndex that = (FileIndex) o; + + return new EqualsBuilder() + .append(getFileName(), that.getFileName()) + .append(getBlobs(), that.getBlobs()) + .append(fileMetadata, that.fileMetadata) + .append(getChecksum(), that.getChecksum()) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(getFileName()) + .append(getBlobs()) + .append(fileMetadata) + .append(getChecksum()) + .toHashCode(); + } + + @Override + public String toString() { + return "FileIndex{" + + "fileName='" + fileName + '\'' + + ", fileBlobs=" + fileBlobs + + ", fileMetadata=" + fileMetadata + + ", checksum='" + checksum + '\'' + + '}'; + } +} \ No newline at end of file diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileMetadata.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileMetadata.java new file mode 100644 index 0000000000..41db1264d6 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileMetadata.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.index; + +import com.google.common.base.Preconditions; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermissions; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + + +/** + * Representation of metadata associated with a File. + */ +public class FileMetadata { + + private final long creationTimeMillis; + private final long lastModifiedTimeMillis; + private final long size; + private final String owner; + private final String group; + private final String permissions; + + public FileMetadata(long creationTimeMillis, long lastModifiedTimeMillis, long size, + String owner, String group, String permissions) { + Preconditions.checkState(creationTimeMillis >= 0); + Preconditions.checkState(lastModifiedTimeMillis >= 0); + Preconditions.checkState(size >= 0); + Preconditions.checkState(StringUtils.isNotBlank(owner)); + Preconditions.checkState(StringUtils.isNotBlank(group)); + Preconditions.checkState(StringUtils.isNotBlank(permissions)); + this.creationTimeMillis = creationTimeMillis; + this.lastModifiedTimeMillis = lastModifiedTimeMillis; + this.size = size; + this.owner = owner; + this.group = group; + this.permissions = permissions; + } + + public static FileMetadata fromFile(File file) throws IOException { + PosixFileAttributes attributes = Files.readAttributes(file.toPath(), PosixFileAttributes.class); + + return new FileMetadata(attributes.creationTime().toMillis(), attributes.lastModifiedTime().toMillis(), + attributes.size(), attributes.owner().toString(), attributes.group().toString(), + PosixFilePermissions.toString(attributes.permissions())); + } + + public long getCreationTimeMillis() { + return creationTimeMillis; + } + + public long getLastModifiedTimeMillis() { + return lastModifiedTimeMillis; + } + + public long getSize() { + return size; + } + + public String getOwner() { + return owner; + } + + public String getGroup() { + return group; + } + + public String getPermissions() { + return permissions; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (!(o instanceof FileMetadata)) { + return false; + } + + FileMetadata that = (FileMetadata) o; + + return new EqualsBuilder() + .append(getCreationTimeMillis(), that.getCreationTimeMillis()) + .append(getLastModifiedTimeMillis(), that.getLastModifiedTimeMillis()) + .append(getSize(), that.getSize()) + .append(getOwner(), that.getOwner()) + .append(getGroup(), that.getGroup()) + .append(getPermissions(), that.getPermissions()) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(getCreationTimeMillis()) + .append(getLastModifiedTimeMillis()) + .append(getSize()) + .append(getOwner()) + .append(getGroup()) + .append(getPermissions()) + .toHashCode(); + } + + @Override + public String toString() { + return "FileMetadata{" + + "creationTimeMillis=" + creationTimeMillis + + ", lastModifiedTimeMillis=" + lastModifiedTimeMillis + + ", size=" + size + + ", owner='" + owner + '\'' + + ", group='" + group + '\'' + + ", permissions=" + permissions + + '}'; + } +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/SnapshotIndex.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/SnapshotIndex.java new file mode 100644 index 0000000000..b3f68dc31d --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/SnapshotIndex.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.index; + +import com.google.common.base.Preconditions; +import java.util.Optional; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + + +/** + * A {@link SnapshotIndex} contains all the information necessary for recreating the local store by + * downloading its contents from the remote blob store. The {@link SnapshotIndex} is itself serialized + * and stored as a blob in the remote store, and its blob id tracked in the Task checkpoint. + */ +public class SnapshotIndex { + private static final short SCHEMA_VERSION = 1; + + private final long creationTimeMillis; + /** + * Metadata for a snapshot like job name, job Id, store name etc. + */ + private final SnapshotMetadata snapshotMetadata; + private final DirIndex dirIndex; + + /** + * Blob ID of previous snapshot index blob. Tracked here to be cleaned up + * in cleanup phase of commit lifecycle. + */ + private final Optional prevSnapshotIndexBlobId; + + public SnapshotIndex(long creationTimeMillis, SnapshotMetadata snapshotMetadata, DirIndex dirIndex, + Optional prevSnapshotIndexBlobId) { + Preconditions.checkState(creationTimeMillis >= 0); + Preconditions.checkNotNull(snapshotMetadata); + Preconditions.checkNotNull(dirIndex); + Preconditions.checkNotNull(prevSnapshotIndexBlobId); + Preconditions.checkState( + !(prevSnapshotIndexBlobId.isPresent() && StringUtils.isBlank(prevSnapshotIndexBlobId.get()))); + this.creationTimeMillis = creationTimeMillis; + this.snapshotMetadata = snapshotMetadata; + this.dirIndex = dirIndex; + // if no previous snapshot index blob was present, this can be null + this.prevSnapshotIndexBlobId = prevSnapshotIndexBlobId; + } + + public static short getSchemaVersion() { + return SCHEMA_VERSION; + } + + public long getCreationTimeMillis() { + return creationTimeMillis; + } + + public SnapshotMetadata getSnapshotMetadata() { + return snapshotMetadata; + } + + public DirIndex getDirIndex() { + return dirIndex; + } + + public Optional getPrevSnapshotIndexBlobId() { + return prevSnapshotIndexBlobId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (!(o instanceof SnapshotIndex)) { + return false; + } + + SnapshotIndex that = (SnapshotIndex) o; + + return new EqualsBuilder() + .append(getCreationTimeMillis(), that.getCreationTimeMillis()) + .append(getSnapshotMetadata(), that.getSnapshotMetadata()) + .append(getDirIndex(), that.getDirIndex()) + .append(getPrevSnapshotIndexBlobId(), that.getPrevSnapshotIndexBlobId()) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(getCreationTimeMillis()) + .append(getSnapshotMetadata()) + .append(getDirIndex()) + .append(prevSnapshotIndexBlobId) + .toHashCode(); + } + + @Override + public String toString() { + return "SnapshotIndex{" + + "creationTimeMillis=" + creationTimeMillis + + ", snapshotMetadata=" + snapshotMetadata + + ", dirIndex=" + dirIndex + + ", prevSnapshotIndexBlobId" + prevSnapshotIndexBlobId + + '}'; + } +} \ No newline at end of file diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/SnapshotMetadata.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/SnapshotMetadata.java new file mode 100644 index 0000000000..7f059b9645 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/SnapshotMetadata.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.index; + +import com.google.common.base.Preconditions; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.samza.checkpoint.CheckpointId; + + +/** + * Represents metadata associated with a remote snapshot. + */ +public class SnapshotMetadata { + private final CheckpointId checkpointId; + private final String jobName; + private final String jobId; + private final String taskName; + private final String storeName; + + public SnapshotMetadata(CheckpointId checkpointId, String jobName, String jobId, String taskName, + String storeName) { + Preconditions.checkNotNull(checkpointId); + Preconditions.checkState(StringUtils.isNotBlank(jobName)); + Preconditions.checkState(StringUtils.isNotBlank(jobId)); + Preconditions.checkState(StringUtils.isNotBlank(taskName)); + Preconditions.checkState(StringUtils.isNotBlank(storeName)); + this.checkpointId = checkpointId; + this.jobName = jobName; + this.jobId = jobId; + this.taskName = taskName; + this.storeName = storeName; + } + + public CheckpointId getCheckpointId() { + return checkpointId; + } + + public String getJobName() { + return jobName; + } + + public String getJobId() { + return jobId; + } + + public String getTaskName() { + return taskName; + } + + public String getStoreName() { + return storeName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (!(o instanceof SnapshotMetadata)) { + return false; + } + + SnapshotMetadata that = (SnapshotMetadata) o; + + return new EqualsBuilder() + .append(getCheckpointId(), that.getCheckpointId()) + .append(getJobName(), that.getJobName()) + .append(getJobId(), that.getJobId()) + .append(getTaskName(), that.getTaskName()) + .append(getStoreName(), that.getStoreName()) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(getCheckpointId()) + .append(getJobName()) + .append(getJobId()) + .append(getTaskName()) + .append(getStoreName()) + .toHashCode(); + } + + @Override + public String toString() { + return "SnapshotMetadata{" + + "checkpointId=" + checkpointId + + ", jobName='" + jobName + '\'' + + ", jobId='" + jobId + '\'' + + ", taskName='" + taskName + '\'' + + ", storeName='" + storeName + '\'' + + '}'; + } +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonDirIndexMixin.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonDirIndexMixin.java new file mode 100644 index 0000000000..7a29c83f98 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonDirIndexMixin.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.index.serde; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import org.apache.samza.storage.blobstore.index.DirIndex; +import org.apache.samza.storage.blobstore.index.FileIndex; + + +/** + * A mix-in Jackson class to convert {@link DirIndex} to/from JSON. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public abstract class JsonDirIndexMixin { + + @JsonCreator + public JsonDirIndexMixin(@JsonProperty("dir-name") String dirName, + @JsonProperty("files-present") List filesPresent, + @JsonProperty("files-removed") List filesRemoved, + @JsonProperty("sub-dirs-present") List subDirsPresent, + @JsonProperty("sub-dirs-removed") List subDirsRemoved) { + } + + @JsonProperty("dir-name") + abstract String getDirName(); + + @JsonProperty("files-present") + abstract List getFilesPresent(); + + @JsonProperty("files-removed") + abstract List getFilesRemoved(); + + @JsonProperty("sub-dirs-present") + abstract List getSubDirsPresent(); + + @JsonProperty("sub-dirs-removed") + abstract List getSubDirsRemoved(); +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonFileBlobMixin.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonFileBlobMixin.java new file mode 100644 index 0000000000..03a91d1c66 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonFileBlobMixin.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.index.serde; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + + +/** + * A mix-in Jackson class to convert FileBlob to/from JSON. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public abstract class JsonFileBlobMixin { + @JsonCreator + public JsonFileBlobMixin(@JsonProperty("blob-id") String blobId, @JsonProperty("offset") int offset) { + } + + @JsonProperty("blob-id") + abstract String getBlobId(); + + @JsonProperty("offset") + abstract int getOffset(); +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonFileIndexMixin.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonFileIndexMixin.java new file mode 100644 index 0000000000..d8ee10a901 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonFileIndexMixin.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.index.serde; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import org.apache.samza.storage.blobstore.index.FileBlob; +import org.apache.samza.storage.blobstore.index.FileMetadata; + + +/** + * A mix-in Jackson class to convert FileIndex to/from JSON. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public abstract class JsonFileIndexMixin { + @JsonCreator + public JsonFileIndexMixin(@JsonProperty("file-name") String fileName, + @JsonProperty("blobs") List blobs, @JsonProperty("file-metadata") FileMetadata fileMetadata, + @JsonProperty("checksum") long checksum) { + + } + + @JsonProperty("file-name") + abstract String getFileName(); + + @JsonProperty("blobs") + abstract List getBlobs(); + + @JsonProperty("file-metadata") + abstract FileMetadata getFileMetadata(); + + @JsonProperty("checksum") + abstract long getChecksum(); +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonFileMetadataMixin.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonFileMetadataMixin.java new file mode 100644 index 0000000000..71eea611d1 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonFileMetadataMixin.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.index.serde; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * A mix-in Jackson class to convert FileMetadata to/from JSON. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public abstract class JsonFileMetadataMixin { + @JsonCreator + public JsonFileMetadataMixin(@JsonProperty("creation-time-millis") long creationTimeMillis, + @JsonProperty("last-modified-time-millis") long lastModifiedTimeMillis, @JsonProperty("size") long size, + @JsonProperty("owner") String owner, @JsonProperty("group") String group, + @JsonProperty("permissions") String permissions) { + } + + @JsonProperty("creation-time-millis") + abstract long getCreationTimeMillis(); + + @JsonProperty("last-modified-time-millis") + abstract long getLastModifiedTimeMillis(); + + @JsonProperty("size") + abstract long getSize(); + + @JsonProperty("owner") + abstract String getOwner(); + + @JsonProperty("group") + abstract String getGroup(); + + @JsonProperty("permissions") + abstract String getPermissions(); +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonSnapshotIndexMixin.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonSnapshotIndexMixin.java new file mode 100644 index 0000000000..3cce2538fd --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonSnapshotIndexMixin.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.index.serde; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Optional; +import org.apache.samza.storage.blobstore.index.DirIndex; +import org.apache.samza.storage.blobstore.index.SnapshotMetadata; + + +/** + * A mix-in Jackson class to convert SnapshotIndex to/from JSON. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public abstract class JsonSnapshotIndexMixin { + @JsonCreator + public JsonSnapshotIndexMixin(@JsonProperty("creation-time-millis") long creationTimeMillis, + @JsonProperty("snapshot-metadata") SnapshotMetadata snapshotMetadata, + @JsonProperty("dir-index") DirIndex dirIndex, + @JsonProperty("prev-snapshot-index-blob-id") Optional prevSnapshotIndexBlobId) { + } + + @JsonProperty("creation-time-millis") + abstract long getCreationTimeMillis(); + + @JsonProperty("snapshot-metadata") + abstract SnapshotMetadata getSnapshotMetadata(); + + @JsonProperty("dir-index") + abstract DirIndex getDirIndex(); + + @JsonProperty("prev-snapshot-index-blob-id") + abstract Optional getPrevSnapshotIndexBlobId(); +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonSnapshotMetadataMixin.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonSnapshotMetadataMixin.java new file mode 100644 index 0000000000..804e3304a3 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/JsonSnapshotMetadataMixin.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.index.serde; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.samza.checkpoint.CheckpointId; + +/** + * A mix-in Jackson class to convert SnapshotMetadata to/from JSON. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public abstract class JsonSnapshotMetadataMixin { + @JsonCreator + public JsonSnapshotMetadataMixin(@JsonProperty("checkpoint-id") CheckpointId checkpointId, + @JsonProperty("job-name") String jobName, @JsonProperty("job-id") String jobId, + @JsonProperty("task-name") String taskName, @JsonProperty("store-name") String storeName) { + + } + + @JsonProperty("checkpoint-id") + abstract CheckpointId getCheckpointId(); + + @JsonProperty("job-name") + abstract String getJobName(); + + @JsonProperty("job-id") + abstract String getJobId(); + + @JsonProperty("task-name") + abstract String getTaskName(); + + @JsonProperty("store-name") + abstract String getStoreName(); +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/SnapshotIndexSerde.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/SnapshotIndexSerde.java new file mode 100644 index 0000000000..9e316ada23 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/SnapshotIndexSerde.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.index.serde; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import org.apache.samza.SamzaException; +import org.apache.samza.serializers.Serde; +import org.apache.samza.serializers.model.SamzaObjectMapper; +import org.apache.samza.storage.blobstore.index.DirIndex; +import org.apache.samza.storage.blobstore.index.FileBlob; +import org.apache.samza.storage.blobstore.index.FileIndex; +import org.apache.samza.storage.blobstore.index.FileMetadata; +import org.apache.samza.storage.blobstore.index.SnapshotIndex; +import org.apache.samza.storage.blobstore.index.SnapshotMetadata; + + +public class SnapshotIndexSerde implements Serde { + + private final static ObjectMapper MAPPER = SamzaObjectMapper.getObjectMapper(); + private TypeReference typeReference; + private final ObjectWriter objectWriter; + + public SnapshotIndexSerde() { + MAPPER.registerModule(new Jdk8Module()); + MAPPER.addMixIn(SnapshotIndex.class, JsonSnapshotIndexMixin.class) + .addMixIn(SnapshotMetadata.class, JsonSnapshotMetadataMixin.class) + .addMixIn(DirIndex.class, JsonDirIndexMixin.class) + .addMixIn(FileIndex.class, JsonFileIndexMixin.class) + .addMixIn(FileMetadata.class, JsonFileMetadataMixin.class) + .addMixIn(FileBlob.class, JsonFileBlobMixin.class); + + this.typeReference = new TypeReference() { }; + this.objectWriter = MAPPER.writerFor(typeReference); + } + + @Override + public SnapshotIndex fromBytes(byte[] bytes) { + try { + return MAPPER.readerFor(typeReference).readValue(bytes); + } catch (Exception exception) { + throw new SamzaException(String.format("Exception in deserializing SnapshotIndex bytes %s", + new String(bytes)), exception); + } + } + + @Override + public byte[] toBytes(SnapshotIndex snapshotIndex) { + try { + return objectWriter.writeValueAsBytes(snapshotIndex); + } catch (Exception exception) { + throw new SamzaException(String.format("Exception in serializing SnapshotIndex bytes %s", snapshotIndex), exception); + } + } +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/metrics/BlobStoreBackupManagerMetrics.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/metrics/BlobStoreBackupManagerMetrics.java new file mode 100644 index 0000000000..5e41073f27 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/metrics/BlobStoreBackupManagerMetrics.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.metrics; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.samza.metrics.Counter; +import org.apache.samza.metrics.Gauge; +import org.apache.samza.metrics.MetricsRegistry; +import org.apache.samza.metrics.Timer; + + +public class BlobStoreBackupManagerMetrics { + private static final String GROUP = BlobStoreBackupManagerMetrics.class.getName(); + private final MetricsRegistry metricsRegistry; + + // TODO LOW shesharm per-task throughput + public final Gauge initNs; + + public final Timer uploadNs; + // gauges of AtomicLong so that the value can be incremented/decremented atomically in a thread-safe way. + // don't set the gauge value directly. use gauge.getValue().incrementAndGet() etc instead. + public final Gauge filesToUpload; + public final Gauge bytesToUpload; + public final Gauge filesUploaded; + public final Gauge bytesUploaded; + public final Gauge filesRemaining; + public final Gauge bytesRemaining; + public final Gauge filesToRetain; + public final Gauge bytesToRetain; + public final Counter uploadRate; + + // per store breakdowns + public final Map storeDirDiffNs; + public final Map storeUploadNs; + + public final Map> storeFilesToUpload; + public final Map> storeFilesToRetain; + public final Map> storeFilesToRemove; + public final Map> storeSubDirsToUpload; + public final Map> storeSubDirsToRetain; + public final Map> storeSubDirsToRemove; + public final Map> storeBytesToUpload; + public final Map> storeBytesToRetain; + public final Map> storeBytesToRemove; + + public final Timer cleanupNs; + + // TODO shesharm LOW move to SamzaHistogram + public final Timer avgFileUploadNs; // avg time for each file uploaded + public final Timer avgFileSizeBytes; // avg size of each file uploaded + + public BlobStoreBackupManagerMetrics(MetricsRegistry metricsRegistry) { + this.metricsRegistry = metricsRegistry; + + this.initNs = metricsRegistry.newGauge(GROUP, "init-ns", 0L); + + this.uploadNs = metricsRegistry.newTimer(GROUP, "upload-ns"); + + this.filesToUpload = metricsRegistry.newGauge(GROUP, "files-to-upload", new AtomicLong(0L)); + this.bytesToUpload = metricsRegistry.newGauge(GROUP, "bytes-to-upload", new AtomicLong(0L)); + this.filesUploaded = metricsRegistry.newGauge(GROUP, "files-uploaded", new AtomicLong(0L)); + this.bytesUploaded = metricsRegistry.newGauge(GROUP, "bytes-uploaded", new AtomicLong(0L)); + this.filesRemaining = metricsRegistry.newGauge(GROUP, "files-remaining", new AtomicLong(0L)); + this.bytesRemaining = metricsRegistry.newGauge(GROUP, "bytes-remaining", new AtomicLong(0L)); + this.filesToRetain = metricsRegistry.newGauge(GROUP, "files-to-retain", new AtomicLong(0L)); + this.bytesToRetain = metricsRegistry.newGauge(GROUP, "bytes-to-retain", new AtomicLong(0L)); + + this.storeDirDiffNs = new ConcurrentHashMap<>(); + this.storeUploadNs = new ConcurrentHashMap<>(); + + this.storeFilesToUpload = new ConcurrentHashMap<>(); + this.storeFilesToRetain = new ConcurrentHashMap<>(); + this.storeFilesToRemove = new ConcurrentHashMap<>(); + this.storeSubDirsToUpload = new ConcurrentHashMap<>(); + this.storeSubDirsToRetain = new ConcurrentHashMap<>(); + this.storeSubDirsToRemove = new ConcurrentHashMap<>(); + this.storeBytesToUpload = new ConcurrentHashMap<>(); + this.storeBytesToRetain = new ConcurrentHashMap<>(); + this.storeBytesToRemove = new ConcurrentHashMap<>(); + + this.uploadRate = metricsRegistry.newCounter(GROUP, "upload-rate"); + + this.cleanupNs = metricsRegistry.newTimer(GROUP, "cleanup-ns"); + + this.avgFileUploadNs = metricsRegistry.newTimer(GROUP, "avg-file-upload-ns"); + this.avgFileSizeBytes = metricsRegistry.newTimer(GROUP, "avg-file-size-bytes"); + } + + public void initStoreMetrics(Collection storeNames) { + for (String storeName: storeNames) { + storeDirDiffNs.putIfAbsent(storeName, + metricsRegistry.newTimer(GROUP, String.format("%s-dir-diff-ns", storeName))); + storeUploadNs.putIfAbsent(storeName, + metricsRegistry.newTimer(GROUP, String.format("%s-upload-ns", storeName))); + + storeFilesToUpload.putIfAbsent(storeName, + metricsRegistry.newGauge(GROUP, String.format("%s-files-to-upload", storeName), 0L)); + storeFilesToRetain.putIfAbsent(storeName, + metricsRegistry.newGauge(GROUP, String.format("%s-files-to-retain", storeName), 0L)); + storeFilesToRemove.putIfAbsent(storeName, + metricsRegistry.newGauge(GROUP, String.format("%s-files-to-remove", storeName), 0L)); + storeSubDirsToUpload.putIfAbsent(storeName, + metricsRegistry.newGauge(GROUP, String.format("%s-sub-dirs-to-upload", storeName), 0L)); + storeSubDirsToRetain.putIfAbsent(storeName, + metricsRegistry.newGauge(GROUP, String.format("%s-sub-dirs-to-retain", storeName), 0L)); + storeSubDirsToRemove.putIfAbsent(storeName, + metricsRegistry.newGauge(GROUP, String.format("%s-sub-dirs-to-remove", storeName), 0L)); + storeBytesToUpload.putIfAbsent(storeName, + metricsRegistry.newGauge(GROUP, String.format("%s-bytes-to-upload", storeName), 0L)); + storeBytesToRetain.putIfAbsent(storeName, + metricsRegistry.newGauge(GROUP, String.format("%s-bytes-to-retain", storeName), 0L)); + storeBytesToRemove.putIfAbsent(storeName, + metricsRegistry.newGauge(GROUP, String.format("%s-bytes-to-remove", storeName), 0L)); + } + } +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/metrics/BlobStoreRestoreManagerMetrics.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/metrics/BlobStoreRestoreManagerMetrics.java new file mode 100644 index 0000000000..ac07f608a9 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/metrics/BlobStoreRestoreManagerMetrics.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.metrics; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.samza.metrics.Counter; +import org.apache.samza.metrics.Gauge; +import org.apache.samza.metrics.MetricsRegistry; +import org.apache.samza.metrics.Timer; + + +public class BlobStoreRestoreManagerMetrics { + private static final String GROUP = BlobStoreRestoreManagerMetrics.class.getName(); + private final MetricsRegistry metricsRegistry; + + // TODO LOW shesharma per-task throughput + public final Gauge initNs; + public final Gauge getSnapshotIndexNs; + + public final Gauge restoreNs; + // gauges of AtomicLong so that the value can be incremented/decremented atomically in a thread-safe way. + // don't set the gauge value directly. use gauge.getValue().incrementAndGet() etc instead. + public final Gauge filesToRestore; + public final Gauge bytesToRestore; + public final Gauge filesRestored; + public final Gauge bytesRestored; + public final Gauge filesRemaining; + public final Gauge bytesRemaining; + + public final Counter restoreRate; + + // per store breakdowns + public final Map> storePreRestoreNs; + public final Map> storeRestoreNs; + public final Map> storePostRestoreNs; + + // TODO LOW shesharma move to SamzaHistogram + public final Timer avgFileRestoreNs; // avg time for each file restored + + public BlobStoreRestoreManagerMetrics(MetricsRegistry metricsRegistry) { + this.metricsRegistry = metricsRegistry; + + this.initNs = metricsRegistry.newGauge(GROUP, "init-ns", 0L); + this.getSnapshotIndexNs = metricsRegistry.newGauge(GROUP, "get-snapshot-index-ns", 0L); + + this.restoreNs = metricsRegistry.newGauge(GROUP, "restore-ns", 0L); + this.filesToRestore = metricsRegistry.newGauge(GROUP, "files-to-restore", new AtomicLong()); + this.bytesToRestore = metricsRegistry.newGauge(GROUP, "bytes-to-restore", new AtomicLong()); + this.filesRestored = metricsRegistry.newGauge(GROUP, "files-restored", new AtomicLong()); + this.bytesRestored = metricsRegistry.newGauge(GROUP, "bytes-restored", new AtomicLong()); + this.filesRemaining = metricsRegistry.newGauge(GROUP, "files-remaining", new AtomicLong()); + this.bytesRemaining = metricsRegistry.newGauge(GROUP, "bytes-remaining", new AtomicLong()); + + + this.restoreRate = metricsRegistry.newCounter(GROUP, "restore-rate"); + + this.storePreRestoreNs = new ConcurrentHashMap<>(); + this.storeRestoreNs = new ConcurrentHashMap<>(); + this.storePostRestoreNs = new ConcurrentHashMap<>(); + + this.avgFileRestoreNs = metricsRegistry.newTimer(GROUP, "avg-file-restore-ns"); + } + + public void initStoreMetrics(Collection storeNames) { + for (String storeName: storeNames) { + storePreRestoreNs.putIfAbsent(storeName, + metricsRegistry.newGauge(GROUP, String.format("%s-pre-restore-ns", storeName), 0L)); + storeRestoreNs.putIfAbsent(storeName, + metricsRegistry.newGauge(GROUP, String.format("%s-restore-ns", storeName), 0L)); + storePostRestoreNs.putIfAbsent(storeName, + metricsRegistry.newGauge(GROUP, String.format("%s-post-restore-ns", storeName), 0L)); + } + } +} diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/util/BlobStoreUtil.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/util/BlobStoreUtil.java new file mode 100644 index 0000000000..b312b368a1 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/util/BlobStoreUtil.java @@ -0,0 +1,622 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.util; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutorService; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.zip.CRC32; +import java.util.zip.CheckedInputStream; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.samza.SamzaException; +import org.apache.samza.checkpoint.Checkpoint; +import org.apache.samza.checkpoint.CheckpointV2; +import org.apache.samza.storage.blobstore.BlobStoreManager; +import org.apache.samza.storage.blobstore.BlobStoreStateBackendFactory; +import org.apache.samza.storage.blobstore.Metadata; +import org.apache.samza.storage.blobstore.diff.DirDiff; +import org.apache.samza.storage.blobstore.exceptions.DeletedException; +import org.apache.samza.storage.blobstore.exceptions.RetriableException; +import org.apache.samza.storage.blobstore.index.DirIndex; +import org.apache.samza.storage.blobstore.index.FileBlob; +import org.apache.samza.storage.blobstore.index.FileIndex; +import org.apache.samza.storage.blobstore.index.FileMetadata; +import org.apache.samza.storage.blobstore.index.SnapshotIndex; +import org.apache.samza.storage.blobstore.index.SnapshotMetadata; +import org.apache.samza.storage.blobstore.index.serde.SnapshotIndexSerde; +import org.apache.samza.storage.blobstore.metrics.BlobStoreBackupManagerMetrics; +import org.apache.samza.storage.blobstore.metrics.BlobStoreRestoreManagerMetrics; +import org.apache.samza.util.FutureUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Helper methods to interact with remote blob store service and GET/PUT/DELETE a + * {@link SnapshotIndex} or {@link DirDiff}. + */ +public class BlobStoreUtil { + private static final Logger LOG = LoggerFactory.getLogger(BlobStoreUtil.class); + + private final BlobStoreManager blobStoreManager; + private final ExecutorService executor; + private final BlobStoreBackupManagerMetrics backupMetrics; + private final BlobStoreRestoreManagerMetrics restoreMetrics; + + public BlobStoreUtil(BlobStoreManager blobStoreManager, ExecutorService executor, + BlobStoreBackupManagerMetrics backupMetrics, BlobStoreRestoreManagerMetrics restoreMetrics) { + this.blobStoreManager = blobStoreManager; + this.executor = executor; + this.backupMetrics = backupMetrics; + this.restoreMetrics = restoreMetrics; + } + + /** + * Get the blob id of {@link SnapshotIndex} and {@link SnapshotIndex}es for the provided {@code task} + * in the provided {@code checkpoint}. + * @param jobName job name is used to build request metadata + * @param jobId job id is used to build request metadata + * @param taskName task name to get the store state checkpoint markers and snapshot indexes for + * @param checkpoint {@link Checkpoint} instance to get the store state checkpoint markers from. Only + * {@link CheckpointV2} and newer are supported for blob stores. + * @return Map of store name to its blob id of snapshot indices and their corresponding snapshot indices for the task. + */ + public Map> getStoreSnapshotIndexes( + String jobName, String jobId, String taskName, Checkpoint checkpoint) { + //TODO MED shesharma document error handling (checkpoint ver, blob not found, getBlob) + if (checkpoint == null) { + LOG.debug("No previous checkpoint found for taskName: {}", taskName); + return ImmutableMap.of(); + } + + if (checkpoint.getVersion() == 1) { + throw new SamzaException("Checkpoint version 1 is not supported for blob store backup and restore."); + } + + Map>> + storeSnapshotIndexFutures = new HashMap<>(); + + CheckpointV2 checkpointV2 = (CheckpointV2) checkpoint; + Map> factoryToStoreSCMs = checkpointV2.getStateCheckpointMarkers(); + Map storeSnapshotIndexBlobIds = factoryToStoreSCMs.get(BlobStoreStateBackendFactory.class.getName()); + + if (storeSnapshotIndexBlobIds != null) { + storeSnapshotIndexBlobIds.forEach((storeName, snapshotIndexBlobId) -> { + try { + LOG.debug("Getting snapshot index for taskName: {} store: {} blobId: {}", taskName, storeName, snapshotIndexBlobId); + Metadata requestMetadata = + new Metadata(Metadata.SNAPSHOT_INDEX_PAYLOAD_PATH, Optional.empty(), jobName, jobId, taskName, storeName); + CompletableFuture snapshotIndexFuture = + getSnapshotIndex(snapshotIndexBlobId, requestMetadata).toCompletableFuture(); + Pair, CompletableFuture> pairOfFutures = + Pair.of(CompletableFuture.completedFuture(snapshotIndexBlobId), snapshotIndexFuture); + + // save the future and block once in the end instead of blocking for each request. + storeSnapshotIndexFutures.put(storeName, FutureUtil.toFutureOfPair(pairOfFutures)); + } catch (Exception e) { + throw new SamzaException( + String.format("Error getting SnapshotIndex for blobId: %s for taskName: %s store: %s", + snapshotIndexBlobId, taskName, storeName), e); + } + }); + } else { + LOG.debug("No store SCMs found for blob store state backend in for taskName: {} in checkpoint {}", + taskName, checkpointV2.getCheckpointId()); + } + + try { + return FutureUtil.toFutureOfMap(t -> { + Throwable unwrappedException = FutureUtil.unwrapExceptions(CompletionException.class, t); + if (unwrappedException instanceof DeletedException) { + LOG.warn("Ignoring already deleted snapshot index for taskName: {}", taskName, t); + return true; + } else { + return false; + } + }, storeSnapshotIndexFutures).join(); + } catch (Exception e) { + throw new SamzaException( + String.format("Error while waiting to get store snapshot indexes for task %s", taskName), e); + } + } + + /** + * GETs the {@link SnapshotIndex} from the blob store. + * @param blobId blob ID of the {@link SnapshotIndex} to get + * @return a Future containing the {@link SnapshotIndex} + */ + public CompletableFuture getSnapshotIndex(String blobId, Metadata metadata) { + Preconditions.checkState(StringUtils.isNotBlank(blobId)); + String opName = "getSnapshotIndex: " + blobId; + return FutureUtil.executeAsyncWithRetries(opName, () -> { + ByteArrayOutputStream indexBlobStream = new ByteArrayOutputStream(); // no need to close ByteArrayOutputStream + return blobStoreManager.get(blobId, indexBlobStream, metadata).toCompletableFuture() + .thenApplyAsync(f -> new SnapshotIndexSerde().fromBytes(indexBlobStream.toByteArray()), executor); + }, isCauseNonRetriable(), executor); + } + + /** + * PUTs the {@link SnapshotIndex} to the blob store. + * @param snapshotIndex SnapshotIndex to put. + * @return a Future containing the blob ID of the {@link SnapshotIndex}. + */ + public CompletableFuture putSnapshotIndex(SnapshotIndex snapshotIndex) { + byte[] bytes = new SnapshotIndexSerde().toBytes(snapshotIndex); + String opName = "putSnapshotIndex for checkpointId: " + snapshotIndex.getSnapshotMetadata().getCheckpointId(); + return FutureUtil.executeAsyncWithRetries(opName, () -> { + InputStream inputStream = new ByteArrayInputStream(bytes); // no need to close ByteArrayInputStream + SnapshotMetadata snapshotMetadata = snapshotIndex.getSnapshotMetadata(); + Metadata metadata = new Metadata(Metadata.SNAPSHOT_INDEX_PAYLOAD_PATH, Optional.of((long) bytes.length), + snapshotMetadata.getJobName(), snapshotMetadata.getJobId(), snapshotMetadata.getTaskName(), + snapshotMetadata.getStoreName()); + return blobStoreManager.put(inputStream, metadata).toCompletableFuture(); + }, isCauseNonRetriable(), executor); + } + + /** + * WARNING: This method deletes the **SnapshotIndex blob** from the snapshot. This should only be called to clean + * up an older snapshot **AFTER** all the files and sub-dirs to be deleted from this snapshot are already deleted + * using {@link #cleanUpDir(DirIndex, Metadata)} + * + * @param snapshotIndexBlobId blob ID of SnapshotIndex blob to delete + * @return a future that completes when the index blob is deleted from remote store. + */ + public CompletionStage deleteSnapshotIndexBlob(String snapshotIndexBlobId, Metadata metadata) { + Preconditions.checkState(StringUtils.isNotBlank(snapshotIndexBlobId)); + LOG.debug("Deleting SnapshotIndex blob: {} from blob store", snapshotIndexBlobId); + String opName = "deleteSnapshotIndexBlob: " + snapshotIndexBlobId; + return FutureUtil.executeAsyncWithRetries(opName, () -> + blobStoreManager.delete(snapshotIndexBlobId, metadata).toCompletableFuture(), isCauseNonRetriable(), executor); + } + + /** + * Non-blocking restore of a {@link SnapshotIndex} to local store by downloading all the files and sub-dirs associated + * with this remote snapshot. + * @return A future that completes when all the async downloads completes + */ + public CompletableFuture restoreDir(File baseDir, DirIndex dirIndex, Metadata metadata) { + LOG.debug("Restoring contents of directory: {} from remote snapshot.", baseDir); + + List> downloadFutures = new ArrayList<>(); + + try { + // create parent directories if they don't exist + Files.createDirectories(baseDir.toPath()); + } catch (IOException exception) { + LOG.error("Error creating directory: {} for restore", baseDir.getAbsolutePath(), exception); + throw new SamzaException(String.format("Error creating directory: %s for restore", + baseDir.getAbsolutePath()), exception); + } + + // restore all files in the directory + for (FileIndex fileIndex : dirIndex.getFilesPresent()) { + File fileToRestore = Paths.get(baseDir.getAbsolutePath(), fileIndex.getFileName()).toFile(); + Metadata requestMetadata = + new Metadata(fileToRestore.getAbsolutePath(), Optional.of(fileIndex.getFileMetadata().getSize()), + metadata.getJobName(), metadata.getJobId(), metadata.getTaskName(), metadata.getStoreName()); + List fileBlobs = fileIndex.getBlobs(); + + String opName = "restoreFile: " + fileToRestore.getAbsolutePath(); + CompletableFuture fileRestoreFuture = + FutureUtil.executeAsyncWithRetries(opName, () -> getFile(fileBlobs, fileToRestore, requestMetadata), + isCauseNonRetriable(), executor); + downloadFutures.add(fileRestoreFuture); + } + + // restore any sub-directories + List subDirs = dirIndex.getSubDirsPresent(); + for (DirIndex subDir : subDirs) { + File subDirFile = Paths.get(baseDir.getAbsolutePath(), subDir.getDirName()).toFile(); + downloadFutures.add(restoreDir(subDirFile, subDir, metadata)); + } + + return FutureUtil.allOf(downloadFutures); + } + + /** + * Recursively upload all new files and upload or update contents of all subdirs in the {@link DirDiff} and return a + * Future containing the {@link DirIndex} associated with the directory. + * @param dirDiff diff for the contents of this directory + * @return A future with the {@link DirIndex} if the upload completed successfully. + */ + public CompletionStage putDir(DirDiff dirDiff, SnapshotMetadata snapshotMetadata) { + // Upload all new files in the dir + List filesToUpload = dirDiff.getFilesAdded(); + List> fileFutures = filesToUpload.stream() + .map(file -> putFile(file, snapshotMetadata)) + .collect(Collectors.toList()); + + CompletableFuture allFilesFuture = + CompletableFuture.allOf(fileFutures.toArray(new CompletableFuture[0])); + + List> subDirFutures = new ArrayList<>(); + // recursively upload all new subdirs of this dir + for (DirDiff subDirAdded: dirDiff.getSubDirsAdded()) { + subDirFutures.add(putDir(subDirAdded, snapshotMetadata)); + } + // recursively update contents of all subdirs that are retained but might have been modified + for (DirDiff subDirRetained: dirDiff.getSubDirsRetained()) { + subDirFutures.add(putDir(subDirRetained, snapshotMetadata)); + } + CompletableFuture allDirBlobsFuture = + CompletableFuture.allOf(subDirFutures.toArray(new CompletableFuture[0])); + + return CompletableFuture.allOf(allDirBlobsFuture, allFilesFuture) + .thenApplyAsync(f -> { + LOG.trace("All file and dir uploads complete for task: {} store: {}", + snapshotMetadata.getTaskName(), snapshotMetadata.getStoreName()); + List filesPresent = fileFutures.stream() + .map(blob -> blob.toCompletableFuture().join()) + .collect(Collectors.toList()); + + filesPresent.addAll(dirDiff.getFilesRetained()); + + List subDirsPresent = subDirFutures.stream() + .map(subDir -> subDir.toCompletableFuture().join()) + .collect(Collectors.toList()); + + LOG.debug("Uploaded diff for task: {} store: {} with statistics: {}", + snapshotMetadata.getTaskName(), snapshotMetadata.getStoreName(), DirDiff.getStats(dirDiff)); + + LOG.trace("Returning new DirIndex for task: {} store: {}", + snapshotMetadata.getTaskName(), snapshotMetadata.getStoreName()); + return new DirIndex(dirDiff.getDirName(), + filesPresent, + dirDiff.getFilesRemoved(), + subDirsPresent, + dirDiff.getSubDirsRemoved()); + }, executor); + } + + /** + * WARNING: Recursively delete **ALL** the associated files and subdirs within the provided {@link DirIndex}. + * @param dirIndex {@link DirIndex} whose entire contents are to be deleted. + * @param metadata {@link Metadata} related to the request + * @return a future that completes when ALL the files and subdirs associated with the dirIndex have been + * marked for deleted in the remote blob store. + */ + public CompletionStage deleteDir(DirIndex dirIndex, Metadata metadata) { + LOG.debug("Completely deleting dir: {} in blob store", dirIndex.getDirName()); + List> deleteFutures = new ArrayList<>(); + // Delete all files present in subDir + for (FileIndex file: dirIndex.getFilesPresent()) { + Metadata requestMetadata = + new Metadata(file.getFileName(), Optional.of(file.getFileMetadata().getSize()), + metadata.getJobName(), metadata.getJobId(), metadata.getTaskName(), metadata.getStoreName()); + deleteFutures.add(deleteFile(file, requestMetadata)); + } + + // Delete all subDirs present recursively + for (DirIndex subDir: dirIndex.getSubDirsPresent()) { + deleteFutures.add(deleteDir(subDir, metadata)); + } + + return CompletableFuture.allOf(deleteFutures.toArray(new CompletableFuture[0])); + } + + + /** + * Recursively issue delete requests for files and dirs marked to be removed in a previously created remote snapshot. + * Note: We do not immediately delete files/dirs to be removed when uploading a snapshot to the remote + * store. We just track them for deletion during the upload, and delete them AFTER the snapshot is uploaded, and the + * blob IDs have been persisted as part of the checkpoint. This is to prevent data loss if a failure happens + * part way through the commit. We issue delete these file/subdirs in cleanUp() phase of commit lifecycle. + * @param dirIndex the dir in the remote snapshot to clean up. + * @param metadata Metadata related to the request + * @return a future that completes when all the files and subdirs marked for deletion are cleaned up. + */ + public CompletionStage cleanUpDir(DirIndex dirIndex, Metadata metadata) { + String dirName = dirIndex.getDirName(); + if (DirIndex.ROOT_DIR_NAME.equals(dirName)) { + LOG.debug("Cleaning up root dir in blob store."); + } else { + LOG.debug("Cleaning up dir: {} in blob store.", dirIndex.getDirName()); + } + + List> cleanUpFuture = new ArrayList<>(); + List files = dirIndex.getFilesRemoved(); + for (FileIndex file: files) { + Metadata requestMetadata = + new Metadata(file.getFileName(), Optional.of(file.getFileMetadata().getSize()), metadata.getJobName(), + metadata.getJobId(), metadata.getTaskName(), metadata.getStoreName()); + cleanUpFuture.add(deleteFile(file, requestMetadata)); + } + + for (DirIndex subDirToDelete : dirIndex.getSubDirsRemoved()) { + // recursively delete ALL contents of the subDirToDelete. + cleanUpFuture.add(deleteDir(subDirToDelete, metadata)); + } + + for (DirIndex subDirToRetain : dirIndex.getSubDirsPresent()) { + // recursively clean up the subDir, only deleting files and subdirs marked for deletion. + cleanUpFuture.add(cleanUpDir(subDirToRetain, metadata)); + } + + return CompletableFuture.allOf(cleanUpFuture.toArray(new CompletableFuture[0])); + } + + /** + * Gets a file from the blob store. + * @param fileBlobs List of {@link FileBlob}s that constitute this file. + * @param fileToRestore File pointing to the local path where the file will be restored. + * @param requestMetadata {@link Metadata} associated with this request + * @return a future that completes when the file is downloaded and written or if an exception occurs. + */ + @VisibleForTesting + CompletableFuture getFile(List fileBlobs, File fileToRestore, Metadata requestMetadata) { + FileOutputStream outputStream = null; + try { + long restoreFileStartTime = System.nanoTime(); + if (fileToRestore.exists()) { + // delete the file if it already exists, e.g. from a previous retry. + Files.delete(fileToRestore.toPath()); + } + + outputStream = new FileOutputStream(fileToRestore); + final FileOutputStream finalOutputStream = outputStream; + // TODO HIGH shesharm add integration tests to ensure empty files and directories are handled correctly E2E. + fileToRestore.createNewFile(); // create file for 0 byte files (fileIndex entry but no fileBlobs). + // create a copy to ensure list being sorted is mutable. + List fileBlobsCopy = new ArrayList<>(fileBlobs); + fileBlobsCopy.sort(Comparator.comparingInt(FileBlob::getOffset)); // sort by offset. + + // chain the futures such that write to file for blobs is sequential. + // can be optimized to write concurrently to the file later. + CompletableFuture resultFuture = CompletableFuture.completedFuture(null); + for (FileBlob fileBlob : fileBlobsCopy) { + resultFuture = resultFuture.thenComposeAsync(v -> { + LOG.debug("Starting restore for file: {} with blob id: {} at offset: {}", fileToRestore, fileBlob.getBlobId(), + fileBlob.getOffset()); + return blobStoreManager.get(fileBlob.getBlobId(), finalOutputStream, requestMetadata); + }, executor); + } + + resultFuture = resultFuture.thenRunAsync(() -> { + LOG.debug("Finished restore for file: {}. Closing output stream.", fileToRestore); + try { + // flush the file contents to disk + finalOutputStream.getFD().sync(); + finalOutputStream.close(); + } catch (Exception e) { + throw new SamzaException(String.format("Error closing output stream for file: %s", fileToRestore.getAbsolutePath()), e); + } + }, executor); + + resultFuture.whenComplete((res, ex) -> { + if (restoreMetrics != null) { + restoreMetrics.avgFileRestoreNs.update(System.nanoTime() - restoreFileStartTime); + + long fileSize = requestMetadata.getPayloadSize(); + restoreMetrics.restoreRate.inc(fileSize); + restoreMetrics.filesRestored.getValue().addAndGet(1); + restoreMetrics.bytesRestored.getValue().addAndGet(fileSize); + restoreMetrics.filesRemaining.getValue().addAndGet(-1); + restoreMetrics.bytesRemaining.getValue().addAndGet(-1 * fileSize); + } + }); + return resultFuture; + } catch (Exception exception) { + try { + if (outputStream != null) { + outputStream.close(); + } + } catch (Exception err) { + LOG.error("Error closing output stream for file: {}", fileToRestore.getAbsolutePath(), err); + } + + throw new SamzaException(String.format("Error restoring file: %s in path: %s", + fileToRestore.getName(), requestMetadata.getPayloadPath()), exception); + } + } + + /** + * Upload a File to blob store. + * @param file File to upload to blob store. + * @return A future containing the {@link FileIndex} for the uploaded file. + */ + @VisibleForTesting + public CompletableFuture putFile(File file, SnapshotMetadata snapshotMetadata) { + if (file == null || !file.isFile()) { + String message = file != null ? "Dir or Symbolic link" : "null"; + throw new SamzaException(String.format("Required a non-null parameter of type file, provided: %s", message)); + } + long putFileStartTime = System.nanoTime(); + + String opName = "putFile: " + file.getAbsolutePath(); + Supplier> fileUploadAction = () -> { + LOG.debug("Putting file: {} to blob store.", file.getPath()); + CompletableFuture fileBlobFuture; + CheckedInputStream inputStream = null; + try { + // TODO HIGH shesharm maybe use the more efficient CRC32C / PureJavaCRC32 impl + inputStream = new CheckedInputStream(new FileInputStream(file), new CRC32()); + CheckedInputStream finalInputStream = inputStream; + FileMetadata fileMetadata = FileMetadata.fromFile(file); + if (backupMetrics != null) { + backupMetrics.avgFileSizeBytes.update(fileMetadata.getSize()); + } + + Metadata metadata = + new Metadata(file.getAbsolutePath(), Optional.of(fileMetadata.getSize()), snapshotMetadata.getJobName(), + snapshotMetadata.getJobId(), snapshotMetadata.getTaskName(), snapshotMetadata.getStoreName()); + + fileBlobFuture = blobStoreManager.put(inputStream, metadata) + .thenApplyAsync(id -> { + LOG.trace("Put complete. Received Blob ID {}. Closing input stream for file: {}.", id, file.getPath()); + try { + finalInputStream.close(); + } catch (Exception e) { + throw new SamzaException(String.format("Error closing input stream for file: %s", + file.getAbsolutePath()), e); + } + + LOG.trace("Returning new FileIndex for file: {}.", file.getPath()); + return new FileIndex( + file.getName(), + Collections.singletonList(new FileBlob(id, 0)), + fileMetadata, + finalInputStream.getChecksum().getValue()); + }, executor).toCompletableFuture(); + } catch (Exception e) { + try { + if (inputStream != null) { + inputStream.close(); + } + } catch (Exception err) { + LOG.error("Error closing input stream for file: {}", file.getName(), err); + } + LOG.error("Error putting file: {}", file.getName(), e); + throw new SamzaException(String.format("Error putting file %s", file.getAbsolutePath()), e); + } + return fileBlobFuture; + }; + + return FutureUtil.executeAsyncWithRetries(opName, fileUploadAction, isCauseNonRetriable(), executor) + .whenComplete((res, ex) -> { + if (backupMetrics != null) { + backupMetrics.avgFileUploadNs.update(System.nanoTime() - putFileStartTime); + + long fileSize = file.length(); + backupMetrics.uploadRate.inc(fileSize); + backupMetrics.filesUploaded.getValue().addAndGet(1); + backupMetrics.bytesUploaded.getValue().addAndGet(fileSize); + backupMetrics.filesRemaining.getValue().addAndGet(-1); + backupMetrics.bytesRemaining.getValue().addAndGet(-1 * fileSize); + } + }); + } + + /** + * Delete a {@link FileIndex} from the remote store by deleting all {@link FileBlob}s associated with it. + * @param fileIndex FileIndex of the file to delete from the remote store. + * @param metadata + * @return a future that completes when the FileIndex has been marked for deletion in the remote blob store. + */ + private CompletionStage deleteFile(FileIndex fileIndex, Metadata metadata) { + List> deleteFutures = new ArrayList<>(); + List fileBlobs = fileIndex.getBlobs(); + for (FileBlob fileBlob : fileBlobs) { + LOG.debug("Deleting file: {} blobId: {} from blob store.", fileIndex.getFileName(), fileBlob.getBlobId()); + String opName = "deleteFile: " + fileIndex.getFileName() + " blobId: " + fileBlob.getBlobId(); + Supplier> fileDeletionAction = () -> + blobStoreManager.delete(fileBlob.getBlobId(), metadata).toCompletableFuture(); + CompletableFuture fileDeletionFuture = + FutureUtil.executeAsyncWithRetries(opName, fileDeletionAction, isCauseNonRetriable(), executor); + deleteFutures.add(fileDeletionFuture); + } + + return CompletableFuture.allOf(deleteFutures.toArray(new CompletableFuture[0])); + } + + /** + * Recursively mark all the blobs associated with the {@link DirIndex} to never expire (remove TTL). + * @param dirIndex the {@link DirIndex} whose contents' TTL needs to be removed + * @param metadata {@link Metadata} related to the request + * @return A future that completes when all the blobs associated with this dirIndex are marked to + * never expire. + */ + private CompletableFuture removeTTL(DirIndex dirIndex, Metadata metadata) { + String dirName = dirIndex.getDirName(); + if (DirIndex.ROOT_DIR_NAME.equals(dirName)) { + LOG.debug("Removing TTL for files and dirs present in DirIndex for root dir."); + } else { + LOG.debug("Removing TTL for files and dirs present in DirIndex for dir: {}", dirName); + } + + List> updateTTLsFuture = new ArrayList<>(); + for (DirIndex subDir: dirIndex.getSubDirsPresent()) { + updateTTLsFuture.add(removeTTL(subDir, metadata)); + } + + for (FileIndex file: dirIndex.getFilesPresent()) { + Metadata requestMetadata = + new Metadata(file.getFileName(), Optional.of(file.getFileMetadata().getSize()), + metadata.getJobName(), metadata.getJobId(), metadata.getTaskName(), metadata.getStoreName()); + List fileBlobs = file.getBlobs(); + for (FileBlob fileBlob : fileBlobs) { + String opname = "removeTTL for fileBlob: " + file.getFileName() + " with blobId: {}" + fileBlob.getBlobId(); + Supplier> ttlRemovalAction = () -> + blobStoreManager.removeTTL(fileBlob.getBlobId(), requestMetadata).toCompletableFuture(); + CompletableFuture ttlRemovalFuture = + FutureUtil.executeAsyncWithRetries(opname, ttlRemovalAction, isCauseNonRetriable(), executor); + updateTTLsFuture.add(ttlRemovalFuture); + } + } + + return CompletableFuture.allOf(updateTTLsFuture.toArray(new CompletableFuture[0])); + } + + + /** + * Marks all the blobs associated with an {@link SnapshotIndex} to never expire. + * @param snapshotIndex {@link SnapshotIndex} of the remote snapshot + * @param metadata {@link Metadata} related to the request + * @return A future that completes when all the files and subdirs associated with this remote snapshot are marked to + * never expire. + */ + public CompletionStage removeTTL(String indexBlobId, SnapshotIndex snapshotIndex, Metadata metadata) { + SnapshotMetadata snapshotMetadata = snapshotIndex.getSnapshotMetadata(); + LOG.debug("Marking contents of SnapshotIndex: {} to never expire", snapshotMetadata.toString()); + + String opName = "removeTTL for SnapshotIndex for checkpointId: " + snapshotMetadata.getCheckpointId(); + Supplier> removeDirIndexTTLAction = + () -> removeTTL(snapshotIndex.getDirIndex(), metadata).toCompletableFuture(); + CompletableFuture dirIndexTTLRemovalFuture = + FutureUtil.executeAsyncWithRetries(opName, removeDirIndexTTLAction, isCauseNonRetriable(), executor); + + return dirIndexTTLRemovalFuture.thenComposeAsync(aVoid -> { + String op2Name = "removeTTL for indexBlobId: " + indexBlobId; + Supplier> removeIndexBlobTTLAction = + () -> blobStoreManager.removeTTL(indexBlobId, metadata).toCompletableFuture(); + return FutureUtil.executeAsyncWithRetries(op2Name, removeIndexBlobTTLAction, isCauseNonRetriable(), executor); + }, executor); + } + + private static Predicate isCauseNonRetriable() { + return throwable -> { + Throwable unwrapped = FutureUtil.unwrapExceptions(CompletionException.class, throwable); + return unwrapped != null && !RetriableException.class.isAssignableFrom(unwrapped.getClass()); + }; + } +} \ No newline at end of file diff --git a/samza-core/src/main/java/org/apache/samza/storage/blobstore/util/DirDiffUtil.java b/samza-core/src/main/java/org/apache/samza/storage/blobstore/util/DirDiffUtil.java new file mode 100644 index 0000000000..f09b56f8a2 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/storage/blobstore/util/DirDiffUtil.java @@ -0,0 +1,403 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.util; + +import com.google.common.base.Preconditions; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.BiPredicate; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.zip.CRC32; +import java.util.zip.CheckedInputStream; +import org.apache.commons.lang3.StringUtils; +import org.apache.samza.SamzaException; +import org.apache.samza.storage.blobstore.diff.DirDiff; +import org.apache.samza.storage.blobstore.index.DirIndex; +import org.apache.samza.storage.blobstore.index.FileIndex; +import org.apache.samza.storage.blobstore.index.FileMetadata; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Provides helper methods to create a {@link DirDiff} between local and remote snapshots. + */ +public class DirDiffUtil { + private static final Logger LOG = LoggerFactory.getLogger(DirDiffUtil.class); + + /** + * Checks if a local directory and a remote directory are identical. Local and remote directories are identical iff: + * 1. The local directory has exactly the same set of files as the remote directory, and the files are themselves + * identical, as determined by {@link #areSameFile(boolean)}, except for those allowed to differ according to + * {@code filesToIgnore}. + * 2. The local directory has exactly the same set of sub-directories as the remote directory. + * + * @param filesToIgnore a set of file names to ignore during the directory comparisons + * (does not exclude directory names) + * @param compareLargeFileChecksums whether to compare checksums for large files (> 1 MB). + * @return boolean indicating whether the local and remote directory are identical. + */ + // TODO HIGH shesharm add unit tests + public BiPredicate areSameDir(Set filesToIgnore, boolean compareLargeFileChecksums) { + return (localDir, remoteDir) -> { + String remoteDirName = remoteDir.getDirName().equals(DirIndex.ROOT_DIR_NAME) ? "root" : remoteDir.getDirName(); + LOG.debug("Creating diff between local dir: {} and remote dir: {} for comparison.", + localDir.getAbsolutePath(), remoteDirName); + DirDiff dirDiff = DirDiffUtil.getDirDiff(localDir, remoteDir, DirDiffUtil.areSameFile(compareLargeFileChecksums)); + + boolean areSameDir = true; + List filesRemoved = dirDiff.getFilesRemoved().stream() + .map(FileIndex::getFileName) + .filter(name -> !filesToIgnore.contains(name)) + .collect(Collectors.toList()); + + if (!filesRemoved.isEmpty()) { + areSameDir = false; + LOG.error("Local directory: {} is missing files that are present in remote snapshot: {}", + localDir.getAbsolutePath(), StringUtils.join(filesRemoved, ", ")); + } + + List subDirsRemoved = dirDiff.getSubDirsRemoved(); + if (!subDirsRemoved.isEmpty()) { + areSameDir = false; + List missingSubDirs = subDirsRemoved.stream().map(DirIndex::getDirName).collect(Collectors.toList()); + LOG.error("Local directory: {} is missing sub-dirs that are present in remote snapshot: {}", + localDir.getAbsolutePath(), StringUtils.join(missingSubDirs, ", ")); + } + + List filesAdded = dirDiff.getFilesAdded().stream() + .map(File::getName) + .filter(name -> !filesToIgnore.contains(name)) + .collect(Collectors.toList()); + if (!filesAdded.isEmpty()) { + areSameDir = false; + LOG.error("Local directory: {} has additional files that are not present in remote snapshot: {}", + localDir.getAbsolutePath(), StringUtils.join(filesAdded, ", ")); + } + + List subDirsAdded = dirDiff.getSubDirsAdded(); + if (!subDirsAdded.isEmpty()) { + areSameDir = false; + List addedDirs = subDirsAdded.stream().map(DirDiff::getDirName).collect(Collectors.toList()); + LOG.error("Local directory: {} has additional sub-dirs that are not present in remote snapshot: {}", + localDir.getAbsolutePath(), StringUtils.join(addedDirs, ", ")); + } + + // dir diff calculation already ensures that all retained files are equal (by definition) + // recursively test that all retained sub-dirs are equal as well + Map remoteSubDirs = new HashMap<>(); + for (DirIndex subDir: remoteDir.getSubDirsPresent()) { + remoteSubDirs.put(subDir.getDirName(), subDir); + } + for (DirDiff subDirRetained: dirDiff.getSubDirsRetained()) { + String localSubDirName = subDirRetained.getDirName(); + File localSubDirFile = Paths.get(localDir.getAbsolutePath(), localSubDirName).toFile(); + DirIndex remoteSubDir = remoteSubDirs.get(localSubDirName); + boolean areSameSubDir = areSameDir(filesToIgnore, false).test(localSubDirFile, remoteSubDir); + if (!areSameSubDir) { + LOG.debug("Local sub-dir: {} and remote sub-dir: {} are not same.", + localSubDirFile.getAbsolutePath(), remoteSubDir.getDirName()); + areSameDir = false; + } + } + + LOG.debug("Local dir: {} and remote dir: {} are {}the same.", + localDir.getAbsolutePath(), remoteDirName, areSameDir ? "" : "not "); + return areSameDir; + }; + } + + /** + * Bipredicate to test a local file in the filesystem and a remote file {@link FileIndex} and find out if they represent + * the same file. Files with same attributes as well as content are same file. A SST file in a special case. They are + * immutable, so we only compare their attributes but not the content. + * @param compareLargeFileChecksums whether to compare checksums for large files (> 1 MB). + * @return BiPredicate to test similarity of local and remote files + */ + public static BiPredicate areSameFile(boolean compareLargeFileChecksums) { + return (localFile, remoteFile) -> { + if (localFile.getName().equals(remoteFile.getFileName())) { + FileMetadata remoteFileMetadata = remoteFile.getFileMetadata(); + + PosixFileAttributes localFileAttrs = null; + try { + localFileAttrs = Files.readAttributes(localFile.toPath(), PosixFileAttributes.class); + } catch (IOException e) { + LOG.error("Error reading attributes for file: {}", localFile.getAbsolutePath()); + throw new RuntimeException(String.format("Error reading attributes for file: %s", localFile.getAbsolutePath())); + } + + // Don't compare file timestamps. The ctime of a local file just restored will be different than the + // remote file, and will cause the file to be uploaded again during the first commit after restore. + + boolean areSameFiles = + localFileAttrs.size() == remoteFileMetadata.getSize() && + localFileAttrs.group().getName().equals(remoteFileMetadata.getGroup()) && + localFileAttrs.owner().getName().equals(remoteFileMetadata.getOwner()) && + PosixFilePermissions.toString(localFileAttrs.permissions()).equals(remoteFileMetadata.getPermissions()); + + if (!areSameFiles) { + LOG.debug("Local file: {} and remote file: {} are not same. " + + "Local file attributes: {}. Remote file attributes: {}.", + localFile.getAbsolutePath(), remoteFile.getFileName(), + fileAttributesToString(localFileAttrs), remoteFile.getFileMetadata().toString()); + return false; + } else { + LOG.trace("Local file: {}. Remote file: {}. " + + "Local file attributes: {}. Remote file attributes: {}.", + localFile.getAbsolutePath(), remoteFile.getFileName(), + fileAttributesToString(localFileAttrs), remoteFile.getFileMetadata().toString()); + } + + boolean isLargeFile = localFileAttrs.size() > 1024 * 1024; + if (!compareLargeFileChecksums && isLargeFile) { + // Since RocksDB SST files are immutable after creation, we can skip the expensive checksum computations + // which requires reading the entire file. + LOG.debug("Local file: {} and remote file: {} are same. " + + "Skipping checksum calculation for large file of size: {}.", + localFile.getAbsolutePath(), remoteFile.getFileName(), localFileAttrs.size()); + return true; + } else { + try { + FileInputStream fis = new FileInputStream(localFile); + CheckedInputStream cis = new CheckedInputStream(fis, new CRC32()); + byte[] buffer = new byte[8 * 1024]; // 8 KB + while (cis.read(buffer, 0, buffer.length) >= 0) { } + long localFileChecksum = cis.getChecksum().getValue(); + cis.close(); + + boolean areSameChecksum = localFileChecksum == remoteFile.getChecksum(); + if (!areSameChecksum) { + LOG.debug("Local file: {} and remote file: {} are not same. " + + "Local checksum: {}. Remote checksum: {}", + localFile.getAbsolutePath(), remoteFile.getFileName(), localFileChecksum, remoteFile.getChecksum()); + } else { + LOG.debug("Local file: {} and remote file: {} are same. Local checksum: {}. Remote checksum: {}", + localFile.getAbsolutePath(), remoteFile.getFileName(), localFileChecksum, remoteFile.getChecksum()); + } + return areSameChecksum; + } catch (IOException e) { + throw new SamzaException("Error calculating checksum for local file: " + localFile.getAbsolutePath(), e); + } + } + } + + return false; + }; + } + + /** + * Compare the local snapshot directory and the remote snapshot directory and return the recursive diff of the two as + * a {@link DirDiff}. + * @param localSnapshotDir File representing local snapshot root directory + * @param remoteSnapshotDir {@link DirIndex} representing the remote snapshot directory + * @param areSameFile A BiPredicate to test if a local and remote file are the same file + * @return {@link DirDiff} representing the recursive diff of local and remote snapshots directories + */ + public static DirDiff getDirDiff(File localSnapshotDir, DirIndex remoteSnapshotDir, + BiPredicate areSameFile) { + return getDirDiff(localSnapshotDir, remoteSnapshotDir, areSameFile, true); + } + + private static DirDiff getDirDiff(File localSnapshotDir, DirIndex remoteSnapshotDir, + BiPredicate areSameFile, boolean isRootDir) { + Preconditions.checkState(localSnapshotDir != null && localSnapshotDir.isDirectory()); + Preconditions.checkNotNull(remoteSnapshotDir); + + LOG.debug("Creating DirDiff between local dir: {} and remote dir: {}", + localSnapshotDir.getPath(), remoteSnapshotDir.getDirName()); + List subDirsAdded = new ArrayList<>(); + List subDirsRetained = new ArrayList<>(); + List subDirsRemoved = new ArrayList<>(); + + // list files returns empty list if local snapshot directory is empty + List localSnapshotFiles = Arrays.asList(Objects.requireNonNull(localSnapshotDir.listFiles(File::isFile))); + List remoteSnapshotFiles = remoteSnapshotDir.getFilesPresent(); + + // list files returns empty list if local snapshot directory is empty + List localSnapshotSubDirs = Arrays.asList(Objects.requireNonNull(localSnapshotDir.listFiles(File::isDirectory))); + Set localSnapshotSubDirNames = localSnapshotSubDirs.stream() + .map(File::getName) + .collect(Collectors.toCollection(HashSet::new)); + + List remoteSnapshotSubDirs = remoteSnapshotDir.getSubDirsPresent(); + Set remoteSnapshotSubDirNames = remoteSnapshotSubDirs.stream() + .map(DirIndex::getDirName) + .collect(Collectors.toCollection(HashSet::new)); + + // TODO MED shesharm: this compares each file in directory 3 times. Categorize files in one traversal instead. + List filesToUpload = getNewFilesToUpload(remoteSnapshotFiles, localSnapshotFiles, areSameFile); + List filesToRetain = getFilesToRetain(remoteSnapshotFiles, localSnapshotFiles, areSameFile); + List filesToRemove = getFilesToRemove(remoteSnapshotFiles, localSnapshotFiles, areSameFile); + + for (File localSnapshotSubDir: localSnapshotSubDirs) { + if (!remoteSnapshotSubDirNames.contains(localSnapshotSubDir.getName())) { + LOG.debug("Subdir {} present in local snapshot but not in remote snapshot. " + + "Recursively adding subdir contents.", localSnapshotSubDir.getPath()); + subDirsAdded.add(getDiffForNewDir(localSnapshotSubDir)); + } else { + LOG.debug("Subdir {} present in local snapshot and in remote snapshot. " + + "Recursively comparing local and remote subdirs.", localSnapshotSubDir.getPath()); + DirIndex remoteSubDirIndex = + remoteSnapshotSubDirs.stream() + .filter(indexBlob -> indexBlob.getDirName().equals(localSnapshotSubDir.getName())) + .findFirst().get(); + subDirsRetained.add(getDirDiff(localSnapshotSubDir, remoteSubDirIndex, areSameFile, false)); + } + } + + // 3. Subdir in remote snapshot but not in local snapshot + for (DirIndex remoteSnapshotSubDir: remoteSnapshotSubDirs) { + if (!localSnapshotSubDirNames.contains(remoteSnapshotSubDir.getDirName())) { + LOG.debug("Subdir {} present in remote snapshot but not in local snapshot. " + + "Marking for removal from remote snapshot. ", remoteSnapshotDir.getDirName()); + subDirsRemoved.add(remoteSnapshotSubDir); + } + } + + String dirName = isRootDir ? DirIndex.ROOT_DIR_NAME : localSnapshotDir.getName(); + return new DirDiff(dirName, + filesToUpload, filesToRetain, filesToRemove, + subDirsAdded, subDirsRetained, subDirsRemoved); + } + + /** + * Builds a {@link DirDiff} from a new local directory that is not already present in the remote snapshot. + * @param localSubDir File representing the local directory to create the new {@link DirDiff} for. + */ + private static DirDiff getDiffForNewDir(File localSubDir) { + List filesAdded = new ArrayList<>(); + List subDirsAdded = new ArrayList<>(); + + File[] files = localSubDir.listFiles(); + if (files != null) { + for (File file: files) { + if (file.isFile()) { + LOG.debug("Adding file {} to local sub dir {}", file.getName(), localSubDir.getPath()); + filesAdded.add(file); + } else { + LOG.debug("Adding sub dir {} to sub dir {}", file.getName(), localSubDir.getPath()); + subDirsAdded.add(getDiffForNewDir(file)); + } + } + } + + return new DirDiff(localSubDir.getName(), filesAdded, Collections.emptyList(), Collections.emptyList(), + subDirsAdded, Collections.emptyList(), Collections.emptyList()); + } + + /** + * Returns a list of files uploaded in remote checkpoint that are not present in new local snapshot and needs to be + * deleted/reclaimed from remote store. + */ + private static List getFilesToRemove( + List remoteSnapshotFiles, List localSnapshotFiles, + BiPredicate areSameFile) { + List filesToRemove = new ArrayList<>(); + + Map localFiles = localSnapshotFiles.stream() + .collect(Collectors.toMap(File::getName, Function.identity())); + + for (FileIndex remoteFile : remoteSnapshotFiles) { + String remoteFileName = remoteFile.getFileName(); + if (!localFiles.containsKey(remoteFileName) || + !areSameFile.test(localFiles.get(remoteFileName), remoteFile)) { + LOG.debug("File {} only present in remote snapshot or is not the same as local file.", remoteFile.getFileName()); + filesToRemove.add(remoteFile); + } + } + + return filesToRemove; + } + + /** + * Returns a list of files to be uploaded to remote store that are part of new snapshot created locally. + */ + private static List getNewFilesToUpload( + List remoteSnapshotFiles, List localSnapshotFiles, + BiPredicate areSameFile) { + List filesToUpload = new ArrayList<>(); + + Map remoteFiles = remoteSnapshotFiles.stream() + .collect(Collectors.toMap(FileIndex::getFileName, Function.identity())); + + for (File localFile: localSnapshotFiles) { + String localFileName = localFile.getName(); + if (!remoteFiles.containsKey(localFileName) || + !areSameFile.test(localFile, remoteFiles.get(localFileName))) { + LOG.debug("File {} only present in local snapshot or is not the same as remote file.", localFile.getPath()); + filesToUpload.add(localFile); + } + } + + return filesToUpload; + } + + /** + * Returns a list of common files between local and remote snapshot. These files are reused from prev remote snapshot + * and do not need to be uploaded again. + */ + private static List getFilesToRetain( + List remoteSnapshotFiles, List localSnapshotFiles, + BiPredicate areSameFile) { + List filesToRetain = new ArrayList<>(); + + Map localFiles = localSnapshotFiles.stream() + .collect(Collectors.toMap(File::getName, Function.identity())); + + for (FileIndex remoteFile : remoteSnapshotFiles) { + String remoteFileName = remoteFile.getFileName(); + if (localFiles.containsKey(remoteFileName) && + areSameFile.test(localFiles.get(remoteFileName), remoteFile)) { + String localFilePath = localFiles.get(remoteFileName).getPath(); + LOG.debug("File {} present in both local and remote snapshot and is the same.", localFilePath); + filesToRetain.add(remoteFile); + } + } + + return filesToRetain; + } + + private static String fileAttributesToString(PosixFileAttributes fileAttributes) { + return "PosixFileAttributes{" + + "creationTimeMillis=" + fileAttributes.creationTime().toMillis() + + ", lastModifiedTimeMillis=" + fileAttributes.lastModifiedTime().toMillis() + + ", size=" + fileAttributes.size() + + ", owner='" + fileAttributes.owner() + '\'' + + ", group='" + fileAttributes.group() + '\'' + + ", permissions=" + PosixFilePermissions.toString(fileAttributes.permissions()) + + '}'; + } +} \ No newline at end of file diff --git a/samza-core/src/main/java/org/apache/samza/system/SystemAdmins.java b/samza-core/src/main/java/org/apache/samza/system/SystemAdmins.java index 2ca81caa7e..987e0c6481 100644 --- a/samza-core/src/main/java/org/apache/samza/system/SystemAdmins.java +++ b/samza-core/src/main/java/org/apache/samza/system/SystemAdmins.java @@ -43,6 +43,10 @@ public SystemAdmins(Config config, String adminLabel) { this.systemAdminMap = systemConfig.getSystemAdmins(adminLabel); } + public SystemAdmins(Map systemAdminMap) { + this.systemAdminMap = systemAdminMap; + } + /** * Creates a new instance of {@link SystemAdmins} with an empty admin mapping. * @return New empty instance of {@link SystemAdmins} @@ -70,6 +74,10 @@ public SystemAdmin getSystemAdmin(String systemName) { return systemAdminMap.get(systemName); } + public Map getSystemAdmins() { + return systemAdminMap; + } + public Set getSystemNames() { return systemAdminMap.keySet(); } diff --git a/samza-core/src/main/java/org/apache/samza/system/inmemory/InMemoryManager.java b/samza-core/src/main/java/org/apache/samza/system/inmemory/InMemoryManager.java index 4d44a86d1f..87429ed0ac 100644 --- a/samza-core/src/main/java/org/apache/samza/system/inmemory/InMemoryManager.java +++ b/samza-core/src/main/java/org/apache/samza/system/inmemory/InMemoryManager.java @@ -54,7 +54,7 @@ public InMemoryManager() { } private List newSynchronizedLinkedList() { - return Collections.synchronizedList(new LinkedList()); + return Collections.synchronizedList(new LinkedList()); } /** diff --git a/samza-core/src/main/java/org/apache/samza/table/retry/AsyncRetriableTable.java b/samza-core/src/main/java/org/apache/samza/table/retry/AsyncRetriableTable.java index 589fb144a0..938a1d3d50 100644 --- a/samza-core/src/main/java/org/apache/samza/table/retry/AsyncRetriableTable.java +++ b/samza-core/src/main/java/org/apache/samza/table/retry/AsyncRetriableTable.java @@ -20,15 +20,13 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import net.jodah.failsafe.RetryPolicy; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; - import java.util.function.Predicate; -import net.jodah.failsafe.RetryPolicy; - import org.apache.samza.context.Context; import org.apache.samza.storage.kv.Entry; import org.apache.samza.table.AsyncReadWriteTable; @@ -36,9 +34,8 @@ import org.apache.samza.table.remote.TableWriteFunction; import org.apache.samza.table.utils.TableMetricsUtil; -import static org.apache.samza.table.retry.FailsafeAdapter.failsafe; - import static org.apache.samza.table.BaseReadWriteTable.Func1; +import static org.apache.samza.table.retry.FailsafeAdapter.failsafe; /** @@ -156,13 +153,13 @@ public void close() { private CompletableFuture doRead(Func1 func) { return readRetryPolicy != null - ? failsafe(readRetryPolicy, readRetryMetrics, retryExecutor).future(() -> func.apply()) + ? failsafe(readRetryPolicy, readRetryMetrics, retryExecutor).getStageAsync(() -> func.apply()) : func.apply(); } private CompletableFuture doWrite(Func1 func) { return writeRetryPolicy != null - ? failsafe(writeRetryPolicy, writeRetryMetrics, retryExecutor).future(() -> func.apply()) + ? failsafe(writeRetryPolicy, writeRetryMetrics, retryExecutor).getStageAsync(() -> func.apply()) : func.apply(); } } diff --git a/samza-core/src/main/java/org/apache/samza/table/retry/FailsafeAdapter.java b/samza-core/src/main/java/org/apache/samza/table/retry/FailsafeAdapter.java index 650d03af01..a9466b0e9a 100644 --- a/samza-core/src/main/java/org/apache/samza/table/retry/FailsafeAdapter.java +++ b/samza-core/src/main/java/org/apache/samza/table/retry/FailsafeAdapter.java @@ -19,15 +19,14 @@ package org.apache.samza.table.retry; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import org.apache.samza.SamzaException; - -import net.jodah.failsafe.AsyncFailsafe; import net.jodah.failsafe.Failsafe; +import net.jodah.failsafe.FailsafeExecutor; import net.jodah.failsafe.RetryPolicy; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.ScheduledExecutorService; +import org.apache.samza.SamzaException; + /** * Helper class adapting the generic {@link TableRetryPolicy} to a failsafe {@link RetryPolicy} and @@ -39,23 +38,24 @@ class FailsafeAdapter { * @return this policy instance */ static RetryPolicy valueOf(TableRetryPolicy policy) { - RetryPolicy failSafePolicy = new RetryPolicy(); + // max retries default changed to 2 in v2.0. switching back to infinite retries by default for back compat. + RetryPolicy failSafePolicy = new RetryPolicy().withMaxRetries(-1); switch (policy.getBackoffType()) { case NONE: break; case FIXED: - failSafePolicy.withDelay(policy.getSleepTime().toMillis(), TimeUnit.MILLISECONDS); + failSafePolicy.withDelay(policy.getSleepTime()); break; case RANDOM: - failSafePolicy.withDelay(policy.getRandomMin().toMillis(), policy.getRandomMax().toMillis(), TimeUnit.MILLISECONDS); + failSafePolicy.withDelay(policy.getRandomMin().toMillis(), policy.getRandomMax().toMillis(), ChronoUnit.MILLIS); break; case EXPONENTIAL: - failSafePolicy.withBackoff(policy.getSleepTime().toMillis(), policy.getExponentialMaxSleep().toMillis(), TimeUnit.MILLISECONDS, - policy.getExponentialFactor()); + failSafePolicy.withBackoff(policy.getSleepTime().toMillis(), policy.getExponentialMaxSleep().toMillis(), + ChronoUnit.MILLIS, policy.getExponentialFactor()); break; default: @@ -63,17 +63,16 @@ static RetryPolicy valueOf(TableRetryPolicy policy) { } if (policy.getMaxDuration() != null) { - failSafePolicy.withMaxDuration(policy.getMaxDuration().toMillis(), TimeUnit.MILLISECONDS); + failSafePolicy.withMaxDuration(policy.getMaxDuration()); } if (policy.getMaxAttempts() != null) { failSafePolicy.withMaxRetries(policy.getMaxAttempts()); } if (policy.getJitter() != null && policy.getBackoffType() != TableRetryPolicy.BackoffType.RANDOM) { - failSafePolicy.withJitter(policy.getJitter().toMillis(), TimeUnit.MILLISECONDS); + failSafePolicy.withJitter(policy.getJitter()); } - failSafePolicy.retryOn(e -> policy.getRetryPredicate().test(e)); - + failSafePolicy.abortOn(policy.getRetryPredicate().negate()); return failSafePolicy; } @@ -82,22 +81,24 @@ static RetryPolicy valueOf(TableRetryPolicy policy) { * @param retryPolicy retry policy * @param metrics retry metrics * @param retryExec executor service for scheduling async retries - * @return {@link net.jodah.failsafe.AsyncFailsafe} instance + * @return {@link net.jodah.failsafe.FailsafeExecutor} instance */ - static AsyncFailsafe failsafe(RetryPolicy retryPolicy, RetryMetrics metrics, ScheduledExecutorService retryExec) { + static FailsafeExecutor failsafe(RetryPolicy retryPolicy, RetryMetrics metrics, ScheduledExecutorService retryExec) { long startMs = System.currentTimeMillis(); - return Failsafe.with(retryPolicy).with(retryExec) + + RetryPolicy retryPolicyWithMetrics = retryPolicy .onRetry(e -> metrics.retryCount.inc()) .onRetriesExceeded(e -> { metrics.retryTimer.update(System.currentTimeMillis() - startMs); metrics.permFailureCount.inc(); - }) - .onSuccess((e, ctx) -> { - if (ctx.getExecutions() > 1) { + }).onSuccess((e) -> { + if (e.getAttemptCount() > 1) { metrics.retryTimer.update(System.currentTimeMillis() - startMs); } else { metrics.successCount.inc(); } }); + + return Failsafe.with(retryPolicyWithMetrics).with(retryExec); } } diff --git a/samza-core/src/main/java/org/apache/samza/util/FutureUtil.java b/samza-core/src/main/java/org/apache/samza/util/FutureUtil.java new file mode 100644 index 0000000000..16e38bcf21 --- /dev/null +++ b/samza-core/src/main/java/org/apache/samza/util/FutureUtil.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.util; + +import net.jodah.failsafe.Failsafe; +import net.jodah.failsafe.RetryPolicy; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutorService; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class FutureUtil { + private static final Logger LOG = LoggerFactory.getLogger(FutureUtil.class); + + /** + * Returns a future that completes when all the futures in the provided collections of futures are complete. + * @param futureCollections collections of futures to complete before the returned future is complete + */ + @SafeVarargs + public static CompletableFuture allOf(Collection>... futureCollections) { + List> fvs = new ArrayList<>(); + for (Collection> futureCollection : futureCollections) { + if (!futureCollection.isEmpty()) { + fvs.add(CompletableFuture.allOf(futureCollection.toArray(new CompletableFuture[0]))); + } + } + + return CompletableFuture.allOf(fvs.toArray(new CompletableFuture[0])); + } + + /** + * Returns a future that completes when all the futures futures are complete. + * Returned future completes exceptionally if any future complete with a non-ignored error. + */ + public static CompletableFuture allOf(Predicate ignoreError, CompletableFuture... futures) { + CompletableFuture allFuture = CompletableFuture.allOf(futures); + return allFuture.handle((aVoid, t) -> { + for (CompletableFuture future : futures) { + try { + future.join(); + } catch (Throwable th) { + if (ignoreError.test(th)) { + // continue + } else { + throw th; + } + } + } + return null; + }); + } + + /** + * Helper method to convert: {@code Pair, CompletableFuture>} + * to: {@code CompletableFuture>} + * + * Returns a future that completes when both futures complete. + * Returned future completes exceptionally if either of the futures complete exceptionally. + */ + public static CompletableFuture> toFutureOfPair( + Pair, CompletableFuture> pairOfFutures) { + return CompletableFuture + .allOf(pairOfFutures.getLeft(), pairOfFutures.getRight()) + .thenApply(v -> Pair.of(pairOfFutures.getLeft().join(), pairOfFutures.getRight().join())); + } + + /** + * Helper method to convert: {@code Map>} + * to: {@code CompletableFuture>} + * + * Returns a future that completes when all value futures complete. + * Returned future completes exceptionally if any of the value futures complete exceptionally. + */ + public static CompletableFuture> toFutureOfMap(Map> keyToValueFutures) { + return CompletableFuture + .allOf(keyToValueFutures.values().toArray(new CompletableFuture[0])) + .thenApply(v -> keyToValueFutures.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().join()))); + } + + /** + * Helper method to convert: {@code Map>} + * to: {@code CompletableFuture>} + * + * Returns a future that completes with successful map entries, skipping any entries with ignored errors, + * when all value futures complete. + * Returned future completes exceptionally if any of the futures complete with a non-ignored error. + */ + public static CompletableFuture> toFutureOfMap( + Predicate ignoreError, Map> keyToValueFutures) { + CompletableFuture allEntriesFuture = + CompletableFuture.allOf(keyToValueFutures.values().toArray(new CompletableFuture[]{})); + + return allEntriesFuture.handle((aVoid, t) -> { + Map successfulResults = new HashMap<>(); + for (Map.Entry> entry : keyToValueFutures.entrySet()) { + K key = entry.getKey(); + try { + V value = entry.getValue().join(); + successfulResults.put(key, value); + } catch (Throwable th) { + if (ignoreError.test(th)) { + // else ignore and continue + LOG.warn("Ignoring value future completion error for key: {}", key, th); + } else { + throw th; + } + } + } + return successfulResults; + }); + } + + public static CompletableFuture executeAsyncWithRetries(String opName, + Supplier> action, + Predicate abortRetries, + ExecutorService executor) { + Duration maxDuration = Duration.ofMinutes(10); + + RetryPolicy retryPolicy = new RetryPolicy<>() + .withBackoff(100, 312500, ChronoUnit.MILLIS, 5) // 100 ms, 500 ms, 2500 ms, 12.5 s, 1.05 min, 5.20 min, 5.20 min + .withMaxDuration(maxDuration) + .abortOn(abortRetries) // stop retrying if predicate returns true + .onRetry(e -> LOG.warn("Action: {} attempt: {} completed with error {} ms after start. Retrying up to {} ms.", + opName, e.getAttemptCount(), e.getElapsedTime().toMillis(), maxDuration.toMillis(), e.getLastFailure())); + + return Failsafe.with(retryPolicy).with(executor).getStageAsync(action::get); + } + + public static CompletableFuture failedFuture(Throwable t) { + final CompletableFuture cf = new CompletableFuture<>(); + cf.completeExceptionally(t); + return cf; + } + + /** + * Removes wrapper exceptions of the provided type from the provided throwable and returns the first cause + * that does not match the wrapper type. Useful for unwrapping CompletionException / SamzaException + * in stack traces and getting to the underlying cause. + * + * Returns null if provided Throwable is null or if there is no cause of non-wrapper type in the stack. + */ + public static Throwable unwrapExceptions(Class wrapperClassToUnwrap, T t) { + if (t == null) return null; + if (wrapperClassToUnwrap == null) return t; + + Throwable originalException = t; + while (wrapperClassToUnwrap.isAssignableFrom(originalException.getClass()) && + originalException.getCause() != null) { + originalException = originalException.getCause(); + } + + // can still be the wrapper class if no other cause was found. + if (wrapperClassToUnwrap.isAssignableFrom(originalException.getClass())) { + return null; + } else { + return originalException; + } + } +} diff --git a/samza-core/src/main/scala/org/apache/samza/checkpoint/CheckpointTool.scala b/samza-core/src/main/scala/org/apache/samza/checkpoint/CheckpointTool.scala index 7936c0a63d..1f98b22b52 100644 --- a/samza-core/src/main/scala/org/apache/samza/checkpoint/CheckpointTool.scala +++ b/samza-core/src/main/scala/org/apache/samza/checkpoint/CheckpointTool.scala @@ -185,21 +185,38 @@ class CheckpointTool(newOffsets: TaskNameToCheckpointMap, coordinatorStreamStore taskNames.foreach(checkpointManager.register) checkpointManager.start() + // Get preferred read version for the checkpoint application + val checkpointReadVersion = taskConfig.getCheckpointReadVersions.get(0) + val defaultCheckpoint = if (checkpointReadVersion == 1) { + new CheckpointV1(new java.util.HashMap[SystemStreamPartition, String]()) + } else if (checkpointReadVersion == 2) { + new CheckpointV2(CheckpointId.create(), new java.util.HashMap[SystemStreamPartition, String](), + new java.util.HashMap[String, util.Map[String, String]]()) + } else { + throw new SamzaException("Unrecognized checkpoint read version: " + checkpointReadVersion) + } + val lastCheckpoints = taskNames.map(taskName => { taskName -> Option(checkpointManager.readLastCheckpoint(taskName)) - .getOrElse(new Checkpoint(new java.util.HashMap[SystemStreamPartition, String]())) - .getOffsets - .asScala - .toMap + .getOrElse(defaultCheckpoint) }).toMap - lastCheckpoints.foreach(lcp => logCheckpoint(lcp._1, lcp._2, "Current checkpoint for task: "+ lcp._1)) + lastCheckpoints.foreach(lcp => logCheckpoint(lcp._1, lcp._2.getOffsets.asScala.toMap, + "Current checkpoint for task: " + lcp._1)) if (newOffsets != null) { newOffsets.foreach { case (taskName: TaskName, offsets: Map[SystemStreamPartition, String]) => logCheckpoint(taskName, offsets, "New offset to be written for task: " + taskName) - val checkpoint = new Checkpoint(offsets.asJava) + val checkpoint = if (checkpointReadVersion == 1) { + new CheckpointV1(offsets.asJava) + } else if (checkpointReadVersion == 2) { + val lastSCMs = lastCheckpoints.getOrElse(taskName, defaultCheckpoint) + .asInstanceOf[CheckpointV2].getStateCheckpointMarkers + new CheckpointV2(CheckpointId.create(), offsets.asJava, lastSCMs) + } else { + throw new SamzaException("Unrecognized checkpoint read version: " + checkpointReadVersion) + } checkpointManager.writeCheckpoint(taskName, checkpoint) info(s"Updated the checkpoint of the task: $taskName to: $offsets") } diff --git a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala index 442d83f035..7491eaaaf9 100644 --- a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala +++ b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala @@ -19,6 +19,7 @@ package org.apache.samza.checkpoint +import java.util import java.util.HashMap import java.util.concurrent.ConcurrentHashMap @@ -219,6 +220,16 @@ class OffsetManager( Option(lastProcessedOffsets.get(taskName)).map(_.get(systemStreamPartition)) } + /** + * Get the last checkpoint saved in the checkpoint manager or null if there is no recorded checkpoints for the task + */ + def getLastTaskCheckpoint(taskName: TaskName): Checkpoint = { + if (checkpointManager != null) { + checkpointManager.readLastCheckpoint(taskName) + } + null + } + /** * Get the starting offset for a SystemStreamPartition. This is the offset * where a SamzaContainer begins reading from when it starts up. @@ -267,9 +278,9 @@ class OffsetManager( * ensure there are no concurrent updates to the offsets between when this method is * invoked and the corresponding call to [[OffsetManager.writeCheckpoint()]] */ - def buildCheckpoint(taskName: TaskName): Checkpoint = { + def getLastProcessedOffsets(taskName: TaskName): util.Map[SystemStreamPartition, String] = { if (checkpointManager != null || checkpointListeners.nonEmpty) { - debug("Getting checkpoint offsets for taskName %s." format taskName) + debug("Getting last processed offsets to checkpoint for taskName %s." format taskName) val taskStartingOffsets = startingOffsets.getOrElse(taskName, throw new SamzaException("Couldn't find starting offsets for task: " + taskName)) @@ -283,10 +294,10 @@ class OffsetManager( .filterKeys(taskSSPs.contains) val modifiedTaskOffsets = getModifiedOffsets(taskStartingOffsets, taskLastProcessedOffsets) - new Checkpoint(new HashMap(modifiedTaskOffsets)) // Copy into new Map to prevent mutation + new util.HashMap(modifiedTaskOffsets) } else { - debug("Returning null checkpoint for taskName %s because no checkpoint manager/callback is defined." format taskName) - null + debug("Returning empty offsets for taskName %s because no checkpoint manager/callback is defined." format taskName) + new util.HashMap() } } @@ -336,11 +347,13 @@ class OffsetManager( */ def writeCheckpoint(taskName: TaskName, checkpoint: Checkpoint) { if (checkpoint != null && (checkpointManager != null || checkpointListeners.nonEmpty)) { - debug("Writing checkpoint for taskName %s with offsets %s." format (taskName, checkpoint)) + debug("Writing checkpoint for taskName: %s as: %s." format (taskName, checkpoint)) + + val sspToOffsets = checkpoint.getOffsets if(checkpointManager != null) { checkpointManager.writeCheckpoint(taskName, checkpoint) - val sspToOffsets = checkpoint.getOffsets + if(sspToOffsets != null) { sspToOffsets.asScala.foreach { case (ssp, cp) => { @@ -357,7 +370,7 @@ class OffsetManager( // changelog SSPs are not registered but may be present in the Checkpoint if transactional state checkpointing // is enabled. val registeredSSPs = systemStreamPartitions.getOrElse(taskName, Set[SystemStreamPartition]()) - checkpoint.getOffsets.asScala + sspToOffsets.asScala .filterKeys(registeredSSPs.contains) .groupBy { case (ssp, _) => ssp.getSystem }.foreach { case (systemName:String, offsets: Map[SystemStreamPartition, String]) => { @@ -452,7 +465,6 @@ class OffsetManager( Map(taskName -> checkpoint.getOffsets.asScala.toMap) } else { info("Did not receive a checkpoint for taskName %s. Proceeding without a checkpoint." format taskName) - Map(taskName -> Map()) } } diff --git a/samza-core/src/main/scala/org/apache/samza/checkpoint/file/FileSystemCheckpointManager.scala b/samza-core/src/main/scala/org/apache/samza/checkpoint/file/FileSystemCheckpointManager.scala index a9780b6e93..a068e2adc5 100644 --- a/samza-core/src/main/scala/org/apache/samza/checkpoint/file/FileSystemCheckpointManager.scala +++ b/samza-core/src/main/scala/org/apache/samza/checkpoint/file/FileSystemCheckpointManager.scala @@ -24,13 +24,11 @@ import java.io.FileNotFoundException import java.io.FileOutputStream import org.apache.samza.SamzaException -import org.apache.samza.checkpoint.Checkpoint -import org.apache.samza.checkpoint.CheckpointManager -import org.apache.samza.checkpoint.CheckpointManagerFactory +import org.apache.samza.checkpoint.{Checkpoint, CheckpointManager, CheckpointManagerFactory, CheckpointV1} import org.apache.samza.config.{Config, FileSystemCheckpointManagerConfig, JobConfig} import org.apache.samza.container.TaskName import org.apache.samza.metrics.MetricsRegistry -import org.apache.samza.serializers.CheckpointSerde +import org.apache.samza.serializers.CheckpointV1Serde import org.apache.samza.util.ScalaJavaUtil.JavaOptionals import scala.io.Source @@ -38,14 +36,17 @@ import scala.io.Source class FileSystemCheckpointManager( jobName: String, root: File, - serde: CheckpointSerde = new CheckpointSerde) extends CheckpointManager { + serde: CheckpointV1Serde = new CheckpointV1Serde) extends CheckpointManager { override def register(taskName: TaskName):Unit = Unit def getCheckpointFile(taskName: TaskName) = getFile(jobName, taskName, "checkpoints") def writeCheckpoint(taskName: TaskName, checkpoint: Checkpoint) { - val bytes = serde.toBytes(checkpoint) + if (!checkpoint.isInstanceOf[CheckpointV1]) { + throw new SamzaException("Unsupported checkpoint version: " + checkpoint.getVersion) + } + val bytes = serde.toBytes(checkpoint.asInstanceOf[CheckpointV1]) val fos = new FileOutputStream(getCheckpointFile(taskName)) fos.write(bytes) diff --git a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala index 83f5c9df4b..fe275528de 100644 --- a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala +++ b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala @@ -23,14 +23,15 @@ import java.io.File import java.lang.management.ManagementFactory import java.net.{URL, UnknownHostException} import java.nio.file.Path -import java.time.Duration import java.util +import java.util.concurrent._ +import java.util.function.Consumer import java.util.{Base64, Optional} -import java.util.concurrent.{CountDownLatch, ExecutorService, Executors, ScheduledExecutorService, ThreadPoolExecutor, TimeUnit} - import com.google.common.annotations.VisibleForTesting import com.google.common.util.concurrent.ThreadFactoryBuilder +import org.apache.samza.SamzaException import org.apache.samza.checkpoint.{CheckpointListener, OffsetManager, OffsetManagerMetrics} +import org.apache.samza.clustermanager.StandbyTaskUtil import org.apache.samza.config.{StreamConfig, _} import org.apache.samza.container.disk.DiskSpaceMonitor.Listener import org.apache.samza.container.disk.{DiskQuotaPolicyFactory, DiskSpaceMonitor, NoThrottlingDiskQuotaPolicyFactory, PollingScanDiskSpaceMonitor} @@ -49,10 +50,7 @@ import org.apache.samza.table.TableManager import org.apache.samza.task._ import org.apache.samza.util.ScalaJavaUtil.JavaOptionals import org.apache.samza.util.{Util, _} -import org.apache.samza.SamzaException -import org.apache.samza.clustermanager.StandbyTaskUtil -import scala.collection.JavaConversions import scala.collection.JavaConverters._ object SamzaContainer extends Logging { @@ -146,7 +144,6 @@ object SamzaContainer extends Logging { val systemConfig = new SystemConfig(config) val containerModel = jobModel.getContainers.get(containerId) val containerName = "samza-container-%s" format containerId - val maxChangeLogStreamPartitions = jobModel.maxChangeLogStreamPartitions val containerPID = ManagementFactory.getRuntimeMXBean().getName() @@ -343,31 +340,10 @@ object SamzaContainer extends Logging { debug("Got system stream message serdes: %s" format systemStreamMessageSerdes) - val storeChangelogs = storageConfig - .getStoreNames.asScala - .filter(storageConfig.getChangelogStream(_).isPresent) - .map(name => (name, storageConfig.getChangelogStream(name).get)).toMap - .mapValues(StreamUtil.getSystemStreamFromNames(_)) + val storeChangelogs = storageConfig.getStoreChangelogs info("Got change log system streams: %s" format storeChangelogs) - /* - * This keeps track of the changelog SSPs that are associated with the whole container. This is used so that we can - * prefetch the metadata about the all of the changelog SSPs associated with the container whenever we need the - * metadata about some of the changelog SSPs. - * An example use case is when Samza writes offset files for stores ({@link TaskStorageManager}). Each task is - * responsible for its own offset file, but if we can do prefetching, then most tasks will already have cached - * metadata by the time they need the offset metadata. - * Note: By using all changelog streams to build the sspsToPrefetch, any fetches done for persisted stores will - * include the ssps for non-persisted stores, so this is slightly suboptimal. However, this does not increase the - * actual number of calls to the {@link SystemAdmin}, and we can decouple this logic from the per-task objects (e.g. - * {@link TaskStorageManager}). - */ - val changelogSSPMetadataCache = new SSPMetadataCache(systemAdmins, - Duration.ofSeconds(5), - SystemClock.instance, - getChangelogSSPsForContainer(containerModel, storeChangelogs).asJava) - val intermediateStreams = streamConfig .getStreamIds() .asScala @@ -398,7 +374,7 @@ object SamzaContainer extends Logging { systemMessageSerdes = systemMessageSerdes, systemStreamKeySerdes = systemStreamKeySerdes, systemStreamMessageSerdes = systemStreamMessageSerdes, - changeLogSystemStreams = storeChangelogs.values.toSet, + changeLogSystemStreams = storeChangelogs.asScala.values.toSet, controlMessageKeySerdes = controlMessageKeySerdes, intermediateMessageSerdes = intermediateStreamMessageSerdes) @@ -476,6 +452,7 @@ object SamzaContainer extends Logging { val threadPoolSize = jobConfig.getThreadPoolSize info("Got thread pool size: " + threadPoolSize) + samzaContainerMetrics.containerThreadPoolSize.set(threadPoolSize) val taskThreadPool = if (threadPoolSize > 0) { Executors.newFixedThreadPool(threadPoolSize, @@ -484,11 +461,19 @@ object SamzaContainer extends Logging { null } - val finalTaskFactory = TaskFactoryUtil.finalizeTaskFactory( taskFactory, taskThreadPool) + // executor for performing async commit operations for a task. + val commitThreadPoolSize = + Math.min( + Math.max(containerModel.getTasks.size() * 2, jobConfig.getCommitThreadPoolSize), + jobConfig.getCommitThreadPoolMaxSize + ) + val commitThreadPool = Executors.newFixedThreadPool(commitThreadPoolSize, + new ThreadFactoryBuilder().setNameFormat("Samza Task Commit Thread-%d").setDaemon(true).build()) + val taskModels = containerModel.getTasks.values.asScala val containerContext = new ContainerContextImpl(containerModel, samzaContainerMetrics.registry) val applicationContainerContextOption = applicationContainerContextFactoryOption @@ -498,8 +483,6 @@ object SamzaContainer extends Logging { val timerExecutor = Executors.newSingleThreadScheduledExecutor - var taskStorageManagers : Map[TaskName, TaskStorageManager] = Map() - val taskInstanceMetrics: Map[TaskName, TaskInstanceMetrics] = taskModels.map(taskModel => { (taskModel.getTaskName, new TaskInstanceMetrics("TaskName-%s" format taskModel.getTaskName)) }).toMap @@ -517,13 +500,29 @@ object SamzaContainer extends Logging { val loggedStorageBaseDir = getLoggedStorageBaseDir(jobConfig, defaultStoreBaseDir) info("Got base directory for logged data stores: %s" format loggedStorageBaseDir) + val backupFactoryNames = storageConfig.getBackupFactories + val restoreFactoryNames = storageConfig.getRestoreFactories + + // Restore factories should be a subset of backup factories + if (!backupFactoryNames.containsAll(restoreFactoryNames)) { + backupFactoryNames.removeAll(restoreFactoryNames) + throw new SamzaException("Restore state backend factories is not a subset of backup state backend factories, " + + "missing factories: " + backupFactoryNames.toString) + } + + val stateStorageBackendBackupFactories = backupFactoryNames.asScala.map( + ReflectionUtil.getObj(_, classOf[StateBackendFactory]) + ) + val stateStorageBackendRestoreFactories = restoreFactoryNames.asScala.map( + factoryName => (factoryName , ReflectionUtil.getObj(factoryName, classOf[StateBackendFactory]))) + .toMap.asJava + val containerStorageManager = new ContainerStorageManager( checkpointManager, containerModel, streamMetadataCache, - changelogSSPMetadataCache, systemAdmins, - storeChangelogs.asJava, + storeChangelogs, sideInputStoresToSystemStreams.mapValues(systemStreamSet => systemStreamSet.toSet.asJava).asJava, storageEngineFactories.asJava, systemFactories.asJava, @@ -533,15 +532,17 @@ object SamzaContainer extends Logging { samzaContainerMetrics, jobContext, containerContext, + stateStorageBackendRestoreFactories, taskCollectors.asJava, loggedStorageBaseDir, nonLoggedStorageBaseDir, - maxChangeLogStreamPartitions, serdeManager, new SystemClock) storeWatchPaths.addAll(containerStorageManager.getStoreDirectoryPaths) + + // Create taskInstances val taskInstances: Map[TaskName, TaskInstance] = taskModels .filter(taskModel => taskModel.getTaskMode.eq(TaskMode.Active)).map(taskModel => { @@ -563,15 +564,23 @@ object SamzaContainer extends Logging { val taskSideInputSSPs = sideInputStoresToSSPs.values.flatMap(_.asScala).toSet info ("Got task side input SSPs: %s" format taskSideInputSSPs) - val storageManager = TaskStorageManagerFactory.create( - taskName, - containerStorageManager, - storeChangelogs, - systemAdmins, - loggedStorageBaseDir, - taskModel.getChangelogPartition, - config, - taskModel.getTaskMode) + val taskBackupManagerMap = new util.HashMap[String, TaskBackupManager]() + stateStorageBackendBackupFactories.asJava.forEach(new Consumer[StateBackendFactory] { + override def accept(factory: StateBackendFactory): Unit = { + val taskMetricsRegistry = + if (taskInstanceMetrics.contains(taskName) && + taskInstanceMetrics.get(taskName).isDefined) taskInstanceMetrics.get(taskName).get.registry + else new MetricsRegistryMap + val taskBackupManager = factory.getBackupManager(jobContext, containerModel, + taskModel, commitThreadPool, taskMetricsRegistry, config, new SystemClock, + loggedStorageBaseDir, nonLoggedStorageBaseDir) + taskBackupManagerMap.put(factory.getClass.getName, taskBackupManager) + } + }) + + val commitManager = new TaskStorageCommitManager(taskName, taskBackupManagerMap, + containerStorageManager, storeChangelogs, taskModel.getChangelogPartition, checkpointManager, config, + commitThreadPool, new StorageManagerUtil, loggedStorageBaseDir, taskInstanceMetrics.get(taskName).get) val tableManager = new TableManager(config) @@ -585,14 +594,16 @@ object SamzaContainer extends Logging { consumerMultiplexer = consumerMultiplexer, collector = taskCollectors.get(taskName).get, offsetManager = offsetManager, - storageManager = storageManager, + commitManager = commitManager, + containerStorageManager = containerStorageManager, tableManager = tableManager, - systemStreamPartitions = JavaConversions.setAsJavaSet(taskSSPs -- taskSideInputSSPs), + systemStreamPartitions = (taskSSPs -- taskSideInputSSPs).asJava, exceptionHandler = TaskInstanceExceptionHandler(taskInstanceMetrics.get(taskName).get, taskConfig), jobModel = jobModel, streamMetadataCache = streamMetadataCache, inputStreamMetadata = inputStreamMetadata, timerExecutor = timerExecutor, + commitThreadPool = commitThreadPool, jobContext = jobContext, containerContext = containerContext, applicationContainerContextOption = applicationContainerContextOption, @@ -601,7 +612,6 @@ object SamzaContainer extends Logging { val taskInstance = createTaskInstance(task) - taskStorageManagers += taskInstance.taskName -> storageManager (taskName, taskInstance) }).toMap @@ -618,8 +628,8 @@ object SamzaContainer extends Logging { val containerMemoryMb : Int = new ClusterManagerConfig(config).getContainerMemoryMb - val hostStatisticsMonitor : SystemStatisticsMonitor = new StatisticsMonitorImpl() - hostStatisticsMonitor.registerListener(new SystemStatisticsMonitor.Listener { + val memoryStatisticsMonitor : SystemStatisticsMonitor = new StatisticsMonitorImpl() + memoryStatisticsMonitor.registerListener(new SystemStatisticsMonitor.Listener { override def onUpdate(sample: SystemMemoryStatistics): Unit = { val physicalMemoryBytes : Long = sample.getPhysicalMemoryBytes val physicalMemoryMb : Float = physicalMemoryBytes / (1024.0F * 1024.0F) @@ -627,16 +637,7 @@ object SamzaContainer extends Logging { logger.debug("Container physical memory utilization (mb): " + physicalMemoryMb) logger.debug("Container physical memory utilization: " + memoryUtilization) samzaContainerMetrics.physicalMemoryMb.set(physicalMemoryMb) - samzaContainerMetrics.physicalMemoryUtilization.set(memoryUtilization) - - var containerThreadPoolSize : Long = 0 - var containerActiveThreads : Long = 0 - if (taskThreadPool != null) { - containerThreadPoolSize = taskThreadPool.asInstanceOf[ThreadPoolExecutor].getPoolSize - containerActiveThreads = taskThreadPool.asInstanceOf[ThreadPoolExecutor].getActiveCount - } - samzaContainerMetrics.containerThreadPoolSize.set(containerThreadPoolSize) - samzaContainerMetrics.containerActiveThreads.set(containerActiveThreads) + samzaContainerMetrics.physicalMemoryUtilization.set(memoryUtilization); } }) @@ -682,8 +683,9 @@ object SamzaContainer extends Logging { reporters = reporters, jvm = jvm, diskSpaceMonitor = diskSpaceMonitor, - hostStatisticsMonitor = hostStatisticsMonitor, + hostStatisticsMonitor = memoryStatisticsMonitor, taskThreadPool = taskThreadPool, + commitThreadPool = commitThreadPool, timerExecutor = timerExecutor, containerContext = containerContext, applicationContainerContextOption = applicationContainerContextOption, @@ -691,19 +693,6 @@ object SamzaContainer extends Logging { containerStorageManager = containerStorageManager, diagnosticsManager = diagnosticsManager) } - - /** - * Builds the set of SSPs for all changelogs on this container. - */ - @VisibleForTesting - private[container] def getChangelogSSPsForContainer(containerModel: ContainerModel, - changeLogSystemStreams: Map[String, SystemStream]): Set[SystemStreamPartition] = { - containerModel.getTasks.values().asScala - .map(taskModel => taskModel.getChangelogPartition) - .flatMap(changelogPartition => changeLogSystemStreams.map { case (_, systemStream) => - new SystemStreamPartition(systemStream, changelogPartition) }) - .toSet - } } class SamzaContainer( @@ -723,6 +712,7 @@ class SamzaContainer( reporters: Map[String, MetricsReporter] = Map(), jvm: JvmMetrics = null, taskThreadPool: ExecutorService = null, + commitThreadPool: ExecutorService = null, timerExecutor: ScheduledExecutorService = Executors.newSingleThreadScheduledExecutor, containerContext: ContainerContext, applicationContainerContextOption: Option[ApplicationContainerContext], @@ -732,7 +722,10 @@ class SamzaContainer( private val jobConfig = new JobConfig(config) private val taskConfig = new TaskConfig(config) - val shutdownMs: Long = taskConfig.getShutdownMs + + val shutdownMs: Long = taskConfig.getLong(TaskConfig.TASK_SHUTDOWN_MS, 5000) + + var shutdownHookThread: Thread = null var jmxServer: JmxServer = null @volatile private var status = SamzaContainerStatus.NOT_STARTED @@ -1053,7 +1046,7 @@ class SamzaContainer( info("Shutting down task thread pool") try { taskThreadPool.shutdown() - if(taskThreadPool.awaitTermination(shutdownMs, TimeUnit.MILLISECONDS)) { + if (!taskThreadPool.awaitTermination(shutdownMs, TimeUnit.MILLISECONDS)) { taskThreadPool.shutdownNow() } } catch { @@ -1061,11 +1054,23 @@ class SamzaContainer( } } + if (commitThreadPool != null) { + info("Shutting down task commit thread pool") + try { + commitThreadPool.shutdown() + if(!commitThreadPool.awaitTermination(shutdownMs, TimeUnit.MILLISECONDS)) { + commitThreadPool.shutdownNow() + } + } catch { + case e: Exception => error(e.getMessage, e) + } + } + if (timerExecutor != null) { info("Shutting down timer executor") try { timerExecutor.shutdown() - if (timerExecutor.awaitTermination(shutdownMs, TimeUnit.MILLISECONDS)) { + if (!timerExecutor.awaitTermination(shutdownMs, TimeUnit.MILLISECONDS)) { timerExecutor.shutdownNow() } } catch { @@ -1133,4 +1138,4 @@ class SamzaContainer( hostStatisticsMonitor.stop() } } -} \ No newline at end of file +} diff --git a/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala b/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala index 801c6bcf30..4e1c41aba2 100644 --- a/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala +++ b/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala @@ -21,21 +21,26 @@ package org.apache.samza.container import java.util.{Collections, Objects, Optional} -import java.util.concurrent.ScheduledExecutorService - +import java.util.concurrent.{CompletableFuture, ExecutorService, ScheduledExecutorService, Semaphore, TimeUnit} import org.apache.samza.SamzaException -import org.apache.samza.checkpoint.{Checkpoint, CheckpointId, CheckpointedChangelogOffset, OffsetManager} +import org.apache.samza.checkpoint.kafka.{KafkaChangelogSSPOffset, KafkaStateCheckpointMarker} +import org.apache.samza.checkpoint.{CheckpointId, CheckpointV1, CheckpointV2, OffsetManager} import org.apache.samza.config.{Config, StreamConfig, TaskConfig} import org.apache.samza.context._ import org.apache.samza.job.model.{JobModel, TaskModel} import org.apache.samza.scheduler.{CallbackSchedulerImpl, EpochTimeScheduler, ScheduledCallback} import org.apache.samza.storage.kv.KeyValueStore -import org.apache.samza.storage.TaskStorageManager +import org.apache.samza.storage.{ContainerStorageManager, TaskStorageCommitManager} import org.apache.samza.system._ import org.apache.samza.table.TableManager import org.apache.samza.task._ +import org.apache.samza.util.ScalaJavaUtil.JavaOptionals.toRichOptional import org.apache.samza.util.{Logging, ScalaJavaUtil} +import java.util +import java.util.concurrent.atomic.AtomicReference +import java.util.function.BiConsumer +import java.util.function.Function import scala.collection.JavaConversions._ import scala.collection.JavaConverters._ import scala.collection.{JavaConverters, Map} @@ -48,14 +53,16 @@ class TaskInstance( consumerMultiplexer: SystemConsumers, collector: TaskInstanceCollector, override val offsetManager: OffsetManager = new OffsetManager, - storageManager: TaskStorageManager = null, + commitManager: TaskStorageCommitManager = null, + containerStorageManager: ContainerStorageManager = null, tableManager: TableManager = null, val systemStreamPartitions: java.util.Set[SystemStreamPartition] = Collections.emptySet(), val exceptionHandler: TaskInstanceExceptionHandler = new TaskInstanceExceptionHandler, jobModel: JobModel = null, streamMetadataCache: StreamMetadataCache = null, inputStreamMetadata: Map[SystemStream, SystemStreamMetadata] = Map(), - timerExecutor : ScheduledExecutorService = null, + timerExecutor: ScheduledExecutorService = null, + commitThreadPool: ExecutorService = null, jobContext: JobContext, containerContext: ContainerContext, applicationContainerContextOption: Option[ApplicationContainerContext], @@ -73,8 +80,9 @@ class TaskInstance( private val kvStoreSupplier = ScalaJavaUtil.toJavaFunction( (storeName: String) => { - if (storageManager != null && storageManager.getStore(storeName).isDefined) { - storageManager.getStore(storeName).get.asInstanceOf[KeyValueStore[_, _]] + if (containerStorageManager != null) { + val storeOption = containerStorageManager.getStore(taskName, storeName).toOption + if (storeOption.isDefined) storeOption.get.asInstanceOf[KeyValueStore[_, _]] else null } else { null } @@ -97,12 +105,21 @@ class TaskInstance( systemStreamPartitions.foreach(ssp2CaughtupMapping += _ -> false) private val config: Config = jobContext.getConfig + val taskConfig = new TaskConfig(config) val streamConfig: StreamConfig = new StreamConfig(config) override val intermediateStreams: java.util.Set[String] = JavaConverters.setAsJavaSetConverter(streamConfig.getStreamIds.filter(streamConfig.getIsIntermediateStream)).asJava val streamsToDeleteCommittedMessages: Set[String] = streamConfig.getStreamIds.filter(streamConfig.getDeleteCommittedMessages).map(streamConfig.getPhysicalName).toSet + val checkpointWriteVersions = new TaskConfig(config).getCheckpointWriteVersions + + @volatile var lastCommitStartTimeMs = System.currentTimeMillis() + val commitMaxDelayMs = taskConfig.getCommitMaxDelayMs + val commitTimeoutMs = taskConfig.getCommitTimeoutMs + val commitInProgress = new Semaphore(1) + val commitException = new AtomicReference[Exception]() + def registerOffsets { debug("Registering offsets for taskName: %s" format taskName) offsetManager.register(taskName, systemStreamPartitions) @@ -121,10 +138,33 @@ class TaskInstance( def initTask { initCaughtUpMapping() - val taskConfig = new TaskConfig(config) + if (commitManager != null) { + debug("Starting commit manager for taskName: %s" format taskName) + + commitManager.init() + } else { + debug("Skipping commit manager initialization for taskName: %s" format taskName) + } + + if (offsetManager != null) { + val checkpoint = offsetManager.getLastTaskCheckpoint(taskName) + // Only required for checkpointV2 + if (checkpoint != null && checkpoint.getVersion == 2) { + val checkpointV2 = checkpoint.asInstanceOf[CheckpointV2] + // call cleanUp on backup managers in case the container previously failed during commit + // before completing this step + + // WARNING: cleanUp is NOT optional with blob stores since this is where we reset the TTL for + // tracked blobs. if this TTL reset is skipped, some of the blobs retained by future commits may + // be deleted in the background by the blob store, leading to data loss. + debug("Cleaning up stale state from previous run for taskName: %s" format taskName) + commitManager.cleanUp(checkpointV2.getCheckpointId, checkpointV2.getStateCheckpointMarkers) + } + } + if (taskConfig.getTransactionalStateRestoreEnabled() && taskConfig.getCommitMs > 0) { - // Commit immediately so the trimmed changelog messages - // will be sealed in a checkpoint + debug("Committing immediately on startup for taskName: %s so that the trimmed changelog " + + "messages will be sealed in a checkpoint" format taskName) commit } @@ -178,8 +218,9 @@ class TaskInstance( if (ssp2CaughtupMapping(incomingMessageSsp)) { metrics.messagesActuallyProcessed.inc - trace("Processing incoming message envelope for taskName and SSP: %s, %s" - format (taskName, incomingMessageSsp)) + // TODO BLOCKER pmaheshw reenable after demo +// trace("Processing incoming message envelope for taskName: %s SSP: %s offset: %s" +// format (taskName, incomingMessageSsp, envelope.getOffset)) exceptionHandler.maybeHandle { val callback = callbackFactory.createCallback() @@ -219,68 +260,247 @@ class TaskInstance( } def commit { - metrics.commits.inc + // ensure that only one commit (including sync and async phases) is ever in progress for a task. + + val commitStartNs = System.nanoTime() + // first check if there were any unrecoverable errors during the async stage of the pending commit + // and if so, shut down the container. + if (commitException.get() != null) { + throw new SamzaException("Unrecoverable error during pending commit for taskName: %s." format taskName, + commitException.get()) + } - val allCheckpointOffsets = new java.util.HashMap[SystemStreamPartition, String]() - val inputCheckpoint = offsetManager.buildCheckpoint(taskName) - if (inputCheckpoint != null) { - trace("Got input offsets for taskName: %s as: %s" format(taskName, inputCheckpoint.getOffsets)) - allCheckpointOffsets.putAll(inputCheckpoint.getOffsets) + // if no commit is in progress for this task, continue with this commit. + // if a previous commit is in progress but less than {@code task.commit.max.delay.ms} + // have elapsed since it started, skip this commit request. + // if more time has elapsed than that, block this commit until the previous commit + // is complete, then continue with this commit. + if (!commitInProgress.tryAcquire()) { + val timeSinceLastCommit = System.currentTimeMillis() - lastCommitStartTimeMs + if (timeSinceLastCommit < commitMaxDelayMs) { + info("Skipping commit for taskName: %s since another commit is in progress. " + + "%s ms have elapsed since the pending commit started." format (taskName, timeSinceLastCommit)) + metrics.commitsSkipped.set(metrics.commitsSkipped.getValue + 1) + return + } else { + warn("Blocking processing for taskName: %s until in-flight commit is complete. " + + "%s ms have elapsed since the pending commit started, " + + "which is greater than the max allowed commit delay: %s." + format (taskName, timeSinceLastCommit, commitMaxDelayMs)) + + if (!commitInProgress.tryAcquire(commitTimeoutMs, TimeUnit.MILLISECONDS)) { + val timeSinceLastCommit = System.currentTimeMillis() - lastCommitStartTimeMs + metrics.commitsTimedOut.set(metrics.commitsTimedOut.getValue + 1) + throw new SamzaException("Timeout waiting for pending commit for taskName: %s to finish. " + + "%s ms have elapsed since the pending commit started. Max allowed commit delay is %s ms " + + "and commit timeout beyond that is %s ms" format (taskName, timeSinceLastCommit, + commitMaxDelayMs, commitTimeoutMs)) + } + } } + // at this point the permit for semaphore has been acquired, proceed with commit. + // the first part of the commit needs to be exclusive with processing, so do it on the caller thread. + lastCommitStartTimeMs = System.currentTimeMillis() + + metrics.commits.inc + val checkpointId = CheckpointId.create() + + debug("Starting sync stage of commit for taskName: %s checkpointId: %s" format (taskName, checkpointId)) + + val inputOffsets = offsetManager.getLastProcessedOffsets(taskName) + trace("Got last processed input offsets for taskName: %s checkpointId: %s as: %s" + format(taskName, checkpointId, inputOffsets)) - trace("Flushing producers for taskName: %s" format taskName) + trace("Flushing producers for taskName: %s checkpointId: %s" format (taskName, checkpointId)) + // Flushes output, checkpoint and changelog producers collector.flush if (tableManager != null) { - trace("Flushing tables for taskName: %s" format taskName) + trace("Flushing tables for taskName: %s checkpointId: %s" format (taskName, checkpointId)) tableManager.flush() } - var newestChangelogOffsets: Map[SystemStreamPartition, Option[String]] = null - if (storageManager != null) { - trace("Flushing state stores for taskName: %s" format taskName) - newestChangelogOffsets = storageManager.flush() - trace("Got newest changelog offsets for taskName: %s as: %s " format(taskName, newestChangelogOffsets)) - } + // create a synchronous snapshot of stores for commit + debug("Creating synchronous state store snapshots for taskName: %s checkpointId: %s" + format (taskName, checkpointId)) + val snapshotStartTimeNs = System.nanoTime() + val snapshotSCMs = commitManager.snapshot(checkpointId) + metrics.snapshotNs.update(System.nanoTime() - snapshotStartTimeNs) + trace("Got synchronous snapshot SCMs for taskName: %s checkpointId: %s as: %s " + format(taskName, checkpointId, snapshotSCMs)) + + debug("Submitting async stage of commit for taskName: %s checkpointId: %s for execution" + format (taskName, checkpointId)) + val asyncStageStartNs = System.nanoTime() + // rest of the commit can happen asynchronously and concurrently with processing. + // schedule it on the commit executor and return. submitted runnable releases the + // commit semaphore permit when this commit is complete. + commitThreadPool.submit(new Runnable { + override def run(): Unit = { + debug("Starting async stage of commit for taskName: %s checkpointId: %s" format (taskName, checkpointId)) + + try { + val uploadStartTimeNs = System.nanoTime() + val uploadSCMsFuture = commitManager.upload(checkpointId, snapshotSCMs) + uploadSCMsFuture.whenComplete(new BiConsumer[util.Map[String, util.Map[String, String]], Throwable] { + override def accept(t: util.Map[String, util.Map[String, String]], throwable: Throwable): Unit = { + if (throwable == null) { + metrics.asyncUploadNs.update(System.nanoTime() - uploadStartTimeNs) + } else { + warn("Commit upload did not complete successfully for taskName: %s checkpointId: %s with error msg: %s" + format (taskName, checkpointId, throwable.getMessage)) + } + } + }) + + // explicit types required to make scala compiler happy + val checkpointWriteFuture: CompletableFuture[util.Map[String, util.Map[String, String]]] = + uploadSCMsFuture.thenApplyAsync(writeCheckpoint(checkpointId, inputOffsets), commitThreadPool) + + val cleanUpFuture: CompletableFuture[Void] = + checkpointWriteFuture.thenComposeAsync(cleanUp(checkpointId), commitThreadPool) + cleanUpFuture.whenComplete(new BiConsumer[Void, Throwable] { + override def accept(v: Void, throwable: Throwable): Unit = { + if (throwable != null) { + warn("Commit cleanup did not complete successfully for taskName: %s checkpointId: %s with error msg: %s" + format (taskName, checkpointId, throwable.getMessage)) + } + } + }) - val checkpointId = CheckpointId.create() - if (storageManager != null && newestChangelogOffsets != null) { - trace("Checkpointing stores for taskName: %s with checkpoint id: %s" format (taskName, checkpointId)) - storageManager.checkpoint(checkpointId, newestChangelogOffsets.toMap) - } + val trimFuture = cleanUpFuture.thenRunAsync( + trim(checkpointId, inputOffsets), commitThreadPool) + + trimFuture.whenCompleteAsync(handleCompletion(checkpointId, commitStartNs, asyncStageStartNs), commitThreadPool) + } catch { + case t: Throwable => handleCompletion(checkpointId, commitStartNs, asyncStageStartNs).accept(null, t) + } + } + }) + + metrics.lastCommitNs.set(System.nanoTime() - commitStartNs) + metrics.commitSyncNs.update(System.nanoTime() - commitStartNs) + debug("Finishing sync stage of commit for taskName: %s checkpointId: %s" format (taskName, checkpointId)) + } + + private def writeCheckpoint(checkpointId: CheckpointId, inputOffsets: util.Map[SystemStreamPartition, String]) = { + new Function[util.Map[String, util.Map[String, String]], util.Map[String, util.Map[String, String]]]() { + override def apply(uploadSCMs: util.Map[String, util.Map[String, String]]) = { + trace("Got asynchronous upload SCMs for taskName: %s checkpointId: %s as: %s " + format(taskName, checkpointId, uploadSCMs)) + + debug("Creating and writing checkpoints for taskName: %s checkpointId: %s" format (taskName, checkpointId)) + checkpointWriteVersions.foreach(checkpointWriteVersion => { + val checkpoint = if (checkpointWriteVersion == 1) { + // build CheckpointV1 with KafkaChangelogSSPOffset for backwards compatibility + val allCheckpointOffsets = new util.HashMap[SystemStreamPartition, String]() + allCheckpointOffsets.putAll(inputOffsets) + val newestChangelogOffsets = KafkaStateCheckpointMarker.scmsToSSPOffsetMap(uploadSCMs) + newestChangelogOffsets.foreach { case (ssp, newestOffsetOption) => + val offset = new KafkaChangelogSSPOffset(checkpointId, newestOffsetOption.orNull).toString + allCheckpointOffsets.put(ssp, offset) + } + new CheckpointV1(allCheckpointOffsets) + } else if (checkpointWriteVersion == 2) { + new CheckpointV2(checkpointId, inputOffsets, uploadSCMs) + } else { + throw new SamzaException("Unsupported checkpoint write version: " + checkpointWriteVersion) + } + + trace("Writing checkpoint for taskName: %s checkpointId: %s as: %s" + format(taskName, checkpointId, checkpoint)) + + // Write input offsets and state checkpoint markers to task store and checkpoint directories + commitManager.writeCheckpointToStoreDirectories(checkpoint) + + // Write input offsets and state checkpoint markers to the checkpoint topic atomically + offsetManager.writeCheckpoint(taskName, checkpoint) + }) - if (newestChangelogOffsets != null) { - newestChangelogOffsets.foreach {case (ssp, newestOffsetOption) => - val offset = new CheckpointedChangelogOffset(checkpointId, newestOffsetOption.orNull).toString - allCheckpointOffsets.put(ssp, offset) + uploadSCMs } } - val checkpoint = new Checkpoint(allCheckpointOffsets) - trace("Got combined checkpoint offsets for taskName: %s as: %s" format (taskName, allCheckpointOffsets)) + } - offsetManager.writeCheckpoint(taskName, checkpoint) + private def cleanUp(checkpointId: CheckpointId) = { + new Function[util.Map[String, util.Map[String, String]], CompletableFuture[Void]] { + override def apply(uploadSCMs: util.Map[String, util.Map[String, String]]): CompletableFuture[Void] = { + // Perform cleanup on unused checkpoints + debug("Cleaning up old checkpoint state for taskName: %s checkpointId: %s" format(taskName, checkpointId)) + val cleanUpStartTime = System.nanoTime() + try { + commitManager.cleanUp(checkpointId, uploadSCMs) + } catch { + case e: Exception => + // WARNING: cleanUp is NOT optional with blob stores since this is where we reset the TTL for + // tracked blobs. if this TTL reset is skipped, some of the blobs retained by future commits may + // be deleted in the background by the blob store, leading to data loss. + throw new SamzaException( + "Failed to remove old checkpoint state for taskName: %s checkpointId: %s." + format(taskName, checkpointId), e) + } finally { + metrics.asyncCleanupNs.update(System.nanoTime() - cleanUpStartTime) + } + } + } + } - if (storageManager != null) { - trace("Remove old checkpoint stores for taskName: %s" format taskName) - try { - storageManager.removeOldCheckpoints(checkpointId) - } catch { - case e: Exception => error("Failed to remove old checkpoints for task: %s. Current checkpointId: %s" format (taskName, checkpointId), e) + private def trim(checkpointId: CheckpointId, inputOffsets: util.Map[SystemStreamPartition, String]) = { + new Runnable { + override def run(): Unit = { + trace("Deleting committed input offsets from intermediate topics for taskName: %s checkpointId: %s" + format (taskName, checkpointId)) + inputOffsets.asScala + .filter { case (ssp, _) => streamsToDeleteCommittedMessages.contains(ssp.getStream) } // Only delete data of intermediate streams + .groupBy { case (ssp, _) => ssp.getSystem } + .foreach { case (systemName: String, offsets: Map[SystemStreamPartition, String]) => + systemAdmins.getSystemAdmin(systemName).deleteMessages(offsets.asJava) + } } } + } - if (inputCheckpoint != null) { - trace("Deleting committed input offsets for taskName: %s" format taskName) - inputCheckpoint.getOffsets.asScala - .filter { case (ssp, _) => streamsToDeleteCommittedMessages.contains(ssp.getStream) } // Only delete data of intermediate streams - .groupBy { case (ssp, _) => ssp.getSystem } - .foreach { case (systemName: String, offsets: Map[SystemStreamPartition, String]) => - systemAdmins.getSystemAdmin(systemName).deleteMessages(offsets.asJava) + private def handleCompletion(checkpointId: CheckpointId, commitStartNs: Long, asyncStageStartNs: Long) = { + new BiConsumer[Void, Throwable] { + override def accept(v: Void, e: Throwable): Unit = { + try { + debug("%s finishing async stage of commit for taskName: %s checkpointId: %s." + format (if (e == null) "Successfully" else "Unsuccessfully", taskName, checkpointId)) + if (e != null) { + val exception = new SamzaException("Unrecoverable error during async stage of commit " + + "for taskName: %s checkpointId: %s" format(taskName, checkpointId), e) + val exceptionSet = commitException.compareAndSet(null, exception) + if (!exceptionSet) { + // should never happen because there should be at most one async stage of commit in progress + // for a task and another one shouldn't be schedule if the previous one failed. throw a new + // exception on the caller thread for logging and debugging if this happens. + error("Should not have encountered a non-null saved exception during async stage of " + + "commit for taskName: %s checkpointId: %s" format(taskName, checkpointId), commitException.get()) + error("New exception during async stage of commit for taskName: %s checkpointId: %s" + format(taskName, checkpointId), exception) + throw new SamzaException("Should not have encountered a non-null saved exception " + + "during async stage of commit for taskName: %s checkpointId: %s. New exception logged above. " + + "Saved exception under Caused By.", commitException.get()) + } + } else { + metrics.commitAsyncNs.update(System.nanoTime() - asyncStageStartNs) + metrics.commitNs.update(System.nanoTime() - commitStartNs) + } + } finally { + // release the permit indicating that previous commit is complete. + commitInProgress.release() } + } } } def shutdownTask { + if (commitManager != null) { + debug("Shutting down commit manager for taskName: %s" format taskName) + commitManager.close() + } else { + debug("Skipping commit manager shutdown for taskName: %s" format taskName) + } applicationTaskContextOption.foreach(applicationTaskContext => { debug("Stopping application-defined task context for taskName: %s" format taskName) applicationTaskContext.stop() diff --git a/samza-core/src/main/scala/org/apache/samza/container/TaskInstanceMetrics.scala b/samza-core/src/main/scala/org/apache/samza/container/TaskInstanceMetrics.scala index bdd773c281..54d3665253 100644 --- a/samza-core/src/main/scala/org/apache/samza/container/TaskInstanceMetrics.scala +++ b/samza-core/src/main/scala/org/apache/samza/container/TaskInstanceMetrics.scala @@ -37,7 +37,17 @@ class TaskInstanceMetrics( val flushes = newCounter("flush-calls") val pendingMessages = newGauge("pending-messages", 0) val messagesInFlight = newGauge("messages-in-flight", 0) - val asyncCallbackCompleted = newCounter("async-callback-complete-calls"); + val asyncCallbackCompleted = newCounter("async-callback-complete-calls") + val commitsTimedOut = newGauge("commits-timed-out", 0) + val commitsSkipped = newGauge("commits-skipped", 0) + val commitNs = newTimer("commit-ns") + val lastCommitNs = newGauge("last-commit-ns", 0L) + val commitSyncNs = newTimer("commit-sync-ns") + val commitAsyncNs = newTimer("commit-async-ns") + val snapshotNs = newTimer("snapshot-ns") + val storeCheckpointNs = newTimer("store-checkpoint-ns") + val asyncUploadNs = newTimer("async-upload-ns") + val asyncCleanupNs = newTimer("async-cleanup-ns") def addOffsetGauge(systemStreamPartition: SystemStreamPartition, getValue: () => String) { newGauge("%s-%s-%d-offset" format (systemStreamPartition.getSystem, systemStreamPartition.getStream, systemStreamPartition.getPartition.getPartitionId), getValue) diff --git a/samza-core/src/main/scala/org/apache/samza/job/local/ProcessJobFactory.scala b/samza-core/src/main/scala/org/apache/samza/job/local/ProcessJobFactory.scala index 32649329b2..64e24086f8 100644 --- a/samza-core/src/main/scala/org/apache/samza/job/local/ProcessJobFactory.scala +++ b/samza-core/src/main/scala/org/apache/samza/job/local/ProcessJobFactory.scala @@ -24,7 +24,7 @@ import java.util import org.apache.samza.SamzaException import org.apache.samza.application.ApplicationUtil import org.apache.samza.application.descriptors.ApplicationDescriptorUtil -import org.apache.samza.config.{Config, JobConfig, TaskConfig} +import org.apache.samza.config.{BlobStoreConfig, Config, JobConfig, StorageConfig, TaskConfig} import org.apache.samza.container.TaskName import org.apache.samza.coordinator.metadatastore.{CoordinatorStreamStore, NamespaceAwareCoordinatorStreamStore} import org.apache.samza.coordinator.stream.messages.SetChangelogMapping @@ -34,7 +34,7 @@ import org.apache.samza.job.model.JobModelUtil import org.apache.samza.job.{CommandBuilder, ShellCommandBuilder, StreamJob, StreamJobFactory} import org.apache.samza.metrics.MetricsRegistryMap import org.apache.samza.startpoint.StartpointManager -import org.apache.samza.storage.ChangelogStreamManager +import org.apache.samza.storage.{ChangelogStreamManager, StateBackendFactory} import org.apache.samza.util.{ConfigUtil, CoordinatorStreamUtil, DiagnosticsUtil, Logging, ReflectionUtil} import scala.collection.JavaConversions._ @@ -91,6 +91,17 @@ class ProcessJobFactory extends StreamJobFactory with Logging { val metadataResourceUtil = new MetadataResourceUtil(jobModel, metricsRegistry, config) metadataResourceUtil.createResources() + val storageConfig = new StorageConfig(config) + storageConfig.getBackupFactories.foreach(stateStorageBackendBackupFactory => { + val stateBackendFactory : StateBackendFactory = + ReflectionUtil.getObj(stateStorageBackendBackupFactory, classOf[StateBackendFactory]) + val stateBackendAdmin = stateBackendFactory.getAdmin(jobModel, config) + // Create resources required for state backend admin + stateBackendAdmin.createResources() + // Validate resources required for state backend admin + stateBackendAdmin.validateResources() + }) + if (new JobConfig(config).getStartpointEnabled()) { // fan out the startpoints val startpointManager = new StartpointManager(coordinatorStreamStore) diff --git a/samza-core/src/main/scala/org/apache/samza/serializers/CheckpointSerde.scala b/samza-core/src/main/scala/org/apache/samza/serializers/CheckpointV1Serde.scala similarity index 54% rename from samza-core/src/main/scala/org/apache/samza/serializers/CheckpointSerde.scala rename to samza-core/src/main/scala/org/apache/samza/serializers/CheckpointV1Serde.scala index 02246675e2..7db709e06f 100644 --- a/samza-core/src/main/scala/org/apache/samza/serializers/CheckpointSerde.scala +++ b/samza-core/src/main/scala/org/apache/samza/serializers/CheckpointV1Serde.scala @@ -19,14 +19,13 @@ package org.apache.samza.serializers -import com.fasterxml.jackson.core.`type`.TypeReference import com.fasterxml.jackson.databind.ObjectMapper -import org.apache.samza.util.Logging import java.util -import org.apache.samza.checkpoint.Checkpoint -import org.apache.samza.container.TaskName +import org.apache.samza.Partition +import org.apache.samza.checkpoint.CheckpointV1 import org.apache.samza.system.SystemStreamPartition -import org.apache.samza.{SamzaException, Partition} +import org.apache.samza.util.Logging + import scala.collection.JavaConverters._ /** @@ -35,34 +34,26 @@ import scala.collection.JavaConverters._ * then interfere with JSON's decoding of the overall map. We'll sidestep the whole issue by turning the * map into a list[String] of (System, Stream, Partition, Offset) serializing that. */ -class CheckpointSerde extends Serde[Checkpoint] with Logging { - import CheckpointSerde._ - // TODO: Elucidate the CheckpointSerde relationshiop to Serde. Should Serde also have keyTo/FromBytes? Should - // we just take CheckpointSerde here as interface and have this be JSONCheckpointSerde? - // TODO: Add more tests. This class currently only has direct test and is mainly tested by the other checkpoint managers +class CheckpointV1Serde extends Serde[CheckpointV1] with Logging { val jsonMapper = new ObjectMapper() - // Jackson absolutely hates Scala types and hidden conversions hate you, so we're going to be very, very - // explicit about the Java (not Scala) types used here and never let Scala get its grubby little hands - // on any instance. - - // Store checkpoint as maps keyed of the SSP.toString to the another map of the constituent SSP components - // and offset. Jackson can't automatically serialize the SSP since it's not a POJO and this avoids - // having to wrap it another class while maintaing readability. - - def fromBytes(bytes: Array[Byte]): Checkpoint = { + // Serialize checkpoint as maps keyed by the SSP.toString() to the another map of the constituent SSP components + // and offset. Jackson can't automatically serialize the SSP since it's not a POJO and this avoids + // having to wrap it another class while maintaining readability. + // { "SSP.toString()" -> {"system": system, "stream": stream, "partition": partition, "offset": offset)} + def fromBytes(bytes: Array[Byte]): CheckpointV1 = { try { val jMap = jsonMapper.readValue(bytes, classOf[util.HashMap[String, util.HashMap[String, String]]]) - def deserializeJSONMap(m:util.HashMap[String, String]) = { - require(m.size() == 4, "All JSON-encoded SystemStreamPartitions must have four keys") - val system = m.get("system") + def deserializeJSONMap(sspInfo:util.HashMap[String, String]) = { + require(sspInfo.size() == 4, "All JSON-encoded SystemStreamPartitions must have four keys") + val system = sspInfo.get("system") require(system != null, "System must be present in JSON-encoded SystemStreamPartition") - val stream = m.get("stream") + val stream = sspInfo.get("stream") require(stream != null, "Stream must be present in JSON-encoded SystemStreamPartition") - val partition = m.get("partition") + val partition = sspInfo.get("partition") require(partition != null, "Partition must be present in JSON-encoded SystemStreamPartition") - val offset = m.get("offset") + val offset = sspInfo.get("offset") // allow null offsets, e.g. for changelog ssps new SystemStreamPartition(system, stream, new Partition(partition.toInt)) -> offset @@ -70,16 +61,15 @@ class CheckpointSerde extends Serde[Checkpoint] with Logging { val cpMap = jMap.values.asScala.map(deserializeJSONMap).toMap - new Checkpoint(cpMap.asJava) - }catch { + new CheckpointV1(cpMap.asJava) + } catch { case e : Exception => - warn("Exception while deserializing checkpoint: " + e) - debug("Exception detail:", e) + warn("Exception while deserializing checkpoint: {}", util.Arrays.toString(bytes), e) null } } - def toBytes(checkpoint: Checkpoint): Array[Byte] = { + def toBytes(checkpoint: CheckpointV1): Array[Byte] = { val offsets = checkpoint.getOffsets val asMap = new util.HashMap[String, util.HashMap[String, String]](offsets.size()) @@ -96,21 +86,4 @@ class CheckpointSerde extends Serde[Checkpoint] with Logging { jsonMapper.writeValueAsBytes(asMap) } - - def changelogPartitionMappingFromBytes(bytes: Array[Byte]): util.Map[TaskName, java.lang.Integer] = { - try { - jsonMapper.readValue(bytes, PARTITION_MAPPING_TYPEREFERENCE) - } catch { - case e : Exception => - throw new SamzaException("Exception while deserializing changelog partition mapping", e) - } - } - - def changelogPartitionMappingToBytes(mapping: util.Map[TaskName, java.lang.Integer]) = { - jsonMapper.writeValueAsBytes(new util.HashMap[TaskName, java.lang.Integer](mapping)) - } -} - -object CheckpointSerde { - val PARTITION_MAPPING_TYPEREFERENCE = new TypeReference[util.HashMap[TaskName, java.lang.Integer]]() {} -} +} \ No newline at end of file diff --git a/samza-core/src/main/scala/org/apache/samza/storage/ContainerStorageManager.java b/samza-core/src/main/scala/org/apache/samza/storage/ContainerStorageManager.java index c333748585..f804a5e7c9 100644 --- a/samza-core/src/main/scala/org/apache/samza/storage/ContainerStorageManager.java +++ b/samza-core/src/main/scala/org/apache/samza/storage/ContainerStorageManager.java @@ -44,7 +44,9 @@ import org.apache.samza.SamzaException; import org.apache.samza.checkpoint.Checkpoint; import org.apache.samza.checkpoint.CheckpointManager; +import org.apache.samza.checkpoint.CheckpointV2; import org.apache.samza.config.Config; +import org.apache.samza.config.JobConfig; import org.apache.samza.config.StorageConfig; import org.apache.samza.config.TaskConfig; import org.apache.samza.container.RunLoop; @@ -65,7 +67,6 @@ import org.apache.samza.storage.kv.Entry; import org.apache.samza.storage.kv.KeyValueStore; import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.SSPMetadataCache; import org.apache.samza.system.StreamMetadataCache; import org.apache.samza.system.SystemAdmins; import org.apache.samza.system.SystemConsumer; @@ -79,6 +80,7 @@ import org.apache.samza.system.chooser.MessageChooser; import org.apache.samza.system.chooser.RoundRobinChooserFactory; import org.apache.samza.table.utils.SerdeUtils; +import org.apache.samza.task.MessageCollector; import org.apache.samza.task.TaskInstanceCollector; import org.apache.samza.util.Clock; import org.apache.samza.util.ReflectionUtil; @@ -116,20 +118,23 @@ public class ContainerStorageManager { private static final int SIDE_INPUT_CHECK_TIMEOUT_SECONDS = 10; private static final int SIDE_INPUT_SHUTDOWN_TIMEOUT_SECONDS = 60; + private static final int RESTORE_THREAD_POOL_SHUTDOWN_TIMEOUT_SECONDS = 60; + /** Maps containing relevant per-task objects */ - private final Map> taskStores; - private final Map taskRestoreManagers; private final Map taskInstanceMetrics; private final Map taskInstanceCollectors; + private final Map> inMemoryStores; // subset of taskStores after #start() + private Map> taskStores; // Will be available after #start() private final Map storeConsumers; // Mapping from store name to SystemConsumers private final Map> storageEngineFactories; // Map of storageEngineFactories indexed by store name private final Map changelogSystemStreams; // Map of changelog system-streams indexed by store name private final Map> serdes; // Map of Serde objects indexed by serde name (specified in config) private final SystemAdmins systemAdmins; + private final Clock clock; + private final Map restoreStateBackendFactories; private final StreamMetadataCache streamMetadataCache; - private final SSPMetadataCache sspMetadataCache; private final SamzaContainerMetrics samzaContainerMetrics; private final CheckpointManager checkpointManager; @@ -142,32 +147,33 @@ public class ContainerStorageManager { private final File nonLoggedStoreBaseDirectory; private final Set storeDirectoryPaths; // the set of store directory paths, used by SamzaContainer to initialize its disk-space-monitor - private final int parallelRestoreThreadPoolSize; - private final int maxChangeLogStreamPartitions; // The partition count of each changelog-stream topic. This is used for validating changelog streams before restoring. - /* Sideinput related parameters */ private final boolean hasSideInputs; + private final Map> sideInputStores; // subset of taskStores after #start() // side inputs indexed first by task, then store name private final Map>> taskSideInputStoreSSPs; + private final Set sideInputStoreNames; private final Map sspSideInputHandlers; private SystemConsumers sideInputSystemConsumers; - private volatile Map sideInputTaskLatches; // Used by the sideInput-read thread to signal to the main thread + private final Map sideInputTaskLatches; // Used by the sideInput-read thread to signal to the main thread private volatile boolean shouldShutdown = false; private RunLoop sideInputRunLoop; private final ExecutorService sideInputsExecutor = Executors.newSingleThreadExecutor( new ThreadFactoryBuilder().setDaemon(true).setNameFormat(SIDEINPUTS_THREAD_NAME).build()); + private final ExecutorService restoreExecutor; private volatile Throwable sideInputException = null; private final Config config; private final StorageManagerUtil storageManagerUtil = new StorageManagerUtil(); + private boolean isStarted = false; + public ContainerStorageManager( CheckpointManager checkpointManager, ContainerModel containerModel, StreamMetadataCache streamMetadataCache, - SSPMetadataCache sspMetadataCache, SystemAdmins systemAdmins, Map changelogSystemStreams, Map> sideInputSystemStreams, @@ -179,26 +185,28 @@ public ContainerStorageManager( SamzaContainerMetrics samzaContainerMetrics, JobContext jobContext, ContainerContext containerContext, + Map restoreStateBackendFactories, Map taskInstanceCollectors, File loggedStoreBaseDirectory, File nonLoggedStoreBaseDirectory, - int maxChangeLogStreamPartitions, SerdeManager serdeManager, Clock clock) { this.checkpointManager = checkpointManager; this.containerModel = containerModel; this.taskSideInputStoreSSPs = getTaskSideInputSSPs(containerModel, sideInputSystemStreams); + this.sideInputStoreNames = sideInputSystemStreams.keySet(); this.sideInputTaskLatches = new HashMap<>(); this.hasSideInputs = this.taskSideInputStoreSSPs.values().stream() .flatMap(m -> m.values().stream()) .flatMap(Collection::stream) .findAny() .isPresent(); - this.sspMetadataCache = sspMetadataCache; this.changelogSystemStreams = getChangelogSystemStreams(containerModel, changelogSystemStreams); // handling standby tasks LOG.info("Starting with changelogSystemStreams = {} taskSideInputStoreSSPs = {}", this.changelogSystemStreams, this.taskSideInputStoreSSPs); + this.clock = clock; + this.restoreStateBackendFactories = restoreStateBackendFactories; this.storageEngineFactories = storageEngineFactories; this.serdes = serdes; this.loggedStoreBaseDirectory = loggedStoreBaseDirectory; @@ -226,15 +234,22 @@ public ContainerStorageManager( // initializing the set of store directory paths this.storeDirectoryPaths = new HashSet<>(); - // Setting the restore thread pool size equal to the number of taskInstances - this.parallelRestoreThreadPoolSize = containerModel.getTasks().size(); - - this.maxChangeLogStreamPartitions = maxChangeLogStreamPartitions; this.streamMetadataCache = streamMetadataCache; this.systemAdmins = systemAdmins; - // create taskStores for all tasks in the containerModel and each store in storageEngineFactories - this.taskStores = createTaskStores(containerModel, jobContext, containerContext, storageEngineFactories, serdes, taskInstanceMetrics, taskInstanceCollectors); + // create side input taskStores for all tasks in the containerModel and each store in storageEngineFactories + this.sideInputStores = createTaskStores(sideInputStoreNames, containerModel, jobContext, containerContext, + storageEngineFactories, serdes, taskInstanceMetrics, taskInstanceCollectors); + StorageConfig storageConfig = new StorageConfig(config); + Set inMemoryStoreNames = storageEngineFactories.keySet().stream() + .filter(storeName -> { + Optional storeFactory = storageConfig.getStorageFactoryClassName(storeName); + return storeFactory.isPresent() && !storeFactory.get() + .equals(StorageConfig.INMEMORY_KV_STORAGE_ENGINE_FACTORY); + }) + .collect(Collectors.toSet()); + this.inMemoryStores = createTaskStores(inMemoryStoreNames, + this.containerModel, jobContext, containerContext, storageEngineFactories, serdes, taskInstanceMetrics, taskInstanceCollectors); Set containerChangelogSystems = this.changelogSystemStreams.values().stream() .map(SystemStream::getSystem) @@ -245,8 +260,16 @@ public ContainerStorageManager( containerChangelogSystems, systemFactories, config, this.samzaContainerMetrics.registry()); this.storeConsumers = createStoreIndexedMap(this.changelogSystemStreams, storeSystemConsumers); - // creating task restore managers - this.taskRestoreManagers = createTaskRestoreManagers(systemAdmins, clock, this.samzaContainerMetrics); + // TODO HIGH dchen tune based on observed concurrency + JobConfig jobConfig = new JobConfig(config); + int restoreThreadPoolSize = + Math.min( + Math.max(containerModel.getTasks().size() * restoreStateBackendFactories.size() * 2, + jobConfig.getRestoreThreadPoolSize()), + jobConfig.getRestoreThreadPoolMaxSize() + ); + this.restoreExecutor = Executors.newFixedThreadPool(restoreThreadPoolSize, + new ThreadFactoryBuilder().setDaemon(true).setNameFormat(RESTORE_THREAD_NAME).build()); this.sspSideInputHandlers = createSideInputHandlers(clock); @@ -311,7 +334,8 @@ private Map>> getTaskSideInputS * @param changelogSystemStreams the passed in set of changelogSystemStreams * @return A map of changeLogSSP to storeName across all tasks, assuming no two stores have the same changelogSSP */ - private Map getChangelogSystemStreams(ContainerModel containerModel, Map changelogSystemStreams) { + private Map getChangelogSystemStreams(ContainerModel containerModel, + Map changelogSystemStreams) { if (MapUtils.invertMap(changelogSystemStreams).size() != changelogSystemStreams.size()) { throw new SamzaException("Two stores cannot have the same changelog system-stream"); @@ -319,20 +343,22 @@ private Map getChangelogSystemStreams(ContainerModel conta Map changelogSSPToStore = new HashMap<>(); changelogSystemStreams.forEach((storeName, systemStream) -> - containerModel.getTasks().forEach((taskName, taskModel) -> { changelogSSPToStore.put(new SystemStreamPartition(systemStream, taskModel.getChangelogPartition()), storeName); }) + containerModel.getTasks().forEach((taskName, taskModel) -> + changelogSSPToStore.put(new SystemStreamPartition(systemStream, taskModel.getChangelogPartition()), storeName)) ); getTasks(containerModel, TaskMode.Standby).forEach((taskName, taskModel) -> { - this.taskSideInputStoreSSPs.putIfAbsent(taskName, new HashMap<>()); + taskSideInputStoreSSPs.putIfAbsent(taskName, new HashMap<>()); changelogSystemStreams.forEach((storeName, systemStream) -> { SystemStreamPartition ssp = new SystemStreamPartition(systemStream, taskModel.getChangelogPartition()); changelogSSPToStore.remove(ssp); - this.taskSideInputStoreSSPs.get(taskName).put(storeName, Collections.singleton(ssp)); + taskSideInputStoreSSPs.get(taskName).put(storeName, Collections.singleton(ssp)); }); }); // changelogSystemStreams correspond only to active tasks (since those of standby-tasks moved to sideInputs above) - return MapUtils.invertMap(changelogSSPToStore).entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, x -> x.getValue().getSystemStream())); + return MapUtils.invertMap(changelogSSPToStore).entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, x -> x.getValue().getSystemStream())); } @@ -357,28 +383,106 @@ private static Map createConsumers(Set storeSyst } private static Map createStoreIndexedMap(Map changelogSystemStreams, - Map storeSystemConsumers) { + Map systemNameToSystemConsumers) { // Map of each storeName to its respective systemConsumer Map storeConsumers = new HashMap<>(); // Populate the map of storeName to its relevant systemConsumer for (String storeName : changelogSystemStreams.keySet()) { - storeConsumers.put(storeName, storeSystemConsumers.get(changelogSystemStreams.get(storeName).getSystem())); + storeConsumers.put(storeName, systemNameToSystemConsumers.get(changelogSystemStreams.get(storeName).getSystem())); } return storeConsumers; } - private Map createTaskRestoreManagers(SystemAdmins systemAdmins, Clock clock, SamzaContainerMetrics samzaContainerMetrics) { - Map taskRestoreManagers = new HashMap<>(); - containerModel.getTasks().forEach((taskName, taskModel) -> { - taskRestoreManagers.put(taskName, - TaskRestoreManagerFactory.create( - taskModel, changelogSystemStreams, getNonSideInputStores(taskName), systemAdmins, - streamMetadataCache, sspMetadataCache, storeConsumers, maxChangeLogStreamPartitions, - loggedStoreBaseDirectory, nonLoggedStoreBaseDirectory, config, clock)); - samzaContainerMetrics.addStoresRestorationGauge(taskName); + private Map createTaskRestoreManagers(Map factories, + Map> backendFactoryStoreNames, Clock clock, SamzaContainerMetrics samzaContainerMetrics, TaskName taskName, + TaskModel taskModel) { + // Get the factories for the task based on the stores of the tasks to be restored from the factory + Map backendFactoryRestoreManagers = new HashMap<>(); // backendFactoryName -> restoreManager + MetricsRegistry taskMetricsRegistry = + taskInstanceMetrics.get(taskName) != null ? taskInstanceMetrics.get(taskName).registry() : new MetricsRegistryMap(); + + backendFactoryStoreNames.forEach((factoryName, storeNames) -> { + StateBackendFactory factory = factories.get(factoryName); + KafkaChangelogRestoreParams kafkaChangelogRestoreParams = new KafkaChangelogRestoreParams(storeConsumers, + inMemoryStores.get(taskName), systemAdmins.getSystemAdmins(), storageEngineFactories, serdes, + taskInstanceCollectors.get(taskName)); + TaskRestoreManager restoreManager = factory.getRestoreManager(jobContext, containerContext, taskModel, restoreExecutor, + taskMetricsRegistry, storeNames, config, clock, loggedStoreBaseDirectory, nonLoggedStoreBaseDirectory, + kafkaChangelogRestoreParams); + + backendFactoryRestoreManagers.put(factoryName, restoreManager); }); - return taskRestoreManagers; + samzaContainerMetrics.addStoresRestorationGauge(taskName); + return backendFactoryRestoreManagers; + } + + /** + * Return a map of backend factory names to set of stores that should be restored using it + */ + @VisibleForTesting + Map> getBackendFactoryStoreNames(Checkpoint checkpoint, Set storeNames, + StorageConfig storageConfig) { + Map> backendFactoryStoreNames = new HashMap<>(); // backendFactoryName -> set(storeNames) + + if (checkpoint != null && checkpoint.getVersion() == 1) { + // Only restore stores with changelog streams configured + Set changelogStores = storeNames.stream() + .filter(storeName -> storageConfig.getChangelogStream(storeName).isPresent()) + .collect(Collectors.toSet()); + // Default to changelog backend factory when using checkpoint v1 for backwards compatibility + if (!changelogStores.isEmpty()) { + backendFactoryStoreNames.put(StorageConfig.KAFKA_STATE_BACKEND_FACTORY, changelogStores); + } + if (storeNames.size() > changelogStores.size()) { + Set nonChangelogStores = storeNames.stream() + .filter(storeName -> !changelogStores.contains(storeName)) + .collect(Collectors.toSet()); + LOG.info("non-Side input stores: {}, do not have a configured store changelogs for checkpoint V1," + + "restore for the store will be skipped", + nonChangelogStores); + } + } else if (checkpoint == null || checkpoint.getVersion() == 2) { + // Extract the state checkpoint markers if checkpoint exists + Map> stateCheckpointMarkers = checkpoint == null ? Collections.emptyMap() : + ((CheckpointV2) checkpoint).getStateCheckpointMarkers(); + + // Find stores associated to each state backend factory + storeNames.forEach(storeName -> { + List storeFactories = storageConfig.getStoreRestoreFactories(storeName); + + if (storeFactories.isEmpty()) { + // If the restore factory is not configured for the store and the store does not have a changelog topic + LOG.info("non-Side input store: {}, does not have a configured restore factories nor store changelogs," + + "restore for the store will be skipped", + storeName); + } else { + // Search the ordered list for the first matched state backend factory in the checkpoint + // If the checkpoint does not exist or state checkpoint markers does not exist, we match the first configured + // restore manager + Optional factoryNameOpt = storeFactories.stream() + .filter(factoryName -> stateCheckpointMarkers.containsKey(factoryName) && + stateCheckpointMarkers.get(factoryName).containsKey(storeName)) + .findFirst(); + String factoryName; + if (factoryNameOpt.isPresent()) { + factoryName = factoryNameOpt.get(); + } else { // Restore factories configured but no checkpoints found + // Use first configured restore factory + factoryName = storeFactories.get(0); + LOG.warn("No matching checkpoints found for configured factories: {}, " + + "defaulting to using the first configured factory with no checkpoints", storeFactories); + } + if (!backendFactoryStoreNames.containsKey(factoryName)) { + backendFactoryStoreNames.put(factoryName, new HashSet<>()); + } + backendFactoryStoreNames.get(factoryName).add(storeName); + } + }); + } else { + throw new SamzaException(String.format("Unsupported checkpoint version %s", checkpoint.getVersion())); + } + return backendFactoryStoreNames; } // Helper method to filter active Tasks from the container model @@ -388,106 +492,77 @@ private static Map getTasks(ContainerModel containerModel, } /** - * Create taskStores for all stores in storageEngineFactories. - * The store mode is chosen as bulk-load if its a non-sideinput store, and readWrite if its a sideInput store + * Create taskStores for all stores in storesToCreate. + * The store mode is chosen as read-write mode. */ - private Map> createTaskStores(ContainerModel containerModel, JobContext jobContext, ContainerContext containerContext, + private Map> createTaskStores(Set storesToCreate, + ContainerModel containerModel, JobContext jobContext, ContainerContext containerContext, Map> storageEngineFactories, Map> serdes, Map taskInstanceMetrics, Map taskInstanceCollectors) { - Map> taskStores = new HashMap<>(); + StorageConfig storageConfig = new StorageConfig(config); - // iterate over each task in the containerModel, and each store in storageEngineFactories + // iterate over each task and each storeName for (Map.Entry task : containerModel.getTasks().entrySet()) { TaskName taskName = task.getKey(); TaskModel taskModel = task.getValue(); - if (!taskStores.containsKey(taskName)) { taskStores.put(taskName, new HashMap<>()); } - for (String storeName : storageEngineFactories.keySet()) { - - StorageEngineFactory.StoreMode storeMode = this.taskSideInputStoreSSPs.get(taskName).containsKey(storeName) ? - StorageEngineFactory.StoreMode.ReadWrite : StorageEngineFactory.StoreMode.BulkLoad; + for (String storeName : storesToCreate) { + List storeBackupManagers = storageConfig.getStoreBackupFactories(storeName); + // A store is considered durable if it is backed by a changelog or another backupManager factory + boolean isDurable = changelogSystemStreams.containsKey(storeName) || !storeBackupManagers.isEmpty(); + boolean isSideInput = this.sideInputStoreNames.contains(storeName); + // Use the logged-store-base-directory for change logged stores and sideInput stores, and non-logged-store-base-dir + // for non logged stores + File storeBaseDir = isDurable || isSideInput ? this.loggedStoreBaseDirectory : this.nonLoggedStoreBaseDirectory; + File storeDirectory = storageManagerUtil.getTaskStoreDir(storeBaseDir, storeName, taskName, + taskModel.getTaskMode()); + this.storeDirectoryPaths.add(storeDirectory.toPath()); + + // if taskInstanceMetrics are specified use those for store metrics, + // otherwise (in case of StorageRecovery) use a blank MetricsRegistryMap + MetricsRegistry storeMetricsRegistry = + taskInstanceMetrics.get(taskName) != null ? taskInstanceMetrics.get(taskName).registry() : new MetricsRegistryMap(); StorageEngine storageEngine = - createStore(storeName, taskName, taskModel, jobContext, containerContext, storageEngineFactories, serdes, taskInstanceMetrics, taskInstanceCollectors, storeMode); + createStore(storeName, storeDirectory, taskModel, jobContext, containerContext, storageEngineFactories, + serdes, storeMetricsRegistry, taskInstanceCollectors.get(taskName), + StorageEngineFactory.StoreMode.ReadWrite, this.changelogSystemStreams, this.config); // add created store to map taskStores.get(taskName).put(storeName, storageEngine); - LOG.info("Created store {} for task {} in mode {}", storeName, taskName, storeMode); + LOG.info("Created task store {} in read-write mode for task {} in path {}", storeName, taskName, storeDirectory.getAbsolutePath()); } } - return taskStores; } - /** - * Recreate all non-sideInput persistent stores in ReadWrite mode. - * - */ - private void recreatePersistentTaskStoresInReadWriteMode(ContainerModel containerModel, JobContext jobContext, - ContainerContext containerContext, Map> storageEngineFactories, - Map> serdes, Map taskInstanceMetrics, - Map taskInstanceCollectors) { - - // iterate over each task and each storeName - for (Map.Entry task : containerModel.getTasks().entrySet()) { - TaskName taskName = task.getKey(); - TaskModel taskModel = task.getValue(); - Map nonSideInputStores = getNonSideInputStores(taskName); - - for (String storeName : nonSideInputStores.keySet()) { - - // if this store has been already created then re-create and overwrite it only if it is a - // persistentStore and a non-sideInputStore, because sideInputStores are always created in RW mode - if (nonSideInputStores.get(storeName).getStoreProperties().isPersistedToDisk()) { - - StorageEngine storageEngine = - createStore(storeName, taskName, taskModel, jobContext, containerContext, storageEngineFactories, serdes, taskInstanceMetrics, taskInstanceCollectors, - StorageEngineFactory.StoreMode.ReadWrite); - - // add created store to map - this.taskStores.get(taskName).put(storeName, storageEngine); - - LOG.info("Re-created store {} in read-write mode for task {} because it a persistent store", storeName, taskName); - } else { - LOG.info("Skipping re-creation of store {} for task {}", storeName, taskName); - } - } - } - } - /** * Method to instantiate a StorageEngine with the given parameters, and populate the storeDirectory paths (used to monitor * disk space). */ - private StorageEngine createStore(String storeName, TaskName taskName, TaskModel taskModel, JobContext jobContext, - ContainerContext containerContext, Map> storageEngineFactories, - Map> serdes, Map taskInstanceMetrics, - Map taskInstanceCollectors, StorageEngineFactory.StoreMode storeMode) { + public static StorageEngine createStore( + String storeName, + File storeDirectory, + TaskModel taskModel, + JobContext jobContext, + ContainerContext containerContext, + Map> storageEngineFactories, + Map> serdes, + MetricsRegistry storeMetricsRegistry, + MessageCollector messageCollector, + StorageEngineFactory.StoreMode storeMode, + Map changelogSystemStreams, + Config config) { StorageConfig storageConfig = new StorageConfig(config); - - SystemStreamPartition changeLogSystemStreamPartition = - (changelogSystemStreams.containsKey(storeName)) ? new SystemStreamPartition( - changelogSystemStreams.get(storeName), taskModel.getChangelogPartition()) : null; - - // Use the logged-store-base-directory for change logged stores and sideInput stores, and non-logged-store-base-dir - // for non logged stores - File storeDirectory; - if (changeLogSystemStreamPartition != null || this.taskSideInputStoreSSPs.get(taskName).containsKey(storeName)) { - storeDirectory = storageManagerUtil.getTaskStoreDir(this.loggedStoreBaseDirectory, storeName, taskName, - taskModel.getTaskMode()); - } else { - storeDirectory = storageManagerUtil.getTaskStoreDir(this.nonLoggedStoreBaseDirectory, storeName, taskName, - taskModel.getTaskMode()); - } - - this.storeDirectoryPaths.add(storeDirectory.toPath()); + SystemStreamPartition changeLogSystemStreamPartition = changelogSystemStreams.containsKey(storeName) ? + new SystemStreamPartition(changelogSystemStreams.get(storeName), taskModel.getChangelogPartition()) : null; Optional storageKeySerde = storageConfig.getStorageKeySerde(storeName); Serde keySerde = null; @@ -500,14 +575,8 @@ private StorageEngine createStore(String storeName, TaskName taskName, TaskModel messageSerde = serdes.get(storageMsgSerde.get()); } - // if taskInstanceMetrics are specified use those for store metrics, - // otherwise (in case of StorageRecovery) use a blank MetricsRegistryMap - MetricsRegistry storeMetricsRegistry = - taskInstanceMetrics.get(taskName) != null ? taskInstanceMetrics.get(taskName).registry() - : new MetricsRegistryMap(); - return storageEngineFactories.get(storeName) - .getStorageEngine(storeName, storeDirectory, keySerde, messageSerde, taskInstanceCollectors.get(taskName), + .getStorageEngine(storeName, storeDirectory, keySerde, messageSerde, messageCollector, storeMetricsRegistry, changeLogSystemStreamPartition, jobContext, containerContext, storeMode); } @@ -584,10 +653,10 @@ private Map createSideInputHandlers if (this.hasSideInputs) { containerModel.getTasks().forEach((taskName, taskModel) -> { - Map sideInputStores = getSideInputStores(taskName); + Map taskSideInputStores = sideInputStores.get(taskName); Map> sideInputStoresToSSPs = new HashMap<>(); boolean taskHasSideInputs = false; - for (String storeName : sideInputStores.keySet()) { + for (String storeName : taskSideInputStores.keySet()) { Set storeSSPs = this.taskSideInputStoreSSPs.get(taskName).get(storeName); taskHasSideInputs = taskHasSideInputs || !storeSSPs.isEmpty(); sideInputStoresToSSPs.put(storeName, storeSSPs); @@ -600,7 +669,7 @@ private Map createSideInputHandlers TaskSideInputHandler taskSideInputHandler = new TaskSideInputHandler(taskName, taskModel.getTaskMode(), loggedStoreBaseDirectory, - sideInputStores, + taskSideInputStores, sideInputStoresToSSPs, taskSideInputProcessors.get(taskName), this.systemAdmins, @@ -612,77 +681,87 @@ private Map createSideInputHandlers handlers.put(ssp, taskSideInputHandler); }); - LOG.info("Created TaskSideInputHandler for task {}, sideInputStores {} and loggedStoreBaseDirectory {}", - taskName, sideInputStores, loggedStoreBaseDirectory); + LOG.info("Created TaskSideInputHandler for task {}, taskSideInputStores {} and loggedStoreBaseDirectory {}", + taskName, taskSideInputStores, loggedStoreBaseDirectory); } }); } return handlers; } - private Map getSideInputStores(TaskName taskName) { - return taskStores.get(taskName).entrySet().stream(). - filter(e -> this.taskSideInputStoreSSPs.get(taskName).containsKey(e.getKey())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - private Map getNonSideInputStores(TaskName taskName) { - return taskStores.get(taskName).entrySet().stream(). - filter(e -> !this.taskSideInputStoreSSPs.get(taskName).containsKey(e.getKey())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - private Set getSideInputHandlers() { return this.sspSideInputHandlers.values().stream().collect(Collectors.toSet()); } public void start() throws SamzaException, InterruptedException { - Map checkpointedChangelogSSPOffsets = new HashMap<>(); - if (new TaskConfig(config).getTransactionalStateRestoreEnabled()) { - getTasks(containerModel, TaskMode.Active).forEach((taskName, taskModel) -> { - if (checkpointManager != null) { - Set changelogSystemStreams = new HashSet<>(this.changelogSystemStreams.values()); - Checkpoint checkpoint = checkpointManager.readLastCheckpoint(taskName); - if (checkpoint != null) { - checkpoint.getOffsets().forEach((ssp, offset) -> { - if (changelogSystemStreams.contains(new SystemStream(ssp.getSystem(), ssp.getStream()))) { - checkpointedChangelogSSPOffsets.put(ssp, offset); - } - }); - } - } - }); + // Restores and recreates + restoreStores(); + // Shutdown restore executor since it will no longer be used + try { + restoreExecutor.shutdown(); + if (restoreExecutor.awaitTermination(RESTORE_THREAD_POOL_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.MILLISECONDS)) { + restoreExecutor.shutdownNow(); + } + } catch (Exception e) { + LOG.error(e.getMessage()); } - LOG.info("Checkpointed changelog ssp offsets: {}", checkpointedChangelogSSPOffsets); - restoreStores(checkpointedChangelogSSPOffsets); if (this.hasSideInputs) { startSideInputs(); } + isStarted = true; } // Restoration of all stores, in parallel across tasks - private void restoreStores(Map checkpointedChangelogSSPOffsets) - throws InterruptedException { + private void restoreStores() throws InterruptedException { LOG.info("Store Restore started"); + Set activeTasks = getTasks(containerModel, TaskMode.Active).keySet(); + // TODO HIGH dchen verify davinci lifecycle + // Find all non-side input stores + Set nonSideInputStoreNames = storageEngineFactories.keySet() + .stream() + .filter(storeName -> !sideInputStoreNames.contains(storeName)) + .collect(Collectors.toSet()); - // initialize each TaskStorageManager - this.taskRestoreManagers.values().forEach(taskStorageManager -> - taskStorageManager.init(checkpointedChangelogSSPOffsets)); + // Obtain the checkpoints for each task + Map> taskRestoreManagers = new HashMap<>(); + Map taskCheckpoints = new HashMap<>(); + containerModel.getTasks().forEach((taskName, taskModel) -> { + Checkpoint taskCheckpoint = null; + if (checkpointManager != null && activeTasks.contains(taskName)) { + // only pass in checkpoints for active tasks + taskCheckpoint = checkpointManager.readLastCheckpoint(taskName); + LOG.info("Obtained checkpoint: {} for state restore for taskName: {}", taskCheckpoint, taskName); + } + taskCheckpoints.put(taskName, taskCheckpoint); + Map> backendFactoryStoreNames = getBackendFactoryStoreNames(taskCheckpoint, nonSideInputStoreNames, + new StorageConfig(config)); + Map taskStoreRestoreManagers = createTaskRestoreManagers(restoreStateBackendFactories, + backendFactoryStoreNames, clock, samzaContainerMetrics, taskName, taskModel); + taskRestoreManagers.put(taskName, taskStoreRestoreManagers); + }); - // Start each store consumer once - this.storeConsumers.values().stream().distinct().forEach(SystemConsumer::start); + // Initialize each TaskStorageManager + taskRestoreManagers.forEach((taskName, restoreManagers) -> + restoreManagers.forEach((factoryName, taskRestoreManager) -> + taskRestoreManager.init(taskCheckpoints.get(taskName)) + ) + ); - // Create a thread pool for parallel restores (and stopping of persistent stores) - ExecutorService executorService = Executors.newFixedThreadPool(this.parallelRestoreThreadPoolSize, - new ThreadFactoryBuilder().setDaemon(true).setNameFormat(RESTORE_THREAD_NAME).build()); + // Start each store consumer once. + // Note: These consumers are per system and only changelog system store consumers will be started. + // Some TaskRestoreManagers may not require the consumer to to be started, but due to the agnostic nature of + // ContainerStorageManager we always start the changelog consumer here in case it is required + this.storeConsumers.values().stream().distinct().forEach(SystemConsumer::start); - List taskRestoreFutures = new ArrayList<>(this.taskRestoreManagers.entrySet().size()); + List taskRestoreFutures = new ArrayList<>(); // Submit restore callable for each taskInstance - this.taskRestoreManagers.forEach((taskInstance, taskRestoreManager) -> { - taskRestoreFutures.add(executorService.submit( - new TaskRestoreCallable(this.samzaContainerMetrics, taskInstance, taskRestoreManager))); - }); + taskRestoreManagers.forEach((taskInstance, restoreManagersMap) -> + // Submit for each restore factory + restoreManagersMap.forEach((factoryName, taskRestoreManager) -> taskRestoreFutures.add(restoreExecutor.submit( + new TaskRestoreCallable(this.samzaContainerMetrics, taskInstance, taskRestoreManager))))); - // loop-over the future list to wait for each thread to finish, catch any exceptions during restore and throw + // Loop-over the future list to wait for each thread to finish, catch any exceptions during restore and throw // as samza exceptions for (Future future : taskRestoreFutures) { try { @@ -690,7 +769,7 @@ private void restoreStores(Map checkpointedChange } catch (InterruptedException e) { LOG.warn("Received an interrupt during store restoration. Issuing interrupts to the store restoration workers to exit " + "prematurely without restoring full state."); - executorService.shutdownNow(); + restoreExecutor.shutdownNow(); throw e; } catch (Exception e) { LOG.error("Exception when restoring ", e); @@ -698,14 +777,26 @@ private void restoreStores(Map checkpointedChange } } - executorService.shutdown(); - // Stop each store consumer once this.storeConsumers.values().stream().distinct().forEach(SystemConsumer::stop); - // Now re-create persistent stores in read-write mode, leave non-persistent stores as-is - recreatePersistentTaskStoresInReadWriteMode(this.containerModel, jobContext, containerContext, + // Now create persistent non side input stores in read-write mode, leave non-persistent stores as-is + this.taskStores = createTaskStores(nonSideInputStoreNames, this.containerModel, jobContext, containerContext, storageEngineFactories, serdes, taskInstanceMetrics, taskInstanceCollectors); + // Add in memory stores + this.inMemoryStores.forEach((taskName, stores) -> { + if (!this.taskStores.containsKey(taskName)) { + taskStores.put(taskName, new HashMap<>()); + } + taskStores.get(taskName).putAll(stores); + }); + // Add side input stores + this.sideInputStores.forEach((taskName, stores) -> { + if (!this.taskStores.containsKey(taskName)) { + taskStores.put(taskName, new HashMap<>()); + } + taskStores.get(taskName).putAll(stores); + }); LOG.info("Store Restore complete"); } @@ -838,15 +929,24 @@ private boolean awaitSideInputTasks() throws InterruptedException { * @return the task store. */ public Optional getStore(TaskName taskName, String storeName) { + if (!isStarted) { + throw new SamzaException(String.format( + "Attempting to access store %s for task %s before ContainerStorageManager is started.", + storeName, taskName)); + } return Optional.ofNullable(this.taskStores.get(taskName).get(storeName)); } /** - * Get all {@link StorageEngine} instance used by a given task. - * @param taskName the task name, all stores for which are desired. + * Get all {@link StorageEngine} instance used by a given task. + * @param taskName the task name, all stores for which are desired. * @return map of stores used by the given task, indexed by storename */ public Map getAllStores(TaskName taskName) { + if (!isStarted) { + throw new SamzaException(String.format( + "Attempting to access stores for task %s before ContainerStorageManager is started.", taskName)); + } return this.taskStores.get(taskName); } @@ -865,9 +965,13 @@ public void stopStores() { public void shutdown() { // stop all nonsideinputstores including persistent and non-persistent stores - this.containerModel.getTasks().forEach((taskName, taskModel) -> - getNonSideInputStores(taskName).forEach((storeName, store) -> store.stop()) - ); + if (taskStores != null) { + this.containerModel.getTasks() + .forEach((taskName, taskModel) -> taskStores.get(taskName) + .entrySet().stream() + .filter(e -> !sideInputStoreNames.contains(e.getKey())) + .forEach(e -> e.getValue().stop())); + } this.shouldShutdown = true; @@ -926,7 +1030,7 @@ public Void call() { } finally { // Stop all persistent stores after restoring. Certain persistent stores opened in BulkLoad mode are compacted // on stop, so paralleling stop() also parallelizes their compaction (a time-intensive operation). - taskRestoreManager.stopPersistentStores(); + taskRestoreManager.close(); long timeToRestore = System.currentTimeMillis() - startTime; if (this.samzaContainerMetrics != null) { diff --git a/samza-core/src/main/scala/org/apache/samza/storage/KafkaNonTransactionalStateTaskBackupManager.scala b/samza-core/src/main/scala/org/apache/samza/storage/KafkaNonTransactionalStateTaskBackupManager.scala new file mode 100644 index 0000000000..633191bd89 --- /dev/null +++ b/samza-core/src/main/scala/org/apache/samza/storage/KafkaNonTransactionalStateTaskBackupManager.scala @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage + +import java.util +import java.util.concurrent.CompletableFuture + +import com.google.common.collect.ImmutableSet +import org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker +import org.apache.samza.checkpoint.{Checkpoint, CheckpointId} +import org.apache.samza.container.TaskName +import org.apache.samza.system._ +import org.apache.samza.util.Logging +import org.apache.samza.{Partition, SamzaException} + +import scala.collection.JavaConverters._ + +/** + * Manage all the storage engines for a given task + */ +class KafkaNonTransactionalStateTaskBackupManager( + taskName: TaskName, + storeChangelogs: util.Map[String, SystemStream] = new util.HashMap[String, SystemStream](), + systemAdmins: SystemAdmins, + partition: Partition) extends Logging with TaskBackupManager { + + override def init(checkpoint: Checkpoint): Unit = {} + + override def snapshot(checkpointId: CheckpointId): util.Map[String, String] = { + debug("Getting newest offsets for kafka changelog SSPs.") + getNewestChangelogSSPOffsets() + } + + override def upload(checkpointId: CheckpointId, + stateCheckpointMarkers: util.Map[String, String]): CompletableFuture[util.Map[String, String]] = { + CompletableFuture.completedFuture(stateCheckpointMarkers) + } + + override def cleanUp(checkpointId: CheckpointId, + stateCheckpointMarker: util.Map[String, String]): CompletableFuture[Void] = { + CompletableFuture.completedFuture(null) + } + + override def close() {} + + /** + * Returns the newest offset for each store changelog SSP for this task. + * @return A map of changelog SSPs for this task to their newest offset (or None if ssp is empty) + * @throws SamzaException if there was an error fetching newest offset for any SSP + */ + private def getNewestChangelogSSPOffsets(): util.Map[String, String] = { + storeChangelogs.asScala + .map { case (storeName, systemStream) => { + debug("Fetching newest offset for taskName %s store %s changelog %s" format (taskName, storeName, systemStream)) + val ssp = new SystemStreamPartition(systemStream.getSystem, systemStream.getStream, partition) + val systemAdmin = systemAdmins.getSystemAdmin(systemStream.getSystem) + + try { + val sspMetadataOption = Option(systemAdmin.getSSPMetadata(ImmutableSet.of(ssp)).get(ssp)) + + // newest offset == null implies topic is empty + val newestOffsetOption = sspMetadataOption.flatMap(sspMetadata => Option(sspMetadata.getNewestOffset)) + newestOffsetOption.foreach(newestOffset => + debug("Got newest offset %s for taskName %s store %s changelog %s" format(newestOffset, taskName, storeName, systemStream))) + + (storeName, KafkaStateCheckpointMarker.serialize(new KafkaStateCheckpointMarker(ssp, newestOffsetOption.orNull))) + } catch { + case e: Exception => + throw new SamzaException("Error getting newest changelog offset for taskName %s store %s changelog %s." + format(taskName, storeName, systemStream), e) + } + }}.asJava + } +} diff --git a/samza-core/src/main/scala/org/apache/samza/storage/KafkaTransactionalStateTaskBackupManager.scala b/samza-core/src/main/scala/org/apache/samza/storage/KafkaTransactionalStateTaskBackupManager.scala new file mode 100644 index 0000000000..dc28fe77fa --- /dev/null +++ b/samza-core/src/main/scala/org/apache/samza/storage/KafkaTransactionalStateTaskBackupManager.scala @@ -0,0 +1,97 @@ +package org.apache.samza.storage + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util +import java.util.concurrent.CompletableFuture + +import com.google.common.annotations.VisibleForTesting +import com.google.common.collect.ImmutableSet +import org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker +import org.apache.samza.checkpoint.{Checkpoint, CheckpointId} +import org.apache.samza.container.TaskName +import org.apache.samza.system._ +import org.apache.samza.util.Logging +import org.apache.samza.{Partition, SamzaException} + +import scala.collection.JavaConverters._ + +/** + * Manage all the storage engines for a given task + */ +class KafkaTransactionalStateTaskBackupManager( + taskName: TaskName, + storeChangelogs: util.Map[String, SystemStream] = new util.HashMap[String, SystemStream](), + systemAdmins: SystemAdmins, + partition: Partition) extends Logging with TaskBackupManager { + + override def init(checkpoint: Checkpoint): Unit = {} + + override def snapshot(checkpointId: CheckpointId): util.Map[String, String] = { + debug("Getting newest offsets for kafka changelog SSPs.") + getNewestChangelogSSPOffsets(taskName, storeChangelogs, partition, systemAdmins) + } + + override def upload(checkpointId: CheckpointId, snapshotCheckpointsMap: util.Map[String, String]): + CompletableFuture[util.Map[String, String]] = { + CompletableFuture.completedFuture(snapshotCheckpointsMap) + } + + override def cleanUp(checkpointId: CheckpointId, + stateCheckpointMarker: util.Map[String, String]): CompletableFuture[Void] = { + CompletableFuture.completedFuture(null) + } + + override def close() {} + + /** + * Returns the newest offset for each store changelog SSP for this task. Returned map will + * always contain an entry for every changelog SSP. + * @return A map of storenames for this task to their ssp and newest offset (null if empty) wrapped in KafkaStateCheckpointMarker + * @throws SamzaException if there was an error fetching newest offset for any SSP + */ + @VisibleForTesting + def getNewestChangelogSSPOffsets(taskName: TaskName, storeChangelogs: util.Map[String, SystemStream], + partition: Partition, systemAdmins: SystemAdmins): util.Map[String, String] = { + storeChangelogs.asScala + .map { case (storeName, systemStream) => { + try { + debug("Fetching newest offset for taskName %s store %s changelog %s" format (taskName, storeName, systemStream)) + val ssp = new SystemStreamPartition(systemStream.getSystem, systemStream.getStream, partition) + val systemAdmin = systemAdmins.getSystemAdmin(systemStream.getSystem) + + val sspMetadata = Option(systemAdmin.getSSPMetadata(ImmutableSet.of(ssp)).get(ssp)) + .getOrElse(throw new SamzaException("Received null metadata for ssp: %s" format ssp)) + + // newest offset == null implies topic is empty + val newestOffsetOption = Option(sspMetadata.getNewestOffset) + newestOffsetOption.foreach(newestOffset => + debug("Got newest offset %s for taskName %s store %s changelog %s" format(newestOffset, taskName, storeName, systemStream))) + + (storeName, KafkaStateCheckpointMarker.serialize(new KafkaStateCheckpointMarker(ssp, newestOffsetOption.orNull))) + } catch { + case e: Exception => + throw new SamzaException("Error getting newest changelog offset for taskName %s store %s changelog %s." + format(taskName, storeName, systemStream), e) + } + }} + .toMap.asJava + } +} diff --git a/samza-core/src/main/scala/org/apache/samza/storage/NonTransactionalStateTaskStorageManager.scala b/samza-core/src/main/scala/org/apache/samza/storage/NonTransactionalStateTaskStorageManager.scala deleted file mode 100644 index 7b3874907e..0000000000 --- a/samza-core/src/main/scala/org/apache/samza/storage/NonTransactionalStateTaskStorageManager.scala +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.samza.storage - -import java.io._ - -import com.google.common.annotations.VisibleForTesting -import com.google.common.collect.ImmutableSet -import org.apache.samza.checkpoint.CheckpointId -import org.apache.samza.container.TaskName -import org.apache.samza.job.model.TaskMode -import org.apache.samza.system._ -import org.apache.samza.util.Logging -import org.apache.samza.util.ScalaJavaUtil.JavaOptionals -import org.apache.samza.{Partition, SamzaException} - -import scala.collection.JavaConverters._ - -/** - * Manage all the storage engines for a given task - */ -class NonTransactionalStateTaskStorageManager( - taskName: TaskName, - containerStorageManager: ContainerStorageManager, - storeChangelogs: Map[String, SystemStream] = Map(), - systemAdmins: SystemAdmins, - loggedStoreBaseDir: File = new File(System.getProperty("user.dir"), "state"), - partition: Partition) extends Logging with TaskStorageManager { - - private val storageManagerUtil = new StorageManagerUtil - private val persistedStores = containerStorageManager.getAllStores(taskName).asScala - .filter { case (storeName, storageEngine) => storageEngine.getStoreProperties.isPersistedToDisk } - - def getStore(storeName: String): Option[StorageEngine] = JavaOptionals.toRichOptional(containerStorageManager.getStore(taskName, storeName)).toOption - - def flush(): Map[SystemStreamPartition, Option[String]] = { - debug("Flushing stores.") - containerStorageManager.getAllStores(taskName).asScala.values.foreach(_.flush) - val newestChangelogSSPOffsets = getNewestChangelogSSPOffsets() - writeChangelogOffsetFiles(newestChangelogSSPOffsets) - newestChangelogSSPOffsets - } - - override def checkpoint(checkpointId: CheckpointId, - newestChangelogOffsets: Map[SystemStreamPartition, Option[String]]): Unit = {} - - override def removeOldCheckpoints(checkpointId: CheckpointId): Unit = {} - - @VisibleForTesting - def stop() { - debug("Stopping stores.") - containerStorageManager.stopStores() - } - - /** - * Returns the newest offset for each store changelog SSP for this task. - * @return A map of changelog SSPs for this task to their newest offset (or None if ssp is empty) - * @throws SamzaException if there was an error fetching newest offset for any SSP - */ - private def getNewestChangelogSSPOffsets(): Map[SystemStreamPartition, Option[String]] = { - storeChangelogs - .map { case (storeName, systemStream) => { - debug("Fetching newest offset for taskName %s store %s changelog %s" format (taskName, storeName, systemStream)) - val ssp = new SystemStreamPartition(systemStream.getSystem, systemStream.getStream, partition) - val systemAdmin = systemAdmins.getSystemAdmin(systemStream.getSystem) - - try { - val sspMetadataOption = Option(systemAdmin.getSSPMetadata(ImmutableSet.of(ssp)).get(ssp)) - - // newest offset == null implies topic is empty - val newestOffsetOption = sspMetadataOption.flatMap(sspMetadata => Option(sspMetadata.getNewestOffset)) - newestOffsetOption.foreach(newestOffset => - debug("Got newest offset %s for taskName %s store %s changelog %s" format(newestOffset, taskName, storeName, systemStream))) - - (ssp, newestOffsetOption) - } catch { - case e: Exception => - throw new SamzaException("Error getting newest changelog offset for taskName %s store %s changelog %s." - format(taskName, storeName, systemStream), e) - } - }} - } - - /** - * Writes the newest changelog ssp offset for each persistent store to the OFFSET file on disk. - * These files are used during container startup to determine whether there is any new information in the - * changelog that is not reflected in the on-disk copy of the store. If there is any delta, it is replayed - * from the changelog e.g. This can happen if the job was run on this host, then another - * host and back to this host. - */ - private def writeChangelogOffsetFiles(newestChangelogOffsets: Map[SystemStreamPartition, Option[String]]) { - debug("Writing OFFSET files for logged persistent key value stores for task %s." format(taskName)) - - storeChangelogs - .filterKeys(storeName => persistedStores.contains(storeName)) - .foreach { case (storeName, systemStream) => { - debug("Writing changelog offset for taskName %s store %s changelog %s." format(taskName, storeName, systemStream)) - val currentStoreDir = storageManagerUtil.getTaskStoreDir(loggedStoreBaseDir, storeName, taskName, TaskMode.Active) - try { - val ssp = new SystemStreamPartition(systemStream.getSystem, systemStream.getStream, partition) - newestChangelogOffsets(ssp) match { - case Some(newestOffset) => { - debug("Storing newest offset %s for taskName %s store %s changelog %s in OFFSET file." - format(newestOffset, taskName, storeName, systemStream)) - // TaskStorageManagers are only created for active tasks - storageManagerUtil.writeOffsetFile(currentStoreDir, Map(ssp -> newestOffset).asJava, false) - debug("Successfully stored offset %s for taskName %s store %s changelog %s in OFFSET file." - format(newestOffset, taskName, storeName, systemStream)) - } - case None => { - // if newestOffset is null, then it means the changelog ssp is (or has become) empty. This could be - // either because the changelog topic was newly added, repartitioned, or manually deleted and recreated. - // No need to persist the offset file. - storageManagerUtil.deleteOffsetFile(currentStoreDir) - debug("Deleting OFFSET file for taskName %s store %s changelog ssp %s since the newestOffset is null." - format (taskName, storeName, ssp)) - } - } - } catch { - case e: Exception => - throw new SamzaException("Error storing offset for taskName %s store %s changelog %s." - format(taskName, storeName, systemStream), e) - } - }} - debug("Done writing OFFSET files for logged persistent key value stores for task %s" format(taskName)) - } -} diff --git a/samza-core/src/main/scala/org/apache/samza/storage/TaskStorageManagerFactory.java b/samza-core/src/main/scala/org/apache/samza/storage/TaskStorageManagerFactory.java deleted file mode 100644 index 97e45043b5..0000000000 --- a/samza-core/src/main/scala/org/apache/samza/storage/TaskStorageManagerFactory.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.samza.storage; - -import scala.collection.immutable.Map; - -import java.io.File; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.config.TaskConfig; -import org.apache.samza.container.TaskName; -import org.apache.samza.job.model.TaskMode; -import org.apache.samza.system.SystemAdmins; -import org.apache.samza.system.SystemStream; - -public class TaskStorageManagerFactory { - public static TaskStorageManager create(TaskName taskName, ContainerStorageManager containerStorageManager, - Map storeChangelogs, SystemAdmins systemAdmins, - File loggedStoreBaseDir, Partition changelogPartition, - Config config, TaskMode taskMode) { - if (new TaskConfig(config).getTransactionalStateCheckpointEnabled()) { - return new TransactionalStateTaskStorageManager(taskName, containerStorageManager, storeChangelogs, systemAdmins, - loggedStoreBaseDir, changelogPartition, taskMode, new StorageManagerUtil()); - } else { - return new NonTransactionalStateTaskStorageManager(taskName, containerStorageManager, storeChangelogs, systemAdmins, - loggedStoreBaseDir, changelogPartition); - } - } -} diff --git a/samza-core/src/main/scala/org/apache/samza/storage/TransactionalStateTaskStorageManager.scala b/samza-core/src/main/scala/org/apache/samza/storage/TransactionalStateTaskStorageManager.scala deleted file mode 100644 index 0335710a35..0000000000 --- a/samza-core/src/main/scala/org/apache/samza/storage/TransactionalStateTaskStorageManager.scala +++ /dev/null @@ -1,201 +0,0 @@ -package org.apache.samza.storage - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.io._ -import java.nio.file.Path - -import com.google.common.annotations.VisibleForTesting -import com.google.common.collect.ImmutableSet -import org.apache.commons.io.FileUtils -import org.apache.commons.io.filefilter.WildcardFileFilter -import org.apache.samza.checkpoint.CheckpointId -import org.apache.samza.{Partition, SamzaException} -import org.apache.samza.container.TaskName -import org.apache.samza.job.model.TaskMode -import org.apache.samza.system._ -import org.apache.samza.util.ScalaJavaUtil.JavaOptionals -import org.apache.samza.util.Logging - -import scala.collection.JavaConverters._ - -/** - * Manage all the storage engines for a given task - */ -class TransactionalStateTaskStorageManager( - taskName: TaskName, - containerStorageManager: ContainerStorageManager, - storeChangelogs: Map[String, SystemStream] = Map(), - systemAdmins: SystemAdmins, - loggedStoreBaseDir: File = new File(System.getProperty("user.dir"), "state"), - partition: Partition, - taskMode: TaskMode, - storageManagerUtil: StorageManagerUtil) extends Logging with TaskStorageManager { - - def getStore(storeName: String): Option[StorageEngine] = JavaOptionals.toRichOptional(containerStorageManager.getStore(taskName, storeName)).toOption - - def flush(): Map[SystemStreamPartition, Option[String]] = { - debug("Flushing stores.") - containerStorageManager.getAllStores(taskName).asScala.values.foreach(_.flush) - getNewestChangelogSSPOffsets(taskName, storeChangelogs, partition, systemAdmins) - } - - def checkpoint(checkpointId: CheckpointId, newestChangelogOffsets: Map[SystemStreamPartition, Option[String]]): Unit = { - debug("Checkpointing stores.") - - val checkpointPaths = containerStorageManager.getAllStores(taskName).asScala - .filter { case (storeName, storeEngine) => - storeEngine.getStoreProperties.isLoggedStore && storeEngine.getStoreProperties.isPersistedToDisk} - .flatMap { case (storeName, storeEngine) => { - val pathOptional = storeEngine.checkpoint(checkpointId) - if (pathOptional.isPresent) { - Some(storeName, pathOptional.get()) - } else { - None - } - }} - .toMap - - writeChangelogOffsetFiles(checkpointPaths, storeChangelogs, newestChangelogOffsets) - } - - def removeOldCheckpoints(latestCheckpointId: CheckpointId): Unit = { - if (latestCheckpointId != null) { - debug("Removing older checkpoints before " + latestCheckpointId) - - val files = loggedStoreBaseDir.listFiles() - if (files != null) { - files - .foreach(storeDir => { - val storeName = storeDir.getName - val taskStoreName = storageManagerUtil.getTaskStoreDir( - loggedStoreBaseDir, storeName, taskName, taskMode).getName - val fileFilter: FileFilter = new WildcardFileFilter(taskStoreName + "-*") - val checkpointDirs = storeDir.listFiles(fileFilter) - - if (checkpointDirs != null) { - checkpointDirs - .filter(!_.getName.contains(latestCheckpointId.toString)) - .foreach(checkpointDir => { - FileUtils.deleteDirectory(checkpointDir) - }) - } - }) - } - } - } - - @VisibleForTesting - def stop() { - debug("Stopping stores.") - containerStorageManager.stopStores() - } - - /** - * Returns the newest offset for each store changelog SSP for this task. Returned map will - * always contain an entry for every changelog SSP. - * @return A map of changelog SSPs for this task to their newest offset (or None if ssp is empty) - * @throws SamzaException if there was an error fetching newest offset for any SSP - */ - @VisibleForTesting - def getNewestChangelogSSPOffsets(taskName: TaskName, storeChangelogs: Map[String, SystemStream], - partition: Partition, systemAdmins: SystemAdmins): Map[SystemStreamPartition, Option[String]] = { - storeChangelogs - .map { case (storeName, systemStream) => { - try { - debug("Fetching newest offset for taskName %s store %s changelog %s" format (taskName, storeName, systemStream)) - val ssp = new SystemStreamPartition(systemStream.getSystem, systemStream.getStream, partition) - val systemAdmin = systemAdmins.getSystemAdmin(systemStream.getSystem) - - val sspMetadata = Option(systemAdmin.getSSPMetadata(ImmutableSet.of(ssp)).get(ssp)) - .getOrElse(throw new SamzaException("Received null metadata for ssp: %s" format ssp)) - - // newest offset == null implies topic is empty - val newestOffsetOption = Option(sspMetadata.getNewestOffset) - newestOffsetOption.foreach(newestOffset => - debug("Got newest offset %s for taskName %s store %s changelog %s" format(newestOffset, taskName, storeName, systemStream))) - - (ssp, newestOffsetOption) - } catch { - case e: Exception => - throw new SamzaException("Error getting newest changelog offset for taskName %s store %s changelog %s." - format(taskName, storeName, systemStream), e) - } - }} - .toMap - } - - /** - * Writes the newest changelog ssp offset for each persistent store the OFFSET file in both the checkpoint - * and the current store directory (the latter for allowing rollbacks). - * - * These files are used during container startup to ensure transactional state, and to determine whether the - * there is any new information in the changelog that is not reflected in the on-disk copy of the store. - * If there is any delta, it is replayed from the changelog e.g. This can happen if the job was run on this host, - * then another host, and then back to this host. - */ - @VisibleForTesting - def writeChangelogOffsetFiles(checkpointPaths: Map[String, Path], storeChangelogs: Map[String, SystemStream], - newestChangelogOffsets: Map[SystemStreamPartition, Option[String]]): Unit = { - debug("Writing OFFSET files for logged persistent key value stores for task %s." format(checkpointPaths)) - - storeChangelogs - .filterKeys(storeName => checkpointPaths.contains(storeName)) - .foreach { case (storeName, systemStream) => { - try { - val ssp = new SystemStreamPartition(systemStream.getSystem, systemStream.getStream, partition) - val currentStoreDir = storageManagerUtil.getTaskStoreDir(loggedStoreBaseDir, storeName, taskName, TaskMode.Active) - newestChangelogOffsets(ssp) match { - case Some(newestOffset) => { - // write the offset file for the checkpoint directory - val checkpointPath = checkpointPaths(storeName) - writeChangelogOffsetFile(storeName, ssp, newestOffset, checkpointPath.toFile) - // write the OFFSET file for the current store (for backwards compatibility / allowing rollbacks) - writeChangelogOffsetFile(storeName, ssp, newestOffset, currentStoreDir) - } - case None => { - // retain existing behavior for current store directory for backwards compatibility / allowing rollbacks - - // if newestOffset is null, then it means the changelog ssp is (or has become) empty. This could be - // either because the changelog topic was newly added, repartitioned, or manually deleted and recreated. - // No need to persist the offset file. - storageManagerUtil.deleteOffsetFile(currentStoreDir) - debug("Deleting OFFSET file for taskName %s current store %s changelog ssp %s since the newestOffset is null." - format (taskName, storeName, ssp)) - } - } - } catch { - case e: Exception => - throw new SamzaException("Error storing offset for taskName %s store %s changelog %s." - format(taskName, storeName, systemStream), e) - } - }} - debug("Done writing OFFSET files for logged persistent key value stores for task %s" format(taskName)) - } - - private def writeChangelogOffsetFile(storeName: String, ssp: SystemStreamPartition, - newestOffset: String, dir: File): Unit = { - debug("Storing newest offset: %s for taskName: %s store: %s changelog: %s in OFFSET file at path: %s." - format(newestOffset, taskName, storeName, ssp, dir)) - storageManagerUtil.writeOffsetFile(dir, Map(ssp -> newestOffset).asJava, false) - debug("Successfully stored offset: %s for taskName: %s store: %s changelog: %s in OFFSET file at path: %s." - format(newestOffset, taskName, storeName, ssp, dir)) - } -} diff --git a/samza-core/src/main/scala/org/apache/samza/util/FileUtil.scala b/samza-core/src/main/scala/org/apache/samza/util/FileUtil.scala index d416340686..029868722b 100644 --- a/samza-core/src/main/scala/org/apache/samza/util/FileUtil.scala +++ b/samza-core/src/main/scala/org/apache/samza/util/FileUtil.scala @@ -146,7 +146,14 @@ class FileUtil extends Logging { } def createDirectories(path: Path): Path = { - Files.createDirectories(path) + // Files.createDirectories throws FileAlreadyExistsException if the path already exists + // but the last dir in the path is a symlink to another dir. Check explicitly if the path + // already exists to avoid this behavior. + if (!Files.exists(path)) { + Files.createDirectories(path) + } else { + path + } } /** diff --git a/samza-core/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaChangelogSSPOffset.java b/samza-core/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaChangelogSSPOffset.java new file mode 100644 index 0000000000..324a349daf --- /dev/null +++ b/samza-core/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaChangelogSSPOffset.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.checkpoint.kafka; + +import org.apache.samza.checkpoint.CheckpointId; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + + +public class TestKafkaChangelogSSPOffset { + @Test + public void testSerializeDeserialize() { + KafkaChangelogSSPOffset kafkaChangelogSSPOffset = new KafkaChangelogSSPOffset(CheckpointId.create(), "offset"); + KafkaChangelogSSPOffset deserializedKafkaChangelogSSPOffset = KafkaChangelogSSPOffset.fromString(kafkaChangelogSSPOffset.toString()); + + assertEquals(kafkaChangelogSSPOffset.getCheckpointId(), deserializedKafkaChangelogSSPOffset.getCheckpointId()); + assertEquals("offset", deserializedKafkaChangelogSSPOffset.getChangelogOffset()); + assertEquals(kafkaChangelogSSPOffset, deserializedKafkaChangelogSSPOffset); + } + + @Test + public void testSerializeDeserializeNullOffsets() { + KafkaChangelogSSPOffset kafkaChangelogSSPOffset = new KafkaChangelogSSPOffset(CheckpointId.create(), null); + KafkaChangelogSSPOffset deserializedKafkaChangelogSSPOffset = KafkaChangelogSSPOffset.fromString(kafkaChangelogSSPOffset.toString()); + + assertEquals(kafkaChangelogSSPOffset.getCheckpointId(), deserializedKafkaChangelogSSPOffset.getCheckpointId()); + assertNull(deserializedKafkaChangelogSSPOffset.getChangelogOffset()); + assertEquals(kafkaChangelogSSPOffset, deserializedKafkaChangelogSSPOffset); + } + + @Test + public void testSerializationFormatForBackwardsCompatibility() { + KafkaChangelogSSPOffset kafkaChangelogSSPOffset = new KafkaChangelogSSPOffset(CheckpointId.create(), "offset"); + + // WARNING: This format is written to persisted remotes stores and local files, making a change in the format + // would be backwards incompatible + String expectedSerializationFormat = kafkaChangelogSSPOffset.getCheckpointId() + KafkaChangelogSSPOffset.SEPARATOR + + kafkaChangelogSSPOffset.getChangelogOffset(); + assertEquals(expectedSerializationFormat, kafkaChangelogSSPOffset.toString()); + assertEquals(kafkaChangelogSSPOffset, KafkaChangelogSSPOffset.fromString(expectedSerializationFormat)); + } + + @Test + public void testNullSerializationFormatForBackwardsCompatibility() { + KafkaChangelogSSPOffset kafkaChangelogSSPOffset = new KafkaChangelogSSPOffset(CheckpointId.create(), null); + + // WARNING: This format is written to persisted remotes stores and local files, making a change in the format + // would be backwards incompatible + String expectedSerializationFormat = kafkaChangelogSSPOffset.getCheckpointId() + KafkaChangelogSSPOffset.SEPARATOR + + "null"; + assertEquals(expectedSerializationFormat, kafkaChangelogSSPOffset.toString()); + assertEquals(kafkaChangelogSSPOffset, KafkaChangelogSSPOffset.fromString(expectedSerializationFormat)); + } +} diff --git a/samza-core/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaStateCheckpointMarker.java b/samza-core/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaStateCheckpointMarker.java new file mode 100644 index 0000000000..75eacff252 --- /dev/null +++ b/samza-core/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaStateCheckpointMarker.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.checkpoint.kafka; + +import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import java.util.Map; +import org.apache.samza.Partition; +import org.apache.samza.system.SystemStreamPartition; +import org.junit.Test; +import scala.Option; + +import static org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + + +public class TestKafkaStateCheckpointMarker { + @Test + public void testSerializeDeserialize() { + SystemStreamPartition ssp = new SystemStreamPartition("system", "stream", new Partition(1)); + KafkaStateCheckpointMarker marker = new KafkaStateCheckpointMarker(ssp, "offset"); + KafkaStateCheckpointMarker deserializedMarker = KafkaStateCheckpointMarker + .deserialize(KafkaStateCheckpointMarker.serialize(marker)); + + assertEquals(MARKER_VERSION, deserializedMarker.getVersion()); + assertEquals(marker.getChangelogOffset(), deserializedMarker.getChangelogOffset()); + assertEquals(marker.getChangelogSSP(), deserializedMarker.getChangelogSSP()); + assertEquals(marker, deserializedMarker); + } + + @Test + public void testSerializeDeserializeNullOffsets() { + SystemStreamPartition ssp = new SystemStreamPartition("system", "stream", new Partition(1)); + KafkaStateCheckpointMarker marker = new KafkaStateCheckpointMarker(ssp, null); + KafkaStateCheckpointMarker deserializedMarker = KafkaStateCheckpointMarker + .deserialize(KafkaStateCheckpointMarker.serialize(marker)); + + assertEquals(MARKER_VERSION, deserializedMarker.getVersion()); + assertNull(deserializedMarker.getChangelogOffset()); + assertEquals(marker.getChangelogSSP(), deserializedMarker.getChangelogSSP()); + assertEquals(marker, deserializedMarker); + } + + @Test + public void testStateCheckpointMarkerToSSPOffsetMap() { + SystemStreamPartition ssp1 = new SystemStreamPartition("system1", "stream1", new Partition(1)); + KafkaStateCheckpointMarker marker1 = new KafkaStateCheckpointMarker(ssp1, "offset1"); + SystemStreamPartition ssp2 = new SystemStreamPartition("system2", "stream2", new Partition(2)); + KafkaStateCheckpointMarker marker2 = new KafkaStateCheckpointMarker(ssp2, null); + Map storesToKSCM = ImmutableMap.of( + "store1", KafkaStateCheckpointMarker.serialize(marker1), + "store2", KafkaStateCheckpointMarker.serialize(marker2) + ); + Map> factoryToSCMs = ImmutableMap.of( + KAFKA_STATE_BACKEND_FACTORY_NAME, storesToKSCM, + "factory2", Collections.EMPTY_MAP // factory2 should be ignored + ); + + Map> sspToOffsetOption = KafkaStateCheckpointMarker + .scmsToSSPOffsetMap(factoryToSCMs); + + assertEquals(2, sspToOffsetOption.size()); + assertTrue(sspToOffsetOption.containsKey(ssp1)); + assertEquals(sspToOffsetOption.get(ssp1).get(), marker1.getChangelogOffset()); + assertEquals(ssp1, marker1.getChangelogSSP()); + assertTrue(sspToOffsetOption.containsKey(ssp2)); + assertTrue(sspToOffsetOption.get(ssp2).isEmpty()); + } + + @Test + public void testStateCheckpointMarkerToSSPOffsetMapNoFactoryFound() { + Map> factoryToSCMs = ImmutableMap.of( + "factory1", Collections.EMPTY_MAP, // factory1 should be ignored + "factory2", Collections.EMPTY_MAP // factory2 should be ignored + ); + + Map> sspToOffsetOption = KafkaStateCheckpointMarker + .scmsToSSPOffsetMap(factoryToSCMs); + + assertEquals(0, sspToOffsetOption.size()); + } + + @Test(expected = IllegalArgumentException.class) + public void testStateCheckpointMarkerToSSPOffsetMapDeserializationError() { + Map storesToSCM = ImmutableMap.of( + "store1", "blobId-1234" + ); + Map> factoryToSCMs = ImmutableMap.of( + "factory2", Collections.EMPTY_MAP, // factory2 should be ignored + KAFKA_STATE_BACKEND_FACTORY_NAME, storesToSCM + ); + + KafkaStateCheckpointMarker.scmsToSSPOffsetMap(factoryToSCMs); + } +} diff --git a/samza-core/src/test/java/org/apache/samza/config/TestStorageConfig.java b/samza-core/src/test/java/org/apache/samza/config/TestStorageConfig.java index ec9b2638dd..0d06c3b28a 100644 --- a/samza-core/src/test/java/org/apache/samza/config/TestStorageConfig.java +++ b/samza-core/src/test/java/org/apache/samza/config/TestStorageConfig.java @@ -30,6 +30,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.samza.SamzaException; +import org.apache.samza.system.SystemStream; import org.junit.Test; import static org.apache.samza.config.StorageConfig.*; @@ -41,6 +42,8 @@ public class TestStorageConfig { private static final String STORE_NAME0 = "store0"; private static final String STORE_NAME1 = "store1"; + private static final String STORE_NAME2 = "store2"; + private static final String STORE_NAME3 = "store3"; @Test public void testGetStoreNames() { @@ -68,6 +71,27 @@ public void testGetStoreNames() { assertEquals(expectedStoreNames, ImmutableSet.copyOf(actual)); } + /** + * Test verifies that the {@link StorageConfig#STORE_RESTORE_FACTORIES} which matches pattern for store.%s.factory + * is not picked up as in store names list + */ + @Test + public void testGetStoreNamesIgnoreStateRestoreFactory() { + // empty config, so no stores + assertEquals(Collections.emptyList(), new StorageConfig(new MapConfig()).getStoreNames()); + + Set expectedStoreNames = ImmutableSet.of(STORE_NAME0, STORE_NAME1); + // has stores + StorageConfig storageConfig = new StorageConfig(new MapConfig( + ImmutableMap.of(String.format(StorageConfig.FACTORY, STORE_NAME0), "store0.factory.class", + String.format(StorageConfig.FACTORY, STORE_NAME1), "store1.factory.class", STORE_RESTORE_FACTORIES, "org.apache.class"))); + + List actual = storageConfig.getStoreNames(); + // ordering shouldn't matter + assertEquals(2, actual.size()); + assertEquals(expectedStoreNames, ImmutableSet.copyOf(actual)); + } + @Test public void testGetChangelogStream() { // empty config, so no changelog stream @@ -77,57 +101,72 @@ public void testGetChangelogStream() { StorageConfig storageConfig = new StorageConfig( new MapConfig(ImmutableMap.of(String.format(StorageConfig.CHANGELOG_STREAM, STORE_NAME0), ""))); assertEquals(Optional.empty(), storageConfig.getChangelogStream(STORE_NAME0)); + assertEquals(Collections.emptyMap(), storageConfig.getStoreChangelogs()); // store has full changelog system-stream defined storageConfig = new StorageConfig(new MapConfig( ImmutableMap.of(String.format(StorageConfig.CHANGELOG_STREAM, STORE_NAME0), - "changelog-system.changelog-stream0"))); + "changelog-system.changelog-stream0", String.format(FACTORY, STORE_NAME0), "store0.factory.class"))); assertEquals(Optional.of("changelog-system.changelog-stream0"), storageConfig.getChangelogStream(STORE_NAME0)); + assertEquals(ImmutableMap.of(STORE_NAME0, new SystemStream("changelog-system", "changelog-stream0")), storageConfig.getStoreChangelogs()); // store has changelog stream defined, but system comes from job.changelog.system storageConfig = new StorageConfig(new MapConfig( ImmutableMap.of(String.format(StorageConfig.CHANGELOG_STREAM, STORE_NAME0), "changelog-stream0", + String.format(FACTORY, STORE_NAME0), "store0.factory.class", StorageConfig.CHANGELOG_SYSTEM, "changelog-system"))); assertEquals(Optional.of("changelog-system.changelog-stream0"), storageConfig.getChangelogStream(STORE_NAME0)); + assertEquals(ImmutableMap.of(STORE_NAME0, new SystemStream("changelog-system", "changelog-stream0")), storageConfig.getStoreChangelogs()); // batch mode: create unique stream name storageConfig = new StorageConfig(new MapConfig( ImmutableMap.of(String.format(StorageConfig.CHANGELOG_STREAM, STORE_NAME0), - "changelog-system.changelog-stream0", ApplicationConfig.APP_MODE, + "changelog-system.changelog-stream0", String.format(FACTORY, STORE_NAME0), "store0.factory.class", + ApplicationConfig.APP_MODE, ApplicationConfig.ApplicationMode.BATCH.name().toLowerCase(), ApplicationConfig.APP_RUN_ID, "run-id"))); assertEquals(Optional.of("changelog-system.changelog-stream0-run-id"), storageConfig.getChangelogStream(STORE_NAME0)); + assertEquals(ImmutableMap.of(STORE_NAME0, new SystemStream("changelog-system", "changelog-stream0-run-id")), storageConfig.getStoreChangelogs()); // job has no changelog stream defined storageConfig = new StorageConfig(new MapConfig( ImmutableMap.of(StorageConfig.CHANGELOG_SYSTEM, "changelog-system", JobConfig.JOB_DEFAULT_SYSTEM, - "should-not-be-used"))); + "should-not-be-used", String.format(FACTORY, STORE_NAME0), "store0.factory.class"))); assertEquals(Optional.empty(), storageConfig.getChangelogStream(STORE_NAME0)); + assertEquals(Collections.emptyMap(), storageConfig.getStoreChangelogs()); // job.changelog.system takes precedence over job.default.system when changelog is specified as just streamName storageConfig = new StorageConfig(new MapConfig( ImmutableMap.of(StorageConfig.CHANGELOG_SYSTEM, "changelog-system", JobConfig.JOB_DEFAULT_SYSTEM, - "should-not-be-used", String.format(CHANGELOG_STREAM, STORE_NAME0), "streamName"))); + "should-not-be-used", String.format(CHANGELOG_STREAM, STORE_NAME0), "streamName", + String.format(FACTORY, STORE_NAME0), "store0.factory.class"))); assertEquals("changelog-system.streamName", storageConfig.getChangelogStream(STORE_NAME0).get()); + assertEquals(ImmutableMap.of(STORE_NAME0, new SystemStream("changelog-system", "streamName")), storageConfig.getStoreChangelogs()); // job.changelog.system takes precedence over job.default.system when changelog is specified as {systemName}.{streamName} storageConfig = new StorageConfig(new MapConfig( ImmutableMap.of(StorageConfig.CHANGELOG_SYSTEM, "changelog-system", JobConfig.JOB_DEFAULT_SYSTEM, - "should-not-be-used", String.format(CHANGELOG_STREAM, STORE_NAME0), "changelog-system.streamName"))); + "should-not-be-used", String.format(CHANGELOG_STREAM, STORE_NAME0), "changelog-system.streamName", + String.format(FACTORY, STORE_NAME0), "store0.factory.class"))); assertEquals("changelog-system.streamName", storageConfig.getChangelogStream(STORE_NAME0).get()); + assertEquals(ImmutableMap.of(STORE_NAME0, new SystemStream("changelog-system", "streamName")), storageConfig.getStoreChangelogs()); // systemName specified using stores.{storeName}.changelog = {systemName}.{streamName} should take precedence even // when job.changelog.system and job.default.system are specified storageConfig = new StorageConfig(new MapConfig( ImmutableMap.of(StorageConfig.CHANGELOG_SYSTEM, "default-changelog-system", JobConfig.JOB_DEFAULT_SYSTEM, "default-system", - String.format(CHANGELOG_STREAM, STORE_NAME0), "nondefault-changelog-system.streamName"))); + String.format(CHANGELOG_STREAM, STORE_NAME0), "nondefault-changelog-system.streamName", + String.format(FACTORY, STORE_NAME0), "store0.factory.class"))); assertEquals("nondefault-changelog-system.streamName", storageConfig.getChangelogStream(STORE_NAME0).get()); + assertEquals(ImmutableMap.of(STORE_NAME0, new SystemStream("nondefault-changelog-system", "streamName")), storageConfig.getStoreChangelogs()); // fall back to job.default.system if job.changelog.system is not specified storageConfig = new StorageConfig(new MapConfig( - ImmutableMap.of(JobConfig.JOB_DEFAULT_SYSTEM, "default-system", String.format(CHANGELOG_STREAM, STORE_NAME0), "streamName"))); + ImmutableMap.of(JobConfig.JOB_DEFAULT_SYSTEM, "default-system", String.format(CHANGELOG_STREAM, STORE_NAME0), + "streamName", String.format(FACTORY, STORE_NAME0), "store0.factory.class"))); assertEquals("default-system.streamName", storageConfig.getChangelogStream(STORE_NAME0).get()); + assertEquals(ImmutableMap.of(STORE_NAME0, new SystemStream("default-system", "streamName")), storageConfig.getStoreChangelogs()); } @Test(expected = SamzaException.class) @@ -137,6 +176,68 @@ public void testGetChangelogStreamMissingSystem() { storageConfig.getChangelogStream(STORE_NAME0); } + @Test + public void testGetBackupManagerFactories() { + String factory1 = "factory1"; + String factory2 = "factory2"; + String factory3 = "factory3"; + StorageConfig storageConfig = new StorageConfig(new MapConfig( + ImmutableMap.of( + String.format(STORE_BACKUP_FACTORIES, STORE_NAME0), factory1 + "," + factory2, + String.format(STORE_BACKUP_FACTORIES, STORE_NAME1), factory1, + String.format(STORE_BACKUP_FACTORIES, STORE_NAME2), factory3, + // store_name3 should use DEFAULT_STATE_BACKEND_FACTORY due to changelog presence + String.format(CHANGELOG_STREAM, STORE_NAME3), "nondefault-changelog-system.streamName"), + ImmutableMap.of( + String.format(FACTORY, STORE_NAME0), "store0.factory.class", + String.format(FACTORY, STORE_NAME1), "store1.factory.class", + String.format(FACTORY, STORE_NAME2), "store2.factory.class", + String.format(FACTORY, STORE_NAME3), "store3.factory.class", + // this store should have no backend factory configured + String.format(FACTORY, "noFactoryStore"), "noFactory.factory.class" + ) + )); + Set factories = storageConfig.getBackupFactories(); + assertTrue(factories.contains(factory1)); + assertTrue(factories.contains(factory2)); + assertTrue(factories.contains(factory3)); + assertTrue(factories.contains(KAFKA_STATE_BACKEND_FACTORY)); + assertEquals(4, factories.size()); + assertEquals(ImmutableList.of(factory1, factory2), storageConfig.getStoreBackupFactories(STORE_NAME0)); + assertEquals(ImmutableList.of(factory1), storageConfig.getStoreBackupFactories(STORE_NAME1)); + assertEquals(ImmutableList.of(factory3), storageConfig.getStoreBackupFactories(STORE_NAME2)); + assertEquals(DEFAULT_BACKUP_FACTORIES, storageConfig.getStoreBackupFactories(STORE_NAME3)); + assertTrue(storageConfig.getStoreBackupFactories("emptyStore").isEmpty()); + assertTrue(storageConfig.getStoreBackupFactories("noFactoryStore").isEmpty()); + } + + @Test + public void testGetStoreToBackup() { + String targetFactory = "target.class"; + StorageConfig config = new StorageConfig(new MapConfig( + ImmutableMap.of( + String.format(StorageConfig.STORE_BACKUP_FACTORIES, STORE_NAME0), targetFactory, + String.format(StorageConfig.STORE_BACKUP_FACTORIES, STORE_NAME1), targetFactory + "," + + KAFKA_STATE_BACKEND_FACTORY, + String.format(StorageConfig.STORE_BACKUP_FACTORIES, STORE_NAME2), KAFKA_STATE_BACKEND_FACTORY), + ImmutableMap.of( + String.format(FACTORY, STORE_NAME0), "store0.factory.class", + String.format(FACTORY, STORE_NAME1), "store1.factory.class", + String.format(FACTORY, STORE_NAME2), "store2.factory.class", + String.format(FACTORY, STORE_NAME3), "store3.factory.class", + String.format(CHANGELOG_STREAM, STORE_NAME3), "nondefault-changelog-system.streamName" + ) + )); + + List targetStoreNames = config.getStoresWithBackupFactory(targetFactory); + List defaultStoreNames = config.getStoresWithBackupFactory( + KAFKA_STATE_BACKEND_FACTORY); + assertTrue(targetStoreNames.containsAll(ImmutableList.of(STORE_NAME0, STORE_NAME1))); + assertEquals(2, targetStoreNames.size()); + assertTrue(defaultStoreNames.containsAll(ImmutableList.of(STORE_NAME2, STORE_NAME1, STORE_NAME3))); + assertEquals(3, defaultStoreNames.size()); + } + @Test public void testGetAccessLogEnabled() { // empty config, access log disabled @@ -317,4 +418,148 @@ public void testGetChangelogMinCompactionLagMs() { configMap.put(String.format(CHANGELOG_MIN_COMPACTION_LAG_MS, STORE_NAME0), String.valueOf(storeSpecificLagOverride)); assertEquals(storeSpecificLagOverride, new StorageConfig(new MapConfig(configMap)).getChangelogMinCompactionLagMs(STORE_NAME0)); } + + @Test + public void testGetRestoreManagers() { + String storeName = "store1"; + String storeName2 = "store2"; + Map configMap = new HashMap<>(); + configMap.put(String.format(FACTORY, storeName), "store1.factory.class"); + configMap.put(String.format(FACTORY, storeName2), "store2.factory.class"); + + // empty config, return no restore managers + assertEquals(Collections.emptySet(), new StorageConfig(new MapConfig(configMap)).getRestoreFactories()); + assertEquals(Collections.emptyList(), new StorageConfig(new MapConfig(configMap)).getStoreRestoreFactories(storeName)); + assertEquals(Collections.emptyList(), new StorageConfig(new MapConfig(configMap)).getStoreRestoreFactories(storeName2)); + + // changelog set, should default to kafka state backend restore + String changelogStreamOverride = "changelogStream"; + configMap.put(String.format(CHANGELOG_STREAM, storeName), changelogStreamOverride); + configMap.put(String.format(CHANGELOG_STREAM, storeName2), changelogStreamOverride); + configMap.put(StorageConfig.CHANGELOG_SYSTEM, "changelog-system"); + configMap.put(String.format(StorageConfig.CHANGELOG_STREAM, storeName), "changelog-stream0"); + assertEquals(ImmutableSet.of(KAFKA_STATE_BACKEND_FACTORY), new StorageConfig(new MapConfig(configMap)).getRestoreFactories()); + assertEquals(DEFAULT_RESTORE_FACTORIES, new StorageConfig(new MapConfig(configMap)).getStoreRestoreFactories(storeName)); + assertEquals(DEFAULT_RESTORE_FACTORIES, new StorageConfig(new MapConfig(configMap)).getStoreRestoreFactories(storeName2)); + + // job restore manager config set should override to job backend factory + String jobRestoreFactory1 = "jobBackendRestoreFactory1"; + String jobRestoreFactory2 = "jobBackendRestoreFactory2"; + String jobRestoreFactoryOverride = jobRestoreFactory1 + "," + jobRestoreFactory2; + configMap.put(JOB_RESTORE_FACTORIES, jobRestoreFactoryOverride); + assertEquals(ImmutableSet.of(jobRestoreFactory1, jobRestoreFactory2), + new StorageConfig(new MapConfig(configMap)).getRestoreFactories()); + assertEquals(ImmutableList.of(jobRestoreFactory1, jobRestoreFactory2), + new StorageConfig(new MapConfig(configMap)).getStoreRestoreFactories(storeName)); + assertEquals(ImmutableList.of(jobRestoreFactory1, jobRestoreFactory2), + new StorageConfig(new MapConfig(configMap)).getStoreRestoreFactories(storeName2)); + + // store specific restore managers set + String storeRestoreFactory1 = "storeBackendRestoreFactory1"; + String storeRestoreFactory2 = "storeBackendRestoreFactory2"; + String storeRestoreFactoryOverride = storeRestoreFactory1 + "," + storeRestoreFactory2; + configMap.put(String.format(STORE_RESTORE_FACTORIES, storeName), storeRestoreFactoryOverride); + assertEquals(ImmutableSet.of(jobRestoreFactory1, jobRestoreFactory2, storeRestoreFactory1, storeRestoreFactory2), + new StorageConfig(new MapConfig(configMap)).getRestoreFactories()); + assertEquals(ImmutableList.of(storeRestoreFactory1, storeRestoreFactory2), + new StorageConfig(new MapConfig(configMap)).getStoreRestoreFactories(storeName)); + assertEquals(ImmutableList.of(jobRestoreFactory1, jobRestoreFactory2), + new StorageConfig(new MapConfig(configMap)).getStoreRestoreFactories(storeName2)); + + String emptyBackupFactory = ""; + configMap.put(String.format(STORE_RESTORE_FACTORIES, storeName), emptyBackupFactory); + assertEquals(ImmutableSet.of(jobRestoreFactory1, jobRestoreFactory2), + new StorageConfig(new MapConfig(configMap)).getRestoreFactories()); + assertEquals(Collections.emptyList(), + new StorageConfig(new MapConfig(configMap)).getStoreRestoreFactories(storeName)); + assertEquals(ImmutableList.of(jobRestoreFactory1, jobRestoreFactory2), + new StorageConfig(new MapConfig(configMap)).getStoreRestoreFactories(storeName2)); + assertEquals(Collections.emptyList(), + new StorageConfig(new MapConfig(configMap)).getStoresWithRestoreFactory(KAFKA_STATE_BACKEND_FACTORY)); + assertEquals(Collections.emptyList(), + new StorageConfig(new MapConfig(configMap)).getStoresWithRestoreFactory(storeRestoreFactory1)); + assertEquals(Collections.emptyList(), + new StorageConfig(new MapConfig(configMap)).getStoresWithRestoreFactory(storeRestoreFactory2)); + } + + @Test + public void testGetBackupManagers() { + String storeName = "store1"; + String storeName2 = "store2"; + Map configMap = new HashMap<>(); + configMap.put(String.format(FACTORY, storeName), "store1.factory.class"); + configMap.put(String.format(FACTORY, storeName2), "store2.factory.class"); + + // empty config, return no restore managers + assertEquals(Collections.emptySet(), new StorageConfig(new MapConfig(configMap)).getBackupFactories()); + assertEquals(Collections.emptyList(), new StorageConfig(new MapConfig(configMap)).getStoreBackupFactories(storeName)); + assertEquals(Collections.emptyList(), new StorageConfig(new MapConfig(configMap)).getStoreBackupFactories(storeName2)); + + // changelog set, should default to kafka state backend restore + String changelogStreamOverride = "changelogStream"; + configMap.put(String.format(CHANGELOG_STREAM, storeName), changelogStreamOverride); + configMap.put(String.format(CHANGELOG_STREAM, storeName2), changelogStreamOverride); + configMap.put(StorageConfig.CHANGELOG_SYSTEM, "changelog-system"); + configMap.put(String.format(StorageConfig.CHANGELOG_STREAM, storeName), "changelog-stream0"); + assertEquals(ImmutableSet.of(KAFKA_STATE_BACKEND_FACTORY), new StorageConfig(new MapConfig(configMap)).getBackupFactories()); + assertEquals(DEFAULT_BACKUP_FACTORIES, new StorageConfig(new MapConfig(configMap)).getStoreBackupFactories(storeName)); + assertEquals(DEFAULT_BACKUP_FACTORIES, new StorageConfig(new MapConfig(configMap)).getStoreBackupFactories(storeName2)); + assertEquals(ImmutableList.of(storeName2, storeName), + new StorageConfig(new MapConfig(configMap)).getStoresWithBackupFactory(KAFKA_STATE_BACKEND_FACTORY)); + + // job restore manager config set should override to job backend factory + String jobBackupFactory1 = "jobBackendBackupFactory1"; + String jobBackupFactory2 = "jobBackendBackupFactory2"; + String jobBackupFactoryOverride = jobBackupFactory1 + "," + jobBackupFactory2; + configMap.put(JOB_BACKUP_FACTORIES, jobBackupFactoryOverride); + assertEquals(ImmutableSet.of(jobBackupFactory1, jobBackupFactory2), + new StorageConfig(new MapConfig(configMap)).getBackupFactories()); + assertEquals(ImmutableList.of(jobBackupFactory1, jobBackupFactory2), + new StorageConfig(new MapConfig(configMap)).getStoreBackupFactories(storeName)); + assertEquals(ImmutableList.of(jobBackupFactory1, jobBackupFactory2), + new StorageConfig(new MapConfig(configMap)).getStoreBackupFactories(storeName2)); + assertEquals(Collections.emptyList(), + new StorageConfig(new MapConfig(configMap)).getStoresWithBackupFactory(KAFKA_STATE_BACKEND_FACTORY)); + assertEquals(ImmutableList.of(storeName2, storeName), + new StorageConfig(new MapConfig(configMap)).getStoresWithBackupFactory(jobBackupFactory1)); + assertEquals(ImmutableList.of(storeName2, storeName), + new StorageConfig(new MapConfig(configMap)).getStoresWithBackupFactory(jobBackupFactory2)); + + // store specific restore managers set + String storeBackupFactory1 = "storeBackendBackupFactory1"; + String storeBackupFactory2 = "storeBackendBackupFactory2"; + String storeBackupFactoryOverride = storeBackupFactory1 + "," + storeBackupFactory2; + configMap.put(String.format(STORE_BACKUP_FACTORIES, storeName), storeBackupFactoryOverride); + assertEquals(ImmutableSet.of(jobBackupFactory1, jobBackupFactory2, storeBackupFactory1, storeBackupFactory2), + new StorageConfig(new MapConfig(configMap)).getBackupFactories()); + assertEquals(ImmutableList.of(storeBackupFactory1, storeBackupFactory2), + new StorageConfig(new MapConfig(configMap)).getStoreBackupFactories(storeName)); + assertEquals(ImmutableList.of(jobBackupFactory1, jobBackupFactory2), + new StorageConfig(new MapConfig(configMap)).getStoreBackupFactories(storeName2)); + assertEquals(Collections.emptyList(), + new StorageConfig(new MapConfig(configMap)).getStoresWithBackupFactory(KAFKA_STATE_BACKEND_FACTORY)); + assertEquals(ImmutableList.of(storeName2), + new StorageConfig(new MapConfig(configMap)).getStoresWithBackupFactory(jobBackupFactory1)); + assertEquals(ImmutableList.of(storeName2), + new StorageConfig(new MapConfig(configMap)).getStoresWithBackupFactory(jobBackupFactory2)); + assertEquals(ImmutableList.of(storeName), + new StorageConfig(new MapConfig(configMap)).getStoresWithBackupFactory(storeBackupFactory1)); + assertEquals(ImmutableList.of(storeName), + new StorageConfig(new MapConfig(configMap)).getStoresWithBackupFactory(storeBackupFactory2)); + + String emptyBackupFactory = ""; + configMap.put(String.format(STORE_BACKUP_FACTORIES, storeName), emptyBackupFactory); + assertEquals(ImmutableSet.of(jobBackupFactory1, jobBackupFactory2), + new StorageConfig(new MapConfig(configMap)).getBackupFactories()); + assertEquals(Collections.emptyList(), + new StorageConfig(new MapConfig(configMap)).getStoreBackupFactories(storeName)); + assertEquals(ImmutableList.of(jobBackupFactory1, jobBackupFactory2), + new StorageConfig(new MapConfig(configMap)).getStoreBackupFactories(storeName2)); + assertEquals(Collections.emptyList(), + new StorageConfig(new MapConfig(configMap)).getStoresWithBackupFactory(KAFKA_STATE_BACKEND_FACTORY)); + assertEquals(Collections.emptyList(), + new StorageConfig(new MapConfig(configMap)).getStoresWithBackupFactory(storeBackupFactory1)); + assertEquals(Collections.emptyList(), + new StorageConfig(new MapConfig(configMap)).getStoresWithBackupFactory(storeBackupFactory2)); + } } diff --git a/samza-core/src/test/java/org/apache/samza/job/model/TestJobModel.java b/samza-core/src/test/java/org/apache/samza/job/model/TestJobModel.java index 77fe639f67..e66e213d6f 100644 --- a/samza-core/src/test/java/org/apache/samza/job/model/TestJobModel.java +++ b/samza-core/src/test/java/org/apache/samza/job/model/TestJobModel.java @@ -45,6 +45,6 @@ public void testMaxChangeLogStreamPartitions() { ContainerModel containerModel2 = new ContainerModel("1", tasksForContainer2); Map containers = ImmutableMap.of("0", containerModel1, "1", containerModel2); JobModel jobModel = new JobModel(config, containers); - assertEquals(jobModel.maxChangeLogStreamPartitions, 5); + assertEquals(jobModel.getMaxChangeLogStreamPartitions(), 5); } } \ No newline at end of file diff --git a/samza-core/src/test/java/org/apache/samza/serializers/TestCheckpointV2Serde.java b/samza-core/src/test/java/org/apache/samza/serializers/TestCheckpointV2Serde.java new file mode 100644 index 0000000000..d3b2d40079 --- /dev/null +++ b/samza-core/src/test/java/org/apache/samza/serializers/TestCheckpointV2Serde.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.serializers; + +import java.util.HashMap; +import java.util.Map; +import org.apache.samza.Partition; +import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.checkpoint.CheckpointV2; +import org.apache.samza.system.SystemStreamPartition; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + + +public class TestCheckpointV2Serde { + + @Test + public void testCheckpointV2Serde() { + CheckpointV2Serde serde = new CheckpointV2Serde(); + Map offsets = new HashMap<>(); + SystemStreamPartition systemStreamPartition = new SystemStreamPartition("test-system", "test-stream", new Partition(777)); + offsets.put(systemStreamPartition, "1"); + + // State Checkpoint marker + Map> factoryStateCheckpointMarkersMap = new HashMap<>(); + Map stateCheckpointMarkersMap = new HashMap<>(); + stateCheckpointMarkersMap.put("store1", "marker1"); + stateCheckpointMarkersMap.put("store2", "marker2"); + + Map stateCheckpointMarkersMap2 = new HashMap<>(); + stateCheckpointMarkersMap2.put("store1", "marker3"); + stateCheckpointMarkersMap2.put("store2", "marker4"); + + factoryStateCheckpointMarkersMap.put("factory1", stateCheckpointMarkersMap); + factoryStateCheckpointMarkersMap.put("factory2", stateCheckpointMarkersMap2); + + CheckpointId checkpointId = CheckpointId.create(); + + CheckpointV2 checkpoint = new CheckpointV2(checkpointId, offsets, factoryStateCheckpointMarkersMap); + CheckpointV2 deserializedCheckpoint = serde.fromBytes(serde.toBytes(checkpoint)); + + // Validate input checkpoints + assertEquals(checkpointId, deserializedCheckpoint.getCheckpointId()); + assertEquals("1", deserializedCheckpoint.getOffsets().get(systemStreamPartition)); + assertEquals(1, deserializedCheckpoint.getOffsets().size()); + + // Validate state checkpoints + assertEquals(2, deserializedCheckpoint.getStateCheckpointMarkers().size()); + assertTrue(deserializedCheckpoint.getStateCheckpointMarkers().containsKey("factory1")); + assertEquals(stateCheckpointMarkersMap, deserializedCheckpoint.getStateCheckpointMarkers().get("factory1")); + assertTrue(deserializedCheckpoint.getStateCheckpointMarkers().containsKey("factory2")); + assertEquals(stateCheckpointMarkersMap2, deserializedCheckpoint.getStateCheckpointMarkers().get("factory2")); + } + + @Test + public void testCheckpointV2SerdeStatelessJob() { + CheckpointV2Serde serde = new CheckpointV2Serde(); + Map offsets = new HashMap<>(); + SystemStreamPartition systemStreamPartition = new SystemStreamPartition("test-system", "test-stream", new Partition(777)); + offsets.put(systemStreamPartition, "1"); + + // State Checkpoint marker + CheckpointId checkpointId = CheckpointId.create(); + + + CheckpointV2 checkpoint = new CheckpointV2(checkpointId, offsets, new HashMap<>()); + CheckpointV2 deserializedCheckpoint = serde.fromBytes(serde.toBytes(checkpoint)); + + // Validate input checkpoints + assertEquals(checkpointId, deserializedCheckpoint.getCheckpointId()); + assertEquals("1", deserializedCheckpoint.getOffsets().get(systemStreamPartition)); + assertEquals(1, deserializedCheckpoint.getOffsets().size()); + + // No state checkpoints, but a map is still created + assertEquals(0, deserializedCheckpoint.getStateCheckpointMarkers().size()); + } +} diff --git a/samza-core/src/test/java/org/apache/samza/storage/TestKafkaChangelogStateBackendFactory.java b/samza-core/src/test/java/org/apache/samza/storage/TestKafkaChangelogStateBackendFactory.java new file mode 100644 index 0000000000..5782a75331 --- /dev/null +++ b/samza-core/src/test/java/org/apache/samza/storage/TestKafkaChangelogStateBackendFactory.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import org.apache.samza.Partition; +import org.apache.samza.container.TaskName; +import org.apache.samza.context.ContainerContextImpl; +import org.apache.samza.job.model.ContainerModel; +import org.apache.samza.job.model.TaskModel; +import org.apache.samza.system.SystemStream; +import org.apache.samza.system.SystemStreamPartition; +import org.junit.Assert; +import org.junit.Test; + + +public class TestKafkaChangelogStateBackendFactory { + + @Test + public void testGetChangelogSSP() { + KafkaChangelogStateBackendFactory factory = new KafkaChangelogStateBackendFactory(); + TaskName taskName0 = new TaskName("task0"); + TaskName taskName1 = new TaskName("task1"); + TaskModel taskModel0 = new TaskModel(taskName0, + ImmutableSet.of(new SystemStreamPartition("input", "stream", new Partition(0))), + new Partition(10)); + TaskModel taskModel1 = new TaskModel(taskName1, + ImmutableSet.of(new SystemStreamPartition("input", "stream", new Partition(1))), new Partition(11)); + ContainerModel containerModel = new ContainerModel("processorId", + ImmutableMap.of(taskName0, taskModel0, taskName1, taskModel1)); + Map changeLogSystemStreams = ImmutableMap.of( + "store0", new SystemStream("changelogSystem0", "store0-changelog"), + "store1", new SystemStream("changelogSystem1", "store1-changelog")); + Set expected = ImmutableSet.of( + new SystemStreamPartition("changelogSystem0", "store0-changelog", new Partition(10)), + new SystemStreamPartition("changelogSystem1", "store1-changelog", new Partition(10)), + new SystemStreamPartition("changelogSystem0", "store0-changelog", new Partition(11)), + new SystemStreamPartition("changelogSystem1", "store1-changelog", new Partition(11))); + Assert.assertEquals(expected, factory.getChangelogSSPForContainer(changeLogSystemStreams, + new ContainerContextImpl(containerModel, null))); + } + + @Test + public void testGetChangelogSSPsForContainerNoChangelogs() { + KafkaChangelogStateBackendFactory factory = new KafkaChangelogStateBackendFactory(); + TaskName taskName0 = new TaskName("task0"); + TaskName taskName1 = new TaskName("task1"); + TaskModel taskModel0 = new TaskModel(taskName0, + ImmutableSet.of(new SystemStreamPartition("input", "stream", new Partition(0))), + new Partition(10)); + TaskModel taskModel1 = new TaskModel(taskName1, + ImmutableSet.of(new SystemStreamPartition("input", "stream", new Partition(1))), + new Partition(11)); + ContainerModel containerModel = new ContainerModel("processorId", + ImmutableMap.of(taskName0, taskModel0, taskName1, taskModel1)); + Assert.assertEquals(Collections.emptySet(), factory.getChangelogSSPForContainer(Collections.emptyMap(), + new ContainerContextImpl(containerModel, null))); + } +} diff --git a/samza-core/src/test/java/org/apache/samza/storage/TestTaskStorageCommitManager.java b/samza-core/src/test/java/org/apache/samza/storage/TestTaskStorageCommitManager.java new file mode 100644 index 0000000000..ea00433c35 --- /dev/null +++ b/samza-core/src/test/java/org/apache/samza/storage/TestTaskStorageCommitManager.java @@ -0,0 +1,916 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage; + +import com.google.common.collect.ImmutableMap; + +import java.io.File; +import java.io.FileFilter; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ForkJoinPool; +import org.apache.samza.Partition; +import org.apache.samza.SamzaException; +import org.apache.samza.checkpoint.Checkpoint; +import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.checkpoint.CheckpointManager; +import org.apache.samza.checkpoint.CheckpointV1; +import org.apache.samza.checkpoint.CheckpointV2; +import org.apache.samza.checkpoint.kafka.KafkaChangelogSSPOffset; +import org.apache.samza.config.MapConfig; +import org.apache.samza.container.TaskInstanceMetrics; +import org.apache.samza.container.TaskName; +import org.apache.samza.job.model.TaskMode; +import org.apache.samza.metrics.Timer; +import org.apache.samza.system.SystemStream; +import org.apache.samza.system.SystemStreamPartition; +import org.junit.Test; +import org.mockito.stubbing.Answer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + + +public class TestTaskStorageCommitManager { + @Test + public void testCommitManagerStart() { + CheckpointManager checkpointManager = mock(CheckpointManager.class); + TaskBackupManager taskBackupManager1 = mock(TaskBackupManager.class); + TaskBackupManager taskBackupManager2 = mock(TaskBackupManager.class); + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + Checkpoint checkpoint = mock(Checkpoint.class); + + TaskName taskName = new TaskName("task1"); + Map backupManagers = ImmutableMap.of( + "factory1", taskBackupManager1, + "factory2", taskBackupManager2 + ); + TaskStorageCommitManager cm = new TaskStorageCommitManager(taskName, backupManagers, containerStorageManager, + Collections.emptyMap(), new Partition(1), checkpointManager, new MapConfig(), + ForkJoinPool.commonPool(), new StorageManagerUtil(), null, null); + + when(checkpointManager.readLastCheckpoint(taskName)).thenReturn(checkpoint); + cm.init(); + verify(taskBackupManager1).init(eq(checkpoint)); + verify(taskBackupManager2).init(eq(checkpoint)); + } + + @Test + public void testCommitManagerStartNullCheckpointManager() { + TaskBackupManager taskBackupManager1 = mock(TaskBackupManager.class); + TaskBackupManager taskBackupManager2 = mock(TaskBackupManager.class); + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + + TaskName task = new TaskName("task1"); + Map backupManagers = ImmutableMap.of( + "factory1", taskBackupManager1, + "factory2", taskBackupManager2 + ); + TaskStorageCommitManager cm = new TaskStorageCommitManager(task, backupManagers, containerStorageManager, + Collections.emptyMap(), new Partition(1), null, new MapConfig(), + ForkJoinPool.commonPool(), new StorageManagerUtil(), null, null); + cm.init(); + verify(taskBackupManager1).init(eq(null)); + verify(taskBackupManager2).init(eq(null)); + } + + @Test + public void testSnapshotAndCommitAllFactories() { + CheckpointManager checkpointManager = mock(CheckpointManager.class); + TaskBackupManager taskBackupManager1 = mock(TaskBackupManager.class); + TaskBackupManager taskBackupManager2 = mock(TaskBackupManager.class); + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + Checkpoint checkpoint = mock(Checkpoint.class); + TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class); + Timer checkpointTimer = mock(Timer.class); + when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer); + + TaskName taskName = new TaskName("task1"); + Map backupManagers = ImmutableMap.of( + "factory1", taskBackupManager1, + "factory2", taskBackupManager2 + ); + TaskStorageCommitManager cm = new TaskStorageCommitManager(taskName, backupManagers, containerStorageManager, + Collections.emptyMap(), new Partition(1), checkpointManager, new MapConfig(), + ForkJoinPool.commonPool(), new StorageManagerUtil(), null, metrics); + when(checkpointManager.readLastCheckpoint(taskName)).thenReturn(checkpoint); + cm.init(); + verify(taskBackupManager1).init(eq(checkpoint)); + verify(taskBackupManager2).init(eq(checkpoint)); + + CheckpointId newCheckpointId = CheckpointId.create(); + Map factory1Checkpoints = ImmutableMap.of( + "store1", "system;stream;1", + "store2", "system;stream;2" + ); + Map factory2Checkpoints = ImmutableMap.of( + "store1", "blobId1", + "store2", "blobId2" + ); + + when(containerStorageManager.getAllStores(taskName)).thenReturn(Collections.emptyMap()); + when(taskBackupManager1.snapshot(newCheckpointId)).thenReturn(factory1Checkpoints); + when(taskBackupManager2.snapshot(newCheckpointId)).thenReturn(factory2Checkpoints); + + when(taskBackupManager1.upload(newCheckpointId, factory1Checkpoints)) + .thenReturn(CompletableFuture.completedFuture(factory1Checkpoints)); + when(taskBackupManager2.upload(newCheckpointId, factory2Checkpoints)) + .thenReturn(CompletableFuture.completedFuture(factory2Checkpoints)); + + Map> snapshotSCMs = cm.snapshot(newCheckpointId); + cm.upload(newCheckpointId, snapshotSCMs); + + // Test flow for snapshot + verify(taskBackupManager1).snapshot(newCheckpointId); + verify(taskBackupManager2).snapshot(newCheckpointId); + + // Test flow for upload + verify(taskBackupManager1).upload(newCheckpointId, factory1Checkpoints); + verify(taskBackupManager2).upload(newCheckpointId, factory2Checkpoints); + verify(checkpointTimer).update(anyLong()); + } + + @Test + public void testFlushAndCheckpointOnSnapshot() { + CheckpointManager checkpointManager = mock(CheckpointManager.class); + TaskBackupManager taskBackupManager1 = mock(TaskBackupManager.class); + TaskBackupManager taskBackupManager2 = mock(TaskBackupManager.class); + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + Checkpoint checkpoint = mock(Checkpoint.class); + + StorageEngine mockLPStore = mock(StorageEngine.class); + StoreProperties lpStoreProps = mock(StoreProperties.class); + when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps); + when(lpStoreProps.isPersistedToDisk()).thenReturn(true); + when(lpStoreProps.isDurableStore()).thenReturn(true); + Path mockPath = mock(Path.class); + when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath)); + + StorageEngine mockPStore = mock(StorageEngine.class); + StoreProperties pStoreProps = mock(StoreProperties.class); + when(mockPStore.getStoreProperties()).thenReturn(pStoreProps); + when(pStoreProps.isPersistedToDisk()).thenReturn(true); + when(pStoreProps.isDurableStore()).thenReturn(false); + + StorageEngine mockLIStore = mock(StorageEngine.class); + StoreProperties liStoreProps = mock(StoreProperties.class); + when(mockLIStore.getStoreProperties()).thenReturn(liStoreProps); + when(liStoreProps.isPersistedToDisk()).thenReturn(false); + when(liStoreProps.isDurableStore()).thenReturn(true); + + StorageEngine mockIStore = mock(StorageEngine.class); + StoreProperties iStoreProps = mock(StoreProperties.class); + when(mockIStore.getStoreProperties()).thenReturn(iStoreProps); + when(iStoreProps.isPersistedToDisk()).thenReturn(false); + when(iStoreProps.isDurableStore()).thenReturn(false); + + TaskName taskName = new TaskName("task1"); + Map backupManagers = ImmutableMap.of( + "factory1", taskBackupManager1, + "factory2", taskBackupManager2 + ); + Map storageEngines = ImmutableMap.of( + "storeLP", mockLPStore, + "storeP", mockPStore, + "storeLI", mockLIStore, + "storeI", mockIStore + ); + + TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class); + Timer checkpointTimer = mock(Timer.class); + when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer); + + TaskStorageCommitManager cm = new TaskStorageCommitManager(taskName, backupManagers, containerStorageManager, + Collections.emptyMap(), new Partition(1), checkpointManager, new MapConfig(), + ForkJoinPool.commonPool(), new StorageManagerUtil(), null, metrics); + when(checkpointManager.readLastCheckpoint(taskName)).thenReturn(checkpoint); + cm.init(); + verify(taskBackupManager1).init(eq(checkpoint)); + verify(taskBackupManager2).init(eq(checkpoint)); + + CheckpointId newCheckpointId = CheckpointId.create(); + Map factory1Checkpoints = ImmutableMap.of( + "store1", "system;stream;1", + "store2", "system;stream;2" + ); + Map factory2Checkpoints = ImmutableMap.of( + "store1", "blobId1", + "store2", "blobId2" + ); + + when(containerStorageManager.getAllStores(taskName)).thenReturn(storageEngines); + when(taskBackupManager1.snapshot(newCheckpointId)).thenReturn(factory1Checkpoints); + when(taskBackupManager1.upload(newCheckpointId, factory1Checkpoints)) + .thenReturn(CompletableFuture.completedFuture(factory1Checkpoints)); + when(taskBackupManager2.snapshot(newCheckpointId)).thenReturn(factory2Checkpoints); + when(taskBackupManager2.upload(newCheckpointId, factory2Checkpoints)) + .thenReturn(CompletableFuture.completedFuture(factory2Checkpoints)); + when(mockLIStore.checkpoint(newCheckpointId)).thenReturn(Optional.empty()); + + cm.init(); + cm.snapshot(newCheckpointId); + + // Assert stores where flushed + verify(mockIStore).flush(); + verify(mockPStore).flush(); + verify(mockLIStore).flush(); + verify(mockLPStore).flush(); + // only logged and persisted stores are checkpointed + verify(mockLPStore).checkpoint(newCheckpointId); + // ensure that checkpoint is never called for non-logged persistent stores since they're + // always cleared on restart. + verify(mockPStore, never()).checkpoint(any()); + // ensure that checkpoint is never called for non-persistent stores + verify(mockIStore, never()).checkpoint(any()); + verify(mockLIStore, never()).checkpoint(any()); + verify(checkpointTimer).update(anyLong()); + } + + @Test(expected = IllegalStateException.class) + public void testSnapshotFailsIfErrorCreatingCheckpoint() { + CheckpointManager checkpointManager = mock(CheckpointManager.class); + TaskBackupManager taskBackupManager1 = mock(TaskBackupManager.class); + TaskBackupManager taskBackupManager2 = mock(TaskBackupManager.class); + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + + StorageEngine mockLPStore = mock(StorageEngine.class); + StoreProperties lpStoreProps = mock(StoreProperties.class); + when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps); + when(lpStoreProps.isPersistedToDisk()).thenReturn(true); + when(lpStoreProps.isDurableStore()).thenReturn(true); + when(mockLPStore.checkpoint(any())).thenThrow(new IllegalStateException()); + + TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class); + Timer checkpointTimer = mock(Timer.class); + when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer); + + TaskName taskName = new TaskName("task1"); + Map backupManagers = ImmutableMap.of( + "factory1", taskBackupManager1, + "factory2", taskBackupManager2 + ); + Map storageEngines = ImmutableMap.of( + "storeLP", mockLPStore + ); + + TaskStorageCommitManager cm = new TaskStorageCommitManager(taskName, backupManagers, containerStorageManager, + Collections.emptyMap(), new Partition(1), checkpointManager, new MapConfig(), + ForkJoinPool.commonPool(), new StorageManagerUtil(), null, metrics); + + when(containerStorageManager.getAllStores(taskName)).thenReturn(storageEngines); + CheckpointId newCheckpointId = CheckpointId.create(); + cm.init(); + cm.snapshot(newCheckpointId); + + // Assert stores where flushed + verify(mockLPStore).flush(); + // only logged and persisted stores are checkpointed + verify(mockLPStore).checkpoint(newCheckpointId); + verify(taskBackupManager1, never()).snapshot(any()); + verify(taskBackupManager2, never()).snapshot(any()); + verify(taskBackupManager1, never()).upload(any(), any()); + verify(taskBackupManager2, never()).upload(any(), any()); + fail("Should have thrown an exception when the storageEngine#checkpoint did not succeed"); + } + + @Test + public void testCleanupAllBackupManagers() { + CheckpointManager checkpointManager = mock(CheckpointManager.class); + TaskBackupManager taskBackupManager1 = mock(TaskBackupManager.class); + TaskBackupManager taskBackupManager2 = mock(TaskBackupManager.class); + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + Checkpoint checkpoint = mock(Checkpoint.class); + File durableStoreDir = mock(File.class); + when(durableStoreDir.listFiles()).thenReturn(new File[0]); + TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class); + Timer checkpointTimer = mock(Timer.class); + when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer); + + TaskName taskName = new TaskName("task1"); + Map backupManagers = ImmutableMap.of( + "factory1", taskBackupManager1, + "factory2", taskBackupManager2 + ); + TaskStorageCommitManager cm = new TaskStorageCommitManager(taskName, backupManagers, containerStorageManager, + Collections.emptyMap(), new Partition(1), checkpointManager, new MapConfig(), + ForkJoinPool.commonPool(), new StorageManagerUtil(), durableStoreDir, metrics); + when(checkpointManager.readLastCheckpoint(taskName)).thenReturn(checkpoint); + when(containerStorageManager.getAllStores(taskName)).thenReturn(Collections.emptyMap()); + when(taskBackupManager1.cleanUp(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + when(taskBackupManager2.cleanUp(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + Map factory1Checkpoints = ImmutableMap.of( + "store1", "system;stream;1", + "store2", "system;stream;2" + ); + Map factory2Checkpoints = ImmutableMap.of( + "store1", "blobId1", + "store2", "blobId2" + ); + Map> factoryCheckpointsMap = ImmutableMap.of( + "factory1", factory1Checkpoints, + "factory2", factory2Checkpoints + ); + + when(taskBackupManager1.cleanUp(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + when(taskBackupManager2.cleanUp(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + + CheckpointId newCheckpointId = CheckpointId.create(); + cm.cleanUp(newCheckpointId, factoryCheckpointsMap).join(); + + verify(taskBackupManager1).cleanUp(newCheckpointId, factory1Checkpoints); + verify(taskBackupManager2).cleanUp(newCheckpointId, factory2Checkpoints); + } + + @Test + public void testCleanupFailsIfBackupManagerNotInitiated() { + CheckpointManager checkpointManager = mock(CheckpointManager.class); + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + Checkpoint checkpoint = mock(Checkpoint.class); + File durableStoreDir = mock(File.class); + when(durableStoreDir.listFiles()).thenReturn(new File[0]); + TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class); + Timer checkpointTimer = mock(Timer.class); + when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer); + + TaskName taskName = new TaskName("task1"); + when(containerStorageManager.getAllStores(taskName)).thenReturn(Collections.emptyMap()); + TaskStorageCommitManager cm = new TaskStorageCommitManager(taskName, Collections.emptyMap(), containerStorageManager, + Collections.emptyMap(), new Partition(1), checkpointManager, new MapConfig(), + ForkJoinPool.commonPool(), new StorageManagerUtil(), durableStoreDir, metrics); + when(checkpointManager.readLastCheckpoint(taskName)).thenReturn(checkpoint); + + Map> factoryCheckpointsMap = ImmutableMap.of( + "factory3", Collections.emptyMap() // factory 3 should be ignored + ); + + CheckpointId newCheckpointId = CheckpointId.create(); + cm.cleanUp(newCheckpointId, factoryCheckpointsMap); + // should not fail the commit because the job should ignore any factories checkpoints not initialized + // in case the user is in a migration phase from on state backend to another + } + + @Test + public void testPersistToFileSystemCheckpointV1AndV2Checkpoint() throws IOException { + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + StorageEngine mockLPStore = mock(StorageEngine.class); + StoreProperties lpStoreProps = mock(StoreProperties.class); + when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps); + when(lpStoreProps.isPersistedToDisk()).thenReturn(true); + when(lpStoreProps.isDurableStore()).thenReturn(true); + Path mockPath = mock(Path.class); + when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath)); + + StorageEngine mockPStore = mock(StorageEngine.class); + StoreProperties pStoreProps = mock(StoreProperties.class); + when(mockPStore.getStoreProperties()).thenReturn(pStoreProps); + when(pStoreProps.isPersistedToDisk()).thenReturn(true); + when(pStoreProps.isDurableStore()).thenReturn(false); + + StorageEngine mockLIStore = mock(StorageEngine.class); + StoreProperties liStoreProps = mock(StoreProperties.class); + when(mockLIStore.getStoreProperties()).thenReturn(liStoreProps); + when(liStoreProps.isPersistedToDisk()).thenReturn(false); + when(liStoreProps.isDurableStore()).thenReturn(true); + + StorageEngine mockIStore = mock(StorageEngine.class); + StoreProperties iStoreProps = mock(StoreProperties.class); + when(mockIStore.getStoreProperties()).thenReturn(iStoreProps); + when(iStoreProps.isPersistedToDisk()).thenReturn(false); + when(iStoreProps.isDurableStore()).thenReturn(false); + + Map taskStores = ImmutableMap.of( + "loggedPersistentStore", mockLPStore, + "persistentStore", mockPStore, + "loggedInMemStore", mockLIStore, + "inMemStore", mockIStore + ); + + Partition changelogPartition = new Partition(0); + SystemStream changelogSystemStream = new SystemStream("changelogSystem", "changelogStream"); + SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); + Map storeChangelogsStreams = ImmutableMap.of( + "loggedPersistentStore", changelogSystemStream, + "loggedInMemStore", new SystemStream("system", "stream") + ); + + StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class); + File durableStoreDir = new File("durableStorePath"); + when(storageManagerUtil.getTaskStoreDir(eq(durableStoreDir), any(), any(), any())).thenReturn(durableStoreDir); + TaskName taskName = new TaskName("task"); + + TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class); + Timer checkpointTimer = mock(Timer.class); + when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer); + + when(containerStorageManager.getAllStores(taskName)).thenReturn(taskStores); + TaskStorageCommitManager commitManager = spy(new TaskStorageCommitManager(taskName, + Collections.emptyMap(), containerStorageManager, storeChangelogsStreams, changelogPartition, + null, null, ForkJoinPool.commonPool(), storageManagerUtil, durableStoreDir, metrics)); + doNothing().when(commitManager).writeChangelogOffsetFile(any(), any(), any(), any()); + when(storageManagerUtil.getStoreCheckpointDir(any(File.class), any(CheckpointId.class))) + .thenAnswer((Answer) invocation -> { + File file = invocation.getArgumentAt(0, File.class); + CheckpointId checkpointId = invocation.getArgumentAt(1, CheckpointId.class); + return file + "-" + checkpointId; + }); + CheckpointId newCheckpointId = CheckpointId.create(); + + String newestOffset = "1"; + KafkaChangelogSSPOffset kafkaChangelogSSPOffset = new KafkaChangelogSSPOffset(newCheckpointId, newestOffset); + Map offsetsJava = ImmutableMap.of( + changelogSSP, kafkaChangelogSSPOffset.toString() + ); + + commitManager.init(); + // invoke persist to file system for v2 checkpoint + commitManager.writeCheckpointToStoreDirectories(new CheckpointV1(offsetsJava)); + + verify(commitManager).writeChangelogOffsetFiles(offsetsJava); + // evoked twice, for OFFSET-V1 and OFFSET-V2 + verify(commitManager).writeChangelogOffsetFile( + eq("loggedPersistentStore"), eq(changelogSSP), eq(newestOffset), eq(durableStoreDir)); + File checkpointFile = Paths.get(storageManagerUtil.getStoreCheckpointDir( + durableStoreDir, kafkaChangelogSSPOffset.getCheckpointId())).toFile(); + verify(commitManager).writeChangelogOffsetFile( + eq("loggedPersistentStore"), eq(changelogSSP), eq(newestOffset), eq(checkpointFile)); + + Map storeSCM = ImmutableMap.of( + "loggedPersistentStore", "system;loggedPersistentStoreStream;1", + "persistentStore", "system;persistentStoreStream;1", + "loggedInMemStore", "system;loggedInMemStoreStream;1", + "inMemStore", "system;inMemStoreStream;1" + ); + CheckpointV2 checkpoint = new CheckpointV2(newCheckpointId, Collections.emptyMap(), Collections.singletonMap("factory", storeSCM)); + + // invoke persist to file system for v2 checkpoint + commitManager.writeCheckpointToStoreDirectories(checkpoint); + // Validate only durable and persisted stores are persisted + // This should be evoked twice, for checkpointV1 and checkpointV2 + verify(storageManagerUtil, times(2)).getTaskStoreDir(eq(durableStoreDir), eq("loggedPersistentStore"), eq(taskName), any()); + File checkpointPath = Paths.get(storageManagerUtil.getStoreCheckpointDir(durableStoreDir, newCheckpointId)).toFile(); + verify(storageManagerUtil).writeCheckpointV2File(eq(checkpointPath), eq(checkpoint)); + } + + @Test + public void testPersistToFileSystemCheckpointV2Only() throws IOException { + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + StorageEngine mockLPStore = mock(StorageEngine.class); + StoreProperties lpStoreProps = mock(StoreProperties.class); + when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps); + when(lpStoreProps.isPersistedToDisk()).thenReturn(true); + when(lpStoreProps.isDurableStore()).thenReturn(true); + Path mockPath = mock(Path.class); + when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath)); + + StorageEngine mockPStore = mock(StorageEngine.class); + StoreProperties pStoreProps = mock(StoreProperties.class); + when(mockPStore.getStoreProperties()).thenReturn(pStoreProps); + when(pStoreProps.isPersistedToDisk()).thenReturn(true); + when(pStoreProps.isDurableStore()).thenReturn(false); + + StorageEngine mockLIStore = mock(StorageEngine.class); + StoreProperties liStoreProps = mock(StoreProperties.class); + when(mockLIStore.getStoreProperties()).thenReturn(liStoreProps); + when(liStoreProps.isPersistedToDisk()).thenReturn(false); + when(liStoreProps.isDurableStore()).thenReturn(true); + + StorageEngine mockIStore = mock(StorageEngine.class); + StoreProperties iStoreProps = mock(StoreProperties.class); + when(mockIStore.getStoreProperties()).thenReturn(iStoreProps); + when(iStoreProps.isPersistedToDisk()).thenReturn(false); + when(iStoreProps.isDurableStore()).thenReturn(false); + + java.util.Map taskStores = ImmutableMap.of( + "loggedPersistentStore", mockLPStore, + "persistentStore", mockPStore, + "loggedInMemStore", mockLIStore, + "inMemStore", mockIStore + ); + + Partition changelogPartition = new Partition(0); + SystemStream changelogSystemStream = new SystemStream("changelogSystem", "changelogStream"); + SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); + java.util.Map storeChangelogsStreams = ImmutableMap.of( + "loggedPersistentStore", changelogSystemStream, + "loggedInMemStore", new SystemStream("system", "stream") + ); + + StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class); + File durableStoreDir = new File("durableStorePath"); + when(storageManagerUtil.getTaskStoreDir(eq(durableStoreDir), eq("loggedPersistentStore"), any(), any())) + .thenReturn(durableStoreDir); + TaskName taskName = new TaskName("task"); + + TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class); + Timer checkpointTimer = mock(Timer.class); + when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer); + + when(containerStorageManager.getAllStores(taskName)).thenReturn(taskStores); + TaskStorageCommitManager commitManager = spy(new TaskStorageCommitManager(taskName, + Collections.emptyMap(), containerStorageManager, storeChangelogsStreams, changelogPartition, + null, null, ForkJoinPool.commonPool(), storageManagerUtil, durableStoreDir, metrics)); + doNothing().when(commitManager).writeChangelogOffsetFile(any(), any(), any(), any()); + + when(storageManagerUtil.getStoreCheckpointDir(any(File.class), any(CheckpointId.class))) + .thenAnswer((Answer) invocation -> { + File file = invocation.getArgumentAt(0, File.class); + CheckpointId checkpointId = invocation.getArgumentAt(1, CheckpointId.class); + return file + "-" + checkpointId; + }); + + CheckpointId newCheckpointId = CheckpointId.create(); + + java.util.Map storeSCM = ImmutableMap.of( + "loggedPersistentStore", "system;loggedPersistentStoreStream;1", + "persistentStore", "system;persistentStoreStream;1", + "loggedInMemStore", "system;loggedInMemStoreStream;1", + "inMemStore", "system;inMemStoreStream;1" + ); + CheckpointV2 checkpoint = new CheckpointV2(newCheckpointId, Collections.emptyMap(), Collections.singletonMap("factory", storeSCM)); + + commitManager.init(); + // invoke persist to file system + commitManager.writeCheckpointToStoreDirectories(checkpoint); + // Validate only durable and persisted stores are persisted + verify(storageManagerUtil).getTaskStoreDir(eq(durableStoreDir), eq("loggedPersistentStore"), eq(taskName), any()); + File checkpointPath = Paths.get(storageManagerUtil.getStoreCheckpointDir(durableStoreDir, newCheckpointId)).toFile(); + verify(storageManagerUtil).writeCheckpointV2File(eq(checkpointPath), eq(checkpoint)); + } + + @Test + public void testWriteChangelogOffsetFilesV1() throws IOException { + Map> mockFileSystem = new HashMap<>(); + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + StorageEngine mockLPStore = mock(StorageEngine.class); + StoreProperties lpStoreProps = mock(StoreProperties.class); + when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps); + when(lpStoreProps.isPersistedToDisk()).thenReturn(true); + when(lpStoreProps.isDurableStore()).thenReturn(true); + Path mockPath = mock(Path.class); + when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath)); + TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class); + Timer checkpointTimer = mock(Timer.class); + when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer); + + java.util.Map taskStores = ImmutableMap.of("loggedPersistentStore", mockLPStore); + + Partition changelogPartition = new Partition(0); + SystemStream changelogSystemStream = new SystemStream("changelogSystem", "changelogStream"); + SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); + java.util.Map storeChangelogsStreams = ImmutableMap.of("loggedPersistentStore", changelogSystemStream); + + StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class); + File tmpTestPath = new File("store-checkpoint-test"); + when(storageManagerUtil.getTaskStoreDir(eq(tmpTestPath), eq("loggedPersistentStore"), any(), any())).thenReturn(tmpTestPath); + TaskName taskName = new TaskName("task"); + + when(containerStorageManager.getAllStores(taskName)).thenReturn(taskStores); + TaskStorageCommitManager commitManager = spy(new TaskStorageCommitManager(taskName, + Collections.emptyMap(), containerStorageManager, storeChangelogsStreams, changelogPartition, + null, null, ForkJoinPool.commonPool(), storageManagerUtil, tmpTestPath, metrics)); + + when(storageManagerUtil.getStoreCheckpointDir(any(File.class), any(CheckpointId.class))) + .thenAnswer((Answer) invocation -> { + File file = invocation.getArgumentAt(0, File.class); + CheckpointId checkpointId = invocation.getArgumentAt(1, CheckpointId.class); + return file + "-" + checkpointId; + }); + + doAnswer(invocation -> { + String fileDir = invocation.getArgumentAt(3, File.class).getName(); + SystemStreamPartition ssp = invocation.getArgumentAt(1, SystemStreamPartition.class); + String offset = invocation.getArgumentAt(2, String.class); + if (mockFileSystem.containsKey(fileDir)) { + mockFileSystem.get(fileDir).put(ssp, offset); + } else { + Map sspOffsets = new HashMap<>(); + sspOffsets.put(ssp, offset); + mockFileSystem.put(fileDir, sspOffsets); + } + return null; + }).when(commitManager).writeChangelogOffsetFile(any(), any(), any(), any()); + + CheckpointId newCheckpointId = CheckpointId.create(); + + String newestOffset = "1"; + KafkaChangelogSSPOffset kafkaChangelogSSPOffset = new KafkaChangelogSSPOffset(newCheckpointId, newestOffset); + java.util.Map offsetsJava = ImmutableMap.of( + changelogSSP, kafkaChangelogSSPOffset.toString() + ); + + commitManager.init(); + // invoke persist to file system for v2 checkpoint + commitManager.writeCheckpointToStoreDirectories(new CheckpointV1(offsetsJava)); + + assertEquals(2, mockFileSystem.size()); + // check if v2 offsets are written correctly + String v2FilePath = storageManagerUtil.getStoreCheckpointDir(tmpTestPath, newCheckpointId); + assertTrue(mockFileSystem.containsKey(v2FilePath)); + assertTrue(mockFileSystem.get(v2FilePath).containsKey(changelogSSP)); + assertEquals(1, mockFileSystem.get(v2FilePath).size()); + assertEquals(newestOffset, mockFileSystem.get(v2FilePath).get(changelogSSP)); + // check if v1 offsets are written correctly + String v1FilePath = tmpTestPath.getPath(); + assertTrue(mockFileSystem.containsKey(v1FilePath)); + assertTrue(mockFileSystem.get(v1FilePath).containsKey(changelogSSP)); + assertEquals(1, mockFileSystem.get(v1FilePath).size()); + assertEquals(newestOffset, mockFileSystem.get(v1FilePath).get(changelogSSP)); + } + + @Test + public void testWriteChangelogOffsetFilesV2andV1() throws IOException { + Map> mockFileSystem = new HashMap<>(); + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + Map mockCheckpointFileSystem = new HashMap<>(); + StorageEngine mockLPStore = mock(StorageEngine.class); + StoreProperties lpStoreProps = mock(StoreProperties.class); + when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps); + when(lpStoreProps.isPersistedToDisk()).thenReturn(true); + when(lpStoreProps.isDurableStore()).thenReturn(true); + Path mockPath = mock(Path.class); + when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath)); + TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class); + Timer checkpointTimer = mock(Timer.class); + when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer); + + java.util.Map taskStores = ImmutableMap.of("loggedPersistentStore", mockLPStore); + + Partition changelogPartition = new Partition(0); + SystemStream changelogSystemStream = new SystemStream("changelogSystem", "changelogStream"); + SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); + java.util.Map storeChangelogsStreams = ImmutableMap.of("loggedPersistentStore", changelogSystemStream); + + StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class); + File tmpTestPath = new File("store-checkpoint-test"); + when(storageManagerUtil.getTaskStoreDir(eq(tmpTestPath), eq("loggedPersistentStore"), any(), any())).thenReturn(tmpTestPath); + TaskName taskName = new TaskName("task"); + + when(containerStorageManager.getAllStores(taskName)).thenReturn(taskStores); + TaskStorageCommitManager commitManager = spy(new TaskStorageCommitManager(taskName, + Collections.emptyMap(), containerStorageManager, storeChangelogsStreams, changelogPartition, + null, null, ForkJoinPool.commonPool(), storageManagerUtil, tmpTestPath, metrics)); + + doAnswer(invocation -> { + String fileDir = invocation.getArgumentAt(3, File.class).getName(); + SystemStreamPartition ssp = invocation.getArgumentAt(1, SystemStreamPartition.class); + String offset = invocation.getArgumentAt(2, String.class); + if (mockFileSystem.containsKey(fileDir)) { + mockFileSystem.get(fileDir).put(ssp, offset); + } else { + Map sspOffsets = new HashMap<>(); + sspOffsets.put(ssp, offset); + mockFileSystem.put(fileDir, sspOffsets); + } + return null; + }).when(commitManager).writeChangelogOffsetFile(any(), any(), any(), any()); + + doAnswer(invocation -> { + String storeDir = invocation.getArgumentAt(0, File.class).getName(); + CheckpointV2 checkpointV2 = invocation.getArgumentAt(1, CheckpointV2.class); + mockCheckpointFileSystem.put(storeDir, checkpointV2); + return null; + }).when(storageManagerUtil).writeCheckpointV2File(any(), any()); + + when(storageManagerUtil.getStoreCheckpointDir(any(File.class), any(CheckpointId.class))) + .thenAnswer((Answer) invocation -> { + File file = invocation.getArgumentAt(0, File.class); + CheckpointId checkpointId = invocation.getArgumentAt(1, CheckpointId.class); + return file + "-" + checkpointId; + }); + + CheckpointId newCheckpointId = CheckpointId.create(); + + String newestOffset = "1"; + KafkaChangelogSSPOffset kafkaChangelogSSPOffset = new KafkaChangelogSSPOffset(newCheckpointId, newestOffset); + java.util.Map offsetsJava = ImmutableMap.of( + changelogSSP, kafkaChangelogSSPOffset.toString() + ); + + commitManager.init(); + // invoke persist to file system for v1 checkpoint + commitManager.writeCheckpointToStoreDirectories(new CheckpointV1(offsetsJava)); + + assertEquals(2, mockFileSystem.size()); + // check if v2 offsets are written correctly + String v2FilePath = storageManagerUtil.getStoreCheckpointDir(tmpTestPath, newCheckpointId); + assertTrue(mockFileSystem.containsKey(v2FilePath)); + assertTrue(mockFileSystem.get(v2FilePath).containsKey(changelogSSP)); + assertEquals(1, mockFileSystem.get(v2FilePath).size()); + assertEquals(newestOffset, mockFileSystem.get(v2FilePath).get(changelogSSP)); + // check if v1 offsets are written correctly + String v1FilePath = tmpTestPath.getPath(); + assertTrue(mockFileSystem.containsKey(v1FilePath)); + assertTrue(mockFileSystem.get(v1FilePath).containsKey(changelogSSP)); + assertEquals(1, mockFileSystem.get(v1FilePath).size()); + assertEquals(newestOffset, mockFileSystem.get(v1FilePath).get(changelogSSP)); + + java.util.Map storeSCM = ImmutableMap.of( + "loggedPersistentStore", "system;loggedPersistentStoreStream;1", + "persistentStore", "system;persistentStoreStream;1", + "loggedInMemStore", "system;loggedInMemStoreStream;1", + "inMemStore", "system;inMemStoreStream;1" + ); + CheckpointV2 checkpoint = new CheckpointV2(newCheckpointId, Collections.emptyMap(), Collections.singletonMap("factory", storeSCM)); + + // invoke persist to file system with checkpoint v2 + commitManager.writeCheckpointToStoreDirectories(checkpoint); + + assertTrue(mockCheckpointFileSystem.containsKey(v2FilePath)); + assertEquals(checkpoint, mockCheckpointFileSystem.get(v2FilePath)); + assertTrue(mockCheckpointFileSystem.containsKey(v1FilePath)); + assertEquals(checkpoint, mockCheckpointFileSystem.get(v1FilePath)); + assertEquals(2, mockCheckpointFileSystem.size()); + + CheckpointV2 updatedCheckpoint = new CheckpointV2( + newCheckpointId, ImmutableMap.of( + new SystemStreamPartition("inputSystem", "inputStream", changelogPartition), "5"), + Collections.singletonMap("factory", storeSCM)); + commitManager.writeCheckpointToStoreDirectories(updatedCheckpoint); + + assertEquals(updatedCheckpoint, mockCheckpointFileSystem.get(v2FilePath)); + assertEquals(updatedCheckpoint, mockCheckpointFileSystem.get(v1FilePath)); + assertEquals(2, mockCheckpointFileSystem.size()); + } + + @Test + public void testWriteChangelogOffsetFilesWithEmptyChangelogTopic() throws IOException { + Map> mockFileSystem = new HashMap<>(); + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + StorageEngine mockLPStore = mock(StorageEngine.class); + StoreProperties lpStoreProps = mock(StoreProperties.class); + when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps); + when(lpStoreProps.isPersistedToDisk()).thenReturn(true); + when(lpStoreProps.isDurableStore()).thenReturn(true); + Path mockPath = mock(Path.class); + when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath)); + TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class); + Timer checkpointTimer = mock(Timer.class); + when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer); + + java.util.Map taskStores = ImmutableMap.of("loggedPersistentStore", mockLPStore); + + Partition changelogPartition = new Partition(0); + SystemStream changelogSystemStream = new SystemStream("changelogSystem", "changelogStream"); + SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); + java.util.Map storeChangelogsStreams = ImmutableMap.of("loggedPersistentStore", changelogSystemStream); + + StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class); + File tmpTestPath = new File("store-checkpoint-test"); + when(storageManagerUtil.getTaskStoreDir(eq(tmpTestPath), any(), any(), any())).thenReturn(tmpTestPath); + TaskName taskName = new TaskName("task"); + + when(containerStorageManager.getAllStores(taskName)).thenReturn(taskStores); + TaskStorageCommitManager commitManager = spy(new TaskStorageCommitManager(taskName, + Collections.emptyMap(), containerStorageManager, storeChangelogsStreams, changelogPartition, + null, null, ForkJoinPool.commonPool(), storageManagerUtil, tmpTestPath, metrics)); + + doAnswer(invocation -> { + String storeName = invocation.getArgumentAt(0, String.class); + String fileDir = invocation.getArgumentAt(3, File.class).getName(); + String mockKey = storeName + fileDir; + SystemStreamPartition ssp = invocation.getArgumentAt(1, SystemStreamPartition.class); + String offset = invocation.getArgumentAt(2, String.class); + if (mockFileSystem.containsKey(mockKey)) { + mockFileSystem.get(mockKey).put(ssp, offset); + } else { + Map sspOffsets = new HashMap<>(); + sspOffsets.put(ssp, offset); + mockFileSystem.put(mockKey, sspOffsets); + } + return null; + }).when(commitManager).writeChangelogOffsetFile(any(), any(), any(), any()); + + CheckpointId newCheckpointId = CheckpointId.create(); + + String newestOffset = null; + KafkaChangelogSSPOffset kafkaChangelogSSPOffset = new KafkaChangelogSSPOffset(newCheckpointId, newestOffset); + java.util.Map offsetsJava = ImmutableMap.of( + changelogSSP, kafkaChangelogSSPOffset.toString() + ); + + commitManager.init(); + // invoke persist to file system for v2 checkpoint + commitManager.writeCheckpointToStoreDirectories(new CheckpointV1(offsetsJava)); + assertTrue(mockFileSystem.isEmpty()); + // verify that delete was called on current store dir offset file + verify(storageManagerUtil, times(1)).deleteOffsetFile(eq(tmpTestPath)); + } + + @Test(expected = SamzaException.class) + public void testThrowOnWriteCheckpointDirIfUnsuccessful() { + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + StorageEngine mockLPStore = mock(StorageEngine.class); + StoreProperties lpStoreProps = mock(StoreProperties.class); + when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps); + when(lpStoreProps.isPersistedToDisk()).thenReturn(true); + when(lpStoreProps.isDurableStore()).thenReturn(true); + Path mockPath = mock(Path.class); + when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath)); + TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class); + Timer checkpointTimer = mock(Timer.class); + when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer); + + java.util.Map taskStores = ImmutableMap.of("loggedPersistentStore", mockLPStore); + + java.util.Map storeChangelogsStreams = ImmutableMap.of("loggedPersistentStore", mock(SystemStream.class)); + + StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class); + File tmpTestPath = new File("store-checkpoint-test"); + when(storageManagerUtil.getTaskStoreDir(eq(tmpTestPath), eq("loggedPersistentStore"), any(), any())).thenReturn(tmpTestPath); + + TaskName taskName = new TaskName("task"); + + TaskStorageCommitManager commitManager = spy(new TaskStorageCommitManager(taskName, + Collections.emptyMap(), containerStorageManager, storeChangelogsStreams, mock(Partition.class), + null, null, ForkJoinPool.commonPool(), storageManagerUtil, tmpTestPath, metrics)); + + java.util.Map storeSCM = ImmutableMap.of( + "loggedPersistentStore", "system;loggedPersistentStoreStream;1", + "persistentStore", "system;persistentStoreStream;1", + "loggedInMemStore", "system;loggedInMemStoreStream;1", + "inMemStore", "system;inMemStoreStream;1" + ); + when(containerStorageManager.getAllStores(taskName)).thenReturn(taskStores); + CheckpointV2 checkpoint = new CheckpointV2(CheckpointId.create(), Collections.emptyMap(), Collections.singletonMap("factory", storeSCM)); + doThrow(IOException.class).when(storageManagerUtil).writeCheckpointV2File(eq(tmpTestPath), eq(checkpoint)); + + commitManager.init(); + // Should throw samza exception since writeCheckpointV2 failed + commitManager.writeCheckpointToStoreDirectories(checkpoint); + } + + @Test + public void testRemoveOldCheckpointsWhenBaseDirContainsRegularFiles() { + ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); + CheckpointManager checkpointManager = mock(CheckpointManager.class); + TaskBackupManager taskBackupManager1 = mock(TaskBackupManager.class); + TaskBackupManager taskBackupManager2 = mock(TaskBackupManager.class); + File durableStoreDir = mock(File.class); + TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class); + Timer checkpointTimer = mock(Timer.class); + when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer); + + StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class); + + TaskName taskName = new TaskName("task1"); + Map backupManagers = ImmutableMap.of( + "factory1", taskBackupManager1, + "factory2", taskBackupManager2 + ); + + when(containerStorageManager.getAllStores(taskName)).thenReturn(Collections.emptyMap()); + TaskStorageCommitManager cm = new TaskStorageCommitManager(taskName, backupManagers, containerStorageManager, + Collections.emptyMap(), new Partition(1), checkpointManager, new MapConfig(), + ForkJoinPool.commonPool(), storageManagerUtil, durableStoreDir, metrics); + + + File mockStoreDir = mock(File.class); + String mockStoreDirName = "notDirectory"; + when(durableStoreDir.listFiles()).thenReturn(new File[] {mockStoreDir}); + when(mockStoreDir.getName()).thenReturn(mockStoreDirName); + when(storageManagerUtil.getTaskStoreDir(eq(durableStoreDir), eq(mockStoreDirName), eq(taskName), eq(TaskMode.Active))).thenReturn(mockStoreDir); + // null here can happen if listFiles is called on a non-directory + when(mockStoreDir.listFiles(any(FileFilter.class))).thenReturn(null); + + cm.cleanUp(CheckpointId.create(), new HashMap<>()).join(); + verify(durableStoreDir).listFiles(); + verify(mockStoreDir).listFiles(any(FileFilter.class)); + verify(storageManagerUtil).getTaskStoreDir(eq(durableStoreDir), eq(mockStoreDirName), eq(taskName), eq(TaskMode.Active)); + } +} diff --git a/samza-core/src/test/java/org/apache/samza/storage/TestTransactionalStateTaskRestoreManager.java b/samza-core/src/test/java/org/apache/samza/storage/TestTransactionalStateTaskRestoreManager.java index 2bdd6c371e..6d5b45c66d 100644 --- a/samza-core/src/test/java/org/apache/samza/storage/TestTransactionalStateTaskRestoreManager.java +++ b/samza-core/src/test/java/org/apache/samza/storage/TestTransactionalStateTaskRestoreManager.java @@ -33,7 +33,7 @@ import java.util.Set; import org.apache.samza.Partition; import org.apache.samza.checkpoint.CheckpointId; -import org.apache.samza.checkpoint.CheckpointedChangelogOffset; +import org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; import org.apache.samza.config.TaskConfig; @@ -134,7 +134,7 @@ public void testGetStoreActionsForNonLoggedPersistentStore_AlwaysClearStore() { Map mockStoreChangelogs = ImmutableMap.of(); - Map mockCheckpointedChangelogOffset = ImmutableMap.of(); + Map mockCheckpointedChangelogOffset = ImmutableMap.of(); Map mockCurrentChangelogOffsets = ImmutableMap.of(); SystemAdmins mockSystemAdmins = mock(SystemAdmins.class); @@ -149,7 +149,7 @@ public void testGetStoreActionsForNonLoggedPersistentStore_AlwaysClearStore() { .thenReturn(mockCurrentStoreDir); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, null, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -185,10 +185,11 @@ public void testStoreDeletedWhenCleanDirsFlagSet() { Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "5"; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - ImmutableMap mockCheckpointedChangelogOffset = - ImmutableMap.of(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + ImmutableMap mockCheckpointedChangelogOffset = + ImmutableMap.of(store1Name, kafkaStateCheckpointMarker); Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -218,7 +219,7 @@ public void testStoreDeletedWhenCleanDirsFlagSet() { .thenReturn(ImmutableList.of(dummyCheckpointDir)); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -255,10 +256,11 @@ public void testGetStoreActionsForLoggedNonPersistentStore_RestoreToCheckpointed Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "5"; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - ImmutableMap mockCheckpointedChangelogOffset = - ImmutableMap.of(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + ImmutableMap mockCheckpointedChangelogOffset = + ImmutableMap.of(store1Name, kafkaStateCheckpointMarker); Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -279,7 +281,7 @@ public void testGetStoreActionsForLoggedNonPersistentStore_RestoreToCheckpointed }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -319,11 +321,12 @@ public void testGetStoreActionsForLoggedNonPersistentStore_FullRestoreIfCheckpoi Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "21"; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - Map mockCheckpointedChangelogOffset = - new HashMap() { { - put(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + Map mockCheckpointedChangelogOffset = + new HashMap() { { + put(store1Name, kafkaStateCheckpointMarker); } }; Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -345,7 +348,7 @@ public void testGetStoreActionsForLoggedNonPersistentStore_FullRestoreIfCheckpoi }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -386,11 +389,12 @@ public void testGetStoreActionsForLoggedNonPersistentStore_FullRestoreIfCheckpoi Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "5"; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - Map mockCheckpointedChangelogOffset = - new HashMap() { { - put(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + Map mockCheckpointedChangelogOffset = + new HashMap() { { + put(store1Name, kafkaStateCheckpointMarker); } }; Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -412,7 +416,7 @@ public void testGetStoreActionsForLoggedNonPersistentStore_FullRestoreIfCheckpoi }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -453,12 +457,12 @@ public void testGetStoreActionsForLoggedNonPersistentStore_FullRestoreIfCheckpoi Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "5"; - CheckpointId checkpointId = CheckpointId.fromString("0-0"); // checkpoint id older than default min.compaction.lag.ms - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(checkpointId, changelog1CheckpointedOffset); - Map mockCheckpointedChangelogOffset = - new HashMap() { { - put(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.deserialize("0-0"); // checkpoint id older than default min.compaction.lag.ms + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + Map mockCheckpointedChangelogOffset = + new HashMap() { { + put(store1Name, kafkaStateCheckpointMarker); } }; Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -480,7 +484,7 @@ public void testGetStoreActionsForLoggedNonPersistentStore_FullRestoreIfCheckpoi }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -524,11 +528,12 @@ public void testGetStoreActionsForLoggedNonPersistentStore_FullTrimIfNullCheckpo Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = null; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - Map mockCheckpointedChangelogOffset = - new HashMap() { { - put(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + Map mockCheckpointedChangelogOffset = + new HashMap() { { + put(store1Name, kafkaStateCheckpointMarker); } }; Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -552,7 +557,7 @@ public void testGetStoreActionsForLoggedNonPersistentStore_FullTrimIfNullCheckpo }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -588,11 +593,12 @@ public void testGetStoreActionsForLoggedNonPersistentStore_FullRestoreIfNullChec Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = null; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - Map mockCheckpointedChangelogOffset = - new HashMap() { { - put(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + Map mockCheckpointedChangelogOffset = + new HashMap() { { + put(store1Name, kafkaStateCheckpointMarker); } }; Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -616,7 +622,7 @@ public void testGetStoreActionsForLoggedNonPersistentStore_FullRestoreIfNullChec }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -652,10 +658,11 @@ public void testGetStoreActionsForLoggedPersistentStore_RestoreToCheckpointedOff Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "5"; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - ImmutableMap mockCheckpointedChangelogOffset = - ImmutableMap.of(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + ImmutableMap mockCheckpointedChangelogOffset = + ImmutableMap.of(store1Name, kafkaStateCheckpointMarker); Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -679,7 +686,7 @@ public void testGetStoreActionsForLoggedPersistentStore_RestoreToCheckpointedOff }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -717,10 +724,11 @@ public void testGetStoreActionsForLoggedPersistentStore_RestoreToCheckpointedOff Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "5"; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - ImmutableMap mockCheckpointedChangelogOffset = - ImmutableMap.of(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + ImmutableMap mockCheckpointedChangelogOffset = + ImmutableMap.of(store1Name, kafkaStateCheckpointMarker); Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -760,7 +768,7 @@ public void testGetStoreActionsForLoggedPersistentStore_RestoreToCheckpointedOff }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -800,10 +808,11 @@ public void testGetStoreActionsForLoggedPersistentStore_RestoreDeltaIfStaleStore Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "5"; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - ImmutableMap mockCheckpointedChangelogOffset = - ImmutableMap.of(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + ImmutableMap mockCheckpointedChangelogOffset = + ImmutableMap.of(store1Name, kafkaStateCheckpointMarker); Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -843,7 +852,7 @@ public void testGetStoreActionsForLoggedPersistentStore_RestoreDeltaIfStaleStore }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -886,10 +895,11 @@ public void testGetStoreActionsForLoggedPersistentStore_NoRestoreButTrimIfUpToDa Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "5"; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - ImmutableMap mockCheckpointedChangelogOffset = - ImmutableMap.of(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + ImmutableMap mockCheckpointedChangelogOffset = + ImmutableMap.of(store1Name, kafkaStateCheckpointMarker); Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -927,7 +937,7 @@ public void testGetStoreActionsForLoggedPersistentStore_NoRestoreButTrimIfUpToDa }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -971,10 +981,11 @@ public void testGetStoreActionsForLoggedPersistentStore_DeleteStoreCheckpointIfL Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "5"; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - ImmutableMap mockCheckpointedChangelogOffset = - ImmutableMap.of(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + ImmutableMap mockCheckpointedChangelogOffset = + ImmutableMap.of(store1Name, kafkaStateCheckpointMarker); Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -1012,7 +1023,7 @@ public void testGetStoreActionsForLoggedPersistentStore_DeleteStoreCheckpointIfL }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -1056,10 +1067,11 @@ public void testGetStoreActionsForLoggedPersistentStore_RetainOneCheckpointIfMul Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "5"; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - ImmutableMap mockCheckpointedChangelogOffset = - ImmutableMap.of(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + ImmutableMap mockCheckpointedChangelogOffset = + ImmutableMap.of(store1Name, kafkaStateCheckpointMarker); Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -1097,7 +1109,7 @@ public void testGetStoreActionsForLoggedPersistentStore_RetainOneCheckpointIfMul }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -1144,11 +1156,12 @@ public void testGetStoreActionsForLoggedPersistentStore_FullRestoreIfNullCheckpo Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = null; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - Map mockCheckpointedChangelogOffset = - new HashMap() { { - put(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + Map mockCheckpointedChangelogOffset = + new HashMap() { { + put(store1Name, kafkaStateCheckpointMarker); } }; Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -1191,7 +1204,7 @@ public void testGetStoreActionsForLoggedPersistentStore_FullRestoreIfNullCheckpo }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -1239,11 +1252,12 @@ public void testGetStoreActionsForLoggedPersistentStore_FullTrimIfNullCheckpoint Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = null; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - Map mockCheckpointedChangelogOffset = - new HashMap() { { - put(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + Map mockCheckpointedChangelogOffset = + new HashMap() { { + put(store1Name, kafkaStateCheckpointMarker); } }; Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -1286,7 +1300,7 @@ public void testGetStoreActionsForLoggedPersistentStore_FullTrimIfNullCheckpoint }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -1332,11 +1346,12 @@ public void testGetStoreActionsForLoggedPersistentStore_FullRestoreIfEqualCheckp Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "5"; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - Map mockCheckpointedChangelogOffset = - new HashMap() { { - put(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + Map mockCheckpointedChangelogOffset = + new HashMap() { { + put(store1Name, kafkaStateCheckpointMarker); } }; Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -1376,7 +1391,7 @@ public void testGetStoreActionsForLoggedPersistentStore_FullRestoreIfEqualCheckp }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -1421,11 +1436,12 @@ public void testGetStoreActionsForLoggedPersistentStore_FullRestoreIfNullCheckpo Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = null; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - Map mockCheckpointedChangelogOffset = - new HashMap() { { - put(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + Map mockCheckpointedChangelogOffset = + new HashMap() { { + put(store1Name, kafkaStateCheckpointMarker); } }; Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -1471,7 +1487,7 @@ public void testGetStoreActionsForLoggedPersistentStore_FullRestoreIfNullCheckpo }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -1516,11 +1532,12 @@ public void testGetStoreActionsForLoggedPersistentStore_FullRestoreIfCheckpointe Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "5"; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - Map mockCheckpointedChangelogOffset = - new HashMap() { { - put(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + Map mockCheckpointedChangelogOffset = + new HashMap() { { + put(store1Name, kafkaStateCheckpointMarker); } }; Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -1561,7 +1578,7 @@ public void testGetStoreActionsForLoggedPersistentStore_FullRestoreIfCheckpointe }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -1606,12 +1623,12 @@ public void testGetStoreActionsForLoggedPersistentStore_RestoreFromLocalToNewest Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "5"; - CheckpointId checkpointId = CheckpointId.fromString("0-0"); // checkpoint timestamp older than default min compaction lag - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(checkpointId, changelog1CheckpointedOffset); - Map mockCheckpointedChangelogOffset = - new HashMap() { { - put(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.deserialize("0-0"); // checkpoint timestamp older than default min compaction lag + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + Map mockCheckpointedChangelogOffset = + new HashMap() { { + put(store1Name, kafkaStateCheckpointMarker); } }; Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -1652,7 +1669,7 @@ public void testGetStoreActionsForLoggedPersistentStore_RestoreFromLocalToNewest }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); @@ -1697,11 +1714,12 @@ public void testGetStoreActionsForLoggedPersistentStore_FullRestoreIfCheckpointe Map mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream); String changelog1CheckpointedOffset = "21"; - CheckpointedChangelogOffset changelog1CheckpointMessage = - new CheckpointedChangelogOffset(CheckpointId.create(), changelog1CheckpointedOffset); - Map mockCheckpointedChangelogOffset = - new HashMap() { { - put(changelog1SSP, changelog1CheckpointMessage.toString()); + CheckpointId checkpointId = CheckpointId.create(); + KafkaStateCheckpointMarker kafkaStateCheckpointMarker = + new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset); + Map mockCheckpointedChangelogOffset = + new HashMap() { { + put(store1Name, kafkaStateCheckpointMarker); } }; Map mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata); @@ -1742,7 +1760,7 @@ public void testGetStoreActionsForLoggedPersistentStore_FullRestoreIfCheckpointe }); StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions( - mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, + mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock); diff --git a/samza-core/src/test/java/org/apache/samza/storage/blobstore/TestBlobStoreBackupManager.java b/samza-core/src/test/java/org/apache/samza/storage/blobstore/TestBlobStoreBackupManager.java new file mode 100644 index 0000000000..ca9e211bfd --- /dev/null +++ b/samza-core/src/test/java/org/apache/samza/storage/blobstore/TestBlobStoreBackupManager.java @@ -0,0 +1,542 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore; + +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.MoreExecutors; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.samza.SamzaException; +import org.apache.samza.checkpoint.Checkpoint; +import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.checkpoint.CheckpointV1; +import org.apache.samza.checkpoint.CheckpointV2; +import org.apache.samza.config.Config; +import org.apache.samza.config.MapConfig; +import org.apache.samza.container.TaskName; +import org.apache.samza.job.model.ContainerModel; +import org.apache.samza.job.model.JobModel; +import org.apache.samza.job.model.TaskMode; +import org.apache.samza.job.model.TaskModel; +import org.apache.samza.metrics.Counter; +import org.apache.samza.metrics.Gauge; +import org.apache.samza.metrics.MetricsRegistry; +import org.apache.samza.metrics.Timer; +import org.apache.samza.storage.StorageEngine; +import org.apache.samza.storage.StorageManagerUtil; +import org.apache.samza.storage.blobstore.diff.DirDiff; +import org.apache.samza.storage.blobstore.index.DirIndex; +import org.apache.samza.storage.blobstore.index.SnapshotIndex; +import org.apache.samza.storage.blobstore.index.SnapshotMetadata; +import org.apache.samza.storage.blobstore.metrics.BlobStoreBackupManagerMetrics; +import org.apache.samza.storage.blobstore.util.BlobStoreTestUtil; +import org.apache.samza.storage.blobstore.util.BlobStoreUtil; +import org.apache.samza.storage.blobstore.util.DirDiffUtil; +import org.apache.samza.util.Clock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import static org.mockito.Matchers.*; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.*; + + +public class TestBlobStoreBackupManager { + private final ExecutorService mockExecutor = MoreExecutors.newDirectExecutorService(); + // mock container - task - job models + private final JobModel jobModel = mock(JobModel.class); + private final ContainerModel containerModel = mock(ContainerModel.class); + private final TaskModel taskModel = mock(TaskModel.class, RETURNS_DEEP_STUBS); + private final Clock clock = mock(Clock.class); + private final BlobStoreUtil blobStoreUtil = mock(BlobStoreUtil.class); + private final BlobStoreManager blobStoreManager = mock(BlobStoreManager.class); + private final StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class); + + //job and store definition + private final CheckpointId checkpointId = CheckpointId.deserialize("1234-567"); + private final String jobName = "testJobName"; + private final String jobId = "testJobID"; + private final String taskName = "testTaskName"; + private final String prevSnapshotIndexBlobId = "testPrevBlobId"; + private Map storeStorageEngineMap = new HashMap<>(); + private Map mapConfig = new HashMap<>(); + + private final MetricsRegistry metricsRegistry = mock(MetricsRegistry.class); + private final Counter counter = mock(Counter.class); + private final Timer timer = mock(Timer.class); + private final Gauge longGauge = mock(Gauge.class); + private final Gauge atomicLongGauge = mock(Gauge.class); + + private BlobStoreBackupManager blobStoreBackupManager; + private BlobStoreBackupManagerMetrics blobStoreTaskBackupMetrics; + + // Remote and local snapshot definitions + private Map testBlobStore = new HashMap<>(); + private Map> indexBlobIdAndLocalRemoteSnapshotsPair; + private Map testStoreNameAndSCMMap; + + @Before + public void setup() throws Exception { + when(clock.currentTimeMillis()).thenReturn(1234567L); + // setup test local and remote snapshots + indexBlobIdAndLocalRemoteSnapshotsPair = setupRemoteAndLocalSnapshots(true); + // setup test store name and SCMs map + testStoreNameAndSCMMap = setupTestStoreSCMMapAndStoreBackedFactoryConfig(indexBlobIdAndLocalRemoteSnapshotsPair); + // setup: setup task backup manager with expected storeName->storageEngine map + testStoreNameAndSCMMap.forEach((storeName, scm) -> storeStorageEngineMap.put(storeName, null)); + + mapConfig.putAll(new MapConfig(ImmutableMap.of("job.name", jobName, "job.id", jobId))); + + Config config = new MapConfig(mapConfig); + + // Mock - return snapshot index for blob id from test blob store map + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + when(blobStoreUtil.getSnapshotIndex(captor.capture(), any(Metadata.class))) + .then((Answer>) invocation -> { + String blobId = invocation.getArgumentAt(0, String.class); + return CompletableFuture.completedFuture(testBlobStore.get(blobId)); + }); + +// doNothing().when(blobStoreManager).init(); + when(taskModel.getTaskName().getTaskName()).thenReturn(taskName); + when(taskModel.getTaskMode()).thenReturn(TaskMode.Active); + + when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(counter); + when(metricsRegistry.newGauge(anyString(), anyString(), anyLong())).thenReturn(longGauge); + when(metricsRegistry.newGauge(anyString(), anyString(), any(AtomicLong.class))).thenReturn(atomicLongGauge); + when(atomicLongGauge.getValue()).thenReturn(new AtomicLong()); + when(metricsRegistry.newTimer(anyString(), anyString())).thenReturn(timer); + blobStoreTaskBackupMetrics = new BlobStoreBackupManagerMetrics(metricsRegistry); + + blobStoreBackupManager = + new MockBlobStoreBackupManager(jobModel, containerModel, taskModel, mockExecutor, + blobStoreTaskBackupMetrics, config, + Files.createTempDirectory("logged-store-").toFile(), storageManagerUtil, blobStoreManager); + } + + @Test + public void testInitWithInvalidCheckpointFails() { + // init called with null checkpoint storeStorageEngineMap + blobStoreBackupManager.init(null); + // verify delete snapshot index blob called from init 0 times because prevSnapshotMap returned from init is empty + // in case of null checkpoint. + verify(blobStoreUtil, times(0)).deleteSnapshotIndexBlob(anyString(), any(Metadata.class)); + when(blobStoreUtil.getStoreSnapshotIndexes(anyString(), anyString(), anyString(), any(Checkpoint.class))).thenCallRealMethod(); + + // init called with Checkpoint V1 -> unsupported + Checkpoint checkpoint = new CheckpointV1(new HashMap<>()); + String expectedException = "Checkpoint version 1 is not supported for blob store backup and restore."; + try { + blobStoreBackupManager.init(checkpoint); + Assert.fail("Checkpoint V1 is exepcted to fail."); + } catch (SamzaException exception) { + Assert.assertEquals(exception.getMessage(), expectedException); + } + } + + @Test + public void testUploadWithNoPreviousCheckpoints() throws IOException { + // Track directory for post cleanup + List checkpointDirsToClean = new ArrayList<>(); + + // Setup: init local/remote snapshots and back manager with no previous checkpoints + indexBlobIdAndLocalRemoteSnapshotsPair = setupRemoteAndLocalSnapshots(false); + Checkpoint checkpoint = + new CheckpointV2(checkpointId, new HashMap<>(), + ImmutableMap.of(BlobStoreStateBackendFactory.class.getName(), new HashMap<>())); + blobStoreBackupManager.init(checkpoint); + + // mock: set task store dir to return corresponding test local store and create checkpoint dir + ArgumentCaptor stringCaptor = ArgumentCaptor.forClass(String.class); + when(storageManagerUtil.getTaskStoreDir(any(File.class), stringCaptor.capture(), any(TaskName.class), any(TaskMode.class))) + .then((Answer) invocation -> { + String storeName = invocation.getArgumentAt(1, String.class); + String snapshotIndexBlobId = testStoreNameAndSCMMap.get(storeName); + String storeDir = indexBlobIdAndLocalRemoteSnapshotsPair.get(snapshotIndexBlobId).getLeft(); + try { + BlobStoreTestUtil.createTestCheckpointDirectory(storeDir, checkpointId.serialize()); // create test checkpoint dir + checkpointDirsToClean.add(storeDir + "-" + checkpointId.serialize()); // track checkpoint dir to cleanup later + } catch (IOException e) { + Assert.fail("Couldn't create checkpoint directory. Test failed."); + } + return new File(storeDir); + }); + + + ArgumentCaptor storeDirCaptor = ArgumentCaptor.forClass(File.class); + when(storageManagerUtil.getStoreCheckpointDir(storeDirCaptor.capture(), eq(checkpointId))) + .thenAnswer(new Answer() { + @Override + public String answer(InvocationOnMock invocation) throws Throwable { + File storeDir = invocation.getArgumentAt(0, File.class); + return storeDir.getAbsolutePath() + "-" + checkpointId.serialize(); + } + }); + + SortedSet actualDirDiffs = new TreeSet<>(Comparator.comparing(DirDiff::getDirName)); + // mock: mock putDir and capture DirDiff + ArgumentCaptor dirDiffCaptor = ArgumentCaptor.forClass(DirDiff.class); + ArgumentCaptor snapshotMetadataCaptor = ArgumentCaptor.forClass(SnapshotMetadata.class); + when(blobStoreUtil.putDir(dirDiffCaptor.capture(), snapshotMetadataCaptor.capture())) + .then((Answer>) invocation -> { + DirDiff dirDiff = invocation.getArgumentAt(0, DirDiff.class); + SnapshotMetadata snapshotMetadata = invocation.getArgumentAt(1, SnapshotMetadata.class); + actualDirDiffs.add(dirDiff); + SnapshotIndex snapshotIndex = testBlobStore.get(testStoreNameAndSCMMap.get(snapshotMetadata.getStoreName())); + return CompletableFuture.completedFuture(snapshotIndex.getDirIndex()); + }); + + SortedSet expectedSnapshotIndexesUploaded = indexBlobIdAndLocalRemoteSnapshotsPair.values().stream() + .map(Pair::getRight) + .collect(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(SnapshotIndex::getCreationTimeMillis)))); + String expectedPreviousSnapshotIndexBlobId = "empty"; + // mock: mock putSnapshotIndex and capture previous snapshot index + SortedSet actualSnapshotIndexesUploaded = + new TreeSet<>(Comparator.comparing(SnapshotIndex::getCreationTimeMillis)); + final String[] actualPreviousSnapshotIndexBlobId = {"empty"}; + ArgumentCaptor snapshotIndexCaptor = ArgumentCaptor.forClass(SnapshotIndex.class); + when(blobStoreUtil.putSnapshotIndex(snapshotIndexCaptor.capture())) + .then((Answer>) invocation -> { + SnapshotIndex snapshotIndex = invocation.getArgumentAt(0, SnapshotIndex.class); + actualSnapshotIndexesUploaded.add(snapshotIndex); + if (!snapshotIndex.getPrevSnapshotIndexBlobId().equals(Optional.empty())) { + actualPreviousSnapshotIndexBlobId[0] = "not-empty"; + } + return CompletableFuture.completedFuture("random-blob-id"); + }); + + // execute + blobStoreBackupManager.upload(checkpointId, testStoreNameAndSCMMap); + + // setup expected dir diffs after execute: needs checkpoint dirs created in upload() + TreeSet expectedDirDiffs = indexBlobIdAndLocalRemoteSnapshotsPair.values().stream() + .map(localRemoteSnapshotPair -> { + File localCheckpointDir = new File(localRemoteSnapshotPair.getLeft() + "-" + checkpointId.serialize()); + DirIndex dirIndex = new DirIndex(localCheckpointDir.getName(), Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), Collections.emptyList()); + return DirDiffUtil.getDirDiff(localCheckpointDir, dirIndex, DirDiffUtil.areSameFile(false)); + }).collect(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(DirDiff::getDirName)))); + + // assert - asset all DirDiff are put to blob store + Assert.assertEquals(actualDirDiffs, expectedDirDiffs); + // assert - assert no previous snapshot indexes were found + Assert.assertEquals(actualPreviousSnapshotIndexBlobId[0], expectedPreviousSnapshotIndexBlobId); + // assert - assert all snapshot indexes are uploaded + Assert.assertEquals(actualSnapshotIndexesUploaded, expectedSnapshotIndexesUploaded); + + // cleanup + checkpointDirsToClean.forEach(path -> { + try { + FileUtils.deleteDirectory(new File(path)); + } catch (IOException exception) { + Assert.fail("Failed to cleanup temporary checkpoint dirs."); + } + }); + } + + @Test + public void testUploadWithPreviousCheckpoints() throws IOException { + // Track directory for post cleanup + List checkpointDirsToClean = new ArrayList<>(); + + // Setup: init back manager with previous checkpoints + //indexBlobIdAndLocalRemoteSnapshotsPair = setupRemoteAndLocalSnapshots(true); + Map previousCheckpoints = + // map store name, previous snapshot index blob id + indexBlobIdAndLocalRemoteSnapshotsPair.entrySet().stream() + .collect(Collectors.toMap(e -> e.getValue().getLeft(), + e -> e.getValue().getRight().getPrevSnapshotIndexBlobId().get())); + + Checkpoint checkpoint = + new CheckpointV2(checkpointId, new HashMap<>(), + ImmutableMap.of(BlobStoreStateBackendFactory.class.getName(), previousCheckpoints)); + when(blobStoreUtil.getStoreSnapshotIndexes(anyString(), anyString(), anyString(), any(Checkpoint.class))).thenCallRealMethod(); + blobStoreBackupManager.init(checkpoint); + + // mock: set task store dir to return corresponding test local store and create checkpoint dir + ArgumentCaptor stringCaptor = ArgumentCaptor.forClass(String.class); + when(storageManagerUtil.getTaskStoreDir(any(File.class), stringCaptor.capture(), any(TaskName.class), any(TaskMode.class))) + .then((Answer) invocation -> { + String storeName = invocation.getArgumentAt(1, String.class); + String snapshotIndexBlobId = testStoreNameAndSCMMap.get(storeName); + String storeDir = indexBlobIdAndLocalRemoteSnapshotsPair.get(snapshotIndexBlobId).getLeft(); + try { // create test checkpoint dir + BlobStoreTestUtil.createTestCheckpointDirectory(storeDir, checkpointId.serialize()); + checkpointDirsToClean.add(storeDir + "-" + checkpointId.serialize()); + } catch (IOException e) { + Assert.fail("Couldn't create checkpoint directory. Test failed."); + } + return new File(storeDir); + }); + + ArgumentCaptor storeDirCaptor = ArgumentCaptor.forClass(File.class); + when(storageManagerUtil.getStoreCheckpointDir(storeDirCaptor.capture(), eq(checkpointId))) + .thenAnswer(new Answer() { + @Override + public String answer(InvocationOnMock invocation) throws Throwable { + File storeDir = invocation.getArgumentAt(0, File.class); + return storeDir.getAbsolutePath() + "-" + checkpointId.serialize(); + } + }); + + // mock: mock putDir and capture DirDiff + SortedSet actualDirDiffs = new TreeSet<>(Comparator.comparing(DirDiff::getDirName)); + ArgumentCaptor dirDiffCaptor = ArgumentCaptor.forClass(DirDiff.class); + ArgumentCaptor snapshotMetadataCaptor = ArgumentCaptor.forClass(SnapshotMetadata.class); + when(blobStoreUtil.putDir(dirDiffCaptor.capture(), snapshotMetadataCaptor.capture())) + .then((Answer>) invocation -> { + DirDiff dirDiff = invocation.getArgumentAt(0, DirDiff.class); + SnapshotMetadata snapshotMetadata = invocation.getArgumentAt(1, SnapshotMetadata.class); + actualDirDiffs.add(dirDiff); + SnapshotIndex snapshotIndex = testBlobStore.get(testStoreNameAndSCMMap.get(snapshotMetadata.getStoreName())); + return CompletableFuture.completedFuture(snapshotIndex.getDirIndex()); + }); + + // mock: mock putSnapshotIndex and capture previous snapshot index + SortedSet expectedSnapshotIndexesUploaded = indexBlobIdAndLocalRemoteSnapshotsPair.values().stream() + .map(Pair::getRight) + .collect(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(SnapshotIndex::getCreationTimeMillis)))); + SortedSet actualSnapshotIndexesUploaded = new TreeSet<>(Comparator.comparing(SnapshotIndex::getCreationTimeMillis)); + SortedSet actualPreviousSnapshotIndexBlobIds = new TreeSet<>(); + SortedSet expectedPreviousSnapshotIndexBlobIds = new TreeSet<>(previousCheckpoints.values()); + ArgumentCaptor snapshotIndexCaptor = ArgumentCaptor.forClass(SnapshotIndex.class); + when(blobStoreUtil.putSnapshotIndex(snapshotIndexCaptor.capture())) + .then((Answer>) invocation -> { + SnapshotIndex snapshotIndex = invocation.getArgumentAt(0, SnapshotIndex.class); + actualSnapshotIndexesUploaded.add(snapshotIndex); + if (snapshotIndex.getPrevSnapshotIndexBlobId().isPresent()) { + actualPreviousSnapshotIndexBlobIds.add(snapshotIndex.getPrevSnapshotIndexBlobId().get()); + } + return CompletableFuture.completedFuture("random-blob-id"); + }); + + // execute + blobStoreBackupManager.upload(checkpointId, ImmutableMap.of()); + + TreeSet expectedDirDiffs = indexBlobIdAndLocalRemoteSnapshotsPair.values() + .stream() + .map(localRemoteSnapshotPair -> + DirDiffUtil.getDirDiff(new File(localRemoteSnapshotPair.getLeft() + "-" + checkpointId.serialize()), + localRemoteSnapshotPair.getRight().getDirIndex(), DirDiffUtil.areSameFile(false))) + .collect(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(DirDiff::getDirName)))); + + // assert - asset all DirDiff are put to blob store + Assert.assertEquals(actualDirDiffs, expectedDirDiffs); + // assert - assert no previous snapshot indexes were found + Assert.assertEquals(actualPreviousSnapshotIndexBlobIds, expectedPreviousSnapshotIndexBlobIds); + // assert - assert all snapshot indexes are uploaded + Assert.assertEquals(actualSnapshotIndexesUploaded, expectedSnapshotIndexesUploaded); + + // cleanup + checkpointDirsToClean.forEach(path -> { + try { + FileUtils.deleteDirectory(new File(path)); + } catch (IOException exception) { + Assert.fail("Failed to cleanup temporary checkpoint dirs."); + } + }); + } + + + + @Test + public void testCleanupRemovesTTLForAllIndexBlobs() { + SortedSet actualRemoveTTLsResult = new TreeSet<>(testStoreNameAndSCMMap.values()); + + SortedSet expectedRemoveTTLsResult = new TreeSet<>(); + + // mock + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + when(blobStoreUtil.removeTTL(captor.capture(), any(SnapshotIndex.class), any(Metadata.class))) + .then((Answer>) invocation -> { + String blobId = invocation.getArgumentAt(0, String.class); + expectedRemoveTTLsResult.add(blobId); + return CompletableFuture.completedFuture(null); + }); + + // stub out non-tested methods + when(blobStoreUtil.cleanUpDir(any(DirIndex.class), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(null)); + when(blobStoreUtil.deleteSnapshotIndexBlob(any(String.class), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(null)); + + // execute + blobStoreBackupManager.cleanUp(checkpointId, testStoreNameAndSCMMap); + + // Assert + Assert.assertEquals(actualRemoveTTLsResult, expectedRemoveTTLsResult); + } + + @Test + public void testCleanupCleansUpRemoteSnapshot() throws Exception { + SortedSet actualCleanedupDirs = indexBlobIdAndLocalRemoteSnapshotsPair.values().stream() + .map(remoteLocalPair -> remoteLocalPair.getRight().getDirIndex()) + .collect(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(DirIndex::getDirName)))); + + SortedSet expectedCleanupDirs = new TreeSet<>(Comparator.comparing(DirIndex::getDirName)); + + // mock + ArgumentCaptor captor = ArgumentCaptor.forClass(DirIndex.class); + when(blobStoreUtil.cleanUpDir(captor.capture(), any(Metadata.class))) + .then((Answer>) invocation -> { + DirIndex dirIndex = invocation.getArgumentAt(0, DirIndex.class); + expectedCleanupDirs.add(dirIndex); + return CompletableFuture.completedFuture(null); + }); + + // stub out non-tested methods + when(blobStoreUtil.removeTTL(anyString(), any(SnapshotIndex.class), any(Metadata.class))) + .thenReturn(CompletableFuture.completedFuture(null)); + when(blobStoreUtil.deleteSnapshotIndexBlob(any(String.class), any(Metadata.class))) + .thenReturn(CompletableFuture.completedFuture(null)); + + // execute + blobStoreBackupManager.cleanUp(checkpointId, testStoreNameAndSCMMap); + + // Assert + Assert.assertEquals(actualCleanedupDirs, expectedCleanupDirs); + } + + @Test + public void testCleanupRemovesOldSnapshots() throws Exception { + TreeSet expectedOldSnapshotsRemoved = indexBlobIdAndLocalRemoteSnapshotsPair.values().stream() + .map(remoteLocalPair -> { + Optional prevSnapshotIndexBlobId = remoteLocalPair.getRight().getPrevSnapshotIndexBlobId(); + return prevSnapshotIndexBlobId.orElse(null); + }) + .collect(Collectors.toCollection(TreeSet::new)); + + SortedSet actualOldSnapshotsRemoved = new TreeSet<>(); + + // mock + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + when(blobStoreUtil.deleteSnapshotIndexBlob(captor.capture(), any(Metadata.class))) + .then((Answer>) invocation -> { + String prevIndexBlobId = invocation.getArgumentAt(0, String.class); + actualOldSnapshotsRemoved.add(prevIndexBlobId); + return CompletableFuture.completedFuture(null); + }); + + // stub out non-tested methods + when(blobStoreUtil.removeTTL(anyString(), any(SnapshotIndex.class), any(Metadata.class))) + .thenReturn(CompletableFuture.completedFuture(null)); + when(blobStoreUtil.cleanUpDir(any(DirIndex.class), any(Metadata.class))) + .thenReturn(CompletableFuture.completedFuture(null)); + + // execute + blobStoreBackupManager.cleanUp(checkpointId, testStoreNameAndSCMMap); + + // Assert + Assert.assertEquals(actualOldSnapshotsRemoved, expectedOldSnapshotsRemoved); + } + + @Test + public void testCleanupIgnoresStoresNotConfiguredWithBlobStoreStateBackend() throws Exception { + // TODO HIGH shesharm Complete test + } + + private Map setupTestStoreSCMMapAndStoreBackedFactoryConfig(Map> indexBlobIdAndRemoteAndLocalSnapshotMap) { + Map storeNameSCMMap = new HashMap<>(); + indexBlobIdAndRemoteAndLocalSnapshotMap + .forEach((blobId, localRemoteSnapshots) -> { + mapConfig.put("stores." + localRemoteSnapshots.getLeft() + ".factory", + BlobStoreStateBackendFactory.class.getName()); + mapConfig.put("stores." + localRemoteSnapshots.getLeft() + ".backup.factories", + BlobStoreStateBackendFactory.class.getName()); + storeNameSCMMap.put(localRemoteSnapshots.getLeft(), blobId); + }); + return storeNameSCMMap; + } + + private Map> setupRemoteAndLocalSnapshots(boolean addPrevCheckpoints) throws IOException { + testBlobStore = new HashMap<>(); // reset blob store + Map> indexBlobIdAndRemoteAndLocalSnapshotMap = new HashMap<>(); + List localSnapshots = new ArrayList<>(); + List previousRemoteSnapshots = new ArrayList<>(); + + localSnapshots.add("[a, c, z/1, y/2, p/m/3, q/n/4]"); + previousRemoteSnapshots.add("[a, b, z/1, x/5, p/m/3, r/o/6]"); + + localSnapshots.add("[a, c, z/1, y/1, p/m/1, q/n/1]"); + previousRemoteSnapshots.add("[a, z/1, p/m/1]"); + + localSnapshots.add("[z/i/1, y/j/1]"); + previousRemoteSnapshots.add("[z/i/1, x/k/1]"); + + // setup local and corresponding remote snapshots + for (int i = 0; i < localSnapshots.size(); i++) { + Path localSnapshot = BlobStoreTestUtil.createLocalDir(localSnapshots.get(i)); + String testLocalSnapshot = localSnapshot.toAbsolutePath().toString(); + DirIndex dirIndex = BlobStoreTestUtil.createDirIndex(localSnapshots.get(i)); + SnapshotMetadata snapshotMetadata = new SnapshotMetadata(checkpointId, jobName, jobId, taskName, testLocalSnapshot); + Optional prevCheckpointId = Optional.empty(); + if (addPrevCheckpoints) { + prevCheckpointId = Optional.of(prevSnapshotIndexBlobId + "-" + i); + DirIndex prevDirIndex = BlobStoreTestUtil.createDirIndex(previousRemoteSnapshots.get(i)); + testBlobStore.put(prevCheckpointId.get(), + new SnapshotIndex(clock.currentTimeMillis(), snapshotMetadata, prevDirIndex, Optional.empty())); + } + SnapshotIndex testRemoteSnapshot = + new SnapshotIndex(clock.currentTimeMillis(), snapshotMetadata, dirIndex, prevCheckpointId); + indexBlobIdAndRemoteAndLocalSnapshotMap.put("blobId-" + i, Pair.of(testLocalSnapshot, testRemoteSnapshot)); + testBlobStore.put("blobId-" + i, testRemoteSnapshot); + } + return indexBlobIdAndRemoteAndLocalSnapshotMap; + } + + private class MockBlobStoreBackupManager extends BlobStoreBackupManager { + + public MockBlobStoreBackupManager(JobModel jobModel, ContainerModel containerModel, TaskModel taskModel, + ExecutorService backupExecutor, BlobStoreBackupManagerMetrics blobStoreTaskBackupMetrics, Config config, + File loggedStoreBaseDir, StorageManagerUtil storageManagerUtil, + BlobStoreManager blobStoreManager) { + super(jobModel, containerModel, taskModel, backupExecutor, blobStoreTaskBackupMetrics, config, clock, + loggedStoreBaseDir, storageManagerUtil, blobStoreManager); + } + + @Override + protected BlobStoreUtil createBlobStoreUtil(BlobStoreManager blobStoreManager, ExecutorService executor, + BlobStoreBackupManagerMetrics metrics) { + return blobStoreUtil; + } + } +} diff --git a/samza-core/src/test/java/org/apache/samza/storage/blobstore/TestBlobStoreRestoreManager.java b/samza-core/src/test/java/org/apache/samza/storage/blobstore/TestBlobStoreRestoreManager.java new file mode 100644 index 0000000000..ddc0c8e19a --- /dev/null +++ b/samza-core/src/test/java/org/apache/samza/storage/blobstore/TestBlobStoreRestoreManager.java @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.MoreExecutors; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.config.BlobStoreConfig; +import org.apache.samza.config.StorageConfig; +import org.apache.samza.container.TaskName; +import org.apache.samza.job.model.TaskMode; +import org.apache.samza.metrics.MetricsRegistryMap; +import org.apache.samza.storage.StorageManagerUtil; +import org.apache.samza.storage.blobstore.index.DirIndex; +import org.apache.samza.storage.blobstore.index.SnapshotIndex; +import org.apache.samza.storage.blobstore.index.SnapshotMetadata; +import org.apache.samza.storage.blobstore.metrics.BlobStoreRestoreManagerMetrics; +import org.apache.samza.storage.blobstore.util.BlobStoreTestUtil; +import org.apache.samza.storage.blobstore.util.BlobStoreUtil; +import org.apache.samza.storage.blobstore.util.DirDiffUtil; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.anySet; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + + +public class TestBlobStoreRestoreManager { + private static final ExecutorService EXECUTOR = MoreExecutors.newDirectExecutorService(); + + @Test + public void testDeleteUnusedStoresRemovesStoresDeletedFromConfig() { + String jobName = "testJobName"; + String jobId = "testJobId"; + String taskName = "taskName"; + StorageConfig storageConfig = mock(StorageConfig.class); + BlobStoreConfig blobStoreConfig = mock(BlobStoreConfig.class); + SnapshotIndex mockSnapshotIndex = mock(SnapshotIndex.class); + String blobId = "blobId"; + Map> initialStoreSnapshotIndexes = + ImmutableMap.of("oldStoreName", Pair.of(blobId, mockSnapshotIndex)); + + when(storageConfig.getStoresWithBackupFactory(eq(BlobStoreStateBackendFactory.class.getName()))) + .thenReturn(ImmutableList.of("newStoreName")); + when(storageConfig.getStoresWithRestoreFactory(eq(BlobStoreStateBackendFactory.class.getName()))) + .thenReturn(ImmutableList.of("newStoreName")); + + DirIndex dirIndex = mock(DirIndex.class); + when(mockSnapshotIndex.getDirIndex()).thenReturn(dirIndex); + + BlobStoreUtil blobStoreUtil = mock(BlobStoreUtil.class); + when(blobStoreUtil.cleanUpDir(any(DirIndex.class), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(null)); + when(blobStoreUtil.deleteDir(any(DirIndex.class), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(null)); + when(blobStoreUtil.deleteSnapshotIndexBlob(anyString(), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(null)); + + BlobStoreRestoreManager.deleteUnusedStoresFromBlobStore( + jobName, jobId, taskName, storageConfig, blobStoreConfig, initialStoreSnapshotIndexes, blobStoreUtil, EXECUTOR); + + verify(blobStoreUtil, times(1)).cleanUpDir(eq(dirIndex), any(Metadata.class)); + verify(blobStoreUtil, times(1)).deleteDir(eq(dirIndex), any(Metadata.class)); + verify(blobStoreUtil, times(1)).deleteSnapshotIndexBlob(eq(blobId), any(Metadata.class)); + + } + + @Test + public void testShouldRestoreIfNoCheckpointDir() throws IOException { + String taskName = "taskName"; + String storeName = "storeName"; + DirIndex dirIndex = mock(DirIndex.class); + Path storeCheckpointDir = Paths.get("/tmp/non-existent-checkpoint-dir"); + StorageConfig storageConfig = mock(StorageConfig.class); + when(storageConfig.cleanLoggedStoreDirsOnStart(anyString())).thenReturn(false); + DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class); + + boolean shouldRestore = BlobStoreRestoreManager.shouldRestore( + taskName, storeName, dirIndex, storeCheckpointDir, storageConfig, dirDiffUtil); + + verifyZeroInteractions(dirDiffUtil); + assertTrue(shouldRestore); + } + + @Test + public void testShouldRestoreIfCleanStateOnRestartEnabled() throws IOException { + String taskName = "taskName"; + String storeName = "storeName"; + DirIndex dirIndex = mock(DirIndex.class); + Path storeCheckpointDir = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX); // must exist + StorageConfig storageConfig = mock(StorageConfig.class); + when(storageConfig.cleanLoggedStoreDirsOnStart(anyString())).thenReturn(true); // clean on restart + DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class); + + boolean shouldRestore = BlobStoreRestoreManager.shouldRestore( + taskName, storeName, dirIndex, storeCheckpointDir, storageConfig, dirDiffUtil); + + verifyZeroInteractions(dirDiffUtil); + assertTrue(shouldRestore); // should not restore, should retain checkpoint dir instead + } + + @Test + public void testShouldRestoreIfCheckpointDirNotIdenticalToRemoteSnapshot() throws IOException { + String taskName = "taskName"; + String storeName = "storeName"; + DirIndex dirIndex = mock(DirIndex.class); + Path storeCheckpointDir = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX); // must exist + StorageConfig storageConfig = mock(StorageConfig.class); + when(storageConfig.cleanLoggedStoreDirsOnStart(anyString())).thenReturn(false); + DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class); + when(dirDiffUtil.areSameDir(anySet(), anyBoolean())).thenReturn((arg1, arg2) -> false); + + boolean shouldRestore = BlobStoreRestoreManager.shouldRestore( + taskName, storeName, dirIndex, storeCheckpointDir, storageConfig, dirDiffUtil); + + assertTrue(shouldRestore); + } + + @Test + public void testShouldNotRestoreIfPreviousCheckpointDirIdenticalToRemoteSnapshot() throws IOException { + String taskName = "taskName"; + String storeName = "storeName"; + DirIndex dirIndex = mock(DirIndex.class); + Path storeCheckpointDir = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX); // must exist + StorageConfig storageConfig = mock(StorageConfig.class); + when(storageConfig.cleanLoggedStoreDirsOnStart(anyString())).thenReturn(false); + DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class); + when(dirDiffUtil.areSameDir(anySet(), anyBoolean())).thenReturn((arg1, arg2) -> true); // are same dir + + boolean shouldRestore = BlobStoreRestoreManager.shouldRestore( + taskName, storeName, dirIndex, storeCheckpointDir, storageConfig, dirDiffUtil); + + verify(dirDiffUtil, times(1)).areSameDir(anySet(), anyBoolean()); + assertFalse(shouldRestore); // should not restore, should retain checkpoint dir instead + } + + @Test + public void testRestoreDeletesStoreDir() throws IOException { + String jobName = "testJobName"; + String jobId = "testJobId"; + TaskName taskName = mock(TaskName.class); + BlobStoreRestoreManagerMetrics metrics = new BlobStoreRestoreManagerMetrics(new MetricsRegistryMap()); + metrics.initStoreMetrics(ImmutableList.of("storeName")); + Set storesToRestore = ImmutableSet.of("storeName"); + SnapshotIndex snapshotIndex = mock(SnapshotIndex.class); + Map> prevStoreSnapshotIndexes = + ImmutableMap.of("storeName", Pair.of("blobId", snapshotIndex)); + DirIndex dirIndex = BlobStoreTestUtil.createDirIndex("[a]"); + when(snapshotIndex.getDirIndex()).thenReturn(dirIndex); + when(snapshotIndex.getSnapshotMetadata()) + .thenReturn(new SnapshotMetadata(CheckpointId.create(), "jobName", "jobId", "taskName", "storeName")); + + Path loggedBaseDir = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX); + + // create store dir to be deleted during restore + Path storeDir = Files.createTempDirectory(loggedBaseDir, "storeDir"); + StorageConfig storageConfig = mock(StorageConfig.class); + StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class); + when(storageManagerUtil.getStoreCheckpointDir(any(File.class), any(CheckpointId.class))) + .thenReturn(Paths.get(storeDir.toString(), "checkpointId").toString()); + when(storageManagerUtil.getTaskStoreDir( + eq(loggedBaseDir.toFile()), eq("storeName"), eq(taskName), eq(TaskMode.Active))) + .thenReturn(storeDir.toFile()); + BlobStoreUtil blobStoreUtil = mock(BlobStoreUtil.class); + DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class); + + // return immediately without restoring. + when(blobStoreUtil.restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class))) + .thenReturn(CompletableFuture.completedFuture(null)); + when(dirDiffUtil.areSameDir(anySet(), anyBoolean())).thenReturn((arg1, arg2) -> true); + + BlobStoreRestoreManager.restoreStores(jobName, jobId, taskName, storesToRestore, prevStoreSnapshotIndexes, + loggedBaseDir.toFile(), storageConfig, metrics, + storageManagerUtil, blobStoreUtil, dirDiffUtil, EXECUTOR); + + // verify that the store directory restore was called and skipped (i.e. shouldRestore == true) + verify(blobStoreUtil, times(1)).restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class)); + // verify that the store directory was deleted prior to restore + // (should still not exist at the end since restore is no-op) + assertFalse(storeDir.toFile().exists()); + } + + @Test + public void testRestoreDeletesCheckpointDirsIfRestoring() throws IOException { + String jobName = "testJobName"; + String jobId = "testJobId"; + TaskName taskName = mock(TaskName.class); + BlobStoreRestoreManagerMetrics metrics = new BlobStoreRestoreManagerMetrics(new MetricsRegistryMap()); + metrics.initStoreMetrics(ImmutableList.of("storeName")); + Set storesToRestore = ImmutableSet.of("storeName"); + SnapshotIndex snapshotIndex = mock(SnapshotIndex.class); + Map> prevStoreSnapshotIndexes = + ImmutableMap.of("storeName", Pair.of("blobId", snapshotIndex)); + DirIndex dirIndex = BlobStoreTestUtil.createDirIndex("[a]"); + when(snapshotIndex.getDirIndex()).thenReturn(dirIndex); + CheckpointId checkpointId = CheckpointId.create(); + when(snapshotIndex.getSnapshotMetadata()) + .thenReturn(new SnapshotMetadata(checkpointId, "jobName", "jobId", "taskName", "storeName")); + + Path loggedBaseDir = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX); + + // create store dir to be deleted during restore + Path storeDir = Files.createTempDirectory(loggedBaseDir, "storeDir"); + Path storeCheckpointDir1 = Files.createTempDirectory(loggedBaseDir, "storeDir-" + checkpointId); + CheckpointId olderCheckpoint = CheckpointId.create(); + Path storeCheckpointDir2 = Files.createTempDirectory(loggedBaseDir, "storeDir-" + olderCheckpoint); + StorageConfig storageConfig = mock(StorageConfig.class); + StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class); + when(storageManagerUtil.getTaskStoreDir( + eq(loggedBaseDir.toFile()), eq("storeName"), eq(taskName), eq(TaskMode.Active))) + .thenReturn(storeDir.toFile()); + when(storageManagerUtil.getStoreCheckpointDir(eq(storeDir.toFile()), eq(checkpointId))) + .thenReturn(Paths.get(storeDir.toString(), checkpointId.toString()).toString()); + when(storageManagerUtil.getTaskStoreCheckpointDirs(any(File.class), anyString(), any(TaskName.class), any(TaskMode.class))) + .thenReturn(ImmutableList.of(storeCheckpointDir1.toFile(), storeCheckpointDir2.toFile())); + BlobStoreUtil blobStoreUtil = mock(BlobStoreUtil.class); + DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class); + + when(dirDiffUtil.areSameDir(anySet(), anyBoolean())).thenReturn((arg1, arg2) -> true); + // return immediately without restoring. + when(blobStoreUtil.restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class))) + .thenReturn(CompletableFuture.completedFuture(null)); + + BlobStoreRestoreManager.restoreStores(jobName, jobId, taskName, storesToRestore, prevStoreSnapshotIndexes, + loggedBaseDir.toFile(), storageConfig, metrics, + storageManagerUtil, blobStoreUtil, dirDiffUtil, EXECUTOR); + + // verify that the store directory restore was called and skipped (i.e. shouldRestore == true) + verify(blobStoreUtil, times(1)).restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class)); + // verify that the checkpoint directories were deleted prior to restore (should not exist at the end) + assertFalse(storeCheckpointDir1.toFile().exists()); + assertFalse(storeCheckpointDir2.toFile().exists()); + } + + @Test + public void testRestoreRetainsCheckpointDirsIfValid() throws IOException { + String jobName = "testJobName"; + String jobId = "testJobId"; + TaskName taskName = mock(TaskName.class); + BlobStoreRestoreManagerMetrics metrics = new BlobStoreRestoreManagerMetrics(new MetricsRegistryMap()); + metrics.initStoreMetrics(ImmutableList.of("storeName")); + Set storesToRestore = ImmutableSet.of("storeName"); + SnapshotIndex snapshotIndex = mock(SnapshotIndex.class); + Map> prevStoreSnapshotIndexes = + ImmutableMap.of("storeName", Pair.of("blobId", snapshotIndex)); + DirIndex dirIndex = BlobStoreTestUtil.createDirIndex("[a]"); + when(snapshotIndex.getDirIndex()).thenReturn(dirIndex); + CheckpointId checkpointId = CheckpointId.create(); + when(snapshotIndex.getSnapshotMetadata()) + .thenReturn(new SnapshotMetadata(checkpointId, "jobName", "jobId", "taskName", "storeName")); + + Path loggedBaseDir = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX); + + // create store dir to be deleted during restore + Path storeDir = Files.createTempDirectory(loggedBaseDir, "storeDir-"); + + // create checkpoint dir so that shouldRestore = false (areSameDir == true later) + Path storeCheckpointDir = Files.createTempDirectory(loggedBaseDir, "storeDir-" + checkpointId + "-"); + // create a dummy file to verify after dir rename. + Path tempFile = Files.createTempFile(storeCheckpointDir, "tempFile-", null); + + StorageConfig storageConfig = mock(StorageConfig.class); + StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class); + when(storageManagerUtil.getTaskStoreDir( + eq(loggedBaseDir.toFile()), eq("storeName"), eq(taskName), eq(TaskMode.Active))) + .thenReturn(storeDir.toFile()); + when(storageManagerUtil.getStoreCheckpointDir(any(File.class), eq(checkpointId))) + .thenReturn(storeCheckpointDir.toString()); + when(storageManagerUtil.getTaskStoreCheckpointDirs(any(File.class), anyString(), any(TaskName.class), any(TaskMode.class))) + .thenReturn(ImmutableList.of(storeCheckpointDir.toFile())); + BlobStoreUtil blobStoreUtil = mock(BlobStoreUtil.class); + DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class); + + // ensures shouldRestore is not called + when(dirDiffUtil.areSameDir(anySet(), anyBoolean())).thenReturn((arg1, arg2) -> true); + // return immediately without restoring. + when(blobStoreUtil.restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class))) + .thenReturn(CompletableFuture.completedFuture(null)); + + BlobStoreRestoreManager.restoreStores(jobName, jobId, taskName, storesToRestore, prevStoreSnapshotIndexes, + loggedBaseDir.toFile(), storageConfig, metrics, + storageManagerUtil, blobStoreUtil, dirDiffUtil, EXECUTOR); + + // verify that the store directory restore was not called (should have restored from checkpoint dir) + verify(blobStoreUtil, times(0)).restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class)); + // verify that the checkpoint dir was renamed to store dir + assertFalse(storeCheckpointDir.toFile().exists()); + assertTrue(storeDir.toFile().exists()); + assertTrue(Files.exists(Paths.get(storeDir.toString(), tempFile.getFileName().toString()))); + } + + @Test + public void testRestoreSkipsStoresWithMissingCheckpointSCM() { + // store renamed from oldStoreName to newStoreName. No SCM for newStoreName in previous checkpoint. + String jobName = "testJobName"; + String jobId = "testJobId"; + TaskName taskName = mock(TaskName.class); + BlobStoreRestoreManagerMetrics metrics = new BlobStoreRestoreManagerMetrics(new MetricsRegistryMap()); + metrics.initStoreMetrics(ImmutableList.of("newStoreName")); + Set storesToRestore = ImmutableSet.of("newStoreName"); // new store in config + SnapshotIndex snapshotIndex = mock(SnapshotIndex.class); + Map> prevStoreSnapshotIndexes = mock(Map.class); + when(prevStoreSnapshotIndexes.containsKey("newStoreName")).thenReturn(false); + DirIndex dirIndex = mock(DirIndex.class); + when(snapshotIndex.getDirIndex()).thenReturn(dirIndex); + CheckpointId checkpointId = CheckpointId.create(); + when(snapshotIndex.getSnapshotMetadata()) + .thenReturn(new SnapshotMetadata(checkpointId, "jobName", "jobId", "taskName", "storeName")); + Path loggedBaseDir = mock(Path.class); + + // create store dir to be deleted during restore + StorageConfig storageConfig = mock(StorageConfig.class); + StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class); + BlobStoreUtil blobStoreUtil = mock(BlobStoreUtil.class); + DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class); + + BlobStoreRestoreManager.restoreStores(jobName, jobId, taskName, storesToRestore, prevStoreSnapshotIndexes, + loggedBaseDir.toFile(), storageConfig, metrics, + storageManagerUtil, blobStoreUtil, dirDiffUtil, EXECUTOR); + + // verify that we checked the previously checkpointed SCMs. + verify(prevStoreSnapshotIndexes, times(1)).containsKey(eq("newStoreName")); + // verify that the store directory restore was never called + verify(blobStoreUtil, times(0)).restoreDir(any(File.class), any(DirIndex.class), any(Metadata.class)); + } +} diff --git a/samza-core/src/test/java/org/apache/samza/storage/blobstore/serde/TestSnapshotIndexSerde.java b/samza-core/src/test/java/org/apache/samza/storage/blobstore/serde/TestSnapshotIndexSerde.java new file mode 100644 index 0000000000..8103a5bb34 --- /dev/null +++ b/samza-core/src/test/java/org/apache/samza/storage/blobstore/serde/TestSnapshotIndexSerde.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.serde; + +import org.apache.samza.storage.blobstore.index.DirIndex; +import org.apache.samza.storage.blobstore.index.SnapshotIndex; +import org.apache.samza.storage.blobstore.index.SnapshotMetadata; +import org.apache.samza.storage.blobstore.index.serde.SnapshotIndexSerde; +import org.apache.samza.storage.blobstore.util.BlobStoreTestUtil; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Optional; +import org.apache.samza.checkpoint.CheckpointId; +import org.junit.Assert; +import org.junit.Test; + + +public class TestSnapshotIndexSerde { + @Test + public void testSnapshotIndexSerde() throws IOException { + // create local and remote snapshots + String local = "[a, b, c/1, d/1/2]"; + String remote = "[a, b, z, c/1/2, e/1]"; + + Path localSnapshot = BlobStoreTestUtil.createLocalDir(local); + DirIndex dirIndex = BlobStoreTestUtil.createDirIndex(remote); + SnapshotMetadata snapshotMetadata = + new SnapshotMetadata(CheckpointId.create(), "job", "123", "task", "store"); + SnapshotIndex testRemoteSnapshot = + new SnapshotIndex(System.currentTimeMillis(), snapshotMetadata, dirIndex, Optional.empty()); + + SnapshotIndexSerde snapshotIndexSerde = new SnapshotIndexSerde(); + byte[] serialized = snapshotIndexSerde.toBytes(testRemoteSnapshot); + SnapshotIndex deserialized = snapshotIndexSerde.fromBytes(serialized); + + Assert.assertNotNull(deserialized); + Assert.assertEquals(deserialized, testRemoteSnapshot); + } +} diff --git a/samza-core/src/test/java/org/apache/samza/storage/blobstore/util/BlobStoreTestUtil.java b/samza-core/src/test/java/org/apache/samza/storage/blobstore/util/BlobStoreTestUtil.java new file mode 100644 index 0000000000..2d72451951 --- /dev/null +++ b/samza-core/src/test/java/org/apache/samza/storage/blobstore/util/BlobStoreTestUtil.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.util; + +import com.google.common.collect.ImmutableList; +import org.apache.samza.storage.blobstore.diff.DirDiff; +import org.apache.samza.storage.blobstore.index.DirIndex; +import org.apache.samza.storage.blobstore.index.FileBlob; +import org.apache.samza.storage.blobstore.index.FileIndex; +import org.apache.samza.storage.blobstore.index.FileMetadata; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.zip.CRC32; +import java.util.zip.Checksum; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.samza.util.FileUtil; + +/** + * Test util methods to work with local dirs, {@link DirDiff}s and {@link DirIndex}es. + */ +public class BlobStoreTestUtil { + public static final String TEMP_DIR_PREFIX = "samza-blob-store-test-"; + + public static Path createLocalDir(String files) throws IOException { + Path tempDirPath = Files.createTempDirectory(TEMP_DIR_PREFIX); + File tempDirFile = tempDirPath.toFile(); + String tempDirPathString = tempDirPath.toAbsolutePath().toString(); + + if (files.length() == 2) return tempDirPath; + String[] paths = files.substring(1, files.length() - 1).split(","); + + for (String path: paths) { + path = path.trim(); + if (!path.contains("/")) { + Path filePath = Files.createFile(Paths.get(tempDirPathString, path)); + new FileUtil().writeToTextFile(filePath.toFile(), path, false); // file contents == file name + filePath.toFile().deleteOnExit(); + } else { + String dirs = path.substring(0, path.lastIndexOf("/")); + String file = path.substring(path.lastIndexOf("/") + 1); + Path directories = Files.createDirectories(Paths.get(tempDirPathString, dirs)); + if (!StringUtils.isBlank(file)) { // can be blank for empty directories + Path filePath = Paths.get(directories.toAbsolutePath().toString(), file); + Files.createFile(filePath); + new FileUtil().writeToTextFile(filePath.toFile(), file, false); // file contents == file name + filePath.toFile().deleteOnExit(); + } + } + } + + deleteDirRecursivelyOnExit(tempDirFile); + return tempDirPath; + } + + public static DirIndex createDirIndex(String files) throws IOException { + if (files.equals("[]")) { // empty dir + return new DirIndex(DirIndex.ROOT_DIR_NAME, Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), Collections.emptyList()); + } + + String[] paths = files.substring(1, files.length() - 1).split(","); + Arrays.sort(paths); + // actually create the directory structure in a temp dir so that file properties and checksums can be computed + Path localDir = createLocalDir(files); + DirTreeNode dirTree = createDirTree(localDir.toAbsolutePath().toString(), paths); + return createDirIndex(localDir.toAbsolutePath().toString(), dirTree); + } + + public static void getAllAddedInDiff(String basePath, DirDiff dirDiff, Set allAdded) { + for (File fileAdded: dirDiff.getFilesAdded()) { + allAdded.add(fileAdded.getAbsolutePath().substring(basePath.length() + 1)); + } + + for (DirDiff dirAdded: dirDiff.getSubDirsAdded()) { + getAllAddedInDiff(basePath, dirAdded, allAdded); + } + + for (DirDiff dirRetained: dirDiff.getSubDirsRetained()) { + getAllAddedInDiff(basePath, dirRetained, allAdded); + } + } + + public static void getAllRemovedInDiff(String basePath, DirDiff dirDiff, Set allRemoved) { + String prefix = basePath.isEmpty() ? basePath : basePath + "/"; + for (FileIndex fileRemoved: dirDiff.getFilesRemoved()) { + allRemoved.add(prefix + fileRemoved.getFileName()); + } + + for (DirIndex dirRemoved: dirDiff.getSubDirsRemoved()) { + getAllRemovedInRemovedSubDir(prefix + dirRemoved.getDirName(), dirRemoved, allRemoved); + } + + for (DirDiff dirRetained: dirDiff.getSubDirsRetained()) { + getAllRemovedInRetainedSubDir(prefix + dirRetained.getDirName(), dirRetained, allRemoved); + } + } + + public static void getAllRetainedInDiff(String basePath, DirDiff dirDiff, Set allRetained) { + String prefix = basePath.isEmpty() ? basePath : basePath + "/"; + for (FileIndex fileRetained: dirDiff.getFilesRetained()) { + allRetained.add(prefix + fileRetained.getFileName()); + } + + for (DirDiff dirRetained: dirDiff.getSubDirsRetained()) { + getAllRetainedInDiff(prefix + dirRetained.getDirName(), dirRetained, allRetained); + } + } + + public static void getAllPresentInIndex(String basePath, DirIndex dirIndex, Set allPresent) { + String prefix = basePath.isEmpty() ? basePath : basePath + "/"; + + for (FileIndex filePresent: dirIndex.getFilesPresent()) { + allPresent.add(prefix + filePresent.getFileName()); + } + + for (DirIndex dirPresent: dirIndex.getSubDirsPresent()) { + getAllPresentInIndex(prefix + dirPresent.getDirName(), dirPresent, allPresent); + } + } + + public static void getAllRemovedInIndex(String basePath, DirIndex dirIndex, Set allRemoved) { + String prefix = basePath.isEmpty() ? basePath : basePath + "/"; + + for (FileIndex fileRemoved: dirIndex.getFilesRemoved()) { + allRemoved.add(prefix + fileRemoved.getFileName()); + } + + for (DirIndex dirRemoved: dirIndex.getSubDirsRemoved()) { + getAllRemovedInRemovedSubDir(prefix + dirRemoved.getDirName(), dirRemoved, allRemoved); + } + + for (DirIndex dirPresent: dirIndex.getSubDirsPresent()) { + getAllRemovedInIndex(prefix + dirPresent.getDirName(), dirPresent, allRemoved); + } + } + + public static SortedSet getExpected(String expectedFiles) { + if (expectedFiles.length() == 2) return new TreeSet<>(); + String[] paths = expectedFiles.substring(1, expectedFiles.length() - 1).split(","); + SortedSet result = new TreeSet<>(); + for (String path: paths) { + result.add(path.trim()); + } + return result; + } + + private static DirIndex createDirIndex(String baseDir, DirTreeNode root) { + String dirName = root.fileName; + List filesPresent = new ArrayList<>(); + List subDirsPresent = new ArrayList<>(); + + List filesRemoved = Collections.emptyList(); + List subDirsRemoved = Collections.emptyList(); + + for (DirTreeNode child: root.children) { + if (!child.children.isEmpty()) { + subDirsPresent.add(createDirIndex(baseDir + "/" + child.fileName, child)); + } else { + filesPresent.add(createFileIndex(baseDir + "/" + child.fileName, child)); + } + } + + return new DirIndex(dirName, filesPresent, filesRemoved, subDirsPresent, subDirsRemoved); + } + + private static DirTreeNode createDirTree(String baseDir, String[] paths) { + DirTreeNode root = new DirTreeNode(); + root.fileName = DirIndex.ROOT_DIR_NAME; + + for (String path: paths) { + DirTreeNode pathRoot = root; + path = path.trim(); + String[] pathParts = path.split("/"); + for (String pathPart: pathParts) { + DirTreeNode childNode; + + Optional childNodeOptional = pathRoot.children.stream() + .filter(dtn -> dtn.fileName.equals(pathPart)).findFirst(); + + if (childNodeOptional.isPresent()) { + childNode = childNodeOptional.get(); + } else { + childNode = new DirTreeNode(); + childNode.fileName = pathPart; + pathRoot.children.add(childNode); + } + pathRoot = childNode; + } + } + + return root; + } + + public static File createTestCheckpointDirectory(String storePath, String checkpointId) throws IOException { + File checkpointDir = new File(storePath + "-" + checkpointId); + FileUtils.copyDirectory(new File(storePath), checkpointDir); + return checkpointDir; + } + + private static FileIndex createFileIndex(String filePath, DirTreeNode node) { + long checksum; + FileMetadata fileMetadata; + try { + Path path = Paths.get(filePath); + Checksum crc32 = new CRC32(); + byte[] fileBytes = Files.readAllBytes(path); + crc32.update(fileBytes, 0, fileBytes.length); + checksum = crc32.getValue(); + fileMetadata = FileMetadata.fromFile(path.toFile()); + } catch (Exception e) { + throw new RuntimeException(e); + } + return new FileIndex(node.fileName, ImmutableList.of(new FileBlob(node.fileName, 0)), fileMetadata, checksum); + } + + // recursively adds all present files to allRemoved in removed subdirs + private static void getAllRemovedInRemovedSubDir(String basePath, DirIndex dirIndex, Set allRemoved) { + String prefix = basePath.isEmpty() ? basePath : basePath + "/"; + for (FileIndex f: dirIndex.getFilesPresent()) { + allRemoved.add(prefix + f.getFileName()); + } + + for (DirIndex d: dirIndex.getSubDirsPresent()) { + getAllRemovedInRemovedSubDir(prefix + d.getDirName(), d, allRemoved); + } + } + + // only adds removed files in retained subdirs + private static void getAllRemovedInRetainedSubDir(String basePath, DirDiff dirDiff, Set allRemoved) { + String prefix = basePath.isEmpty() ? basePath : basePath + "/"; + for (FileIndex f: dirDiff.getFilesRemoved()) { + allRemoved.add(prefix + f.getFileName()); + } + + for (DirIndex dirRemoved: dirDiff.getSubDirsRemoved()) { + getAllRemovedInRemovedSubDir(prefix + dirRemoved.getDirName(), dirRemoved, allRemoved); + } + + for (DirDiff dirRetained: dirDiff.getSubDirsRetained()) { + getAllRemovedInRetainedSubDir(prefix + dirRetained.getDirName(), dirRetained, allRemoved); + } + } + + private static class DirTreeNode { + String fileName; + Set children = new HashSet<>(); + } + + private static class MockFileMetadata extends FileMetadata { + public MockFileMetadata() { + super(0, 0, 0, "owner", "group", "rwxrw-r--"); + } + } + + private static void deleteDirRecursivelyOnExit(File dir) { + dir.deleteOnExit(); + for (File f: dir.listFiles()) { + if (f.isDirectory()) { + deleteDirRecursivelyOnExit(f); + } else { + f.deleteOnExit(); + } + } + } +} diff --git a/samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestBlobStoreUtil.java b/samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestBlobStoreUtil.java new file mode 100644 index 0000000000..b22827e0a1 --- /dev/null +++ b/samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestBlobStoreUtil.java @@ -0,0 +1,936 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.util; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.MoreExecutors; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.zip.CRC32; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.NullOutputStream; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.samza.SamzaException; +import org.apache.samza.checkpoint.Checkpoint; +import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.checkpoint.CheckpointV2; +import org.apache.samza.storage.blobstore.BlobStoreManager; +import org.apache.samza.storage.blobstore.BlobStoreStateBackendFactory; +import org.apache.samza.storage.blobstore.Metadata; +import org.apache.samza.storage.blobstore.diff.DirDiff; +import org.apache.samza.storage.blobstore.exceptions.DeletedException; +import org.apache.samza.storage.blobstore.exceptions.RetriableException; +import org.apache.samza.storage.blobstore.index.DirIndex; +import org.apache.samza.storage.blobstore.index.FileBlob; +import org.apache.samza.storage.blobstore.index.FileIndex; +import org.apache.samza.storage.blobstore.index.FileMetadata; +import org.apache.samza.storage.blobstore.index.SnapshotIndex; +import org.apache.samza.storage.blobstore.index.SnapshotMetadata; +import org.apache.samza.util.FileUtil; +import org.apache.samza.util.FutureUtil; +import org.junit.Ignore; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.stubbing.Answer; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.*; +import static org.mockito.Mockito.*; + + +public class TestBlobStoreUtil { + private static final ExecutorService EXECUTOR = MoreExecutors.newDirectExecutorService(); + private final CheckpointId checkpointId = CheckpointId.deserialize("1234-567"); + private final String jobName = "jobName"; + private final String jobId = "jobId"; + private final String taskName = "taskName"; + private final String storeName = "storeName"; + private final Metadata metadata = + new Metadata("payload-path", Optional.empty(), jobName, jobId, taskName, storeName); + + @Test + // TODO HIGH shesharm test with empty (0 byte) files + public void testPutDir() throws IOException, InterruptedException, ExecutionException { + BlobStoreManager blobStoreManager = mock(BlobStoreManager.class); + + // File, dir and recursive dir added, retained and removed in local + String local = "[a, c, z/1, y/1, p/m/1, q/n/1]"; + String remote = "[a, b, z/1, x/1, p/m/1, p/m/2, r/o/1]"; + String expectedAdded = "[c, y/1, q/n/1]"; + String expectedRetained = "[a, z/1, p/m/1]"; + String expectedRemoved = "[b, x/1, r/o/1, p/m/2]"; + SortedSet expectedAddedFiles = BlobStoreTestUtil.getExpected(expectedAdded); + SortedSet expectedRetainedFiles = BlobStoreTestUtil.getExpected(expectedRetained); + SortedSet expectedPresentFiles = new TreeSet<>(expectedAddedFiles); + expectedPresentFiles.addAll(expectedRetainedFiles); + SortedSet expectedRemovedFiles = BlobStoreTestUtil.getExpected(expectedRemoved); + + // Set up environment + Path localSnapshotDir = BlobStoreTestUtil.createLocalDir(local); + String basePath = localSnapshotDir.toAbsolutePath().toString(); + DirIndex remoteSnapshotDir = BlobStoreTestUtil.createDirIndex(remote); + SnapshotMetadata snapshotMetadata = new SnapshotMetadata(checkpointId, jobName, jobId, taskName, storeName); + DirDiff dirDiff = DirDiffUtil.getDirDiff(localSnapshotDir.toFile(), remoteSnapshotDir, + (localFile, remoteFile) -> localFile.getName().equals(remoteFile.getFileName())); + + SortedSet allUploaded = new TreeSet<>(); + // Set up mocks + when(blobStoreManager.put(any(InputStream.class), any(Metadata.class))) + .thenAnswer((Answer>) invocation -> { + Metadata metadata = invocation.getArgumentAt(1, Metadata.class); + String path = metadata.getPayloadPath(); + allUploaded.add(path.substring(localSnapshotDir.toAbsolutePath().toString().length() + 1)); + return CompletableFuture.completedFuture(path); + }); + + // Execute + BlobStoreUtil blobStoreUtil = new BlobStoreUtil(blobStoreManager, EXECUTOR, null, null); + CompletionStage dirIndexFuture = blobStoreUtil.putDir(dirDiff, snapshotMetadata); + DirIndex dirIndex = null; + try { + // should be already complete. if not, future composition in putDir is broken. + dirIndex = dirIndexFuture.toCompletableFuture().get(0, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + fail("Future returned from putDir should be already complete."); + } + + SortedSet allPresent = new TreeSet<>(); + SortedSet allRemoved = new TreeSet<>(); + BlobStoreTestUtil.getAllPresentInIndex("", dirIndex, allPresent); + BlobStoreTestUtil.getAllRemovedInIndex("", dirIndex, allRemoved); + + // Assert + assertEquals(expectedAddedFiles, allUploaded); + assertEquals(expectedPresentFiles, allPresent); + assertEquals(expectedRemovedFiles, allRemoved); + } + + @Test + public void testPutDirFailsIfAnyFileUploadFails() throws IOException, TimeoutException, InterruptedException { + BlobStoreManager blobStoreManager = mock(BlobStoreManager.class); + + // File, dir and recursive dir added, retained and removed in local + String local = "[a, b]"; + String remote = "[]"; + + // Set up environment + Path localSnapshotDir = BlobStoreTestUtil.createLocalDir(local); + String basePath = localSnapshotDir.toAbsolutePath().toString(); + DirIndex remoteSnapshotDir = BlobStoreTestUtil.createDirIndex(remote); + SnapshotMetadata snapshotMetadata = new SnapshotMetadata(checkpointId, jobName, jobId, taskName, storeName); + DirDiff dirDiff = DirDiffUtil.getDirDiff(localSnapshotDir.toFile(), remoteSnapshotDir, + (localFile, remoteFile) -> localFile.getName().equals(remoteFile.getFileName())); + + // Set up mocks + SamzaException exception = new SamzaException("Error uploading file"); + CompletableFuture failedFuture = new CompletableFuture<>(); + failedFuture.completeExceptionally(exception); + when(blobStoreManager.put(any(InputStream.class), any(Metadata.class))) + .thenAnswer((Answer>) invocation -> { + Metadata metadata = invocation.getArgumentAt(1, Metadata.class); + String path = metadata.getPayloadPath(); + if (path.endsWith("a")) { + return CompletableFuture.completedFuture("aBlobId"); + } else { + return failedFuture; + } + }); + + // Execute + BlobStoreUtil blobStoreUtil = new BlobStoreUtil(blobStoreManager, EXECUTOR, null, null); + CompletionStage dirIndexFuture = blobStoreUtil.putDir(dirDiff, snapshotMetadata); + try { + // should be already complete. if not, future composition in putDir is broken. + dirIndexFuture.toCompletableFuture().get(0, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + // Assert that the result future fails and that the cause is propagated correctly + assertEquals(exception, cause); + return; + } + + fail("DirIndex future should have been completed with an exception"); + } + + @Test + public void testPutDirFailsIfAnySubDirFileUploadFails() throws IOException, TimeoutException, InterruptedException { + BlobStoreManager blobStoreManager = mock(BlobStoreManager.class); + + // File, dir and recursive dir added, retained and removed in local + String local = "[a/1, b/2]"; + String remote = "[]"; + + // Set up environment + Path localSnapshotDir = BlobStoreTestUtil.createLocalDir(local); + String basePath = localSnapshotDir.toAbsolutePath().toString(); + DirIndex remoteSnapshotDir = BlobStoreTestUtil.createDirIndex(remote); + SnapshotMetadata snapshotMetadata = new SnapshotMetadata(checkpointId, jobName, jobId, taskName, storeName); + DirDiff dirDiff = DirDiffUtil.getDirDiff(localSnapshotDir.toFile(), remoteSnapshotDir, + (localFile, remoteFile) -> localFile.getName().equals(remoteFile.getFileName())); + + // Set up mocks + SamzaException exception = new SamzaException("Error uploading file"); + CompletableFuture failedFuture = new CompletableFuture<>(); + failedFuture.completeExceptionally(exception); + when(blobStoreManager.put(any(InputStream.class), any(Metadata.class))) + .thenAnswer((Answer>) invocation -> { + Metadata metadata = invocation.getArgumentAt(1, Metadata.class); + String path = metadata.getPayloadPath(); + if (path.endsWith("1")) { + return CompletableFuture.completedFuture("a1BlobId"); + } else { + return failedFuture; + } + }); + + // Execute + BlobStoreUtil blobStoreUtil = new BlobStoreUtil(blobStoreManager, EXECUTOR, null, null); + CompletionStage dirIndexFuture = blobStoreUtil.putDir(dirDiff, snapshotMetadata); + try { + // should be already complete. if not, future composition in putDir is broken. + dirIndexFuture.toCompletableFuture().get(0, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + // Assert that the result future fails and that the cause is propagated correctly + assertEquals(exception, cause); + return; + } + + fail("DirIndex future should have been completed with an exception"); + } + + @Test + public void testCleanup() throws IOException, ExecutionException, InterruptedException { + BlobStoreManager blobStoreManager = mock(BlobStoreManager.class); + + // File, dir and recursive dir added, retained and removed in local + // Using unique file names since test util uses only the file name (leaf node) + // as the mock blob id, not the full file path. + String local = "[a, c, z/1, y/2, p/m/3, q/n/4]"; + String remote = "[a, b, z/1, x/5, p/m/3, r/o/6]"; + String expectedRemoved = "[b, 5, 6]"; + // keep only the last character (the file name). + SortedSet expectedRemovedFiles = BlobStoreTestUtil.getExpected(expectedRemoved); + + // Set up environment + Path localSnapshotDir = BlobStoreTestUtil.createLocalDir(local); + String basePath = localSnapshotDir.toAbsolutePath().toString(); + DirIndex remoteSnapshotDir = BlobStoreTestUtil.createDirIndex(remote); + SnapshotMetadata snapshotMetadata = new SnapshotMetadata(checkpointId, jobName, jobId, taskName, storeName); + DirDiff dirDiff = DirDiffUtil.getDirDiff(localSnapshotDir.toFile(), remoteSnapshotDir, + (localFile, remoteFile) -> localFile.getName().equals(remoteFile.getFileName())); + + BlobStoreUtil blobStoreUtil = new BlobStoreUtil(blobStoreManager, EXECUTOR, null, null); + when(blobStoreManager.put(any(InputStream.class), any(Metadata.class))) + .thenReturn(CompletableFuture.completedFuture("blobId")); + CompletionStage dirIndexFuture = blobStoreUtil.putDir(dirDiff, snapshotMetadata); + DirIndex dirIndex = null; + try { + // should be already complete. if not, future composition in putDir is broken. + dirIndex = dirIndexFuture.toCompletableFuture().get(0, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + fail("Future returned from putDir should be already complete."); + } + + // Set up mocks + SortedSet allDeleted = new TreeSet<>(); + when(blobStoreManager.delete(anyString(), any(Metadata.class))) + .thenAnswer((Answer>) invocation -> { + String blobId = invocation.getArgumentAt(0, String.class); + allDeleted.add(blobId); + return CompletableFuture.completedFuture(null); + }); + + // Execute + CompletionStage cleanUpFuture = blobStoreUtil.cleanUpDir(dirIndex, metadata); + try { + // should be already complete. if not, future composition in putDir is broken. + cleanUpFuture.toCompletableFuture().get(0, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + fail("Future returned from putDir should be already complete."); + } + + // Assert + assertEquals(expectedRemovedFiles, allDeleted); + } + + @Test + public void testCleanUpFailsIfAnyFileDeleteFails() + throws IOException, TimeoutException, InterruptedException, ExecutionException { + BlobStoreManager blobStoreManager = mock(BlobStoreManager.class); + + // File, dir and recursive dir added, retained and removed in local + // Using unique file names since test util uses only the file name (leaf node) + // as the mock blob id, not the full file path. + String local = "[a, b]"; + String remote = "[c, d]"; + + // Set up environment + Path localSnapshotDir = BlobStoreTestUtil.createLocalDir(local); + String basePath = localSnapshotDir.toAbsolutePath().toString(); + DirIndex remoteSnapshotDir = BlobStoreTestUtil.createDirIndex(remote); + SnapshotMetadata snapshotMetadata = new SnapshotMetadata(checkpointId, jobName, jobId, taskName, storeName); + DirDiff dirDiff = DirDiffUtil.getDirDiff(localSnapshotDir.toFile(), remoteSnapshotDir, + (localFile, remoteFile) -> localFile.getName().equals(remoteFile.getFileName())); + + BlobStoreUtil blobStoreUtil = new BlobStoreUtil(blobStoreManager, EXECUTOR, null, null); + when(blobStoreManager.put(any(InputStream.class), any(Metadata.class))) + .thenReturn(CompletableFuture.completedFuture("blobId")); + CompletionStage dirIndexFuture = blobStoreUtil.putDir(dirDiff, snapshotMetadata); + DirIndex dirIndex = null; + try { + // should be already complete. if not, future composition in putDir is broken. + dirIndex = dirIndexFuture.toCompletableFuture().get(0, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + fail("Future returned from putDir should be already complete."); + } + + // Set up mocks + SamzaException exception = new SamzaException("Error deleting file"); + CompletableFuture failedFuture = new CompletableFuture<>(); + failedFuture.completeExceptionally(exception); + when(blobStoreManager.delete(anyString(), any(Metadata.class))) + .thenAnswer((Answer>) invocation -> { + String blobId = invocation.getArgumentAt(0, String.class); + if (blobId.equals("c")) { + return CompletableFuture.completedFuture(null); + } else { + return failedFuture; + } + }); + + // Execute + CompletionStage cleanUpFuture = blobStoreUtil.cleanUpDir(dirIndex, metadata); + try { + // should be already complete. if not, future composition in putDir is broken. + cleanUpFuture.toCompletableFuture().get(0, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + // Assert that the result future fails and that the cause is propagated correctly + assertEquals(exception, cause); + return; + } + + fail("Clean up future should have been completed with an exception"); + } + + @Test + public void testCleanUpFailsIfAnySubDirFileDeleteFails() + throws IOException, TimeoutException, InterruptedException, ExecutionException { + BlobStoreManager blobStoreManager = mock(BlobStoreManager.class); + + // File, dir and recursive dir added, retained and removed in local + // Using unique file names since test util uses only the file name (leaf node) + // as the mock blob id, not the full file path. + String local = "[a/1, b/2]"; + String remote = "[c/3, d/4]"; + + // Set up environment + Path localSnapshotDir = BlobStoreTestUtil.createLocalDir(local); + String basePath = localSnapshotDir.toAbsolutePath().toString(); + DirIndex remoteSnapshotDir = BlobStoreTestUtil.createDirIndex(remote); + SnapshotMetadata snapshotMetadata = new SnapshotMetadata(checkpointId, jobName, jobId, taskName, storeName); + DirDiff dirDiff = DirDiffUtil.getDirDiff(localSnapshotDir.toFile(), remoteSnapshotDir, + (localFile, remoteFile) -> localFile.getName().equals(remoteFile.getFileName())); + + BlobStoreUtil blobStoreUtil = new BlobStoreUtil(blobStoreManager, EXECUTOR, null, null); + when(blobStoreManager.put(any(InputStream.class), any(Metadata.class))) + .thenReturn(CompletableFuture.completedFuture("blobId")); + CompletionStage dirIndexFuture = blobStoreUtil.putDir(dirDiff, snapshotMetadata); + DirIndex dirIndex = null; + try { + // should be already complete. if not, future composition in putDir is broken. + dirIndex = dirIndexFuture.toCompletableFuture().get(0, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + fail("Future returned from putDir should be already complete."); + } + + // Set up mocks + SamzaException exception = new SamzaException("Error deleting file"); + CompletableFuture failedFuture = new CompletableFuture<>(); + failedFuture.completeExceptionally(exception); + when(blobStoreManager.delete(anyString(), any(Metadata.class))) + .thenAnswer((Answer>) invocation -> { + String blobId = invocation.getArgumentAt(0, String.class); + if (blobId.equals("3")) { // blob ID == file name (leaf node) in blob store test util + return CompletableFuture.completedFuture(null); + } else { + return failedFuture; + } + }); + + // Execute + CompletionStage cleanUpFuture = blobStoreUtil.cleanUpDir(dirIndex, metadata); + try { + // should be already complete. if not, future composition in putDir is broken. + cleanUpFuture.toCompletableFuture().get(0, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + // Assert that the result future fails and that the cause is propagated correctly + assertEquals(exception, cause); + return; + } + + fail("Clean up future should have been completed with an exception"); + } + + @Test + public void testRemoveTTL() throws IOException, ExecutionException, InterruptedException { + BlobStoreManager blobStoreManager = mock(BlobStoreManager.class); + + // File, dir and recursive dir added, retained and removed in local + // Using unique file names since test setup returns it as the blob id + String local = "[a, c, z/1, y/2, p/m/3, q/n/4]"; + String remote = "[a, b, z/1, x/5, p/m/3, r/o/6]"; + String expectedAdded = "[c, y/2, q/n/4]"; + String expectedRetained = "[a, z/1, p/m/3]"; + SortedSet expectedAddedFiles = BlobStoreTestUtil.getExpected(expectedAdded); + SortedSet expectedRetainedFiles = BlobStoreTestUtil.getExpected(expectedRetained); + SortedSet expectedPresentFiles = new TreeSet<>(expectedAddedFiles); + expectedPresentFiles.addAll(expectedRetainedFiles); + + // Set up environment + Path localSnapshotDir = BlobStoreTestUtil.createLocalDir(local); + String basePath = localSnapshotDir.toAbsolutePath().toString(); + DirIndex remoteSnapshotDir = BlobStoreTestUtil.createDirIndex(remote); + SnapshotMetadata snapshotMetadata = new SnapshotMetadata(checkpointId, jobName, jobId, taskName, storeName); + DirDiff dirDiff = DirDiffUtil.getDirDiff(localSnapshotDir.toFile(), remoteSnapshotDir, + (localFile, remoteFile) -> localFile.getName().equals(remoteFile.getFileName())); + + when(blobStoreManager.put(any(InputStream.class), any(Metadata.class))) + .thenAnswer((Answer>) invocation -> { + Metadata metadata = invocation.getArgumentAt(1, Metadata.class); + String path = metadata.getPayloadPath(); + String fileName = path.substring(path.length() - 1); // use only the last character as file name + return CompletableFuture.completedFuture(fileName); + }); + + BlobStoreUtil blobStoreUtil = new BlobStoreUtil(blobStoreManager, EXECUTOR, null, null); + CompletionStage dirIndexFuture = blobStoreUtil.putDir(dirDiff, snapshotMetadata); + DirIndex dirIndex = null; + try { + // should be already complete. if not, future composition in putDir is broken. + dirIndex = dirIndexFuture.toCompletableFuture().get(0, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + fail("Future returned from putDir should be already complete."); + } + + SnapshotIndex mockSnapshotIndex = mock(SnapshotIndex.class); + when(mockSnapshotIndex.getSnapshotMetadata()).thenReturn(snapshotMetadata); + when(mockSnapshotIndex.getDirIndex()).thenReturn(dirIndex); + + SortedSet allTTLRemoved = new TreeSet<>(); + when(blobStoreManager.removeTTL(anyString(), any(Metadata.class))) + .thenAnswer((Answer>) invocation -> { + String blobId = invocation.getArgumentAt(0, String.class); + allTTLRemoved.add(blobId); + return CompletableFuture.completedFuture(null); + }); + + // Execute + blobStoreUtil.removeTTL("snapshotIndexBlobId", mockSnapshotIndex, metadata); + + // Assert + SortedSet expectedBlobIds = new TreeSet<>(); + // test uses unique file name (last char) as the blob ID. + expectedPresentFiles.forEach(f -> expectedBlobIds.add(f.substring(f.length() - 1))); + expectedBlobIds.add("snapshotIndexBlobId"); + + assertEquals(expectedBlobIds, allTTLRemoved); + } + + @Test + public void testPutFileChecksumAndMetadata() throws IOException, ExecutionException, InterruptedException { + // Setup + SnapshotMetadata snapshotMetadata = new SnapshotMetadata(checkpointId, jobName, jobId, taskName, storeName); + Path path = Files.createTempFile("samza-testPutFileChecksum-", ".tmp"); + FileUtil fileUtil = new FileUtil(); + fileUtil.writeToTextFile(path.toFile(), RandomStringUtils.random(1000), false); + long expectedChecksum = FileUtils.checksumCRC32(path.toFile()); + + BlobStoreManager blobStoreManager = mock(BlobStoreManager.class); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Metadata.class); + when(blobStoreManager.put(any(InputStream.class), argumentCaptor.capture())).thenAnswer( + (Answer>) invocation -> { + InputStream inputStream = invocation.getArgumentAt(0, InputStream.class); + // consume input stream to ensure checksum is calculated + IOUtils.copy(inputStream, NullOutputStream.NULL_OUTPUT_STREAM); + return CompletableFuture.completedFuture("blobId"); + }); + + BlobStoreUtil blobStoreUtil = new BlobStoreUtil(blobStoreManager, EXECUTOR, null, null); + + CompletionStage fileIndexFuture = blobStoreUtil.putFile(path.toFile(), snapshotMetadata); + FileIndex fileIndex = null; + try { + // should be already complete. if not, future composition in putFile is broken. + fileIndex = fileIndexFuture.toCompletableFuture().get(0, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + fail("Future returned from putFile should be already complete."); + } + + // Assert + Metadata metadata = (Metadata) argumentCaptor.getValue(); + assertEquals(path.toAbsolutePath().toString(), metadata.getPayloadPath()); + assertEquals(path.toFile().length(), Long.valueOf(metadata.getPayloadSize()).longValue()); + assertEquals(expectedChecksum, fileIndex.getChecksum()); + } + + @Test + public void testAreSameFile() throws IOException { + FileUtil fileUtil = new FileUtil(); + // 1. test with sst file with same attributes + Path sstFile = Files.createTempFile("samza-testAreSameFiles-", ".sst"); + + PosixFileAttributes sstFileAttribs = Files.readAttributes(sstFile, PosixFileAttributes.class); + FileMetadata sstFileMetadata = new FileMetadata(sstFileAttribs.creationTime().toMillis(), + sstFileAttribs.lastModifiedTime().toMillis(), sstFileAttribs.size(), sstFileAttribs.owner().toString(), + sstFileAttribs.group().toString(), PosixFilePermissions.toString(sstFileAttribs.permissions())); + // checksum should be ignored for sst file. Set any dummy value + FileIndex sstFileIndex = new FileIndex(sstFile.getFileName().toString(), Collections.emptyList(), sstFileMetadata, 0L); + + assertTrue(DirDiffUtil.areSameFile(false).test(sstFile.toFile(), sstFileIndex)); + + // 2. test with sst file with different timestamps + // Update last modified time + Files.setLastModifiedTime(sstFile, FileTime.fromMillis(System.currentTimeMillis() + 1000L)); + assertTrue(DirDiffUtil.areSameFile(false).test(sstFile.toFile(), sstFileIndex)); + + // 3. test with non-sst files with same metadata and content + Path tmpFile = Files.createTempFile("samza-testAreSameFiles-", ".tmp"); + fileUtil.writeToTextFile(tmpFile.toFile(), RandomStringUtils.random(1000), false); + + PosixFileAttributes tmpFileAttribs = Files.readAttributes(tmpFile, PosixFileAttributes.class); + FileMetadata tmpFileMetadata = + new FileMetadata(tmpFileAttribs.creationTime().toMillis(), tmpFileAttribs.lastModifiedTime().toMillis(), + tmpFileAttribs.size(), tmpFileAttribs.owner().toString(), tmpFileAttribs.group().toString(), + PosixFilePermissions.toString(tmpFileAttribs.permissions())); + FileIndex tmpFileIndex = new FileIndex(tmpFile.getFileName().toString(), Collections.emptyList(), tmpFileMetadata, + FileUtils.checksumCRC32(tmpFile.toFile())); + + assertTrue(DirDiffUtil.areSameFile(false).test(tmpFile.toFile(), tmpFileIndex)); + + // 4. test with non-sst files with different attributes + // change lastModifiedTime of local file + FileTime prevLastModified = tmpFileAttribs.lastModifiedTime(); + Files.setLastModifiedTime(tmpFile, FileTime.fromMillis(System.currentTimeMillis() + 1000L)); + assertTrue(DirDiffUtil.areSameFile(false).test(tmpFile.toFile(), tmpFileIndex)); + + // change content/checksum of local file + Files.setLastModifiedTime(tmpFile, prevLastModified); // reset attributes to match with remote file + fileUtil.writeToTextFile(tmpFile.toFile(), RandomStringUtils.random(1000), false); //new content + assertFalse(DirDiffUtil.areSameFile(false).test(tmpFile.toFile(), tmpFileIndex)); + } + + @Test + public void testRestoreDirRestoresMultiPartFilesCorrectly() throws IOException { + Path restoreDirBasePath = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX); + + // remote file == 26 blobs, blob ids from a to z, blob contents from a to z, offsets 0 to 25. + DirIndex mockDirIndex = mock(DirIndex.class); + when(mockDirIndex.getDirName()).thenReturn(DirIndex.ROOT_DIR_NAME); + FileIndex mockFileIndex = mock(FileIndex.class); + when(mockFileIndex.getFileName()).thenReturn("1.sst"); + + // setup mock file attributes. create a temp file to get current user/group/permissions so that they + // match with restored files. + File tmpFile = Paths.get(restoreDirBasePath.toString(), "tempfile-" + new Random().nextInt()).toFile(); + tmpFile.createNewFile(); + PosixFileAttributes attrs = Files.readAttributes(tmpFile.toPath(), PosixFileAttributes.class); + FileMetadata fileMetadata = new FileMetadata(1234L, 1243L, 26, // ctime mtime does not matter. size == 26 + attrs.owner().getName(), attrs.group().getName(), PosixFilePermissions.toString(attrs.permissions())); + when(mockFileIndex.getFileMetadata()).thenReturn(fileMetadata); + Files.delete(tmpFile.toPath()); // delete so that it doesn't show up in restored dir contents. + + List mockFileBlobs = new ArrayList<>(); + StringBuilder fileContents = new StringBuilder(); + for (int i = 0; i < 26; i++) { + FileBlob mockFileBlob = mock(FileBlob.class); + char c = (char) ('a' + i); + fileContents.append(c); // blob contents == blobId + when(mockFileBlob.getBlobId()).thenReturn(String.valueOf(c)); + when(mockFileBlob.getOffset()).thenReturn(i); + mockFileBlobs.add(mockFileBlob); + } + when(mockFileIndex.getBlobs()).thenReturn(mockFileBlobs); + CRC32 checksum = new CRC32(); + checksum.update(fileContents.toString().getBytes()); + when(mockFileIndex.getChecksum()).thenReturn(checksum.getValue()); + when(mockDirIndex.getFilesPresent()).thenReturn(ImmutableList.of(mockFileIndex)); + + BlobStoreManager mockBlobStoreManager = mock(BlobStoreManager.class); + when(mockBlobStoreManager.get(anyString(), any(OutputStream.class), any(Metadata.class))).thenAnswer( + (Answer>) invocationOnMock -> { + String blobId = invocationOnMock.getArgumentAt(0, String.class); + OutputStream outputStream = invocationOnMock.getArgumentAt(1, OutputStream.class); + // blob contents = blob id + outputStream.write(blobId.getBytes()); + + // force flush so that the checksum calculation later uses the full file contents. + ((FileOutputStream) outputStream).getFD().sync(); + return CompletableFuture.completedFuture(null); + }); + + BlobStoreUtil blobStoreUtil = new BlobStoreUtil(mockBlobStoreManager, EXECUTOR, null, null); + blobStoreUtil.restoreDir(restoreDirBasePath.toFile(), mockDirIndex, metadata).join(); + + assertTrue( + new DirDiffUtil().areSameDir(Collections.emptySet(), false).test(restoreDirBasePath.toFile(), mockDirIndex)); + } + + @Test + public void testRestoreDirRetriesFileRestoreOnRetriableExceptions() throws IOException { + Path restoreDirBasePath = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX); + + DirIndex mockDirIndex = mock(DirIndex.class); + when(mockDirIndex.getDirName()).thenReturn(DirIndex.ROOT_DIR_NAME); + FileIndex mockFileIndex = mock(FileIndex.class); + when(mockFileIndex.getFileName()).thenReturn("1.sst"); + + // setup mock file attributes. create a temp file to get current user/group/permissions so that they + // match with restored files. + File tmpFile = Paths.get(restoreDirBasePath.toString(), "tempfile-" + new Random().nextInt()).toFile(); + tmpFile.createNewFile(); + byte[] fileContents = "fileContents".getBytes(); + PosixFileAttributes attrs = Files.readAttributes(tmpFile.toPath(), PosixFileAttributes.class); + FileMetadata fileMetadata = + new FileMetadata(1234L, 1243L, fileContents.length, // ctime mtime does not matter. size == 26 + attrs.owner().getName(), attrs.group().getName(), PosixFilePermissions.toString(attrs.permissions())); + when(mockFileIndex.getFileMetadata()).thenReturn(fileMetadata); + Files.delete(tmpFile.toPath()); // delete so that it doesn't show up in restored dir contents. + + List mockFileBlobs = new ArrayList<>(); + FileBlob mockFileBlob = mock(FileBlob.class); + when(mockFileBlob.getBlobId()).thenReturn("fileBlobId"); + when(mockFileBlob.getOffset()).thenReturn(0); + mockFileBlobs.add(mockFileBlob); + when(mockFileIndex.getBlobs()).thenReturn(mockFileBlobs); + + CRC32 checksum = new CRC32(); + checksum.update(fileContents); + when(mockFileIndex.getChecksum()).thenReturn(checksum.getValue()); + when(mockDirIndex.getFilesPresent()).thenReturn(ImmutableList.of(mockFileIndex)); + + BlobStoreManager mockBlobStoreManager = mock(BlobStoreManager.class); + when(mockBlobStoreManager.get(anyString(), any(OutputStream.class), any(Metadata.class))).thenAnswer( + (Answer>) invocationOnMock -> { // first try, retriable error + String blobId = invocationOnMock.getArgumentAt(0, String.class); + OutputStream outputStream = invocationOnMock.getArgumentAt(1, OutputStream.class); + // write garbage data on first retry to verify that final file contents are correct + outputStream.write("bad-data".getBytes()); + ((FileOutputStream) outputStream).getFD().sync(); + return FutureUtil.failedFuture(new RetriableException()); // retriable error + }).thenAnswer((Answer>) invocationOnMock -> { // 2nd try + String blobId = invocationOnMock.getArgumentAt(0, String.class); + OutputStream outputStream = invocationOnMock.getArgumentAt(1, OutputStream.class); + // write correct data on first retry to verify that final file contents are correct + outputStream.write(fileContents); + ((FileOutputStream) outputStream).getFD().sync(); + return CompletableFuture.completedFuture(null); // success + }); + + BlobStoreUtil blobStoreUtil = new BlobStoreUtil(mockBlobStoreManager, EXECUTOR, null, null); + blobStoreUtil.restoreDir(restoreDirBasePath.toFile(), mockDirIndex, metadata).join(); + + assertTrue( + new DirDiffUtil().areSameDir(Collections.emptySet(), false).test(restoreDirBasePath.toFile(), mockDirIndex)); + } + + @Test + public void testRestoreDirFailsRestoreOnNonRetriableExceptions() throws IOException { + Path restoreDirBasePath = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX); + + DirIndex mockDirIndex = mock(DirIndex.class); + when(mockDirIndex.getDirName()).thenReturn(DirIndex.ROOT_DIR_NAME); + FileIndex mockFileIndex = mock(FileIndex.class); + when(mockFileIndex.getFileName()).thenReturn("1.sst"); + + // setup mock file attributes. create a temp file to get current user/group/permissions so that they + // match with restored files. + File tmpFile = Paths.get(restoreDirBasePath.toString(), "tempfile-" + new Random().nextInt()).toFile(); + tmpFile.createNewFile(); + byte[] fileContents = "fileContents".getBytes(); + PosixFileAttributes attrs = Files.readAttributes(tmpFile.toPath(), PosixFileAttributes.class); + FileMetadata fileMetadata = + new FileMetadata(1234L, 1243L, fileContents.length, // ctime mtime does not matter. size == 26 + attrs.owner().getName(), attrs.group().getName(), PosixFilePermissions.toString(attrs.permissions())); + when(mockFileIndex.getFileMetadata()).thenReturn(fileMetadata); + Files.delete(tmpFile.toPath()); // delete so that it doesn't show up in restored dir contents. + + List mockFileBlobs = new ArrayList<>(); + FileBlob mockFileBlob = mock(FileBlob.class); + when(mockFileBlob.getBlobId()).thenReturn("fileBlobId"); + when(mockFileBlob.getOffset()).thenReturn(0); + mockFileBlobs.add(mockFileBlob); + when(mockFileIndex.getBlobs()).thenReturn(mockFileBlobs); + + CRC32 checksum = new CRC32(); + checksum.update(fileContents); + when(mockFileIndex.getChecksum()).thenReturn(checksum.getValue()); + when(mockDirIndex.getFilesPresent()).thenReturn(ImmutableList.of(mockFileIndex)); + + BlobStoreManager mockBlobStoreManager = mock(BlobStoreManager.class); + when(mockBlobStoreManager.get(anyString(), any(OutputStream.class), any(Metadata.class))).thenReturn( + FutureUtil.failedFuture(new IllegalArgumentException())) // non retriable error + .thenAnswer((Answer>) invocationOnMock -> { + String blobId = invocationOnMock.getArgumentAt(0, String.class); + OutputStream outputStream = invocationOnMock.getArgumentAt(1, OutputStream.class); + outputStream.write(fileContents); + + // force flush so that the checksum calculation later uses the full file contents. + ((FileOutputStream) outputStream).getFD().sync(); + return CompletableFuture.completedFuture(null); + }); + + BlobStoreUtil blobStoreUtil = new BlobStoreUtil(mockBlobStoreManager, EXECUTOR, null, null); + try { + blobStoreUtil.restoreDir(restoreDirBasePath.toFile(), mockDirIndex, metadata).join(); + fail("Should have failed on non-retriable errors during file restore"); + } catch (CompletionException e) { + assertTrue(e.getCause() instanceof IllegalArgumentException); + } + } + + @Test + @Ignore // TODO remove + public void testRestoreDirRecreatesEmptyFilesAndDirs() throws IOException { + String prevSnapshotFiles = "[a, b, z/1, y/1, p/m/1, q/n/1]"; + DirIndex dirIndex = BlobStoreTestUtil.createDirIndex(prevSnapshotFiles); + String localSnapshotFiles = "[a, b, z/1, y/1, p/m/1, q/n/1]"; + Path localSnapshot = BlobStoreTestUtil.createLocalDir(localSnapshotFiles); + BlobStoreManager mockBlobStoreManager = mock(BlobStoreManager.class); + when(mockBlobStoreManager.get(anyString(), any(OutputStream.class), any(Metadata.class))).thenAnswer( + (Answer>) invocationOnMock -> { + String blobId = invocationOnMock.getArgumentAt(0, String.class); + OutputStream outputStream = invocationOnMock.getArgumentAt(1, OutputStream.class); + // blob contents = blob id + outputStream.write(blobId.getBytes()); + return CompletableFuture.completedFuture(null); + }); + boolean result = new DirDiffUtil().areSameDir(new TreeSet<>(), false).test(localSnapshot.toFile(), dirIndex); + assertFalse(result); + //ToDo complete + } + + @Test + public void testRestoreDirVerifiesFileChecksums() { + // ToDo shesharma restore dir only restores SST files. Since other metadata files are in ignore list, + // no checksum matching would be done? Check later. + } + + @Test + public void testRestoreDirCreatesCorrectDirectoryStructure() throws IOException { + String prevSnapshotFiles = "[a, b, z/1, y/1, p/m/1, q/n/1]"; + DirIndex dirIndex = BlobStoreTestUtil.createDirIndex(prevSnapshotFiles); + + BlobStoreManager mockBlobStoreManager = mock(BlobStoreManager.class); + when(mockBlobStoreManager.get(anyString(), any(OutputStream.class), any(Metadata.class))).thenAnswer( + (Answer>) invocationOnMock -> { + String blobId = invocationOnMock.getArgumentAt(0, String.class); + OutputStream outputStream = invocationOnMock.getArgumentAt(1, OutputStream.class); + // blob contents = blob id + outputStream.write(blobId.getBytes()); + return CompletableFuture.completedFuture(null); + }); + + Path restoreDirBasePath = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX); + BlobStoreUtil blobStoreUtil = new BlobStoreUtil(mockBlobStoreManager, EXECUTOR, null, null); + blobStoreUtil.restoreDir(restoreDirBasePath.toFile(), dirIndex, metadata).join(); + + assertTrue(new DirDiffUtil().areSameDir(Collections.emptySet(), false).test(restoreDirBasePath.toFile(), dirIndex)); + } + + /** + * Tests related to {@link BlobStoreUtil#getStoreSnapshotIndexes} + */ + + @Test + public void testGetSSIReturnsEmptyMapForNullCheckpoint() { + BlobStoreUtil blobStoreUtil = + new BlobStoreUtil(mock(BlobStoreManager.class), MoreExecutors.newDirectExecutorService(), null, null); + Map> snapshotIndexes = + blobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", null); + assertTrue(snapshotIndexes.isEmpty()); + } + + @Test(expected = SamzaException.class) + public void testGetSSIThrowsExceptionForCheckpointV1() { + Checkpoint mockCheckpoint = mock(Checkpoint.class); + when(mockCheckpoint.getVersion()).thenReturn((short) 1); + BlobStoreUtil blobStoreUtil = + new BlobStoreUtil(mock(BlobStoreManager.class), MoreExecutors.newDirectExecutorService(), null, null); + blobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", mockCheckpoint); + } + + @Test + public void testGetSSIReturnsEmptyMapIfNoEntryForBlobStoreBackendFactory() { + CheckpointV2 mockCheckpoint = mock(CheckpointV2.class); + when(mockCheckpoint.getVersion()).thenReturn((short) 2); + when(mockCheckpoint.getStateCheckpointMarkers()).thenReturn( + ImmutableMap.of("com.OtherStateBackendFactory", ImmutableMap.of("storeName", "otherSCM"))); + + BlobStoreUtil blobStoreUtil = + new BlobStoreUtil(mock(BlobStoreManager.class), MoreExecutors.newDirectExecutorService(), null, null); + Map> snapshotIndexes = + blobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", mockCheckpoint); + assertTrue(snapshotIndexes.isEmpty()); + } + + @Test + public void testGetSSIReturnsEmptyMapIfNoStoreForBlobStoreBackendFactory() { + CheckpointV2 mockCheckpoint = mock(CheckpointV2.class); + when(mockCheckpoint.getVersion()).thenReturn((short) 2); + when(mockCheckpoint.getStateCheckpointMarkers()).thenReturn( + ImmutableMap.of(BlobStoreStateBackendFactory.class.getName(), ImmutableMap.of())); + + BlobStoreUtil blobStoreUtil = + new BlobStoreUtil(mock(BlobStoreManager.class), MoreExecutors.newDirectExecutorService(), null, null); + Map> snapshotIndexes = + blobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", mockCheckpoint); + assertTrue(snapshotIndexes.isEmpty()); + } + + @Test(expected = SamzaException.class) + public void testGetSSIThrowsExceptionOnSyncBlobStoreErrors() { + Checkpoint checkpoint = createCheckpointV2(BlobStoreStateBackendFactory.class.getName(), + ImmutableMap.of("storeName", "snapshotIndexBlobId")); + BlobStoreUtil mockBlobStoreUtil = mock(BlobStoreUtil.class); + when(mockBlobStoreUtil.getSnapshotIndex(anyString(), any(Metadata.class))).thenThrow(new RuntimeException()); + when(mockBlobStoreUtil.getStoreSnapshotIndexes(anyString(), anyString(), anyString(), + any(Checkpoint.class))).thenCallRealMethod(); + mockBlobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", checkpoint); + } + + @Test + public void testGetSSISkipsStoresWithSnapshotIndexAlreadyDeleted() { + Checkpoint checkpoint = createCheckpointV2(BlobStoreStateBackendFactory.class.getName(), + ImmutableMap.of("storeName1", "snapshotIndexBlobId1", "storeName2", "snapshotIndexBlobId2")); + SnapshotIndex store1SnapshotIndex = mock(SnapshotIndex.class); + BlobStoreUtil mockBlobStoreUtil = mock(BlobStoreUtil.class); + + CompletableFuture failedFuture = FutureUtil.failedFuture(new DeletedException()); + when(mockBlobStoreUtil.getSnapshotIndex(eq("snapshotIndexBlobId1"), any(Metadata.class))).thenReturn( + CompletableFuture.completedFuture(store1SnapshotIndex)); + when(mockBlobStoreUtil.getSnapshotIndex(eq("snapshotIndexBlobId2"), any(Metadata.class))).thenReturn(failedFuture); + when(mockBlobStoreUtil.getStoreSnapshotIndexes(anyString(), anyString(), anyString(), + any(Checkpoint.class))).thenCallRealMethod(); + + Map> snapshotIndexes = + mockBlobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", checkpoint); + assertEquals(1, snapshotIndexes.size()); + assertEquals("snapshotIndexBlobId1", snapshotIndexes.get("storeName1").getLeft()); + assertEquals(store1SnapshotIndex, snapshotIndexes.get("storeName1").getRight()); + } + + @Test + public void testGetSSIThrowsExceptionIfAnyNonIgnoredAsyncBlobStoreErrors() { + Checkpoint checkpoint = createCheckpointV2(BlobStoreStateBackendFactory.class.getName(), + ImmutableMap.of("storeName1", "snapshotIndexBlobId1", "storeName2", "snapshotIndexBlobId2")); + SnapshotIndex store1SnapshotIndex = mock(SnapshotIndex.class); + BlobStoreUtil mockBlobStoreUtil = mock(BlobStoreUtil.class); + when(mockBlobStoreUtil.getStoreSnapshotIndexes(anyString(), anyString(), anyString(), + any(Checkpoint.class))).thenCallRealMethod(); + RuntimeException nonIgnoredException = new RuntimeException(); + CompletableFuture failedFuture = FutureUtil.failedFuture(nonIgnoredException); + when(mockBlobStoreUtil.getSnapshotIndex(eq("snapshotIndexBlobId1"), any(Metadata.class))).thenReturn( + FutureUtil.failedFuture(new DeletedException())); // should fail even if some errors are ignored + when(mockBlobStoreUtil.getSnapshotIndex(eq("snapshotIndexBlobId2"), any(Metadata.class))).thenReturn(failedFuture); + + try { + mockBlobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", checkpoint); + fail("Should have thrown an exception"); + } catch (Exception e) { + Throwable cause = + FutureUtil.unwrapExceptions(CompletionException.class, FutureUtil.unwrapExceptions(SamzaException.class, e)); + assertEquals(nonIgnoredException, cause); + } + } + + @Test + public void testGetSSIReturnsCorrectSCMSnapshotIndexPair() { + String storeName = "storeName"; + String otherStoreName = "otherStoreName"; + String storeSnapshotIndexBlobId = "snapshotIndexBlobId"; + String otherStoreSnapshotIndexBlobId = "otherSnapshotIndexBlobId"; + SnapshotIndex mockStoreSnapshotIndex = mock(SnapshotIndex.class); + SnapshotIndex mockOtherStooreSnapshotIndex = mock(SnapshotIndex.class); + + CheckpointV2 checkpoint = createCheckpointV2(BlobStoreStateBackendFactory.class.getName(), + ImmutableMap.of(storeName, storeSnapshotIndexBlobId, otherStoreName, otherStoreSnapshotIndexBlobId)); + + BlobStoreUtil mockBlobStoreUtil = mock(BlobStoreUtil.class); + + when(mockBlobStoreUtil.getSnapshotIndex(eq(storeSnapshotIndexBlobId), any(Metadata.class))).thenReturn( + CompletableFuture.completedFuture(mockStoreSnapshotIndex)); + when(mockBlobStoreUtil.getSnapshotIndex(eq(otherStoreSnapshotIndexBlobId), any(Metadata.class))).thenReturn( + CompletableFuture.completedFuture(mockOtherStooreSnapshotIndex)); + when(mockBlobStoreUtil.getStoreSnapshotIndexes(anyString(), anyString(), anyString(), + any(Checkpoint.class))).thenCallRealMethod(); + + Map> snapshotIndexes = + mockBlobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", checkpoint); + + assertEquals(storeSnapshotIndexBlobId, snapshotIndexes.get(storeName).getKey()); + assertEquals(mockStoreSnapshotIndex, snapshotIndexes.get(storeName).getValue()); + assertEquals(otherStoreSnapshotIndexBlobId, snapshotIndexes.get(otherStoreName).getKey()); + assertEquals(mockOtherStooreSnapshotIndex, snapshotIndexes.get(otherStoreName).getValue()); + verify(mockBlobStoreUtil, times(2)).getSnapshotIndex(anyString(), any(Metadata.class)); + } + + private CheckpointV2 createCheckpointV2(String stateBackendFactory, Map storeSnapshotIndexBlobIds) { + CheckpointId checkpointId = CheckpointId.create(); + Map> factoryStoreSCMs = new HashMap<>(); + Map storeSCMs = new HashMap<>(); + for (Map.Entry entry : storeSnapshotIndexBlobIds.entrySet()) { + storeSCMs.put(entry.getKey(), entry.getValue()); + } + + factoryStoreSCMs.put(stateBackendFactory, storeSCMs); + return new CheckpointV2(checkpointId, ImmutableMap.of(), factoryStoreSCMs); + } +} diff --git a/samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestDirDiffUtil.java b/samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestDirDiffUtil.java new file mode 100644 index 0000000000..fa08670426 --- /dev/null +++ b/samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestDirDiffUtil.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.util; + +import org.apache.samza.storage.blobstore.diff.DirDiff; +import org.apache.samza.storage.blobstore.index.DirIndex; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collection; +import java.util.SortedSet; +import java.util.TreeSet; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import static org.junit.Assert.assertEquals; + + +@RunWith(value = Parameterized.class) +public class TestDirDiffUtil { + + private final String local; + private final String remote; + private final String expectedAdded; + private final String expectedRetained; + private final String expectedRemoved; + private final String description; + + public TestDirDiffUtil(String local, String remote, + String expectedAdded, String expectedRetained, String expectedRemoved, + String description) { + this.local = local; + this.remote = remote; + this.expectedAdded = expectedAdded; + this.expectedRetained = expectedRetained; + this.expectedRemoved = expectedRemoved; + this.description = description; + } + + // TODO HIGH shesharm test with empty subdirectories + @Parameterized.Parameters(name = "testGetDirDiff: {5}") + public static Collection data() { + return Arrays.asList(new Object[][]{ + // Local Remote Expected Added Expected Retained Expected Removed Description + // -------------------------------- -------------------------------- ----------------- ------------------ ----------------- ------------------------------------------------------------------ + {"[]", "[]", "[]", "[]", "[]", "Nothing in local or remote"}, + {"[a]", "[]", "[a]", "[]", "[]", "New file in local"}, + {"[z/1]", "[]", "[z/1]", "[]", "[]", "New dir in local"}, + {"[z/i/1]", "[]", "[z/i/1]", "[]", "[]", "New recursive dir in local"}, + {"[a, z/1]", "[]", "[a, z/1]", "[]", "[]", "New file and dir in local"}, + {"[a, z/1, y/j/1]", "[]", "[a, z/1, y/j/1]", "[]", "[]", "New file, dir and recursive dir in local"}, + {"[a]", "[a]", "[]", "[a]", "[]", "File retained in local"}, + {"[z/1]", "[z/1]", "[]", "[z/1]", "[]", "Dir retained in local"}, + {"[z/i/1]", "[z/i/1]", "[]", "[z/i/1]", "[]", "Recursive dir retained in local"}, + {"[a, z/1]", "[a, z/1]", "[]", "[a, z/1]", "[]", "File and dir retained in local"}, + {"[a, z/1, y/j/1]", "[a, z/1, y/j/1]", "[]", "[a, z/1, y/j/1]", "[]", "File, dir and recursive dir retained in local"}, + {"[]", "[a]", "[]", "[]", "[a]", "File removed in local"}, + {"[]", "[z/1]", "[]", "[]", "[z/1]", "Dir removed in local"}, + {"[]", "[z/i/1]", "[]", "[]", "[z/i/1]", "Recursive dir removed in local"}, + {"[]", "[a, z/1]", "[]", "[]", "[a, z/1]", "File and dir removed in local"}, + {"[]", "[a, z/1, y/j/1]", "[]", "[]", "[a, z/1, y/j/1]", "File, dir and recursive dir removed in local"}, + {"[b]", "[a]", "[b]", "[]", "[a]", "File added and removed in local"}, + {"[y/1]", "[z/1]", "[y/1]", "[]", "[z/1]", "Dir added and removed in local"}, + {"[y/j/1]", "[z/i/1]", "[y/j/1]", "[]", "[z/i/1]", "Recursive dir added and removed in local"}, + {"[b, y/1]", "[a, z/1]", "[b, y/1]", "[]", "[a, z/1]", "File and dir added and removed in local"}, + {"[b, y/1, x/k/1]", "[a, z/1, w/m/1]", "[b, y/1, x/k/1]", "[]", "[a, z/1, w/m/1]", "File, dir and recursive dir added and removed in local"}, + {"[a, c]", "[a]", "[c]", "[a]", "[]", "File added and retained in local"}, + {"[z/1, y/1]", "[z/1]", "[y/1]", "[z/1]", "[]", "Dir added and retained in local"}, + {"[z/i/1, y/j/1]", "[z/i/1]", "[y/j/1]", "[z/i/1]", "[]", "Recursive dir added and retained in local"}, + {"[a, c, z/1, y/1]", "[a, z/1]", "[c, y/1]", "[a, z/1]", "[]", "File and dir added and retained in local"}, + {"[a, c, z/1, y/1, p/m/1, q/n/1]", "[a, z/1, p/m/1]", "[c, y/1, q/n/1]", "[a, z/1, p/m/1]", "[]", "File, dir and recursive dir added and retained in local"}, + {"[a, c]", "[a, b]", "[c]", "[a]", "[b]", "File added, retained and removed in local"}, + {"[z/1, y/1]", "[z/1, x/1]", "[y/1]", "[z/1]", "[x/1]", "Dir added, retained and removed in local"}, + {"[z/1, z/3]", "[z/1, z/2]", "[z/3]", "[z/1]", "[z/2]", "File added, retained and removed in dir in local"}, + {"[z/i/1, y/j/1]", "[z/i/1, x/k/1]", "[y/j/1]", "[z/i/1]", "[x/k/1]", "Recursive dir added, retained and removed in local"}, + {"[a, c, z/1, y/1]", "[a, b, z/1, x/1]", "[c, y/1]", "[a, z/1]", "[b, x/1]", "File and dir added, retained and removed in local"}, + {"[a, c, z/1, y/1, p/m/1, q/n/1]", "[a, b, z/1, x/1, p/m/1, r/o/1]", "[c, y/1, q/n/1]", "[a, z/1, p/m/1]", "[b, x/1, r/o/1]", "File, dir and recursive dir added, retained and removed in local"}, + {"[a, c, z/1, p/m/1, p/m/2, q/n/1]", "[a, b, z/1, x/1, p/m/1, r/o/1]", "[c, p/m/2, q/n/1]", "[a, z/1, p/m/1]", "[b, x/1, r/o/1]", "File, File in recursive subdir, dir and recursive dir added, retained and removed in local"} + }); + } + + @Test + public void testGetDirDiff() throws IOException { + // Setup + Path localSnapshotDir = BlobStoreTestUtil.createLocalDir(this.local); + String basePath = localSnapshotDir.toAbsolutePath().toString(); + DirIndex remoteSnapshotDir = BlobStoreTestUtil.createDirIndex(this.remote); + + // Execute + DirDiff dirDiff = DirDiffUtil.getDirDiff(localSnapshotDir.toFile(), remoteSnapshotDir, + (localFile, remoteFile) -> localFile.getName().equals(remoteFile.getFileName())); + + SortedSet allAdded = new TreeSet<>(); + SortedSet allRemoved = new TreeSet<>(); + SortedSet allRetained = new TreeSet<>(); + BlobStoreTestUtil.getAllAddedInDiff(basePath, dirDiff, allAdded); + BlobStoreTestUtil.getAllRemovedInDiff("", dirDiff, allRemoved); + BlobStoreTestUtil.getAllRetainedInDiff("", dirDiff, allRetained); + + // Assert + SortedSet expectedAddedFiles = BlobStoreTestUtil.getExpected(this.expectedAdded); + SortedSet expectedRetainedFiles = BlobStoreTestUtil.getExpected(this.expectedRetained); + SortedSet expectedRemovedFiles = BlobStoreTestUtil.getExpected(this.expectedRemoved); + assertEquals(expectedAddedFiles, allAdded); + assertEquals(expectedRetainedFiles, allRetained); + assertEquals(expectedRemovedFiles, allRemoved); + } +} diff --git a/samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestDirDiffUtilMisc.java b/samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestDirDiffUtilMisc.java new file mode 100644 index 0000000000..e61297bf4b --- /dev/null +++ b/samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestDirDiffUtilMisc.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage.blobstore.util; + +import org.apache.samza.storage.blobstore.diff.DirDiff; +import org.apache.samza.storage.blobstore.index.DirIndex; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.SortedSet; +import java.util.TreeSet; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestDirDiffUtilMisc { + + + /** + * Test the case when a file has been modified locally. I.e., when isSameFile returns false for a local file with + * the same name as the remote file. The file should be marked for both deletion and addition. + */ + @Test + public void testGetDirDiffWhenIsSameFileReturnsFalseForSameFileName() throws IOException { + String local = "[a]"; + String remote = "[a]"; + String expectedAdded = "[a]"; + String expectedRetained = "[]"; + String expectedRemoved = "[a]"; + + Path localSnapshotDir = BlobStoreTestUtil.createLocalDir(local); + String basePath = localSnapshotDir.toAbsolutePath().toString(); + DirIndex remoteSnapshotDir = BlobStoreTestUtil.createDirIndex(remote); + + // Execute + DirDiff dirDiff = DirDiffUtil.getDirDiff(localSnapshotDir.toFile(), remoteSnapshotDir, + (localFile, remoteFile) -> false); + + SortedSet allAdded = new TreeSet<>(); + SortedSet allRemoved = new TreeSet<>(); + SortedSet allRetained = new TreeSet<>(); + BlobStoreTestUtil.getAllAddedInDiff(basePath, dirDiff, allAdded); + BlobStoreTestUtil.getAllRemovedInDiff("", dirDiff, allRemoved); + BlobStoreTestUtil.getAllRetainedInDiff("", dirDiff, allRetained); + + // Assert + SortedSet expectedAddedFiles = BlobStoreTestUtil.getExpected(expectedAdded); + SortedSet expectedRetainedFiles = BlobStoreTestUtil.getExpected(expectedRetained); + SortedSet expectedRemovedFiles = BlobStoreTestUtil.getExpected(expectedRemoved); + assertEquals(expectedAddedFiles, allAdded); + assertEquals(expectedRetainedFiles, allRetained); + assertEquals(expectedRemovedFiles, allRemoved); + } +} diff --git a/samza-core/src/test/java/org/apache/samza/util/TestFutureUtil.java b/samza-core/src/test/java/org/apache/samza/util/TestFutureUtil.java new file mode 100644 index 0000000000..815eb34a71 --- /dev/null +++ b/samza-core/src/test/java/org/apache/samza/util/TestFutureUtil.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.util; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CompletionStage; +import java.util.function.Predicate; +import org.apache.samza.SamzaException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TestFutureUtil { + + /** + * Test all futures in all collections complete before allOf completes. + * Test completes exceptionally if any complete exceptionally. + * Test works with heterogeneous value types. + * Test works with heterogeneous collection types. + * Test works with completion stages as well as completable futures. + */ + @Test + public void testAllOf() { + // verify that there is no short circuiting + CompletableFuture future1 = new CompletableFuture<>(); + CompletableFuture future2 = new CompletableFuture<>(); + CompletableFuture future3 = new CompletableFuture<>(); + CompletableFuture future4 = new CompletableFuture<>(); + ImmutableList> collection1 = + ImmutableList.of(future1, future2); + ImmutableSet> collection2 = + ImmutableSet.of(future3, future4); + + CompletableFuture allFuture = FutureUtil.allOf(collection1, collection2); + future1.complete("1"); + assertFalse(allFuture.isDone()); + RuntimeException ex2 = new RuntimeException("2"); + future2.completeExceptionally(ex2); + assertFalse(allFuture.isDone()); + assertFalse(allFuture.isCompletedExceptionally()); + future3.complete("3"); + assertFalse(allFuture.isDone()); + assertFalse(allFuture.isCompletedExceptionally()); + future4.complete(4); + assertTrue(allFuture.isDone()); + assertTrue(allFuture.isCompletedExceptionally()); + + try { + allFuture.join(); + } catch (Exception e) { + assertEquals(ex2, FutureUtil.unwrapExceptions(CompletionException.class, e)); + } + } + + @Test + public void testAllOfIgnoringErrorsCompletesSuccessfullyIfNoErrors() { + CompletableFuture future1 = new CompletableFuture<>(); + CompletableFuture future2 = new CompletableFuture<>(); + + CompletableFuture allFuture = FutureUtil.allOf(t -> false, future1, future2); + future1.complete("1"); + assertFalse(allFuture.isDone()); + future2.complete("2"); + assertTrue(allFuture.isDone()); + assertFalse(allFuture.isCompletedExceptionally()); + } + + @Test + public void testAllOfIgnoringErrorsCompletesSuccessfullyIfOnlyIgnoredErrors() { + CompletableFuture future1 = new CompletableFuture<>(); + CompletableFuture future2 = new CompletableFuture<>(); + + CompletableFuture allFuture = FutureUtil.allOf(t -> true, future1, future2); + future1.complete("1"); + assertFalse(allFuture.isDone()); + RuntimeException ex2 = new RuntimeException("2"); + future2.completeExceptionally(ex2); + assertTrue(allFuture.isDone()); + assertFalse(allFuture.isCompletedExceptionally()); + } + + @Test + public void testAllOfIgnoringErrorsCompletesExceptionallyIfNonIgnoredErrors() { + // also test that each future is checked individually + CompletableFuture future1 = new CompletableFuture<>(); + CompletableFuture future2 = new CompletableFuture<>(); + + Predicate mockPredicate = mock(Predicate.class); + when(mockPredicate.test(any())) + .thenReturn(true) + .thenReturn(false); + CompletableFuture allFuture = FutureUtil.allOf(mockPredicate, future1, future2); + future1.completeExceptionally(new SamzaException()); + assertFalse(allFuture.isDone()); + RuntimeException ex2 = new RuntimeException("2"); + future2.completeExceptionally(ex2); + assertTrue(allFuture.isDone()); + assertTrue(allFuture.isCompletedExceptionally()); + verify(mockPredicate, times(2)).test(any()); + } + + @Test + public void testFutureOfMapCompletesExceptionallyIfAValueFutureCompletesExceptionally() { + Map> map = new HashMap<>(); + map.put("1", CompletableFuture.completedFuture("1")); + map.put("2", FutureUtil.failedFuture(new SamzaException())); + + assertTrue(FutureUtil.toFutureOfMap(map).isCompletedExceptionally()); + } + + @Test + public void testFutureOfMapCompletesSuccessfullyIfNoErrors() { + Map> map = new HashMap<>(); + map.put("1", CompletableFuture.completedFuture("1")); + map.put("2", CompletableFuture.completedFuture("2")); + + CompletableFuture> result = FutureUtil.toFutureOfMap(t -> true, map); + assertTrue(result.isDone()); + assertFalse(result.isCompletedExceptionally()); + } + + @Test + public void testFutureOfMapCompletesSuccessfullyIfOnlyIgnoredErrors() { + Map> map = new HashMap<>(); + map.put("1", CompletableFuture.completedFuture("1")); + map.put("2", FutureUtil.failedFuture(new SamzaException())); + + CompletableFuture> result = FutureUtil + .toFutureOfMap(t -> FutureUtil.unwrapExceptions(CompletionException.class, t) instanceof SamzaException, map); + assertTrue(result.isDone()); + result.join(); + assertFalse(result.isCompletedExceptionally()); + assertEquals("1", result.join().get("1")); + assertFalse(result.join().containsKey("2")); + } + + @Test + public void testFutureOfMapCompletesExceptionallyIfAnyNonIgnoredErrors() { + Map> map = new HashMap<>(); + map.put("1", FutureUtil.failedFuture(new RuntimeException())); + SamzaException samzaException = new SamzaException(); + map.put("2", FutureUtil.failedFuture(samzaException)); + + Predicate mockPredicate = mock(Predicate.class); + when(mockPredicate.test(any())) + .thenReturn(true) + .thenReturn(false); + + CompletableFuture> result = FutureUtil.toFutureOfMap(mockPredicate, map); + assertTrue(result.isDone()); + assertTrue(result.isCompletedExceptionally()); + verify(mockPredicate, times(2)).test(any()); // verify that each failed value future is tested + + try { + result.join(); + fail("Should have thrown an exception."); + } catch (Exception e) { + assertEquals(samzaException, FutureUtil.unwrapExceptions(CompletionException.class, e)); + } + } + + @Test + public void testUnwrapExceptionUnwrapsMultipleExceptions() { + IllegalArgumentException cause = new IllegalArgumentException(); + Throwable t = new SamzaException(new SamzaException(cause)); + Throwable unwrappedThrowable = FutureUtil.unwrapExceptions(SamzaException.class, t); + assertEquals(cause, unwrappedThrowable); + } + + @Test + public void testUnwrapExceptionReturnsOriginalExceptionIfNoWrapper() { + IllegalArgumentException cause = new IllegalArgumentException(); + Throwable unwrappedThrowable = FutureUtil.unwrapExceptions(SamzaException.class, cause); + assertEquals(cause, unwrappedThrowable); + } + + @Test + public void testUnwrapExceptionReturnsNullIfNoNonWrapperCause() { + Throwable t = new SamzaException(new SamzaException()); + Throwable unwrappedThrowable = FutureUtil.unwrapExceptions(SamzaException.class, t); + assertNull(unwrappedThrowable); + } + + @Test + public void testUnwrapExceptionReturnsNullIfOriginalExceptionIsNull() { + Throwable unwrappedThrowable = FutureUtil.unwrapExceptions(SamzaException.class, null); + assertNull(unwrappedThrowable); + } +} + diff --git a/samza-core/src/test/scala/org/apache/samza/checkpoint/TestCheckpointTool.scala b/samza-core/src/test/scala/org/apache/samza/checkpoint/TestCheckpointTool.scala index 777a6a06d5..85b2011ddb 100644 --- a/samza-core/src/test/scala/org/apache/samza/checkpoint/TestCheckpointTool.scala +++ b/samza-core/src/test/scala/org/apache/samza/checkpoint/TestCheckpointTool.scala @@ -19,8 +19,7 @@ package org.apache.samza.checkpoint -import java.util.Properties - +import java.util.{Collections, Properties} import org.apache.samza.Partition import org.apache.samza.checkpoint.CheckpointTool.{CheckpointToolCommandLine, TaskNameToCheckpointMap} import org.apache.samza.checkpoint.TestCheckpointTool.{MockCheckpointManagerFactory, MockSystemFactory} @@ -32,9 +31,10 @@ import org.apache.samza.execution.JobPlanner import org.apache.samza.metrics.MetricsRegistry import org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata import org.apache.samza.system._ +import org.junit.Assert.{assertEquals, assertNotNull} import org.junit.{Before, Test} import org.mockito.Matchers._ -import org.mockito.Mockito +import org.mockito.{ArgumentCaptor, Matchers, Mockito} import org.mockito.Mockito._ import org.scalatest.junit.AssertionsForJUnit import org.scalatest.mockito.MockitoSugar @@ -67,8 +67,10 @@ class TestCheckpointTool extends AssertionsForJUnit with MockitoSugar { val tn0 = new TaskName("Partition 0") val tn1 = new TaskName("Partition 1") + val tn2 = new TaskName("Partition 2") val p0 = new Partition(0) val p1 = new Partition(1) + val p2 = new Partition(2) @Before def setup() { @@ -83,17 +85,23 @@ class TestCheckpointTool extends AssertionsForJUnit with MockitoSugar { ).asJava) config = JobPlanner.generateSingleJobConfig(userDefinedConfig) val metadata = new SystemStreamMetadata("foo", Map[Partition, SystemStreamPartitionMetadata]( - new Partition(0) -> new SystemStreamPartitionMetadata("0", "100", "101"), - new Partition(1) -> new SystemStreamPartitionMetadata("0", "200", "201") + p0 -> new SystemStreamPartitionMetadata("0", "100", "101"), + p1 -> new SystemStreamPartitionMetadata("0", "200", "201"), + p2 -> new SystemStreamPartitionMetadata("0", "300", "301") ).asJava) TestCheckpointTool.checkpointManager = mock[CheckpointManager] TestCheckpointTool.systemAdmin = mock[SystemAdmin] when(TestCheckpointTool.systemAdmin.getSystemStreamPartitionCounts(Set("foo").asJava, 0)) .thenReturn(Map("foo" -> metadata).asJava) when(TestCheckpointTool.checkpointManager.readLastCheckpoint(tn0)) - .thenReturn(new Checkpoint(Map(new SystemStreamPartition("test", "foo", p0) -> "1234").asJava)) + .thenReturn(new CheckpointV1(Map(new SystemStreamPartition("test", "foo", p0) -> "1234").asJava)) when(TestCheckpointTool.checkpointManager.readLastCheckpoint(tn1)) - .thenReturn(new Checkpoint(Map(new SystemStreamPartition("test", "foo", p1) -> "4321").asJava)) + .thenReturn(new CheckpointV1(Map(new SystemStreamPartition("test", "foo", p1) -> "4321").asJava)) + when(TestCheckpointTool.checkpointManager.readLastCheckpoint(tn2)) + .thenReturn(new CheckpointV2(null, + Map(new SystemStreamPartition("test", "foo", p2) -> "5678").asJava, + Map("BackupFactory"-> Map("StoreName"-> "offset").asJava).asJava + )) } @Test @@ -113,9 +121,45 @@ class TestCheckpointTool extends AssertionsForJUnit with MockitoSugar { val checkpointTool = CheckpointTool(config, toOverwrite) checkpointTool.run() verify(TestCheckpointTool.checkpointManager) - .writeCheckpoint(tn0, new Checkpoint(Map(new SystemStreamPartition("test", "foo", p0) -> "42").asJava)) + .writeCheckpoint(tn0, new CheckpointV1(Map(new SystemStreamPartition("test", "foo", p0) -> "42").asJava)) + verify(TestCheckpointTool.checkpointManager) + .writeCheckpoint(tn1, new CheckpointV1(Map(new SystemStreamPartition("test", "foo", p1) -> "43").asJava)) + } + + @Test + def testOverwriteCheckpointV2() { + // Skips the v1 checkpoints for the task + when(TestCheckpointTool.checkpointManager.readLastCheckpoint(tn0)) + .thenReturn(null) + when(TestCheckpointTool.checkpointManager.readLastCheckpoint(tn1)) + .thenReturn(null) + + val toOverwrite = Map( + tn0 -> Map(new SystemStreamPartition("test", "foo", p0) -> "42"), + tn1 -> Map(new SystemStreamPartition("test", "foo", p1) -> "43"), + tn2 -> Map(new SystemStreamPartition("test", "foo", p2) -> "45")) + + val checkpointV2Config = new MapConfig(config, Map(TaskConfig.CHECKPOINT_READ_VERSIONS -> "2").asJava) + + val argument = ArgumentCaptor.forClass(classOf[CheckpointV2]) + val checkpointTool = CheckpointTool(checkpointV2Config, toOverwrite) + + checkpointTool.run() + verify(TestCheckpointTool.checkpointManager) + .writeCheckpoint(Matchers.same(tn0), argument.capture()) + assertNotNull(argument.getValue.getCheckpointId) + assertEquals(Map(new SystemStreamPartition("test", "foo", p0) -> "42").asJava, argument.getValue.getOffsets) + assertEquals(Collections.emptyMap(), argument.getValue.getStateCheckpointMarkers) + verify(TestCheckpointTool.checkpointManager) + .writeCheckpoint(Matchers.same(tn1), argument.capture()) + assertNotNull(argument.getValue.getCheckpointId) + assertEquals(Map(new SystemStreamPartition("test", "foo", p1) -> "43").asJava, argument.getValue.getOffsets) + assertEquals(Collections.emptyMap(), argument.getValue.getStateCheckpointMarkers) verify(TestCheckpointTool.checkpointManager) - .writeCheckpoint(tn1, new Checkpoint(Map(new SystemStreamPartition("test", "foo", p1) -> "43").asJava)) + .writeCheckpoint(Matchers.same(tn2), argument.capture()) + assertNotNull(argument.getValue.getCheckpointId) + assertEquals(Map(new SystemStreamPartition("test", "foo", p2) -> "45").asJava, argument.getValue.getOffsets) + assertEquals(Map("BackupFactory"-> Map("StoreName"-> "offset").asJava).asJava, argument.getValue.getStateCheckpointMarkers) } @Test @@ -157,9 +201,9 @@ class TestCheckpointTool extends AssertionsForJUnit with MockitoSugar { checkpointTool.run() verify(TestCheckpointTool.checkpointManager) - .writeCheckpoint(tn0, new Checkpoint(Map(new SystemStreamPartition("test", "foo", p0) -> "42").asJava)) + .writeCheckpoint(tn0, new CheckpointV1(Map(new SystemStreamPartition("test", "foo", p0) -> "42").asJava)) verify(TestCheckpointTool.checkpointManager) - .writeCheckpoint(tn1, new Checkpoint(Map(new SystemStreamPartition("test", "foo", p1) -> "43").asJava)) + .writeCheckpoint(tn1, new CheckpointV1(Map(new SystemStreamPartition("test", "foo", p1) -> "43").asJava)) // Two configurations job.id, job.name are populated in the coordinator config by SamzaRuntime and it is not present in generated config. assert(generatedConfigs.entrySet().containsAll(TestCheckpointTool.coordinatorConfig.entrySet())) diff --git a/samza-core/src/test/scala/org/apache/samza/checkpoint/TestOffsetManager.scala b/samza-core/src/test/scala/org/apache/samza/checkpoint/TestOffsetManager.scala index 677504d8c6..3949ecf186 100644 --- a/samza-core/src/test/scala/org/apache/samza/checkpoint/TestOffsetManager.scala +++ b/samza-core/src/test/scala/org/apache/samza/checkpoint/TestOffsetManager.scala @@ -100,7 +100,7 @@ class TestOffsetManager { } startpointManagerUtil.getStartpointManager.start assertFalse(startpointManagerUtil.getStartpointManager.getFanOutForTask(taskName).containsKey(systemStreamPartition)) // Startpoint should delete after checkpoint commit - val expectedCheckpoint = new Checkpoint(Map(systemStreamPartition -> "47").asJava) + val expectedCheckpoint = new CheckpointV1(Map(systemStreamPartition -> "47").asJava) assertEquals(expectedCheckpoint, checkpointManager.readLastCheckpoint(taskName)) startpointManagerUtil.stop } @@ -260,7 +260,7 @@ class TestOffsetManager { val systemStreamPartition = new SystemStreamPartition(systemStream, partition) val testStreamMetadata = new SystemStreamMetadata(systemStream.getStream, Map(partition -> new SystemStreamPartitionMetadata("0", "1", "2")).asJava) val systemStreamMetadata = Map(systemStream -> testStreamMetadata) - val checkpoint = new Checkpoint(Map(systemStreamPartition -> "45").asJava) + val checkpoint = new CheckpointV1(Map(systemStreamPartition -> "45").asJava) val checkpointManager = getCheckpointManager(systemStreamPartition, taskName) val config = new MapConfig(Map( "systems.test-system.samza.offset.default" -> "oldest", @@ -289,7 +289,7 @@ class TestOffsetManager { partition1 -> new SystemStreamPartitionMetadata("0", "1", "2"), partition2 -> new SystemStreamPartitionMetadata("3", "4", "5")).asJava) val systemStreamMetadata = Map(systemStream -> testStreamMetadata) - val checkpoint = new Checkpoint(Map(systemStreamPartition1 -> "45").asJava) + val checkpoint = new CheckpointV1(Map(systemStreamPartition1 -> "45").asJava) // Checkpoint manager only has partition 1. val checkpointManager = getCheckpointManager(systemStreamPartition1, taskName1) val startpointManagerUtil = getStartpointManagerUtil() @@ -397,8 +397,7 @@ class TestOffsetManager { val testStreamMetadata2 = new SystemStreamMetadata(systemStream2.getStream, Map(partition -> new SystemStreamPartitionMetadata("0", "1", "2")).asJava) val systemStreamMetadata = Map(systemStream -> testStreamMetadata, systemStream2->testStreamMetadata2) val config = new MapConfig - val checkpointManager = getCheckpointManager1(systemStreamPartition, - new Checkpoint(Map(systemStreamPartition -> "45", systemStreamPartition2 -> "100").asJava), + val checkpointManager = getCheckpointManager1(new CheckpointV1(Map(systemStreamPartition -> "45", systemStreamPartition2 -> "100").asJava), taskName) val startpointManagerUtil = getStartpointManagerUtil() val consumer = new SystemConsumerWithCheckpointCallback @@ -423,9 +422,9 @@ class TestOffsetManager { // Should get offset 45 back from the checkpoint manager, which is last processed, and system admin should return 46 as starting offset. assertTrue(startpointManagerUtil.getStartpointManager.getFanOutForTask(taskName).containsKey(systemStreamPartition)) val offsetsToCheckpoint = new java.util.HashMap[SystemStreamPartition, String]() - offsetsToCheckpoint.putAll(offsetManager.buildCheckpoint(taskName).getOffsets) + offsetsToCheckpoint.putAll(offsetManager.getLastProcessedOffsets(taskName)) offsetsToCheckpoint.put(unregisteredSystemStreamPartition, "50") - offsetManager.writeCheckpoint(taskName, new Checkpoint(offsetsToCheckpoint)) + offsetManager.writeCheckpoint(taskName, new CheckpointV1(offsetsToCheckpoint)) intercept[IllegalStateException] { // StartpointManager should stop after last fan out is removed @@ -499,15 +498,15 @@ class TestOffsetManager { offsetManager.update(taskName, systemStreamPartition, "46") // Get checkpoint snapshot like we do at the beginning of TaskInstance.commit() - val checkpoint46 = offsetManager.buildCheckpoint(taskName) + val checkpoint46 = offsetManager.getLastProcessedOffsets(taskName) offsetManager.update(taskName, systemStreamPartition, "47") // Offset updated before checkpoint - offsetManager.writeCheckpoint(taskName, checkpoint46) + offsetManager.writeCheckpoint(taskName, new CheckpointV1(checkpoint46)) assertEquals(Some("47"), offsetManager.getLastProcessedOffset(taskName, systemStreamPartition)) assertEquals("46", offsetManager.offsetManagerMetrics.checkpointedOffsets.get(systemStreamPartition).getValue) // Now write the checkpoint for the latest offset - val checkpoint47 = offsetManager.buildCheckpoint(taskName) - offsetManager.writeCheckpoint(taskName, checkpoint47) + val checkpoint47 = offsetManager.getLastProcessedOffsets(taskName) + offsetManager.writeCheckpoint(taskName, new CheckpointV1(checkpoint47)) startpointManagerUtil.stop assertEquals(Some("47"), offsetManager.getLastProcessedOffset(taskName, systemStreamPartition)) assertEquals("47", offsetManager.offsetManagerMetrics.checkpointedOffsets.get(systemStreamPartition).getValue) @@ -582,7 +581,7 @@ class TestOffsetManager { // Utility method to create and write checkpoint in one statement def checkpoint(offsetManager: OffsetManager, taskName: TaskName): Unit = { - offsetManager.writeCheckpoint(taskName, offsetManager.buildCheckpoint(taskName)) + offsetManager.writeCheckpoint(taskName, new CheckpointV1(offsetManager.getLastProcessedOffsets(taskName))) } class SystemConsumerWithCheckpointCallback extends SystemConsumer with CheckpointListener{ @@ -602,10 +601,10 @@ class TestOffsetManager { } private def getCheckpointManager(systemStreamPartition: SystemStreamPartition, taskName:TaskName = new TaskName("taskName")) = { - getCheckpointManager1(systemStreamPartition, new Checkpoint(Map(systemStreamPartition -> "45").asJava), taskName) + getCheckpointManager1(new CheckpointV1(Map(systemStreamPartition -> "45").asJava), taskName) } - private def getCheckpointManager1(systemStreamPartition: SystemStreamPartition, checkpoint: Checkpoint, taskName:TaskName = new TaskName("taskName")) = { + private def getCheckpointManager1(checkpoint: Checkpoint, taskName:TaskName = new TaskName("taskName")) = { new CheckpointManager { var isStarted = false var isStopped = false @@ -617,9 +616,6 @@ class TestOffsetManager { def writeCheckpoint(taskName: TaskName, checkpoint: Checkpoint) { checkpoints += taskName -> checkpoint } def readLastCheckpoint(taskName: TaskName) = checkpoints.getOrElse(taskName, null) def stop { isStopped = true } - - // Only for testing purposes - not present in actual checkpoint manager - def getOffets = Map(taskName -> checkpoint.getOffsets.asScala.toMap) } } diff --git a/samza-core/src/test/scala/org/apache/samza/checkpoint/file/TestFileSystemCheckpointManager.scala b/samza-core/src/test/scala/org/apache/samza/checkpoint/file/TestFileSystemCheckpointManager.scala index f7839af876..8cc6770b6a 100644 --- a/samza-core/src/test/scala/org/apache/samza/checkpoint/file/TestFileSystemCheckpointManager.scala +++ b/samza-core/src/test/scala/org/apache/samza/checkpoint/file/TestFileSystemCheckpointManager.scala @@ -20,13 +20,15 @@ package org.apache.samza.checkpoint.file import java.io.File + import scala.collection.JavaConverters._ import java.util.Random + import org.junit.Assert._ import org.junit.{After, Before, Test} import org.apache.samza.SamzaException import org.apache.samza.Partition -import org.apache.samza.checkpoint.Checkpoint +import org.apache.samza.checkpoint.{Checkpoint, CheckpointV1} import org.apache.samza.system.SystemStreamPartition import org.apache.samza.container.TaskName import org.junit.rules.TemporaryFolder @@ -52,17 +54,17 @@ class TestFileSystemCheckpointManager { @Test def testReadForCheckpointFileThatDoesExistShouldReturnProperCheckpoint { - val cp = new Checkpoint(Map( + val cp = new CheckpointV1(Map( new SystemStreamPartition("a", "b", new Partition(0)) -> "c", new SystemStreamPartition("a", "c", new Partition(1)) -> "d", new SystemStreamPartition("b", "d", new Partition(2)) -> "e").asJava) - var readCp:Checkpoint = null + var readCp:CheckpointV1 = null val cpm = new FileSystemCheckpointManager("some-job-name", tempFolder.getRoot) cpm.start cpm.writeCheckpoint(taskName, cp) - readCp = cpm.readLastCheckpoint(taskName) + readCp = cpm.readLastCheckpoint(taskName).asInstanceOf[CheckpointV1] cpm.stop assertNotNull(readCp) diff --git a/samza-core/src/test/scala/org/apache/samza/container/TestSamzaContainer.scala b/samza-core/src/test/scala/org/apache/samza/container/TestSamzaContainer.scala index 7d180339eb..a86c49f458 100644 --- a/samza-core/src/test/scala/org/apache/samza/container/TestSamzaContainer.scala +++ b/samza-core/src/test/scala/org/apache/samza/container/TestSamzaContainer.scala @@ -308,40 +308,6 @@ class TestSamzaContainer extends AssertionsForJUnit with MockitoSugar { assertEquals(2, mockJobServlet.exceptionCount) } - @Test - def testGetChangelogSSPsForContainer() { - val taskName0 = new TaskName("task0") - val taskName1 = new TaskName("task1") - val taskModel0 = new TaskModel(taskName0, - Set(new SystemStreamPartition("input", "stream", new Partition(0))), - new Partition(10)) - val taskModel1 = new TaskModel(taskName1, - Set(new SystemStreamPartition("input", "stream", new Partition(1))), - new Partition(11)) - val containerModel = new ContainerModel("processorId", Map(taskName0 -> taskModel0, taskName1 -> taskModel1)) - val changeLogSystemStreams = Map("store0" -> new SystemStream("changelogSystem0", "store0-changelog"), - "store1" -> new SystemStream("changelogSystem1", "store1-changelog")) - val expected = Set(new SystemStreamPartition("changelogSystem0", "store0-changelog", new Partition(10)), - new SystemStreamPartition("changelogSystem1", "store1-changelog", new Partition(10)), - new SystemStreamPartition("changelogSystem0", "store0-changelog", new Partition(11)), - new SystemStreamPartition("changelogSystem1", "store1-changelog", new Partition(11))) - assertEquals(expected, SamzaContainer.getChangelogSSPsForContainer(containerModel, changeLogSystemStreams)) - } - - @Test - def testGetChangelogSSPsForContainerNoChangelogs() { - val taskName0 = new TaskName("task0") - val taskName1 = new TaskName("task1") - val taskModel0 = new TaskModel(taskName0, - Set(new SystemStreamPartition("input", "stream", new Partition(0))), - new Partition(10)) - val taskModel1 = new TaskModel(taskName1, - Set(new SystemStreamPartition("input", "stream", new Partition(1))), - new Partition(11)) - val containerModel = new ContainerModel("processorId", Map(taskName0 -> taskModel0, taskName1 -> taskModel1)) - assertEquals(Set(), SamzaContainer.getChangelogSSPsForContainer(containerModel, Map())) - } - @Test def testStoreContainerLocality():Unit = { this.config = new MapConfig(Map(ClusterManagerConfig.JOB_HOST_AFFINITY_ENABLED -> "true")) diff --git a/samza-core/src/test/scala/org/apache/samza/container/TestTaskInstance.scala b/samza-core/src/test/scala/org/apache/samza/container/TestTaskInstance.scala index 4cab1859f6..e357fe5e0a 100644 --- a/samza-core/src/test/scala/org/apache/samza/container/TestTaskInstance.scala +++ b/samza-core/src/test/scala/org/apache/samza/container/TestTaskInstance.scala @@ -19,19 +19,20 @@ package org.apache.samza.container -import java.util.Collections - -import com.google.common.collect.ImmutableSet -import org.apache.samza.{Partition, SamzaException} -import org.apache.samza.checkpoint.{Checkpoint, CheckpointedChangelogOffset, OffsetManager} +import com.google.common.collect.{ImmutableMap, ImmutableSet} +import com.google.common.util.concurrent.MoreExecutors +import org.apache.samza.checkpoint._ +import org.apache.samza.checkpoint.kafka.{KafkaChangelogSSPOffset, KafkaStateCheckpointMarker} import org.apache.samza.config.MapConfig import org.apache.samza.context.{TaskContext => _, _} import org.apache.samza.job.model.TaskModel -import org.apache.samza.metrics.Counter -import org.apache.samza.storage.NonTransactionalStateTaskStorageManager +import org.apache.samza.metrics.{Counter, Gauge, Timer} +import org.apache.samza.storage.TaskStorageCommitManager import org.apache.samza.system.{IncomingMessageEnvelope, StreamMetadataCache, SystemAdmin, SystemConsumers, SystemStream, SystemStreamMetadata, _} import org.apache.samza.table.TableManager import org.apache.samza.task._ +import org.apache.samza.util.FutureUtil +import org.apache.samza.{Partition, SamzaException} import org.junit.Assert._ import org.junit.{Before, Test} import org.mockito.Matchers._ @@ -42,6 +43,10 @@ import org.mockito.{ArgumentCaptor, Matchers, Mock, MockitoAnnotations} import org.scalatest.junit.AssertionsForJUnit import org.scalatest.mockito.MockitoSugar +import java.util +import java.util.Collections +import java.util.concurrent.{CompletableFuture, ExecutorService, Executors, ForkJoinPool} +import java.util.function.Consumer import scala.collection.JavaConverters._ class TestTaskInstance extends AssertionsForJUnit with MockitoSugar { @@ -68,7 +73,9 @@ class TestTaskInstance extends AssertionsForJUnit with MockitoSugar { @Mock private var offsetManager: OffsetManager = null @Mock - private var taskStorageManager: NonTransactionalStateTaskStorageManager = null + private var taskCommitManager: TaskStorageCommitManager = null + @Mock + private var checkpointManager: CheckpointManager = null @Mock private var taskTableManager: TableManager = null // not a mock; using MockTaskInstanceExceptionHandler @@ -88,6 +95,8 @@ class TestTaskInstance extends AssertionsForJUnit with MockitoSugar { private var taskInstance: TaskInstance = null + private val numCheckpointVersions = 2 // checkpoint versions count + @Before def setup(): Unit = { MockitoAnnotations.initMocks(this) @@ -98,7 +107,10 @@ class TestTaskInstance extends AssertionsForJUnit with MockitoSugar { Matchers.eq(this.containerContext), any(), Matchers.eq(this.applicationContainerContext))) .thenReturn(this.applicationTaskContext) when(this.systemAdmins.getSystemAdmin(SYSTEM_NAME)).thenReturn(this.systemAdmin) - when(this.jobContext.getConfig).thenReturn(new MapConfig(Collections.singletonMap("task.commit.ms", "-1"))) + val taskConfigsMap = new util.HashMap[String, String]() + taskConfigsMap.put("task.commit.ms", "-1") + taskConfigsMap.put("task.commit.max.delay.ms", "100000") + when(this.jobContext.getConfig).thenReturn(new MapConfig(taskConfigsMap)) setupTaskInstance(Some(this.applicationTaskContextFactory)) } @@ -172,7 +184,6 @@ class TestTaskInstance extends AssertionsForJUnit with MockitoSugar { taskInstance.initTask verify(this.offsetManager).setStartingOffset(TASK_NAME, SYSTEM_STREAM_PARTITION, "10") - verifyNoMoreInteractions(this.offsetManager) } @Test @@ -212,18 +223,44 @@ class TestTaskInstance extends AssertionsForJUnit with MockitoSugar { def testCommitOrder() { val commitsCounter = mock[Counter] when(this.metrics.commits).thenReturn(commitsCounter) - val inputOffsets = new Checkpoint(Map(SYSTEM_STREAM_PARTITION -> "4").asJava) + val snapshotTimer = mock[Timer] + when(this.metrics.snapshotNs).thenReturn(snapshotTimer) + val commitTimer = mock[Timer] + when(this.metrics.commitNs).thenReturn(commitTimer) + val lastCommitGauge = mock[Gauge[Long]] + when(this.metrics.lastCommitNs).thenReturn(lastCommitGauge) + val commitSyncTimer = mock[Timer] + when(this.metrics.commitSyncNs).thenReturn(commitSyncTimer) + val commitAsyncTimer = mock[Timer] + when(this.metrics.commitAsyncNs).thenReturn(commitAsyncTimer) + val uploadTimer = mock[Timer] + when(this.metrics.asyncUploadNs).thenReturn(uploadTimer) + val cleanUpTimer = mock[Timer] + when(this.metrics.asyncCleanupNs).thenReturn(cleanUpTimer) + val skippedCounter = mock[Gauge[Int]] + when(this.metrics.commitsSkipped).thenReturn(skippedCounter) + val inputOffsets = new util.HashMap[SystemStreamPartition, String]() + inputOffsets.put(SYSTEM_STREAM_PARTITION,"4") val changelogSSP = new SystemStreamPartition(new SystemStream(SYSTEM_NAME, "test-changelog-stream"), new Partition(0)) - val changelogOffsets = Map(changelogSSP -> Some("5")) - when(this.offsetManager.buildCheckpoint(TASK_NAME)).thenReturn(inputOffsets) - when(this.taskStorageManager.flush()).thenReturn(changelogOffsets) - doNothing().when(this.taskStorageManager).checkpoint(any(), any[Map[SystemStreamPartition, Option[String]]]) + val stateCheckpointMarkers: util.Map[String, String] = new util.HashMap[String, String]() + val stateCheckpointMarker = KafkaStateCheckpointMarker.serialize(new KafkaStateCheckpointMarker(changelogSSP, "5")) + stateCheckpointMarkers.put("storeName", stateCheckpointMarker) + when(this.offsetManager.getLastProcessedOffsets(TASK_NAME)).thenReturn(inputOffsets) + + val snapshotSCMs = ImmutableMap.of(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME, stateCheckpointMarkers) + when(this.taskCommitManager.snapshot(any())).thenReturn(snapshotSCMs) + val snapshotSCMFuture: CompletableFuture[util.Map[String, util.Map[String, String]]] = + CompletableFuture.completedFuture(snapshotSCMs) + when(this.taskCommitManager.upload(any(), Matchers.eq(snapshotSCMs))).thenReturn(snapshotSCMFuture) // kafka is no-op + when(this.taskCommitManager.cleanUp(any(), any())).thenReturn(CompletableFuture.completedFuture[Void](null)) + + taskInstance.commit - val mockOrder = inOrder(this.offsetManager, this.collector, this.taskTableManager, this.taskStorageManager) + val mockOrder = inOrder(this.offsetManager, this.collector, this.taskTableManager, this.taskCommitManager) // We must first get a snapshot of the input offsets so it doesn't change while we flush. SAMZA-1384 - mockOrder.verify(this.offsetManager).buildCheckpoint(TASK_NAME) + mockOrder.verify(this.offsetManager).getLastProcessedOffsets(TASK_NAME) // Producers must be flushed next and ideally the output would be flushed before the changelog // s.t. the changelog and checkpoints (state and inputs) are captured last @@ -233,128 +270,701 @@ class TestTaskInstance extends AssertionsForJUnit with MockitoSugar { mockOrder.verify(this.taskTableManager).flush() // Local state should be flushed next next - mockOrder.verify(this.taskStorageManager).flush() + mockOrder.verify(this.taskCommitManager).snapshot(any()) + + // Upload should be called next with the snapshot SCMs. + mockOrder.verify(this.taskCommitManager).upload(any(), Matchers.eq(snapshotSCMs)) // Stores checkpoints should be created next with the newest changelog offsets - mockOrder.verify(this.taskStorageManager).checkpoint(any(), Matchers.eq(changelogOffsets)) + mockOrder.verify(this.taskCommitManager).writeCheckpointToStoreDirectories(any()) // Input checkpoint should be written with the snapshot captured at the beginning of commit and the // newest changelog offset captured during storage manager flush val captor = ArgumentCaptor.forClass(classOf[Checkpoint]) - mockOrder.verify(offsetManager).writeCheckpoint(any(), captor.capture) - val cp = captor.getValue - assertEquals("4", cp.getOffsets.get(SYSTEM_STREAM_PARTITION)) - assertEquals("5", CheckpointedChangelogOffset.fromString(cp.getOffsets.get(changelogSSP)).getOffset) + mockOrder.verify(offsetManager, times(numCheckpointVersions)).writeCheckpoint(any(), captor.capture) + val cp = captor.getAllValues + assertEquals(numCheckpointVersions, cp.size()) + cp.forEach(new Consumer[Checkpoint] { + override def accept(c: Checkpoint): Unit = { + assertEquals("4", c.getOffsets.get(SYSTEM_STREAM_PARTITION)) + if (c.getVersion == 2) { + assertEquals(1, c.getOffsets.size()) + assertTrue(c.isInstanceOf[CheckpointV2]) + val checkpointedStateCheckpointMarkers = c.asInstanceOf[CheckpointV2] + .getStateCheckpointMarkers.get(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME) + assertTrue(checkpointedStateCheckpointMarkers.size() == 1) + val checkpointedStateCheckpointMarker = checkpointedStateCheckpointMarkers.get("storeName") + assertTrue(checkpointedStateCheckpointMarker.equals(stateCheckpointMarker)) + val kafkaMarker = KafkaStateCheckpointMarker.deserialize(checkpointedStateCheckpointMarker) + assertEquals(kafkaMarker.getChangelogOffset, "5") + assertEquals(kafkaMarker.getChangelogSSP, changelogSSP) + } else { // c.getVersion == 1 + assertEquals(2, c.getOffsets.size()) + assertTrue(c.isInstanceOf[CheckpointV1]) + assertEquals("5", KafkaChangelogSSPOffset.fromString(c.getOffsets.get(changelogSSP)).getChangelogOffset) + } + } + }) // Old checkpointed stores should be cleared - mockOrder.verify(this.taskStorageManager).removeOldCheckpoints(any()) + mockOrder.verify(this.taskCommitManager).cleanUp(any(), any()) verify(commitsCounter).inc() + verify(snapshotTimer).update(anyLong()) + verify(uploadTimer).update(anyLong()) + verify(commitTimer).update(anyLong()) } @Test def testEmptyChangelogSSPOffsetInCommit() { // e.g. if changelog topic is empty val commitsCounter = mock[Counter] when(this.metrics.commits).thenReturn(commitsCounter) + val snapshotTimer = mock[Timer] + when(this.metrics.snapshotNs).thenReturn(snapshotTimer) + val commitTimer = mock[Timer] + when(this.metrics.commitNs).thenReturn(commitTimer) + val lastCommitGauge = mock[Gauge[Long]] + when(this.metrics.lastCommitNs).thenReturn(lastCommitGauge) + val commitSyncTimer = mock[Timer] + when(this.metrics.commitSyncNs).thenReturn(commitSyncTimer) + val commitAsyncTimer = mock[Timer] + when(this.metrics.commitAsyncNs).thenReturn(commitAsyncTimer) + val uploadTimer = mock[Timer] + when(this.metrics.asyncUploadNs).thenReturn(uploadTimer) + val skippedCounter = mock[Gauge[Int]] + when(this.metrics.commitsSkipped).thenReturn(skippedCounter) - val inputOffsets = new Checkpoint(Map(SYSTEM_STREAM_PARTITION -> "4").asJava) + val inputOffsets = Map(SYSTEM_STREAM_PARTITION -> "4").asJava val changelogSSP = new SystemStreamPartition(new SystemStream(SYSTEM_NAME, "test-changelog-stream"), new Partition(0)) - val changelogOffsets = Map(changelogSSP -> None) - when(this.offsetManager.buildCheckpoint(TASK_NAME)).thenReturn(inputOffsets) - when(this.taskStorageManager.flush()).thenReturn(changelogOffsets) + val stateCheckpointMarkers: util.Map[String, String] = new util.HashMap[String, String]() + val nullStateCheckpointMarker = KafkaStateCheckpointMarker.serialize(new KafkaStateCheckpointMarker(changelogSSP, null)) + stateCheckpointMarkers.put("storeName", nullStateCheckpointMarker) + when(this.offsetManager.getLastProcessedOffsets(TASK_NAME)).thenReturn(inputOffsets) + when(this.taskCommitManager.upload(any(), any())) + .thenReturn(CompletableFuture.completedFuture( + Collections.singletonMap(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME, stateCheckpointMarkers))) taskInstance.commit val captor = ArgumentCaptor.forClass(classOf[Checkpoint]) - verify(offsetManager).writeCheckpoint(any(), captor.capture) - val cp = captor.getValue - assertEquals("4", cp.getOffsets.get(SYSTEM_STREAM_PARTITION)) - val message = cp.getOffsets.get(changelogSSP) - val checkpointedOffset = CheckpointedChangelogOffset.fromString(message) - assertNull(checkpointedOffset.getOffset) - assertNotNull(checkpointedOffset.getCheckpointId) + verify(offsetManager, times(numCheckpointVersions)).writeCheckpoint(any(), captor.capture) + val cp = captor.getAllValues + assertEquals(numCheckpointVersions, cp.size()) + cp.forEach(new Consumer[Checkpoint] { + override def accept(checkpoint: Checkpoint): Unit = { + assertEquals("4", checkpoint.getOffsets.get(SYSTEM_STREAM_PARTITION)) + if (checkpoint.getVersion == 2) { + assertEquals(1, checkpoint.getOffsets.size()) + assertTrue(checkpoint.isInstanceOf[CheckpointV2]) + val checkpointedStateCheckpointMarkers = checkpoint.asInstanceOf[CheckpointV2] + .getStateCheckpointMarkers.get(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME) + assertTrue(checkpointedStateCheckpointMarkers.size() == 1) + val checkpointedStateCheckpointMarker = checkpointedStateCheckpointMarkers.get("storeName") + assertTrue(checkpointedStateCheckpointMarker.equals(nullStateCheckpointMarker)) + val kafkaMarker = KafkaStateCheckpointMarker.deserialize(checkpointedStateCheckpointMarker) + assertNull(kafkaMarker.getChangelogOffset) + assertEquals(kafkaMarker.getChangelogSSP, changelogSSP) + } else { // c.getVersion == 1 + assertEquals(2, checkpoint.getOffsets.size()) + assertTrue(checkpoint.isInstanceOf[CheckpointV1]) + val message = checkpoint.getOffsets.get(changelogSSP) + val checkpointedOffset = KafkaChangelogSSPOffset.fromString(message) + assertNull(checkpointedOffset.getChangelogOffset) + assertNotNull(checkpointedOffset.getCheckpointId) + } + } + }) verify(commitsCounter).inc() + verify(snapshotTimer).update(anyLong()) + verify(uploadTimer).update(anyLong()) } @Test def testEmptyChangelogOffsetsInCommit() { // e.g. if stores have no changelogs val commitsCounter = mock[Counter] when(this.metrics.commits).thenReturn(commitsCounter) + val snapshotTimer = mock[Timer] + when(this.metrics.snapshotNs).thenReturn(snapshotTimer) + val commitTimer = mock[Timer] + when(this.metrics.commitNs).thenReturn(commitTimer) + val commitSyncTimer = mock[Timer] + when(this.metrics.commitSyncNs).thenReturn(commitSyncTimer) + val commitAsyncTimer = mock[Timer] + when(this.metrics.commitAsyncNs).thenReturn(commitAsyncTimer) + val uploadTimer = mock[Timer] + when(this.metrics.asyncUploadNs).thenReturn(uploadTimer) + val skippedCounter = mock[Gauge[Int]] + when(this.metrics.commitsSkipped).thenReturn(skippedCounter) + val lastCommitGauge = mock[Gauge[Long]] + when(this.metrics.lastCommitNs).thenReturn(lastCommitGauge) - val inputOffsets = new Checkpoint(Map(SYSTEM_STREAM_PARTITION -> "4").asJava) - val changelogOffsets = Map[SystemStreamPartition, Option[String]]() - when(this.offsetManager.buildCheckpoint(TASK_NAME)).thenReturn(inputOffsets) - when(this.taskStorageManager.flush()).thenReturn(changelogOffsets) + val inputOffsets = Map(SYSTEM_STREAM_PARTITION -> "4").asJava + val stateCheckpointMarkers: util.Map[String, String] = new util.HashMap[String, String]() + when(this.offsetManager.getLastProcessedOffsets(TASK_NAME)).thenReturn(inputOffsets) + when(this.taskCommitManager.upload(any(), any())) + .thenReturn(CompletableFuture.completedFuture( + Collections.singletonMap(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME, stateCheckpointMarkers))) taskInstance.commit val captor = ArgumentCaptor.forClass(classOf[Checkpoint]) - verify(offsetManager).writeCheckpoint(any(), captor.capture) - val cp = captor.getValue - assertEquals("4", cp.getOffsets.get(SYSTEM_STREAM_PARTITION)) - assertEquals(1, cp.getOffsets.size()) + // verify the write checkpoint is evoked twice, once per checkpoint version + verify(offsetManager, times(numCheckpointVersions)).writeCheckpoint(any(), captor.capture) + val cp = captor.getAllValues + assertEquals(numCheckpointVersions, cp.size()) + cp.forEach(new Consumer[Checkpoint] { + override def accept(c: Checkpoint): Unit = { + assertEquals("4", c.getOffsets.get(SYSTEM_STREAM_PARTITION)) + assertEquals(1, c.getOffsets.size()) + } + }) verify(commitsCounter).inc() + verify(snapshotTimer).update(anyLong()) + verify(uploadTimer).update(anyLong()) } @Test def testCommitFailsIfErrorGettingChangelogOffset() { // required for transactional state val commitsCounter = mock[Counter] when(this.metrics.commits).thenReturn(commitsCounter) + val snapshotTimer = mock[Timer] + when(this.metrics.snapshotNs).thenReturn(snapshotTimer) - val inputOffsets = new Checkpoint(Map(SYSTEM_STREAM_PARTITION -> "4").asJava) - when(this.offsetManager.buildCheckpoint(TASK_NAME)).thenReturn(inputOffsets) - when(this.taskStorageManager.flush()).thenThrow(new SamzaException("Error getting changelog offsets")) + val inputOffsets = new util.HashMap[SystemStreamPartition, String]() + inputOffsets.put(SYSTEM_STREAM_PARTITION,"4") + when(this.offsetManager.getLastProcessedOffsets(TASK_NAME)).thenReturn(inputOffsets) + when(this.taskCommitManager.snapshot(any())).thenThrow(new SamzaException("Error getting changelog offsets")) try { + // sync stage exception should be caught and rethrown immediately taskInstance.commit + + verify(commitsCounter).inc() + verifyZeroInteractions(snapshotTimer) } catch { case e: SamzaException => + val msg = e.getMessage // exception is expected, container should fail if could not get changelog offsets. return } - fail("Should have failed commit if error getting newest changelog offests") + fail("Should have failed commit if error getting newest changelog offsets") } @Test - def testCommitFailsIfErrorCreatingStoreCheckpoints() { // required for transactional state + def testCommitFailsIfPreviousAsyncUploadFails() { val commitsCounter = mock[Counter] when(this.metrics.commits).thenReturn(commitsCounter) + val snapshotTimer = mock[Timer] + when(this.metrics.snapshotNs).thenReturn(snapshotTimer) + val commitTimer = mock[Timer] + when(this.metrics.commitNs).thenReturn(commitTimer) + val commitSyncTimer = mock[Timer] + when(this.metrics.commitSyncNs).thenReturn(commitSyncTimer) + val commitAsyncTimer = mock[Timer] + when(this.metrics.commitAsyncNs).thenReturn(commitAsyncTimer) + val uploadTimer = mock[Timer] + when(this.metrics.asyncUploadNs).thenReturn(uploadTimer) + val cleanUpTimer = mock[Timer] + when(this.metrics.asyncCleanupNs).thenReturn(cleanUpTimer) + val skippedCounter = mock[Gauge[Int]] + when(this.metrics.commitsSkipped).thenReturn(skippedCounter) + val lastCommitGauge = mock[Gauge[Long]] + when(this.metrics.lastCommitNs).thenReturn(lastCommitGauge) - val inputOffsets = new Checkpoint(Map(SYSTEM_STREAM_PARTITION -> "4").asJava) - when(this.offsetManager.buildCheckpoint(TASK_NAME)).thenReturn(inputOffsets) - when(this.taskStorageManager.flush()).thenReturn(Map[SystemStreamPartition, Option[String]]()) - when(this.taskStorageManager.checkpoint(any(), any())).thenThrow(new SamzaException("Error creating store checkpoint")) + val inputOffsets = new util.HashMap[SystemStreamPartition, String]() + inputOffsets.put(SYSTEM_STREAM_PARTITION,"4") + val stateCheckpointMarkers: util.Map[String, String] = new util.HashMap[String, String]() + when(this.offsetManager.getLastProcessedOffsets(TASK_NAME)).thenReturn(inputOffsets) + when(this.taskCommitManager.upload(any(), any())) + .thenReturn(CompletableFuture.completedFuture( + Collections.singletonMap(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME, stateCheckpointMarkers))) + when(this.taskCommitManager.upload(any(), any())) + .thenReturn(FutureUtil.failedFuture[util.Map[String, util.Map[String, String]]](new RuntimeException)) try { taskInstance.commit + + verify(commitsCounter).inc() + verify(snapshotTimer).update(anyLong()) + verifyZeroInteractions(uploadTimer) + verifyZeroInteractions(commitTimer) + verifyZeroInteractions(skippedCounter) + + // async stage exception in first commit should be caught and rethrown by the subsequent commit + taskInstance.commit + verifyNoMoreInteractions(commitsCounter) } catch { case e: SamzaException => - // exception is expected, container should fail if could not get changelog offsets. + // exception is expected, container should fail if could not upload previous snapshot. return } - fail("Should have failed commit if error getting newest changelog offests") + fail("Should have failed commit if error uploading store contents") } @Test - def testCommitContinuesIfErrorClearingOldCheckpoints() { // required for transactional state + def testCommitFailsIfAsyncStoreDirCheckpointWriteFails() { // required for transactional state val commitsCounter = mock[Counter] when(this.metrics.commits).thenReturn(commitsCounter) + val snapshotTimer = mock[Timer] + when(this.metrics.snapshotNs).thenReturn(snapshotTimer) + val commitTimer = mock[Timer] + when(this.metrics.commitNs).thenReturn(commitTimer) + val commitSyncTimer = mock[Timer] + when(this.metrics.commitSyncNs).thenReturn(commitSyncTimer) + val commitAsyncTimer = mock[Timer] + when(this.metrics.commitAsyncNs).thenReturn(commitAsyncTimer) + val uploadTimer = mock[Timer] + when(this.metrics.asyncUploadNs).thenReturn(uploadTimer) + val cleanUpTimer = mock[Timer] + when(this.metrics.asyncCleanupNs).thenReturn(cleanUpTimer) + val skippedCounter = mock[Gauge[Int]] + when(this.metrics.commitsSkipped).thenReturn(skippedCounter) + val lastCommitGauge = mock[Gauge[Long]] + when(this.metrics.lastCommitNs).thenReturn(lastCommitGauge) - val inputOffsets = new Checkpoint(Map(SYSTEM_STREAM_PARTITION -> "4").asJava) - when(this.offsetManager.buildCheckpoint(TASK_NAME)).thenReturn(inputOffsets) - when(this.taskStorageManager.flush()).thenReturn(Map[SystemStreamPartition, Option[String]]()) - doNothing().when(this.taskStorageManager).checkpoint(any(), any()) - when(this.taskStorageManager.removeOldCheckpoints(any())) - .thenThrow(new SamzaException("Error clearing old checkpoints")) + val inputOffsets = new util.HashMap[SystemStreamPartition, String]() + inputOffsets.put(SYSTEM_STREAM_PARTITION,"4") + val stateCheckpointMarkers: util.Map[String, String] = new util.HashMap[String, String]() + when(this.offsetManager.getLastProcessedOffsets(TASK_NAME)).thenReturn(inputOffsets) + when(this.taskCommitManager.upload(any(), any())) + .thenReturn(CompletableFuture.completedFuture( + Collections.singletonMap(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME, stateCheckpointMarkers))) + when(this.taskCommitManager.writeCheckpointToStoreDirectories(any())) + .thenThrow(new SamzaException("Error creating store checkpoint")) try { taskInstance.commit + + verify(commitsCounter).inc() + verify(snapshotTimer).update(anyLong()) + verify(uploadTimer).update(anyLong()) + verifyZeroInteractions(commitTimer) + verifyZeroInteractions(skippedCounter) + + // async stage exception in first commit should be caught and rethrown by the subsequent commit + taskInstance.commit + verifyNoMoreInteractions(commitsCounter) } catch { case e: SamzaException => // exception is expected, container should fail if could not get changelog offsets. - fail("Exception from removeOldCheckpoints should have been caught") + return + } + + fail("Should have failed commit if error writing checkpoint to store dirs") + } + + @Test + def testCommitFailsIfPreviousAsyncCheckpointTopicWriteFails() { + val commitsCounter = mock[Counter] + when(this.metrics.commits).thenReturn(commitsCounter) + val snapshotTimer = mock[Timer] + when(this.metrics.snapshotNs).thenReturn(snapshotTimer) + val commitTimer = mock[Timer] + when(this.metrics.commitNs).thenReturn(commitTimer) + val commitSyncTimer = mock[Timer] + when(this.metrics.commitSyncNs).thenReturn(commitSyncTimer) + val commitAsyncTimer = mock[Timer] + when(this.metrics.commitAsyncNs).thenReturn(commitAsyncTimer) + val uploadTimer = mock[Timer] + when(this.metrics.asyncUploadNs).thenReturn(uploadTimer) + val cleanUpTimer = mock[Timer] + when(this.metrics.asyncCleanupNs).thenReturn(cleanUpTimer) + val skippedCounter = mock[Gauge[Int]] + when(this.metrics.commitsSkipped).thenReturn(skippedCounter) + val lastCommitGauge = mock[Gauge[Long]] + when(this.metrics.lastCommitNs).thenReturn(lastCommitGauge) + + val inputOffsets = new util.HashMap[SystemStreamPartition, String]() + inputOffsets.put(SYSTEM_STREAM_PARTITION,"4") + val stateCheckpointMarkers: util.Map[String, String] = new util.HashMap[String, String]() + when(this.offsetManager.getLastProcessedOffsets(TASK_NAME)).thenReturn(inputOffsets) + when(this.taskCommitManager.upload(any(), any())) + .thenReturn(CompletableFuture.completedFuture( + Collections.singletonMap(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME, stateCheckpointMarkers))) + doNothing().when(this.taskCommitManager).writeCheckpointToStoreDirectories(any()) + when(this.offsetManager.writeCheckpoint(any(), any())) + .thenThrow(new SamzaException("Error writing checkpoint")) + + try { + taskInstance.commit + + verify(commitsCounter).inc() + verify(snapshotTimer).update(anyLong()) + verify(uploadTimer).update(anyLong()) + verifyZeroInteractions(commitTimer) + verifyZeroInteractions(skippedCounter) + + // async stage exception in first commit should be caught and rethrown by the subsequent commit + taskInstance.commit + verifyNoMoreInteractions(commitsCounter) + } catch { + case e: SamzaException => + // exception is expected, container should fail if could not write previous checkpoint. + return + } + + fail("Should have failed commit if error writing checkpoints to checkpoint topic") + } + + @Test + def testCommitFailsIfPreviousAsyncCleanUpFails() { // required for blob store backend + val commitsCounter = mock[Counter] + when(this.metrics.commits).thenReturn(commitsCounter) + val snapshotTimer = mock[Timer] + when(this.metrics.snapshotNs).thenReturn(snapshotTimer) + val commitTimer = mock[Timer] + when(this.metrics.commitNs).thenReturn(commitTimer) + val commitSyncTimer = mock[Timer] + when(this.metrics.commitSyncNs).thenReturn(commitSyncTimer) + val commitAsyncTimer = mock[Timer] + when(this.metrics.commitAsyncNs).thenReturn(commitAsyncTimer) + val uploadTimer = mock[Timer] + when(this.metrics.asyncUploadNs).thenReturn(uploadTimer) + val cleanUpTimer = mock[Timer] + when(this.metrics.asyncCleanupNs).thenReturn(cleanUpTimer) + val skippedCounter = mock[Gauge[Int]] + when(this.metrics.commitsSkipped).thenReturn(skippedCounter) + val lastCommitGauge = mock[Gauge[Long]] + when(this.metrics.lastCommitNs).thenReturn(lastCommitGauge) + + val inputOffsets = new util.HashMap[SystemStreamPartition, String]() + inputOffsets.put(SYSTEM_STREAM_PARTITION,"4") + val stateCheckpointMarkers: util.Map[String, String] = new util.HashMap[String, String]() + when(this.offsetManager.getLastProcessedOffsets(TASK_NAME)).thenReturn(inputOffsets) + when(this.taskCommitManager.upload(any(), any())) + .thenReturn(CompletableFuture.completedFuture( + Collections.singletonMap(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME, stateCheckpointMarkers))) + doNothing().when(this.taskCommitManager).writeCheckpointToStoreDirectories(any()) + when(this.taskCommitManager.cleanUp(any(), any())) + .thenReturn(FutureUtil.failedFuture[Void](new SamzaException("Error during cleanup"))) + + try { + taskInstance.commit + + verify(commitsCounter).inc() + verify(snapshotTimer).update(anyLong()) + verify(uploadTimer).update(anyLong()) + verifyZeroInteractions(commitTimer) + verifyZeroInteractions(skippedCounter) + + // async stage exception in first commit should be caught and rethrown by the subsequent commit + taskInstance.commit + verifyNoMoreInteractions(commitsCounter) + } catch { + case e: SamzaException => + // exception is expected, container should fail if could not clean up old checkpoint. + return + } + + fail("Should have failed commit if error cleaning up previous commit") + } + + @Test + def testCommitFailsIfPreviousAsyncUploadFailsSynchronously() { + val commitsCounter = mock[Counter] + when(this.metrics.commits).thenReturn(commitsCounter) + val snapshotTimer = mock[Timer] + when(this.metrics.snapshotNs).thenReturn(snapshotTimer) + val commitTimer = mock[Timer] + when(this.metrics.commitNs).thenReturn(commitTimer) + val commitSyncTimer = mock[Timer] + when(this.metrics.commitSyncNs).thenReturn(commitSyncTimer) + val commitAsyncTimer = mock[Timer] + when(this.metrics.commitAsyncNs).thenReturn(commitAsyncTimer) + val uploadTimer = mock[Timer] + when(this.metrics.asyncUploadNs).thenReturn(uploadTimer) + val cleanUpTimer = mock[Timer] + when(this.metrics.asyncCleanupNs).thenReturn(cleanUpTimer) + val skippedCounter = mock[Gauge[Int]] + when(this.metrics.commitsSkipped).thenReturn(skippedCounter) + val lastCommitGauge = mock[Gauge[Long]] + when(this.metrics.lastCommitNs).thenReturn(lastCommitGauge) + + val inputOffsets = new util.HashMap[SystemStreamPartition, String]() + inputOffsets.put(SYSTEM_STREAM_PARTITION,"4") + val stateCheckpointMarkers: util.Map[String, String] = new util.HashMap[String, String]() + when(this.offsetManager.getLastProcessedOffsets(TASK_NAME)).thenReturn(inputOffsets) + when(this.taskCommitManager.upload(any(), any())) + .thenReturn(CompletableFuture.completedFuture( + Collections.singletonMap(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME, stateCheckpointMarkers))) + + // Fail synchronously instead of returning a failed future. + when(this.taskCommitManager.upload(any(), any())) + .thenThrow(new RuntimeException) + + try { + taskInstance.commit + + verify(commitsCounter).inc() + verify(snapshotTimer).update(anyLong()) + verifyZeroInteractions(uploadTimer) + verifyZeroInteractions(commitTimer) + verifyZeroInteractions(skippedCounter) + + // async stage exception in first commit should be caught and rethrown by the subsequent commit + taskInstance.commit + verifyNoMoreInteractions(commitsCounter) + } catch { + case e: SamzaException => + // exception is expected, container should fail if could not upload previous snapshot. + return } + + fail("Should have failed commit if synchronous error during upload in async stage of previous commit") + } + + @Test + def testCommitSucceedsIfPreviousAsyncStageSucceeds() { + val commitsCounter = mock[Counter] + when(this.metrics.commits).thenReturn(commitsCounter) + val snapshotTimer = mock[Timer] + when(this.metrics.snapshotNs).thenReturn(snapshotTimer) + val commitTimer = mock[Timer] + when(this.metrics.commitNs).thenReturn(commitTimer) + val commitSyncTimer = mock[Timer] + when(this.metrics.commitSyncNs).thenReturn(commitSyncTimer) + val commitAsyncTimer = mock[Timer] + when(this.metrics.commitAsyncNs).thenReturn(commitAsyncTimer) + val uploadTimer = mock[Timer] + when(this.metrics.asyncUploadNs).thenReturn(uploadTimer) + val cleanUpTimer = mock[Timer] + when(this.metrics.asyncCleanupNs).thenReturn(cleanUpTimer) + val skippedCounter = mock[Gauge[Int]] + when(this.metrics.commitsSkipped).thenReturn(skippedCounter) + val lastCommitGauge = mock[Gauge[Long]] + when(this.metrics.lastCommitNs).thenReturn(lastCommitGauge) + + val inputOffsets = new util.HashMap[SystemStreamPartition, String]() + inputOffsets.put(SYSTEM_STREAM_PARTITION,"4") + val stateCheckpointMarkers: util.Map[String, String] = new util.HashMap[String, String]() + when(this.offsetManager.getLastProcessedOffsets(TASK_NAME)).thenReturn(inputOffsets) + when(this.taskCommitManager.upload(any(), any())) + .thenReturn(CompletableFuture.completedFuture( + Collections.singletonMap(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME, stateCheckpointMarkers))) + doNothing().when(this.taskCommitManager).writeCheckpointToStoreDirectories(any()) + when(this.taskCommitManager.cleanUp(any(), any())) + .thenReturn(CompletableFuture.completedFuture[Void](null)) + + taskInstance.commit // async stage will be run by caller due to direct executor + + val asyncCommitTimeCaptor = ArgumentCaptor.forClass(classOf[Long]) + val uploadTimeCaptor = ArgumentCaptor.forClass(classOf[Long]) + val cleanUpTimeCaptor = ArgumentCaptor.forClass(classOf[Long]) + + verify(commitsCounter).inc() + verify(snapshotTimer).update(anyLong()) + verify(uploadTimer).update(anyLong()) + verify(commitTimer).update(anyLong()) + verify(commitAsyncTimer).update(asyncCommitTimeCaptor.capture()) + verify(uploadTimer).update(uploadTimeCaptor.capture()) + verify(cleanUpTimer).update(cleanUpTimeCaptor.capture()) + + assertTrue((cleanUpTimeCaptor.getValue + uploadTimeCaptor.getValue) < asyncCommitTimeCaptor.getValue) + + taskInstance.commit + + // verify that all commit operations ran twice + verify(taskCommitManager, times(2)).snapshot(any()) + verify(taskCommitManager, times(2)).upload(any(), any()) + // called 2x per commit, once for each checkpoint version + verify(taskCommitManager, times(4)).writeCheckpointToStoreDirectories(any()) + verify(offsetManager, times(4)).writeCheckpoint(any(), any()) + verify(taskCommitManager, times(2)).cleanUp(any(), any()) + verify(commitsCounter, times(2)).inc() + } + + @Test + def testCommitSkipsIfPreviousAsyncCommitInProgressWithinMaxCommitDelay() { + val commitsCounter = mock[Counter] + when(this.metrics.commits).thenReturn(commitsCounter) + val snapshotTimer = mock[Timer] + when(this.metrics.snapshotNs).thenReturn(snapshotTimer) + val commitTimer = mock[Timer] + when(this.metrics.commitNs).thenReturn(commitTimer) + val commitSyncTimer = mock[Timer] + when(this.metrics.commitSyncNs).thenReturn(commitSyncTimer) + val commitAsyncTimer = mock[Timer] + when(this.metrics.commitAsyncNs).thenReturn(commitAsyncTimer) + val uploadTimer = mock[Timer] + when(this.metrics.asyncUploadNs).thenReturn(uploadTimer) + val cleanUpTimer = mock[Timer] + when(this.metrics.asyncCleanupNs).thenReturn(cleanUpTimer) + val skippedCounter = mock[Gauge[Int]] + when(this.metrics.commitsSkipped).thenReturn(skippedCounter) + val lastCommitGauge = mock[Gauge[Long]] + when(this.metrics.lastCommitNs).thenReturn(lastCommitGauge) + + val inputOffsets = new util.HashMap[SystemStreamPartition, String]() + inputOffsets.put(SYSTEM_STREAM_PARTITION,"4") + val changelogSSP = new SystemStreamPartition(new SystemStream(SYSTEM_NAME, "test-changelog-stream"), new Partition(0)) + + val stateCheckpointMarkers: util.Map[String, String] = new util.HashMap[String, String]() + val stateCheckpointMarker = KafkaStateCheckpointMarker.serialize(new KafkaStateCheckpointMarker(changelogSSP, "5")) + stateCheckpointMarkers.put("storeName", stateCheckpointMarker) + when(this.offsetManager.getLastProcessedOffsets(TASK_NAME)).thenReturn(inputOffsets) + + val snapshotSCMs = ImmutableMap.of(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME, stateCheckpointMarkers) + when(this.taskCommitManager.snapshot(any())).thenReturn(snapshotSCMs) + val snapshotSCMFuture: CompletableFuture[util.Map[String, util.Map[String, String]]] = + CompletableFuture.completedFuture(snapshotSCMs) + + when(this.taskCommitManager.upload(any(), Matchers.eq(snapshotSCMs))).thenReturn(snapshotSCMFuture) // kafka is no-op + + val cleanUpFuture = new CompletableFuture[Void]() // not completed until subsequent commit starts + when(this.taskCommitManager.cleanUp(any(), any())).thenReturn(cleanUpFuture) + + // use a separate executor to perform async operations on to test caller thread blocking behavior + setupTaskInstance(None, ForkJoinPool.commonPool()) + + taskInstance.commit // async stage will not complete until cleanUpFuture is completed + + taskInstance.commit + + verify(skippedCounter).set(1) + + verify(commitsCounter, times(1)).inc() // should only have been incremented once on the initial commit + verify(snapshotTimer).update(anyLong()) + verifyZeroInteractions(commitTimer) + + cleanUpFuture.complete(null) // just to unblock shared executor } + @Test + def testCommitThrowsIfPreviousAsyncCommitInProgressAfterMaxCommitDelayAndBlockTime() { + val commitsCounter = mock[Counter] + when(this.metrics.commits).thenReturn(commitsCounter) + val snapshotTimer = mock[Timer] + when(this.metrics.snapshotNs).thenReturn(snapshotTimer) + val commitTimer = mock[Timer] + when(this.metrics.commitNs).thenReturn(commitTimer) + val commitSyncTimer = mock[Timer] + when(this.metrics.commitSyncNs).thenReturn(commitSyncTimer) + val commitAsyncTimer = mock[Timer] + when(this.metrics.commitAsyncNs).thenReturn(commitAsyncTimer) + val uploadTimer = mock[Timer] + when(this.metrics.asyncUploadNs).thenReturn(uploadTimer) + val cleanUpTimer = mock[Timer] + when(this.metrics.asyncCleanupNs).thenReturn(cleanUpTimer) + val skippedCounter = mock[Gauge[Int]] + when(this.metrics.commitsSkipped).thenReturn(skippedCounter) + val lastCommitGauge = mock[Gauge[Long]] + when(this.metrics.lastCommitNs).thenReturn(lastCommitGauge) + + val inputOffsets = new util.HashMap[SystemStreamPartition, String]() + inputOffsets.put(SYSTEM_STREAM_PARTITION,"4") + val changelogSSP = new SystemStreamPartition(new SystemStream(SYSTEM_NAME, "test-changelog-stream"), new Partition(0)) + + val stateCheckpointMarkers: util.Map[String, String] = new util.HashMap[String, String]() + val stateCheckpointMarker = KafkaStateCheckpointMarker.serialize(new KafkaStateCheckpointMarker(changelogSSP, "5")) + stateCheckpointMarkers.put("storeName", stateCheckpointMarker) + when(this.offsetManager.getLastProcessedOffsets(TASK_NAME)).thenReturn(inputOffsets) + + val snapshotSCMs = ImmutableMap.of(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME, stateCheckpointMarkers) + when(this.taskCommitManager.snapshot(any())).thenReturn(snapshotSCMs) + val snapshotSCMFuture: CompletableFuture[util.Map[String, util.Map[String, String]]] = + CompletableFuture.completedFuture(snapshotSCMs) + + when(this.taskCommitManager.upload(any(), Matchers.eq(snapshotSCMs))).thenReturn(snapshotSCMFuture) // kafka is no-op + + val cleanUpFuture = new CompletableFuture[Void]() + when(this.taskCommitManager.cleanUp(any(), any())).thenReturn(cleanUpFuture) + + // use a separate executor to perform async operations on to test caller thread blocking behavior + val taskConfigsMap = new util.HashMap[String, String]() + taskConfigsMap.put("task.commit.ms", "-1") + // "block" immediately if previous commit async stage not complete + taskConfigsMap.put("task.commit.max.delay.ms", "-1") + taskConfigsMap.put("task.commit.timeout.ms", "0") // throw exception immediately if blocked + when(this.jobContext.getConfig).thenReturn(new MapConfig(taskConfigsMap)) // override default behavior + + setupTaskInstance(None, ForkJoinPool.commonPool()) + + taskInstance.commit // async stage will not complete until cleanUpFuture is completed + + try { + taskInstance.commit // should throw exception + fail("Should have thrown an exception if blocked for previous commit async stage.") + } catch { + case e: Exception => + verify(commitsCounter, times(1)).inc() // should only have been incremented once on the initial commit + } + + cleanUpFuture.complete(null) // just to unblock shared executor + } + + @Test + def testCommitBlocksIfPreviousAsyncCommitInProgressAfterMaxCommitDelayButWithinBlockTime() { + val commitsCounter = mock[Counter] + when(this.metrics.commits).thenReturn(commitsCounter) + val snapshotTimer = mock[Timer] + when(this.metrics.snapshotNs).thenReturn(snapshotTimer) + val commitTimer = mock[Timer] + when(this.metrics.commitNs).thenReturn(commitTimer) + val uploadTimer = mock[Timer] + val commitSyncTimer = mock[Timer] + when(this.metrics.commitSyncNs).thenReturn(commitSyncTimer) + val commitAsyncTimer = mock[Timer] + when(this.metrics.commitAsyncNs).thenReturn(commitAsyncTimer) + when(this.metrics.asyncUploadNs).thenReturn(uploadTimer) + val cleanUpTimer = mock[Timer] + when(this.metrics.asyncCleanupNs).thenReturn(cleanUpTimer) + val skippedCounter = mock[Gauge[Int]] + when(this.metrics.commitsSkipped).thenReturn(skippedCounter) + val lastCommitGauge = mock[Gauge[Long]] + when(this.metrics.lastCommitNs).thenReturn(lastCommitGauge) + + val inputOffsets = new util.HashMap[SystemStreamPartition, String]() + inputOffsets.put(SYSTEM_STREAM_PARTITION,"4") + val changelogSSP = new SystemStreamPartition(new SystemStream(SYSTEM_NAME, "test-changelog-stream"), new Partition(0)) + + val stateCheckpointMarkers: util.Map[String, String] = new util.HashMap[String, String]() + val stateCheckpointMarker = KafkaStateCheckpointMarker.serialize(new KafkaStateCheckpointMarker(changelogSSP, "5")) + stateCheckpointMarkers.put("storeName", stateCheckpointMarker) + when(this.offsetManager.getLastProcessedOffsets(TASK_NAME)).thenReturn(inputOffsets) + + val snapshotSCMs = ImmutableMap.of(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME, stateCheckpointMarkers) + when(this.taskCommitManager.snapshot(any())).thenReturn(snapshotSCMs) + val snapshotSCMFuture: CompletableFuture[util.Map[String, util.Map[String, String]]] = + CompletableFuture.completedFuture(snapshotSCMs) + + when(this.taskCommitManager.upload(any(), Matchers.eq(snapshotSCMs))).thenReturn(snapshotSCMFuture) // kafka is no-op + + val cleanUpFuture = new CompletableFuture[Void]() + when(this.taskCommitManager.cleanUp(any(), any())).thenReturn(cleanUpFuture) + + // use a separate executor to perform async operations on to test caller thread blocking behavior + val taskConfigsMap = new util.HashMap[String, String]() + taskConfigsMap.put("task.commit.ms", "-1") + // "block" immediately if previous commit async stage not complete + taskConfigsMap.put("task.commit.max.delay.ms", "-1") + taskConfigsMap.put("task.commit.timeout.ms", "1000000") // block until previous stage is complete + when(this.jobContext.getConfig).thenReturn(new MapConfig(taskConfigsMap)) // override default behavior + + setupTaskInstance(None, ForkJoinPool.commonPool()) + + taskInstance.commit // async stage will not complete until cleanUpFuture is completed + + val executorService = Executors.newSingleThreadExecutor() + val secondCommitFuture = CompletableFuture.runAsync(new Runnable { + override def run(): Unit = taskInstance.commit // will block on executor + }, executorService) + + var retries = 0 // wait no more than ~100 millis + while (!taskInstance.commitInProgress.hasQueuedThreads && retries < 10) { + retries += 1 + Thread.sleep(10) // wait until commit in other thread blocks on the semaphore. + } + if (!taskInstance.commitInProgress.hasQueuedThreads) { + fail("Other thread should have blocked on semaphore acquisition. " + + "May need to increase retries if transient failure.") + } + + cleanUpFuture.complete(null) // will eventually unblock the 2nd commit in other thread. + secondCommitFuture.join() // will complete when the sync phase of 2nd commit is complete. + verify(commitsCounter, times(2)).inc() // should only have been incremented twice - once for each commit + verify(snapshotTimer, times(2)).update(anyLong()) + } + + /** * Given that no application task context factory is provided, then no lifecycle calls should be made. */ @@ -400,7 +1010,7 @@ class TestTaskInstance extends AssertionsForJUnit with MockitoSugar { this.consumerMultiplexer, this.collector, offsetManager = offsetManagerMock, - storageManager = this.taskStorageManager, + commitManager = this.taskCommitManager, tableManager = this.taskTableManager, systemStreamPartitions = ImmutableSet.of(ssp), exceptionHandler = this.taskInstanceExceptionHandler, @@ -418,7 +1028,8 @@ class TestTaskInstance extends AssertionsForJUnit with MockitoSugar { } private def setupTaskInstance( - applicationTaskContextFactory: Option[ApplicationTaskContextFactory[ApplicationTaskContext]]): Unit = { + applicationTaskContextFactory: Option[ApplicationTaskContextFactory[ApplicationTaskContext]], + commitThreadPool: ExecutorService = MoreExecutors.newDirectExecutorService()): Unit = { this.taskInstance = new TaskInstance(this.task, this.taskModel, this.metrics, @@ -426,10 +1037,11 @@ class TestTaskInstance extends AssertionsForJUnit with MockitoSugar { this.consumerMultiplexer, this.collector, offsetManager = this.offsetManager, - storageManager = this.taskStorageManager, + commitManager = this.taskCommitManager, tableManager = this.taskTableManager, systemStreamPartitions = SYSTEM_STREAM_PARTITIONS, exceptionHandler = this.taskInstanceExceptionHandler, + commitThreadPool = commitThreadPool, jobContext = this.jobContext, containerContext = this.containerContext, applicationContainerContextOption = Some(this.applicationContainerContext), diff --git a/samza-core/src/test/scala/org/apache/samza/serializers/TestCheckpointSerde.scala b/samza-core/src/test/scala/org/apache/samza/serializers/TestCheckpointV1Serde.scala similarity index 69% rename from samza-core/src/test/scala/org/apache/samza/serializers/TestCheckpointSerde.scala rename to samza-core/src/test/scala/org/apache/samza/serializers/TestCheckpointV1Serde.scala index c2060e0d03..02f2e59474 100644 --- a/samza-core/src/test/scala/org/apache/samza/serializers/TestCheckpointSerde.scala +++ b/samza-core/src/test/scala/org/apache/samza/serializers/TestCheckpointV1Serde.scala @@ -22,46 +22,30 @@ package org.apache.samza.serializers import java.util import org.apache.samza.Partition -import org.apache.samza.checkpoint.Checkpoint +import org.apache.samza.checkpoint.{CheckpointV1} import org.apache.samza.container.TaskName import org.apache.samza.system.SystemStreamPartition import org.junit.Assert._ import org.junit.Test import scala.collection.JavaConverters._ -import scala.collection.mutable -class TestCheckpointSerde { +class TestCheckpointV1Serde { @Test def testExactlyOneOffset { - val serde = new CheckpointSerde + val serde = new CheckpointV1Serde var offsets = Map[SystemStreamPartition, String]() val systemStreamPartition = new SystemStreamPartition("test-system", "test-stream", new Partition(777)) offsets += systemStreamPartition -> "1" - val deserializedOffsets = serde.fromBytes(serde.toBytes(new Checkpoint(offsets.asJava))) + val deserializedOffsets = serde.fromBytes(serde.toBytes(new CheckpointV1(offsets.asJava))) assertEquals("1", deserializedOffsets.getOffsets.get(systemStreamPartition)) assertEquals(1, deserializedOffsets.getOffsets.size) } - @Test - def testChangelogPartitionMappingRoundTrip { - val mapping = new util.HashMap[TaskName, java.lang.Integer]() - mapping.put(new TaskName("Ted"), 0) - mapping.put(new TaskName("Dougal"), 1) - mapping.put(new TaskName("Jack"), 2) - - val checkpointSerde = new CheckpointSerde - val asBytes = checkpointSerde.changelogPartitionMappingToBytes(mapping) - val backToMap = checkpointSerde.changelogPartitionMappingFromBytes(asBytes) - - assertEquals(mapping, backToMap) - assertNotSame(mapping, backToMap) - } - @Test def testNullCheckpointSerde: Unit = { val checkpointBytes = null.asInstanceOf[Array[Byte]] - val checkpointSerde = new CheckpointSerde + val checkpointSerde = new CheckpointV1Serde val checkpoint = checkpointSerde.fromBytes(checkpointBytes) assertNull(checkpoint) } diff --git a/samza-core/src/test/scala/org/apache/samza/storage/TestContainerStorageManager.java b/samza-core/src/test/scala/org/apache/samza/storage/TestContainerStorageManager.java index c36a3bebde..8645e4a21c 100644 --- a/samza-core/src/test/scala/org/apache/samza/storage/TestContainerStorageManager.java +++ b/samza-core/src/test/scala/org/apache/samza/storage/TestContainerStorageManager.java @@ -18,16 +18,24 @@ */ package org.apache.samza.storage; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import java.io.File; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Optional; +import java.util.Set; import org.apache.samza.Partition; -import org.apache.samza.checkpoint.Checkpoint; import org.apache.samza.checkpoint.CheckpointManager; +import org.apache.samza.checkpoint.CheckpointV1; +import org.apache.samza.checkpoint.CheckpointV2; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; +import org.apache.samza.config.StorageConfig; import org.apache.samza.config.TaskConfig; import org.apache.samza.container.SamzaContainerMetrics; import org.apache.samza.container.TaskInstance; @@ -170,6 +178,8 @@ public void setUp() throws InterruptedException { Map configMap = new HashMap<>(); configMap.put("stores." + STORE_NAME + ".key.serde", "stringserde"); configMap.put("stores." + STORE_NAME + ".msg.serde", "stringserde"); + configMap.put("stores." + STORE_NAME + ".factory", mockStorageEngineFactory.getClass().getName()); + configMap.put("stores." + STORE_NAME + ".changelog", SYSTEM_NAME + "." + STREAM_NAME); configMap.put("serializers.registry.stringserde.class", StringSerdeFactory.class.getName()); configMap.put(TaskConfig.TRANSACTIONAL_STATE_RETAIN_EXISTING_STATE, "true"); Config config = new MapConfig(configMap); @@ -205,24 +215,36 @@ public Void answer(InvocationOnMock invocation) { new scala.collection.immutable.Map.Map1(new SystemStream(SYSTEM_NAME, STREAM_NAME), systemStreamMetadata)); CheckpointManager checkpointManager = mock(CheckpointManager.class); - when(checkpointManager.readLastCheckpoint(any(TaskName.class))).thenReturn(new Checkpoint(new HashMap<>())); + when(checkpointManager.readLastCheckpoint(any(TaskName.class))).thenReturn(new CheckpointV1(new HashMap<>())); SSPMetadataCache mockSSPMetadataCache = mock(SSPMetadataCache.class); when(mockSSPMetadataCache.getMetadata(any(SystemStreamPartition.class))) .thenReturn(new SystemStreamMetadata.SystemStreamPartitionMetadata("0", "10", "11")); - // Reset the expected number of sysConsumer create, start and stop calls, and store.restore() calls + ContainerContext mockContainerContext = mock(ContainerContext.class); + ContainerModel mockContainerModel = new ContainerModel("samza-container-test", tasks); + when(mockContainerContext.getContainerModel()).thenReturn(mockContainerModel); + + // Reset the expected number of sysConsumer create, start and stop calls, and store.restore() calls this.systemConsumerCreationCount = 0; this.systemConsumerStartCount = 0; this.systemConsumerStopCount = 0; this.storeRestoreCallCount = 0; + StateBackendFactory backendFactory = mock(StateBackendFactory.class); + TaskRestoreManager restoreManager = mock(TaskRestoreManager.class); + when(backendFactory.getRestoreManager(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any())) + .thenReturn(restoreManager); + doAnswer(invocation -> { + storeRestoreCallCount++; + return null; + }).when(restoreManager).restore(); + // Create the container storage manager this.containerStorageManager = new ContainerStorageManager( checkpointManager, - new ContainerModel("samza-container-test", tasks), + mockContainerModel, mockStreamMetadataCache, - mockSSPMetadataCache, mockSystemAdmins, changelogSystemStreams, new HashMap<>(), @@ -233,11 +255,11 @@ public Void answer(InvocationOnMock invocation) { taskInstanceMetrics, samzaContainerMetrics, mock(JobContext.class), - mock(ContainerContext.class), + mockContainerContext, + ImmutableMap.of(StorageConfig.KAFKA_STATE_BACKEND_FACTORY, backendFactory), mock(Map.class), DEFAULT_LOGGED_STORE_BASE_DIR, DEFAULT_STORE_BASE_DIR, - 2, null, new SystemClock()); } @@ -252,10 +274,230 @@ public void testParallelismAndMetrics() throws InterruptedException { mockingDetails(gauge).getInvocations().size() >= 1); } - Assert.assertTrue("Store restore count should be 2 because there are 2 tasks", this.storeRestoreCallCount == 2); - Assert.assertTrue("systemConsumerCreation count should be 1 (1 consumer per system)", - this.systemConsumerCreationCount == 1); - Assert.assertTrue("systemConsumerStopCount count should be 1", this.systemConsumerStopCount == 1); - Assert.assertTrue("systemConsumerStartCount count should be 1", this.systemConsumerStartCount == 1); + Assert.assertEquals("Store restore count should be 2 because there are 2 tasks", 2, this.storeRestoreCallCount); + Assert.assertEquals("systemConsumerCreation count should be 1 (1 consumer per system)", 1, + this.systemConsumerCreationCount); + Assert.assertEquals("systemConsumerStopCount count should be 1", 1, this.systemConsumerStopCount); + Assert.assertEquals("systemConsumerStartCount count should be 1", 1, this.systemConsumerStartCount); + } + + @Test + public void testNoConfiguredDurableStores() throws InterruptedException { + taskRestoreMetricGauges = new HashMap<>(); + this.tasks = new HashMap<>(); + this.taskInstanceMetrics = new HashMap<>(); + + // Add two mocked tasks + addMockedTask("task 0", 0); + addMockedTask("task 1", 1); + + // Mock container metrics + samzaContainerMetrics = mock(SamzaContainerMetrics.class); + when(samzaContainerMetrics.taskStoreRestorationMetrics()).thenReturn(taskRestoreMetricGauges); + + // Create mocked configs for specifying serdes + Map configMap = new HashMap<>(); + configMap.put("serializers.registry.stringserde.class", StringSerdeFactory.class.getName()); + configMap.put(TaskConfig.TRANSACTIONAL_STATE_RETAIN_EXISTING_STATE, "true"); + Config config = new MapConfig(configMap); + + Map> serdes = new HashMap<>(); + serdes.put("stringserde", mock(Serde.class)); + + CheckpointManager checkpointManager = mock(CheckpointManager.class); + when(checkpointManager.readLastCheckpoint(any(TaskName.class))).thenReturn(new CheckpointV1(new HashMap<>())); + + ContainerContext mockContainerContext = mock(ContainerContext.class); + ContainerModel mockContainerModel = new ContainerModel("samza-container-test", tasks); + when(mockContainerContext.getContainerModel()).thenReturn(mockContainerModel); + + // Reset the expected number of sysConsumer create, start and stop calls, and store.restore() calls + this.systemConsumerCreationCount = 0; + this.systemConsumerStartCount = 0; + this.systemConsumerStopCount = 0; + this.storeRestoreCallCount = 0; + + StateBackendFactory backendFactory = mock(StateBackendFactory.class); + TaskRestoreManager restoreManager = mock(TaskRestoreManager.class); + when(backendFactory.getRestoreManager(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any())) + .thenReturn(restoreManager); + doAnswer(invocation -> { + storeRestoreCallCount++; + return null; + }).when(restoreManager).restore(); + + // Create the container storage manager + ContainerStorageManager containerStorageManager = new ContainerStorageManager( + checkpointManager, + mockContainerModel, + mock(StreamMetadataCache.class), + mock(SystemAdmins.class), + new HashMap<>(), + new HashMap<>(), + new HashMap<>(), + new HashMap<>(), + serdes, + config, + taskInstanceMetrics, + samzaContainerMetrics, + mock(JobContext.class), + mockContainerContext, + new HashMap<>(), + mock(Map.class), + DEFAULT_LOGGED_STORE_BASE_DIR, + DEFAULT_STORE_BASE_DIR, + null, + new SystemClock()); + + containerStorageManager.start(); + containerStorageManager.shutdown(); + + for (Gauge gauge : taskRestoreMetricGauges.values()) { + Assert.assertTrue("Restoration time gauge value should never be invoked", + mockingDetails(gauge).getInvocations().size() == 0); + } + + Assert.assertEquals("Store restore count should be 2 because there are 0 stores", 0, this.storeRestoreCallCount); + Assert.assertEquals(0, + this.systemConsumerCreationCount); + Assert.assertEquals(0, this.systemConsumerStopCount); + Assert.assertEquals(0, this.systemConsumerStartCount); + } + + @Test + public void testCheckpointBasedRestoreFactoryCreation() { + Set storeNames = ImmutableSet.of("storeName0", "storeName1", "storeName2"); + + StorageConfig mockConfig = mock(StorageConfig.class); + when(mockConfig.getStoreRestoreFactories("storeName0")) + .thenReturn(ImmutableList.of("factory0", "factory1", "factory2")); + when(mockConfig.getStoreRestoreFactories("storeName1")) + .thenReturn(ImmutableList.of("factory2", "factory1")); + when(mockConfig.getStoreRestoreFactories("storeName2")) + .thenReturn(Collections.emptyList()); + + when(mockConfig.getChangelogStream("storeName0")) + .thenReturn(Optional.empty()); + when(mockConfig.getChangelogStream("storeName1")) + .thenReturn(Optional.of("changelog")); + when(mockConfig.getChangelogStream("storeName2")) + .thenReturn(Optional.of("changelog")); + + CheckpointV1 checkpointV1 = mock(CheckpointV1.class); + when(checkpointV1.getVersion()).thenReturn((short) 1); + Map> factoriesToStores = this.containerStorageManager + .getBackendFactoryStoreNames(checkpointV1, storeNames, mockConfig); + + Assert.assertEquals(1, factoriesToStores.size()); + Assert.assertEquals(ImmutableSet.of("storeName1", "storeName2"), + factoriesToStores.get(StorageConfig.KAFKA_STATE_BACKEND_FACTORY)); + + factoriesToStores = this.containerStorageManager + .getBackendFactoryStoreNames(null, storeNames, mockConfig); + + Assert.assertEquals(2, factoriesToStores.size()); + Assert.assertEquals(ImmutableSet.of("storeName0"), + factoriesToStores.get("factory0")); + Assert.assertEquals(ImmutableSet.of("storeName1"), + factoriesToStores.get("factory2")); + } + + @Test + public void testCheckpointV2BasedRestoreFactoryCreation() { + Set storeNames = ImmutableSet.of("storeName0", "storeName1", "storeName2"); + + StorageConfig mockConfig = mock(StorageConfig.class); + when(mockConfig.getStoreRestoreFactories("storeName0")) + .thenReturn(ImmutableList.of("factory0", "factory1", "factory2")); + when(mockConfig.getStoreRestoreFactories("storeName1")) + .thenReturn(ImmutableList.of("factory2", "factory1")); + when(mockConfig.getStoreRestoreFactories("storeName2")) + .thenReturn(Collections.emptyList()); + + when(mockConfig.getChangelogStream("storeName0")) + .thenReturn(Optional.empty()); + when(mockConfig.getChangelogStream("storeName1")) + .thenReturn(Optional.of("changelog")); + when(mockConfig.getChangelogStream("storeName2")) + .thenReturn(Optional.of("changelog")); + + CheckpointV2 checkpointV2 = mock(CheckpointV2.class); + when(checkpointV2.getVersion()).thenReturn((short) 2); + when(checkpointV2.getStateCheckpointMarkers()) + .thenReturn(ImmutableMap.of( + "factory0", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", ""), + "factory1", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", ""), + "factory2", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", ""))); + + Map> factoriesToStores = this.containerStorageManager + .getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig); + Assert.assertEquals(2, factoriesToStores.size()); + Assert.assertEquals(ImmutableSet.of("storeName0"), + factoriesToStores.get("factory0")); + Assert.assertEquals(ImmutableSet.of("storeName1"), + factoriesToStores.get("factory2")); + + when(checkpointV2.getStateCheckpointMarkers()) + .thenReturn(ImmutableMap.of( + "factory2", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", ""))); + factoriesToStores = this.containerStorageManager + .getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig); + Assert.assertEquals(1, factoriesToStores.size()); + Assert.assertEquals(ImmutableSet.of("storeName1", "storeName0"), + factoriesToStores.get("factory2")); + + when(checkpointV2.getStateCheckpointMarkers()) + .thenReturn(ImmutableMap.of( + "factory1", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", ""), + "factory2", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", ""))); + factoriesToStores = this.containerStorageManager + .getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig); + Assert.assertEquals(2, factoriesToStores.size()); + Assert.assertEquals(ImmutableSet.of("storeName0"), + factoriesToStores.get("factory1")); + Assert.assertEquals(ImmutableSet.of("storeName1"), + factoriesToStores.get("factory2")); + + when(checkpointV2.getStateCheckpointMarkers()) + .thenReturn(ImmutableMap.of( + "factory1", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", ""), + "factory2", ImmutableMap.of("storeName0", "", "storeName2", ""))); + factoriesToStores = this.containerStorageManager + .getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig); + Assert.assertEquals(1, factoriesToStores.size()); + Assert.assertEquals(ImmutableSet.of("storeName0", "storeName1"), + factoriesToStores.get("factory1")); + + when(checkpointV2.getStateCheckpointMarkers()) + .thenReturn(ImmutableMap.of( + "factory1", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", ""))); + factoriesToStores = this.containerStorageManager + .getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig); + Assert.assertEquals(1, factoriesToStores.size()); + Assert.assertEquals(ImmutableSet.of("storeName0", "storeName1"), + factoriesToStores.get("factory1")); + + when(checkpointV2.getStateCheckpointMarkers()) + .thenReturn(Collections.emptyMap()); + factoriesToStores = this.containerStorageManager + .getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig); + Assert.assertEquals(2, factoriesToStores.size()); + Assert.assertEquals(ImmutableSet.of("storeName0"), + factoriesToStores.get("factory0")); + Assert.assertEquals(ImmutableSet.of("storeName1"), + factoriesToStores.get("factory2")); + + when(checkpointV2.getStateCheckpointMarkers()) + .thenReturn(ImmutableMap.of( + "factory0", ImmutableMap.of("storeName1", "", "storeName2", ""), + "factory1", ImmutableMap.of("storeName1", "", "storeName2", ""), + "factory2", ImmutableMap.of("storeName0", "", "storeName2", ""))); + factoriesToStores = this.containerStorageManager + .getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig); + Assert.assertEquals(2, factoriesToStores.size()); + Assert.assertEquals(ImmutableSet.of("storeName1"), + factoriesToStores.get("factory1")); + Assert.assertEquals(ImmutableSet.of("storeName0"), + factoriesToStores.get("factory2")); } } diff --git a/samza-core/src/test/scala/org/apache/samza/storage/TestTransactionalStateTaskStorageManager.java b/samza-core/src/test/scala/org/apache/samza/storage/TestTransactionalStateTaskStorageManager.java deleted file mode 100644 index 244a35b824..0000000000 --- a/samza-core/src/test/scala/org/apache/samza/storage/TestTransactionalStateTaskStorageManager.java +++ /dev/null @@ -1,534 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.samza.storage; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import java.io.FileFilter; -import scala.Option; -import scala.collection.immutable.Map; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Optional; -import org.apache.samza.Partition; -import org.apache.samza.SamzaException; -import org.apache.samza.checkpoint.CheckpointId; -import org.apache.samza.container.TaskName; -import org.apache.samza.job.model.TaskMode; -import org.apache.samza.system.SystemAdmin; -import org.apache.samza.system.SystemAdmins; -import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.util.ScalaJavaUtil; -import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.InOrder; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class TestTransactionalStateTaskStorageManager { - @Test - public void testFlushOrder() { - ContainerStorageManager csm = mock(ContainerStorageManager.class); - StorageEngine mockStore = mock(StorageEngine.class); - java.util.Map taskStores = ImmutableMap.of("mockStore", mockStore); - when(csm.getAllStores(any())).thenReturn(taskStores); - - TransactionalStateTaskStorageManager tsm = spy(buildTSM(csm, mock(Partition.class), new StorageManagerUtil())); - // stub actual method call - doReturn(mock(Map.class)).when(tsm).getNewestChangelogSSPOffsets(any(), any(), any(), any()); - - // invoke flush - tsm.flush(); - - // ensure that stores are flushed before we get newest changelog offsets - InOrder inOrder = inOrder(mockStore, tsm); - inOrder.verify(mockStore).flush(); - inOrder.verify(tsm).getNewestChangelogSSPOffsets(any(), any(), any(), any()); - } - - @Test - public void testGetNewestOffsetsReturnsCorrectOffset() { - ContainerStorageManager csm = mock(ContainerStorageManager.class); - TransactionalStateTaskStorageManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil()); - - TaskName taskName = mock(TaskName.class); - String changelogSystemName = "systemName"; - String storeName = "storeName"; - String changelogStreamName = "changelogName"; - String newestChangelogSSPOffset = "1"; - SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName); - Partition changelogPartition = new Partition(0); - SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); - - Map storeChangelogs = - ScalaJavaUtil.toScalaMap(ImmutableMap.of(storeName, changelogSystemStream)); - - SystemAdmins systemAdmins = mock(SystemAdmins.class); - SystemAdmin systemAdmin = mock(SystemAdmin.class); - SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class); - - when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset); - when(systemAdmins.getSystemAdmin(changelogSystemName)).thenReturn(systemAdmin); - when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(ImmutableMap.of(changelogSSP, metadata)); - - // invoke the method - Map> offsets = - tsm.getNewestChangelogSSPOffsets( - taskName, storeChangelogs, changelogPartition, systemAdmins); - - // verify results - assertEquals(1, offsets.size()); - assertEquals(Option.apply(newestChangelogSSPOffset), offsets.apply(changelogSSP)); - } - - @Test - public void testGetNewestOffsetsReturnsNoneForEmptyTopic() { - // empty topic == null newest offset - ContainerStorageManager csm = mock(ContainerStorageManager.class); - TransactionalStateTaskStorageManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil()); - - TaskName taskName = mock(TaskName.class); - String changelogSystemName = "systemName"; - String storeName = "storeName"; - String changelogStreamName = "changelogName"; - String newestChangelogSSPOffset = null; - SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName); - Partition changelogPartition = new Partition(0); - SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); - - Map storeChangelogs = - ScalaJavaUtil.toScalaMap(ImmutableMap.of(storeName, changelogSystemStream)); - - SystemAdmins systemAdmins = mock(SystemAdmins.class); - SystemAdmin systemAdmin = mock(SystemAdmin.class); - SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class); - - when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset); - when(systemAdmins.getSystemAdmin(changelogSystemName)).thenReturn(systemAdmin); - when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(ImmutableMap.of(changelogSSP, metadata)); - - // invoke the method - Map> offsets = - tsm.getNewestChangelogSSPOffsets( - taskName, storeChangelogs, changelogPartition, systemAdmins); - - // verify results - assertEquals(1, offsets.size()); - assertEquals(Option.empty(), offsets.apply(changelogSSP)); - } - - @Test(expected = SamzaException.class) - public void testGetNewestOffsetsThrowsIfNullMetadata() { - // empty topic == null newest offset - ContainerStorageManager csm = mock(ContainerStorageManager.class); - TransactionalStateTaskStorageManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil()); - - TaskName taskName = mock(TaskName.class); - String changelogSystemName = "systemName"; - String storeName = "storeName"; - String changelogStreamName = "changelogName"; - String newestChangelogSSPOffset = null; - SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName); - Partition changelogPartition = new Partition(0); - SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); - - Map storeChangelogs = - ScalaJavaUtil.toScalaMap(ImmutableMap.of(storeName, changelogSystemStream)); - - SystemAdmins systemAdmins = mock(SystemAdmins.class); - SystemAdmin systemAdmin = mock(SystemAdmin.class); - SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class); - - when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset); - when(systemAdmins.getSystemAdmin(changelogSystemName)).thenReturn(systemAdmin); - when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(null); - - // invoke the method - Map> offsets = - tsm.getNewestChangelogSSPOffsets( - taskName, storeChangelogs, changelogPartition, systemAdmins); - - // verify results - fail("Should have thrown an exception if admin didn't return any metadata"); - } - - @Test(expected = SamzaException.class) - public void testGetNewestOffsetsThrowsIfNullSSPMetadata() { - // empty topic == null newest offset - ContainerStorageManager csm = mock(ContainerStorageManager.class); - TransactionalStateTaskStorageManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil()); - - TaskName taskName = mock(TaskName.class); - String changelogSystemName = "systemName"; - String storeName = "storeName"; - String changelogStreamName = "changelogName"; - String newestChangelogSSPOffset = null; - SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName); - Partition changelogPartition = new Partition(0); - SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); - - Map storeChangelogs = - ScalaJavaUtil.toScalaMap(ImmutableMap.of(storeName, changelogSystemStream)); - - SystemAdmins systemAdmins = mock(SystemAdmins.class); - SystemAdmin systemAdmin = mock(SystemAdmin.class); - SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class); - - when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset); - when(systemAdmins.getSystemAdmin(changelogSystemName)).thenReturn(systemAdmin); - java.util.Map metadataMap = new HashMap() { { - put(changelogSSP, null); - } }; - when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(metadataMap); - - // invoke the method - Map> offsets = - tsm.getNewestChangelogSSPOffsets( - taskName, storeChangelogs, changelogPartition, systemAdmins); - - // verify results - fail("Should have thrown an exception if admin returned null metadata for changelog SSP"); - } - - @Test(expected = SamzaException.class) - public void testGetNewestOffsetsThrowsIfErrorGettingMetadata() { - // empty topic == null newest offset - ContainerStorageManager csm = mock(ContainerStorageManager.class); - TransactionalStateTaskStorageManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil()); - - TaskName taskName = mock(TaskName.class); - String changelogSystemName = "systemName"; - String storeName = "storeName"; - String changelogStreamName = "changelogName"; - String newestChangelogSSPOffset = null; - SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName); - Partition changelogPartition = new Partition(0); - SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); - - Map storeChangelogs = - ScalaJavaUtil.toScalaMap(ImmutableMap.of(storeName, changelogSystemStream)); - - SystemAdmins systemAdmins = mock(SystemAdmins.class); - SystemAdmin systemAdmin = mock(SystemAdmin.class); - SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class); - - when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset); - when(systemAdmins.getSystemAdmin(changelogSystemName)).thenThrow(new SamzaException("Error getting metadata")); - when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(null); - - // invoke the method - Map> offsets = - tsm.getNewestChangelogSSPOffsets( - taskName, storeChangelogs, changelogPartition, systemAdmins); - - // verify results - fail("Should have thrown an exception if admin had an error getting metadata"); - } - - @Test - public void testCheckpoint() { - ContainerStorageManager csm = mock(ContainerStorageManager.class); - - StorageEngine mockLPStore = mock(StorageEngine.class); - StoreProperties lpStoreProps = mock(StoreProperties.class); - when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps); - when(lpStoreProps.isPersistedToDisk()).thenReturn(true); - when(lpStoreProps.isLoggedStore()).thenReturn(true); - Path mockPath = mock(Path.class); - when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath)); - - StorageEngine mockPStore = mock(StorageEngine.class); - StoreProperties pStoreProps = mock(StoreProperties.class); - when(mockPStore.getStoreProperties()).thenReturn(pStoreProps); - when(pStoreProps.isPersistedToDisk()).thenReturn(true); - when(pStoreProps.isLoggedStore()).thenReturn(false); - - StorageEngine mockLIStore = mock(StorageEngine.class); - StoreProperties liStoreProps = mock(StoreProperties.class); - when(mockLIStore.getStoreProperties()).thenReturn(liStoreProps); - when(liStoreProps.isPersistedToDisk()).thenReturn(false); - when(liStoreProps.isLoggedStore()).thenReturn(true); - - StorageEngine mockIStore = mock(StorageEngine.class); - StoreProperties iStoreProps = mock(StoreProperties.class); - when(mockIStore.getStoreProperties()).thenReturn(iStoreProps); - when(iStoreProps.isPersistedToDisk()).thenReturn(false); - when(iStoreProps.isLoggedStore()).thenReturn(false); - - java.util.Map taskStores = ImmutableMap.of( - "loggedPersistentStore", mockLPStore, - "persistentStore", mockPStore, - "loggedInMemStore", mockLIStore, - "inMemStore", mockIStore - ); - when(csm.getAllStores(any())).thenReturn(taskStores); - - TransactionalStateTaskStorageManager tsm = spy(buildTSM(csm, mock(Partition.class), new StorageManagerUtil())); - // stub actual method call - ArgumentCaptor checkpointPathsCaptor = ArgumentCaptor.forClass(Map.class); - doNothing().when(tsm).writeChangelogOffsetFiles(any(), any(), any()); - - Map> offsets = ScalaJavaUtil.toScalaMap( - ImmutableMap.of(mock(SystemStreamPartition.class), Option.apply("1"))); - - // invoke checkpoint - tsm.checkpoint(CheckpointId.create(), offsets); - - // ensure that checkpoint is never called for non-logged persistent stores since they're - // always cleared on restart. - verify(mockPStore, never()).checkpoint(any()); - // ensure that checkpoint is never called for in-memory stores since they're not persistent. - verify(mockIStore, never()).checkpoint(any()); - verify(mockLIStore, never()).checkpoint(any()); - verify(tsm).writeChangelogOffsetFiles(checkpointPathsCaptor.capture(), any(), eq(offsets)); - Map checkpointPaths = checkpointPathsCaptor.getValue(); - assertEquals(1, checkpointPaths.size()); - assertEquals(mockPath, checkpointPaths.apply("loggedPersistentStore")); - } - - @Test(expected = IllegalStateException.class) - public void testCheckpointFailsIfErrorCreatingCheckpoint() { - ContainerStorageManager csm = mock(ContainerStorageManager.class); - - StorageEngine mockLPStore = mock(StorageEngine.class); - StoreProperties lpStoreProps = mock(StoreProperties.class); - when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps); - when(lpStoreProps.isPersistedToDisk()).thenReturn(true); - when(lpStoreProps.isLoggedStore()).thenReturn(true); - when(mockLPStore.checkpoint(any())).thenThrow(new IllegalStateException()); - java.util.Map taskStores = - ImmutableMap.of("loggedPersistentStore", mockLPStore); - when(csm.getAllStores(any())).thenReturn(taskStores); - - TransactionalStateTaskStorageManager tsm = spy(buildTSM(csm, mock(Partition.class), new StorageManagerUtil())); - - Map> offsets = ScalaJavaUtil.toScalaMap( - ImmutableMap.of(mock(SystemStreamPartition.class), Option.apply("1"))); - - // invoke checkpoint - tsm.checkpoint(CheckpointId.create(), offsets); - verify(tsm, never()).writeChangelogOffsetFiles(any(), any(), any()); - fail("Should have thrown an exception if error creating store checkpoint"); - } - - @Test(expected = SamzaException.class) - public void testCheckpointFailsIfErrorWritingOffsetFiles() { - ContainerStorageManager csm = mock(ContainerStorageManager.class); - - StorageEngine mockLPStore = mock(StorageEngine.class); - StoreProperties lpStoreProps = mock(StoreProperties.class); - when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps); - when(lpStoreProps.isPersistedToDisk()).thenReturn(true); - when(lpStoreProps.isLoggedStore()).thenReturn(true); - Path mockPath = mock(Path.class); - when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath)); - java.util.Map taskStores = - ImmutableMap.of("loggedPersistentStore", mockLPStore); - when(csm.getAllStores(any())).thenReturn(taskStores); - - TransactionalStateTaskStorageManager tsm = spy(buildTSM(csm, mock(Partition.class), new StorageManagerUtil())); - doThrow(new SamzaException("Error writing offset file")) - .when(tsm).writeChangelogOffsetFiles(any(), any(), any()); - - Map> offsets = ScalaJavaUtil.toScalaMap( - ImmutableMap.of(mock(SystemStreamPartition.class), Option.apply("1"))); - - // invoke checkpoint - tsm.checkpoint(CheckpointId.create(), offsets); - - fail("Should have thrown an exception if error writing offset file."); - } - - @Test - public void testWriteChangelogOffsetFiles() throws IOException { - String storeName = "mockStore"; - ContainerStorageManager csm = mock(ContainerStorageManager.class); - StorageEngine mockStore = mock(StorageEngine.class); - java.util.Map taskStores = ImmutableMap.of(storeName, mockStore); - when(csm.getAllStores(any())).thenReturn(taskStores); - - Partition changelogPartition = new Partition(0); - SystemStream changelogSS = new SystemStream("system", "changelog"); - SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSS, changelogPartition); - StorageManagerUtil smu = spy(new StorageManagerUtil()); - File mockCurrentStoreDir = mock(File.class); - doReturn(mockCurrentStoreDir).when(smu).getTaskStoreDir(any(), eq(storeName), any(), any()); - doNothing().when(smu).writeOffsetFile(eq(mockCurrentStoreDir), any(), anyBoolean()); - TransactionalStateTaskStorageManager tsm = spy(buildTSM(csm, changelogPartition, smu)); - - String changelogNewestOffset = "1"; - Map> offsets = ScalaJavaUtil.toScalaMap( - ImmutableMap.of(changelogSSP, Option.apply(changelogNewestOffset))); - - Path checkpointPath = Files.createTempDirectory("store-checkpoint-test").toAbsolutePath(); - - Map checkpointPaths = ScalaJavaUtil.toScalaMap( - ImmutableMap.of(storeName, checkpointPath)); - Map storeChangelogs = ScalaJavaUtil.toScalaMap( - ImmutableMap.of(storeName, changelogSS)); - - // invoke method - tsm.writeChangelogOffsetFiles(checkpointPaths, storeChangelogs, offsets); - - // verify that offset file was written to the checkpoint dir - java.util.Map fileOffsets = new StorageManagerUtil() - .readOffsetFile(checkpointPath.toFile(), ImmutableSet.of(changelogSSP), false); - assertEquals(1, fileOffsets.size()); - assertEquals(changelogNewestOffset, fileOffsets.get(changelogSSP)); - - // verify that offset file write was called on the current dir - verify(smu, times(1)).writeOffsetFile(eq(mockCurrentStoreDir), any(), anyBoolean()); - } - - @Test - public void testWriteChangelogOffsetFilesWithEmptyChangelogTopic() throws IOException { - String storeName = "mockStore"; - ContainerStorageManager csm = mock(ContainerStorageManager.class); - StorageEngine mockStore = mock(StorageEngine.class); - java.util.Map taskStores = ImmutableMap.of(storeName, mockStore); - when(csm.getAllStores(any())).thenReturn(taskStores); - - Partition changelogPartition = new Partition(0); - SystemStream changelogSS = new SystemStream("system", "changelog"); - SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSS, changelogPartition); - StorageManagerUtil mockSMU = mock(StorageManagerUtil.class); - File mockCurrentStoreDir = mock(File.class); - when(mockSMU.getTaskStoreDir(any(), eq(storeName), any(), any())).thenReturn(mockCurrentStoreDir); - TransactionalStateTaskStorageManager tsm = spy(buildTSM(csm, changelogPartition, mockSMU)); - - String changelogNewestOffset = null; - Map> offsets = ScalaJavaUtil.toScalaMap( - ImmutableMap.of(changelogSSP, Option.apply(changelogNewestOffset))); - - Path checkpointPath = Files.createTempDirectory("store-checkpoint-test").toAbsolutePath(); - - Map checkpointPaths = ScalaJavaUtil.toScalaMap( - ImmutableMap.of(storeName, checkpointPath)); - Map storeChangelogs = ScalaJavaUtil.toScalaMap( - ImmutableMap.of(storeName, changelogSS)); - - // invoke method - tsm.writeChangelogOffsetFiles(checkpointPaths, storeChangelogs, offsets); - - // verify that the offset files were not written to the checkpoint dir - assertFalse(Files.exists(new File(checkpointPath.toFile(), StorageManagerUtil.OFFSET_FILE_NAME_LEGACY).toPath())); - assertFalse(Files.exists(new File(checkpointPath.toFile(), StorageManagerUtil.OFFSET_FILE_NAME_NEW).toPath())); - java.util.Map fileOffsets = new StorageManagerUtil() - .readOffsetFile(checkpointPath.toFile(), ImmutableSet.of(changelogSSP), false); - assertEquals(0, fileOffsets.size()); - - // verify that delete was called on current store dir offset file - verify(mockSMU, times(1)).deleteOffsetFile(eq(mockCurrentStoreDir)); - } - - /** - * This should never happen with CheckpointingTaskStorageManager. #getNewestChangelogSSPOffset must - * return a key for every changelog SSP. If the SSP is empty, the value should be none. If it could - * not fetch metadata, it should throw an exception instead of skipping the SSP. - * If this contract is accidentally broken, ensure that we fail the commit - */ - @Test(expected = SamzaException.class) - public void testWriteChangelogOffsetFilesWithNoChangelogOffset() throws IOException { - String storeName = "mockStore"; - ContainerStorageManager csm = mock(ContainerStorageManager.class); - StorageEngine mockStore = mock(StorageEngine.class); - java.util.Map taskStores = ImmutableMap.of(storeName, mockStore); - when(csm.getAllStores(any())).thenReturn(taskStores); - - Partition changelogPartition = new Partition(0); - SystemStream changelogSS = new SystemStream("system", "changelog"); - SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSS, changelogPartition); - TransactionalStateTaskStorageManager tsm = spy(buildTSM(csm, changelogPartition, new StorageManagerUtil())); - - // no mapping present for changelog newest offset - Map> offsets = ScalaJavaUtil.toScalaMap(ImmutableMap.of()); - - Path checkpointPath = Files.createTempDirectory("store-checkpoint-test").toAbsolutePath(); - Map checkpointPaths = ScalaJavaUtil.toScalaMap( - ImmutableMap.of(storeName, checkpointPath)); - Map storeChangelogs = ScalaJavaUtil.toScalaMap( - ImmutableMap.of(storeName, changelogSS)); - - // invoke method - tsm.writeChangelogOffsetFiles(checkpointPaths, storeChangelogs, offsets); - - fail("Should have thrown an exception if no changelog offset found for checkpointed store"); - } - - @Test - public void testRemoveOldCheckpointsWhenBaseDirContainsRegularFiles() { - TaskName taskName = new TaskName("Partition 0"); - ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class); - Map changelogSystemStreams = mock(Map.class); - SystemAdmins systemAdmins = mock(SystemAdmins.class); - File loggedStoreBaseDir = mock(File.class); - Partition changelogPartition = new Partition(0); - TaskMode taskMode = TaskMode.Active; - StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class); - - File mockStoreDir = mock(File.class); - String mockStoreDirName = "notDirectory"; - - when(loggedStoreBaseDir.listFiles()).thenReturn(new File[] {mockStoreDir}); - when(mockStoreDir.getName()).thenReturn(mockStoreDirName); - when(storageManagerUtil.getTaskStoreDir(eq(loggedStoreBaseDir), eq(mockStoreDirName), eq(taskName), eq(taskMode))).thenReturn(mockStoreDir); - // null here can happen if listFiles is called on a non-directory - when(mockStoreDir.listFiles(any(FileFilter.class))).thenReturn(null); - - TransactionalStateTaskStorageManager tsm = new TransactionalStateTaskStorageManager(taskName, containerStorageManager, - changelogSystemStreams, systemAdmins, loggedStoreBaseDir, changelogPartition, taskMode, storageManagerUtil); - - tsm.removeOldCheckpoints(CheckpointId.create()); - } - - private TransactionalStateTaskStorageManager buildTSM(ContainerStorageManager csm, Partition changelogPartition, - StorageManagerUtil smu) { - TaskName taskName = new TaskName("Partition 0"); - Map changelogSystemStreams = mock(Map.class); - SystemAdmins systemAdmins = mock(SystemAdmins.class); - File loggedStoreBaseDir = mock(File.class); - TaskMode taskMode = TaskMode.Active; - - return new TransactionalStateTaskStorageManager( - taskName, csm, changelogSystemStreams, systemAdmins, - loggedStoreBaseDir, changelogPartition, taskMode, smu); - } -} \ No newline at end of file diff --git a/samza-core/src/test/scala/org/apache/samza/util/TestFileUtil.scala b/samza-core/src/test/scala/org/apache/samza/util/TestFileUtil.scala index 6604ae0262..104f983d9a 100644 --- a/samza-core/src/test/scala/org/apache/samza/util/TestFileUtil.scala +++ b/samza-core/src/test/scala/org/apache/samza/util/TestFileUtil.scala @@ -21,11 +21,15 @@ package org.apache.samza.util -import java.io.{File, FileInputStream, FileOutputStream, ObjectInputStream, ObjectOutputStream} +import org.apache.samza.testUtils.FileUtil -import org.junit.Assert.{assertEquals, assertNull, assertTrue} +import java.io.{File, FileInputStream, FileOutputStream, ObjectInputStream, ObjectOutputStream} +import org.junit.Assert.{assertEquals, assertNull, assertTrue, fail} import org.junit.Test +import java.nio.file.{FileAlreadyExistsException, Files, Paths} +import scala.util.Random + class TestFileUtil { val data = "100" val fileUtil = new FileUtil() @@ -104,4 +108,45 @@ class TestFileUtil { // Check data returned assertNull(result) } + + /** + * Files.createDirectories fails with a FileAlreadyExistsException if the last directory + * in the path already exists but is a symlink to another directory. It works correctly + * if one of the intermediate directory is a symlink. Verify this behavior and + * test that the util method handles this correctly. + */ + @Test + def testCreateDirectoriesWithSymlinks(): Unit = { + /** + * Directory structure: + * /tmp/samza-file-util-RANDOM + * /tmp/samza-file-util-RANDOM-symlink (symlink to dir above) + * /tmp/samza-file-util-RANDOM/subdir (created via the symlink above) + */ + val tmpDirPath = Paths.get(FileUtil.TMP_DIR) + val tmpSubDirName = "samza-file-util-" + Random.nextInt() + val tmpSubDirSymlinkName = tmpSubDirName + "-symlink" + + val tmpSubDirPath = Paths.get(FileUtil.TMP_DIR, tmpSubDirName); + fileUtil.createDirectories(tmpSubDirPath) + + val tmpSymlinkPath = Paths.get(FileUtil.TMP_DIR, tmpSubDirSymlinkName) + Files.createSymbolicLink(tmpSymlinkPath, tmpDirPath); + + try { + Files.createDirectories(tmpSymlinkPath) + fail("Should have thrown a FileAlreadyExistsException since last dir in path already " + + "exists and is a symlink") + } catch { + case e: FileAlreadyExistsException => + // ignore and continue + } + + // test that the util method handles this correctly and does not throw an exception + fileUtil.createDirectories(tmpSymlinkPath) + + // verify that subdirs can be created via symlinks correctly. + val tmpSubSubDirPath = Paths.get(FileUtil.TMP_DIR, tmpSubDirName + "-symlink", "subdir") + fileUtil.createDirectories(tmpSubSubDirPath) + } } diff --git a/samza-kafka/src/main/java/org/apache/samza/checkpoint/kafka/KafkaCheckpointLogKey.java b/samza-kafka/src/main/java/org/apache/samza/checkpoint/kafka/KafkaCheckpointLogKey.java index a732abaddf..97443e3ec8 100644 --- a/samza-kafka/src/main/java/org/apache/samza/checkpoint/kafka/KafkaCheckpointLogKey.java +++ b/samza-kafka/src/main/java/org/apache/samza/checkpoint/kafka/KafkaCheckpointLogKey.java @@ -19,6 +19,8 @@ package org.apache.samza.checkpoint.kafka; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; import org.apache.samza.container.TaskName; /** @@ -26,7 +28,12 @@ */ public class KafkaCheckpointLogKey { - public static final String CHECKPOINT_KEY_TYPE = "checkpoint"; + public static final String CHECKPOINT_V1_KEY_TYPE = "checkpoint"; + public static final String CHECKPOINT_V2_KEY_TYPE = "checkpoint-v2"; + public static final Map CHECKPOINT_KEY_VERSIONS = ImmutableMap.of( + CHECKPOINT_V1_KEY_TYPE, (short) 1, + CHECKPOINT_V2_KEY_TYPE, (short) 2 + ); /** * The SystemStreamPartitionGrouperFactory configured for this job run. Since, checkpoints of different * groupers are not compatible, we persist and validate them across job runs. diff --git a/samza-kafka/src/main/java/org/apache/samza/checkpoint/kafka/KafkaCheckpointLogKeySerde.java b/samza-kafka/src/main/java/org/apache/samza/checkpoint/kafka/KafkaCheckpointLogKeySerde.java index e738190f3a..59ff34fad3 100644 --- a/samza-kafka/src/main/java/org/apache/samza/checkpoint/kafka/KafkaCheckpointLogKeySerde.java +++ b/samza-kafka/src/main/java/org/apache/samza/checkpoint/kafka/KafkaCheckpointLogKeySerde.java @@ -24,7 +24,6 @@ import org.apache.samza.container.TaskName; import org.apache.samza.serializers.Serde; -import java.util.Arrays; import java.util.LinkedHashMap; /** @@ -59,11 +58,11 @@ public KafkaCheckpointLogKey fromBytes(byte[] bytes) { try { LinkedHashMap deserializedKey = MAPPER.readValue(bytes, LinkedHashMap.class); - return new KafkaCheckpointLogKey(deserializedKey.get(TYPE_FIELD), new TaskName(deserializedKey.get(TASK_NAME_FIELD)), deserializedKey.get(SSP_GROUPER_FACTORY_FIELD) - ); + return new KafkaCheckpointLogKey(deserializedKey.get(TYPE_FIELD), new TaskName(deserializedKey.get(TASK_NAME_FIELD)), + deserializedKey.get(SSP_GROUPER_FACTORY_FIELD)); } catch (Exception e) { throw new SamzaException(String.format("Exception in de-serializing checkpoint bytes: %s", - Arrays.toString(bytes)), e); + new String(bytes)), e); } } } diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaConsumerProxy.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaConsumerProxy.java index 8fd21fe92b..5c061aec3f 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaConsumerProxy.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaConsumerProxy.java @@ -152,9 +152,8 @@ public void start() { } if (topicPartitionToSSP.size() == 0) { - String msg = String.format("Cannot start KafkaConsumerProxy without any registered TopicPartitions for %s", systemName); - LOG.error(msg); - throw new SamzaException(msg); + String msg = String.format("Started KafkaConsumerProxy without any registered TopicPartitions for %s", systemName); + LOG.warn(msg); } } diff --git a/samza-kafka/src/main/scala/org/apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala b/samza-kafka/src/main/scala/org/apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala index d88048e5cf..7dbb9b3c65 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala @@ -22,20 +22,19 @@ package org.apache.samza.checkpoint.kafka import java.util.Collections import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicReference - import com.google.common.annotations.VisibleForTesting import com.google.common.base.Preconditions -import org.apache.samza.checkpoint.{Checkpoint, CheckpointManager} +import org.apache.samza.checkpoint.{Checkpoint, CheckpointManager, CheckpointV1, CheckpointV2} import org.apache.samza.config.{Config, JobConfig, TaskConfig} import org.apache.samza.container.TaskName -import org.apache.samza.serializers.Serde +import org.apache.samza.serializers.{CheckpointV1Serde, CheckpointV2Serde, Serde} import org.apache.samza.metrics.MetricsRegistry -import org.apache.samza.serializers.CheckpointSerde import org.apache.samza.system._ import org.apache.samza.system.kafka.KafkaStreamSpec -import org.apache.samza.util.{ExponentialSleepStrategy, Logging} +import org.apache.samza.util.Logging import org.apache.samza.{Partition, SamzaException} +import java.{lang, util} import scala.collection.mutable /** @@ -53,7 +52,8 @@ class KafkaCheckpointManager(checkpointSpec: KafkaStreamSpec, validateCheckpoint: Boolean, config: Config, metricsRegistry: MetricsRegistry, - checkpointMsgSerde: Serde[Checkpoint] = new CheckpointSerde, + checkpointV1MsgSerde: Serde[CheckpointV1] = new CheckpointV1Serde, + checkpointV2MsgSerde: Serde[CheckpointV2] = new CheckpointV2Serde, checkpointKeySerde: Serde[KafkaCheckpointLogKey] = new KafkaCheckpointLogKeySerde) extends CheckpointManager with Logging { var MaxRetryDurationInMillis: Long = TimeUnit.MINUTES.toMillis(15) @@ -81,6 +81,8 @@ class KafkaCheckpointManager(checkpointSpec: KafkaStreamSpec, // for active containers, this will be set to true, while false for standby containers. val stopConsumerAfterFirstRead: Boolean = new TaskConfig(config).getCheckpointManagerConsumerStopAfterFirstRead + val checkpointReadVersions: util.List[lang.Short] = new TaskConfig(config).getCheckpointReadVersions + /** * Create checkpoint stream prior to start. * @@ -176,7 +178,7 @@ class KafkaCheckpointManager(checkpointSpec: KafkaStreamSpec, } catch { case exception: Exception => { producerException = exception - warn(s"Retrying failed checkpoint write for checkpoint: $checkpoint for task: $taskName", exception) + warn(s"Retrying failed write for checkpoint: $checkpoint for task: $taskName", exception) // TODO: Remove this producer recreation logic after SAMZA-1393. val newProducer: SystemProducer = getSystemProducer() producerCreationLock.synchronized { @@ -264,10 +266,10 @@ class KafkaCheckpointManager(checkpointSpec: KafkaStreamSpec, checkpointKeySerde.fromBytes(keyBytes) } catch { case e: Exception => if (validateCheckpoint) { - throw new SamzaException(s"Exception while serializing checkpoint-key. " + + throw new SamzaException(s"Exception while deserializing checkpoint-key. " + s"Topic: $checkpointTopic Offset: $offset", e) } else { - warn(s"Ignoring exception while serializing checkpoint-key. Topic: $checkpointTopic Offset: $offset", e) + warn(s"Ignoring exception while deserializing checkpoint-key. Topic: $checkpointTopic Offset: $offset", e) null } } @@ -283,17 +285,28 @@ class KafkaCheckpointManager(checkpointSpec: KafkaStreamSpec, } } - // If the type of the key is not KafkaCheckpointLogKey.CHECKPOINT_KEY_TYPE, it can safely be ignored. - if (KafkaCheckpointLogKey.CHECKPOINT_KEY_TYPE.equals(checkpointKey.getType)) { - val checkpointBytes = checkpointEnvelope.getMessage.asInstanceOf[Array[Byte]] - val checkpoint = try { - checkpointMsgSerde.fromBytes(checkpointBytes) - } catch { - case e: Exception => throw new SamzaException(s"Exception while serializing checkpoint-message. " + - s"Topic: $checkpointTopic Offset: $offset", e) + val msgBytes = checkpointEnvelope.getMessage.asInstanceOf[Array[Byte]] + try { + // if checkpoint key version does not match configured checkpoint version to read, skip the message. + if (checkpointReadVersions.contains( + KafkaCheckpointLogKey.CHECKPOINT_KEY_VERSIONS.get(checkpointKey.getType))) { + if (!checkpoints.contains(checkpointKey.getTaskName) || + shouldOverrideCheckpoint(checkpoints.get(checkpointKey.getTaskName), checkpointKey)) { + checkpoints.put(checkpointKey.getTaskName, deserializeCheckpoint(checkpointKey, msgBytes)) + } // else ignore the de-prioritized checkpoint + } else { + // Ignore and skip the unknown checkpoint key type. We do not want to throw any exceptions for this case + // for forwards compatibility with new checkpoints versions in the checkpoint topic + warn(s"Ignoring unknown checkpoint key type for checkpoint key: $checkpointKey") } - - checkpoints.put(checkpointKey.getTaskName, checkpoint) + } catch { + case e: Exception => + if (validateCheckpoint) { + throw new SamzaException(s"Exception while deserializing checkpoint-message. " + + s"Topic: $checkpointTopic Offset: $offset", e) + } else { + warn(s"Ignoring exception while deserializing checkpoint-msg. Topic: $checkpointTopic Offset: $offset", e) + } } } } @@ -322,20 +335,68 @@ class KafkaCheckpointManager(checkpointSpec: KafkaStreamSpec, partitionMetaData.getOldestOffset } - @VisibleForTesting - def buildOutgoingMessageEnvelope(taskName: TaskName, checkpoint: Checkpoint): OutgoingMessageEnvelope = { - val key = new KafkaCheckpointLogKey(KafkaCheckpointLogKey.CHECKPOINT_KEY_TYPE, taskName, expectedGrouperFactory) - val keyBytes = try { - checkpointKeySerde.toBytes(key) - } catch { - case e: Exception => throw new SamzaException(s"Exception when writing checkpoint-key for $taskName: $checkpoint", e) + def buildOutgoingMessageEnvelope[T <: Checkpoint](taskName: TaskName, checkpoint: T): OutgoingMessageEnvelope = { + checkpoint match { + case checkpointV1: CheckpointV1 => { + val key = new KafkaCheckpointLogKey( + KafkaCheckpointLogKey.CHECKPOINT_V1_KEY_TYPE, taskName, expectedGrouperFactory) + val keyBytes = try { + checkpointKeySerde.toBytes(key) + } catch { + case e: Exception => + throw new SamzaException(s"Exception when writing checkpoint-key for $taskName: $checkpoint", e) + } + val msgBytes = try { + checkpointV1MsgSerde.toBytes(checkpointV1) + } catch { + case e: Exception => + throw new SamzaException(s"Exception when writing checkpoint for $taskName: $checkpoint", e) + } + new OutgoingMessageEnvelope(checkpointSsp, keyBytes, msgBytes) + } + case checkpointV2: CheckpointV2 => { + val key = new KafkaCheckpointLogKey( + KafkaCheckpointLogKey.CHECKPOINT_V2_KEY_TYPE, taskName, expectedGrouperFactory) + val keyBytes = try { + checkpointKeySerde.toBytes(key) + } catch { + case e: Exception => + throw new SamzaException(s"Exception when writing checkpoint-key for $taskName: $checkpoint", e) + } + val msgBytes = try { + checkpointV2MsgSerde.toBytes(checkpointV2) + } catch { + case e: Exception => + throw new SamzaException(s"Exception when writing checkpoint for $taskName: $checkpoint", e) + } + new OutgoingMessageEnvelope(checkpointSsp, keyBytes, msgBytes) + } + case _ => throw new SamzaException("Unknown checkpoint version: " + checkpoint.getVersion) } - val msgBytes = try { - checkpointMsgSerde.toBytes(checkpoint) - } catch { - case e: Exception => throw new SamzaException(s"Exception when writing checkpoint for $taskName: $checkpoint", e) + } + + private def shouldOverrideCheckpoint(currentCheckpoint: Option[Checkpoint], + newCheckpointKey: KafkaCheckpointLogKey): Boolean = { + val newCheckpointVersion = KafkaCheckpointLogKey.CHECKPOINT_KEY_VERSIONS.get(newCheckpointKey.getType) + if (newCheckpointVersion == null) { + // Unknown checkpoint version + throw new IllegalArgumentException("Unknown checkpoint key type: " + newCheckpointKey.getType + + " for checkpoint key: " + newCheckpointKey) } + // Override checkpoint if the current checkpoint does not exist or if new checkpoint has a higher restore + // priority than the currently written checkpoint + currentCheckpoint.isEmpty || + checkpointReadVersions.indexOf(newCheckpointVersion) <= + checkpointReadVersions.indexOf(currentCheckpoint.get.getVersion) + } - new OutgoingMessageEnvelope(checkpointSsp, keyBytes, msgBytes) + private def deserializeCheckpoint(checkpointKey: KafkaCheckpointLogKey, checkpointMsgBytes: Array[Byte]): Checkpoint = { + if (KafkaCheckpointLogKey.CHECKPOINT_V1_KEY_TYPE.equals(checkpointKey.getType)) { + checkpointV1MsgSerde.fromBytes(checkpointMsgBytes) + } else if (KafkaCheckpointLogKey.CHECKPOINT_V2_KEY_TYPE.equals(checkpointKey.getType)) { + checkpointV2MsgSerde.fromBytes(checkpointMsgBytes) + } else { + throw new IllegalArgumentException("Unknown checkpoint key type: " + checkpointKey.getType) + } } } diff --git a/samza-kafka/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointLogKeySerde.java b/samza-kafka/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointLogKeySerde.java index 614aaba43c..08fa02c972 100644 --- a/samza-kafka/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointLogKeySerde.java +++ b/samza-kafka/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointLogKeySerde.java @@ -29,7 +29,7 @@ public class TestKafkaCheckpointLogKeySerde { @Test public void testBinaryCompatibility() { - KafkaCheckpointLogKey logKey1 = new KafkaCheckpointLogKey(KafkaCheckpointLogKey.CHECKPOINT_KEY_TYPE, + KafkaCheckpointLogKey logKey1 = new KafkaCheckpointLogKey(KafkaCheckpointLogKey.CHECKPOINT_V1_KEY_TYPE, new TaskName("Partition 0"), GroupByPartitionFactory.class.getCanonicalName()); KafkaCheckpointLogKeySerde checkpointSerde = new KafkaCheckpointLogKeySerde(); @@ -43,7 +43,7 @@ public void testBinaryCompatibility() { @Test public void testSerde() { - KafkaCheckpointLogKey key = new KafkaCheckpointLogKey(KafkaCheckpointLogKey.CHECKPOINT_KEY_TYPE, + KafkaCheckpointLogKey key = new KafkaCheckpointLogKey(KafkaCheckpointLogKey.CHECKPOINT_V1_KEY_TYPE, new TaskName("Partition 0"), GroupByPartitionFactory.class.getCanonicalName()); KafkaCheckpointLogKeySerde checkpointSerde = new KafkaCheckpointLogKeySerde(); @@ -51,6 +51,16 @@ public void testSerde() { Assert.assertEquals(key, checkpointSerde.fromBytes(checkpointSerde.toBytes(key))); } + @Test + public void testCheckpointTypeV2() { + KafkaCheckpointLogKey keyV2 = new KafkaCheckpointLogKey(KafkaCheckpointLogKey.CHECKPOINT_V2_KEY_TYPE, new TaskName("Partition 0"), + GroupByPartitionFactory.class.getCanonicalName()); + KafkaCheckpointLogKeySerde checkpointKeySerde = new KafkaCheckpointLogKeySerde(); + + // test that deserialize(serialize(k)) == k + Assert.assertEquals(keyV2, checkpointKeySerde.fromBytes(checkpointKeySerde.toBytes(keyV2))); + } + @Test public void testForwardsCompatibility() { // Set the key to another value, this is for the future if we want to support multiple checkpoint keys diff --git a/samza-kafka/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManagerJava.java b/samza-kafka/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManagerJava.java index f32040a6bf..d0e927f55c 100644 --- a/samza-kafka/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManagerJava.java +++ b/samza-kafka/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManagerJava.java @@ -24,13 +24,14 @@ import kafka.common.TopicAlreadyMarkedForDeletionException; import org.apache.samza.Partition; import org.apache.samza.SamzaException; -import org.apache.samza.checkpoint.Checkpoint; +import org.apache.samza.checkpoint.CheckpointV1; import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.container.TaskName; import org.apache.samza.container.grouper.stream.GroupByPartitionFactory; import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.serializers.CheckpointSerde; +import org.apache.samza.serializers.CheckpointV1Serde; +import org.apache.samza.serializers.CheckpointV2Serde; import org.apache.samza.system.IncomingMessageEnvelope; import org.apache.samza.system.StreamValidationException; import org.apache.samza.system.SystemAdmin; @@ -72,7 +73,7 @@ public void testStartFailsOnTopicCreationErrors() { SystemFactory factory = newFactory(mock(SystemProducer.class), mock(SystemConsumer.class), mockAdmin); KafkaCheckpointManager checkpointManager = new KafkaCheckpointManager(checkpointSpec, factory, - true, mock(Config.class), mock(MetricsRegistry.class), null, new KafkaCheckpointLogKeySerde()); + true, mock(Config.class), mock(MetricsRegistry.class), null, null, new KafkaCheckpointLogKeySerde()); // expect an exception during startup checkpointManager.createResources(); @@ -91,7 +92,7 @@ public void testStartFailsOnTopicValidationErrors() { SystemFactory factory = newFactory(mock(SystemProducer.class), mock(SystemConsumer.class), mockAdmin); KafkaCheckpointManager checkpointManager = new KafkaCheckpointManager(checkpointSpec, factory, - true, mock(Config.class), mock(MetricsRegistry.class), null, new KafkaCheckpointLogKeySerde()); + true, mock(Config.class), mock(MetricsRegistry.class), null, null, new KafkaCheckpointLogKeySerde()); // expect an exception during startup checkpointManager.createResources(); @@ -116,7 +117,7 @@ public void testReadFailsOnSerdeExceptions() throws Exception { // wire up an exception throwing serde with the checkpointmanager KafkaCheckpointManager checkpointManager = new KafkaCheckpointManager(checkpointSpec, factory, - true, mockConfig, mock(MetricsRegistry.class), new ExceptionThrowingCheckpointSerde(), new KafkaCheckpointLogKeySerde()); + true, mockConfig, mock(MetricsRegistry.class), new ExceptionThrowingCheckpointV1Serde(), null, new KafkaCheckpointLogKeySerde()); checkpointManager.register(TASK1); checkpointManager.start(); @@ -142,7 +143,7 @@ public void testReadSucceedsOnKeySerdeExceptionsWhenValidationIsDisabled() throw // wire up an exception throwing serde with the checkpointmanager KafkaCheckpointManager checkpointManager = new KafkaCheckpointManager(checkpointSpec, factory, - false, mockConfig, mock(MetricsRegistry.class), new ExceptionThrowingCheckpointSerde(), + false, mockConfig, mock(MetricsRegistry.class), new ExceptionThrowingCheckpointV1Serde(), null, new ExceptionThrowingCheckpointKeySerde()); checkpointManager.register(TASK1); checkpointManager.start(); @@ -167,7 +168,8 @@ public void testCheckpointsAreReadFromOldestOffset() throws Exception { SystemAdmin mockAdmin = newAdmin(oldestOffset, "1"); SystemFactory factory = newFactory(mock(SystemProducer.class), mockConsumer, mockAdmin); KafkaCheckpointManager checkpointManager = new KafkaCheckpointManager(checkpointSpec, factory, - true, mockConfig, mock(MetricsRegistry.class), new CheckpointSerde(), new KafkaCheckpointLogKeySerde()); + true, mockConfig, mock(MetricsRegistry.class), new CheckpointV1Serde(), new CheckpointV2Serde(), + new KafkaCheckpointLogKeySerde()); checkpointManager.register(TASK1); // 1. verify that consumer.register is called only during checkpointManager.start. @@ -206,12 +208,13 @@ public void testAllMessagesInTheLogAreRead() throws Exception { SystemFactory factory = newFactory(mock(SystemProducer.class), mockConsumer, mockAdmin); KafkaCheckpointManager checkpointManager = new KafkaCheckpointManager(checkpointSpec, factory, - true, mockConfig, mock(MetricsRegistry.class), new CheckpointSerde(), new KafkaCheckpointLogKeySerde()); + true, mockConfig, mock(MetricsRegistry.class), new CheckpointV1Serde(), new CheckpointV2Serde(), + new KafkaCheckpointLogKeySerde()); checkpointManager.register(TASK1); checkpointManager.start(); // check that all ten messages are read, and the checkpoint is the newest message - Checkpoint checkpoint = checkpointManager.readLastCheckpoint(TASK1); + CheckpointV1 checkpoint = (CheckpointV1) checkpointManager.readLastCheckpoint(TASK1); Assert.assertEquals(checkpoint.getOffsets(), ImmutableMap.of(ssp, Integer.toString(newestOffset))); } @@ -261,15 +264,15 @@ private IncomingMessageEnvelope newCheckpointEnvelope(TaskName taskName, SystemS new KafkaCheckpointLogKey("checkpoint", taskName, GROUPER_FACTORY_CLASS); KafkaCheckpointLogKeySerde checkpointKeySerde = new KafkaCheckpointLogKeySerde(); - Checkpoint checkpointMsg = new Checkpoint(ImmutableMap.of(ssp, offset)); - CheckpointSerde checkpointMsgSerde = new CheckpointSerde(); + CheckpointV1 checkpointMsg = new CheckpointV1(ImmutableMap.of(ssp, offset)); + CheckpointV1Serde checkpointMsgSerde = new CheckpointV1Serde(); return new IncomingMessageEnvelope(CHECKPOINT_SSP, offset, checkpointKeySerde.toBytes(checkpointKey), checkpointMsgSerde.toBytes(checkpointMsg)); } - private static class ExceptionThrowingCheckpointSerde extends CheckpointSerde { - public Checkpoint fromBytes(byte[] bytes) { + private static class ExceptionThrowingCheckpointV1Serde extends CheckpointV1Serde { + public CheckpointV1 fromBytes(byte[] bytes) { throw new KafkaException("exception"); } } diff --git a/samza-core/src/test/scala/org/apache/samza/storage/TestTaskStorageManager.scala b/samza-kafka/src/test/java/org/apache/samza/storage/TestTaskStorageManager.scala similarity index 83% rename from samza-core/src/test/scala/org/apache/samza/storage/TestTaskStorageManager.scala rename to samza-kafka/src/test/java/org/apache/samza/storage/TestTaskStorageManager.scala index 957c00ccc0..4c8d8e489c 100644 --- a/samza-core/src/test/scala/org/apache/samza/storage/TestTaskStorageManager.scala +++ b/samza-kafka/src/test/java/org/apache/samza/storage/TestTaskStorageManager.scala @@ -22,24 +22,27 @@ package org.apache.samza.storage import java.io.{File, FileOutputStream, ObjectOutputStream} import java.util +import com.google.common.collect.{ImmutableMap, ImmutableSet} import org.apache.samza.Partition +import org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker +import org.apache.samza.checkpoint.{CheckpointId, CheckpointManager, CheckpointV1} import org.apache.samza.config._ import org.apache.samza.container.{SamzaContainerMetrics, TaskInstanceMetrics, TaskName} import org.apache.samza.context.{ContainerContext, JobContext} -import org.apache.samza.job.model.{ContainerModel, TaskMode, TaskModel} +import org.apache.samza.job.model.{ContainerModel, JobModel, TaskMode, TaskModel} import org.apache.samza.serializers.{Serde, StringSerdeFactory} import org.apache.samza.storage.StoreProperties.StorePropertiesBuilder import org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata import org.apache.samza.system._ import org.apache.samza.task.TaskInstanceCollector -import org.apache.samza.util.{FileUtil, SystemClock} +import org.apache.samza.util.{Clock, FileUtil, SystemClock} import org.junit.Assert._ import org.junit.runner.RunWith import org.junit.runners.Parameterized import org.junit.runners.Parameterized.Parameters import org.junit.{After, Before, Test} import org.mockito.Matchers._ -import org.mockito.{Mockito} +import org.mockito.Mockito import org.mockito.Mockito._ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer @@ -48,8 +51,6 @@ import org.scalatest.mockito.MockitoSugar import scala.collection.JavaConverters._ import scala.collection.immutable.HashMap import scala.collection.mutable -import com.google.common.collect.{ImmutableMap, ImmutableSet} -import org.apache.samza.checkpoint.{Checkpoint, CheckpointManager} /** * This test is parameterized on the offsetFileName and is run for both @@ -58,7 +59,7 @@ import org.apache.samza.checkpoint.{Checkpoint, CheckpointManager} * @param offsetFileName the name of the offset file. */ @RunWith(value = classOf[Parameterized]) -class TestNonTransactionalStateTaskStorageManager(offsetFileName: String) extends MockitoSugar { +class TestKafkaNonTransactionalStateTaskBackupManager(offsetFileName: String) extends MockitoSugar { val store = "store1" val loggedStore = "loggedStore1" @@ -131,44 +132,6 @@ class TestNonTransactionalStateTaskStorageManager(offsetFileName: String) extend assertTrue(storeFile.exists()) assertFalse(offsetFile.exists()) verify(mockSystemConsumer).register(ssp, "0") - - // Test 2: flush should update the offset file - taskManager.flush() - assertTrue(offsetFile.exists()) - validateOffsetFileContents(offsetFile, "kafka.testStream-loggedStore1.0", "50") - - // Test 3: Update sspMetadata before shutdown and verify that offset file is not updated - when(mockSystemAdmin.getSSPMetadata(ImmutableSet.of(ssp))) - .thenReturn(ImmutableMap.of(ssp, new SystemStreamPartitionMetadata("0", "100", "101"))) - taskManager.stop() - verify(mockStorageEngine, times(1)).flush() // only called once during Test 2. - assertTrue(storeFile.exists()) - assertTrue(offsetFile.exists()) - validateOffsetFileContents(offsetFile, "kafka.testStream-loggedStore1.0", "50") - - // Test 4: Initialize again with an updated sspMetadata; Verify that it restores from the correct offset - sspMetadata = new SystemStreamPartitionMetadata("0", "150", "151") - metadata = new SystemStreamMetadata(getStreamName(loggedStore), new java.util.HashMap[Partition, SystemStreamPartitionMetadata]() { - { - put(partition, sspMetadata) - } - }) - when(mockStreamMetadataCache.getStreamMetadata(any(), any())).thenReturn(Map(ss -> metadata)) - when(mockSystemAdmin.getSSPMetadata(ImmutableSet.of(ssp))) - .thenReturn(ImmutableMap.of(ssp, sspMetadata)) - when(mockSystemAdmin.getOffsetsAfter(Map(ssp -> "50").asJava)).thenReturn(Map(ssp -> "51").asJava) - Mockito.reset(mockSystemConsumer) - - taskManager = new TaskStorageManagerBuilder() - .addStore(loggedStore, mockStorageEngine, mockSystemConsumer) - .setStreamMetadataCache(mockStreamMetadataCache) - .setSystemAdmin("kafka", mockSystemAdmin) - .initializeContainerStorageManager() - .build - - assertTrue(storeFile.exists()) - assertTrue(offsetFile.exists()) - verify(mockSystemConsumer).register(ssp, "51") } /** @@ -217,7 +180,9 @@ class TestNonTransactionalStateTaskStorageManager(offsetFileName: String) extend verify(mockSystemConsumer).register(ssp, "0") // Test 2: flush should NOT create/update the offset file. Store directory has no files - taskManager.flush() + val checkpointId = CheckpointId.create() + val snapshot = taskManager.snapshot(checkpointId) + val stateCheckpointMarkers = taskManager.upload(checkpointId, snapshot) assertTrue(storeDirectory.list().isEmpty) // Test 3: Update sspMetadata before shutdown and verify that offset file is NOT created @@ -228,7 +193,7 @@ class TestNonTransactionalStateTaskStorageManager(offsetFileName: String) extend }) when(mockStreamMetadataCache.getStreamMetadata(any(), any())).thenReturn(Map(ss -> metadata)) when(mockSystemAdmin.getSSPMetadata(ImmutableSet.of(ssp))).thenReturn(ImmutableMap.of(ssp, sspMetadata)) - taskManager.stop() + taskManager.close() assertTrue(storeDirectory.list().isEmpty) // Test 4: Initialize again with an updated sspMetadata; Verify that it restores from the earliest offset @@ -368,51 +333,12 @@ class TestNonTransactionalStateTaskStorageManager(offsetFileName: String) extend .build //Invoke test method - taskStorageManager.stop() + taskStorageManager.close() //Check conditions assertFalse("Offset file doesn't exist!", offsetFile.exists()) } - /** - * Given that the SSPMetadataCache returns metadata, flush should create the offset files. - */ - @Test - def testFlushCreatesOffsetFileForLoggedStore() { - val partition = new Partition(0) - - val offsetFilePath = new File(storageManagerUtil.getTaskStoreDir(TaskStorageManagerBuilder.defaultLoggedStoreBaseDir, loggedStore, taskName, TaskMode.Active) + File.separator + offsetFileName) - val anotherOffsetPath = new File( - storageManagerUtil.getTaskStoreDir(TaskStorageManagerBuilder.defaultLoggedStoreBaseDir, store, taskName, TaskMode.Active) + File.separator + offsetFileName) - - val ssp1 = new SystemStreamPartition("kafka", getStreamName(loggedStore), partition) - val ssp2 = new SystemStreamPartition("kafka", getStreamName(store), partition) - val sspMetadata = new SystemStreamPartitionMetadata("20", "100", "101") - - val mockSystemAdmin = mock[SystemAdmin] - when(mockSystemAdmin.getSSPMetadata(ImmutableSet.of(ssp1))).thenReturn(ImmutableMap.of(ssp1, sspMetadata)) - when(mockSystemAdmin.getSSPMetadata(ImmutableSet.of(ssp2))).thenReturn(ImmutableMap.of(ssp2, sspMetadata)) - - //Build TaskStorageManager - val taskStorageManager = new TaskStorageManagerBuilder() - .addLoggedStore(loggedStore, true) - .addStore(store, false) - .setSystemAdmin("kafka", mockSystemAdmin) - .setStreamMetadataCache(createMockStreamMetadataCache("20", "100", "101")) - .setPartition(partition) - .initializeContainerStorageManager() - .build - - //Invoke test method - taskStorageManager.flush() - - //Check conditions - assertTrue("Offset file doesn't exist!", offsetFilePath.exists()) - validateOffsetFileContents(offsetFilePath, "kafka.testStream-loggedStore1.0", "100") - - assertTrue("Offset file got created for a store that is not persisted to the disk!!", !anotherOffsetPath.exists()) - } - /** * Flush should delete the existing OFFSET file if the changelog partition (for some reason) becomes empty */ @@ -420,8 +346,6 @@ class TestNonTransactionalStateTaskStorageManager(offsetFileName: String) extend def testFlushDeletesOffsetFileForLoggedStoreForEmptyPartition() { val partition = new Partition(0) - val offsetFilePath = new File(storageManagerUtil.getTaskStoreDir(TaskStorageManagerBuilder.defaultLoggedStoreBaseDir, loggedStore, taskName, TaskMode.Active) + File.separator + offsetFileName) - val ssp = new SystemStreamPartition("kafka", getStreamName(loggedStore), partition) val sspMetadata = new SystemStreamPartitionMetadata("0", "100", "101") val nullSspMetadata = new SystemStreamPartitionMetadata(null, null, null) @@ -430,7 +354,7 @@ class TestNonTransactionalStateTaskStorageManager(offsetFileName: String) extend .thenReturn(ImmutableMap.of(ssp, sspMetadata)) .thenReturn(ImmutableMap.of(ssp, nullSspMetadata)) - var metadata = new SystemStreamMetadata(getStreamName(loggedStore), new java.util.HashMap[Partition, SystemStreamPartitionMetadata]() { + val metadata = new SystemStreamMetadata(getStreamName(loggedStore), new java.util.HashMap[Partition, SystemStreamPartitionMetadata]() { { put(partition, sspMetadata) } @@ -449,67 +373,15 @@ class TestNonTransactionalStateTaskStorageManager(offsetFileName: String) extend .build //Invoke test method - taskStorageManager.flush() - - //Check conditions - assertTrue("Offset file doesn't exist!", offsetFilePath.exists()) - validateOffsetFileContents(offsetFilePath, "kafka.testStream-loggedStore1.0", "100") + val checkpointId = CheckpointId.create() + var snapshot = taskStorageManager.snapshot(checkpointId) + taskStorageManager.upload(checkpointId, snapshot) //Invoke test method again - taskStorageManager.flush() + snapshot = taskStorageManager.snapshot(checkpointId) + val stateCheckpointMarkers2 = taskStorageManager.upload(checkpointId, snapshot) - //Check conditions - assertFalse("Offset file for null offset exists!", offsetFilePath.exists()) - } - - @Test - def testFlushOverwritesOffsetFileForLoggedStore() { - val partition = new Partition(0) - val ssp = new SystemStreamPartition("kafka", getStreamName(loggedStore), partition) - - val offsetFilePath = new File(storageManagerUtil.getTaskStoreDir(TaskStorageManagerBuilder.defaultLoggedStoreBaseDir, loggedStore, taskName, TaskMode.Active) + File.separator + offsetFileName) - fileUtil.writeWithChecksum(offsetFilePath, "100") - - val sspMetadata = new SystemStreamPartitionMetadata("20", "139", "140") - val mockSystemAdmin = mock[SystemAdmin] - when(mockSystemAdmin.getSSPMetadata(ImmutableSet.of(ssp))).thenReturn(ImmutableMap.of(ssp, sspMetadata)) - - - var metadata = new SystemStreamMetadata(getStreamName(loggedStore), new java.util.HashMap[Partition, SystemStreamPartitionMetadata]() { - { - put(partition, sspMetadata) - } - }) - - val mockStreamMetadataCache = mock[StreamMetadataCache] - when(mockStreamMetadataCache.getStreamMetadata(any(), any())).thenReturn(Map(new SystemStream("kafka", getStreamName(loggedStore)) -> metadata)) - - //Build TaskStorageManager - val taskStorageManager = new TaskStorageManagerBuilder() - .addLoggedStore(loggedStore, true) - .setSystemAdmin("kafka", mockSystemAdmin) - .setPartition(partition) - .setStreamMetadataCache(mockStreamMetadataCache) - .initializeContainerStorageManager() - .build - - //Invoke test method - taskStorageManager.flush() - - //Check conditions - assertTrue("Offset file doesn't exist!", offsetFilePath.exists()) - validateOffsetFileContents(offsetFilePath, "kafka.testStream-loggedStore1.0", "139") - - // Flush again - when(mockSystemAdmin.getSSPMetadata(ImmutableSet.of(ssp))) - .thenReturn(ImmutableMap.of(ssp, new SystemStreamPartitionMetadata("20", "193", "194"))) - - //Invoke test method - taskStorageManager.flush() - - //Check conditions - assertTrue("Offset file doesn't exist!", offsetFilePath.exists()) - validateOffsetFileContents(offsetFilePath, "kafka.testStream-loggedStore1.0", "193") + assertNull(KafkaStateCheckpointMarker.deserialize(stateCheckpointMarkers2.get.get(loggedStore)).getChangelogOffset) } /** @@ -517,7 +389,7 @@ class TestNonTransactionalStateTaskStorageManager(offsetFileName: String) extend * The legacy offset file only contains the offset as a string, while the new offset file contains a map of * ssp to offset in json format. * The name of the two offset files are given in {@link StorageManagerUtil.OFFSET_FILE_NAME_NEW} and - * {@link StorageManagerUtil.OFFSET_FILE_LEGACY}. + * {@link StorageManagerUtil.OFFSET_FILE_NAME_LEGACY}. */ private def validateOffsetFileContents(offsetFile: File, ssp: String, offset: String): Unit = { @@ -549,7 +421,7 @@ class TestNonTransactionalStateTaskStorageManager(offsetFileName: String) extend .build //Invoke test method - taskStorageManager.stop() + taskStorageManager.close() //Check conditions assertTrue("Offset file should not exist!", !offsetFilePath.exists()) @@ -777,7 +649,7 @@ class TestNonTransactionalStateTaskStorageManager(offsetFileName: String) extend } } -object TestNonTransactionalStateTaskStorageManager { +object TestKafkaNonTransactionalStateTaskBackupManager { @Parameters def parameters: util.Collection[Array[String]] = { val offsetFileNames = new util.ArrayList[Array[String]]() @@ -864,7 +736,10 @@ class TaskStorageManagerBuilder extends MockitoSugar { var containerModel = new ContainerModel("container", tasks.asJava) val mockSystemAdmins = Mockito.mock(classOf[SystemAdmins]) - Mockito.when(mockSystemAdmins.getSystemAdmin(org.mockito.Matchers.eq("kafka"))).thenReturn(systemAdminsMap.get("kafka").get) + Mockito.when(mockSystemAdmins.getSystemAdmin(org.mockito.Matchers.eq("kafka"))) + .thenReturn(systemAdminsMap.get("kafka").get) + Mockito.when(mockSystemAdmins.getSystemAdmins) + .thenReturn(systemAdminsMap.asJava) var mockStorageEngineFactory : StorageEngineFactory[AnyRef, AnyRef] = Mockito.mock(classOf[StorageEngineFactory[AnyRef, AnyRef]]) @@ -892,23 +767,32 @@ class TaskStorageManagerBuilder extends MockitoSugar { "stores.loggedStore1.clean.on.container.start" -> cleanStoreDirsOnStart.toString, "stores.store1.key.serde" -> classOf[StringSerdeFactory].getCanonicalName, "stores.store1.msg.serde" -> classOf[StringSerdeFactory].getCanonicalName, + "stores.store1.factory" -> mockStorageEngineFactory.getClass.getName, + "stores.store1.changelog" -> "system.stream", + "stores.loggedStore1.factory" -> mockStorageEngineFactory.getClass.getName, "stores.loggedStore1.key.serde" -> classOf[StringSerdeFactory].getCanonicalName, "stores.loggedStore1.msg.serde" -> classOf[StringSerdeFactory].getCanonicalName, + "stores.loggedStore1.changelog" -> "system.stream", TaskConfig.TRANSACTIONAL_STATE_RESTORE_ENABLED -> "false").asJava) var mockSerdes: Map[String, Serde[AnyRef]] = HashMap[String, Serde[AnyRef]]((classOf[StringSerdeFactory].getCanonicalName, Mockito.mock(classOf[Serde[AnyRef]]))) val mockCheckpointManager = Mockito.mock(classOf[CheckpointManager]) when(mockCheckpointManager.readLastCheckpoint(any(classOf[TaskName]))) - .thenReturn(new Checkpoint(new util.HashMap[SystemStreamPartition, String]())) + .thenReturn(new CheckpointV1(new util.HashMap[SystemStreamPartition, String]())) + + val mockContainerContext = Mockito.mock(classOf[ContainerContext]) + when(mockContainerContext.getContainerModel).thenReturn(containerModel); - val mockSSPMetadataCache = Mockito.mock(classOf[SSPMetadataCache]) + val mockJobContext = Mockito.mock(classOf[JobContext]) + val mockJobModel = Mockito.mock(classOf[JobModel]) + when(mockJobContext.getJobModel).thenReturn(mockJobModel) + when(mockJobModel.getMaxChangeLogStreamPartitions).thenReturn(1) containerStorageManager = new ContainerStorageManager( mockCheckpointManager, containerModel, streamMetadataCache, - mockSSPMetadataCache, mockSystemAdmins, changeLogSystemStreams.asJava, Map[String, util.Set[SystemStream]]().asJava, @@ -918,12 +802,13 @@ class TaskStorageManagerBuilder extends MockitoSugar { config, new HashMap[TaskName, TaskInstanceMetrics]().asJava, Mockito.mock(classOf[SamzaContainerMetrics]), - Mockito.mock(classOf[JobContext]), - Mockito.mock(classOf[ContainerContext]), + mockJobContext, + mockContainerContext, + ImmutableMap.of(StorageConfig.KAFKA_STATE_BACKEND_FACTORY, + new mockKafkaChangelogBackendManager(changeLogSystemStreams)), new HashMap[TaskName, TaskInstanceCollector].asJava, loggedStoreBaseDir, TaskStorageManagerBuilder.defaultStoreBaseDir, - 1, null, new SystemClock) this @@ -931,18 +816,16 @@ class TaskStorageManagerBuilder extends MockitoSugar { - def build: NonTransactionalStateTaskStorageManager = { + def build: KafkaNonTransactionalStateTaskBackupManager = { if (containerStorageManager != null) { containerStorageManager.start() } - new NonTransactionalStateTaskStorageManager( + new KafkaNonTransactionalStateTaskBackupManager( taskName = taskName, - containerStorageManager = containerStorageManager, - storeChangelogs = changeLogSystemStreams, + storeChangelogs = changeLogSystemStreams.asJava, systemAdmins = buildSystemAdmins(systemAdminsMap), - loggedStoreBaseDir = loggedStoreBaseDir, partition = partition ) } @@ -954,4 +837,12 @@ class TaskStorageManagerBuilder extends MockitoSugar { } systemAdmins } + + private class mockKafkaChangelogBackendManager(storeSystemStream: Map[String, SystemStream]) + extends KafkaChangelogStateBackendFactory { + override def filterStandbySystemStreams(changelogSystemStreams: util.Map[String, SystemStream], containerModel: ContainerModel): + util.Map[String, SystemStream] = storeSystemStream.asJava + + override def getStreamCache(admins: SystemAdmins, clock: Clock): StreamMetadataCache = streamMetadataCache + } } diff --git a/samza-kafka/src/test/java/org/apache/samza/storage/TestTransactionalStateTaskBackupManager.java b/samza-kafka/src/test/java/org/apache/samza/storage/TestTransactionalStateTaskBackupManager.java new file mode 100644 index 0000000000..8bb93f769f --- /dev/null +++ b/samza-kafka/src/test/java/org/apache/samza/storage/TestTransactionalStateTaskBackupManager.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker; + +import java.util.HashMap; +import java.util.concurrent.ForkJoinPool; +import org.apache.samza.Partition; +import org.apache.samza.SamzaException; +import org.apache.samza.checkpoint.CheckpointId; +import org.apache.samza.container.TaskInstanceMetrics; +import org.apache.samza.container.TaskName; +import org.apache.samza.metrics.Timer; +import org.apache.samza.system.SystemAdmin; +import org.apache.samza.system.SystemAdmins; +import org.apache.samza.system.SystemStream; +import org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata; +import org.apache.samza.system.SystemStreamPartition; +import org.junit.Test; +import org.mockito.InOrder; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + + +public class TestTransactionalStateTaskBackupManager { + @Test + public void testFlushOrder() { + ContainerStorageManager csm = mock(ContainerStorageManager.class); + StorageEngine mockStore = mock(StorageEngine.class); + java.util.Map taskStores = ImmutableMap.of("mockStore", mockStore); + when(csm.getAllStores(any())).thenReturn(taskStores); + when(mockStore.getStoreProperties()).thenReturn(new StoreProperties + .StorePropertiesBuilder().setPersistedToDisk(true).setLoggedStore(true).build()); + TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class); + Timer checkpointTimer = mock(Timer.class); + when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer); + + KafkaTransactionalStateTaskBackupManager tsm = spy(buildTSM(csm, mock(Partition.class), new StorageManagerUtil())); + TaskStorageCommitManager commitManager = new TaskStorageCommitManager(new TaskName("task"), + ImmutableMap.of("kafka", tsm), csm, null, null, null, null, + ForkJoinPool.commonPool(), new StorageManagerUtil(), null, metrics); + // stub actual method call + doReturn(mock(java.util.Map.class)).when(tsm).getNewestChangelogSSPOffsets(any(), any(), any(), any()); + + // invoke Kafka flush + commitManager.init(); + commitManager.snapshot(CheckpointId.create()); + + // ensure that stores are flushed before we get newest changelog offsets + InOrder inOrder = inOrder(mockStore, tsm); + inOrder.verify(mockStore).flush(); + inOrder.verify(tsm).getNewestChangelogSSPOffsets(any(), any(), any(), any()); + } + + @Test + public void testGetNewestOffsetsReturnsCorrectOffset() { + ContainerStorageManager csm = mock(ContainerStorageManager.class); + KafkaTransactionalStateTaskBackupManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil()); + + TaskName taskName = mock(TaskName.class); + String changelogSystemName = "systemName"; + String storeName = "storeName"; + String changelogStreamName = "changelogName"; + String newestChangelogSSPOffset = "1"; + SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName); + Partition changelogPartition = new Partition(0); + SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); + + java.util.Map storeChangelogs = new HashMap<>(); + storeChangelogs.put(storeName, changelogSystemStream); + + SystemAdmins systemAdmins = mock(SystemAdmins.class); + SystemAdmin systemAdmin = mock(SystemAdmin.class); + SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class); + + when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset); + when(systemAdmins.getSystemAdmin(changelogSystemName)).thenReturn(systemAdmin); + when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(ImmutableMap.of(changelogSSP, metadata)); + + // invoke the method + java.util.Map stateCheckpointMarkerMap = + tsm.getNewestChangelogSSPOffsets( + taskName, storeChangelogs, changelogPartition, systemAdmins); + + // verify results + assertEquals(1, stateCheckpointMarkerMap.size()); + KafkaStateCheckpointMarker kscm = KafkaStateCheckpointMarker.deserialize(stateCheckpointMarkerMap.get(storeName)); + assertEquals(newestChangelogSSPOffset, kscm.getChangelogOffset()); + assertEquals(changelogSSP, kscm.getChangelogSSP()); + } + + @Test + public void testGetNewestOffsetsReturnsNoneForEmptyTopic() { + // empty topic == null newest offset + ContainerStorageManager csm = mock(ContainerStorageManager.class); + KafkaTransactionalStateTaskBackupManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil()); + + TaskName taskName = mock(TaskName.class); + String changelogSystemName = "systemName"; + String storeName = "storeName"; + String changelogStreamName = "changelogName"; + String newestChangelogSSPOffset = null; + SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName); + Partition changelogPartition = new Partition(0); + SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); + + java.util.Map storeChangelogs = new HashMap(); + storeChangelogs.put(storeName, changelogSystemStream); + + SystemAdmins systemAdmins = mock(SystemAdmins.class); + SystemAdmin systemAdmin = mock(SystemAdmin.class); + SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class); + + when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset); + when(systemAdmins.getSystemAdmin(changelogSystemName)).thenReturn(systemAdmin); + when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(ImmutableMap.of(changelogSSP, metadata)); + + // invoke the method + java.util.Map stateCheckpointMarkerMap = + tsm.getNewestChangelogSSPOffsets( + taskName, storeChangelogs, changelogPartition, systemAdmins); + + // verify results + assertEquals(1, stateCheckpointMarkerMap.size()); + KafkaStateCheckpointMarker kscm = KafkaStateCheckpointMarker.deserialize(stateCheckpointMarkerMap.get(storeName)); + assertEquals(changelogSSP, kscm.getChangelogSSP()); + assertNull(kscm.getChangelogOffset()); + } + + @Test(expected = SamzaException.class) + public void testGetNewestOffsetsThrowsIfNullMetadata() { + // empty topic == null newest offset + ContainerStorageManager csm = mock(ContainerStorageManager.class); + KafkaTransactionalStateTaskBackupManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil()); + + TaskName taskName = mock(TaskName.class); + String changelogSystemName = "systemName"; + String storeName = "storeName"; + String changelogStreamName = "changelogName"; + String newestChangelogSSPOffset = null; + SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName); + Partition changelogPartition = new Partition(0); + SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); + + java.util.Map storeChangelogs = new HashMap<>(); + storeChangelogs.put(storeName, changelogSystemStream); + + SystemAdmins systemAdmins = mock(SystemAdmins.class); + SystemAdmin systemAdmin = mock(SystemAdmin.class); + SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class); + + when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset); + when(systemAdmins.getSystemAdmin(changelogSystemName)).thenReturn(systemAdmin); + when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(null); + + // invoke the method + java.util.Map offsets = + tsm.getNewestChangelogSSPOffsets( + taskName, storeChangelogs, changelogPartition, systemAdmins); + + // verify results + fail("Should have thrown an exception if admin didn't return any metadata"); + } + + @Test(expected = SamzaException.class) + public void testGetNewestOffsetsThrowsIfNullSSPMetadata() { + // empty topic == null newest offset + ContainerStorageManager csm = mock(ContainerStorageManager.class); + KafkaTransactionalStateTaskBackupManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil()); + + TaskName taskName = mock(TaskName.class); + String changelogSystemName = "systemName"; + String storeName = "storeName"; + String changelogStreamName = "changelogName"; + String newestChangelogSSPOffset = null; + SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName); + Partition changelogPartition = new Partition(0); + SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); + + java.util.Map storeChangelogs = new HashMap<>(); + storeChangelogs.put(storeName, changelogSystemStream); + + SystemAdmins systemAdmins = mock(SystemAdmins.class); + SystemAdmin systemAdmin = mock(SystemAdmin.class); + SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class); + + when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset); + when(systemAdmins.getSystemAdmin(changelogSystemName)).thenReturn(systemAdmin); + java.util.Map metadataMap = new HashMap() { { + put(changelogSSP, null); + } }; + when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(metadataMap); + + // invoke the method + java.util.Map offsets = + tsm.getNewestChangelogSSPOffsets( + taskName, storeChangelogs, changelogPartition, systemAdmins); + + // verify results + fail("Should have thrown an exception if admin returned null metadata for changelog SSP"); + } + + @Test(expected = SamzaException.class) + public void testGetNewestOffsetsThrowsIfErrorGettingMetadata() { + // empty topic == null newest offset + ContainerStorageManager csm = mock(ContainerStorageManager.class); + KafkaTransactionalStateTaskBackupManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil()); + + TaskName taskName = mock(TaskName.class); + String changelogSystemName = "systemName"; + String storeName = "storeName"; + String changelogStreamName = "changelogName"; + String newestChangelogSSPOffset = null; + SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName); + Partition changelogPartition = new Partition(0); + SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition); + + java.util.Map storeChangelogs = new HashMap<>(); + storeChangelogs.put(storeName, changelogSystemStream); + + SystemAdmins systemAdmins = mock(SystemAdmins.class); + SystemAdmin systemAdmin = mock(SystemAdmin.class); + SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class); + + when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset); + when(systemAdmins.getSystemAdmin(changelogSystemName)).thenThrow(new SamzaException("Error getting metadata")); + when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(null); + + // invoke the method + java.util.Map offsets = + tsm.getNewestChangelogSSPOffsets( + taskName, storeChangelogs, changelogPartition, systemAdmins); + + // verify results + fail("Should have thrown an exception if admin had an error getting metadata"); + } + + private KafkaTransactionalStateTaskBackupManager buildTSM(ContainerStorageManager csm, Partition changelogPartition, + StorageManagerUtil smu) { + TaskName taskName = new TaskName("Partition 0"); + java.util.Map changelogSystemStreams = mock(java.util.Map.class); + SystemAdmins systemAdmins = mock(SystemAdmins.class); + + return new KafkaTransactionalStateTaskBackupManager( + taskName, changelogSystemStreams, systemAdmins, changelogPartition); + } +} \ No newline at end of file diff --git a/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala b/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala index 2671067561..835f53ed22 100644 --- a/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala +++ b/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala @@ -20,16 +20,15 @@ package org.apache.samza.checkpoint.kafka import java.util.Properties - import kafka.integration.KafkaServerTestHarness import kafka.utils.{CoreUtils, TestUtils} import com.google.common.collect.ImmutableMap -import org.apache.samza.checkpoint.Checkpoint +import org.apache.samza.checkpoint.{Checkpoint, CheckpointId, CheckpointV1, CheckpointV2} import org.apache.samza.config._ import org.apache.samza.container.TaskName import org.apache.samza.container.grouper.stream.GroupByPartitionFactory import org.apache.samza.metrics.MetricsRegistry -import org.apache.samza.serializers.CheckpointSerde +import org.apache.samza.serializers.{CheckpointV1Serde, CheckpointV2Serde} import org.apache.samza.system._ import org.apache.samza.system.kafka.{KafkaStreamSpec, KafkaSystemFactory} import org.apache.samza.util.ScalaJavaUtil.JavaOptionals @@ -48,8 +47,8 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness { val sspGrouperFactoryName = classOf[GroupByPartitionFactory].getCanonicalName val ssp = new SystemStreamPartition("kafka", "topic", new Partition(0)) - val checkpoint1 = new Checkpoint(ImmutableMap.of(ssp, "offset-1")) - val checkpoint2 = new Checkpoint(ImmutableMap.of(ssp, "offset-2")) + val checkpoint1 = new CheckpointV1(ImmutableMap.of(ssp, "offset-1")) + val checkpoint2 = new CheckpointV1(ImmutableMap.of(ssp, "offset-2")) val taskName = new TaskName("Partition 0") var config: Config = null @@ -88,7 +87,7 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness { checkPointManager.register(taskName) checkPointManager.start - checkPointManager.writeCheckpoint(taskName, new Checkpoint(ImmutableMap.of())) + checkPointManager.writeCheckpoint(taskName, new CheckpointV1(ImmutableMap.of())) checkPointManager.stop() // Verifications after the test @@ -127,7 +126,7 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness { } @Test - def testReadCheckpointShouldIgnoreUnknownCheckpointKeys(): Unit = { + def testCheckpointV1AndV2WriteAndReadV1(): Unit = { val checkpointTopic = "checkpoint-topic-1" val kcm1 = createKafkaCheckpointManager(checkpointTopic) kcm1.register(taskName) @@ -146,21 +145,169 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness { val readCp = readCheckpoint(checkpointTopic, taskName) assertNull(readCp) - // skips unknown checkpoints from checkpoint topic - writeCheckpoint(checkpointTopic, taskName, checkpoint1, "checkpoint-v2") + val checkpointV1 = new CheckpointV1(ImmutableMap.of(ssp, "offset-1")) + val checkpointV2 = new CheckpointV2(CheckpointId.create(), ImmutableMap.of(ssp, "offset-2"), + ImmutableMap.of("factory1", ImmutableMap.of("store1", "changelogOffset"))) + + // skips v2 checkpoints from checkpoint topic + writeCheckpoint(checkpointTopic, taskName, checkpointV2) assertNull(readCheckpoint(checkpointTopic, taskName)) // reads latest v1 checkpoints - writeCheckpoint(checkpointTopic, taskName, checkpoint1) - assertEquals(checkpoint1, readCheckpoint(checkpointTopic, taskName)) + writeCheckpoint(checkpointTopic, taskName, checkpointV1) + assertEquals(checkpointV1, readCheckpoint(checkpointTopic, taskName)) // writing checkpoint v2 still returns the previous v1 checkpoint - writeCheckpoint(checkpointTopic, taskName, checkpoint2, "checkpoint-v2") - assertEquals(checkpoint1, readCheckpoint(checkpointTopic, taskName)) + writeCheckpoint(checkpointTopic, taskName, checkpointV2) + assertEquals(checkpointV1, readCheckpoint(checkpointTopic, taskName)) + } + + @Test + def testCheckpointV1AndV2WriteAndReadV2(): Unit = { + val checkpointTopic = "checkpoint-topic-1" + val kcm1 = createKafkaCheckpointManager(checkpointTopic) + kcm1.register(taskName) + kcm1.createResources + kcm1.start + kcm1.stop + + // check that start actually creates the topic with log compaction enabled + val topicConfig = adminZkClient.getAllTopicConfigs().getOrElse(checkpointTopic, new Properties()) + + assertEquals(topicConfig, new KafkaConfig(config).getCheckpointTopicProperties()) + assertEquals("compact", topicConfig.get("cleanup.policy")) + assertEquals("26214400", topicConfig.get("segment.bytes")) + + // read before topic exists should result in a null checkpoint + val readCp = readCheckpoint(checkpointTopic, taskName) + assertNull(readCp) + + val checkpointV1 = new CheckpointV1(ImmutableMap.of(ssp, "offset-1")) + val checkpointV2 = new CheckpointV2(CheckpointId.create(), ImmutableMap.of(ssp, "offset-2"), + ImmutableMap.of("factory1", ImmutableMap.of("store1", "changelogOffset"))) + + val overrideConfig = new MapConfig(new ImmutableMap.Builder[String, String]() + .put(JobConfig.JOB_NAME, "some-job-name") + .put(JobConfig.JOB_ID, "i001") + .put(s"systems.$checkpointSystemName.samza.factory", classOf[KafkaSystemFactory].getCanonicalName) + .put(s"systems.$checkpointSystemName.producer.bootstrap.servers", brokerList) + .put(s"systems.$checkpointSystemName.consumer.zookeeper.connect", zkConnect) + .put("task.checkpoint.system", checkpointSystemName) + .put(TaskConfig.CHECKPOINT_READ_VERSIONS, "2") + .build()) + + // Skips reading any v1 checkpoints + writeCheckpoint(checkpointTopic, taskName, checkpointV1) + assertNull(readCheckpoint(checkpointTopic, taskName, overrideConfig)) + + // writing a v2 checkpoint would allow reading it back + writeCheckpoint(checkpointTopic, taskName, checkpointV2) + assertEquals(checkpointV2, readCheckpoint(checkpointTopic, taskName, overrideConfig)) + + // writing v1 checkpoint is still skipped + writeCheckpoint(checkpointTopic, taskName, checkpointV1) + assertEquals(checkpointV2, readCheckpoint(checkpointTopic, taskName, overrideConfig)) + } + + @Test + def testCheckpointV1AndV2WriteAndReadV1V2PrecedenceList(): Unit = { + val checkpointTopic = "checkpoint-topic-1" + val kcm1 = createKafkaCheckpointManager(checkpointTopic) + kcm1.register(taskName) + kcm1.createResources + kcm1.start + kcm1.stop + + // check that start actually creates the topic with log compaction enabled + val topicConfig = adminZkClient.getAllTopicConfigs().getOrElse(checkpointTopic, new Properties()) + + assertEquals(topicConfig, new KafkaConfig(config).getCheckpointTopicProperties()) + assertEquals("compact", topicConfig.get("cleanup.policy")) + assertEquals("26214400", topicConfig.get("segment.bytes")) + + // read before topic exists should result in a null checkpoint + val readCp = readCheckpoint(checkpointTopic, taskName) + assertNull(readCp) + + val checkpointV1 = new CheckpointV1(ImmutableMap.of(ssp, "offset-1")) + val checkpointV2 = new CheckpointV2(CheckpointId.create(), ImmutableMap.of(ssp, "offset-2"), + ImmutableMap.of("factory1", ImmutableMap.of("store1", "changelogOffset"))) + + val overrideConfig = new MapConfig(new ImmutableMap.Builder[String, String]() + .put(JobConfig.JOB_NAME, "some-job-name") + .put(JobConfig.JOB_ID, "i001") + .put(s"systems.$checkpointSystemName.samza.factory", classOf[KafkaSystemFactory].getCanonicalName) + .put(s"systems.$checkpointSystemName.producer.bootstrap.servers", brokerList) + .put(s"systems.$checkpointSystemName.consumer.zookeeper.connect", zkConnect) + .put("task.checkpoint.system", checkpointSystemName) + .put(TaskConfig.CHECKPOINT_READ_VERSIONS, "2,1") + .build()) + + // Still reads any v1 checkpoints due to precedence list + writeCheckpoint(checkpointTopic, taskName, checkpointV1) + assertEquals(checkpointV1, readCheckpoint(checkpointTopic, taskName, overrideConfig)) + + // writing a v2 checkpoint would allow reading it back + writeCheckpoint(checkpointTopic, taskName, checkpointV2) + assertEquals(checkpointV2, readCheckpoint(checkpointTopic, taskName, overrideConfig)) + + // writing v1 checkpoint is still skipped + writeCheckpoint(checkpointTopic, taskName, checkpointV1) + assertEquals(checkpointV2, readCheckpoint(checkpointTopic, taskName, overrideConfig)) + + val newCheckpointV2 = new CheckpointV2(CheckpointId.create(), ImmutableMap.of(ssp, "offset-3"), + ImmutableMap.of("factory1", ImmutableMap.of("store1", "changelogOffset"))) + // writing v2 returns a new checkpoint v2 + writeCheckpoint(checkpointTopic, taskName, newCheckpointV2) + assertEquals(newCheckpointV2, readCheckpoint(checkpointTopic, taskName, overrideConfig)) + } + + @Test + def testCheckpointValidationSkipped(): Unit = { + val checkpointTopic = "checkpoint-topic-1" + val kcm1 = createKafkaCheckpointManager(checkpointTopic, serde = new MockCheckpointSerde(), + failOnTopicValidation = false) + kcm1.register(taskName) + kcm1.start + kcm1.writeCheckpoint(taskName, new CheckpointV1(ImmutableMap.of(ssp, "offset-1"))) + kcm1.readLastCheckpoint(taskName) + kcm1.stop + } + + @Test + def testReadCheckpointShouldIgnoreUnknownCheckpointKeys(): Unit = { + val checkpointTopic = "checkpoint-topic-1" + val kcm1 = createKafkaCheckpointManager(checkpointTopic) + kcm1.register(taskName) + kcm1.createResources + kcm1.start + kcm1.stop + + // check that start actually creates the topic with log compaction enabled + val topicConfig = adminZkClient.getAllTopicConfigs().getOrElse(checkpointTopic, new Properties()) + + assertEquals(topicConfig, new KafkaConfig(config).getCheckpointTopicProperties()) + assertEquals("compact", topicConfig.get("cleanup.policy")) + assertEquals("26214400", topicConfig.get("segment.bytes")) + + // read before topic exists should result in a null checkpoint + val readCp = readCheckpoint(checkpointTopic, taskName) + assertNull(readCp) + // skips unknown checkpoints from checkpoint topic + writeCheckpoint(checkpointTopic, taskName, checkpoint1, "checkpoint-v2", useMock = true) + assertNull(readCheckpoint(checkpointTopic, taskName, useMock = true)) + + // reads latest v1 checkpoints + writeCheckpoint(checkpointTopic, taskName, checkpoint1, useMock = true) + assertEquals(checkpoint1, readCheckpoint(checkpointTopic, taskName, useMock = true)) + + // writing checkpoint v2 still returns the previous v1 checkpoint + writeCheckpoint(checkpointTopic, taskName, checkpoint2, "checkpoint-v2", useMock = true) + assertEquals(checkpoint1, readCheckpoint(checkpointTopic, taskName, useMock = true)) // writing checkpoint2 with the correct key returns the checkpoint2 - writeCheckpoint(checkpointTopic, taskName, checkpoint2) - assertEquals(checkpoint2, readCheckpoint(checkpointTopic, taskName)) + writeCheckpoint(checkpointTopic, taskName, checkpoint2, useMock = true) + assertEquals(checkpoint2, readCheckpoint(checkpointTopic, taskName, useMock = true)) } @Test @@ -179,7 +326,7 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness { try { checkPointManager.register(taskName) checkPointManager.start - checkPointManager.writeCheckpoint(taskName, new Checkpoint(ImmutableMap.of())) + checkPointManager.writeCheckpoint(taskName, new CheckpointV1(ImmutableMap.of())) } catch { case _: SamzaException => info("Got SamzaException as expected.") case unexpectedException: Throwable => fail("Expected SamzaException but got %s" format unexpectedException) @@ -212,7 +359,7 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness { // create topic with the wrong number of partitions createTopic(checkpointTopic, 8, new KafkaConfig(config).getCheckpointTopicProperties()) val failOnTopicValidation = false - val kcm = createKafkaCheckpointManager(checkpointTopic, new CheckpointSerde, failOnTopicValidation) + val kcm = createKafkaCheckpointManager(checkpointTopic, new CheckpointV1Serde, failOnTopicValidation) kcm.register(taskName) kcm.createResources() kcm.start() @@ -288,25 +435,32 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness { .build()) } - private def createKafkaCheckpointManager(cpTopic: String, serde: CheckpointSerde = new CheckpointSerde, - failOnTopicValidation: Boolean = true, checkpointKey: String = KafkaCheckpointLogKey.CHECKPOINT_KEY_TYPE) = { - val kafkaConfig = new org.apache.samza.config.KafkaConfig(config) + private def createKafkaCheckpointManager(cpTopic: String, serde: CheckpointV1Serde = new CheckpointV1Serde, + failOnTopicValidation: Boolean = true, useMock: Boolean = false, checkpointKey: String = KafkaCheckpointLogKey.CHECKPOINT_V1_KEY_TYPE, + overrideConfig: Config = config) = { + val kafkaConfig = new org.apache.samza.config.KafkaConfig(overrideConfig) val props = kafkaConfig.getCheckpointTopicProperties() val systemName = kafkaConfig.getCheckpointSystem.getOrElse( throw new SamzaException("No system defined for Kafka's checkpoint manager.")) - val systemConfig = new SystemConfig(config) + val systemConfig = new SystemConfig(overrideConfig) val systemFactoryClassName = JavaOptionals.toRichOptional(systemConfig.getSystemFactory(systemName)).toOption .getOrElse(throw new SamzaException("Missing configuration: " + SystemConfig.SYSTEM_FACTORY_FORMAT format systemName)) val systemFactory = ReflectionUtil.getObj(systemFactoryClassName, classOf[SystemFactory]) val spec = new KafkaStreamSpec("id", cpTopic, checkpointSystemName, 1, 1, props) - new MockKafkaCheckpointManager(spec, systemFactory, failOnTopicValidation, serde, checkpointKey) + + if (useMock) { + new MockKafkaCheckpointManager(spec, systemFactory, failOnTopicValidation, serde, checkpointKey) + } else { + new KafkaCheckpointManager(spec, systemFactory, failOnTopicValidation, overrideConfig, new NoOpMetricsRegistry, serde) + } } - private def readCheckpoint(checkpointTopic: String, taskName: TaskName) : Checkpoint = { - val kcm = createKafkaCheckpointManager(checkpointTopic) + private def readCheckpoint(checkpointTopic: String, taskName: TaskName, config: Config = config, + useMock: Boolean = false) : Checkpoint = { + val kcm = createKafkaCheckpointManager(checkpointTopic, overrideConfig = config, useMock = useMock) kcm.register(taskName) kcm.start val checkpoint = kcm.readLastCheckpoint(taskName) @@ -315,8 +469,8 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness { } private def writeCheckpoint(checkpointTopic: String, taskName: TaskName, checkpoint: Checkpoint, - checkpointKey: String = KafkaCheckpointLogKey.CHECKPOINT_KEY_TYPE): Unit = { - val kcm = createKafkaCheckpointManager(checkpointTopic, checkpointKey = checkpointKey) + checkpointKey: String = KafkaCheckpointLogKey.CHECKPOINT_V1_KEY_TYPE, useMock: Boolean = false): Unit = { + val kcm = createKafkaCheckpointManager(checkpointTopic, checkpointKey = checkpointKey, useMock = useMock) kcm.register(taskName) kcm.start kcm.writeCheckpoint(taskName, checkpoint) @@ -339,21 +493,37 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness { } } + class MockCheckpointSerde() extends CheckpointV1Serde { + override def fromBytes(bytes: Array[Byte]): CheckpointV1 = { + throw new SamzaException("Failed to deserialize") + } + } + + class MockKafkaCheckpointManager(spec: KafkaStreamSpec, systemFactory: SystemFactory, failOnTopicValidation: Boolean, - serde: CheckpointSerde = new CheckpointSerde, checkpointKey: String) + serde: CheckpointV1Serde = new CheckpointV1Serde, checkpointKey: String) extends KafkaCheckpointManager(spec, systemFactory, failOnTopicValidation, config, new NoOpMetricsRegistry, serde) { - override def buildOutgoingMessageEnvelope(taskName: TaskName, checkpoint: Checkpoint): OutgoingMessageEnvelope = { + + override def buildOutgoingMessageEnvelope[T <: Checkpoint](taskName: TaskName, checkpoint: T): OutgoingMessageEnvelope = { val key = new KafkaCheckpointLogKey(checkpointKey, taskName, expectedGrouperFactory) val keySerde = new KafkaCheckpointLogKeySerde - val checkpointMsgSerde = new CheckpointSerde + val checkpointMsgSerde = new CheckpointV1Serde + val checkpointV2MsgSerde = new CheckpointV2Serde val keyBytes = try { keySerde.toBytes(key) } catch { case e: Exception => throw new SamzaException(s"Exception when writing checkpoint-key for $taskName: $checkpoint", e) } val msgBytes = try { - checkpointMsgSerde.toBytes(checkpoint) + checkpoint match { + case v1: CheckpointV1 => + checkpointMsgSerde.toBytes(v1) + case v2: CheckpointV2 => + checkpointV2MsgSerde.toBytes(v2) + case _ => + throw new IllegalArgumentException("Unknown checkpoint key type for test, please use Checkpoint v1 or v2") + } } catch { case e: Exception => throw new SamzaException(s"Exception when writing checkpoint for $taskName: $checkpoint", e) } diff --git a/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryKeyValueStorageEngineFactory.java b/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryKeyValueStorageEngineFactory.java index 8cd4e3645a..883766b708 100644 --- a/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryKeyValueStorageEngineFactory.java +++ b/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryKeyValueStorageEngineFactory.java @@ -25,7 +25,6 @@ import org.apache.samza.storage.kv.BaseKeyValueStorageEngineFactory; import org.apache.samza.storage.kv.KeyValueStore; import org.apache.samza.storage.kv.KeyValueStoreMetrics; -import org.apache.samza.system.SystemStreamPartition; public class InMemoryKeyValueStorageEngineFactory extends BaseKeyValueStorageEngineFactory { @@ -33,7 +32,6 @@ public class InMemoryKeyValueStorageEngineFactory extends BaseKeyValueStor protected KeyValueStore getKVStore(String storeName, File storeDir, MetricsRegistry registry, - SystemStreamPartition changeLogSystemStreamPartition, JobContext jobContext, ContainerContext containerContext, StoreMode storeMode) { diff --git a/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbOptionsHelper.java b/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbOptionsHelper.java index 5af2ef89f9..ac2b3f0fc7 100644 --- a/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbOptionsHelper.java +++ b/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbOptionsHelper.java @@ -20,13 +20,17 @@ package org.apache.samza.storage.kv; import java.io.File; +import org.apache.commons.lang3.StringUtils; import org.apache.samza.config.Config; import org.apache.samza.storage.StorageEngineFactory; import org.apache.samza.storage.StorageManagerUtil; import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.CompactionOptionsUniversal; +import org.rocksdb.CompactionStopStyle; import org.rocksdb.CompactionStyle; import org.rocksdb.CompressionType; import org.rocksdb.Options; +import org.rocksdb.WALRecoveryMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,9 +40,30 @@ public class RocksDbOptionsHelper { private static final Logger log = LoggerFactory.getLogger(RocksDbOptionsHelper.class); + // TODO HIGH pmaheshw Add these to RockdDBTableDescriptor + public static final String ROCKSDB_WAL_ENABLED = "rocksdb.wal.enabled"; private static final String ROCKSDB_COMPRESSION = "rocksdb.compression"; private static final String ROCKSDB_BLOCK_SIZE_BYTES = "rocksdb.block.size.bytes"; + + private static final String ROCKSDB_COMPACTION_NUM_LEVELS = "rocksdb.compaction.num.levels"; + private static final String ROCKSDB_COMPACTION_LEVEL0_FILE_NUM_COMPACTION_TRIGGER = + "rocksdb.compaction.level0.file.num.compaction.trigger"; + private static final String ROCKSDB_COMPACTION_MAX_BACKGROUND_COMPACTIONS = "rocksdb.compaction.max.background.compactions"; + private static final String ROCKSDB_COMPACTION_TARGET_FILE_SIZE_BASE = "rocksdb.compaction.target.file.size.base"; + private static final String ROCKSDB_COMPACTION_TARGET_FILE_SIZE_MULTIPLIER = "rocksdb.compaction.target.file.size.multiplier"; + private static final String ROCKSDB_COMPACTION_STYLE = "rocksdb.compaction.style"; + private static final String ROCKSDB_COMPACTION_UNIVERSAL_MAX_SIZE_AMPLIFICATION_PERCENT = + "rocksdb.compaction.universal.max.size.amplification.percent"; + private static final String ROCKSDB_COMPACTION_UNIVERSAL_SIZE_RATIO = + "rocksdb.compaction.universal.size.ratio"; + private static final String ROCKSDB_COMPACTION_UNIVERSAL_MIN_MERGE_WIDTH = + "rocksdb.compaction.universal.min.merge.width"; + private static final String ROCKSDB_COMPACTION_UNIVERSAL_MAX_MERGE_WIDTH = + "rocksdb.compaction.universal.max.merge.width"; + private static final String ROCKSDB_COMPACTION_UNIVERSAL_COMPACTION_STOP_STYLE = + "rocksdb.compaction.universal.compaction.stop.style"; + private static final String ROCKSDB_NUM_WRITE_BUFFERS = "rocksdb.num.write.buffers"; private static final String ROCKSDB_MAX_LOG_FILE_SIZE_BYTES = "rocksdb.max.log.file.size.bytes"; private static final String ROCKSDB_KEEP_LOG_FILE_NUM = "rocksdb.keep.log.file.num"; @@ -49,6 +74,12 @@ public class RocksDbOptionsHelper { public static Options options(Config storeConfig, int numTasksForContainer, File storeDir, StorageEngineFactory.StoreMode storeMode) { Options options = new Options(); + + if (storeConfig.getBoolean(ROCKSDB_WAL_ENABLED, false)) { + options.setManualWalFlush(true); // store.flush() will flushWAL(sync = true) instead + options.setWalRecoveryMode(WALRecoveryMode.AbsoluteConsistency); + } + Long writeBufSize = storeConfig.getLong("container.write.buffer.size.bytes", 32 * 1024 * 1024); // Cache size and write buffer size are specified on a per-container basis. options.setWriteBufferSize((int) (writeBufSize / numTasksForContainer)); @@ -86,23 +117,7 @@ public static Options options(Config storeConfig, int numTasksForContainer, File tableOptions.setBlockCacheSize(blockCacheSize).setBlockSize(blockSize); options.setTableFormatConfig(tableOptions); - CompactionStyle compactionStyle = CompactionStyle.UNIVERSAL; - String compactionStyleInConfig = storeConfig.get(ROCKSDB_COMPACTION_STYLE, "universal"); - switch (compactionStyleInConfig) { - case "universal": - compactionStyle = CompactionStyle.UNIVERSAL; - break; - case "fifo": - compactionStyle = CompactionStyle.FIFO; - break; - case "level": - compactionStyle = CompactionStyle.LEVEL; - break; - default: - log.warn("Unknown rocksdb.compaction.style " + compactionStyleInConfig + - ", overwriting to " + compactionStyle.name()); - } - options.setCompactionStyle(compactionStyle); + setCompactionOptions(storeConfig, options); options.setMaxWriteBufferNumber(storeConfig.getInt(ROCKSDB_NUM_WRITE_BUFFERS, 3)); options.setCreateIfMissing(true); @@ -129,6 +144,81 @@ public static Options options(Config storeConfig, int numTasksForContainer, File return options; } + private static void setCompactionOptions(Config storeConfig, Options options) { + if (storeConfig.containsKey(ROCKSDB_COMPACTION_NUM_LEVELS)) { + options.setNumLevels(storeConfig.getInt(ROCKSDB_COMPACTION_NUM_LEVELS)); + } + + if (storeConfig.containsKey(ROCKSDB_COMPACTION_LEVEL0_FILE_NUM_COMPACTION_TRIGGER)) { + int level0FileNumCompactionTrigger = storeConfig.getInt(ROCKSDB_COMPACTION_LEVEL0_FILE_NUM_COMPACTION_TRIGGER); + options.setLevel0FileNumCompactionTrigger(level0FileNumCompactionTrigger); + } + + if (storeConfig.containsKey(ROCKSDB_COMPACTION_MAX_BACKGROUND_COMPACTIONS)) { + options.setMaxBackgroundCompactions(storeConfig.getInt(ROCKSDB_COMPACTION_MAX_BACKGROUND_COMPACTIONS)); + } + + if (storeConfig.containsKey(ROCKSDB_COMPACTION_TARGET_FILE_SIZE_BASE)) { + options.setTargetFileSizeBase(storeConfig.getLong(ROCKSDB_COMPACTION_TARGET_FILE_SIZE_BASE)); + } + + if (storeConfig.containsKey(ROCKSDB_COMPACTION_TARGET_FILE_SIZE_MULTIPLIER)) { + options.setTargetFileSizeBase(storeConfig.getLong(ROCKSDB_COMPACTION_TARGET_FILE_SIZE_MULTIPLIER)); + } + + CompactionStyle compactionStyle = CompactionStyle.UNIVERSAL; + String compactionStyleInConfig = storeConfig.get(ROCKSDB_COMPACTION_STYLE, "universal"); + switch (compactionStyleInConfig) { + case "universal": + compactionStyle = CompactionStyle.UNIVERSAL; + break; + case "fifo": + compactionStyle = CompactionStyle.FIFO; + break; + case "level": + compactionStyle = CompactionStyle.LEVEL; + break; + default: + log.warn("Unknown rocksdb.compaction.style " + compactionStyleInConfig + + ", overwriting to " + compactionStyle.name()); + } + options.setCompactionStyle(compactionStyle); + + // Universal compaction options + if (compactionStyle.equals(CompactionStyle.UNIVERSAL)) { + CompactionOptionsUniversal compactionOptions = new CompactionOptionsUniversal(); + + if (storeConfig.containsKey(ROCKSDB_COMPACTION_UNIVERSAL_MAX_SIZE_AMPLIFICATION_PERCENT)) { + int val = storeConfig.getInt(ROCKSDB_COMPACTION_UNIVERSAL_MAX_SIZE_AMPLIFICATION_PERCENT); + compactionOptions.setMaxSizeAmplificationPercent(val); + } + + if (storeConfig.containsKey(ROCKSDB_COMPACTION_UNIVERSAL_SIZE_RATIO)) { + int val = storeConfig.getInt(ROCKSDB_COMPACTION_UNIVERSAL_SIZE_RATIO); + compactionOptions.setSizeRatio(val); + } + + if (storeConfig.containsKey(ROCKSDB_COMPACTION_UNIVERSAL_MIN_MERGE_WIDTH)) { + int val = storeConfig.getInt(ROCKSDB_COMPACTION_UNIVERSAL_MIN_MERGE_WIDTH); + compactionOptions.setMinMergeWidth(val); + } + + if (storeConfig.containsKey(ROCKSDB_COMPACTION_UNIVERSAL_MAX_MERGE_WIDTH)) { + int val = storeConfig.getInt(ROCKSDB_COMPACTION_UNIVERSAL_MAX_MERGE_WIDTH); + compactionOptions.setMaxMergeWidth(val); + } + + if (storeConfig.containsKey(ROCKSDB_COMPACTION_UNIVERSAL_COMPACTION_STOP_STYLE)) { + String stopStyle = storeConfig.get(ROCKSDB_COMPACTION_UNIVERSAL_COMPACTION_STOP_STYLE); + if (StringUtils.isNotBlank(stopStyle)) { + compactionOptions.setStopStyle(CompactionStopStyle.valueOf(stopStyle)); + } + } + + options.setCompactionOptionsUniversal(compactionOptions); + } + } + public static Long getBlockCacheSize(Config storeConfig, int numTasksForContainer) { long cacheSize = storeConfig.getLong("container.cache.size.bytes", 100 * 1024 * 1024L); return cacheSize / numTasksForContainer; diff --git a/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStorageEngineFactory.scala b/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStorageEngineFactory.scala index d02d6235f3..afc289ad75 100644 --- a/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStorageEngineFactory.scala +++ b/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStorageEngineFactory.scala @@ -34,14 +34,12 @@ class RocksDbKeyValueStorageEngineFactory [K, V] extends BaseKeyValueStorageEngi * @param storeName Name of the store * @param storeDir The directory of the store * @param registry MetricsRegistry to which to publish store specific metrics. - * @param changeLogSystemStreamPartition Samza stream partition from which to receive the changelog. * @param containerContext Information about the container in which the task is executing. * @return A valid KeyValueStore instance */ override def getKVStore(storeName: String, storeDir: File, registry: MetricsRegistry, - changeLogSystemStreamPartition: SystemStreamPartition, jobContext: JobContext, containerContext: ContainerContext, storeMode: StoreMode): KeyValueStore[Array[Byte], Array[Byte]] = { val storageConfigSubset = jobContext.getConfig.subset("stores." + storeName + ".", true) @@ -52,7 +50,13 @@ class RocksDbKeyValueStorageEngineFactory [K, V] extends BaseKeyValueStorageEngi () => RocksDbOptionsHelper.getBlockCacheSize(storageConfigSubset, numTasksForContainer)) val rocksDbOptions = RocksDbOptionsHelper.options(storageConfigSubset, numTasksForContainer, storeDir, storeMode) - val rocksDbWriteOptions = new WriteOptions().setDisableWAL(true) + val rocksDbWriteOptions = new WriteOptions() + + if (!storageConfigSubset.getBoolean(RocksDbOptionsHelper.ROCKSDB_WAL_ENABLED, false)) { + // if WAL not enabled, explicitly disable it + rocksDbWriteOptions.setDisableWAL(true) + } + val rocksDbFlushOptions = new FlushOptions().setWaitForFlush(true) val rocksDb = new RocksDbKeyValueStore( storeDir, diff --git a/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStore.scala b/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStore.scala index 300177af6a..6f5a7f2f8f 100644 --- a/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStore.scala +++ b/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStore.scala @@ -19,16 +19,18 @@ package org.apache.samza.storage.kv +import com.google.common.annotations.VisibleForTesting + import java.io.File import java.nio.file.{Path, Paths} import java.util.concurrent.TimeUnit import java.util.concurrent.locks.ReentrantReadWriteLock import java.util.{Comparator, Optional} - import org.apache.samza.SamzaException import org.apache.samza.checkpoint.CheckpointId import org.apache.samza.config.Config -import org.apache.samza.util.Logging +import org.apache.samza.storage.StorageManagerUtil +import org.apache.samza.util.{FileUtil, Logging} import org.rocksdb.{TtlDB, _} object RocksDbKeyValueStore extends Logging { @@ -67,6 +69,9 @@ object RocksDbKeyValueStore extends Logging { } try { + // Create the path if it doesn't exist + new FileUtil().createDirectories(dir.toPath) + val rocksDb = if (useTTL) { info("Opening RocksDB store: %s in path: %s with TTL value: %s" format (storeName, dir.toString, ttl)) @@ -122,7 +127,7 @@ class RocksDbKeyValueStore( // lazy val here is important because the store directories do not exist yet, it can only be opened // after the directories are created, which happens much later from now. - private lazy val db = RocksDbKeyValueStore.openDB(dir, options, storeConfig, isLoggedStore, storeName, metrics) + @VisibleForTesting lazy val db = RocksDbKeyValueStore.openDB(dir, options, storeConfig, isLoggedStore, storeName, metrics) private val lexicographic = new LexicographicComparator() /** @@ -233,13 +238,17 @@ class RocksDbKeyValueStore( def flush(): Unit = ifOpen { metrics.flushes.inc trace("Flushing store: %s" format storeName) - db.flush(flushOptions) + if (storeConfig.getBoolean(RocksDbOptionsHelper.ROCKSDB_WAL_ENABLED, false)) { + db.flushWal(true) + } else { + db.flush(flushOptions) + } trace("Flushed store: %s" format storeName) } override def checkpoint(id: CheckpointId): Optional[Path] = { val checkpoint = Checkpoint.create(db) - val checkpointPath = dir.getPath + "-" + id.toString + val checkpointPath = new StorageManagerUtil().getStoreCheckpointDir(dir, id) checkpoint.createCheckpoint(checkpointPath) Optional.of(Paths.get(checkpointPath)) } diff --git a/samza-kv-rocksdb/src/test/scala/org/apache/samza/storage/kv/TestRocksDbKeyValueStore.scala b/samza-kv-rocksdb/src/test/scala/org/apache/samza/storage/kv/TestRocksDbKeyValueStore.scala index 2a4f44eafc..54dca8f4df 100644 --- a/samza-kv-rocksdb/src/test/scala/org/apache/samza/storage/kv/TestRocksDbKeyValueStore.scala +++ b/samza-kv-rocksdb/src/test/scala/org/apache/samza/storage/kv/TestRocksDbKeyValueStore.scala @@ -22,11 +22,10 @@ package org.apache.samza.storage.kv import java.io.File import java.util - import org.apache.samza.SamzaException import org.apache.samza.config.MapConfig import org.apache.samza.metrics.{Counter, Gauge, MetricsRegistryMap, MetricsVisitor, Timer} -import org.apache.samza.util.ExponentialSleepStrategy +import org.apache.samza.util.{ExponentialSleepStrategy, FileUtil} import org.junit.{Assert, Test} import org.rocksdb.{FlushOptions, Options, RocksDB, RocksIterator} @@ -123,6 +122,23 @@ class TestRocksDbKeyValueStore rocksDB.get(key) } + @Test + def testRocksDbCreatePathIfNotExist(): Unit = { + val map = new util.HashMap[String, String]() + val config = new MapConfig(map) + val options = new Options() + options.setCreateIfMissing(true) + + val dbDir = new File(System.getProperty("java.io.tmpdir") + File.separator + "samza-test2", "rocksDbFiles") + val rocksDB = new RocksDbKeyValueStore(dbDir, options, config, false, "dbStore") + val key = "key".getBytes("UTF-8") + rocksDB.put(key, "val".getBytes("UTF-8")) + + rocksDB.close() + + new FileUtil().rm(dbDir) + } + @Test def testIteratorWithRemoval(): Unit = { val lock = new Object diff --git a/samza-kv/src/main/java/org/apache/samza/storage/kv/BaseKeyValueStorageEngineFactory.java b/samza-kv/src/main/java/org/apache/samza/storage/kv/BaseKeyValueStorageEngineFactory.java index d6c1196eb1..34d407f13a 100644 --- a/samza-kv/src/main/java/org/apache/samza/storage/kv/BaseKeyValueStorageEngineFactory.java +++ b/samza-kv/src/main/java/org/apache/samza/storage/kv/BaseKeyValueStorageEngineFactory.java @@ -19,6 +19,7 @@ package org.apache.samza.storage.kv; import java.io.File; +import java.util.List; import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.apache.samza.SamzaException; @@ -60,7 +61,6 @@ public abstract class BaseKeyValueStorageEngineFactory implements StorageE * @param storeName Name of the store * @param storeDir The directory of the store * @param registry MetricsRegistry to which to publish store specific metrics. - * @param changeLogSystemStreamPartition Samza stream partition from which to receive the changelog. * @param jobContext Information about the job in which the task is executing. * @param containerContext Information about the container in which the task is executing. * @return A raw KeyValueStore instance @@ -68,7 +68,6 @@ public abstract class BaseKeyValueStorageEngineFactory implements StorageE protected abstract KeyValueStore getKVStore(String storeName, File storeDir, MetricsRegistry registry, - SystemStreamPartition changeLogSystemStreamPartition, JobContext jobContext, ContainerContext containerContext, StoreMode storeMode); @@ -106,6 +105,10 @@ public StorageEngine getStorageEngine(String storeName, if (!storeFactory.get().equals(INMEMORY_KV_STORAGE_ENGINE_FACTORY)) { storePropertiesBuilder.setPersistedToDisk(true); } + // The store is durable iff it is backed by the task backup manager + List storeBackupManagers = storageConfig.getStoreBackupFactories(storeName); + storePropertiesBuilder.setIsDurable(!storeBackupManagers.isEmpty()); + int batchSize = storageConfigSubset.getInt(WRITE_BATCH_SIZE, DEFAULT_WRITE_BATCH_SIZE); int cacheSize = storageConfigSubset.getInt(OBJECT_CACHE_SIZE, Math.max(batchSize, DEFAULT_OBJECT_CACHE_SIZE)); if (cacheSize > 0 && cacheSize < batchSize) { @@ -123,7 +126,7 @@ public StorageEngine getStorageEngine(String storeName, } KeyValueStore rawStore = - getKVStore(storeName, storeDir, registry, changelogSSP, jobContext, containerContext, storeMode); + getKVStore(storeName, storeDir, registry, jobContext, containerContext, storeMode); KeyValueStore maybeLoggedStore = buildMaybeLoggedStore(changelogSSP, storeName, registry, storePropertiesBuilder, rawStore, changelogCollector); // this also applies serialization and caching layers diff --git a/samza-kv/src/test/java/org/apache/samza/storage/kv/MockKeyValueStorageEngineFactory.java b/samza-kv/src/test/java/org/apache/samza/storage/kv/MockKeyValueStorageEngineFactory.java index 3430ae951e..2c8251e962 100644 --- a/samza-kv/src/test/java/org/apache/samza/storage/kv/MockKeyValueStorageEngineFactory.java +++ b/samza-kv/src/test/java/org/apache/samza/storage/kv/MockKeyValueStorageEngineFactory.java @@ -22,7 +22,6 @@ import org.apache.samza.context.ContainerContext; import org.apache.samza.context.JobContext; import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.SystemStreamPartition; /** @@ -38,8 +37,7 @@ public MockKeyValueStorageEngineFactory(KeyValueStore rawKeyValu @Override protected KeyValueStore getKVStore(String storeName, File storeDir, MetricsRegistry registry, - SystemStreamPartition changeLogSystemStreamPartition, JobContext jobContext, ContainerContext containerContext, - StoreMode storeMode) { + JobContext jobContext, ContainerContext containerContext, StoreMode storeMode) { return this.rawKeyValueStore; } } diff --git a/samza-kv/src/test/java/org/apache/samza/storage/kv/TestBaseKeyValueStorageEngineFactory.java b/samza-kv/src/test/java/org/apache/samza/storage/kv/TestBaseKeyValueStorageEngineFactory.java index 22a1b57027..457fcba8da 100644 --- a/samza-kv/src/test/java/org/apache/samza/storage/kv/TestBaseKeyValueStorageEngineFactory.java +++ b/samza-kv/src/test/java/org/apache/samza/storage/kv/TestBaseKeyValueStorageEngineFactory.java @@ -127,7 +127,23 @@ public void testInMemoryKeyValueStore() { "org.apache.samza.storage.kv.inmemory.InMemoryKeyValueStorageEngineFactory")); StorageEngine storageEngine = callGetStorageEngine(config, null); KeyValueStorageEngine keyValueStorageEngine = baseStorageEngineValidation(storageEngine); - assertStoreProperties(keyValueStorageEngine.getStoreProperties(), false, false); + assertStoreProperties(keyValueStorageEngine.getStoreProperties(), false, false, false); + NullSafeKeyValueStore nullSafeKeyValueStore = + assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class); + SerializedKeyValueStore serializedKeyValueStore = + assertAndCast(nullSafeKeyValueStore.getStore(), SerializedKeyValueStore.class); + // config has the in-memory key-value factory, but still calling the test factory, so store will be the test store + assertEquals(this.rawKeyValueStore, serializedKeyValueStore.getStore()); + } + + @Test + public void testDurableKeyValueStore() { + Config config = new MapConfig(BASE_CONFIG, DISABLE_CACHE, + ImmutableMap.of(String.format(StorageConfig.STORE_BACKUP_FACTORIES, STORE_NAME), + "backendFactory,backendFactory2")); + StorageEngine storageEngine = callGetStorageEngine(config, null); + KeyValueStorageEngine keyValueStorageEngine = baseStorageEngineValidation(storageEngine); + assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false, true); NullSafeKeyValueStore nullSafeKeyValueStore = assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class); SerializedKeyValueStore serializedKeyValueStore = @@ -141,7 +157,7 @@ public void testRawStoreOnly() { Config config = new MapConfig(BASE_CONFIG, DISABLE_CACHE); StorageEngine storageEngine = callGetStorageEngine(config, null); KeyValueStorageEngine keyValueStorageEngine = baseStorageEngineValidation(storageEngine); - assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false); + assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false, false); NullSafeKeyValueStore nullSafeKeyValueStore = assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class); SerializedKeyValueStore serializedKeyValueStore = @@ -154,7 +170,7 @@ public void testWithLoggedStore() { Config config = new MapConfig(BASE_CONFIG, DISABLE_CACHE); StorageEngine storageEngine = callGetStorageEngine(config, CHANGELOG_SSP); KeyValueStorageEngine keyValueStorageEngine = baseStorageEngineValidation(storageEngine); - assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, true); + assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, true, false); NullSafeKeyValueStore nullSafeKeyValueStore = assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class); SerializedKeyValueStore serializedKeyValueStore = @@ -170,7 +186,7 @@ public void testWithLoggedStoreAndCachedStore() { Config config = new MapConfig(BASE_CONFIG); StorageEngine storageEngine = callGetStorageEngine(config, CHANGELOG_SSP); KeyValueStorageEngine keyValueStorageEngine = baseStorageEngineValidation(storageEngine); - assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, true); + assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, true, false); NullSafeKeyValueStore nullSafeKeyValueStore = assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class); CachedStore cachedStore = assertAndCast(nullSafeKeyValueStore.getStore(), CachedStore.class); @@ -187,7 +203,7 @@ public void testDisallowLargeMessages() { Config config = new MapConfig(BASE_CONFIG, DISABLE_CACHE, DISALLOW_LARGE_MESSAGES); StorageEngine storageEngine = callGetStorageEngine(config, null); KeyValueStorageEngine keyValueStorageEngine = baseStorageEngineValidation(storageEngine); - assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false); + assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false, false); NullSafeKeyValueStore nullSafeKeyValueStore = assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class); SerializedKeyValueStore serializedKeyValueStore = @@ -202,7 +218,7 @@ public void testDisallowLargeMessagesWithCache() { Config config = new MapConfig(BASE_CONFIG, DISALLOW_LARGE_MESSAGES); StorageEngine storageEngine = callGetStorageEngine(config, null); KeyValueStorageEngine keyValueStorageEngine = baseStorageEngineValidation(storageEngine); - assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false); + assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false, false); NullSafeKeyValueStore nullSafeKeyValueStore = assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class); SerializedKeyValueStore serializedKeyValueStore = @@ -220,7 +236,7 @@ public void testDropLargeMessages() { Config config = new MapConfig(BASE_CONFIG, DISABLE_CACHE, DROP_LARGE_MESSAGES); StorageEngine storageEngine = callGetStorageEngine(config, null); KeyValueStorageEngine keyValueStorageEngine = baseStorageEngineValidation(storageEngine); - assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false); + assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false, false); NullSafeKeyValueStore nullSafeKeyValueStore = assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class); SerializedKeyValueStore serializedKeyValueStore = @@ -235,7 +251,7 @@ public void testDropLargeMessagesWithCache() { Config config = new MapConfig(BASE_CONFIG, DROP_LARGE_MESSAGES); StorageEngine storageEngine = callGetStorageEngine(config, null); KeyValueStorageEngine keyValueStorageEngine = baseStorageEngineValidation(storageEngine); - assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false); + assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false, false); NullSafeKeyValueStore nullSafeKeyValueStore = assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class); CachedStore cachedStore = assertAndCast(nullSafeKeyValueStore.getStore(), CachedStore.class); @@ -252,7 +268,7 @@ public void testAccessLogStore() { // AccessLoggedStore requires a changelog SSP StorageEngine storageEngine = callGetStorageEngine(config, CHANGELOG_SSP); KeyValueStorageEngine keyValueStorageEngine = baseStorageEngineValidation(storageEngine); - assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, true); + assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, true, false); NullSafeKeyValueStore nullSafeKeyValueStore = assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class); AccessLoggedStore accessLoggedStore = @@ -278,9 +294,10 @@ public void testAccessLogStore() { } private static void assertStoreProperties(StoreProperties storeProperties, boolean expectedPersistedToDisk, - boolean expectedLoggedStore) { + boolean expectedLoggedStore, boolean expectedDurable) { assertEquals(expectedPersistedToDisk, storeProperties.isPersistedToDisk()); assertEquals(expectedLoggedStore, storeProperties.isLoggedStore()); + assertEquals(expectedDurable, storeProperties.isDurableStore()); } /** diff --git a/samza-test/src/test/java/org/apache/samza/checkpoint/CheckpointVersionIntegrationTest.java b/samza-test/src/test/java/org/apache/samza/checkpoint/CheckpointVersionIntegrationTest.java new file mode 100644 index 0000000000..66eb79c551 --- /dev/null +++ b/samza-test/src/test/java/org/apache/samza/checkpoint/CheckpointVersionIntegrationTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.checkpoint; + +import java.io.File; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.samza.config.JobConfig; +import org.apache.samza.config.JobCoordinatorConfig; +import org.apache.samza.config.KafkaConfig; +import org.apache.samza.config.TaskConfig; +import org.apache.samza.storage.MyStatefulApplication; +import org.apache.samza.test.framework.StreamApplicationIntegrationTestHarness; +import org.apache.samza.util.FileUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class CheckpointVersionIntegrationTest extends StreamApplicationIntegrationTestHarness { + + private final static Logger LOG = LoggerFactory.getLogger(CheckpointVersionIntegrationTest.class); + + private static final String INPUT_TOPIC = "inputTopic"; + private static final String INPUT_SYSTEM = "kafka"; + private static final String STORE_NAME = "store"; + private static final String CHANGELOG_TOPIC = "changelog"; + private static final String LOGGED_STORE_BASE_DIR = new File(System.getProperty("java.io.tmpdir"), "logged-store").getAbsolutePath(); + private static final Map CONFIGS = new HashMap() { { + put(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY, "org.apache.samza.standalone.PassthroughJobCoordinatorFactory"); + put(JobConfig.PROCESSOR_ID, "0"); + put(TaskConfig.GROUPER_FACTORY, "org.apache.samza.container.grouper.task.GroupByContainerIdsFactory"); + put(TaskConfig.CHECKPOINT_MANAGER_FACTORY, "org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory"); + put(TaskConfig.COMMIT_MS, "-1"); // manual commit only + put(TaskConfig.TRANSACTIONAL_STATE_RESTORE_ENABLED, "true"); + put(TaskConfig.TRANSACTIONAL_STATE_RETAIN_EXISTING_STATE, "true"); + put(KafkaConfig.CHECKPOINT_REPLICATION_FACTOR(), "1"); + put(JobConfig.JOB_LOGGED_STORE_BASE_DIR, LOGGED_STORE_BASE_DIR); + put(TaskConfig.COMMIT_MAX_DELAY_MS, "0"); // Ensure no commits are skipped due to in progress commits + } }; + + @Before + @Override + public void setUp() { + super.setUp(); + // reset static state shared with task between each parameterized iteration + MyStatefulApplication.resetTestState(); + new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR)); // always clear local store on startup + } + + @Test + public void testStopCheckpointV1V2AndRestartCheckpointV2() { + List inputMessagesOnInitialRun = Arrays.asList("1", "2", "3", "2", "97", "-97", ":98", ":99", ":crash_once"); + // double check collectors.flush + List expectedChangelogMessagesOnInitialRun = Arrays.asList("1", "2", "3", "2", "97", null, "98", "99"); + initialRun(inputMessagesOnInitialRun, expectedChangelogMessagesOnInitialRun); + + // first two are reverts for uncommitted messages from last run for keys 98 and 99 + List expectedChangelogMessagesAfterSecondRun = + Arrays.asList(null, null, "98", "99", "4", "5", "5"); + List expectedInitialStoreContentsOnSecondRun = Arrays.asList("1", "2", "3"); + Map configOverrides = new HashMap<>(CONFIGS); + configOverrides.put(TaskConfig.CHECKPOINT_READ_VERSIONS, "2"); + secondRun(CHANGELOG_TOPIC, + expectedChangelogMessagesAfterSecondRun, expectedInitialStoreContentsOnSecondRun, configOverrides); + } + + private void initialRun(List inputMessages, List expectedChangelogMessages) { + // create input topic and produce the first batch of input messages + createTopic(INPUT_TOPIC, 1); + inputMessages.forEach(m -> produceMessage(INPUT_TOPIC, 0, m, m)); + + // verify that the input messages were produced successfully + if (inputMessages.size() > 0) { + List> inputRecords = + consumeMessages(Collections.singletonList(INPUT_TOPIC), inputMessages.size()); + List readInputMessages = inputRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList()); + Assert.assertEquals(inputMessages, readInputMessages); + } + + // run the application + RunApplicationContext context = runApplication( + new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, Collections.singletonMap(STORE_NAME, CHANGELOG_TOPIC)), "myApp", CONFIGS); + + // wait for the application to finish + context.getRunner().waitForFinish(); + + // consume and verify the changelog messages + if (expectedChangelogMessages.size() > 0) { + List> changelogRecords = + consumeMessages(Collections.singletonList(CHANGELOG_TOPIC), expectedChangelogMessages.size()); + List changelogMessages = changelogRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList()); + Assert.assertEquals(expectedChangelogMessages, changelogMessages); + } + + LOG.info("Finished initial run"); + } + + private void secondRun(String changelogTopic, List expectedChangelogMessages, + List expectedInitialStoreContents, Map overriddenConfigs) { + // remove previous files so restore is from the checkpointV2 + new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR)); + + // produce the second batch of input messages + + List inputMessages = Arrays.asList("4", "5", "5", ":shutdown"); + inputMessages.forEach(m -> produceMessage(INPUT_TOPIC, 0, m, m)); + + // run the application + RunApplicationContext context = runApplication( + new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, Collections.singletonMap(STORE_NAME, changelogTopic)), "myApp", overriddenConfigs); + + // wait for the application to finish + context.getRunner().waitForFinish(); + + // verify the store contents during startup (this is after changelog verification to ensure init has completed) + Assert.assertEquals(expectedInitialStoreContents, MyStatefulApplication.getInitialStoreContents().get(STORE_NAME)); + + // consume and verify any additional changelog messages + List> changelogRecords = + consumeMessages(Collections.singletonList(changelogTopic), expectedChangelogMessages.size()); + List changelogMessages = changelogRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList()); + Assert.assertEquals(expectedChangelogMessages, changelogMessages); + } +} diff --git a/samza-test/src/test/java/org/apache/samza/storage/MyStatefulApplication.java b/samza-test/src/test/java/org/apache/samza/storage/MyStatefulApplication.java new file mode 100644 index 0000000000..99c54b0f0f --- /dev/null +++ b/samza-test/src/test/java/org/apache/samza/storage/MyStatefulApplication.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.storage; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.samza.application.TaskApplication; +import org.apache.samza.application.descriptors.TaskApplicationDescriptor; +import org.apache.samza.context.Context; +import org.apache.samza.operators.KV; +import org.apache.samza.serializers.KVSerde; +import org.apache.samza.serializers.StringSerde; +import org.apache.samza.storage.kv.KeyValueIterator; +import org.apache.samza.storage.kv.KeyValueStore; +import org.apache.samza.storage.kv.descriptors.RocksDbTableDescriptor; +import org.apache.samza.system.IncomingMessageEnvelope; +import org.apache.samza.system.kafka.descriptors.KafkaInputDescriptor; +import org.apache.samza.system.kafka.descriptors.KafkaSystemDescriptor; +import org.apache.samza.task.InitableTask; +import org.apache.samza.task.MessageCollector; +import org.apache.samza.task.StreamTask; +import org.apache.samza.task.StreamTaskFactory; +import org.apache.samza.task.TaskCoordinator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Stateful TaskApplication used for testing task store backup and restore behaviour. + * {@link #resetTestState()} should be invoked in @Before class of the test using this class. + * + * Input Message format: + * "num" -> put (key = num, value = num) and flush + * "-num" -> delete (key = num) and flush + * ":msg" -> act on msg (including flush) but no commit (may be num, shutdown or crash_once) + * "shutdown" -> always shutdown the job + * "crash_once" -> shut down the job the first time but ignore on subsequent run + */ +public class MyStatefulApplication implements TaskApplication { + + public static final Logger LOG = LoggerFactory.getLogger(MyStatefulApplication.class); + + private static Map> initialStoreContents = new HashMap<>(); + private static boolean crashedOnce = false; + private final String inputSystem; + private final String inputTopic; + private final Map storeToChangelog; + + public MyStatefulApplication(String inputSystem, String inputTopic, Map storeToChangelog) { + this.inputSystem = inputSystem; + this.inputTopic = inputTopic; + this.storeToChangelog = storeToChangelog; + } + + @Override + public void describe(TaskApplicationDescriptor appDescriptor) { + KafkaSystemDescriptor ksd = new KafkaSystemDescriptor(inputSystem); + KVSerde serde = KVSerde.of(new StringSerde(), new StringSerde()); + + KafkaInputDescriptor> isd = ksd.getInputDescriptor(inputTopic, serde); + + TaskApplicationDescriptor desc = appDescriptor + .withInputStream(isd) + .withTaskFactory((StreamTaskFactory) () -> new MyTask(storeToChangelog.keySet())); + + storeToChangelog.forEach((storeName, changelogTopic) -> { + RocksDbTableDescriptor td = new RocksDbTableDescriptor<>(storeName, serde) + .withChangelogStream(changelogTopic) + .withChangelogReplicationFactor(1); + desc.withTable(td); + }); + } + + public static void resetTestState() { + initialStoreContents = new HashMap<>(); + crashedOnce = false; + } + + public static Map> getInitialStoreContents() { + return initialStoreContents; + } + + static class MyTask implements StreamTask, InitableTask { + private final Set> stores = new HashSet<>(); + private final Set storeNames; + + MyTask(Set storeNames) { + this.storeNames = storeNames; + } + + @Override + public void init(Context context) { + storeNames.forEach(storeName -> { + KeyValueStore store = (KeyValueStore) context.getTaskContext().getStore(storeName); + stores.add(store); + KeyValueIterator storeEntries = store.all(); + List storeInitialChangelog = new ArrayList<>(); + while (storeEntries.hasNext()) { + storeInitialChangelog.add(storeEntries.next().getValue()); + } + initialStoreContents.put(storeName, storeInitialChangelog); + storeEntries.close(); + }); + } + + @Override + public void process(IncomingMessageEnvelope envelope, + MessageCollector collector, TaskCoordinator coordinator) { + String key = (String) envelope.getKey(); + LOG.info("Received key: {}", key); + + if (key.endsWith("crash_once")) { // endsWith allows :crash_once and crash_once + if (!crashedOnce) { + crashedOnce = true; + coordinator.shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); + } else { + return; + } + } else if (key.endsWith("shutdown")) { + coordinator.shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); + } else if (key.startsWith("-")) { + stores.forEach(store -> store.delete(key.substring(1))); + } else if (key.startsWith(":")) { + // write the message and flush, but don't invoke commit later + String msg = key.substring(1); + stores.forEach(store -> store.put(msg, msg)); + } else { + stores.forEach(store -> store.put(key, key)); + } + stores.forEach(KeyValueStore::flush); + + if (!key.startsWith(":")) { + coordinator.commit(TaskCoordinator.RequestScope.CURRENT_TASK); + } + } + } +} diff --git a/samza-test/src/test/java/org/apache/samza/storage/kv/TransactionalStateIntegrationTest.java b/samza-test/src/test/java/org/apache/samza/storage/kv/TransactionalStateIntegrationTest.java index 05e6737768..6f9427c079 100644 --- a/samza-test/src/test/java/org/apache/samza/storage/kv/TransactionalStateIntegrationTest.java +++ b/samza-test/src/test/java/org/apache/samza/storage/kv/TransactionalStateIntegrationTest.java @@ -22,7 +22,6 @@ import com.google.common.collect.ImmutableList; import java.io.File; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -31,25 +30,11 @@ import java.util.Map; import java.util.stream.Collectors; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.samza.application.TaskApplication; -import org.apache.samza.application.descriptors.TaskApplicationDescriptor; import org.apache.samza.config.JobConfig; import org.apache.samza.config.JobCoordinatorConfig; import org.apache.samza.config.KafkaConfig; import org.apache.samza.config.TaskConfig; -import org.apache.samza.context.Context; -import org.apache.samza.operators.KV; -import org.apache.samza.serializers.KVSerde; -import org.apache.samza.serializers.StringSerde; -import org.apache.samza.storage.kv.descriptors.RocksDbTableDescriptor; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.kafka.descriptors.KafkaInputDescriptor; -import org.apache.samza.system.kafka.descriptors.KafkaSystemDescriptor; -import org.apache.samza.task.InitableTask; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.StreamTask; -import org.apache.samza.task.StreamTaskFactory; -import org.apache.samza.task.TaskCoordinator; +import org.apache.samza.storage.MyStatefulApplication; import org.apache.samza.test.framework.StreamApplicationIntegrationTestHarness; import org.apache.samza.util.FileUtil; import org.junit.Assert; @@ -60,14 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Message format: - * "num" -> put (key = num, value = num) and flush - * "-num" -> delete (key = num) and flush - * ":msg" -> act on msg (including flush) but no commit (may be num, shutdown or crash_once) - * "shutdown" -> always shutdown the job - * "crash_once" -> shut down the job the first time but ignore on subsequent run - */ + @RunWith(value = Parameterized.class) public class TransactionalStateIntegrationTest extends StreamApplicationIntegrationTestHarness { @Parameterized.Parameters(name = "hostAffinity={0}") @@ -92,11 +70,9 @@ public static Collection data() { put(TaskConfig.TRANSACTIONAL_STATE_RETAIN_EXISTING_STATE, "true"); put(KafkaConfig.CHECKPOINT_REPLICATION_FACTOR(), "1"); put(JobConfig.JOB_LOGGED_STORE_BASE_DIR, LOGGED_STORE_BASE_DIR); + put(TaskConfig.COMMIT_MAX_DELAY_MS, "0"); // Ensure no commits are skipped due to in progress commits } }; - private static List actualInitialStoreContents = new ArrayList<>(); - private static boolean crashedOnce = false; - private final boolean hostAffinity; public TransactionalStateIntegrationTest(boolean hostAffinity) { @@ -108,43 +84,43 @@ public TransactionalStateIntegrationTest(boolean hostAffinity) { public void setUp() { super.setUp(); // reset static state shared with task between each parameterized iteration - crashedOnce = false; - actualInitialStoreContents = new ArrayList<>(); + MyStatefulApplication.resetTestState(); new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR)); // always clear local store on startup } @Test public void testStopAndRestart() { List inputMessagesOnInitialRun = Arrays.asList("1", "2", "3", "2", "97", "-97", ":98", ":99", ":crash_once"); + // double check collectors.flush List expectedChangelogMessagesOnInitialRun = Arrays.asList("1", "2", "3", "2", "97", null, "98", "99"); initialRun(inputMessagesOnInitialRun, expectedChangelogMessagesOnInitialRun); - // first two are reverts for uncommitted messages from last run - List expectedChangelogMessagesOnSecondRun = + // first two are reverts for uncommitted messages from last run for keys 98 and 99 + List expectedChangelogMessagesAfterSecondRun = Arrays.asList(null, null, "98", "99", "4", "5", "5"); List expectedInitialStoreContentsOnSecondRun = Arrays.asList("1", "2", "3"); secondRun(CHANGELOG_TOPIC, - expectedChangelogMessagesOnSecondRun, expectedInitialStoreContentsOnSecondRun); + expectedChangelogMessagesAfterSecondRun, expectedInitialStoreContentsOnSecondRun, CONFIGS); } @Test public void testWithEmptyChangelogFromInitialRun() { // expected changelog messages will always match since we'll read 0 messages initialRun(ImmutableList.of("crash_once"), Collections.emptyList()); - secondRun(CHANGELOG_TOPIC, ImmutableList.of("4", "5", "5"), Collections.emptyList()); + secondRun(CHANGELOG_TOPIC, ImmutableList.of("4", "5", "5"), Collections.emptyList(), CONFIGS); } @Test public void testWithNewChangelogAfterInitialRun() { List inputMessagesOnInitialRun = Arrays.asList("1", "2", "3", "2", "97", "-97", ":98", ":99", ":crash_once"); - List expectedChangelogMessagesOnInitialRun = Arrays.asList("1", "2", "3", "2", "97", null, "98", "99"); - initialRun(inputMessagesOnInitialRun, expectedChangelogMessagesOnInitialRun); + List expectedChangelogMessagesAfterInitialRun = Arrays.asList("1", "2", "3", "2", "97", null, "98", "99"); + initialRun(inputMessagesOnInitialRun, expectedChangelogMessagesAfterInitialRun); // admin client delete topic doesn't seem to work, times out up to 60 seconds. // simulate delete topic by changing the changelog topic instead. String newChangelogTopic = "changelog2"; LOG.info("Changing changelog topic to: {}", newChangelogTopic); - secondRun(newChangelogTopic, ImmutableList.of("98", "99", "4", "5", "5"), Collections.emptyList()); + secondRun(newChangelogTopic, ImmutableList.of("98", "99", "4", "5", "5"), Collections.emptyList(), CONFIGS); } private void initialRun(List inputMessages, List expectedChangelogMessages) { @@ -161,7 +137,12 @@ private void initialRun(List inputMessages, List expectedChangel } // run the application - RunApplicationContext context = runApplication(new MyApplication(CHANGELOG_TOPIC), "myApp", CONFIGS); + RunApplicationContext context = runApplication( + new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, Collections.singletonMap(STORE_NAME, CHANGELOG_TOPIC)), + "myApp", CONFIGS); + + // wait for the application to finish + context.getRunner().waitForFinish(); // consume and verify the changelog messages if (expectedChangelogMessages.size() > 0) { @@ -171,13 +152,11 @@ private void initialRun(List inputMessages, List expectedChangel Assert.assertEquals(expectedChangelogMessages, changelogMessages); } - // wait for the application to finish - context.getRunner().waitForFinish(); LOG.info("Finished initial run"); } private void secondRun(String changelogTopic, List expectedChangelogMessages, - List expectedInitialStoreContents) { + List expectedInitialStoreContents, Map overriddenConfigs) { // clear the local store directory if (!hostAffinity) { new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR)); @@ -189,7 +168,9 @@ private void secondRun(String changelogTopic, List expectedChangelogMess inputMessages.forEach(m -> produceMessage(INPUT_TOPIC, 0, m, m)); // run the application - RunApplicationContext context = runApplication(new MyApplication(changelogTopic), "myApp", CONFIGS); + RunApplicationContext context = runApplication( + new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, Collections.singletonMap(STORE_NAME, changelogTopic)), + "myApp", overriddenConfigs); // wait for the application to finish context.getRunner().waitForFinish(); @@ -201,76 +182,6 @@ private void secondRun(String changelogTopic, List expectedChangelogMess Assert.assertEquals(expectedChangelogMessages, changelogMessages); // verify the store contents during startup (this is after changelog verification to ensure init has completed) - Assert.assertEquals(expectedInitialStoreContents, actualInitialStoreContents); - } - - static class MyApplication implements TaskApplication { - private final String changelogTopic; - - public MyApplication(String changelogTopic) { - this.changelogTopic = changelogTopic; - } - - @Override - public void describe(TaskApplicationDescriptor appDescriptor) { - KafkaSystemDescriptor ksd = new KafkaSystemDescriptor(INPUT_SYSTEM); - KVSerde serde = KVSerde.of(new StringSerde(), new StringSerde()); - - KafkaInputDescriptor> isd = ksd.getInputDescriptor(INPUT_TOPIC, serde); - - RocksDbTableDescriptor td = new RocksDbTableDescriptor<>(STORE_NAME, serde) - .withChangelogStream(changelogTopic) - .withChangelogReplicationFactor(1); - - appDescriptor - .withInputStream(isd) - .withTaskFactory((StreamTaskFactory) () -> new MyTask()) - .withTable(td); - } - } - - static class MyTask implements StreamTask, InitableTask { - private KeyValueStore store; - - @Override - public void init(Context context) { - this.store = (KeyValueStore) context.getTaskContext().getStore(STORE_NAME); - KeyValueIterator storeEntries = store.all(); - while (storeEntries.hasNext()) { - actualInitialStoreContents.add(storeEntries.next().getValue()); - } - storeEntries.close(); - } - - @Override - public void process(IncomingMessageEnvelope envelope, - MessageCollector collector, TaskCoordinator coordinator) { - String key = (String) envelope.getKey(); - LOG.info("Received key: {}", key); - - if (key.endsWith("crash_once")) { // endsWith allows :crash_once and crash_once - if (!crashedOnce) { - crashedOnce = true; - coordinator.shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); - } else { - return; - } - } else if (key.endsWith("shutdown")) { - coordinator.shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); - } else if (key.startsWith("-")) { - store.delete(key.substring(1)); - } else if (key.startsWith(":")) { - // write the message and flush, but don't invoke commit later - String msg = key.substring(1); - store.put(msg, msg); - } else { - store.put(key, key); - } - store.flush(); - - if (!key.startsWith(":")) { - coordinator.commit(TaskCoordinator.RequestScope.CURRENT_TASK); - } - } + Assert.assertEquals(expectedInitialStoreContents, MyStatefulApplication.getInitialStoreContents().get(STORE_NAME)); } } \ No newline at end of file diff --git a/samza-test/src/test/java/org/apache/samza/storage/kv/TransactionalStateMultiStoreIntegrationTest.java b/samza-test/src/test/java/org/apache/samza/storage/kv/TransactionalStateMultiStoreIntegrationTest.java index 41eb1ab665..e0fbaca6e3 100644 --- a/samza-test/src/test/java/org/apache/samza/storage/kv/TransactionalStateMultiStoreIntegrationTest.java +++ b/samza-test/src/test/java/org/apache/samza/storage/kv/TransactionalStateMultiStoreIntegrationTest.java @@ -21,8 +21,8 @@ import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.io.File; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -31,25 +31,12 @@ import java.util.Map; import java.util.stream.Collectors; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.samza.application.TaskApplication; -import org.apache.samza.application.descriptors.TaskApplicationDescriptor; +import org.apache.samza.application.SamzaApplication; import org.apache.samza.config.JobConfig; import org.apache.samza.config.JobCoordinatorConfig; import org.apache.samza.config.KafkaConfig; import org.apache.samza.config.TaskConfig; -import org.apache.samza.context.Context; -import org.apache.samza.operators.KV; -import org.apache.samza.serializers.KVSerde; -import org.apache.samza.serializers.StringSerde; -import org.apache.samza.storage.kv.descriptors.RocksDbTableDescriptor; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.kafka.descriptors.KafkaInputDescriptor; -import org.apache.samza.system.kafka.descriptors.KafkaSystemDescriptor; -import org.apache.samza.task.InitableTask; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.StreamTask; -import org.apache.samza.task.StreamTaskFactory; -import org.apache.samza.task.TaskCoordinator; +import org.apache.samza.storage.MyStatefulApplication; import org.apache.samza.test.framework.StreamApplicationIntegrationTestHarness; import org.apache.samza.util.FileUtil; import org.junit.Assert; @@ -80,6 +67,7 @@ public static Collection data() { private static final String STORE_2_NAME = "store2"; private static final String STORE_1_CHANGELOG = "changelog1"; private static final String STORE_2_CHANGELOG = "changelog2"; + private static final String APP_NAME = "myApp"; private static final String LOGGED_STORE_BASE_DIR = new File(System.getProperty("java.io.tmpdir"), "logged-store").getAbsolutePath(); private static final Map CONFIGS = new HashMap() { { put(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY, "org.apache.samza.standalone.PassthroughJobCoordinatorFactory"); @@ -91,11 +79,9 @@ public static Collection data() { put(TaskConfig.TRANSACTIONAL_STATE_RETAIN_EXISTING_STATE, "true"); put(KafkaConfig.CHECKPOINT_REPLICATION_FACTOR(), "1"); put(JobConfig.JOB_LOGGED_STORE_BASE_DIR, LOGGED_STORE_BASE_DIR); + put(TaskConfig.COMMIT_MAX_DELAY_MS, "0"); // Ensure no commits are skipped due to in progress commits } }; - private static List actualInitialStoreContents = new ArrayList<>(); - private static boolean crashedOnce = false; - private final boolean hostAffinity; public TransactionalStateMultiStoreIntegrationTest(boolean hostAffinity) { @@ -107,8 +93,7 @@ public TransactionalStateMultiStoreIntegrationTest(boolean hostAffinity) { public void setUp() { super.setUp(); // reset static state shared with task between each parameterized iteration - crashedOnce = false; - actualInitialStoreContents = new ArrayList<>(); + MyStatefulApplication.resetTestState(); new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR)); // always clear local store on startup } @@ -118,7 +103,7 @@ public void testStopAndRestart() { List expectedChangelogMessagesOnInitialRun = Arrays.asList("1", "2", "3", "2", "97", null, "98", "99"); initialRun(inputMessagesOnInitialRun, expectedChangelogMessagesOnInitialRun); - // first two are reverts for uncommitted messages from last run + // first two are reverts for uncommitted messages from last run for keys 98 and 99 List expectedChangelogMessagesOnSecondRun = Arrays.asList(null, null, "98", "99", "4", "5", "5"); List expectedInitialStoreContentsOnSecondRun = Arrays.asList("1", "2", "3"); @@ -159,8 +144,14 @@ private void initialRun(List inputMessages, List expectedChangel Assert.assertEquals(inputMessages, readInputMessages); } + SamzaApplication app = new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, ImmutableMap.of( + STORE_1_NAME, STORE_1_CHANGELOG, + STORE_2_NAME, STORE_2_CHANGELOG + )); + // run the application - RunApplicationContext context = runApplication(new MyApplication(STORE_1_CHANGELOG), "myApp", CONFIGS); + RunApplicationContext context = runApplication(app, APP_NAME, CONFIGS); + // consume and verify the changelog messages if (expectedChangelogMessages.size() > 0) { @@ -187,8 +178,12 @@ private void secondRun(String changelogTopic, List expectedChangelogMess List inputMessages = Arrays.asList("4", "5", "5", ":shutdown"); inputMessages.forEach(m -> produceMessage(INPUT_TOPIC, 0, m, m)); + SamzaApplication app = new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, ImmutableMap.of( + STORE_1_NAME, changelogTopic, + STORE_2_NAME, STORE_2_CHANGELOG + )); // run the application - RunApplicationContext context = runApplication(new MyApplication(changelogTopic), "myApp", CONFIGS); + RunApplicationContext context = runApplication(app, APP_NAME, CONFIGS); // wait for the application to finish context.getRunner().waitForFinish(); @@ -200,81 +195,6 @@ private void secondRun(String changelogTopic, List expectedChangelogMess Assert.assertEquals(expectedChangelogMessages, changelogMessages); // verify the store contents during startup (this is after changelog verification to ensure init has completed) - Assert.assertEquals(expectedInitialStoreContents, actualInitialStoreContents); - } - - static class MyApplication implements TaskApplication { - private final String changelogTopic; - - public MyApplication(String changelogTopic) { - this.changelogTopic = changelogTopic; - } - - @Override - public void describe(TaskApplicationDescriptor appDescriptor) { - KafkaSystemDescriptor ksd = new KafkaSystemDescriptor(INPUT_SYSTEM); - KVSerde serde = KVSerde.of(new StringSerde(), new StringSerde()); - - KafkaInputDescriptor> isd = ksd.getInputDescriptor(INPUT_TOPIC, serde); - - RocksDbTableDescriptor td1 = new RocksDbTableDescriptor<>(STORE_1_NAME, serde) - .withChangelogStream(changelogTopic) - .withChangelogReplicationFactor(1); - - RocksDbTableDescriptor td2 = new RocksDbTableDescriptor<>(STORE_2_NAME, serde) - .withChangelogStream(STORE_2_CHANGELOG) - .withChangelogReplicationFactor(1); - - appDescriptor - .withInputStream(isd) - .withTaskFactory((StreamTaskFactory) () -> new MyTask()) - .withTable(td1) - .withTable(td2); - } - } - - static class MyTask implements StreamTask, InitableTask { - private KeyValueStore store; - - @Override - public void init(Context context) { - this.store = (KeyValueStore) context.getTaskContext().getStore(STORE_1_NAME); - KeyValueIterator storeEntries = store.all(); - while (storeEntries.hasNext()) { - actualInitialStoreContents.add(storeEntries.next().getValue()); - } - storeEntries.close(); - } - - @Override - public void process(IncomingMessageEnvelope envelope, - MessageCollector collector, TaskCoordinator coordinator) { - String key = (String) envelope.getKey(); - LOG.info("Received key: {}", key); - - if (key.endsWith("crash_once")) { // endsWith allows :crash_once and crash_once - if (!crashedOnce) { - crashedOnce = true; - coordinator.shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); - } else { - return; - } - } else if (key.endsWith("shutdown")) { - coordinator.shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); - } else if (key.startsWith("-")) { - store.delete(key.substring(1)); - } else if (key.startsWith(":")) { - // write the message and flush, but don't invoke commit later - String msg = key.substring(1); - store.put(msg, msg); - } else { - store.put(key, key); - } - store.flush(); - - if (!key.startsWith(":")) { - coordinator.commit(TaskCoordinator.RequestScope.CURRENT_TASK); - } - } + Assert.assertEquals(expectedInitialStoreContents, MyStatefulApplication.getInitialStoreContents().get(STORE_1_NAME)); } } \ No newline at end of file diff --git a/samza-test/src/test/java/org/apache/samza/test/processor/TestZkLocalApplicationRunner.java b/samza-test/src/test/java/org/apache/samza/test/processor/TestZkLocalApplicationRunner.java index 4e693720ae..e2e599da3d 100644 --- a/samza-test/src/test/java/org/apache/samza/test/processor/TestZkLocalApplicationRunner.java +++ b/samza-test/src/test/java/org/apache/samza/test/processor/TestZkLocalApplicationRunner.java @@ -856,7 +856,7 @@ public void testStatefulSamzaApplicationShouldRedistributeInputPartitionsToCorre } Assert.assertEquals(expectedTaskAssignments, actualTaskAssignments); - Assert.assertEquals(32, jobModel.maxChangeLogStreamPartitions); + Assert.assertEquals(32, jobModel.getMaxChangeLogStreamPartitions()); } /** @@ -958,7 +958,7 @@ public void testStatefulSamzaApplicationShouldRedistributeInputPartitionsToCorre // Validate that the new JobModel has the expected task assignments. actualTaskAssignments = getTaskAssignments(jobModel); Assert.assertEquals(expectedTaskAssignments, actualTaskAssignments); - Assert.assertEquals(32, jobModel.maxChangeLogStreamPartitions); + Assert.assertEquals(32, jobModel.getMaxChangeLogStreamPartitions()); } @Test diff --git a/samza-test/src/test/scala/org/apache/samza/test/integration/NonTransactionalStateIntegrationTest.scala b/samza-test/src/test/scala/org/apache/samza/test/integration/NonTransactionalStateIntegrationTest.scala index 00a530df5f..229e026375 100644 --- a/samza-test/src/test/scala/org/apache/samza/test/integration/NonTransactionalStateIntegrationTest.scala +++ b/samza-test/src/test/scala/org/apache/samza/test/integration/NonTransactionalStateIntegrationTest.scala @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.samza.test.integration import org.apache.samza.config.TaskConfig @@ -78,7 +77,9 @@ class NonTransactionalStateIntegrationTest extends StreamTaskTestUtil { "stores.mystore.changelog.replication.factor" -> "1", // However, don't have the inputs use the checkpoint manager // since the second part of the test expects to replay the input streams. - "systems.kafka.streams.input.samza.reset.offset" -> "true")) + "systems.kafka.streams.input.samza.reset.offset" -> "true", + TaskConfig.COMMIT_MAX_DELAY_MS -> "0" // Ensure no commits are skipped due to in progress commits + )) @Test def testShouldStartAndRestore { diff --git a/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala b/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala index edcb159d9f..b375511037 100644 --- a/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala +++ b/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala @@ -34,7 +34,7 @@ import org.apache.kafka.clients.consumer.{ConsumerRecord, KafkaConsumer} import org.apache.kafka.clients.producer.{KafkaProducer, Producer, ProducerConfig, ProducerRecord} import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.samza.Partition -import org.apache.samza.checkpoint.Checkpoint +import org.apache.samza.checkpoint.{Checkpoint, CheckpointV1} import org.apache.samza.config._ import org.apache.samza.container.TaskName import org.apache.samza.context.Context @@ -73,8 +73,8 @@ object StreamTaskTestUtil { var producer: Producer[Array[Byte], Array[Byte]] = null var adminClient: AdminClient = null - val cp1 = new Checkpoint(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "123").asJava) - val cp2 = new Checkpoint(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "12345").asJava) + val cp1 = new CheckpointV1(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "123").asJava) + val cp2 = new CheckpointV1(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "12345").asJava) // use a random store directory for each run. prevents test failures due to left over state from // previously aborted test runs @@ -291,7 +291,7 @@ class StreamTaskTestUtil { case _ => throw new ConfigException("No checkpoint manager factory configured") } - ChangelogStreamManager.createChangelogStreams(jobModel.getConfig, jobModel.maxChangeLogStreamPartitions) + ChangelogStreamManager.createChangelogStreams(jobModel.getConfig, jobModel.getMaxChangeLogStreamPartitions) } }