+ *
+ * The master should not throw away the proc result as soon as the procedure is done but should wait
+ * a result request from the client (see executor.removeResult(procId)) The client will call
+ * something like master.isProcDone() or master.getProcResult() which will return the result/state
+ * to the client, and it will mark the completed proc as ready to delete. note that the client may
+ * not receive the response from the master (e.g. master failover) so, if we delay a bit the real
+ * deletion of the proc result the client will be able to get the result the next try.
+ */
+public class CompletedProcedureCleaner extends InternalProcedure {
+ private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);
+
+ static final long CLEANER_INTERVAL =
+ ProcedureNodeConfigDescriptor.getInstance().getConf().getCompletedCleanInterval();
+ private static final int DEFAULT_BATCH_SIZE = 32;
+
+ private final Map> completed;
+ private final IProcedureStore store;
+
+ public CompletedProcedureCleaner(
+ IProcedureStore store, Map> completedMap) {
+ super(TimeUnit.SECONDS.toMillis(CLEANER_INTERVAL));
+ this.completed = completedMap;
+ this.store = store;
+ }
+
+ @Override
+ protected void periodicExecute(final Env env) {
+ if (completed.isEmpty()) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("No completed procedures to cleanup.");
+ }
+ return;
+ }
+
+ final long evictTtl = ProcedureExecutor.EVICT_TTL;
+ final long[] batchIds = new long[DEFAULT_BATCH_SIZE];
+ int batchCount = 0;
+
+ final long now = System.currentTimeMillis();
+ final Iterator>> it =
+ completed.entrySet().iterator();
+ while (it.hasNext() && store.isRunning()) {
+ final Map.Entry> entry = it.next();
+ final CompletedProcedureRetainer retainer = entry.getValue();
+ final Procedure proc = retainer.getProcedure();
+ if (retainer.isExpired(now, evictTtl)) {
+ // Failed procedures aren't persisted in WAL.
+ batchIds[batchCount++] = entry.getKey();
+ if (batchCount == batchIds.length) {
+ store.delete(batchIds, 0, batchCount);
+ batchCount = 0;
+ }
+ it.remove();
+ LOG.trace("Evict completed {}", proc);
+ }
+ }
+ if (batchCount > 0) {
+ store.delete(batchIds, 0, batchCount);
+ }
+ // let the store do some cleanup works, i.e, delete the place marker for preserving the max
+ // procedure id.
+ store.cleanup();
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/CompletedProcedureRetainer.java b/procedure/src/main/java/org/apache/iotdb/procedure/CompletedProcedureRetainer.java
new file mode 100644
index 000000000000..812173700181
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/CompletedProcedureRetainer.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+public class CompletedProcedureRetainer {
+ private final Procedure procedure;
+
+ public CompletedProcedureRetainer(Procedure procedure) {
+ this.procedure = procedure;
+ }
+
+ public Procedure getProcedure() {
+ return procedure;
+ }
+
+ public boolean isExpired(long now, long evictTtl) {
+ return (now - procedure.getLastUpdate()) >= evictTtl;
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/InternalProcedure.java b/procedure/src/main/java/org/apache/iotdb/procedure/InternalProcedure.java
new file mode 100644
index 000000000000..23fd47756d18
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/InternalProcedure.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+import org.apache.iotdb.procedure.exception.ProcedureSuspendedException;
+import org.apache.iotdb.procedure.exception.ProcedureYieldException;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Internal Procedure, do some preiodic job for framework
+ *
+ * @param
+ */
+public abstract class InternalProcedure extends Procedure {
+ public InternalProcedure(long toMillis) {
+ setTimeout(toMillis);
+ }
+
+ protected abstract void periodicExecute(final Env env);
+
+ @Override
+ protected Procedure[] execute(Env env)
+ throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected void rollback(Env env) throws IOException, InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected boolean abort(Env env) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void serialize(ByteBuffer byteBuffer) throws IOException {}
+
+ @Override
+ public void deserialize(ByteBuffer byteBuffer) throws IOException {}
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/Procedure.java b/procedure/src/main/java/org/apache/iotdb/procedure/Procedure.java
new file mode 100644
index 000000000000..6cbb3ec69af1
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/Procedure.java
@@ -0,0 +1,891 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+import org.apache.iotdb.procedure.exception.*;
+import org.apache.iotdb.procedure.store.IProcedureStore;
+import org.apache.iotdb.service.rpc.thrift.ProcedureState;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Abstract class of all procedures.
+ *
+ * @param
+ */
+public abstract class Procedure implements Comparable> {
+ private static final Logger LOG = LoggerFactory.getLogger(Procedure.class);
+ public static final long NO_PROC_ID = -1;
+ public static final long NO_TIMEOUT = -1;
+
+ private long parentProcId = NO_PROC_ID;
+ private long rootProcId = NO_PROC_ID;
+ private long procId = NO_PROC_ID;
+ private long submittedTime;
+
+ private ProcedureState state = ProcedureState.INITIALIZING;
+ private int childrenLatch = 0;
+ private ProcedureException exception;
+
+ private volatile long timeout = NO_TIMEOUT;
+ private volatile long lastUpdate;
+
+ private volatile byte[] result = null;
+ private volatile boolean locked = false;
+ private boolean lockedWhenLoading = false;
+
+ private int[] stackIndexes = null;
+
+ private boolean persist = true;
+
+ public boolean needPersistance() {
+ return this.persist;
+ }
+
+ public void resetPersistance() {
+ this.persist = true;
+ }
+
+ public final void skipPersistance() {
+ this.persist = false;
+ }
+
+ public final boolean hasLock() {
+ return locked;
+ }
+
+ // User level code, override it if necessary
+
+ /**
+ * The main code of the procedure. It must be idempotent since execute() may be called multiple
+ * times in case of machine failure in the middle of the execution.
+ *
+ * @param env the environment passed to the ProcedureExecutor
+ * @return a set of sub-procedures to run or ourselves if there is more work to do or null if the
+ * procedure is done.
+ * @throws ProcedureYieldException the procedure will be added back to the queue and retried
+ * later.
+ * @throws InterruptedException the procedure will be added back to the queue and retried later.
+ * @throws ProcedureSuspendedException Signal to the executor that Procedure has suspended itself
+ * and has set itself up waiting for an external event to wake it back up again.
+ */
+ protected abstract Procedure[] execute(Env env)
+ throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException;
+
+ /**
+ * The code to undo what was done by the execute() code. It is called when the procedure or one of
+ * the sub-procedures failed or an abort was requested. It should cleanup all the resources
+ * created by the execute() call. The implementation must be idempotent since rollback() may be
+ * called multiple time in case of machine failure in the middle of the execution.
+ *
+ * @param env the environment passed to the ProcedureExecutor
+ * @throws IOException temporary failure, the rollback will retry later
+ * @throws InterruptedException the procedure will be added back to the queue and retried later
+ */
+ protected abstract void rollback(Env env) throws IOException, InterruptedException;
+
+ /**
+ * The abort() call is asynchronous and each procedure must decide how to deal with it, if they
+ * want to be abortable. The simplest implementation is to have an AtomicBoolean set in the
+ * abort() method and then the execute() will check if the abort flag is set or not. abort() may
+ * be called multiple times from the client, so the implementation must be idempotent.
+ *
+ *
NOTE: abort() is not like Thread.interrupt(). It is just a notification that allows the
+ * procedure implementor abort.
+ */
+ protected abstract boolean abort(Env env);
+
+ public void serialize(ByteBuffer byteBuffer) throws IOException {
+ // 1. class name
+ String className = this.getClass().getName();
+ byte[] classNameBytes = className.getBytes(StandardCharsets.UTF_8);
+ byteBuffer.putInt(classNameBytes.length);
+ byteBuffer.put(classNameBytes);
+ // procid
+ byteBuffer.putLong(this.procId);
+ // state
+ byteBuffer.putInt(this.state.getValue());
+ // submit time
+ byteBuffer.putLong(this.submittedTime);
+ // last updated
+ byteBuffer.putLong(this.lastUpdate);
+ // parent id
+ byteBuffer.putLong(this.parentProcId);
+ // time out
+ byteBuffer.putLong(this.timeout);
+ // stack indexes
+ if (stackIndexes != null) {
+ byteBuffer.putInt(stackIndexes.length);
+ Arrays.stream(stackIndexes).forEach(byteBuffer::putInt);
+ } else {
+ byteBuffer.putInt(-1);
+ }
+
+ // exceptions
+ if (hasException()) {
+ byteBuffer.put((byte) 1);
+ String exceptionClassName = exception.getClass().getName();
+ byte[] exceptionClassNameBytes = exceptionClassName.getBytes(StandardCharsets.UTF_8);
+ byteBuffer.putInt(exceptionClassNameBytes.length);
+ byteBuffer.put(exceptionClassNameBytes);
+ String message = this.exception.getMessage();
+ byte[] messageBytes = message.getBytes(StandardCharsets.UTF_8);
+ byteBuffer.putInt(messageBytes.length);
+ byteBuffer.put(messageBytes);
+ } else {
+ byteBuffer.put((byte) 0);
+ }
+
+ // result
+ if (result != null) {
+ byteBuffer.putInt(result.length);
+ byteBuffer.put(result);
+ } else {
+ byteBuffer.putInt(-1);
+ }
+
+ // has lock
+ byteBuffer.put(this.hasLock() ? (byte) 1 : (byte) 0);
+ }
+
+ public void deserialize(ByteBuffer byteBuffer) throws IOException {
+ // procid
+ this.setProcId(byteBuffer.getLong());
+ // state
+ this.setState(ProcedureState.findByValue(byteBuffer.getInt()));
+ // submit time
+ this.setSubmittedTime(byteBuffer.getLong());
+ // last updated
+ this.setLastUpdate(byteBuffer.getLong());
+ // parent id
+ this.setParentProcId(byteBuffer.getLong());
+ // time out
+ this.setTimeout(byteBuffer.getLong());
+ // stack index
+ int stackIndexesLen = byteBuffer.getInt();
+ if (stackIndexesLen >= 0) {
+ List indexList = new ArrayList<>(stackIndexesLen);
+ for (int i = 0; i < stackIndexesLen; i++) {
+ indexList.add(byteBuffer.getInt());
+ }
+ this.setStackIndexes(indexList);
+ }
+ // exceptions
+ if (byteBuffer.get() == 1) {
+ Class exceptionClass = deserializeTypeInfo(byteBuffer);
+ int messageBytesLength = byteBuffer.getInt();
+ byte[] messageBytes = new byte[messageBytesLength];
+ byteBuffer.get(messageBytes);
+ String errMsg = new String(messageBytes, StandardCharsets.UTF_8);
+ ProcedureException exception;
+ try {
+ exception =
+ (ProcedureException) exceptionClass.getConstructor(String.class).newInstance(errMsg);
+ } catch (InstantiationException
+ | IllegalAccessException
+ | InvocationTargetException
+ | NoSuchMethodException e) {
+ LOG.warn("Instantiation exception class failed", e);
+ exception = new ProcedureException(errMsg);
+ }
+
+ setFailure(exception);
+ }
+
+ // result
+ int resultLen = byteBuffer.getInt();
+ if (resultLen > 0) {
+ byte[] resultArr = new byte[resultLen];
+ byteBuffer.get(resultArr);
+ }
+ // has lock
+ if (byteBuffer.get() == 1) {
+ this.lockedWhenLoading();
+ }
+ }
+
+ /**
+ * Deserialize class Name and load class
+ *
+ * @param byteBuffer bytebuffer
+ * @return Procedure
+ */
+ public static Class deserializeTypeInfo(ByteBuffer byteBuffer) {
+ int classNameBytesLen = byteBuffer.getInt();
+ byte[] classNameBytes = new byte[classNameBytesLen];
+ byteBuffer.get(classNameBytes);
+ String className = new String(classNameBytes, StandardCharsets.UTF_8);
+ Class clazz;
+ try {
+ clazz = Class.forName(className);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("Invalid procedure class", e);
+ }
+ return clazz;
+ }
+
+ public static Procedure newInstance(ByteBuffer byteBuffer) {
+ Class procedureClass = deserializeTypeInfo(byteBuffer);
+ Procedure procedure;
+ try {
+ procedure = (Procedure) procedureClass.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new RuntimeException("Instantiation failed", e);
+ }
+ return procedure;
+ }
+
+ /**
+ * The {@link #doAcquireLock(Object, IProcedureStore)} will be split into two steps, first, it
+ * will call us to determine whether we need to wait for initialization, second, it will call
+ * {@link #acquireLock(Object)} to actually handle the lock for this procedure.
+ *
+ * @return true means we need to wait until the environment has been initialized, otherwise true.
+ */
+ protected boolean waitInitialized(Env env) {
+ return false;
+ }
+
+ /**
+ * Acquire a lock, user should override it if necessary.
+ *
+ * @param env environment
+ * @return state of lock
+ */
+ protected ProcedureLockState acquireLock(Env env) {
+ return ProcedureLockState.LOCK_ACQUIRED;
+ }
+
+ /**
+ * Release a lock, user should override it if necessary.
+ *
+ * @param env env
+ */
+ protected void releaseLock(Env env) {
+ // no op
+ }
+
+ /**
+ * Used to keep procedure lock even when the procedure is yielded or suspended.
+ *
+ * @param env env
+ * @return true if hold the lock
+ */
+ protected boolean holdLock(Env env) {
+ return false;
+ }
+
+ /**
+ * Called before the procedure is recovered and added into the queue.
+ *
+ * @param env environment
+ */
+ protected final void beforeRecover(Env env) {
+ // no op
+ }
+
+ /**
+ * Called when the procedure is recovered and added into the queue.
+ *
+ * @param env environment
+ */
+ protected final void afterRecover(Env env) {
+ // no op
+ }
+
+ /**
+ * Called when the procedure is completed (success or rollback). The procedure may use this method
+ * to clean up in-memory states. This operation will not be retried on failure.
+ *
+ * @param env environment
+ */
+ protected void completionCleanup(Env env) {
+ // no op
+ }
+
+ /**
+ * To make executor yield between each execution step to give other procedures a chance to run.
+ *
+ * @param env environment
+ * @return return true if yield is allowed.
+ */
+ protected boolean isYieldAfterExecution(Env env) {
+ return false;
+ }
+
+ // -------------------------Internal methods - called by the procedureExecutor------------------
+ /**
+ * Internal method called by the ProcedureExecutor that starts the user-level code execute().
+ *
+ * @param env execute environment
+ * @return sub procedures
+ */
+ protected Procedure[] doExecute(Env env)
+ throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
+ try {
+ updateTimestamp();
+ return execute(env);
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ /**
+ * Internal method called by the ProcedureExecutor that starts the user-level code rollback().
+ *
+ * @param env execute environment
+ * @throws IOException ioe
+ * @throws InterruptedException interrupted exception
+ */
+ public void doRollback(Env env) throws IOException, InterruptedException {
+ try {
+ updateTimestamp();
+ rollback(env);
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ /**
+ * Internal method called by the ProcedureExecutor that starts the user-level code acquireLock().
+ *
+ * @param env environment
+ * @param store ProcedureStore
+ * @return ProcedureLockState
+ */
+ public final ProcedureLockState doAcquireLock(Env env, IProcedureStore store) {
+ if (waitInitialized(env)) {
+ return ProcedureLockState.LOCK_EVENT_WAIT;
+ }
+ if (lockedWhenLoading) {
+ lockedWhenLoading = false;
+ locked = true;
+ return ProcedureLockState.LOCK_ACQUIRED;
+ }
+ ProcedureLockState state = acquireLock(env);
+ if (state == ProcedureLockState.LOCK_ACQUIRED) {
+ locked = true;
+ store.update(this);
+ }
+ return state;
+ }
+
+ /**
+ * Presist lock state of the procedure
+ *
+ * @param env environment
+ * @param store ProcedureStore
+ */
+ public final void doReleaseLock(Env env, IProcedureStore store) {
+ locked = false;
+ if (getState() != ProcedureState.ROLLEDBACK) {
+ store.update(this);
+ }
+ releaseLock(env);
+ }
+
+ public final void restoreLock(Env env) {
+ if (!lockedWhenLoading) {
+ LOG.debug("{} didn't hold the lock before restarting, skip acquiring lock", this);
+ return;
+ }
+ if (isFinished()) {
+ LOG.debug("{} is already bypassed, skip acquiring lock.", this);
+ return;
+ }
+ if (getState() == ProcedureState.WAITING && !holdLock(env)) {
+ LOG.debug("{} is in WAITING STATE, and holdLock= false , skip acquiring lock.", this);
+ return;
+ }
+ LOG.debug("{} held the lock before restarting, call acquireLock to restore it.", this);
+ acquireLock(env);
+ }
+
+ @Override
+ public String toString() {
+ // Return the simple String presentation of the procedure.
+ return toStringSimpleSB().toString();
+ }
+
+ /**
+ * Build the StringBuilder for the simple form of procedure string.
+ *
+ * @return the StringBuilder
+ */
+ protected StringBuilder toStringSimpleSB() {
+ final StringBuilder sb = new StringBuilder();
+
+ sb.append("pid=");
+ sb.append(getProcId());
+
+ if (hasParent()) {
+ sb.append(", ppid=");
+ sb.append(getParentProcId());
+ }
+
+ /*
+ * TODO
+ * Enable later when this is being used.
+ * Currently owner not used.
+ if (hasOwner()) {
+ sb.append(", owner=");
+ sb.append(getOwner());
+ }*/
+
+ sb.append(", state="); // pState for Procedure State as opposed to any other kind.
+ toStringState(sb);
+
+ // Only print out locked if actually locked. Most of the time it is not.
+ if (this.locked) {
+ sb.append(", locked=").append(locked);
+ }
+ if (hasException()) {
+ sb.append(", exception=" + getException());
+ }
+
+ sb.append("; ");
+ toStringClassDetails(sb);
+
+ return sb;
+ }
+
+ /** Extend the toString() information with more procedure details */
+ public String toStringDetails() {
+ final StringBuilder sb = toStringSimpleSB();
+
+ sb.append(" submittedTime=");
+ sb.append(getSubmittedTime());
+
+ sb.append(", lastUpdate=");
+ sb.append(getLastUpdate());
+
+ final int[] stackIndices = getStackIndexes();
+ if (stackIndices != null) {
+ sb.append("\n");
+ sb.append("stackIndexes=");
+ sb.append(Arrays.toString(stackIndices));
+ }
+
+ return sb.toString();
+ }
+
+ protected String toStringClass() {
+ StringBuilder sb = new StringBuilder();
+ toStringClassDetails(sb);
+ return sb.toString();
+ }
+
+ /**
+ * Called from {@link #toString()} when interpolating {@link Procedure} State. Allows decorating
+ * generic Procedure State with Procedure particulars.
+ *
+ * @param builder Append current {@link ProcedureState}
+ */
+ protected void toStringState(StringBuilder builder) {
+ builder.append(getState());
+ }
+
+ /**
+ * Extend the toString() information with the procedure details e.g. className and parameters
+ *
+ * @param builder the string builder to use to append the proc specific information
+ */
+ protected void toStringClassDetails(StringBuilder builder) {
+ builder.append(getClass().getName());
+ }
+
+ // ==========================================================================
+ // Those fields are unchanged after initialization.
+ //
+ // Each procedure will get created from the user or during
+ // ProcedureExecutor.start() during the load() phase and then submitted
+ // to the executor. these fields will never be changed after initialization
+ // ==========================================================================
+ public long getProcId() {
+ return procId;
+ }
+
+ public boolean hasParent() {
+ return parentProcId != NO_PROC_ID;
+ }
+
+ public long getParentProcId() {
+ return parentProcId;
+ }
+
+ public long getRootProcId() {
+ return rootProcId;
+ }
+
+ public String getProcName() {
+ return toStringClass();
+ }
+
+ public long getSubmittedTime() {
+ return submittedTime;
+ }
+
+ /** Called by the ProcedureExecutor to assign the ID to the newly created procedure. */
+ protected void setProcId(long procId) {
+ this.procId = procId;
+ }
+
+ public void setProcRunnable() {
+ this.submittedTime = System.currentTimeMillis();
+ setState(ProcedureState.RUNNABLE);
+ }
+
+ /** Called by the ProcedureExecutor to assign the parent to the newly created procedure. */
+ protected void setParentProcId(long parentProcId) {
+ this.parentProcId = parentProcId;
+ }
+
+ protected void setRootProcId(long rootProcId) {
+ this.rootProcId = rootProcId;
+ }
+
+ /**
+ * Called on store load to initialize the Procedure internals after the creation/deserialization.
+ */
+ protected void setSubmittedTime(long submittedTime) {
+ this.submittedTime = submittedTime;
+ }
+
+ // ==========================================================================
+ // runtime state - timeout related
+ // ==========================================================================
+ /** @param timeout timeout interval in msec */
+ protected void setTimeout(long timeout) {
+ this.timeout = timeout;
+ }
+
+ public boolean hasTimeout() {
+ return timeout != NO_TIMEOUT;
+ }
+
+ /** @return the timeout in msec */
+ public long getTimeout() {
+ return timeout;
+ }
+
+ /**
+ * Called on store load to initialize the Procedure internals after the creation/deserialization.
+ */
+ protected void setLastUpdate(long lastUpdate) {
+ this.lastUpdate = lastUpdate;
+ }
+
+ /** Called by ProcedureExecutor after each time a procedure step is executed. */
+ protected void updateTimestamp() {
+ this.lastUpdate = System.currentTimeMillis();
+ }
+
+ public long getLastUpdate() {
+ return lastUpdate;
+ }
+
+ /**
+ * Timeout of the next timeout. Called by the ProcedureExecutor if the procedure has timeout set
+ * and the procedure is in the waiting queue.
+ *
+ * @return the timestamp of the next timeout.
+ */
+ protected long getTimeoutTimestamp() {
+ return getLastUpdate() + getTimeout();
+ }
+
+ // ==========================================================================
+ // runtime state
+ // ==========================================================================
+ /** @return the time elapsed between the last update and the start time of the procedure. */
+ public long elapsedTime() {
+ return getLastUpdate() - getSubmittedTime();
+ }
+
+ /** @return the serialized result if any, otherwise null */
+ public byte[] getResult() {
+ return result;
+ }
+
+ /**
+ * The procedure may leave a "result" on completion.
+ *
+ * @param result the serialized result that will be passed to the client
+ */
+ protected void setResult(byte[] result) {
+ this.result = result;
+ }
+
+ /**
+ * Will only be called when loading procedures from procedure store, where we need to record
+ * whether the procedure has already held a lock. Later we will call {@link #restoreLock(Object)}
+ * to actually acquire the lock.
+ */
+ final void lockedWhenLoading() {
+ this.lockedWhenLoading = true;
+ }
+
+ /**
+ * Can only be called when restarting, before the procedure actually being executed, as after we
+ * actually call the {@link #doAcquireLock(Object, IProcedureStore)} method, we will reset {@link
+ * #lockedWhenLoading} to false.
+ *
+ *
Now it is only used in the ProcedureScheduler to determine whether we should put a Procedure
+ * in front of a queue.
+ */
+ public boolean isLockedWhenLoading() {
+ return lockedWhenLoading;
+ }
+
+ // ==============================================================================================
+ // Runtime state, updated every operation by the ProcedureExecutor
+ //
+ // There is always 1 thread at the time operating on the state of the procedure.
+ // The ProcedureExecutor may check and set states, or some Procecedure may
+ // update its own state. but no concurrent updates. we use synchronized here
+ // just because the procedure can get scheduled on different executor threads on each step.
+ // ==============================================================================================
+
+ /** @return true if the procedure is in a RUNNABLE state. */
+ public synchronized boolean isRunnable() {
+ return state == ProcedureState.RUNNABLE;
+ }
+
+ public synchronized boolean isInitializing() {
+ return state == ProcedureState.INITIALIZING;
+ }
+
+ /** @return true if the procedure has failed. It may or may not have rolled back. */
+ public synchronized boolean isFailed() {
+ return state == ProcedureState.FAILED || state == ProcedureState.ROLLEDBACK;
+ }
+
+ /** @return true if the procedure is finished successfully. */
+ public synchronized boolean isSuccess() {
+ return state == ProcedureState.SUCCESS && !hasException();
+ }
+
+ /**
+ * @return true if the procedure is finished. The Procedure may be completed successfully or
+ * rolledback.
+ */
+ public synchronized boolean isFinished() {
+ return isSuccess() || state == ProcedureState.ROLLEDBACK;
+ }
+
+ /** @return true if the procedure is waiting for a child to finish or for an external event. */
+ public synchronized boolean isWaiting() {
+ switch (state) {
+ case WAITING:
+ case WAITING_TIMEOUT:
+ return true;
+ default:
+ break;
+ }
+ return false;
+ }
+
+ protected synchronized void setState(final ProcedureState state) {
+ this.state = state;
+ updateTimestamp();
+ }
+
+ public synchronized ProcedureState getState() {
+ return state;
+ }
+
+ protected void setFailure(final String source, final Throwable cause) {
+ setFailure(new ProcedureException(source, cause));
+ }
+
+ protected synchronized void setFailure(final ProcedureException exception) {
+ this.exception = exception;
+ if (!isFinished()) {
+ setState(ProcedureState.FAILED);
+ }
+ }
+
+ protected void setAbortFailure(final String source, final String msg) {
+ setFailure(source, new ProcedureAbortedException(msg));
+ }
+
+ /**
+ * Called by the ProcedureExecutor when the timeout set by setTimeout() is expired.
+ *
+ *
Another usage for this method is to implement retrying. A procedure can set the state to
+ * {@code WAITING_TIMEOUT} by calling {@code setState} method, and throw a {@link
+ * ProcedureSuspendedException} to halt the execution of the procedure, and do not forget a call
+ * {@link #setTimeout(long)} method to set the timeout. And you should also override this method
+ * to wake up the procedure, and also return false to tell the ProcedureExecutor that the timeout
+ * event has been handled.
+ *
+ * @return true to let the framework handle the timeout as abort, false in case the procedure
+ * handled the timeout itself.
+ */
+ protected synchronized boolean setTimeoutFailure(Env env) {
+ if (state == ProcedureState.WAITING_TIMEOUT) {
+ long timeDiff = System.currentTimeMillis() - lastUpdate;
+ setFailure(
+ "ProcedureExecutor",
+ new ProcedureTimeoutException("Operation timed out after " + timeDiff + " ms."));
+ return true;
+ }
+ return false;
+ }
+
+ public synchronized boolean hasException() {
+ return exception != null;
+ }
+
+ public synchronized ProcedureException getException() {
+ return exception;
+ }
+
+ /** Called by the ProcedureExecutor on procedure-load to restore the latch state */
+ protected synchronized void setChildrenLatch(int numChildren) {
+ this.childrenLatch = numChildren;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("CHILD LATCH INCREMENT SET " + this.childrenLatch, new Throwable(this.toString()));
+ }
+ }
+
+ /** Called by the ProcedureExecutor on procedure-load to restore the latch state */
+ protected synchronized void incChildrenLatch() {
+ // TODO: can this be inferred from the stack? I think so...
+ this.childrenLatch++;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("CHILD LATCH INCREMENT " + this.childrenLatch, new Throwable(this.toString()));
+ }
+ }
+
+ /** Called by the ProcedureExecutor to notify that one of the sub-procedures has completed. */
+ private synchronized boolean childrenCountDown() {
+ assert childrenLatch > 0 : this;
+ boolean b = --childrenLatch == 0;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("CHILD LATCH DECREMENT " + childrenLatch, new Throwable(this.toString()));
+ }
+ return b;
+ }
+
+ /**
+ * Try to set this procedure into RUNNABLE state. Succeeds if all subprocedures/children are done.
+ *
+ * @return True if we were able to move procedure to RUNNABLE state.
+ */
+ synchronized boolean tryRunnable() {
+ // Don't use isWaiting in the below; it returns true for WAITING and WAITING_TIMEOUT
+ if (getState() == ProcedureState.WAITING && childrenCountDown()) {
+ setState(ProcedureState.RUNNABLE);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ protected synchronized boolean hasChildren() {
+ return childrenLatch > 0;
+ }
+
+ protected synchronized int getChildrenLatch() {
+ return childrenLatch;
+ }
+
+ /**
+ * Called by the RootProcedureState on procedure execution. Each procedure store its stack-index
+ * positions.
+ */
+ protected synchronized void addStackIndex(final int index) {
+ if (stackIndexes == null) {
+ stackIndexes = new int[] {index};
+ } else {
+ int count = stackIndexes.length;
+ stackIndexes = Arrays.copyOf(stackIndexes, count + 1);
+ stackIndexes[count] = index;
+ }
+ }
+
+ protected synchronized boolean removeStackIndex() {
+ if (stackIndexes != null && stackIndexes.length > 1) {
+ stackIndexes = Arrays.copyOf(stackIndexes, stackIndexes.length - 1);
+ return false;
+ } else {
+ stackIndexes = null;
+ return true;
+ }
+ }
+
+ /**
+ * Called on store load to initialize the Procedure internals after the creation/deserialization.
+ */
+ protected synchronized void setStackIndexes(final List stackIndexes) {
+ this.stackIndexes = new int[stackIndexes.size()];
+ for (int i = 0; i < this.stackIndexes.length; ++i) {
+ this.stackIndexes[i] = stackIndexes.get(i);
+ }
+ }
+
+ protected synchronized boolean wasExecuted() {
+ return stackIndexes != null;
+ }
+
+ protected synchronized int[] getStackIndexes() {
+ return stackIndexes;
+ }
+
+ /** Helper to lookup the root Procedure ID given a specified procedure. */
+ protected static long getRootProcedureId(Map procedures, Procedure proc) {
+ while (proc.hasParent()) {
+ proc = procedures.get(proc.getParentProcId());
+ if (proc == null) {
+ return NO_PROC_ID;
+ }
+ }
+ return proc.getProcId();
+ }
+
+ public void setRootProcedureId(long rootProcedureId) {
+ this.rootProcId = rootProcedureId;
+ }
+
+ /**
+ * @param a the first procedure to be compared.
+ * @param b the second procedure to be compared.
+ * @return true if the two procedures have the same parent
+ */
+ public static boolean haveSameParent(Procedure a, Procedure b) {
+ return a.hasParent() && b.hasParent() && (a.getParentProcId() == b.getParentProcId());
+ }
+
+ @Override
+ public int compareTo(Procedure other) {
+ return Long.compare(getProcId(), other.getProcId());
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/ProcedureExecutor.java b/procedure/src/main/java/org/apache/iotdb/procedure/ProcedureExecutor.java
new file mode 100644
index 000000000000..c8a8c5b2687a
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/ProcedureExecutor.java
@@ -0,0 +1,985 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+import org.apache.iotdb.procedure.conf.ProcedureNodeConfigDescriptor;
+import org.apache.iotdb.procedure.exception.ProcedureException;
+import org.apache.iotdb.procedure.exception.ProcedureSuspendedException;
+import org.apache.iotdb.procedure.exception.ProcedureYieldException;
+import org.apache.iotdb.procedure.scheduler.ProcedureScheduler;
+import org.apache.iotdb.procedure.scheduler.SimpleProcedureScheduler;
+import org.apache.iotdb.procedure.store.IProcedureStore;
+import org.apache.iotdb.service.rpc.thrift.ProcedureState;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class ProcedureExecutor {
+ private static final Logger LOG = LoggerFactory.getLogger(ProcedureExecutor.class);
+
+ static final int EVICT_TTL =
+ ProcedureNodeConfigDescriptor.getInstance().getConf().getCompletedEvictTTL();
+
+ private final ConcurrentHashMap idLockMap = new ConcurrentHashMap<>();
+
+ private final ConcurrentHashMap> completed =
+ new ConcurrentHashMap<>();
+
+ private final ConcurrentHashMap> rollbackStack =
+ new ConcurrentHashMap<>();
+
+ private final ConcurrentHashMap procedures = new ConcurrentHashMap<>();
+
+ private ThreadGroup threadGroup;
+
+ private CopyOnWriteArrayList workerThreads;
+
+ private TimeoutExecutorThread timeoutExecutor;
+
+ private TimeoutExecutorThread workerMonitorExecutor;
+
+ private int corePoolSize;
+ private int maxPoolSize;
+ private volatile long keepAliveTime;
+
+ private final ProcedureScheduler scheduler;
+
+ private final AtomicLong lastProcId = new AtomicLong(-1);
+ private final AtomicLong workId = new AtomicLong(0);
+ private final AtomicInteger activeExecutorCount = new AtomicInteger(0);
+ private final AtomicBoolean running = new AtomicBoolean(false);
+ private final Env environment;
+ private final IProcedureStore store;
+
+ public ProcedureExecutor(
+ final Env environment, final IProcedureStore store, final ProcedureScheduler scheduler) {
+ this.environment = environment;
+ this.scheduler = scheduler;
+ this.store = store;
+ this.lastProcId.incrementAndGet();
+ }
+
+ public ProcedureExecutor(final Env environment, final IProcedureStore store) {
+ this(environment, store, new SimpleProcedureScheduler());
+ }
+
+ public void init(int numThreads) {
+ this.corePoolSize = numThreads;
+ this.maxPoolSize = 10 * numThreads;
+ this.threadGroup = new ThreadGroup("ProcedureWorkerGroup");
+ this.timeoutExecutor =
+ new TimeoutExecutorThread<>(this, threadGroup, "ProcedureTimeoutExecutor");
+ this.workerMonitorExecutor =
+ new TimeoutExecutorThread<>(this, threadGroup, "ProcedureWorkerThreadMonitor");
+ workId.set(0);
+ workerThreads = new CopyOnWriteArrayList<>();
+ for (int i = 0; i < corePoolSize; i++) {
+ workerThreads.add(new WorkerThread(threadGroup));
+ }
+ // add worker Monitor
+ workerMonitorExecutor.add(new WorkerMonitor());
+
+ scheduler.start();
+ recover();
+ }
+
+ public void setKeepAliveTime(final long keepAliveTime, final TimeUnit timeUnit) {
+ this.keepAliveTime = timeUnit.toMillis(keepAliveTime);
+ this.scheduler.signalAll();
+ }
+
+ public long getKeepAliveTime(final TimeUnit timeUnit) {
+ return timeUnit.convert(keepAliveTime, TimeUnit.MILLISECONDS);
+ }
+
+ private void recover() {
+ // 1.Build rollback stack
+ int runnableCount = 0;
+ int failedCount = 0;
+ int waitingCount = 0;
+ int waitingTimeoutCount = 0;
+ List procedureList = new ArrayList<>();
+ // load procedure wal file
+ store.load(procedureList);
+ for (Procedure proc : procedureList) {
+ if (proc.isFinished()) {
+ completed.putIfAbsent(proc.getProcId(), new CompletedProcedureRetainer(proc));
+ } else {
+ if (!proc.hasParent()) {
+ rollbackStack.put(proc.getProcId(), new RootProcedureStack<>());
+ }
+ }
+ procedures.putIfAbsent(proc.getProcId(), proc);
+ switch (proc.getState()) {
+ case RUNNABLE:
+ runnableCount++;
+ break;
+ case FAILED:
+ failedCount++;
+ break;
+ case WAITING:
+ waitingCount++;
+ break;
+ case WAITING_TIMEOUT:
+ waitingTimeoutCount++;
+ break;
+ default:
+ break;
+ }
+ }
+ List> runnableList = new ArrayList<>(runnableCount);
+ List> failedList = new ArrayList<>(failedCount);
+ List> waitingList = new ArrayList<>(waitingCount);
+ List> waitingTimeoutList = new ArrayList<>(waitingTimeoutCount);
+ for (Procedure proc : procedureList) {
+ if (proc.isFinished() && !proc.hasParent()) {
+ continue;
+ }
+ long rootProcedureId = getRootProcId(proc);
+ if (proc.hasParent()) {
+ Procedure parent = procedures.get(proc.getParentProcId());
+ if (parent != null && !proc.isFinished()) {
+ parent.incChildrenLatch();
+ }
+ }
+ RootProcedureStack rootStack = rollbackStack.get(rootProcedureId);
+ rootStack.loadStack(proc);
+ proc.setRootProcedureId(rootProcedureId);
+ switch (proc.getState()) {
+ case RUNNABLE:
+ runnableList.add(proc);
+ break;
+ case FAILED:
+ failedList.add(proc);
+ break;
+ case WAITING:
+ waitingList.add(proc);
+ break;
+ case WAITING_TIMEOUT:
+ waitingTimeoutList.add(proc);
+ break;
+ case ROLLEDBACK:
+ case INITIALIZING:
+ LOG.error("Unexpected state:{} for {}", proc.getState(), proc);
+ throw new UnsupportedOperationException("Unexpected state");
+ default:
+ break;
+ }
+ }
+
+ waitingList.forEach(
+ procedure -> {
+ if (procedure.hasChildren()) {
+ procedure.setState(ProcedureState.RUNNABLE);
+ runnableList.add(procedure);
+ } else {
+ procedure.afterRecover(environment);
+ }
+ });
+ restoreLocks();
+
+ waitingTimeoutList.forEach(
+ procedure -> {
+ procedure.afterRecover(environment);
+ timeoutExecutor.add(procedure);
+ });
+
+ failedList.forEach(scheduler::addBack);
+ runnableList.forEach(
+ procedure -> {
+ procedure.afterRecover(environment);
+ scheduler.addBack(procedure);
+ });
+ scheduler.signalAll();
+ }
+
+ public long getRootProcId(Procedure proc) {
+ return Procedure.getRootProcedureId(procedures, proc);
+ }
+
+ private void releaseLock(Procedure procedure, boolean force) {
+ if (force || !procedure.holdLock(this.environment) || procedure.isFinished()) {
+ procedure.doReleaseLock(this.environment, store);
+ }
+ }
+
+ private void restoreLock(Procedure procedure, Set restored) {
+ procedure.restoreLock(environment);
+ restored.add(procedure.getProcId());
+ }
+
+ private void restoreLocks(Deque> stack, Set restored) {
+ while (!stack.isEmpty()) {
+ restoreLock(stack.pop(), restored);
+ }
+ }
+
+ private void restoreLocks() {
+ Set restored = new HashSet<>();
+ Deque> stack = new ArrayDeque<>();
+ procedures
+ .values()
+ .forEach(
+ procedure -> {
+ while (true) {
+ if (restored.contains(procedure.getProcId())) {
+ restoreLocks(stack, restored);
+ return;
+ }
+ if (!procedure.hasParent()) {
+ restoreLock(procedure, restored);
+ restoreLocks(stack, restored);
+ return;
+ }
+ stack.push(procedure);
+ procedure = procedures.get(procedure.getParentProcId());
+ }
+ });
+ }
+
+ public void startWorkers() {
+ if (!running.compareAndSet(false, true)) {
+ LOG.warn("Already running");
+ return;
+ }
+ timeoutExecutor.start();
+ workerMonitorExecutor.start();
+ for (WorkerThread workerThread : workerThreads) {
+ workerThread.start();
+ }
+ addInternalProcedure(new CompletedProcedureCleaner(store, completed));
+ }
+
+ private void addInternalProcedure(InternalProcedure interalProcedure) {
+ if (interalProcedure == null) {
+ return;
+ }
+ interalProcedure.setState(ProcedureState.WAITING_TIMEOUT);
+ timeoutExecutor.add(interalProcedure);
+ }
+
+ public boolean removeInternalProcedure(InternalProcedure internalProcedure) {
+ if (internalProcedure == null) {
+ return true;
+ }
+ internalProcedure.setState(ProcedureState.SUCCESS);
+ return timeoutExecutor.remove(internalProcedure);
+ }
+
+ /**
+ * Get next Procedure id
+ *
+ * @return next procedure id
+ */
+ private long nextProcId() {
+ long procId = lastProcId.incrementAndGet();
+ if (procId < 0) {
+ while (!lastProcId.compareAndSet(procId, 0)) {
+ procId = lastProcId.get();
+ if (procId >= 0) {
+ break;
+ }
+ }
+ while (procedures.containsKey(procId)) {
+ procId = lastProcId.incrementAndGet();
+ }
+ }
+ return procId;
+ }
+
+ /**
+ * Executes procedure
+ *
+ *
Calls doExecute() if success and return subprocedures submit sub procs set the state to
+ * WAITING, wait for all sub procs completed. else if no sub procs procedure completed
+ * successfully set procedure's parent to RUNNABLE in case of failure start rollback of the
+ * procedure.
+ *
+ * @param proc procedure
+ */
+ private void executeProcedure(Procedure proc) {
+ if (proc.isFinished()) {
+ LOG.debug("{} is already finished.", proc);
+ return;
+ }
+ final Long rootProcId = getRootProcedureId(proc);
+ if (rootProcId == null) {
+ LOG.warn("Rollback because parent is done/rolledback, proc is {}", proc);
+ executeRollback(proc);
+ return;
+ }
+ RootProcedureStack rootProcStack = rollbackStack.get(rootProcId);
+ if (rootProcStack == null) {
+ LOG.warn("Rollback stack is null for {}", proc.getProcId());
+ return;
+ }
+ do {
+ if (!rootProcStack.acquire()) {
+ if (rootProcStack.setRollback()) {
+ switch (executeRootStackRollback(rootProcId, rootProcStack)) {
+ case LOCK_ACQUIRED:
+ break;
+ case LOCK_YIELD_WAIT:
+ rootProcStack.unsetRollback();
+ scheduler.yield(proc);
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ } else {
+ if (!proc.wasExecuted()) {
+ switch (executeRollback(proc)) {
+ case LOCK_ACQUIRED:
+ break;
+ case LOCK_EVENT_WAIT:
+ LOG.info("LOCK_EVENT_WAIT can't rollback child running for {}", proc);
+ case LOCK_YIELD_WAIT:
+ scheduler.yield(proc);
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+ }
+ break;
+ }
+ ProcedureLockState lockState = acquireLock(proc);
+ switch (lockState) {
+ case LOCK_ACQUIRED:
+ executeProcedure(rootProcStack, proc);
+ break;
+ case LOCK_YIELD_WAIT:
+ case LOCK_EVENT_WAIT:
+ LOG.info("{} lockstate is {}", proc, lockState);
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ rootProcStack.release();
+
+ if (proc.isSuccess()) {
+ LOG.info("{} finished in {} successfully.", proc, proc.elapsedTime());
+ if (proc.getProcId() == rootProcId) {
+ rootProcedureCleanup(proc);
+ } else {
+ executeCompletionCleanup(proc);
+ }
+ return;
+ }
+
+ } while (rootProcStack.isFailed());
+ }
+
+ /**
+ * execute procedure and submit its children
+ *
+ * @param rootProcStack procedure's root proc stack
+ * @param proc procedure
+ */
+ private void executeProcedure(RootProcedureStack rootProcStack, Procedure proc) {
+ Preconditions.checkArgument(
+ proc.getState() == ProcedureState.RUNNABLE, "NOT RUNNABLE! " + proc);
+ boolean suspended = false;
+ boolean reExecute;
+
+ Procedure[] subprocs = null;
+ do {
+ reExecute = false;
+ proc.resetPersistance();
+ try {
+ subprocs = proc.doExecute(this.environment);
+ if (subprocs != null && subprocs.length == 0) {
+ subprocs = null;
+ }
+ } catch (ProcedureSuspendedException e) {
+ LOG.debug("Suspend {}", proc);
+ suspended = true;
+ } catch (ProcedureYieldException e) {
+ LOG.debug("Yield {}", proc);
+ yieldProcedure(proc);
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupt during execution, suspend or retry it later.", e);
+ yieldProcedure(proc);
+ } catch (Throwable e) {
+ LOG.error("CODE-BUG:{}", proc, e);
+ proc.setFailure(new ProcedureException(e.getMessage(), e));
+ }
+
+ if (!proc.isFailed()) {
+ if (subprocs != null) {
+ if (subprocs.length == 1 && subprocs[0] == proc) {
+ subprocs = null;
+ reExecute = true;
+ } else {
+ subprocs = initializeChildren(rootProcStack, proc, subprocs);
+ LOG.info("Initialized sub procs:{}", Arrays.toString(subprocs));
+ }
+ } else if (proc.getState() == ProcedureState.WAITING_TIMEOUT) {
+ LOG.info("Added into timeoutExecutor {}", proc);
+ } else if (!suspended) {
+ proc.setState(ProcedureState.SUCCESS);
+ }
+ }
+ // add procedure into rollback stack.
+ rootProcStack.addRollbackStep(proc);
+
+ if (proc.needPersistance()) {
+ updateStoreOnExecution(rootProcStack, proc, subprocs);
+ }
+
+ if (!store.isRunning()) {
+ return;
+ }
+
+ if (proc.isRunnable() && !suspended && proc.isYieldAfterExecution(this.environment)) {
+ yieldProcedure(proc);
+ return;
+ }
+ } while (reExecute);
+
+ if (subprocs != null && !proc.isFailed()) {
+ submitChildrenProcedures(subprocs);
+ }
+
+ releaseLock(proc, false);
+ if (!suspended && proc.isFinished() && proc.hasParent()) {
+ countDownChildren(rootProcStack, proc);
+ }
+ }
+
+ /**
+ * Serve as a countdown latch to check whether all children has completed.
+ *
+ * @param rootProcStack root procedure stack
+ * @param proc proc
+ */
+ private void countDownChildren(RootProcedureStack rootProcStack, Procedure proc) {
+ Procedure parent = procedures.get(proc.getParentProcId());
+ if (parent == null && rootProcStack.isRollingback()) {
+ return;
+ }
+ if (parent.tryRunnable()) {
+ // if success, means all its children have completed, move parent to front of the queue.
+ store.update(parent);
+ scheduler.addFront(parent);
+ LOG.info(
+ "Finished subprocedure pid={}, resume processing ppid={}",
+ proc.getProcId(),
+ parent.getProcId());
+ }
+ }
+
+ /**
+ * Submit children procedures
+ *
+ * @param subprocs children procedures
+ */
+ private void submitChildrenProcedures(Procedure[] subprocs) {
+ for (Procedure subproc : subprocs) {
+ procedures.put(subproc.getProcId(), subproc);
+ scheduler.addFront(subproc);
+ }
+ }
+
+ private void updateStoreOnExecution(
+ RootProcedureStack rootProcStack, Procedure proc, Procedure[] subprocs) {
+ if (subprocs != null && !proc.isFailed()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Stored {}, children {}", proc, Arrays.toString(subprocs));
+ }
+ store.update(subprocs);
+ } else {
+ LOG.debug("Store update {}", proc);
+ if (proc.isFinished() && !proc.hasParent()) {
+ final long[] childProcIds = rootProcStack.getSubprocedureIds();
+ if (childProcIds != null) {
+ store.delete(childProcIds);
+ for (long childProcId : childProcIds) {
+ procedures.remove(childProcId);
+ }
+ } else {
+ store.update(proc);
+ }
+ } else {
+ store.update(proc);
+ }
+ }
+ }
+
+ private Procedure[] initializeChildren(
+ RootProcedureStack rootProcStack, Procedure proc, Procedure[] subprocs) {
+ final long rootProcedureId = getRootProcedureId(proc);
+ for (int i = 0; i < subprocs.length; i++) {
+ Procedure subproc = subprocs[i];
+ if (subproc == null) {
+ String errMsg = "subproc[" + i + "] is null, aborting procedure";
+ proc.setFailure(new ProcedureException((errMsg), new IllegalArgumentException(errMsg)));
+ return null;
+ }
+ subproc.setParentProcId(proc.getProcId());
+ subproc.setRootProcId(rootProcedureId);
+ subproc.setProcId(nextProcId());
+ subproc.setProcRunnable();
+ rootProcStack.addSubProcedure(subproc);
+ }
+
+ if (!proc.isFailed()) {
+ proc.setChildrenLatch(subprocs.length);
+ switch (proc.getState()) {
+ case RUNNABLE:
+ proc.setState(ProcedureState.WAITING);
+ break;
+ case WAITING_TIMEOUT:
+ timeoutExecutor.add(proc);
+ break;
+ default:
+ break;
+ }
+ }
+ return subprocs;
+ }
+
+ private void yieldProcedure(Procedure proc) {
+ releaseLock(proc, false);
+ scheduler.yield(proc);
+ }
+
+ /**
+ * Rollback full root procedure stack.
+ *
+ * @param rootProcId root procedure id
+ * @param procedureStack root procedure stack
+ * @return lock state
+ */
+ private ProcedureLockState executeRootStackRollback(
+ Long rootProcId, RootProcedureStack procedureStack) {
+ Procedure rootProcedure = procedures.get(rootProcId);
+ ProcedureException exception = rootProcedure.getException();
+ if (exception == null) {
+ exception = procedureStack.getException();
+ rootProcedure.setFailure(exception);
+ store.update(rootProcedure);
+ }
+ List> subprocStack = procedureStack.getSubproceduresStack();
+ int stackTail = subprocStack.size();
+ while (stackTail-- > 0) {
+ Procedure procedure = subprocStack.get(stackTail);
+ if (procedure.isSuccess()) {
+ subprocStack.remove(stackTail);
+ cleanupAfterRollback(procedure);
+ continue;
+ }
+ ProcedureLockState lockState = acquireLock(procedure);
+ if (lockState != ProcedureLockState.LOCK_ACQUIRED) {
+ return lockState;
+ }
+ lockState = executeRollback(procedure);
+ releaseLock(procedure, false);
+
+ boolean abortRollback = lockState != ProcedureLockState.LOCK_ACQUIRED;
+ abortRollback |= !isRunning() || !store.isRunning();
+ if (abortRollback) {
+ return lockState;
+ }
+
+ if (!procedure.isFinished() && procedure.isYieldAfterExecution(this.environment)) {
+ return ProcedureLockState.LOCK_YIELD_WAIT;
+ }
+
+ if (procedure != rootProcedure) {
+ executeCompletionCleanup(procedure);
+ }
+ }
+
+ LOG.info("Rolled back {}, time duration is {}", rootProcedure, rootProcedure.elapsedTime());
+ rootProcedureCleanup(rootProcedure);
+ return ProcedureLockState.LOCK_ACQUIRED;
+ }
+
+ private ProcedureLockState acquireLock(Procedure proc) {
+ if (proc.hasLock()) {
+ return ProcedureLockState.LOCK_ACQUIRED;
+ }
+ return proc.doAcquireLock(this.environment, store);
+ }
+
+ /**
+ * do execute defined in procedure and then update store or remove completely in case it is a
+ * child.
+ *
+ * @param procedure procedure
+ * @return procedure lock state
+ */
+ private ProcedureLockState executeRollback(Procedure procedure) {
+ ReentrantLock idLock =
+ idLockMap.computeIfAbsent(procedure.getProcId(), procId -> new ReentrantLock());
+ try {
+ idLock.lock();
+ procedure.doRollback(this.environment);
+ } catch (IOException e) {
+ LOG.error("Roll back failed for {}", procedure, e);
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted exception occured for {}", procedure, e);
+ } catch (Throwable t) {
+ LOG.error("CODE-BUG: runtime exception for {}", procedure, t);
+ } finally {
+ idLock.unlock();
+ }
+ cleanupAfterRollback(procedure);
+ return ProcedureLockState.LOCK_ACQUIRED;
+ }
+
+ private void cleanupAfterRollback(Procedure procedure) {
+ if (procedure.removeStackIndex()) {
+ if (!procedure.isSuccess()) {
+ procedure.setState(ProcedureState.ROLLEDBACK);
+ }
+ if (procedure.hasParent()) {
+ store.delete(procedure.getProcId());
+ procedures.remove(procedure.getProcId());
+ } else {
+ final long[] childProcIds = rollbackStack.get(procedure.getProcId()).getSubprocedureIds();
+ if (childProcIds != null) {
+ store.delete(childProcIds);
+ } else {
+ store.update(procedure);
+ }
+ }
+ } else {
+ store.update(procedure);
+ }
+ }
+
+ private void executeCompletionCleanup(Procedure proc) {
+ if (proc.hasLock()) {
+ releaseLock(proc, true);
+ }
+ try {
+ proc.completionCleanup(this.environment);
+ } catch (Throwable e) {
+ LOG.error("CODE-BUG:Uncaught runtime exception for procedure {}", proc, e);
+ }
+ }
+
+ private void rootProcedureCleanup(Procedure proc) {
+ executeCompletionCleanup(proc);
+ CompletedProcedureRetainer retainer = new CompletedProcedureRetainer<>(proc);
+ completed.put(proc.getProcId(), retainer);
+ rollbackStack.remove(proc.getProcId());
+ procedures.remove(proc.getProcId());
+ }
+
+ private Long getRootProcedureId(Procedure proc) {
+ return Procedure.getRootProcedureId(procedures, proc);
+ }
+
+ /**
+ * add a Procedure to executor
+ *
+ * @param procedure procedure
+ * @return procedure id
+ */
+ private long pushProcedure(Procedure procedure) {
+ final long currentProcId = procedure.getProcId();
+ RootProcedureStack stack = new RootProcedureStack();
+ rollbackStack.put(currentProcId, stack);
+ procedures.put(currentProcId, procedure);
+ scheduler.addBack(procedure);
+ return procedure.getProcId();
+ }
+
+ private class WorkerThread extends StoppableThread {
+ private final AtomicLong startTime = new AtomicLong(Long.MAX_VALUE);
+ private volatile Procedure activeProcedure;
+
+ public WorkerThread(ThreadGroup threadGroup) {
+ this(threadGroup, "ProcExecWorker-");
+ }
+
+ public WorkerThread(ThreadGroup threadGroup, String prefix) {
+ super(threadGroup, prefix + workId.incrementAndGet());
+ setDaemon(true);
+ }
+
+ @Override
+ public void sendStopSignal() {
+ scheduler.signalAll();
+ }
+
+ @Override
+ public void run() {
+ long lastUpdated = System.currentTimeMillis();
+ try {
+ while (isRunning() && keepAlive(lastUpdated)) {
+ Procedure procedure = scheduler.poll(keepAliveTime, TimeUnit.MILLISECONDS);
+ if (procedure == null) {
+ continue;
+ }
+ this.activeProcedure = procedure;
+ int activeCount = activeExecutorCount.incrementAndGet();
+ startTime.set(System.currentTimeMillis());
+ ReentrantLock idLock =
+ idLockMap.computeIfAbsent(procedure.getProcId(), id -> new ReentrantLock());
+ idLock.lock();
+ executeProcedure(procedure);
+ activeCount = activeExecutorCount.decrementAndGet();
+ LOG.trace("Halt pid={}, activeCount={}", procedure.getProcId(), activeCount);
+ this.activeProcedure = null;
+ lastUpdated = System.currentTimeMillis();
+ startTime.set(lastUpdated);
+ idLock.unlock();
+ }
+
+ } catch (Throwable throwable) {
+ LOG.warn("Worker terminated {}", this.activeProcedure, throwable);
+ } finally {
+ LOG.debug("Worker teminated.");
+ }
+ workerThreads.remove(this);
+ }
+
+ protected boolean keepAlive(long lastUpdated) {
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ Procedure p = this.activeProcedure;
+ return getName() + "(pid=" + (p == null ? Procedure.NO_PROC_ID : p.getProcId() + ")");
+ }
+
+ /** @return the time since the current procedure is running */
+ public long getCurrentRunTime() {
+ return System.currentTimeMillis() - startTime.get();
+ }
+ }
+
+ // A worker thread which can be added when core workers are stuck. Will timeout after
+ // keepAliveTime if there is no procedure to run.
+ private final class KeepAliveWorkerThread extends WorkerThread {
+ public KeepAliveWorkerThread(ThreadGroup group) {
+ super(group, "KAProcExecWorker-");
+ }
+
+ @Override
+ protected boolean keepAlive(long lastUpdate) {
+ return System.currentTimeMillis() - lastUpdate < keepAliveTime;
+ }
+ }
+
+ private final class WorkerMonitor extends InternalProcedure {
+ private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 5000; // 5sec
+
+ private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 10000; // 10sec
+
+ private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck
+
+ private float addWorkerStuckPercentage = DEFAULT_WORKER_ADD_STUCK_PERCENTAGE;
+ private int timeoutInterval = DEFAULT_WORKER_MONITOR_INTERVAL;
+ private int stuckThreshold = DEFAULT_WORKER_STUCK_THRESHOLD;
+
+ public WorkerMonitor() {
+ super(DEFAULT_WORKER_MONITOR_INTERVAL);
+ updateTimestamp();
+ }
+
+ private int checkForStuckWorkers() {
+ // check if any of the worker is stuck
+ int stuckCount = 0;
+ for (WorkerThread worker : workerThreads) {
+ if (worker.activeProcedure == null || worker.getCurrentRunTime() < stuckThreshold) {
+ continue;
+ }
+
+ // WARN the worker is stuck
+ stuckCount++;
+ LOG.warn("Worker stuck {}, run time {} ms", worker, worker.getCurrentRunTime());
+ }
+ return stuckCount;
+ }
+
+ private void checkThreadCount(final int stuckCount) {
+ // nothing to do if there are no runnable tasks
+ if (stuckCount < 1 || !scheduler.hasRunnables()) {
+ return;
+ }
+ // add a new thread if the worker stuck percentage exceed the threshold limit
+ // and every handler is active.
+ final float stuckPerc = ((float) stuckCount) / workerThreads.size();
+ // let's add new worker thread more aggressively, as they will timeout finally if there is no
+ // work to do.
+ if (stuckPerc >= addWorkerStuckPercentage && workerThreads.size() < maxPoolSize) {
+ final KeepAliveWorkerThread worker = new KeepAliveWorkerThread(threadGroup);
+ workerThreads.add(worker);
+ worker.start();
+ LOG.debug("Added new worker thread {}", worker);
+ }
+ }
+
+ @Override
+ protected void periodicExecute(Env env) {
+ final int stuckCount = checkForStuckWorkers();
+ checkThreadCount(stuckCount);
+ updateTimestamp();
+ }
+ }
+
+ public int getWorkerThreadCount() {
+ return workerThreads.size();
+ }
+
+ public boolean isRunning() {
+ return running.get();
+ }
+
+ public void stop() {
+ if (!running.getAndSet(false)) {
+ return;
+ }
+ LOG.info("Stopping");
+ scheduler.stop();
+ timeoutExecutor.sendStopSignal();
+ }
+
+ public void join() {
+ timeoutExecutor.awaitTermination();
+ for (WorkerThread workerThread : workerThreads) {
+ workerThread.awaitTermination();
+ }
+ try {
+ threadGroup.destroy();
+ } catch (IllegalThreadStateException e) {
+ LOG.error(
+ "ThreadGroup {} contains running threads; {}: See STDOUT",
+ this.threadGroup,
+ e.getMessage());
+ this.threadGroup.list();
+ }
+ }
+
+ public boolean isStarted(long procId) {
+ Procedure procedure = procedures.get(procId);
+ if (procedure == null) {
+ return completed.get(procId) != null;
+ }
+ return procedure.wasExecuted();
+ }
+
+ public boolean isFinished(final long procId) {
+ return !procedures.containsKey(procId);
+ }
+
+ public ConcurrentHashMap getProcedures() {
+ return procedures;
+ }
+
+ // -----------------------------CLIENT IMPLEMENTATION-----------------------------------
+ /**
+ * Submit a new root-procedure to the executor, called by client.
+ *
+ * @param procedure root procedure
+ * @return procedure id
+ */
+ public long submitProcedure(Procedure procedure) {
+ Preconditions.checkArgument(lastProcId.get() >= 0);
+ Preconditions.checkArgument(procedure.getState() == ProcedureState.INITIALIZING);
+ Preconditions.checkArgument(!procedure.hasParent(), "Unexpected parent", procedure);
+ final long currentProcId = nextProcId();
+ // Initialize the procedure
+ procedure.setProcId(currentProcId);
+ procedure.setProcRunnable();
+ // Commit the transaction
+ store.update(procedure);
+ LOG.debug("{} is stored.", procedure);
+ // Add the procedure to the executor
+ return pushProcedure(procedure);
+ }
+
+ /**
+ * Abort a specified procedure.
+ *
+ * @param procId procedure id
+ * @param force whether abort the running procdure.
+ * @return true if the procedure exists and has received the abort.
+ */
+ public boolean abort(long procId, boolean force) {
+ Procedure procedure = procedures.get(procId);
+ if (procedure != null) {
+ if (!force && procedure.wasExecuted()) {
+ return false;
+ }
+ return procedure.abort(this.environment);
+ }
+ return false;
+ }
+
+ public boolean abort(long procId) {
+ return abort(procId, true);
+ }
+
+ public Procedure getResult(long procId) {
+ CompletedProcedureRetainer retainer = completed.get(procId);
+ if (retainer == null) {
+ return null;
+ } else {
+ return retainer.getProcedure();
+ }
+ }
+
+ /**
+ * Query a procedure result
+ *
+ * @param procId procedure id
+ * @return procedure or retainer
+ */
+ public Procedure getResultOrProcedure(long procId) {
+ CompletedProcedureRetainer retainer = completed.get(procId);
+ if (retainer == null) {
+ return procedures.get(procId);
+ } else {
+ return retainer.getProcedure();
+ }
+ }
+
+ public ProcedureScheduler getScheduler() {
+ return scheduler;
+ }
+
+ public Env getEnvironment() {
+ return environment;
+ }
+
+ public IProcedureStore getStore() {
+ return store;
+ }
+
+ public RootProcedureStack getRollbackStack(long rootProcId) {
+ return rollbackStack.get(rootProcId);
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/ProcedureLockState.java b/procedure/src/main/java/org/apache/iotdb/procedure/ProcedureLockState.java
new file mode 100644
index 000000000000..2c98174a3620
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/ProcedureLockState.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+public enum ProcedureLockState {
+ LOCK_ACQUIRED,
+ LOCK_YIELD_WAIT,
+ LOCK_EVENT_WAIT
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/RootProcedureStack.java b/procedure/src/main/java/org/apache/iotdb/procedure/RootProcedureStack.java
new file mode 100644
index 000000000000..2491e827d358
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/RootProcedureStack.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+import org.apache.iotdb.procedure.exception.ProcedureException;
+import org.apache.iotdb.service.rpc.thrift.ProcedureState;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class RootProcedureStack {
+ private static final Logger LOG = LoggerFactory.getLogger(RootProcedureStack.class);
+
+ private enum State {
+ RUNNING, // The Procedure is running or ready to run
+ FAILED, // The Procedure failed, waiting for the rollback executing
+ ROLLINGBACK, // The Procedure failed and the execution was rolledback
+ }
+
+ private Set> subprocs = null;
+ private ArrayList> subprocStack = null;
+ private State state = State.RUNNING;
+ private int running = 0;
+
+ public synchronized boolean isFailed() {
+ switch (state) {
+ case ROLLINGBACK:
+ case FAILED:
+ return true;
+ default:
+ break;
+ }
+ return false;
+ }
+
+ public synchronized boolean isRollingback() {
+ return state == State.ROLLINGBACK;
+ }
+
+ /** Called by the ProcedureExecutor to mark rollback execution */
+ protected synchronized boolean setRollback() {
+ if (running == 0 && state == State.FAILED) {
+ state = State.ROLLINGBACK;
+ return true;
+ }
+ return false;
+ }
+
+ /** Called by the ProcedureExecutor to mark rollback execution */
+ protected synchronized void unsetRollback() {
+ assert state == State.ROLLINGBACK;
+ state = State.FAILED;
+ }
+
+ protected synchronized long[] getSubprocedureIds() {
+ if (subprocs == null) {
+ return null;
+ }
+ return subprocs.stream().mapToLong(Procedure::getProcId).toArray();
+ }
+
+ protected synchronized List> getSubproceduresStack() {
+ return subprocStack;
+ }
+
+ protected synchronized ProcedureException getException() {
+ if (subprocStack != null) {
+ for (Procedure proc : subprocStack) {
+ if (proc.hasException()) {
+ return proc.getException();
+ }
+ }
+ }
+ return null;
+ }
+
+ /** Called by the ProcedureExecutor to mark the procedure step as running. */
+ protected synchronized boolean acquire() {
+ if (state != State.RUNNING) {
+ return false;
+ }
+
+ running++;
+ return true;
+ }
+
+ /** Called by the ProcedureExecutor to mark the procedure step as finished. */
+ protected synchronized void release() {
+ running--;
+ }
+
+ protected synchronized void abort() {
+ if (state == State.RUNNING) {
+ state = State.FAILED;
+ }
+ }
+
+ /**
+ * Called by the ProcedureExecutor after the procedure step is completed, to add the step to the
+ * rollback list (or procedure stack)
+ */
+ protected synchronized void addRollbackStep(Procedure proc) {
+ if (proc.isFailed()) {
+ state = State.FAILED;
+ }
+ if (subprocStack == null) {
+ subprocStack = new ArrayList<>();
+ }
+ proc.addStackIndex(subprocStack.size());
+ LOG.trace("Add procedure {} as the {}th rollback step", proc, subprocStack.size());
+ subprocStack.add(proc);
+ }
+
+ protected synchronized void addSubProcedure(Procedure proc) {
+ if (!proc.hasParent()) {
+ return;
+ }
+ if (subprocs == null) {
+ subprocs = new HashSet<>();
+ }
+ subprocs.add(proc);
+ }
+
+ /**
+ * Called on store load by the ProcedureExecutor to load part of the stack.
+ *
+ *
Each procedure has its own stack-positions. Which means we have to write to the store only
+ * the Procedure we executed, and nothing else. on load we recreate the full stack by aggregating
+ * each procedure stack-positions.
+ */
+ protected synchronized void loadStack(Procedure proc) {
+ addSubProcedure(proc);
+ int[] stackIndexes = proc.getStackIndexes();
+ if (stackIndexes != null) {
+ if (subprocStack == null) {
+ subprocStack = new ArrayList<>();
+ }
+ int diff = (1 + stackIndexes[stackIndexes.length - 1]) - subprocStack.size();
+ if (diff > 0) {
+ subprocStack.ensureCapacity(1 + stackIndexes[stackIndexes.length - 1]);
+ while (diff-- > 0) {
+ subprocStack.add(null);
+ }
+ }
+ for (int stackIndex : stackIndexes) {
+ subprocStack.set(stackIndex, proc);
+ }
+ }
+ if (proc.getState() == ProcedureState.ROLLEDBACK) {
+ state = State.ROLLINGBACK;
+ } else if (proc.isFailed()) {
+ state = State.FAILED;
+ }
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/StateMachineProcedure.java b/procedure/src/main/java/org/apache/iotdb/procedure/StateMachineProcedure.java
new file mode 100644
index 000000000000..837452dcef5e
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/StateMachineProcedure.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+import org.apache.iotdb.procedure.exception.ProcedureSuspendedException;
+import org.apache.iotdb.procedure.exception.ProcedureYieldException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Procedure described by a series of steps.
+ *
+ *
The procedure implementor must have an enum of 'states', describing the various step of the
+ * procedure. Once the procedure is running, the procedure-framework will call executeFromState()
+ * using the 'state' provided by the user. The first call to executeFromState() will be performed
+ * with 'state = null'. The implementor can jump between states using
+ * setNextState(MyStateEnum.ordinal()). The rollback will call rollbackState() for each state that
+ * was executed, in reverse order.
+ */
+public abstract class StateMachineProcedure extends Procedure {
+ private static final Logger LOG = LoggerFactory.getLogger(StateMachineProcedure.class);
+
+ private static final int EOF_STATE = Integer.MIN_VALUE;
+
+ private final AtomicBoolean aborted = new AtomicBoolean(false);
+
+ private Flow stateFlow = Flow.HAS_MORE_STATE;
+ protected int stateCount = 0;
+ private int[] states = null;
+
+ private List> subProcList = null;
+
+ protected final int getCycles() {
+ return cycles;
+ }
+
+ /** Cycles on same state. Good for figuring if we are stuck. */
+ private int cycles = 0;
+
+ /** Ordinal of the previous state. So we can tell if we are progressing or not. */
+ private int previousState;
+
+ public enum Flow {
+ HAS_MORE_STATE,
+ NO_MORE_STATE,
+ }
+
+ /**
+ * called to perform a single step of the specified 'state' of the procedure
+ *
+ * @param state state to execute
+ * @return Flow.NO_MORE_STATE if the procedure is completed, Flow.HAS_MORE_STATE if there is
+ * another step.
+ */
+ protected abstract Flow executeFromState(Env env, TState state)
+ throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException;
+
+ /**
+ * called to perform the rollback of the specified state
+ *
+ * @param state state to rollback
+ * @throws IOException temporary failure, the rollback will retry later
+ */
+ protected abstract void rollbackState(Env env, TState state)
+ throws IOException, InterruptedException;
+
+ /**
+ * Convert an ordinal (or state id) to an Enum (or more descriptive) state object.
+ *
+ * @param stateId the ordinal() of the state enum (or state id)
+ * @return the state enum object
+ */
+ protected abstract TState getState(int stateId);
+
+ /**
+ * Convert the Enum (or more descriptive) state object to an ordinal (or state id).
+ *
+ * @param state the state enum object
+ * @return stateId the ordinal() of the state enum (or state id)
+ */
+ protected abstract int getStateId(TState state);
+
+ /**
+ * Return the initial state object that will be used for the first call to executeFromState().
+ *
+ * @return the initial state enum object
+ */
+ protected abstract TState getInitialState();
+
+ /**
+ * Set the next state for the procedure.
+ *
+ * @param state the state enum object
+ */
+ protected void setNextState(final TState state) {
+ setNextState(getStateId(state));
+ failIfAborted();
+ }
+
+ /**
+ * By default, the executor will try ro run all the steps of the procedure start to finish. Return
+ * true to make the executor yield between execution steps to give other procedures time to run
+ * their steps.
+ *
+ * @param state the state we are going to execute next.
+ * @return Return true if the executor should yield before the execution of the specified step.
+ * Defaults to return false.
+ */
+ protected boolean isYieldBeforeExecuteFromState(Env env, TState state) {
+ return false;
+ }
+
+ /**
+ * Add a child procedure to execute
+ *
+ * @param subProcedure the child procedure
+ */
+ protected > void addChildProcedure(T... subProcedure) {
+ if (subProcedure == null) {
+ return;
+ }
+ final int len = subProcedure.length;
+ if (len == 0) {
+ return;
+ }
+ if (subProcList == null) {
+ subProcList = new ArrayList<>(len);
+ }
+ subProcList.addAll(Arrays.asList(subProcedure).subList(0, len));
+ }
+
+ @Override
+ protected Procedure[] execute(final Env env)
+ throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
+ updateTimestamp();
+ try {
+ failIfAborted();
+
+ if (!hasMoreState() || isFailed()) {
+ return null;
+ }
+
+ TState state = getCurrentState();
+ if (stateCount == 0) {
+ setNextState(getStateId(state));
+ }
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(state + " " + this + "; cycles=" + this.cycles);
+ }
+ // Keep running count of cycles
+ if (getStateId(state) != this.previousState) {
+ this.previousState = getStateId(state);
+ this.cycles = 0;
+ } else {
+ this.cycles++;
+ }
+
+ LOG.trace("{}", this);
+ stateFlow = executeFromState(env, state);
+ if (!hasMoreState()) {
+ setNextState(EOF_STATE);
+ }
+
+ if (subProcList != null && !subProcList.isEmpty()) {
+ Procedure[] subProcedures = subProcList.toArray(new Procedure[subProcList.size()]);
+ subProcList = null;
+ return subProcedures;
+ }
+ return (isWaiting() || isFailed() || !hasMoreState()) ? null : new Procedure[] {this};
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ @Override
+ protected void rollback(final Env env) throws IOException, InterruptedException {
+ if (isEofState()) {
+ stateCount--;
+ }
+
+ try {
+ updateTimestamp();
+ rollbackState(env, getCurrentState());
+ } finally {
+ stateCount--;
+ updateTimestamp();
+ }
+ }
+
+ protected boolean isEofState() {
+ return stateCount > 0 && states[stateCount - 1] == EOF_STATE;
+ }
+
+ @Override
+ protected boolean abort(final Env env) {
+ LOG.debug("Abort requested for {}", this);
+ if (!hasMoreState()) {
+ LOG.warn("Ignore abort request on {} because it has already been finished", this);
+ return false;
+ }
+ if (!isRollbackSupported(getCurrentState())) {
+ LOG.warn("Ignore abort request on {} because it does not support rollback", this);
+ return false;
+ }
+ aborted.set(true);
+ return true;
+ }
+
+ /**
+ * If procedure has more states then abort it otherwise procedure is finished and abort can be
+ * ignored.
+ */
+ protected final void failIfAborted() {
+ if (aborted.get()) {
+ if (hasMoreState()) {
+ setAbortFailure(getClass().getSimpleName(), "abort requested");
+ } else {
+ LOG.warn("Ignoring abort request on state='" + getCurrentState() + "' for " + this);
+ }
+ }
+ }
+
+ /**
+ * Used by the default implementation of abort() to know if the current state can be aborted and
+ * rollback can be triggered.
+ */
+ protected boolean isRollbackSupported(final TState state) {
+ return false;
+ }
+
+ @Override
+ protected boolean isYieldAfterExecution(final Env env) {
+ return isYieldBeforeExecuteFromState(env, getCurrentState());
+ }
+
+ private boolean hasMoreState() {
+ return stateFlow != Flow.NO_MORE_STATE;
+ }
+
+ protected TState getCurrentState() {
+ return stateCount > 0 ? getState(states[stateCount - 1]) : getInitialState();
+ }
+
+ /**
+ * This method is used from test code as it cannot be assumed that state transition will happen
+ * sequentially. Some procedures may skip steps/ states, some may add intermediate steps in
+ * future.
+ */
+ public int getCurrentStateId() {
+ return getStateId(getCurrentState());
+ }
+
+ /**
+ * Set the next state for the procedure.
+ *
+ * @param stateId the ordinal() of the state enum (or state id)
+ */
+ private void setNextState(final int stateId) {
+ if (states == null || states.length == stateCount) {
+ int newCapacity = stateCount + 8;
+ if (states != null) {
+ states = Arrays.copyOf(states, newCapacity);
+ } else {
+ states = new int[newCapacity];
+ }
+ }
+ states[stateCount++] = stateId;
+ }
+
+ @Override
+ protected void toStringState(StringBuilder builder) {
+ super.toStringState(builder);
+ if (!isFinished() && !isEofState() && getCurrentState() != null) {
+ builder.append(":").append(getCurrentState());
+ }
+ }
+
+ @Override
+ public void serialize(ByteBuffer byteBuffer) throws IOException {
+ super.serialize(byteBuffer);
+ byteBuffer.putInt(stateCount);
+ for (int i = 0; i < stateCount; ++i) {
+ byteBuffer.putInt(states[i]);
+ }
+ }
+
+ @Override
+ public void deserialize(ByteBuffer byteBuffer) throws IOException {
+ super.deserialize(byteBuffer);
+ stateCount = byteBuffer.getInt();
+ if (stateCount > 0) {
+ states = new int[stateCount];
+ for (int i = 0; i < stateCount; ++i) {
+ states[i] = byteBuffer.getInt();
+ }
+ if (isEofState()) {
+ stateFlow = Flow.NO_MORE_STATE;
+ }
+ } else {
+ states = null;
+ }
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/StoppableThread.java b/procedure/src/main/java/org/apache/iotdb/procedure/StoppableThread.java
new file mode 100644
index 000000000000..9309333a6569
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/StoppableThread.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class StoppableThread extends Thread {
+
+ private static final int JOIN_TIMEOUT = 250;
+ private static final Logger LOG = LoggerFactory.getLogger(StoppableThread.class);
+
+ public StoppableThread(ThreadGroup threadGroup, String name) {
+ super(threadGroup, name);
+ }
+
+ public abstract void sendStopSignal();
+
+ public void awaitTermination() {
+ try {
+ for (int i = 0; isAlive(); i++) {
+ sendStopSignal();
+ join(JOIN_TIMEOUT);
+ if (i > 0 && (i % 8) == 0) {
+ interrupt();
+ }
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("{} join wait got interrupted", getName(), e);
+ }
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/TimeoutExecutorThread.java b/procedure/src/main/java/org/apache/iotdb/procedure/TimeoutExecutorThread.java
new file mode 100644
index 000000000000..5ac861560ffe
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/TimeoutExecutorThread.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+import java.util.concurrent.DelayQueue;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.TimeUnit;
+
+public class TimeoutExecutorThread extends StoppableThread {
+
+ private static final int DELAY_QUEUE_TIMEOUT = 20;
+ private final ProcedureExecutor executor;
+ private final DelayQueue> queue = new DelayQueue<>();
+
+ public TimeoutExecutorThread(
+ ProcedureExecutor envProcedureExecutor, ThreadGroup threadGroup, String name) {
+ super(threadGroup, name);
+ setDaemon(true);
+ this.executor = envProcedureExecutor;
+ }
+
+ public void add(Procedure procedure) {
+ queue.add(new ProcedureDelayContainer<>(procedure));
+ }
+
+ public boolean remove(Procedure procedure) {
+ return queue.remove(new ProcedureDelayContainer<>(procedure));
+ }
+
+ private ProcedureDelayContainer takeQuietly() {
+ try {
+ return queue.poll(DELAY_QUEUE_TIMEOUT, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ currentThread().interrupt();
+ return null;
+ }
+ }
+
+ @Override
+ public void run() {
+ while (executor.isRunning()) {
+ ProcedureDelayContainer delayTask = takeQuietly();
+ if (delayTask == null) {
+ continue;
+ }
+ Procedure procedure = delayTask.getProcedure();
+ if (procedure instanceof InternalProcedure) {
+ InternalProcedure internal = (InternalProcedure) procedure;
+ internal.periodicExecute(executor.getEnvironment());
+ procedure.updateTimestamp();
+ queue.add(delayTask);
+ } else {
+ if (procedure.setTimeoutFailure(executor.getEnvironment())) {
+ long rootProcId = executor.getRootProcId(procedure);
+ RootProcedureStack rollbackStack = executor.getRollbackStack(rootProcId);
+ rollbackStack.abort();
+ executor.getStore().update(procedure);
+ executor.getScheduler().addFront(procedure);
+ }
+ }
+ }
+ }
+
+ public void sendStopSignal() {}
+
+ private static class ProcedureDelayContainer implements Delayed {
+
+ private final Procedure procedure;
+
+ public ProcedureDelayContainer(Procedure procedure) {
+ this.procedure = procedure;
+ }
+
+ public Procedure getProcedure() {
+ return procedure;
+ }
+
+ @Override
+ public long getDelay(TimeUnit unit) {
+ long delay = procedure.getTimeoutTimestamp() - System.currentTimeMillis();
+ return unit.convert(delay, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public int compareTo(Delayed other) {
+ return Long.compareUnsigned(
+ this.getDelay(TimeUnit.MILLISECONDS), other.getDelay(TimeUnit.MILLISECONDS));
+ }
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/conf/ProcedureNodeConfig.java b/procedure/src/main/java/org/apache/iotdb/procedure/conf/ProcedureNodeConfig.java
new file mode 100644
index 000000000000..048f7b0e175e
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/conf/ProcedureNodeConfig.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.conf;
+
+import java.io.File;
+
+public class ProcedureNodeConfig {
+
+ private String rpcAddress = "0.0.0.0";
+ private int rpcPort = 22281;
+ private int confignodePort = 22277;
+ private int datanodePort = 22278;
+ private int rpcMaxConcurrentClientNum = 65535;
+ private boolean rpcAdvancedCompressionEnable = false;
+ private boolean isRpcThriftCompressionEnabled = false;
+ private int thriftMaxFrameSize = 536870912;
+ private int thriftDefaultBufferSize = 1024;
+ private int thriftServerAwaitTimeForStopService = 60;
+ private String procedureWalDir =
+ ProcedureNodeConstant.PROC_DIR + File.separator + ProcedureNodeConstant.WAL_DIR;
+ private int completedEvictTTL = 800;
+ private int completedCleanInterval = 30;
+ private int workerThreadsCoreSize = Math.max(Runtime.getRuntime().availableProcessors() / 4, 16);
+
+ public String getRpcAddress() {
+ return rpcAddress;
+ }
+
+ public void setRpcAddress(String rpcAddress) {
+ this.rpcAddress = rpcAddress;
+ }
+
+ public int getRpcPort() {
+ return rpcPort;
+ }
+
+ public void setRpcPort(int rpcPort) {
+ this.rpcPort = rpcPort;
+ }
+
+ public int getConfignodePort() {
+ return confignodePort;
+ }
+
+ public void setConfignodePort(int confignodePort) {
+ this.confignodePort = confignodePort;
+ }
+
+ public int getDatanodePort() {
+ return datanodePort;
+ }
+
+ public void setDatanodePort(int datanodePort) {
+ this.datanodePort = datanodePort;
+ }
+
+ public int getRpcMaxConcurrentClientNum() {
+ return rpcMaxConcurrentClientNum;
+ }
+
+ public void setRpcMaxConcurrentClientNum(int rpcMaxConcurrentClientNum) {
+ this.rpcMaxConcurrentClientNum = rpcMaxConcurrentClientNum;
+ }
+
+ public boolean isRpcThriftCompressionEnabled() {
+ return isRpcThriftCompressionEnabled;
+ }
+
+ public void setRpcThriftCompressionEnabled(boolean rpcThriftCompressionEnabled) {
+ isRpcThriftCompressionEnabled = rpcThriftCompressionEnabled;
+ }
+
+ public int getThriftMaxFrameSize() {
+ return thriftMaxFrameSize;
+ }
+
+ public void setThriftMaxFrameSize(int thriftMaxFrameSize) {
+ this.thriftMaxFrameSize = thriftMaxFrameSize;
+ }
+
+ public int getThriftDefaultBufferSize() {
+ return thriftDefaultBufferSize;
+ }
+
+ public void setThriftDefaultBufferSize(int thriftDefaultBufferSize) {
+ this.thriftDefaultBufferSize = thriftDefaultBufferSize;
+ }
+
+ public int getThriftServerAwaitTimeForStopService() {
+ return thriftServerAwaitTimeForStopService;
+ }
+
+ public void setThriftServerAwaitTimeForStopService(int thriftServerAwaitTimeForStopService) {
+ this.thriftServerAwaitTimeForStopService = thriftServerAwaitTimeForStopService;
+ }
+
+ public String getProcedureWalDir() {
+ return procedureWalDir;
+ }
+
+ public void setProcedureWalDir(String procedureWalDir) {
+ this.procedureWalDir = procedureWalDir;
+ }
+
+ public int getCompletedEvictTTL() {
+ return completedEvictTTL;
+ }
+
+ public void setCompletedEvictTTL(int completedEvictTTL) {
+ this.completedEvictTTL = completedEvictTTL;
+ }
+
+ public int getCompletedCleanInterval() {
+ return completedCleanInterval;
+ }
+
+ public void setCompletedCleanInterval(int completedCleanInterval) {
+ this.completedCleanInterval = completedCleanInterval;
+ }
+
+ public boolean isRpcAdvancedCompressionEnable() {
+ return rpcAdvancedCompressionEnable;
+ }
+
+ public void setRpcAdvancedCompressionEnable(boolean rpcAdvancedCompressionEnable) {
+ this.rpcAdvancedCompressionEnable = rpcAdvancedCompressionEnable;
+ }
+
+ public int getWorkerThreadsCoreSize() {
+ return workerThreadsCoreSize;
+ }
+
+ public void setWorkerThreadsCoreSize(int workerThreadsCoreSize) {
+ this.workerThreadsCoreSize = workerThreadsCoreSize;
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/conf/ProcedureNodeConfigDescriptor.java b/procedure/src/main/java/org/apache/iotdb/procedure/conf/ProcedureNodeConfigDescriptor.java
new file mode 100644
index 000000000000..cab0bd747318
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/conf/ProcedureNodeConfigDescriptor.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.conf;
+
+import org.apache.iotdb.commons.exception.StartupException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Properties;
+
+public class ProcedureNodeConfigDescriptor {
+ private static final Logger LOG = LoggerFactory.getLogger(ProcedureNodeConfigDescriptor.class);
+
+ private final ProcedureNodeConfig conf = new ProcedureNodeConfig();
+
+ private ProcedureNodeConfigDescriptor() {
+ loadProps();
+ }
+
+ public ProcedureNodeConfig getConf() {
+ return conf;
+ }
+
+ /**
+ * get props url location
+ *
+ * @return url object if location exit, otherwise null.
+ */
+ public URL getPropsUrl() {
+ // Check if a config-directory was specified first.
+ String urlString = System.getProperty(ProcedureNodeConstant.PROCEDURE_CONF_DIR, null);
+ // If it wasn't, check if a home directory was provided
+ if (urlString == null) {
+ urlString = System.getProperty(ProcedureNodeConstant.PROCEDURENODE_HOME, null);
+ if (urlString != null) {
+ urlString =
+ urlString
+ + File.separatorChar
+ + "conf"
+ + File.separatorChar
+ + ProcedureNodeConstant.CONF_NAME;
+ } else {
+ // When start ProcedureNode with the script, the environment variables ProcedureNode_CONF
+ // and ProcedureNode_HOME will be set. But we didn't set these two in developer mode.
+ // Thus, just return null and use default Configuration in developer mode.
+ return null;
+ }
+ }
+ // If a config location was provided, but it doesn't end with a properties file,
+ // append the default location.
+ else if (!urlString.endsWith(".properties")) {
+ urlString += (File.separatorChar + ProcedureNodeConstant.CONF_NAME);
+ }
+
+ // If the url doesn't start with "file:" or "classpath:", it's provided as a no path.
+ // So we need to add it to make it a real URL.
+ if (!urlString.startsWith("file:") && !urlString.startsWith("classpath:")) {
+ urlString = "file:" + urlString;
+ }
+ try {
+ return new URL(urlString);
+ } catch (MalformedURLException e) {
+ return null;
+ }
+ }
+
+ private void loadProps() {
+ URL url = getPropsUrl();
+ if (url == null) {
+ LOG.warn(
+ "Couldn't load the ProcedureNode configuration from any of the known sources. Use default configuration.");
+ return;
+ }
+
+ try (InputStream inputStream = url.openStream()) {
+
+ LOG.info("start reading ProcedureNode conf file: {}", url);
+
+ Properties properties = new Properties();
+ properties.load(inputStream);
+
+ conf.setRpcAddress(
+ properties.getProperty("procedure_node_address", String.valueOf(conf.getRpcAddress())));
+
+ conf.setRpcPort(
+ Integer.parseInt(
+ properties.getProperty("config_node_rpc_port", String.valueOf(conf.getRpcPort()))));
+
+ conf.setConfignodePort(
+ Integer.parseInt(
+ properties.getProperty(
+ "config_node_port", String.valueOf(conf.getConfignodePort()))));
+
+ conf.setDatanodePort(
+ Integer.parseInt(
+ properties.getProperty("date_node_port", String.valueOf(conf.getDatanodePort()))));
+
+ conf.setRpcAdvancedCompressionEnable(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "rpc_advanced_compression_enable",
+ String.valueOf(conf.isRpcAdvancedCompressionEnable()))));
+
+ conf.setRpcThriftCompressionEnabled(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "rpc_thrift_compression_enable",
+ String.valueOf(conf.isRpcThriftCompressionEnabled()))));
+
+ conf.setRpcMaxConcurrentClientNum(
+ Integer.parseInt(
+ properties.getProperty(
+ "rpc_max_concurrent_client_num",
+ String.valueOf(conf.getRpcMaxConcurrentClientNum()))));
+
+ conf.setThriftDefaultBufferSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "thrift_init_buffer_size", String.valueOf(conf.getThriftDefaultBufferSize()))));
+
+ conf.setThriftMaxFrameSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "thrift_max_frame_size", String.valueOf(conf.getThriftMaxFrameSize()))));
+
+ conf.setProcedureWalDir(properties.getProperty("proc_wal_dir", conf.getProcedureWalDir()));
+
+ conf.setCompletedEvictTTL(
+ Integer.parseInt(
+ properties.getProperty(
+ "completed_evict_ttl", String.valueOf(conf.getCompletedEvictTTL()))));
+
+ conf.setCompletedCleanInterval(
+ Integer.parseInt(
+ properties.getProperty(
+ "completed_clean_interval", String.valueOf(conf.getCompletedCleanInterval()))));
+
+ conf.setWorkerThreadsCoreSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "workerthreads_core_size", String.valueOf(conf.getWorkerThreadsCoreSize()))));
+ } catch (IOException e) {
+ LOG.warn("Couldn't load ProcedureNode conf file, use default config", e);
+ } finally {
+ updatePath();
+ }
+ }
+
+ private void updatePath() {
+ formulateFolders();
+ }
+
+ private void formulateFolders() {
+ conf.setProcedureWalDir(addHomeDir(conf.getProcedureWalDir()));
+ }
+
+ private String addHomeDir(String dir) {
+ String homeDir = System.getProperty(ProcedureNodeConstant.PROCEDURENODE_HOME, null);
+ if (!new File(dir).isAbsolute() && homeDir != null && homeDir.length() > 0) {
+ if (!homeDir.endsWith(File.separator)) {
+ dir = homeDir + File.separatorChar + dir;
+ } else {
+ dir = homeDir + dir;
+ }
+ }
+ return dir;
+ }
+
+ public static ProcedureNodeConfigDescriptor getInstance() {
+ return ProcedureNodeDescriptorHolder.INSTANCE;
+ }
+
+ public void checkConfig() throws StartupException {
+ File walDir = new File(conf.getProcedureWalDir());
+ if (!walDir.exists()) {
+ if (walDir.mkdirs()) {
+ LOG.info("Make procedure wall dirs:{}", walDir);
+ } else {
+ throw new StartupException(
+ String.format(
+ "Start procedure node failed, because can not make wal dirs:%s.",
+ walDir.getAbsolutePath()));
+ }
+ }
+ }
+
+ private static class ProcedureNodeDescriptorHolder {
+
+ private static final ProcedureNodeConfigDescriptor INSTANCE =
+ new ProcedureNodeConfigDescriptor();
+
+ private ProcedureNodeDescriptorHolder() {
+ // empty constructor
+ }
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/conf/ProcedureNodeConstant.java b/procedure/src/main/java/org/apache/iotdb/procedure/conf/ProcedureNodeConstant.java
new file mode 100644
index 000000000000..750a97ab11ec
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/conf/ProcedureNodeConstant.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.conf;
+
+public class ProcedureNodeConstant {
+ public static final String PROCEDURE_WAL_SUFFIX = ".proc.wal";
+ public static final String PROCEDURENODE_PACKAGE = "org.apache.iotdb.procedurenode.service";
+ public static final String JMX_TYPE = "type";
+ public static final String GLOBAL_NAME = "IoTDB ProcedureNode";
+ public static final String PROCEDURE_CONF_DIR = "PROCEDURENODE_CONF";
+ public static final String PROCEDURENODE_HOME = "PROCEDURENODE_HOME";
+ public static final String CONF_NAME = "iotdb-procedure.properties";
+ public static final String PROC_DIR = "proc";
+ public static final String WAL_DIR = "wal";
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/env/ClusterProcedureEnvironment.java b/procedure/src/main/java/org/apache/iotdb/procedure/env/ClusterProcedureEnvironment.java
new file mode 100644
index 000000000000..c8a8ad79ffe1
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/env/ClusterProcedureEnvironment.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.env;
+
+/**
+ * Procedure Execute environment which is used to rpc between other nodes (datanode, confignode).
+ */
+public class ClusterProcedureEnvironment {}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureAbortedException.java b/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureAbortedException.java
new file mode 100644
index 000000000000..51143ca14fcd
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureAbortedException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.exception;
+
+public class ProcedureAbortedException extends ProcedureException {
+ public ProcedureAbortedException() {
+ super();
+ }
+
+ public ProcedureAbortedException(String msg) {
+ super(msg);
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureException.java b/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureException.java
new file mode 100644
index 000000000000..7a9749d76d50
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureException.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.exception;
+
+public class ProcedureException extends Exception {
+ /** default constructor */
+ public ProcedureException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ *
+ * @param s message
+ */
+ public ProcedureException(String s) {
+ super(s);
+ }
+
+ public ProcedureException(Throwable t) {
+ super(t);
+ }
+
+ public ProcedureException(String source, Throwable cause) {
+ super(source, cause);
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureSuspendedException.java b/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureSuspendedException.java
new file mode 100644
index 000000000000..7e9dbdab3cbe
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureSuspendedException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.exception;
+
+public class ProcedureSuspendedException extends ProcedureException {
+
+ private static final long serialVersionUID = -8328419627678496269L;
+
+ /** default constructor */
+ public ProcedureSuspendedException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ *
+ * @param s message
+ */
+ public ProcedureSuspendedException(String s) {
+ super(s);
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureTimeoutException.java b/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureTimeoutException.java
new file mode 100644
index 000000000000..422dad201c46
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureTimeoutException.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.exception;
+
+public class ProcedureTimeoutException extends ProcedureException {
+ public ProcedureTimeoutException(String s) {
+ super(s);
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureYieldException.java b/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureYieldException.java
new file mode 100644
index 000000000000..3f5c69a069b7
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/exception/ProcedureYieldException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.exception;
+
+public class ProcedureYieldException extends ProcedureException {
+ /** default constructor */
+ public ProcedureYieldException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ *
+ * @param s message
+ */
+ public ProcedureYieldException(String s) {
+ super(s);
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/scheduler/AbstractProcedureScheduler.java b/procedure/src/main/java/org/apache/iotdb/procedure/scheduler/AbstractProcedureScheduler.java
new file mode 100644
index 000000000000..be98b3986749
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/scheduler/AbstractProcedureScheduler.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.scheduler;
+
+import org.apache.iotdb.procedure.Procedure;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+public abstract class AbstractProcedureScheduler implements ProcedureScheduler {
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractProcedureScheduler.class);
+ private final ReentrantLock schedulerLock = new ReentrantLock();
+ private final Condition schedWaitCond = schedulerLock.newCondition();
+ private boolean running = false;
+
+ @Override
+ public void start() {
+ schedLock();
+ try {
+ running = true;
+ } finally {
+ schedUnlock();
+ }
+ }
+
+ @Override
+ public void stop() {
+ schedLock();
+ try {
+ running = false;
+ schedWaitCond.signalAll();
+ } finally {
+ schedUnlock();
+ }
+ }
+
+ @Override
+ public void signalAll() {
+ schedLock();
+ try {
+ schedWaitCond.signalAll();
+ } finally {
+ schedUnlock();
+ }
+ }
+
+ // ==========================================================================
+ // Add related
+ // ==========================================================================
+ /**
+ * Add the procedure to the queue. NOTE: this method is called with the sched lock held.
+ *
+ * @param procedure the Procedure to add
+ * @param addFront true if the item should be added to the front of the queue
+ */
+ protected abstract void enqueue(Procedure procedure, boolean addFront);
+
+ @Override
+ public void addFront(final Procedure procedure) {
+ push(procedure, true, true);
+ }
+
+ @Override
+ public void addFront(final Procedure procedure, boolean notify) {
+ push(procedure, true, notify);
+ }
+
+ @Override
+ public void addBack(final Procedure procedure) {
+ push(procedure, false, true);
+ }
+
+ @Override
+ public void addBack(final Procedure procedure, boolean notify) {
+ push(procedure, false, notify);
+ }
+
+ protected void push(final Procedure procedure, final boolean addFront, final boolean notify) {
+ schedLock();
+ try {
+ enqueue(procedure, addFront);
+ if (notify) {
+ schedWaitCond.signal();
+ }
+ } finally {
+ schedUnlock();
+ }
+ }
+
+ // ==========================================================================
+ // Poll related
+ // ==========================================================================
+ /**
+ * Fetch one Procedure from the queue NOTE: this method is called with the sched lock held.
+ *
+ * @return the Procedure to execute, or null if nothing is available.
+ */
+ protected abstract Procedure dequeue();
+
+ @Override
+ public Procedure poll() {
+ return poll(-1);
+ }
+
+ @Override
+ public Procedure poll(long timeout, TimeUnit unit) {
+ return poll(unit.toNanos(timeout));
+ }
+
+ public Procedure poll(final long nanos) {
+ schedLock();
+ try {
+ if (!running) {
+ LOG.debug("the scheduler is not running");
+ return null;
+ }
+
+ if (!queueHasRunnables()) {
+ // WA_AWAIT_NOT_IN_LOOP: we are not in a loop because we want the caller
+ // to take decisions after a wake/interruption.
+ if (nanos < 0) {
+ schedWaitCond.await();
+ } else {
+ schedWaitCond.awaitNanos(nanos);
+ }
+ if (!queueHasRunnables()) {
+ return null;
+ }
+ }
+ final Procedure pollResult = dequeue();
+
+ return pollResult;
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return null;
+ } finally {
+ schedUnlock();
+ }
+ }
+
+ // ==========================================================================
+ // Utils
+ // ==========================================================================
+ /**
+ * Returns the number of elements in this queue. NOTE: this method is called with the sched lock
+ * held.
+ *
+ * @return the number of elements in this queue.
+ */
+ protected abstract int queueSize();
+
+ /**
+ * Returns true if there are procedures available to process. NOTE: this method is called with the
+ * sched lock held.
+ *
+ * @return true if there are procedures available to process, otherwise false.
+ */
+ protected abstract boolean queueHasRunnables();
+
+ @Override
+ public int size() {
+ schedLock();
+ try {
+ return queueSize();
+ } finally {
+ schedUnlock();
+ }
+ }
+
+ @Override
+ public boolean hasRunnables() {
+ schedLock();
+ try {
+ return queueHasRunnables();
+ } finally {
+ schedUnlock();
+ }
+ }
+
+ // ==========================================================================
+ // Internal helpers
+ // ==========================================================================
+ protected void schedLock() {
+ schedulerLock.lock();
+ }
+
+ protected void schedUnlock() {
+ schedulerLock.unlock();
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/scheduler/ProcedureScheduler.java b/procedure/src/main/java/org/apache/iotdb/procedure/scheduler/ProcedureScheduler.java
new file mode 100644
index 000000000000..b582e80502eb
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/scheduler/ProcedureScheduler.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.scheduler;
+
+import org.apache.iotdb.procedure.Procedure;
+
+import java.util.concurrent.TimeUnit;
+
+/** Keep track of the runnable procedures */
+public interface ProcedureScheduler {
+ /** Start the scheduler */
+ void start();
+
+ /** Stop the scheduler */
+ void stop();
+
+ /**
+ * In case the class is blocking on poll() waiting for items to be added, this method should awake
+ * poll() and poll() should return.
+ */
+ void signalAll();
+
+ /**
+ * Inserts the specified element at the front of this queue.
+ *
+ * @param proc the Procedure to add
+ */
+ void addFront(Procedure proc);
+
+ /**
+ * Inserts the specified element at the front of this queue.
+ *
+ * @param proc the Procedure to add
+ * @param notify whether need to notify worker
+ */
+ void addFront(Procedure proc, boolean notify);
+
+ /**
+ * Inserts the specified element at the end of this queue.
+ *
+ * @param proc the Procedure to add
+ */
+ void addBack(Procedure proc);
+
+ /**
+ * Inserts the specified element at the end of this queue.
+ *
+ * @param proc the Procedure to add
+ * @param notify whether need to notify worker
+ */
+ void addBack(Procedure proc, boolean notify);
+
+ /**
+ * The procedure can't run at the moment. add it back to the queue, giving priority to someone
+ * else.
+ *
+ * @param proc the Procedure to add back to the list
+ */
+ void yield(Procedure proc);
+
+ /** @return true if there are procedures available to process, otherwise false. */
+ boolean hasRunnables();
+
+ /**
+ * Fetch one Procedure from the queue
+ *
+ * @return the Procedure to execute, or null if nothing present.
+ */
+ Procedure poll();
+
+ /**
+ * Fetch one Procedure from the queue
+ *
+ * @param timeout how long to wait before giving up, in units of unit
+ * @param unit a TimeUnit determining how to interpret the timeout parameter
+ * @return the Procedure to execute, or null if nothing present.
+ */
+ Procedure poll(long timeout, TimeUnit unit);
+
+ /**
+ * Returns the number of elements in this queue.
+ *
+ * @return the number of elements in this queue.
+ */
+ int size();
+
+ /**
+ * Clear current state of scheduler such that it is equivalent to newly created scheduler. Used
+ * for testing failure and recovery.
+ */
+ void clear();
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/scheduler/SimpleProcedureScheduler.java b/procedure/src/main/java/org/apache/iotdb/procedure/scheduler/SimpleProcedureScheduler.java
new file mode 100644
index 000000000000..7024ddc25f8e
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/scheduler/SimpleProcedureScheduler.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.scheduler;
+
+import org.apache.iotdb.procedure.Procedure;
+
+import java.util.ArrayDeque;
+
+/** Simple scheduler for procedures */
+public class SimpleProcedureScheduler extends AbstractProcedureScheduler {
+ private final ArrayDeque runnables = new ArrayDeque<>();
+ private final ArrayDeque waitings = new ArrayDeque<>();
+
+ @Override
+ protected void enqueue(final Procedure procedure, final boolean addFront) {
+ if (addFront) {
+ runnables.addFirst(procedure);
+ } else {
+ runnables.addLast(procedure);
+ }
+ }
+
+ @Override
+ protected Procedure dequeue() {
+ return runnables.poll();
+ }
+
+ @Override
+ public void clear() {
+ schedLock();
+ try {
+ runnables.clear();
+ } finally {
+ schedUnlock();
+ }
+ }
+
+ @Override
+ public void yield(final Procedure proc) {
+ addBack(proc);
+ }
+
+ @Override
+ public boolean queueHasRunnables() {
+ return runnables.size() > 0;
+ }
+
+ @Override
+ public int queueSize() {
+ return runnables.size();
+ }
+
+ public void addWaiting(Procedure proc) {
+ waitings.add(proc);
+ }
+
+ public void releaseWaiting() {
+ runnables.addAll(waitings);
+ waitings.clear();
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureNode.java b/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureNode.java
new file mode 100644
index 000000000000..7fbdabd611c1
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureNode.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.service;
+
+import org.apache.iotdb.commons.exception.ShutdownException;
+import org.apache.iotdb.commons.exception.StartupException;
+import org.apache.iotdb.commons.service.JMXService;
+import org.apache.iotdb.commons.service.RegisterManager;
+import org.apache.iotdb.procedure.conf.ProcedureNodeConstant;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class ProcedureNode implements ProcedureNodeMBean {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ProcedureNode.class);
+ private final String mbeanName =
+ String.format(
+ "%s:%s=%s",
+ ProcedureNodeConstant.PROCEDURENODE_PACKAGE,
+ ProcedureNodeConstant.JMX_TYPE,
+ "ProcedureNode");
+ private final RegisterManager registerManager = new RegisterManager();
+
+ private ProcedureNode() {}
+
+ public static ProcedureNode getInstance() {
+ return ProcedureNodeHolder.INSTANCE;
+ }
+
+ public static void main(String[] args) {
+ new ProcedureServerCommandLine().doMain(args);
+ }
+
+ public void setUp() throws StartupException, IOException {
+ LOGGER.info("Setting up {}...", ProcedureNodeConstant.GLOBAL_NAME);
+ registerManager.register(JMXService.getInstance());
+ JMXService.registerMBean(getInstance(), mbeanName);
+ ProcedureServer.getInstance().initSyncedServiceImpl(new ProcedureServerProcessor());
+ registerManager.register(ProcedureServer.getInstance());
+ LOGGER.info("Init rpc server success");
+ }
+
+ public void active() {
+ try {
+ setUp();
+ } catch (StartupException | IOException e) {
+ LOGGER.error("Meet error while starting up.", e);
+ ProcedureServer.getInstance().stop();
+ LOGGER.warn("Procedure executor has stopped.");
+ deactivate();
+ return;
+ }
+ LOGGER.info("{} has started.", ProcedureNodeConstant.GLOBAL_NAME);
+ }
+
+ public void deactivate() {
+ LOGGER.info("Deactivating {}...", ProcedureNodeConstant.GLOBAL_NAME);
+ registerManager.deregisterAll();
+ JMXService.deregisterMBean(mbeanName);
+ LOGGER.info("{} is deactivated.", ProcedureNodeConstant.GLOBAL_NAME);
+ }
+
+ public void shutdown() throws ShutdownException {
+ LOGGER.info("Deactivating {}...", ProcedureNodeConstant.GLOBAL_NAME);
+ registerManager.shutdownAll();
+ JMXService.deregisterMBean(mbeanName);
+ LOGGER.info("{} is deactivated.", ProcedureNodeConstant.GLOBAL_NAME);
+ }
+
+ public void stop() {
+ deactivate();
+ }
+
+ private static class ProcedureNodeHolder {
+ private static final ProcedureNode INSTANCE = new ProcedureNode();
+
+ private ProcedureNodeHolder() {}
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureNodeMBean.java b/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureNodeMBean.java
new file mode 100644
index 000000000000..05e734eaaa77
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureNodeMBean.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.service;
+
+public interface ProcedureNodeMBean {}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureServer.java b/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureServer.java
new file mode 100644
index 000000000000..2da216e66e49
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureServer.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.service;
+
+import org.apache.iotdb.commons.concurrent.ThreadName;
+import org.apache.iotdb.commons.exception.runtime.RPCServiceException;
+import org.apache.iotdb.commons.service.ServiceType;
+import org.apache.iotdb.commons.service.ThriftService;
+import org.apache.iotdb.commons.service.ThriftServiceThread;
+import org.apache.iotdb.procedure.ProcedureExecutor;
+import org.apache.iotdb.procedure.conf.ProcedureNodeConfig;
+import org.apache.iotdb.procedure.conf.ProcedureNodeConfigDescriptor;
+import org.apache.iotdb.procedure.env.ClusterProcedureEnvironment;
+import org.apache.iotdb.procedure.scheduler.ProcedureScheduler;
+import org.apache.iotdb.procedure.scheduler.SimpleProcedureScheduler;
+import org.apache.iotdb.procedure.store.IProcedureStore;
+import org.apache.iotdb.procedure.store.ProcedureStore;
+import org.apache.iotdb.service.rpc.thrift.ProcedureService;
+
+public class ProcedureServer extends ThriftService implements ProcedureNodeMBean {
+
+ private static final ProcedureNodeConfig conf =
+ ProcedureNodeConfigDescriptor.getInstance().getConf();
+
+ private ProcedureScheduler scheduler = new SimpleProcedureScheduler();
+ private ClusterProcedureEnvironment env = new ClusterProcedureEnvironment();
+ private IProcedureStore store = new ProcedureStore();
+ private ProcedureExecutor executor;
+
+ private ProcedureServerProcessor client;
+
+ public ProcedureServer() {
+ executor = new ProcedureExecutor(env, store, scheduler);
+ client = new ProcedureServerProcessor(executor);
+ }
+
+ public void initExecutor() {
+ executor.init(conf.getWorkerThreadsCoreSize());
+ executor.startWorkers();
+ store.setRunning(true);
+ }
+
+ public void stop() {
+ store.cleanup();
+ store.setRunning(false);
+ executor.stop();
+ }
+
+ public static ProcedureServer getInstance() {
+ return ProcedureServerHolder.INSTANCE;
+ }
+
+ @Override
+ public ServiceType getID() {
+ return ServiceType.PROCEDURE_SERVICE;
+ }
+
+ @Override
+ public ThriftService getImplementation() {
+ return ProcedureServer.getInstance();
+ }
+
+ @Override
+ public void initTProcessor()
+ throws ClassNotFoundException, IllegalAccessException, InstantiationException {
+ this.processor = new ProcedureService.Processor<>(client);
+ super.initSyncedServiceImpl(this.client);
+ }
+
+ @Override
+ public void initThriftServiceThread()
+ throws IllegalAccessException, InstantiationException, ClassNotFoundException {
+ try {
+ thriftServiceThread =
+ new ThriftServiceThread(
+ processor,
+ getID().getName(),
+ ThreadName.PROCEDURE_NODE_CLIENT.getName(),
+ getBindIP(),
+ getBindPort(),
+ conf.getRpcMaxConcurrentClientNum(),
+ conf.getThriftServerAwaitTimeForStopService(),
+ new ProcedureServiceHanlder(client),
+ conf.isRpcThriftCompressionEnabled());
+ } catch (RPCServiceException e) {
+ throw new IllegalAccessException(e.getMessage());
+ }
+ thriftServiceThread.setName(ThreadName.PROCEDURE_NODE_SERVER.getName());
+ }
+
+ @Override
+ public String getBindIP() {
+ return conf.getRpcAddress();
+ }
+
+ @Override
+ public int getBindPort() {
+ return conf.getRpcPort();
+ }
+
+ private static class ProcedureServerHolder {
+ public static final ProcedureServer INSTANCE = new ProcedureServer();
+
+ private ProcedureServerHolder() {}
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureServerCommandLine.java b/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureServerCommandLine.java
new file mode 100644
index 000000000000..4a068bb59225
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureServerCommandLine.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.service;
+
+import org.apache.iotdb.commons.ServerCommandLine;
+import org.apache.iotdb.commons.exception.StartupException;
+import org.apache.iotdb.commons.service.StartupChecks;
+import org.apache.iotdb.procedure.conf.ProcedureNodeConfigDescriptor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ProcedureServerCommandLine extends ServerCommandLine {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ProcedureServerCommandLine.class);
+
+ @Override
+ protected String getUsage() {
+ return null;
+ }
+
+ @Override
+ protected int run(String[] args) {
+ try {
+ StartupChecks checks = new StartupChecks().withDefaultTest();
+ checks.verify();
+ ProcedureNodeConfigDescriptor.getInstance().checkConfig();
+ ProcedureNode procedureNode = ProcedureNode.getInstance();
+ procedureNode.active();
+ } catch (StartupException e) {
+ LOGGER.info("Meet error when doing start checking", e);
+ return -1;
+ }
+ return 0;
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureServerProcessor.java b/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureServerProcessor.java
new file mode 100644
index 000000000000..88bc432d3bf5
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureServerProcessor.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.service;
+
+import org.apache.iotdb.procedure.Procedure;
+import org.apache.iotdb.procedure.ProcedureExecutor;
+import org.apache.iotdb.service.rpc.thrift.ProcedureService;
+import org.apache.iotdb.service.rpc.thrift.SubmitProcedureReq;
+
+import org.apache.thrift.TException;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class ProcedureServerProcessor implements ProcedureService.Iface {
+ private ProcedureExecutor executor;
+
+ private static final String PROC_NOT_FOUND = "Proc %d not found";
+ private static final String PROC_ABORT = "Proc %d is aborted";
+ private static final String PROC_ABORT_FAILED = "Abort %d failed";
+ private static final String PROC_SUBMIT = "Proc is submitted";
+
+ public ProcedureServerProcessor() {}
+
+ public ProcedureServerProcessor(ProcedureExecutor executor) {
+ this.executor = executor;
+ }
+
+ @Override
+ public String query(long procId) {
+ Procedure procedure = executor.getResultOrProcedure(procId);
+ if (null != procedure) {
+ return procedure.toString();
+ } else {
+ return String.format(PROC_NOT_FOUND, procId);
+ }
+ }
+
+ @Override
+ public String abort(long procId) {
+ return executor.abort(procId)
+ ? String.format(PROC_ABORT, procId)
+ : String.format(PROC_ABORT_FAILED, procId);
+ }
+
+ @Override
+ public long submit(SubmitProcedureReq req) throws TException {
+ byte[] procedureBody = req.getProcedureBody();
+ long procId;
+ ByteBuffer byteBuffer = ByteBuffer.wrap(procedureBody);
+ Procedure procedure = Procedure.newInstance(byteBuffer);
+ try {
+ procedure.deserialize(byteBuffer);
+ procId = executor.submitProcedure(procedure);
+ } catch (IOException e) {
+ return -1;
+ }
+ return procId;
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureServiceHanlder.java b/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureServiceHanlder.java
new file mode 100644
index 000000000000..6190cb4035f0
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/service/ProcedureServiceHanlder.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.service;
+
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.server.ServerContext;
+import org.apache.thrift.server.TServerEventHandler;
+import org.apache.thrift.transport.TTransport;
+
+public class ProcedureServiceHanlder implements TServerEventHandler {
+ public ProcedureServiceHanlder(ProcedureServerProcessor client) {}
+
+ @Override
+ public void preServe() {}
+
+ @Override
+ public ServerContext createContext(TProtocol tProtocol, TProtocol tProtocol1) {
+ return null;
+ }
+
+ @Override
+ public void deleteContext(
+ ServerContext serverContext, TProtocol tProtocol, TProtocol tProtocol1) {}
+
+ @Override
+ public void processContext(
+ ServerContext serverContext, TTransport tTransport, TTransport tTransport1) {}
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/store/IProcedureStore.java b/procedure/src/main/java/org/apache/iotdb/procedure/store/IProcedureStore.java
new file mode 100644
index 000000000000..94782dac03b3
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/store/IProcedureStore.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.store;
+
+import org.apache.iotdb.procedure.Procedure;
+
+import java.util.List;
+
+public interface IProcedureStore {
+ boolean isRunning();
+
+ void setRunning(boolean running);
+
+ void load(List procedureList);
+
+ void update(Procedure procedure);
+
+ void update(Procedure[] subprocs);
+
+ void delete(long procId);
+
+ void delete(long[] childProcIds);
+
+ void delete(long[] batchIds, int startIndex, int batchCount);
+
+ void cleanup();
+
+ void stop();
+
+ void start();
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/store/ProcedureStore.java b/procedure/src/main/java/org/apache/iotdb/procedure/store/ProcedureStore.java
new file mode 100644
index 000000000000..bf11e7de1ae7
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/store/ProcedureStore.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.store;
+
+import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.procedure.Procedure;
+import org.apache.iotdb.procedure.conf.ProcedureNodeConfigDescriptor;
+import org.apache.iotdb.procedure.conf.ProcedureNodeConstant;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class ProcedureStore implements IProcedureStore {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ProcedureStore.class);
+ private String procedureWalDir =
+ ProcedureNodeConfigDescriptor.getInstance().getConf().getProcedureWalDir();
+ private final ConcurrentHashMap procWALMap = new ConcurrentHashMap<>();
+ private volatile boolean isRunning = false;
+
+ public ProcedureStore() {
+ try {
+ Files.createDirectories(Paths.get(procedureWalDir));
+ } catch (IOException e) {
+ throw new RuntimeException("Create procedure wal directory failed.", e);
+ }
+ }
+
+ @TestOnly
+ public ProcedureStore(String testWALDir) {
+ try {
+ Files.createDirectories(Paths.get(testWALDir));
+ procedureWalDir = testWALDir;
+ } catch (IOException e) {
+ throw new RuntimeException("Create procedure wal directory failed.", e);
+ }
+ }
+
+ public boolean isRunning() {
+ return this.isRunning;
+ }
+
+ public void setRunning(boolean running) {
+ isRunning = running;
+ }
+
+ /**
+ * Load procedure wal files into memory.
+ *
+ * @param procedureList procedureList
+ */
+ public void load(List procedureList) {
+ try {
+ Files.list(Paths.get(procedureWalDir))
+ .filter(
+ path ->
+ path.getFileName()
+ .toString()
+ .endsWith(ProcedureNodeConstant.PROCEDURE_WAL_SUFFIX))
+ .sorted(
+ (p1, p2) ->
+ Long.compareUnsigned(
+ Long.parseLong(p1.getFileName().toString().split("\\.")[0]),
+ Long.parseLong(p2.getFileName().toString().split("\\.")[0])))
+ .forEach(
+ path -> {
+ String fileName = path.getFileName().toString();
+ long procId = Long.parseLong(fileName.split("\\.")[0]);
+ ProcedureWAL procedureWAL =
+ procWALMap.computeIfAbsent(procId, id -> new ProcedureWAL(path));
+ procedureWAL.load(procedureList);
+ });
+ } catch (IOException e) {
+ LOG.error("Load procedure wal failed.", e);
+ }
+ }
+
+ /**
+ * Update procedure, roughly delete and create a new wal file.
+ *
+ * @param procedure procedure
+ */
+ public void update(Procedure procedure) {
+ if (!procedure.needPersistance()) {
+ procWALMap.remove(procedure.getProcId());
+ return;
+ }
+ long procId = procedure.getProcId();
+ Path path = Paths.get(procedureWalDir, procId + ProcedureNodeConstant.PROCEDURE_WAL_SUFFIX);
+ ProcedureWAL procedureWAL = procWALMap.computeIfAbsent(procId, id -> new ProcedureWAL(path));
+ try {
+ procedureWAL.save(procedure);
+ } catch (IOException e) {
+ LOG.error("Update Procedure (pid={}) wal failed", procedure.getProcId());
+ }
+ }
+
+ /**
+ * Batch update
+ *
+ * @param subprocs procedure array
+ */
+ public void update(Procedure[] subprocs) {
+ for (Procedure subproc : subprocs) {
+ update(subproc);
+ }
+ }
+
+ /**
+ * Delete procedure wal file
+ *
+ * @param procId procedure id
+ */
+ public void delete(long procId) {
+ ProcedureWAL procedureWAL = procWALMap.get(procId);
+ if (procedureWAL != null) {
+ procedureWAL.delete();
+ }
+ procWALMap.remove(procId);
+ }
+
+ /**
+ * Batch delete
+ *
+ * @param childProcIds procedure id array
+ */
+ public void delete(long[] childProcIds) {
+ for (long childProcId : childProcIds) {
+ delete(childProcId);
+ }
+ }
+
+ /**
+ * Batch delete by index
+ *
+ * @param batchIds batchIds
+ * @param startIndex start index
+ * @param batchCount delete procedure count
+ */
+ public void delete(long[] batchIds, int startIndex, int batchCount) {
+ for (int i = startIndex; i < batchCount; i++) {
+ delete(batchIds[i]);
+ }
+ }
+
+ /** clean all the wal, used for unit test. */
+ public void cleanup() {
+ try {
+ FileUtils.cleanDirectory(new File(procedureWalDir));
+ } catch (IOException e) {
+ LOG.error("Clean wal directory failed", e);
+ }
+ }
+
+ public void stop() {
+ isRunning = false;
+ }
+
+ @Override
+ public void start() {
+ if (!isRunning) {
+ isRunning = true;
+ }
+ }
+
+ public static class ProcedureStoreHolder {
+ private static final ProcedureStore INSTANCE = new ProcedureStore();
+ }
+}
diff --git a/procedure/src/main/java/org/apache/iotdb/procedure/store/ProcedureWAL.java b/procedure/src/main/java/org/apache/iotdb/procedure/store/ProcedureWAL.java
new file mode 100644
index 000000000000..2be9831008a5
--- /dev/null
+++ b/procedure/src/main/java/org/apache/iotdb/procedure/store/ProcedureWAL.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.store;
+
+import org.apache.iotdb.procedure.Procedure;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+
+public class ProcedureWAL {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ProcedureWAL.class);
+
+ private static final String TMP_SUFFIX = ".tmp";
+ private static final int PROCEDURE_WAL_BUFFER_SIZE = 8 * 1024 * 1024;
+ private Path walFilePath;
+
+ public ProcedureWAL(Path walFilePath) {
+ this.walFilePath = walFilePath;
+ }
+
+ /**
+ * Create a wal file
+ *
+ * @throws IOException ioe
+ */
+ public void save(Procedure procedure) throws IOException {
+ File walTmp = new File(walFilePath + TMP_SUFFIX);
+ Path walTmpPath = walTmp.toPath();
+ Files.deleteIfExists(walTmpPath);
+ Files.createFile(walTmpPath);
+ try (FileOutputStream fos = new FileOutputStream(walTmp);
+ FileChannel channel = fos.getChannel()) {
+ ByteBuffer byteBuffer = ByteBuffer.allocate(PROCEDURE_WAL_BUFFER_SIZE);
+ procedure.serialize(byteBuffer);
+ byteBuffer.flip();
+ channel.write(byteBuffer);
+ }
+ Files.deleteIfExists(walFilePath);
+ Files.move(walTmpPath, walFilePath);
+ }
+
+ /**
+ * Load wal files into memory
+ *
+ * @param procedureList procedure list
+ */
+ public void load(List procedureList) {
+ Procedure procedure = null;
+ try (FileInputStream fis = new FileInputStream(walFilePath.toFile());
+ FileChannel channel = fis.getChannel()) {
+ ByteBuffer byteBuffer = ByteBuffer.allocate(PROCEDURE_WAL_BUFFER_SIZE);
+ if (channel.read(byteBuffer) > 0) {
+ byteBuffer.flip();
+ procedure = Procedure.newInstance(byteBuffer);
+ if (procedure != null) {
+ procedure.deserialize(byteBuffer);
+ } else {
+ throw new IOException("WAL File is corrupted.");
+ }
+ byteBuffer.clear();
+ }
+ procedureList.add(procedure);
+ } catch (IOException e) {
+ LOG.error("Load {} failed, it will be deleted.", walFilePath, e);
+ walFilePath.toFile().delete();
+ }
+ }
+
+ public void delete() {
+ try {
+ Files.deleteIfExists(Paths.get(walFilePath + TMP_SUFFIX));
+ Files.deleteIfExists(walFilePath);
+ } catch (IOException e) {
+ LOG.error("Delete procedure wal failed.");
+ }
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/NoopProcedureStore.java b/procedure/src/test/java/org/apache/iotdb/procedure/NoopProcedureStore.java
new file mode 100644
index 000000000000..35875af12efe
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/NoopProcedureStore.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+import org.apache.iotdb.procedure.store.IProcedureStore;
+
+import java.util.List;
+
+public class NoopProcedureStore implements IProcedureStore {
+
+ private volatile boolean running = false;
+
+ @Override
+ public boolean isRunning() {
+ return running;
+ }
+
+ @Override
+ public void setRunning(boolean running) {
+ this.running = running;
+ }
+
+ @Override
+ public void load(List procedureList) {}
+
+ @Override
+ public void update(Procedure procedure) {}
+
+ @Override
+ public void update(Procedure[] subprocs) {}
+
+ @Override
+ public void delete(long procId) {}
+
+ @Override
+ public void delete(long[] childProcIds) {}
+
+ @Override
+ public void delete(long[] batchIds, int startIndex, int batchCount) {}
+
+ @Override
+ public void cleanup() {}
+
+ @Override
+ public void stop() {
+ running = false;
+ }
+
+ @Override
+ public void start() {
+ running = true;
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/TestLockRegime.java b/procedure/src/test/java/org/apache/iotdb/procedure/TestLockRegime.java
new file mode 100644
index 000000000000..82e19013d0e8
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/TestLockRegime.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+import org.apache.iotdb.procedure.entity.SimpleLockProcedure;
+import org.apache.iotdb.procedure.util.ProcedureTestUtil;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TestLockRegime extends TestProcedureBase {
+
+ @Test
+ public void testAcquireLock() {
+ List procIdList = new ArrayList<>();
+ for (int i = 1; i < 5; i++) {
+ String procName = "[proc" + i + "]";
+ SimpleLockProcedure stmProcedure = new SimpleLockProcedure(procName);
+ long procId = this.procExecutor.submitProcedure(stmProcedure);
+ procIdList.add(procId);
+ }
+ ProcedureTestUtil.waitForProcedure(
+ this.procExecutor, procIdList.stream().mapToLong(Long::longValue).toArray());
+ Assert.assertEquals(env.lockAcquireSeq.toString(), env.executeSeq.toString());
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/TestProcEnv.java b/procedure/src/test/java/org/apache/iotdb/procedure/TestProcEnv.java
new file mode 100644
index 000000000000..3bf0bd6c0f80
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/TestProcEnv.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+import org.apache.iotdb.procedure.scheduler.ProcedureScheduler;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class TestProcEnv {
+ public ProcedureScheduler scheduler;
+
+ private final ReentrantLock envLock = new ReentrantLock();
+
+ private final AtomicInteger acc = new AtomicInteger(0);
+
+ public final AtomicInteger successCount = new AtomicInteger(0);
+
+ public final AtomicInteger rolledBackCount = new AtomicInteger(0);
+
+ public StringBuilder executeSeq = new StringBuilder();
+
+ public StringBuilder lockAcquireSeq = new StringBuilder();
+
+ public AtomicInteger getAcc() {
+ return acc;
+ }
+
+ public ProcedureScheduler getScheduler() {
+ return scheduler;
+ }
+
+ public ReentrantLock getEnvLock() {
+ return envLock;
+ }
+
+ public void setScheduler(ProcedureScheduler scheduler) {
+ this.scheduler = scheduler;
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/TestProcedureBase.java b/procedure/src/test/java/org/apache/iotdb/procedure/TestProcedureBase.java
new file mode 100644
index 000000000000..b165c2564b5a
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/TestProcedureBase.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+import org.apache.iotdb.procedure.store.IProcedureStore;
+
+import org.junit.After;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestProcedureBase {
+
+ public static final Logger LOG = LoggerFactory.getLogger(TestProcedureBase.class);
+
+ protected TestProcEnv env;
+ protected IProcedureStore procStore;
+ protected ProcedureExecutor procExecutor;
+
+ @Before
+ public void setUp() {
+ initExecutor();
+ this.procStore.start();
+ this.procExecutor.startWorkers();
+ }
+
+ @After
+ public void tearDown() {
+ this.procStore.stop();
+ this.procStore.cleanup();
+ this.procExecutor.stop();
+ this.procExecutor.join();
+ }
+
+ protected void initExecutor() {
+ this.env = new TestProcEnv();
+ this.procStore = new NoopProcedureStore();
+ this.procExecutor = new ProcedureExecutor<>(env, procStore);
+ this.env.setScheduler(this.procExecutor.getScheduler());
+ this.procExecutor.init(4);
+ }
+
+ public TestProcEnv getEnv() {
+ return env;
+ }
+
+ public void setEnv(TestProcEnv env) {
+ this.env = env;
+ }
+
+ public IProcedureStore getProcStore() {
+ return procStore;
+ }
+
+ public void setProcStore(IProcedureStore procStore) {
+ this.procStore = procStore;
+ }
+
+ public ProcedureExecutor getProcExecutor() {
+ return procExecutor;
+ }
+
+ public void setProcExecutor(ProcedureExecutor procExecutor) {
+ this.procExecutor = procExecutor;
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/TestProcedureExecutor.java b/procedure/src/test/java/org/apache/iotdb/procedure/TestProcedureExecutor.java
new file mode 100644
index 000000000000..1f750115112c
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/TestProcedureExecutor.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+import org.apache.iotdb.procedure.entity.IncProcedure;
+import org.apache.iotdb.procedure.entity.NoopProcedure;
+import org.apache.iotdb.procedure.entity.StuckProcedure;
+import org.apache.iotdb.procedure.util.ProcedureTestUtil;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class TestProcedureExecutor extends TestProcedureBase {
+
+ @Override
+ protected void initExecutor() {
+ this.env = new TestProcEnv();
+ this.procStore = new NoopProcedureStore();
+ this.procExecutor = new ProcedureExecutor<>(env, procStore);
+ this.env.setScheduler(this.procExecutor.getScheduler());
+ this.procExecutor.init(2);
+ }
+
+ @Test
+ public void testSubmitProcedure() {
+ IncProcedure incProcedure = new IncProcedure();
+ long procId = this.procExecutor.submitProcedure(incProcedure);
+ ProcedureTestUtil.waitForProcedure(this.procExecutor, procId);
+ TestProcEnv env = this.getEnv();
+ AtomicInteger acc = env.getAcc();
+ Assert.assertEquals(acc.get(), 1);
+ }
+
+ @Test
+ public void testWorkerThreadStuck() throws InterruptedException {
+ procExecutor.setKeepAliveTime(10, TimeUnit.SECONDS);
+ Semaphore latch1 = new Semaphore(2);
+ latch1.acquire(2);
+ StuckProcedure busyProc1 = new StuckProcedure(latch1);
+
+ Semaphore latch2 = new Semaphore(2);
+ latch2.acquire(2);
+ StuckProcedure busyProc2 = new StuckProcedure(latch2);
+
+ long busyProcId1 = procExecutor.submitProcedure(busyProc1);
+ long busyProcId2 = procExecutor.submitProcedure(busyProc2);
+ long otherProcId = procExecutor.submitProcedure(new NoopProcedure());
+
+ // wait until a new worker is being created
+ int threads1 = waitThreadCount(3);
+ LOG.info("new threads got created: " + (threads1 - 2));
+ Assert.assertEquals(3, threads1);
+
+ ProcedureTestUtil.waitForProcedure(procExecutor, otherProcId);
+ Assert.assertEquals(true, procExecutor.isFinished(otherProcId));
+ Assert.assertEquals(true, procExecutor.isRunning());
+ Assert.assertEquals(false, procExecutor.isFinished(busyProcId1));
+ Assert.assertEquals(false, procExecutor.isFinished(busyProcId2));
+
+ // terminate the busy procedures
+ latch1.release();
+ latch2.release();
+
+ LOG.info("set keep alive and wait threads being removed");
+ procExecutor.setKeepAliveTime(500L, TimeUnit.MILLISECONDS);
+ int threads2 = waitThreadCount(2);
+ LOG.info("threads got removed: " + (threads1 - threads2));
+ Assert.assertEquals(2, threads2);
+
+ // terminate the busy procedures
+ latch1.release();
+ latch2.release();
+
+ // wait for all procs to complete
+ ProcedureTestUtil.waitForProcedure(procExecutor, busyProcId1);
+ ProcedureTestUtil.waitForProcedure(procExecutor, busyProcId2);
+ }
+
+ private int waitThreadCount(final int expectedThreads) {
+ long startTime = System.currentTimeMillis();
+ while (procExecutor.isRunning()
+ && TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - startTime) <= 30) {
+ if (procExecutor.getWorkerThreadCount() == expectedThreads) {
+ break;
+ }
+ ProcedureTestUtil.sleepWithoutInterrupt(250);
+ }
+ return procExecutor.getWorkerThreadCount();
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/TestSTMProcedure.java b/procedure/src/test/java/org/apache/iotdb/procedure/TestSTMProcedure.java
new file mode 100644
index 000000000000..286dcbb165c3
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/TestSTMProcedure.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure;
+
+import org.apache.iotdb.procedure.entity.SimpleSTMProcedure;
+import org.apache.iotdb.procedure.util.ProcedureTestUtil;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class TestSTMProcedure extends TestProcedureBase {
+
+ @Test
+ public void testSubmitProcedure() {
+ SimpleSTMProcedure stmProcedure = new SimpleSTMProcedure();
+ long procId = this.procExecutor.submitProcedure(stmProcedure);
+ ProcedureTestUtil.waitForProcedure(this.procExecutor, procId);
+ TestProcEnv env = this.getEnv();
+ AtomicInteger acc = env.getAcc();
+ Assert.assertEquals(acc.get(), 10);
+ }
+
+ @Test
+ public void testRolledBackProcedure() {
+ SimpleSTMProcedure stmProcedure = new SimpleSTMProcedure();
+ stmProcedure.throwAtIndex = 4;
+ long procId = this.procExecutor.submitProcedure(stmProcedure);
+ ProcedureTestUtil.waitForProcedure(this.procExecutor, procId);
+ TestProcEnv env = this.getEnv();
+ AtomicInteger acc = env.getAcc();
+ int success = env.successCount.get();
+ int rolledback = env.rolledBackCount.get();
+ System.out.println(acc.get());
+ System.out.println(success);
+ System.out.println(rolledback);
+ Assert.assertEquals(1 + success - rolledback, acc.get());
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/entity/IncProcedure.java b/procedure/src/test/java/org/apache/iotdb/procedure/entity/IncProcedure.java
new file mode 100644
index 000000000000..8e464c2e0cc9
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/entity/IncProcedure.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.entity;
+
+import org.apache.iotdb.procedure.Procedure;
+import org.apache.iotdb.procedure.TestProcEnv;
+import org.apache.iotdb.procedure.exception.ProcedureSuspendedException;
+import org.apache.iotdb.procedure.exception.ProcedureYieldException;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class IncProcedure extends Procedure {
+
+ public boolean throwEx = false;
+
+ @Override
+ protected Procedure[] execute(TestProcEnv testProcEnv)
+ throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
+ AtomicInteger acc = testProcEnv.getAcc();
+ if (throwEx) {
+ throw new RuntimeException("throw a EXCEPTION");
+ }
+ acc.getAndIncrement();
+ testProcEnv.successCount.getAndIncrement();
+ return null;
+ }
+
+ @Override
+ protected void rollback(TestProcEnv testProcEnv) throws IOException, InterruptedException {
+ AtomicInteger acc = testProcEnv.getAcc();
+ acc.getAndDecrement();
+ testProcEnv.rolledBackCount.getAndIncrement();
+ }
+
+ @Override
+ protected boolean abort(TestProcEnv testProcEnv) {
+ return true;
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/entity/NoopProcedure.java b/procedure/src/test/java/org/apache/iotdb/procedure/entity/NoopProcedure.java
new file mode 100644
index 000000000000..7afe81860b9c
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/entity/NoopProcedure.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.entity;
+
+import org.apache.iotdb.procedure.Procedure;
+import org.apache.iotdb.procedure.TestProcEnv;
+import org.apache.iotdb.procedure.exception.ProcedureSuspendedException;
+import org.apache.iotdb.procedure.exception.ProcedureYieldException;
+
+import java.io.IOException;
+
+public class NoopProcedure extends Procedure {
+
+ @Override
+ protected Procedure[] execute(TestProcEnv testProcEnv)
+ throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
+ return new Procedure[0];
+ }
+
+ @Override
+ protected void rollback(TestProcEnv testProcEnv) throws IOException, InterruptedException {}
+
+ @Override
+ protected boolean abort(TestProcEnv testProcEnv) {
+ return false;
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/entity/SimpleLockProcedure.java b/procedure/src/test/java/org/apache/iotdb/procedure/entity/SimpleLockProcedure.java
new file mode 100644
index 000000000000..6b2c9672925b
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/entity/SimpleLockProcedure.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.entity;
+
+import org.apache.iotdb.procedure.Procedure;
+import org.apache.iotdb.procedure.ProcedureLockState;
+import org.apache.iotdb.procedure.TestProcEnv;
+import org.apache.iotdb.procedure.exception.ProcedureSuspendedException;
+import org.apache.iotdb.procedure.exception.ProcedureYieldException;
+import org.apache.iotdb.procedure.scheduler.SimpleProcedureScheduler;
+
+import java.io.IOException;
+
+public class SimpleLockProcedure extends Procedure {
+
+ private String procName;
+
+ public SimpleLockProcedure(String procName) {
+ this.procName = procName;
+ }
+
+ @Override
+ protected Procedure[] execute(TestProcEnv testProcEnv)
+ throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
+ testProcEnv.executeSeq.append(procName);
+ return null;
+ }
+
+ @Override
+ protected void rollback(TestProcEnv testProcEnv) throws IOException, InterruptedException {}
+
+ @Override
+ protected boolean abort(TestProcEnv testProcEnv) {
+ return false;
+ }
+
+ @Override
+ protected ProcedureLockState acquireLock(TestProcEnv testProcEnv) {
+ if (testProcEnv.getEnvLock().tryLock()) {
+ testProcEnv.lockAcquireSeq.append(procName);
+ System.out.println(procName + " acquired lock.");
+
+ return ProcedureLockState.LOCK_ACQUIRED;
+ }
+ SimpleProcedureScheduler scheduler = (SimpleProcedureScheduler) testProcEnv.getScheduler();
+ scheduler.addWaiting(this);
+ System.out.println(procName + " wait for lock.");
+ return ProcedureLockState.LOCK_EVENT_WAIT;
+ }
+
+ @Override
+ protected void releaseLock(TestProcEnv testProcEnv) {
+ System.out.println(procName + " release lock.");
+ testProcEnv.getEnvLock().unlock();
+ SimpleProcedureScheduler scheduler = (SimpleProcedureScheduler) testProcEnv.getScheduler();
+ scheduler.releaseWaiting();
+ }
+
+ @Override
+ protected boolean holdLock(TestProcEnv testProcEnv) {
+ return testProcEnv.getEnvLock().isHeldByCurrentThread();
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/entity/SimpleSTMProcedure.java b/procedure/src/test/java/org/apache/iotdb/procedure/entity/SimpleSTMProcedure.java
new file mode 100644
index 000000000000..1ab2c36c0d31
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/entity/SimpleSTMProcedure.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.entity;
+
+import org.apache.iotdb.procedure.StateMachineProcedure;
+import org.apache.iotdb.procedure.TestProcEnv;
+import org.apache.iotdb.procedure.exception.ProcedureException;
+import org.apache.iotdb.procedure.exception.ProcedureSuspendedException;
+import org.apache.iotdb.procedure.exception.ProcedureYieldException;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class SimpleSTMProcedure
+ extends StateMachineProcedure {
+
+ public int throwAtIndex = -1;
+
+ public enum TestState {
+ STEP_1,
+ STEP_2,
+ STEP_3
+ }
+
+ @Override
+ protected Flow executeFromState(TestProcEnv testProcEnv, TestState testState)
+ throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
+ AtomicInteger acc = testProcEnv.getAcc();
+ try {
+ switch (testState) {
+ case STEP_1:
+ acc.getAndAdd(1);
+ setNextState(TestState.STEP_2);
+ break;
+ case STEP_2:
+ for (int i = 0; i < 10; i++) {
+ IncProcedure child = new IncProcedure();
+ if (i == throwAtIndex) {
+ child.throwEx = true;
+ }
+ addChildProcedure(child);
+ }
+ setNextState(TestState.STEP_3);
+ break;
+ case STEP_3:
+ acc.getAndAdd(-1);
+ return Flow.NO_MORE_STATE;
+ }
+ } catch (Exception e) {
+ if (isRollbackSupported(testState)) {
+ setFailure("proc failed", new ProcedureException(e));
+ }
+ }
+ return Flow.HAS_MORE_STATE;
+ }
+
+ @Override
+ protected boolean isRollbackSupported(TestState testState) {
+ return true;
+ }
+
+ @Override
+ protected void rollbackState(TestProcEnv testProcEnv, TestState testState)
+ throws IOException, InterruptedException {}
+
+ @Override
+ protected TestState getState(int stateId) {
+ return TestState.values()[stateId];
+ }
+
+ @Override
+ protected int getStateId(TestState testState) {
+ return testState.ordinal();
+ }
+
+ @Override
+ protected TestState getInitialState() {
+ return TestState.STEP_1;
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/entity/SleepProcedure.java b/procedure/src/test/java/org/apache/iotdb/procedure/entity/SleepProcedure.java
new file mode 100644
index 000000000000..0fd12ac415a6
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/entity/SleepProcedure.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.entity;
+
+import org.apache.iotdb.procedure.Procedure;
+import org.apache.iotdb.procedure.TestProcEnv;
+import org.apache.iotdb.procedure.exception.ProcedureSuspendedException;
+import org.apache.iotdb.procedure.exception.ProcedureYieldException;
+import org.apache.iotdb.procedure.util.ProcedureTestUtil;
+
+import java.io.IOException;
+
+public class SleepProcedure extends Procedure {
+ @Override
+ protected Procedure[] execute(TestProcEnv testProcEnv)
+ throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
+ System.out.println("Procedure is sleeping.");
+ ProcedureTestUtil.sleepWithoutInterrupt(2000);
+ return null;
+ }
+
+ @Override
+ protected void rollback(TestProcEnv testProcEnv) throws IOException, InterruptedException {}
+
+ @Override
+ protected boolean abort(TestProcEnv testProcEnv) {
+ return false;
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/entity/StuckProcedure.java b/procedure/src/test/java/org/apache/iotdb/procedure/entity/StuckProcedure.java
new file mode 100644
index 000000000000..876568da7ac6
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/entity/StuckProcedure.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.entity;
+
+import org.apache.iotdb.procedure.Procedure;
+import org.apache.iotdb.procedure.TestProcEnv;
+
+import java.io.IOException;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+
+public class StuckProcedure extends Procedure {
+ private final Semaphore latch;
+
+ public StuckProcedure(final Semaphore latch) {
+ this.latch = latch;
+ }
+
+ @Override
+ protected Procedure[] execute(final TestProcEnv env) {
+ try {
+ if (!latch.tryAcquire(1, 30, TimeUnit.SECONDS)) {
+ throw new Exception("waited too long");
+ }
+
+ if (!latch.tryAcquire(1, 30, TimeUnit.SECONDS)) {
+ throw new Exception("waited too long");
+ }
+ } catch (Exception e) {
+ setFailure("StuckProcedure", e);
+ }
+ return null;
+ }
+
+ @Override
+ protected void rollback(TestProcEnv testProcEnv) throws IOException, InterruptedException {}
+
+ @Override
+ protected boolean abort(TestProcEnv testProcEnv) {
+ return false;
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/entity/StuckSTMProcedure.java b/procedure/src/test/java/org/apache/iotdb/procedure/entity/StuckSTMProcedure.java
new file mode 100644
index 000000000000..5d9ac1a50d1a
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/entity/StuckSTMProcedure.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.entity;
+
+import org.apache.iotdb.procedure.StateMachineProcedure;
+import org.apache.iotdb.procedure.TestProcEnv;
+import org.apache.iotdb.procedure.exception.ProcedureException;
+import org.apache.iotdb.procedure.exception.ProcedureSuspendedException;
+import org.apache.iotdb.procedure.exception.ProcedureYieldException;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class StuckSTMProcedure
+ extends StateMachineProcedure {
+ private int childCount = 0;
+
+ public StuckSTMProcedure() {}
+
+ public StuckSTMProcedure(int childCount) {
+ this.childCount = childCount;
+ }
+
+ public enum TestState {
+ STEP_1,
+ STEP_2,
+ STEP_3
+ }
+
+ @Override
+ protected Flow executeFromState(TestProcEnv testProcEnv, TestState testState)
+ throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
+ AtomicInteger acc = testProcEnv.getAcc();
+ try {
+ switch (testState) {
+ case STEP_1:
+ acc.getAndAdd(1);
+ setNextState(TestState.STEP_2);
+ break;
+ case STEP_2:
+ for (int i = 0; i < childCount; i++) {
+ SleepProcedure child = new SleepProcedure();
+ addChildProcedure(child);
+ }
+ setNextState(TestState.STEP_3);
+ break;
+ case STEP_3:
+ acc.getAndAdd(-1);
+ return Flow.NO_MORE_STATE;
+ }
+ } catch (Exception e) {
+ if (isRollbackSupported(testState)) {
+ setFailure("proc failed", new ProcedureException(e));
+ }
+ }
+ return Flow.HAS_MORE_STATE;
+ }
+
+ @Override
+ protected boolean isRollbackSupported(TestState testState) {
+ return true;
+ }
+
+ @Override
+ protected void rollbackState(TestProcEnv testProcEnv, TestState testState)
+ throws IOException, InterruptedException {}
+
+ @Override
+ protected TestState getState(int stateId) {
+ return TestState.values()[stateId];
+ }
+
+ @Override
+ protected int getStateId(TestState testState) {
+ return testState.ordinal();
+ }
+
+ @Override
+ protected TestState getInitialState() {
+ return TestState.STEP_1;
+ }
+
+ @Override
+ public void serialize(ByteBuffer byteBuffer) throws IOException {
+ super.serialize(byteBuffer);
+ byteBuffer.putInt(childCount);
+ }
+
+ @Override
+ public void deserialize(ByteBuffer byteBuffer) throws IOException {
+ super.deserialize(byteBuffer);
+ this.childCount = byteBuffer.getInt();
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/service/TestProcedureService.java b/procedure/src/test/java/org/apache/iotdb/procedure/service/TestProcedureService.java
new file mode 100644
index 000000000000..de10dd623416
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/service/TestProcedureService.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.service;
+
+import org.apache.iotdb.procedure.NoopProcedureStore;
+import org.apache.iotdb.procedure.ProcedureExecutor;
+import org.apache.iotdb.procedure.TestProcEnv;
+import org.apache.iotdb.procedure.entity.IncProcedure;
+import org.apache.iotdb.procedure.scheduler.ProcedureScheduler;
+import org.apache.iotdb.procedure.scheduler.SimpleProcedureScheduler;
+import org.apache.iotdb.procedure.store.IProcedureStore;
+import org.apache.iotdb.procedure.util.ProcedureTestUtil;
+import org.apache.iotdb.service.rpc.thrift.SubmitProcedureReq;
+
+import org.apache.thrift.TException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class TestProcedureService {
+
+ private int bufferSize = 16 * 1024 * 1024;
+
+ ProcedureExecutor executor;
+ ProcedureServerProcessor client;
+ TestProcEnv env;
+ ProcedureScheduler scheduler;
+ IProcedureStore store;
+
+ @Before
+ public void setUp() {
+ env = new TestProcEnv();
+ scheduler = new SimpleProcedureScheduler();
+ store = new NoopProcedureStore();
+ executor = new ProcedureExecutor(env, store, scheduler);
+ client = new ProcedureServerProcessor(executor);
+ executor.init(4);
+ store.start();
+ scheduler.start();
+ executor.startWorkers();
+ }
+
+ @Test
+ public void testSubmitAndQuery() throws IOException, TException {
+ IncProcedure inc = new IncProcedure();
+ ByteBuffer byteBuffer = ByteBuffer.allocate(bufferSize);
+ inc.serialize(byteBuffer);
+ byteBuffer.flip();
+ byte[] bytes = new byte[byteBuffer.limit() - byteBuffer.position()];
+ byteBuffer.get(bytes);
+ SubmitProcedureReq submitProcedureReq = new SubmitProcedureReq();
+ submitProcedureReq.setProcedureBody(bytes);
+ byteBuffer.flip();
+ long procId = client.submit(submitProcedureReq);
+ ProcedureTestUtil.waitForProcedure(executor, procId);
+ String query = client.query(procId);
+ Assert.assertTrue(query.contains("pid=1"));
+ }
+
+ @After
+ public void tearDown() {
+ executor.stop();
+ store.stop();
+ scheduler.stop();
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/store/TestProcedureStore.java b/procedure/src/test/java/org/apache/iotdb/procedure/store/TestProcedureStore.java
new file mode 100644
index 000000000000..1b50e0ed4bb3
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/store/TestProcedureStore.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.store;
+
+import org.apache.iotdb.procedure.Procedure;
+import org.apache.iotdb.procedure.ProcedureExecutor;
+import org.apache.iotdb.procedure.TestProcEnv;
+import org.apache.iotdb.procedure.TestProcedureBase;
+import org.apache.iotdb.procedure.entity.IncProcedure;
+import org.apache.iotdb.procedure.entity.StuckSTMProcedure;
+import org.apache.iotdb.procedure.util.ProcedureTestUtil;
+import org.apache.iotdb.service.rpc.thrift.ProcedureState;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class TestProcedureStore extends TestProcedureBase {
+
+ private static final String TEST_DIR = "./target/testWAL/";
+ private static final int WORK_THREAD = 2;
+
+ @Override
+ protected void initExecutor() {
+ this.env = new TestProcEnv();
+ this.procStore = new ProcedureStore(TEST_DIR);
+ this.procExecutor = new ProcedureExecutor<>(env, procStore);
+ this.env.setScheduler(this.procExecutor.getScheduler());
+ this.procExecutor.init(WORK_THREAD);
+ }
+
+ @Test
+ public void testUpdate() {
+ ProcedureStore procedureStore = new ProcedureStore(TEST_DIR);
+ IncProcedure incProcedure = new IncProcedure();
+ procedureStore.update(incProcedure);
+ List procedureList = new ArrayList<>();
+ procedureStore.load(procedureList);
+ assertProc(
+ incProcedure,
+ procedureList.get(0).getClass(),
+ procedureList.get(0).getProcId(),
+ procedureList.get(0).getState());
+
+ this.procStore.cleanup();
+ }
+
+ @Test
+ public void testChildProcedureLoad() {
+ int childCount = 10;
+ StuckSTMProcedure STMProcedure = new StuckSTMProcedure(childCount);
+ long rootId = procExecutor.submitProcedure(STMProcedure);
+ ProcedureTestUtil.sleepWithoutInterrupt(50);
+ // stop service
+ ProcedureTestUtil.stopService(procExecutor, procExecutor.getScheduler(), procStore);
+ ConcurrentHashMap procedures = procExecutor.getProcedures();
+ ProcedureStore procedureStore = new ProcedureStore(TEST_DIR);
+ List procedureList = new ArrayList<>();
+ procedureStore.load(procedureList);
+ Assert.assertEquals(childCount + 1, procedureList.size());
+ for (int i = 0; i < procedureList.size(); i++) {
+ Procedure procedure = procedureList.get(i);
+ assertProc(
+ procedure,
+ procedures.get(procedure.getProcId()).getClass(),
+ i + 1,
+ procedures.get(procedure.getProcId()).getState());
+ }
+ // restart service
+ initExecutor();
+ this.procStore.start();
+ this.procExecutor.startWorkers();
+
+ ProcedureTestUtil.waitForProcedure(procExecutor, rootId);
+ Assert.assertEquals(
+ procExecutor.getResultOrProcedure(rootId).getState(), ProcedureState.SUCCESS);
+ }
+
+ private void assertProc(Procedure proc, Class clazz, long procId, ProcedureState state) {
+ Assert.assertEquals(clazz, proc.getClass());
+ Assert.assertEquals(procId, proc.getProcId());
+ Assert.assertEquals(state, proc.getState());
+ }
+}
diff --git a/procedure/src/test/java/org/apache/iotdb/procedure/util/ProcedureTestUtil.java b/procedure/src/test/java/org/apache/iotdb/procedure/util/ProcedureTestUtil.java
new file mode 100644
index 000000000000..e23622381857
--- /dev/null
+++ b/procedure/src/test/java/org/apache/iotdb/procedure/util/ProcedureTestUtil.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.procedure.util;
+
+import org.apache.iotdb.procedure.ProcedureExecutor;
+import org.apache.iotdb.procedure.scheduler.ProcedureScheduler;
+import org.apache.iotdb.procedure.store.IProcedureStore;
+
+import java.util.concurrent.TimeUnit;
+
+public class ProcedureTestUtil {
+ public static void waitForProcedure(ProcedureExecutor executor, long... procIds) {
+ for (long procId : procIds) {
+ long startTimeForProcId = System.currentTimeMillis();
+ while (executor.isRunning()
+ && !executor.isFinished(procId)
+ && TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - startTimeForProcId)
+ < 35) {
+ sleepWithoutInterrupt(250);
+ }
+ }
+ }
+
+ public static void sleepWithoutInterrupt(final long timeToSleep) {
+ long currentTime = System.currentTimeMillis();
+ long endTime = timeToSleep + currentTime;
+ boolean interrupted = false;
+ while (currentTime < endTime) {
+ try {
+ Thread.sleep(endTime - currentTime);
+ } catch (InterruptedException e) {
+ interrupted = true;
+ }
+ currentTime = System.currentTimeMillis();
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ public static void stopService(
+ ProcedureExecutor procExecutor, ProcedureScheduler scheduler, IProcedureStore store) {
+ procExecutor.stop();
+ procExecutor.join();
+ scheduler.clear();
+ scheduler.stop();
+ store.stop();
+ }
+}
diff --git a/thrift-procedure/pom.xml b/thrift-procedure/pom.xml
new file mode 100644
index 000000000000..ca3e5a3b7c7d
--- /dev/null
+++ b/thrift-procedure/pom.xml
@@ -0,0 +1,66 @@
+
+
+
+ 4.0.0
+
+ org.apache.iotdb
+ iotdb-parent
+ 0.14.0-SNAPSHOT
+ ../pom.xml
+
+ iotdb-thrift-procedure
+ rpc-thrift-procedure
+
+
+ org.apache.thrift
+ libthrift
+
+
+ org.apache.iotdb
+ iotdb-thrift
+ ${project.version}
+
+
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+ 3.2.0
+
+
+ add-source
+ generate-sources
+
+ add-source
+
+
+
+ ${project.build.directory}/generated-sources/thrift
+
+
+
+
+
+
+
+
diff --git a/thrift-procedure/src/main/thrift/procedure.thrift b/thrift-procedure/src/main/thrift/procedure.thrift
new file mode 100644
index 000000000000..8c91bd8d41a9
--- /dev/null
+++ b/thrift-procedure/src/main/thrift/procedure.thrift
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace java org.apache.iotdb.service.rpc.thrift
+namespace py iotdb.thrift.rpc
+
+enum ProcedureState {
+ INITIALIZING = 1,
+ RUNNABLE = 2,
+ WAITING = 3,
+ WAITING_TIMEOUT = 4,
+ ROLLEDBACK = 5,
+ SUCCESS = 6,
+ FAILED = 7
+}
+
+struct SubmitProcedureReq{
+ 1: optional binary procedureBody
+}
+
+service ProcedureService{
+ string query(i64 procId);
+ string abort(i64 procId);
+ i64 submit(SubmitProcedureReq req);
+
+}
\ No newline at end of file