Add ResetTriggerFromErrorState functionality (#1904)

changelog.md

@@ -2,15 +2,15 @@
 
 [http://www.quartz-scheduler.net](http://www.quartz-scheduler.net)
 
-## Release 3.5.1, xxx xx 2022
+## Release 3.6.0, Jan 29 2023
 
-This release contains mostly bug fixes and refinement of package dependencies/targets.
+This release contains new API to reset errored trigger state in job store, some bug fixes and refinement of package dependencies/targets.
 
 * NEW FEATURES
 
   * Add explicit netcoreapp3.1 and net6.0 targets to MS integration projects (#1879)
   * Use IHostApplicationLifetime instead of IApplicationLifetime in >= netcoreapp3.1 Hosting targets (#1593)
-
+  * Add ResetTriggerFromErrorState functionality (#1904) 
 
 * FIXES
 

src/Directory.Build.props

@@ -26,7 +26,7 @@
     <Nullable>enable</Nullable>
     <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
 
-    <VersionPrefix>3.5.1</VersionPrefix>
+    <VersionPrefix>3.6.0</VersionPrefix>
 
     <CLSCompliant>True</CLSCompliant>
     <ComVisible>False</ComVisible>

src/Quartz.Tests.Unit/QuartzHostedServiceTests.cs

@@ -182,6 +182,11 @@ public Task<TriggerState> GetTriggerState(TriggerKey triggerKey, CancellationTok
                 throw new NotImplementedException();
             }
 
+            public Task ResetTriggerFromErrorState(TriggerKey triggerKey, CancellationToken cancellationToken = default)
+            {
+                throw new NotImplementedException();
+            }
+
             public Task<bool> Interrupt(JobKey jobKey, CancellationToken cancellationToken = default)
             {
                 throw new NotImplementedException();

src/Quartz.Tests.Unit/Simpl/RAMJobStoreTest.cs

@@ -417,7 +417,47 @@ public async Task TestAcquireTriggersInBatch()
                 Assert.AreEqual("job" + i, triggers[i].Key.Name);
             }
         }
-
+
+        [Test]
+        public async Task TestResetErrorTrigger()
+        {
+            var baseFireTimeDate = DateBuilder.EvenMinuteDateAfterNow();
+
+            // create and store a trigger
+            IOperableTrigger trigger1 = new SimpleTriggerImpl(
+                "trigger1",
+                "triggerGroup1",
+                fJobDetail.Name,
+                fJobDetail.Group,
+                baseFireTimeDate.AddMilliseconds(200000),
+                baseFireTimeDate.AddMilliseconds(200000),
+                2,
+                TimeSpan.FromMilliseconds(2000));
+
+            trigger1.ComputeFirstFireTimeUtc(null);
+            await fJobStore.StoreTrigger(trigger1, false);
+
+            var firstFireTime = trigger1.GetNextFireTimeUtc().Value;
+
+            // pretend to fire it
+            var aqTs = await fJobStore.AcquireNextTriggers(firstFireTime.AddMilliseconds(10000), 1, TimeSpan.Zero);
+            Assert.AreEqual(trigger1.Key, aqTs.First().Key);
+
+            var fTs = await fJobStore.TriggersFired(aqTs);
+            var ft = fTs.First();
+
+            // get the trigger into error state
+            await fJobStore.TriggeredJobComplete(ft.TriggerFiredBundle.Trigger, ft.TriggerFiredBundle.JobDetail, SchedulerInstruction.SetTriggerError);
+
+            var state = await fJobStore.GetTriggerState(trigger1.Key);
+            Assert.AreEqual(TriggerState.Error, state);
+
+            // test reset
+            await fJobStore.ResetTriggerFromErrorState(trigger1.Key);
+            state = await fJobStore.GetTriggerState(trigger1.Key);
+            Assert.AreEqual(TriggerState.Normal, state);
+        }
+
         [Test]
         public async Task TestJobDeleteReturnValue()
         {

src/Quartz/Core/QuartzScheduler.cs

@@ -1395,6 +1395,13 @@ public virtual Task<TriggerState> GetTriggerState(
             return resources.JobStore.GetTriggerState(triggerKey, cancellationToken);
         }
 
+        public Task ResetTriggerFromErrorState(TriggerKey triggerKey, CancellationToken cancellationToken = default)
+        {
+            ValidateState();
+
+            return resources.JobStore.ResetTriggerFromErrorState(triggerKey, cancellationToken);
+        }
+
         /// <summary>
         /// Add (register) the given <see cref="ICalendar" /> to the Scheduler.
         /// </summary>
@@ -2464,6 +2471,11 @@ TriggerState IRemotableQuartzScheduler.GetTriggerState(TriggerKey triggerKey)
             return GetTriggerState(triggerKey).ConfigureAwait(false).GetAwaiter().GetResult();
         }
 
+        void IRemotableQuartzScheduler.ResetTriggerFromErrorState(TriggerKey triggerKey)
+        {
+            ResetTriggerFromErrorState(triggerKey).ConfigureAwait(false).GetAwaiter().GetResult();
+        }
+
         void IRemotableQuartzScheduler.AddCalendar(string calName, ICalendar calendar, bool replace, bool updateTriggers)
         {
             AddCalendar(calName, calendar, replace, updateTriggers).ConfigureAwait(false).GetAwaiter().GetResult();

src/Quartz/IScheduler.cs

@@ -684,6 +684,23 @@ Task<TriggerState> GetTriggerState(
             TriggerKey triggerKey,
             CancellationToken cancellationToken = default);
 
+        /// <summary>
+        /// Reset the current state of the identified <see cref="ITrigger" /> from <see cref="TriggerState.Error" />
+        /// to <see cref="TriggerState.Normal" /> or <see cref="TriggerState.Paused" /> as appropriate.
+        /// </summary>
+        /// <remarks>
+        /// <para>
+        /// Only affects triggers that are in <see cref="TriggerState.Error" /> state - if identified trigger is not
+        /// in that state then the result is a no-op.
+        /// </para>
+        /// <para>
+        /// The result will be the trigger returning to the normal, waiting to be fired state, unless the trigger's
+        /// group has been paused, in which case it will go into the <see cref="TriggerState.Paused" /> state.
+        /// </para>
+        /// </remarks>
+        /// <seealso cref="TriggerState"/>
+        Task ResetTriggerFromErrorState(TriggerKey triggerKey, CancellationToken cancellationToken = default);
+
         /// <summary>
         /// Add (register) the given <see cref="ICalendar" /> to the Scheduler.
         /// </summary>

src/Quartz/Impl/AdoJobStore/JobStoreSupport.cs

@@ -1515,6 +1515,38 @@ protected virtual async Task<TriggerState> GetTriggerState(
             }
         }
 
+        public Task ResetTriggerFromErrorState(TriggerKey triggerKey, CancellationToken cancellationToken = default)
+        {
+            return ExecuteInLock(
+                LockTriggerAccess,
+                conn => ResetTriggerFromErrorState(conn, triggerKey, cancellationToken),
+                cancellationToken);
+        }
+
+        private async Task ResetTriggerFromErrorState(
+            ConnectionAndTransactionHolder conn,
+            TriggerKey triggerKey,
+            CancellationToken cancellationToken = default)
+        {
+            try
+            {
+                var newState = StateWaiting;
+
+                if (await Delegate.IsTriggerGroupPaused(conn, triggerKey.Group, cancellationToken).ConfigureAwait(false))
+                {
+                    newState = StatePaused;
+                }
+
+                await Delegate.UpdateTriggerStateFromOtherState(conn, triggerKey, newState, StateError, cancellationToken).ConfigureAwait(false);
+
+                Log.Info($"Trigger {triggerKey} reset from ERROR state to: {newState}");
+            }
+            catch (Exception e)
+            {
+                throw new JobPersistenceException($"Couldn't reset from error state of trigger ({triggerKey}): {e.Message}", e);
+            }
+        }
+
         /// <summary>
         /// Store the given <see cref="ICalendar" />.
         /// </summary>

src/Quartz/Impl/RemoteScheduler.cs

@@ -586,6 +586,16 @@ public virtual Task<TriggerState> GetTriggerState(
             return CallInGuard(x => x.GetTriggerState(triggerKey));
         }
 
+        /// <summary>
+        /// Calls the equivalent method on the 'proxied' <see cref="QuartzScheduler" />.
+        /// </summary>
+        public virtual Task ResetTriggerFromErrorState(
+            TriggerKey triggerKey,
+            CancellationToken cancellationToken = default)
+        {
+            return CallInGuard(x => x.ResetTriggerFromErrorState(triggerKey));
+        }
+
         /// <summary>
         /// Calls the equivalent method on the 'proxied' <see cref="QuartzScheduler" />.
         /// </summary>

src/Quartz/Impl/StdScheduler.cs

@@ -543,6 +543,14 @@ public virtual Task<TriggerState> GetTriggerState(
             return sched.GetTriggerState(triggerKey, cancellationToken);
         }
 
+        /// <summary>
+        /// Calls the equivalent method on the 'proxied' <see cref="QuartzScheduler" />.
+        /// </summary>
+        public Task ResetTriggerFromErrorState(TriggerKey triggerKey, CancellationToken cancellationToken = default)
+        {
+            return sched.ResetTriggerFromErrorState(triggerKey, cancellationToken);
+        }
+
         /// <summary>
         /// Calls the equivalent method on the 'proxied' <see cref="QuartzScheduler" />.
         /// </summary>

src/Quartz/SPI/IJobStore.cs

@@ -444,6 +444,23 @@ Task<TriggerState> GetTriggerState(
             TriggerKey triggerKey, 
             CancellationToken cancellationToken = default);
 
+        /// <summary>
+        /// Reset the current state of the identified <see cref="ITrigger" /> from <see cref="TriggerState.Error" />
+        /// to <see cref="TriggerState.Normal" /> or <see cref="TriggerState.Paused" /> as appropriate.
+        /// </summary>
+        /// <remarks>
+        /// <para>
+        /// Only affects triggers that are in <see cref="TriggerState.Error" /> state - if identified trigger is not
+        /// in that state then the result is a no-op.
+        /// </para>
+        /// <para>
+        /// The result will be the trigger returning to the normal, waiting to be fired state, unless the trigger's
+        /// group has been paused, in which case it will go into the <see cref="TriggerState.Paused" /> state.
+        /// </para>
+        /// </remarks>
+        /// <seealso cref="TriggerState"/>
+        Task ResetTriggerFromErrorState(TriggerKey triggerKey, CancellationToken cancellationToken = default);
+
         /////////////////////////////////////////////////////////////////////////////
         //
         // Trigger State manipulation methods

src/Quartz/Simpl/IRemotableQuartzScheduler.cs

@@ -150,6 +150,8 @@ public interface IRemotableQuartzScheduler
 
         TriggerState GetTriggerState(TriggerKey triggerKey);
 
+        void ResetTriggerFromErrorState(TriggerKey triggerKey);
+
         void AddCalendar(string calName, ICalendar calendar, bool replace, bool updateTriggers);
 
         bool DeleteCalendar(string calName);

src/Quartz/Simpl/RAMJobStore.cs

@@ -757,6 +757,36 @@ public virtual Task<TriggerState> GetTriggerState(
             return Task.FromResult(TriggerState.Normal);
         }
 
+        public Task ResetTriggerFromErrorState(TriggerKey triggerKey, CancellationToken cancellationToken = default)
+        {
+            lock (lockObject)
+            {
+                // does the trigger exist?
+                if (!triggersByKey.TryGetValue(triggerKey, out var tw) || tw.Trigger == null)
+                {
+                    return Task.CompletedTask;
+                }
+
+                // is the trigger in error state?
+                if (tw.state != InternalTriggerState.Error)
+                {
+                    return Task.CompletedTask;
+                }
+
+                if (pausedTriggerGroups.Contains(triggerKey.Group))
+                {
+                    tw.state = InternalTriggerState.Paused;
+                }
+                else
+                {
+                    tw.state = InternalTriggerState.Waiting;
+                    timeTriggers.Add(tw);
+                }
+            }
+
+            return Task.CompletedTask;
+        }
+
         /// <summary>
         /// Store the given <see cref="ICalendar" />.
         /// </summary>