diff --git a/spec/index.spec.ts b/spec/index.spec.ts
index 63127c9..8591f34 100644
--- a/spec/index.spec.ts
+++ b/spec/index.spec.ts
@@ -328,15 +328,44 @@ describe('axiosRetry(axios, { retries, retryCondition })', () => {
       });
 
       describe('when request is aborted', () => {
-        it('should clearTimeout', (done) => {
-          const client = axios.create({
-            timeout: 100000
+        it('should clear timeout if started', (done) => {
+          const client = axios.create();
+          setupResponses(client, [
+            () => nock('http://example.com').get('/test').reply(429),
+            () => nock('http://example.com').get('/test').reply(429)
+          ]);
+          axiosRetry(client, {
+            retries: 1,
+            retryCondition: () => true,
+            retryDelay: () => 10000
           });
-          setupResponses(client, [() => nock('http://example.com').get('/test').reply(429)]);
+          const abortController = new AbortController();
+          client
+            .get('http://example.com/test', { signal: abortController.signal })
+            .then(
+              () => done.fail(),
+              (error) => {
+                expect(error).toBeInstanceOf(CanceledError);
+                expect(new Date().getTime() - timeStart).toBeLessThan(300);
+                done();
+              }
+            )
+            .catch(done.fail);
+          setTimeout(() => abortController.abort(), 200);
+          const timeStart = new Date().getTime();
+        });
+
+        it('should not start a timeout if not started yet', (done) => {
+          const client = axios.create();
+          setupResponses(client, [
+            () => nock('http://example.com').get('/test').reply(429),
+            () => nock('http://example.com').get('/test').reply(429)
+          ]);
           axiosRetry(client, {
             retries: 1,
             retryCondition: () => true,
-            retryDelay: () => 9999999
+            retryDelay: () => 10000,
+            onRetry: async () => new Promise((resolve) => setTimeout(resolve, 100))
           });
           const abortController = new AbortController();
           client
@@ -345,11 +374,13 @@ describe('axiosRetry(axios, { retries, retryCondition })', () => {
               () => done.fail(),
               (error) => {
                 expect(error).toBeInstanceOf(CanceledError);
+                expect(new Date().getTime() - timeStart).toBeLessThan(300);
                 done();
               }
             )
             .catch(done.fail);
-          abortController.abort();
+          setTimeout(() => abortController.abort(), 50);
+          const timeStart = new Date().getTime();
         });
       });
     });
diff --git a/src/index.ts b/src/index.ts
index f86f5fd..35cec8e 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -252,13 +252,17 @@ async function handleRetry(
   }
   config.transformRequest = [(data) => data];
   await onRetry(currentState.retryCount, error, config);
-  return new Promise((resolve) => {
+  if (config.signal?.aborted) {
+    return Promise.resolve(axiosInstance(config));
+  }
+  return new Promise((resolve, reject) => {
     const timeout = setTimeout(() => resolve(axiosInstance(config)), delay);
     if (config.signal?.addEventListener) {
       config.signal.addEventListener(
         'abort',
         () => {
           clearTimeout(timeout);
+          resolve(axiosInstance(config));
         },
         { once: true }
       );