Merge pull request #6112 from wRAR/test-shutdown-forced

Make shutdown tests more robust.
scrapy · Oct 30, 2023 · 1d81585 · 1d81585
2 parents 9b06f6b + 6b0c18e
commit 1d81585
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 4 deletions.
diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py
@@ -14,7 +14,7 @@ async def parse(self, response):
         from twisted.internet import reactor
 
         d = Deferred()
-        reactor.callLater(3, d.callback, None)
+        reactor.callLater(int(self.sleep), d.callback, None)
         await maybe_deferred_to_future(d)
 
 

diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
@@ -1,3 +1,4 @@
+import os
 import sys
 from io import BytesIO
 from pathlib import Path
@@ -147,8 +148,10 @@ def test_fetch(self):
             "scrapy.cmdline",
             "shell",
         )
+        env = os.environ.copy()
+        env["SCRAPY_PYTHON_SHELL"] = "python"
         logfile = BytesIO()
-        p = PopenSpawn(args, timeout=5)
+        p = PopenSpawn(args, env=env, timeout=5)
         p.logfile_read = logfile
         p.expect_exact("Available Scrapy objects")
         with MockServer() as mockserver:

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
@@ -522,7 +522,7 @@ def test_args_change_settings(self):
 
     def test_shutdown_graceful(self):
         sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
-        args = self.get_script_args("sleeping.py")
+        args = self.get_script_args("sleeping.py", "-a", "sleep=3")
         p = PopenSpawn(args, timeout=5)
         p.expect_exact("Spider opened")
         p.expect_exact("Crawled (200)")
@@ -531,14 +531,21 @@ def test_shutdown_graceful(self):
         p.expect_exact("Spider closed (shutdown)")
         p.wait()
 
+    @defer.inlineCallbacks
     def test_shutdown_forced(self):
+        from twisted.internet import reactor
+
         sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
-        args = self.get_script_args("sleeping.py")
+        args = self.get_script_args("sleeping.py", "-a", "sleep=10")
         p = PopenSpawn(args, timeout=5)
         p.expect_exact("Spider opened")
         p.expect_exact("Crawled (200)")
         p.kill(sig)
         p.expect_exact("shutting down gracefully")
+        # sending the second signal too fast often causes problems
+        d = defer.Deferred()
+        reactor.callLater(0.1, d.callback, None)
+        yield d
         p.kill(sig)
         p.expect_exact("forcing unclean shutdown")
         p.wait()