celery SoftTimeLimitExceeded support

tasks/user.py

@@ -4,6 +4,8 @@
     get_fans_or_followers_ids,
     get_profile, get_user_profile
 )
+from logger import crawler
+from celery.exceptions import SoftTimeLimitExceeded
 
 
 @app.task(ignore_result=True)
@@ -31,16 +33,24 @@ def crawl_person_infos(uid):
     if not uid:
         return
 
-    user, is_crawled = get_profile(uid)
-    # If it's enterprise user, just skip it
-    if user and user.verify_type == 2:
-        SeedidsOper.set_seed_other_crawled(uid)
-        return
+    try:
+        user, is_crawled = get_profile(uid)
+        # If it's enterprise user, just skip it
+        if user and user.verify_type == 2:
+            SeedidsOper.set_seed_other_crawled(uid)
+            return
 
-    # Crawl fans and followers
-    if not is_crawled:
-        app.send_task('tasks.user.crawl_follower_fans', args=(uid,), queue='fans_followers',
-                      routing_key='for_fans_followers')
+        # Crawl fans and followers
+        if not is_crawled:
+            app.send_task('tasks.user.crawl_follower_fans', args=(uid,), queue='fans_followers',
+                          routing_key='for_fans_followers')
+
+    # By adding '--soft-time-limit secs' when you start celery, this will resend task to broker
+    # e.g. celery -A tasks.workers -Q user_crawler worker -l info -c 1 --soft-time-limit 10
+    except SoftTimeLimitExceeded:
+        crawler.error("Exception SoftTimeLimitExceeded    uid={uid}".format(uid=uid))
+        app.send_task('tasks.user.crawl_person_infos', args=(uid, ), queue='user_crawler',
+                      routing_key='for_user_info')
 
 
 @app.task(ignore_result=True)