optimized setup file

optimized reload spider job

SpiderKeeper/app/schedulers/common.py

@@ -47,26 +47,30 @@ def reload_runnable_spider_job_execution():
     :return:
     '''
     running_job_ids = set([job.id for job in scheduler.get_jobs()])
-    app.logger.debug('[running_job_ids] %s' % ','.join(running_job_ids))
+    # app.logger.debug('[running_job_ids] %s' % ','.join(running_job_ids))
     available_job_ids = set()
     # add new job to schedule
     for job_instance in JobInstance.query.filter_by(enabled=0, run_type="periodic").all():
         job_id = "spider_job_%s:%s" % (job_instance.id, int(time.mktime(job_instance.date_modified.timetuple())))
         available_job_ids.add(job_id)
         if job_id not in running_job_ids:
-            scheduler.add_job(run_spider_job,
-                              args=(job_instance.id,),
-                              trigger='cron',
-                              id=job_id,
-                              minute=job_instance.cron_minutes,
-                              hour=job_instance.cron_hour,
-                              day=job_instance.cron_day_of_month,
-                              day_of_week=job_instance.cron_day_of_week,
-                              month=job_instance.cron_month,
-                              second=0,
-                              max_instances=999,
-                              misfire_grace_time=60 * 60,
-                              coalesce=True)
+            try:
+                scheduler.add_job(run_spider_job,
+                                  args=(job_instance.id,),
+                                  trigger='cron',
+                                  id=job_id,
+                                  minute=job_instance.cron_minutes,
+                                  hour=job_instance.cron_hour,
+                                  day=job_instance.cron_day_of_month,
+                                  day_of_week=job_instance.cron_day_of_week,
+                                  month=job_instance.cron_month,
+                                  second=0,
+                                  max_instances=999,
+                                  misfire_grace_time=60 * 60,
+                                  coalesce=True)
+            except Exception as e:
+                app.logger.error(
+                    '[load_spider_job] failed {} {},may be cron expression format error '.format(job_id, str(e)))
             app.logger.info('[load_spider_job][project:%s][spider_name:%s][job_instance_id:%s][job_id:%s]' % (
                 job_instance.project_id, job_instance.spider_name, job_instance.id, job_id))
     # remove invalid jobs

setup.py

@@ -1,9 +1,15 @@
 #!/usr/bin/env python
-
+from os import path
 from setuptools import setup, find_packages
 
 from SpiderKeeper import __version__, __author__
 
+here = path.abspath(path.dirname(__file__))
+with open(path.join(here, 'requirements.txt'), encoding='utf-8') as f:
+    all_reqs = f.read().split('\n')
+
+install_requires = [x.strip() for x in all_reqs if 'git+' not in x]
+
 setup(
     name='SpiderKeeper',
     version=__version__,
@@ -16,27 +22,7 @@
     license='MIT',
     include_package_data=True,
     packages=find_packages(),
-    install_requires=[
-        'aniso8601==1.2.0',
-        'APScheduler==3.3.1',
-        'click==6.7',
-        'Flask==0.12.1',
-        'Flask-BasicAuth==0.2.0',
-        'Flask-RESTful==0.3.5',
-        'flask-restful-swagger==0.19',
-        'Flask-SQLAlchemy==2.2',
-        'itsdangerous==0.24',
-        'Jinja2==2.9.6',
-        'MarkupSafe==1.0',
-        'PyMySQL==0.7.11',
-        'python-dateutil==2.6.0',
-        'pytz==2017.2',
-        'requests==2.13.0',
-        'six==1.10.0',
-        'SQLAlchemy==1.1.9',
-        'tzlocal==1.3',
-        'Werkzeug==0.12.1'
-    ],
+    install_requires=install_requires,
 
     entry_points={
         'console_scripts': {