Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

123 lines (98 sloc) 4.246 kb
import traceback
import uuid
from cStringIO import StringIO
from twisted.python import log
from scrapy.utils.txweb import JsonResource
from .utils import get_spider_list
class WsResource(JsonResource):
def __init__(self, root):
JsonResource.__init__(self)
self.root = root
def render(self, txrequest):
try:
return JsonResource.render(self, txrequest)
except Exception, e:
if self.root.debug:
return traceback.format_exc()
log.err()
r = {"status": "error", "message": str(e)}
return self.render_object(r, txrequest)
class Schedule(WsResource):
def render_POST(self, txrequest):
settings = txrequest.args.pop('setting', [])
settings = dict(x.split('=', 1) for x in settings)
args = dict((k, v[0]) for k, v in txrequest.args.items())
project = args.pop('project')
spider = args.pop('spider')
args['settings'] = settings
jobid = uuid.uuid1().hex
args['_job'] = jobid
self.root.scheduler.schedule(project, spider, **args)
return {"status": "ok", "jobid": jobid}
class Cancel(WsResource):
def render_POST(self, txrequest):
args = dict((k, v[0]) for k, v in txrequest.args.items())
project = args['project']
jobid = args['job']
signal = args.get('signal', 'TERM')
prevstate = None
queue = self.root.poller.queues[project]
c = queue.remove(lambda x: x["_job"] == jobid)
if c:
prevstate = "pending"
spiders = self.root.launcher.processes.values()
for s in spiders:
if s.job == jobid:
s.transport.signalProcess(signal)
prevstate = "running"
return {"status": "ok", "prevstate": prevstate}
class AddVersion(WsResource):
def render_POST(self, txrequest):
project = txrequest.args['project'][0]
version = txrequest.args['version'][0]
eggf = StringIO(txrequest.args['egg'][0])
self.root.eggstorage.put(eggf, project, version)
spiders = get_spider_list(project)
self.root.update_projects()
return {"status": "ok", "project": project, "version": version, \
"spiders": len(spiders)}
class ListProjects(WsResource):
def render_GET(self, txrequest):
projects = self.root.scheduler.list_projects()
return {"status": "ok", "projects": projects}
class ListVersions(WsResource):
def render_GET(self, txrequest):
project = txrequest.args['project'][0]
versions = self.root.eggstorage.list(project)
return {"status": "ok", "versions": versions}
class ListSpiders(WsResource):
def render_GET(self, txrequest):
project = txrequest.args['project'][0]
spiders = get_spider_list(project, runner=self.root.runner)
return {"status": "ok", "spiders": spiders}
class ListJobs(WsResource):
def render_GET(self, txrequest):
project = txrequest.args['project'][0]
spiders = self.root.launcher.processes.values()
running = [{"id": s.job, "spider": s.spider} for s in spiders if s.project == project]
queue = self.root.poller.queues[project]
pending = [{"id": x["_job"], "spider": x["name"]} for x in queue.list()]
finished = [{"id": s.job, "spider": s.spider,
"start_time": s.start_time.isoformat(' '),
"end_time": s.end_time.isoformat(' ')} for s in self.root.launcher.finished
if s.project == project]
return {"status":"ok", "pending": pending, "running": running, "finished": finished}
class DeleteProject(WsResource):
def render_POST(self, txrequest):
project = txrequest.args['project'][0]
self._delete_version(project)
return {"status": "ok"}
def _delete_version(self, project, version=None):
self.root.eggstorage.delete(project, version)
self.root.update_projects()
class DeleteVersion(DeleteProject):
def render_POST(self, txrequest):
project = txrequest.args['project'][0]
version = txrequest.args['version'][0]
self._delete_version(project, version)
return {"status": "ok"}
Jump to Line
Something went wrong with that request. Please try again.