diff --git a/docs/source/usage.rst b/docs/source/usage.rst index c317b3d..5528773 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -204,9 +204,50 @@ See the `delete version endpoint`_ on Scrapyd's documentation. .. code-block:: python - >>> scrapyd.delete_version('project_name', 'ac32a..b21ac') + >>> scrapyd.delete_version('project_name', 'version_name') True +Retrieve the status of a specific job +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. method:: ScrapydAPI.job_status(project, job_id) + +.. versionadded:: 0.2 + +Returns the job status for a single job. The status returned can be one of: +``''``, ``'running'``, ``'pending'`` or ``'finished'``. The empty string is +returned if the job ID could not be found and the status is therefore unknown. + +**Arguments**: + +- **project** *(string)* The name of the project which the version belongs to. +- **job_id** *(string)* The ID of the job you wish to check the status of. + +**Returns**: *(string)* The status of the job, if known. + +.. note:: + Scrapyd does not support an endpoint for this specific action. This + method's result is derived from the list jobs endpoint, and therefore + this is a helper method/shortcut provided by this wrapper itself. This is + why the call requires the `project` argument, as the list jobs endpoint + underlying this method also requires it. + +.. code-block:: python + + >>> scrapyd.job_status('project_name', 'ac32a..bc21') + 'running' + +If you wish, the various strings defining job state can be imported from +the ``scrapyd`` module itself for use in comparisons. e.g: + +.. code-block:: python + + from scrapyd_api import RUNNING, FINISHED, PENDING + + state = scrapyd.job_status('project_name', 'ac32a..bc21') + if state == RUNNING: + print 'Job is running' + List all jobs for a project ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapyd_api/__init__.py b/scrapyd_api/__init__.py index c172adf..9046984 100644 --- a/scrapyd_api/__init__.py +++ b/scrapyd_api/__init__.py @@ -2,6 +2,11 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +from .constants import ( + FINISHED, + PENDING, + RUNNING +) from .exceptions import ScrapydError from .wrapper import ScrapydAPI @@ -13,4 +18,4 @@ VERSION = __version__ -__all__ = ['ScrapydError', 'ScrapydAPI'] +__all__ = ['ScrapydError', 'ScrapydAPI', 'FINISHED', 'PENDING', 'RUNNING'] diff --git a/scrapyd_api/constants.py b/scrapyd_api/constants.py index 124c43c..2df33c5 100644 --- a/scrapyd_api/constants.py +++ b/scrapyd_api/constants.py @@ -21,3 +21,9 @@ LIST_VERSIONS_ENDPOINT: '/listversions.json', SCHEDULE_ENDPOINT: '/schedule.json', } + +FINISHED = 'finished' +PENDING = 'pending' +RUNNING = 'running' + +JOB_STATES = [FINISHED, PENDING, RUNNING] diff --git a/scrapyd_api/wrapper.py b/scrapyd_api/wrapper.py index 14018c8..23afde5 100644 --- a/scrapyd_api/wrapper.py +++ b/scrapyd_api/wrapper.py @@ -12,7 +12,10 @@ class ScrapydAPI(object): """ - Provides a thin Pythonic wrapper around the Scrapyd API. + Provides a thin Pythonic wrapper around the Scrapyd API. The public methods + come in two types: first class, those that wrap a Scrapyd API endpoint + directly; and derived, those that use a one or more Scrapyd API endpoint(s) + to provide functionality that is unique to this wrapper. """ def __init__(self, target='http://localhost:6800', auth=None, @@ -56,7 +59,8 @@ def _build_url(self, endpoint): def add_version(self, project, version, egg): """ - Adds a new project egg to the Scrapyd service. + Adds a new project egg to the Scrapyd service. First class, maps to + Scrapyd's add version endpoint. """ url = self._build_url(constants.ADD_VERSION_ENDPOINT) data = { @@ -71,7 +75,8 @@ def add_version(self, project, version, egg): def cancel(self, project, job): """ - Cancels a job from a specific project. + Cancels a job from a specific project. First class, maps to + Scrapyd's cancel job endpoint. """ url = self._build_url(constants.CANCEL_ENDPOINT) data = { @@ -79,11 +84,12 @@ def cancel(self, project, job): 'job': job } json = self.client.post(url, data=data) - return True if json['prevstate'] == 'running' else False + return True if json['prevstate'] == constants.RUNNING else False def delete_project(self, project): """ - Deletes all versions of a project. + Deletes all versions of a project. First class, maps to Scrapyd's + delete project endpoint. """ url = self._build_url(constants.DELETE_PROJECT_ENDPOINT) data = { @@ -94,7 +100,8 @@ def delete_project(self, project): def delete_version(self, project, version): """ - Deletes a specific version of a project. + Deletes a specific version of a project. First class, maps to + Scrapyd's delete version endpoint. """ url = self._build_url(constants.DELETE_VERSION_ENDPOINT) data = { @@ -104,9 +111,22 @@ def delete_version(self, project, version): self.client.post(url, data=data) return True + def job_status(self, project, job_id): + """ + Retrieves the 'status' of a specific job specified by its id. Derived, + utilises Scrapyd's list jobs endpoint to provide the answer. + """ + all_jobs = self.list_jobs(project) + for state in constants.JOB_STATES: + job_ids = [job['id'] for job in all_jobs[state]] + if job_id in job_ids: + return state + return '' # Job not found, state unknown. + def list_jobs(self, project): """ - Lists all known jobs. + Lists all known jobs for a project. First class, maps to Scrapyd's + list jobs endpoint. """ url = self._build_url(constants.LIST_JOBS_ENDPOINT) params = {'project': project} @@ -115,7 +135,8 @@ def list_jobs(self, project): def list_projects(self): """ - Lists all deployed projects. + Lists all deployed projects. First class, maps to Scrapyd's + list projects endpoint. """ url = self._build_url(constants.LIST_PROJECTS_ENDPOINT) json = self.client.get(url) @@ -123,7 +144,8 @@ def list_projects(self): def list_spiders(self, project): """ - Lists all known spiders for a specific project. + Lists all known spiders for a specific project. First class, maps + to Scrapyd's list spiders endpoint. """ url = self._build_url(constants.LIST_SPIDERS_ENDPOINT) params = {'project': project} @@ -132,7 +154,8 @@ def list_spiders(self, project): def list_versions(self, project): """ - Lists all deployed versions of a specific project. + Lists all deployed versions of a specific project. First class, maps + to Scrapyd's list versions endpoint. """ url = self._build_url(constants.LIST_VERSIONS_ENDPOINT) params = {'project': project} @@ -141,7 +164,8 @@ def list_versions(self, project): def schedule(self, project, spider, settings=None, **kwargs): """ - Schedules a spider from a specific project to run. + Schedules a spider from a specific project to run. First class, maps + to Scrapyd's scheduling endpoint. """ url = self._build_url(constants.SCHEDULE_ENDPOINT) diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index f01be90..8ec0787 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -4,7 +4,9 @@ from scrapyd_api.compat import StringIO from scrapyd_api.constants import ( ADD_VERSION_ENDPOINT, - CANCEL_ENDPOINT + CANCEL_ENDPOINT, + FINISHED, + PENDING ) from scrapyd_api.wrapper import ScrapydAPI @@ -159,22 +161,45 @@ def test_delete_version(): ) +def test_job_status(): + """ + Test the method which handles retrieving the status of a given job. + """ + mock_client = MagicMock() + mock_client.get.return_value = { + 'pending': [{'id': 'abc'}, {'id': 'def'}], + 'running': [], + 'finished': [{'id': 'ghi'}], + } + api = ScrapydAPI(HOST_URL, client=mock_client) + expected_results = ( + ('abc', PENDING), + ('def', PENDING), + ('ghi', FINISHED), + ('xyz', '') + ) + for job_id, expected_result in expected_results: + rtn = api.job_status(PROJECT, job_id) + assert rtn == expected_result + + def test_list_jobs(): """ Test the method which handles listing jobs on the server. """ mock_client = MagicMock() mock_client.get.return_value = { - 'pending': ['abcdef'], - 'running': ['ghijkl'], - 'finished': ['mnopqr'], + 'pending': [{'id': 'abc'}, {'id': 'def'}], + 'running': [], + 'finished': [{'id': 'ghi'}], } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.list_jobs(PROJECT) assert len(rtn) == 3 - assert rtn['pending'] == ['abcdef'] - assert rtn['running'] == ['ghijkl'] - assert rtn['finished'] == ['mnopqr'] + assert sorted(rtn.keys()) == ['finished', 'pending', 'running'] + assert rtn['pending'] == [{'id': 'abc'}, {'id': 'def'}] + assert rtn['finished'] == [{'id': 'ghi'}] + assert rtn['running'] == [] mock_client.get.assert_called_with( 'http://localhost/listjobs.json', params={