Initial support for chronos 3.x for #33

.travis.yml

@@ -1,14 +1,29 @@
-env:
- - CHRONOSVERSION: 2.4.0
-
 language: python
-python:
-  - 2.7
-  - 3.5
-sudo: true
+matrix:
+  include:
+  - os: linux
+    sudo: required
+    services: docker
+    python: 2.7
+    env: CHRONOSVERSION=2.4.0
+  - os: linux
+    sudo: required
+    services: docker
+    python: 3.5
+    env: CHRONOSVERSION=2.4.0
+  - os: linux
+    sudo: required
+    services: docker
+    python: 2.7
+    env: CHRONOSVERSION=3.0.2
+  - os: linux
+    sudo: required
+    services: docker
+    python: 3.5
+    env: CHRONOSVERSION=3.0.2
+before_install:
+  - docker-compose -f itests/docker-compose.yml pull
 install:
   - pip install tox
 script:
-  - ./itests/install-chronos.sh
-  - ./itests/start-chronos.sh
   - make test && make itests

README.md

@@ -70,12 +70,12 @@ Delete all the in flight tasks for a job:
 ## Testing
 
 ``chronos-python`` uses Travis to test against multiple versions of Chronos. You can run the tests locally on any machine
-with [Docker](https://www.docker.com/) on it.
+with [Docker](https://www.docker.com/) and [Docker compose](https://docs.docker.com/compose/) on it.
 
 To run the tests:
 
     make itests
 
 To run against a different version of Chronos:
 
-    CHRONOSVERSION=2.4.0 make itests
+    CHRONOSVERSION=3.0.2 make itests

chronos/__init__.py

@@ -26,13 +26,16 @@
 import socket
 import json
 import logging
+import warnings
 
 # Python 3 changed the submodule for quote
 try:
     from urllib import quote
 except ImportError:
     from urllib.parse import quote
 
+SCHEDULER_API_VERSIONS = ('v1',)
+
 
 class ChronosAPIError(Exception):
     pass
@@ -54,7 +57,11 @@ class ChronosClient(object):
     _user = None
     _password = None
 
-    def __init__(self, servers, proto="http", username=None, password=None, extra_headers=None, level='WARN'):
+    def __init__(
+        self, servers, proto="http", username=None,
+        password=None, extra_headers=None, level='WARN',
+        scheduler_api_version='v1'
+    ):
         server_list = servers if isinstance(servers, list) else [servers]
         self.servers = ["%s://%s" % (proto, server) for server in server_list]
         self.extra_headers = extra_headers
@@ -63,6 +70,14 @@ def __init__(self, servers, proto="http", username=None, password=None, extra_he
             self._password = password
         logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=level)
         self.logger = logging.getLogger(__name__)
+        if scheduler_api_version is None:
+            warnings.warn("Chronos >=3.x requires scheduler_api_version set", FutureWarning)
+            self._prefix = ''
+        else:
+            if scheduler_api_version not in SCHEDULER_API_VERSIONS:
+                raise ChronosAPIError('Wrong scheduler_api_version provided')
+            self._prefix = "/%s" % (scheduler_api_version,)
+        self.scheduler_api_version = scheduler_api_version
 
     def list(self):
         """List all jobs on Chronos."""
@@ -89,10 +104,14 @@ def add(self, job_def, update=False):
         self._check_fields(job_def)
         if "parents" in job_def:
             path = "/scheduler/dependency"
-        if update:
-            method = "PUT"
-        else:
-            method = "POST"
+        # Cool story: chronos >= 3.0 ditched PUT and only allows POST here,
+        # trying to maintain backwards compat with < 3.0
+        method = "POST"
+        if self.scheduler_api_version is None:
+            if update:
+                method = "PUT"
+            else:
+                method = "POST"
         return self._call(path, method, json.dumps(job_def))
 
     def update(self, job_def):
@@ -125,14 +144,19 @@ def scheduler_stat_mean(self):
         return self._call('/scheduler/stats/mean', 'GET')
 
     def metrics(self):
-        return self._call('/metrics', 'GET')
+        # for some reason, /metrics is not prefixed with the version
+        return self._call('/metrics', 'GET', prefix=False)
 
-    def _call(self, url, method="GET", body=None, headers={}):
+    def _call(self, url, method="GET", body=None, headers={}, prefix=True):
         hdrs = {}
         if body:
             hdrs['Content-Type'] = "application/json"
         hdrs.update(headers)
-        self.logger.debug("Fetch: %s %s" % (method, url))
+        if prefix:
+            _url = '%s%s' % (self._prefix, url, )
+        else:
+            _url = url
+        self.logger.debug("Calling: %s %s" % (method, _url))
         if body:
             self.logger.debug("Body: %s" % body)
         conn = httplib2.Http(disable_ssl_certificate_validation=True)
@@ -145,7 +169,7 @@ def _call(self, url, method="GET", body=None, headers={}):
         servers = list(self.servers)
         while servers:
             server = servers.pop(0)
-            endpoint = "%s%s" % (server, quote(url))
+            endpoint = "%s%s" % (server, quote(_url))
             try:
                 resp, content = conn.request(endpoint, method, body=body, headers=hdrs)
             except (socket.error, httplib2.ServerNotFoundError) as e:
@@ -178,10 +202,23 @@ def _check(self, resp, content):
         if payload is None and status != 204:
             raise ChronosAPIError("Request to Chronos API failed: status: %d, response: %s" % (status, content))
 
+        # if the status returned is not an OK status, raise an exception
+        if status >= 400:
+            message = "API returned status %d, content: %s" % (status, payload,)
+            # newer chronos does return the full stack trace in a message field,
+            # grabbing the first 160 chars from it
+            if 'message' in payload:
+                self.logger.debug(payload['message'])
+                message = '%s (...)' % (payload['message'][:120],)
+            raise ChronosAPIError(message)
+
         return payload
 
     def _check_fields(self, job):
-        for k in ChronosJob.fields:
+        fields = ChronosJob.fields
+        if self.scheduler_api_version is None:
+            fields.extend(ChronosJob.legacy_fields)
+        for k in fields:
             if k not in job:
                 raise MissingFieldError("missing required field %s" % k)
 
@@ -202,18 +239,23 @@ def _check_fields(self, job):
 
 class ChronosJob(object):
     fields = [
-        "async",
         "command",
         "name",
         "owner",
         "disabled"
     ]
+    legacy_fields = [
+        "async",
+    ]
     one_of = ["schedule", "parents"]
     container_fields = [
         "type",
         "image"
     ]
 
 
-def connect(servers, proto="http", username=None, password=None, extra_headers=None):
-    return ChronosClient(servers, proto=proto, username=username, password=password, extra_headers=extra_headers)
+def connect(servers, proto="http", username=None, password=None, extra_headers=None, scheduler_api_version='v1'):
+    return ChronosClient(
+        servers, proto=proto, username=username, password=password,
+        extra_headers=extra_headers, scheduler_api_version=scheduler_api_version
+    )

itests/chronos-python.feature

@@ -1,17 +1,24 @@
 Feature: chronos-python can interact with chronos
 
+  @all
   Scenario: Trivial chronos interaction
     Given a working chronos instance
     When we create a trivial chronos job named "myjob"
     Then we should be able to see the job named "myjob" when we list jobs
 
+  @3.0.2
   Scenario: Handling spaces in job names
     Given a working chronos instance
      When we create a trivial chronos job named "job with spaces"
-     Then we should be able to see the job named "job with spaces" when we list jobs
-      And we should be able to delete the job named "job with spaces"
-      And we should not be able to see the job named "job with spaces" when we list jobs
+     Then we should not see the job named "job with spaces" when we list jobs
 
+  @2.4.0
+  Scenario: Handling spaces in job names
+    Given a working chronos instance
+    When we create a trivial chronos job named "job with spaces"
+    Then we should be able to see the job named "job with spaces" when we list jobs
+
+  @all
   Scenario: Gathering job stats for job
     Given a working chronos instance
      When we create a trivial chronos job named "myjob"
@@ -20,12 +27,14 @@ Feature: chronos-python can interact with chronos
       And we should be able to see the median timing for all jobs
       And we should be able to see the mean timing for all jobs
 
+  @all
   Scenario: Gathering scheduler graph
     Given a working chronos instance
      When we create a trivial chronos job named "myjob"
       And we create a trivial chronos job named "myotherjob"
      Then we should be able to see 2 jobs in the job graph
 
+  @all
   Scenario: Getting metrics
     Given a working chronos instance
      Then we should be able to see metrics

itests/docker-compose.yml

@@ -1,5 +1,77 @@
----
-chronos:
-  build: .
-  ports:
-    - 4400
+version: "2"
+
+services:
+  zookeeper:
+    image: zookeeper:3.4.9
+    container_name: zookeeper
+    environment:
+      ZOO_MY_ID: 1
+      ZK_INIT_LIMIT: 10
+      ZK_SYNC_LIMIT: 5
+
+  master:
+    image: mesosphere/mesos-master:1.0.3
+    container_name: mesos-master
+    environment:
+      MESOS_ZK: zk://zookeeper:2181/mesos
+      MESOS_QUORUM: 1
+      MESOS_CLUSTER: chronos-python
+      MESOS_REGISTRY: in_memory
+      MESOS_HOSTNAME: localhost
+    ports:
+      - 5050:5050
+    depends_on:
+      - zookeeper
+    links:
+      - zookeeper
+
+  agent:
+    image: mesosphere/mesos-slave:1.0.3
+    container_name: mesos-agent
+    pid: host
+    environment:
+      MESOS_MASTER: zk://zookeeper:2181/mesos
+      MESOS_WORK_DIR: /tmp/mesos
+      MESOS_PORT: 5051
+      MESOS_RESOURCES: ports(*):[11000-11999]
+      MESOS_CONTAINERIZERS: mesos
+      MESOS_HOSTNAME: localhost
+    ports:
+      - 5051:5051
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup
+    depends_on:
+      - zookeeper
+    links:
+      - zookeeper
+      - master
+
+  chronos-2.4.0:
+    image: mesosphere/chronos:chronos-2.4.0-0.1.20150828104228.ubuntu1404-mesos-0.27.0-0.2.190.ubuntu1404
+    container_name: chronos-2.4.0
+    command: '/usr/bin/chronos run_jar --http_port 4400 --zk_hosts zookeeper:2181 --master zk://zookeeper:2181/mesos'
+    ports:
+      - 4400:4400
+    depends_on:
+      - zookeeper
+      - master
+    links:
+      - zookeeper
+      - master
+
+  chronos-3.0.2:
+    image: mesosphere/chronos:v3.0.2
+    container_name: chronos-3.0.2
+    command: '--zk_hosts zookeeper:2181 --master zk://zookeeper:2181/mesos'
+    environment:
+      PORT0: 4400
+      PORT1: 8080
+    ports:
+      - 4400:4400
+    depends_on:
+      - zookeeper
+      - master
+    links:
+      - zookeeper
+      - master
+

itests/environment.py

@@ -1,5 +1,4 @@
 import time
-
 from itest_utils import wait_for_chronos
 
 
@@ -16,4 +15,4 @@ def after_scenario(context, scenario):
                 break
             for job in jobs:
                 context.client.delete(job['name'])
-            time.sleep(0.5)
+            time.sleep(1.0)