Merge branch 'master' of github.com:pyfarm/pyfarm-agent into ilc_rean…

…nounce

README.rst

@@ -15,22 +15,72 @@
 PyFarm Agent
 ============
 
-.. image:: https://badge.waffle.io/pyfarm/pyfarm-agent.png?label=ready
-    :target: https://waffle.io/pyfarm/pyfarm-agent
-    :align: left
-
 .. image:: https://travis-ci.org/pyfarm/pyfarm-agent.png?branch=master
     :target: https://travis-ci.org/pyfarm/pyfarm-agent
-    :align: left
+    :alt: build status (agent)
 
 .. image:: https://coveralls.io/repos/pyfarm/pyfarm-agent/badge.png?branch=master
     :target: https://coveralls.io/r/pyfarm/pyfarm-agent?branch=master
-    :align: left
+    :alt: coverage
+
 
 Core module containing code to run PyFarm's agent. This will allow a remote
 host to:
-    * add itself to PyFarm
-    * request, receieve, and execute work via job types
-    * track and control individual processes
-    * measure and limit system resource usage
+
+    * Inform the master about itself
+    * Request, receive and execute work via job types
+    * Track and control individual processes
+    * Measure and limit system resource usage
+
+
+Python Version Support
+----------------------
+
+This library supports Python 2.6 and Python 2.7 only for the moment.  Coding
+practices have been geared towards supporting Python 3 when the underlying
+library, Twisted, is ported to Python 3.
+
+Documentation
+-------------
+
+The documentation for this this library is hosted on
+`Read The Docs <https://pyfarm.readthedocs.org/projects/pyfarm-agent/en/latest/>`_.
+It's generated directly from this library using sphinx (setup may vary depending
+on platform)::
+
+    virtualenv env
+    . env/bin/activate
+    pip install sphinx nose
+    pip install -e . --egg
+    make -C docs html
+
+Testing
+-------
+
+Tests are run on `Travis <https://travis-ci.org/pyfarm/pyfarm-agent>`_ for
+every commit.  They can also be run locally too using ``trial``.  Currently,
+the agent tests require:
+
+    * Access to https://httpbin.pyfarm.net for HTTP client testing.  This is
+      configurable however and could be pointed to an internal domain
+      using the ``agent_unittest`` configuration variable.
+    * The ``pyfarm.master`` module to run the API.  So all the setup steps
+      that apply to the master will apply here as well.  This includes the
+      requirement to run Redis, RabbitMQ or another backend that supports
+      ``celery``.
+    * Linux or OS X since the master is designed to operate on these
+      platforms.  The below setup may work on Windows with a few configuration
+      tweaks too.
+
+Newer tests are being designed to be lighter weight so eventually most of the
+above may no longer be required for testing.  For now however these are the
+basic steps to run the tests and are based on the steps in ``.travis.yml``::
+
+    virtualenv env
+    . env/bin/activate
+    pip install pyfarm.master uwsgi mock
+    pyfarm-tables
+    uwsgi resources/uwsgi.ini
+    pip install -e . --egg
+    trial tests  # in a new shell with the same virtualenv
 

docs/source/includes/commands/pyfarm-agent_start.out

@@ -1,6 +1,4 @@
-usage: pyfarm-agent [status|start|stop] start [-h]
-                                              [--projects PROJECTS [PROJECTS ...]]
-                                              [--state STATE]
+usage: pyfarm-agent [status|start|stop] start [-h] [--state STATE]
                                               [--time-offset TIME_OFFSET]
                                               [--ntp-server NTP_SERVER]
                                               [--ntp-server-version NTP_SERVER_VERSION]
@@ -23,7 +21,8 @@ usage: pyfarm-agent [status|start|stop] start [-h]
                                               [--manhole-password MANHOLE_PASSWORD]
                                               [--html-templates-reload]
                                               [--static-files STATIC_FILES]
-                                              [--http-retry-delay HTTP_RETRY_DELAY]
+                                              [--http-retry-delay-offset HTTP_RETRY_DELAY_OFFSET]
+                                              [--http-retry-delay-factor HTTP_RETRY_DELAY_FACTOR]
                                               [--jobtype-no-cache]
 
 optional arguments:
@@ -33,13 +32,6 @@ General Configuration:
   These flags configure parts of the agent related to hardware, state, and
   certain timing and scheduling attributes.
 
-  --projects PROJECTS [PROJECTS ...]
-                        The project or projects this agent is dedicated to. By
-                        default the agent will service any project however
-                        specific projects may be specified. For example if you
-                        wish this agent to service 'Foo Part I' and 'Foo Part
-                        II' only just specify it as `--projects "Foo Part I"
-                        "Foo Part II"`
   --state STATE         The current agent state, valid values are ['disabled',
                         'offline', 'running', 'online']. [default: online]
   --time-offset TIME_OFFSET
@@ -159,9 +151,15 @@ HTTP Configuration:
   --static-files STATIC_FILES
                         The default location where the agent's http server
                         should find static files to serve.
-  --http-retry-delay HTTP_RETRY_DELAY
-                        If a http request to the master has failed, wait this
-                        amount of time before trying again
+  --http-retry-delay-offset HTTP_RETRY_DELAY_OFFSET
+                        If a http request to the master has failed, wait at
+                        least this amount of time before resending the
+                        request.
+  --http-retry-delay-factor HTTP_RETRY_DELAY_FACTOR
+                        The value provided here is used in combination with
+                        --http-retry-delay-offset to calculate the retry
+                        delay. This is used as a multiplier against random()
+                        before being added to the offset.
 
 Job Types:
   --jobtype-no-cache    If provided then do not cache job types, always

docs/source/modules/pyfarm.agent.http.api.rst

@@ -9,7 +9,6 @@ Submodules
    pyfarm.agent.http.api.assign
    pyfarm.agent.http.api.base
    pyfarm.agent.http.api.config
-   pyfarm.agent.http.api.log
    pyfarm.agent.http.api.state
    pyfarm.agent.http.api.tasklogs
    pyfarm.agent.http.api.tasks

docs/source/modules/pyfarm.agent.rst

@@ -19,7 +19,6 @@ Submodules
    pyfarm.agent.config
    pyfarm.agent.manhole
    pyfarm.agent.service
-   pyfarm.agent.tasks
    pyfarm.agent.testutil
    pyfarm.agent.utility
 

tests/test_agent/test_entrypoints_utility.py

@@ -14,35 +14,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from os import urandom
-import re
-
-from pyfarm.agent.config import config
-from pyfarm.agent.testutil import TestCase, ErrorCapturingParser
-
-
-class TestConfigWithParser(TestCase):
-    def test_set(self):
-        key = urandom(16).encode("hex")
-        value = urandom(16).encode("hex")
-        parser = ErrorCapturingParser()
-        parser.add_argument("--foo", config=key, help=key, default=False)
-        self.assertIn(key, config)
-        args = parser.parse_args(["--foo", value])
-        self.assertEqual(args.foo, value)
-        self.assertIn(key, config)
-        self.assertEqual(config[key], value)
-
-    def test_uses_default(self):
-        key = urandom(16).encode("hex")
-        parser = ErrorCapturingParser()
-        parser.add_argument("--foo", config=key, help=key, default=False)
-        args = parser.parse_args()
-        self.assertEqual(args.foo, False)
-        self.assertEqual(config[key], False)
-
-    def test_requires_default(self):
-        parser = ErrorCapturingParser()
-        with self.assertRaisesRegexp(
-                AssertionError, re.compile(".*no default was provided.*")):
-            parser.add_argument("--foo", config="foo", help="foo")

tests/test_agent/test_service.py

@@ -15,7 +15,9 @@
 # limitations under the License.
 
 import os
+import ntplib
 import sys
+import time
 import json
 import uuid
 from contextlib import nested
@@ -42,7 +44,7 @@
 from pyfarm.agent.sysinfo import cpu
 from pyfarm.agent.testutil import TestCase, random_port
 from pyfarm.agent.config import config
-from pyfarm.agent.service import Agent, svclog
+from pyfarm.agent.service import Agent, svclog, ntplog
 from pyfarm.agent.sysinfo import network, graphics, memory
 from pyfarm.agent import utility
 
@@ -76,22 +78,7 @@ def render_POST(self, request):
         return NOT_DONE_YET
 
 
-# TODO: need better tests, these are a little rudimentary at the moment
-class TestAgentServiceAttributes(TestCase):
-    def test_agent_api_url(self):
-        config["agent_id"] = uuid.uuid4()
-        agent = Agent()
-        self.assertEqual(
-            agent.agent_api(),
-            "{master_api}/agents/{agent_id}".format(
-                master_api=config["master_api"],
-                agent_id=config["agent_id"]))
-
-    def test_agent_api_url_keyerror(self):
-        agent = Agent()
-        config.pop("agent_id")
-        self.assertIsNone(agent.agent_api())
-
+class TestSystemData(TestCase):
     def test_system_data(self):
         config["remote_ip"] = os.urandom(16).encode("hex")
         expected = {
@@ -108,20 +95,75 @@ def test_system_data(self):
             "port": config["agent_api_port"],
             "time_offset": config["agent_time_offset"],
             "state": config["state"],
-            "mac_addresses": list(network.mac_addresses())}
-
-        try:
-            gpu_names = graphics.graphics_cards()
-            expected["gpus"] = gpu_names
-        except graphics.GPULookupError:
-            pass
+            "mac_addresses": list(network.mac_addresses()),
+            "gpus": [1, 3, 5]
+        }
 
         agent = Agent()
-        system_data = agent.system_data()
+        with patch.object(graphics, "graphics_cards", return_value=[1, 3, 5]):
+            system_data = agent.system_data()
+
         self.assertApproximates(
             system_data.pop("free_ram"), config["free_ram"], 64)
         self.assertEqual(system_data, expected)
 
+    def test_agent_time_offset(self):
+        config["agent_time_offset"] = "auto"
+        agent = Agent()
+        agent.system_data()
+        self.assertNotEqual(config["agent_time_offset"], "auto")
+
+    def test_get_offset(self):
+        agent = Agent()
+
+        response = Mock()
+        response.tx_time = 10
+
+        with nested(
+            patch.object(ntplib.NTPClient, "request", return_value=response),
+            patch.object(time, "time", return_value=5),
+        ):
+            agent.system_data(requery_timeoffset=True)
+
+        self.assertEqual(config["agent_time_offset"], 5)
+
+    def test_get_offset_error(self):
+        offset = config["agent_time_offset"]
+        agent = Agent()
+
+        error = Exception()
+
+        def raise_(*args, **kwargs):
+            raise error
+
+        with nested(
+            patch.object(ntplib.NTPClient, "request", raise_),
+            patch.object(time, "time", return_value=5),
+            patch.object(ntplog, "warning"),
+        ) as (_, _, warning):
+            agent.system_data(requery_timeoffset=True)
+
+        self.assertEqual(config["agent_time_offset"], offset)
+        warning.assert_called_with(
+            "Failed to determine network time: %s", error)
+
+
+# TODO: need better tests, these are a little rudimentary at the moment
+class TestAgentServiceAttributes(TestCase):
+    def test_agent_api_url(self):
+        config["agent_id"] = uuid.uuid4()
+        agent = Agent()
+        self.assertEqual(
+            agent.agent_api(),
+            "{master_api}/agents/{agent_id}".format(
+                master_api=config["master_api"],
+                agent_id=config["agent_id"]))
+
+    def test_agent_api_url_keyerror(self):
+        agent = Agent()
+        config.pop("agent_id")
+        self.assertIsNone(agent.agent_api())
+
     def test_callback_agent_id_set(self):
         schedule_call_args = []
 
@@ -721,7 +763,7 @@ def test_no_agent_api(self):
             "Cannot post shutdown, agent_api() returned None")
         stop_lock_acquire.assert_called_once()
         stop_lock_release.assert_called_once()
-        post_shutdown.assert_never_called()
+        self.assertFalse(post_shutdown.called)
 
     @inlineCallbacks
     def test_stops_and_removes_jobtypes(self):

tests/test_agent/test_testutil.py

@@ -0,0 +1,48 @@
+# No shebang line, this module is meant to be imported
+#
+# Copyright 2015 Oliver Palmer
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from os import urandom
+import re
+
+from pyfarm.agent.config import config
+from pyfarm.agent.testutil import TestCase, ErrorCapturingParser
+
+
+class TestConfigWithParser(TestCase):
+    def test_set(self):
+        key = urandom(16).encode("hex")
+        value = urandom(16).encode("hex")
+        parser = ErrorCapturingParser()
+        parser.add_argument("--foo", config=key, help=key, default=False)
+        self.assertIn(key, config)
+        args = parser.parse_args(["--foo", value])
+        self.assertEqual(args.foo, value)
+        self.assertIn(key, config)
+        self.assertEqual(config[key], value)
+
+    def test_uses_default(self):
+        key = urandom(16).encode("hex")
+        parser = ErrorCapturingParser()
+        parser.add_argument("--foo", config=key, help=key, default=False)
+        args = parser.parse_args()
+        self.assertEqual(args.foo, False)
+        self.assertEqual(config[key], False)
+
+    def test_requires_default(self):
+        parser = ErrorCapturingParser()
+        with self.assertRaisesRegexp(
+                AssertionError, re.compile(".*no default was provided.*")):
+            parser.add_argument("--foo", config="foo", help="foo")