Merge cd2e342 into b745889

pyfarm/jobtypes/core/jobtype.py

@@ -29,7 +29,6 @@
 import tempfile
 from errno import EEXIST
 from datetime import datetime, timedelta
-from functools import partial
 from string import Template
 from os.path import expanduser, abspath, isdir, join
 from pprint import pformat
@@ -42,9 +41,13 @@
     from http.client import (
         OK, INTERNAL_SERVER_ERROR, CONFLICT, CREATED, NOT_FOUND, BAD_REQUEST)
 
+try:
+    WindowsError
+except NameError:  # pragma: no cover
+    WindowsError = OSError
+
 import treq
 from twisted.internet import reactor
-from twisted.internet.defer import Deferred
 from twisted.internet.error import ProcessDone, ProcessTerminated
 from twisted.python.failure import Failure
 from twisted.internet.defer import inlineCallbacks, Deferred, returnValue
@@ -55,7 +58,7 @@
 from pyfarm.core.enums import INTEGER_TYPES, STRING_TYPES, WorkState, WINDOWS
 from pyfarm.core.utility import ImmutableDict
 from pyfarm.agent.config import config
-from pyfarm.agent.http.core.client import post, http_retry_delay, post_direct
+from pyfarm.agent.http.core.client import http_retry_delay, post_direct
 from pyfarm.agent.logger import getLogger
 from pyfarm.agent.sysinfo import memory, system
 from pyfarm.agent.sysinfo.user import is_administrator, username
@@ -484,7 +487,7 @@ def tempdir(self, new=False, remove_on_finish=True):
 
         try:
             os.makedirs(parent_dir)
-        except OSError as e:  # pragma: no cover
+        except (IOError, OSError, WindowsError) as e:
             if e.errno != EEXIST:
                 logger.error("Failed to create %s: %s", parent_dir, e)
                 raise

tests/test_jobtypes/test_core_jobtype.py

@@ -16,20 +16,24 @@
 
 import os
 import re
+import tempfile
+from contextlib import nested
+from os.path import join, isdir, dirname, basename
 from uuid import UUID, uuid4
 
+from mock import patch
+from twisted.internet.defer import Deferred, inlineCallbacks
 from voluptuous import Schema, MultipleInvalid
 
 from pyfarm.core.utility import ImmutableDict
 from pyfarm.core.enums import INTEGER_TYPES, STRING_TYPES, WINDOWS
 from pyfarm.agent.config import config
 from pyfarm.agent.testutil import TestCase, skipIf
-from pyfarm.agent.sysinfo.user import is_administrator
-from pyfarm.jobtypes.core.internals import USER_GROUP_TYPES
+from pyfarm.agent.sysinfo import system, memory, user
+from pyfarm.agent.utility import remove_directory
+from pyfarm.jobtypes.core.internals import USER_GROUP_TYPES, logpool
 from pyfarm.jobtypes.core.jobtype import JobType, CommandData
 
-IS_ADMIN = is_administrator()
-
 
 def fake_assignment():
     assignment_id = uuid4()
@@ -148,24 +152,24 @@ def test_validate_group_type(self):
             CommandData("", group=1.0).validate()
 
     @skipIf(WINDOWS, "Non-Windows only")
-    @skipIf(IS_ADMIN, "Is Administrator")
+    @skipIf(user.is_administrator(), "Is Administrator")
     def test_validate_change_user_non_admin_failure(self):
         with self.assertRaises(EnvironmentError):
             CommandData("", user=0).validate()
 
     @skipIf(WINDOWS, "Non-Windows only")
-    @skipIf(IS_ADMIN, "Is Administrator")
+    @skipIf(user.is_administrator(), "Is Administrator")
     def test_validate_change_group_non_admin_failure(self):
         with self.assertRaises(EnvironmentError):
             CommandData("", group=0).validate()
 
     @skipIf(WINDOWS, "Non-Windows only")
-    @skipIf(not IS_ADMIN, "Not Administrator")
+    @skipIf(not user.is_administrator(), "Not Administrator")
     def test_validate_change_user_admin(self):
         CommandData("", user=0).validate()
 
     @skipIf(WINDOWS, "Non-Windows only")
-    @skipIf(not IS_ADMIN, "Not Administrator")
+    @skipIf(not user.is_administrator(), "Not Administrator")
     def test_validate_change_group_admin(self):
         CommandData("", group=0).validate()
 
@@ -208,6 +212,179 @@ def test_set_default_environment(self):
 
 
 class TestJobTypeLoad(TestCase):
+    # TODO: test to ensure load() does something useful, the below
+    # are just unittests
+
     def test_schema(self):
         with self.assertRaises(MultipleInvalid):
             JobType.load({})
+
+    def test_download_called(self):
+        assignment = fake_assignment()
+        deferred = Deferred()
+
+        class JobTypePatch(JobType):
+            @classmethod
+            def _download_jobtype(cls, name, version):
+                self.assertEqual(assignment["jobtype"]["name"], name)
+                self.assertEqual(assignment["jobtype"]["version"], version)
+                return deferred
+
+        with nested(
+            patch.object(
+                JobType, "_download_jobtype", JobTypePatch._download_jobtype),
+            patch.object(deferred, "addCallback")
+        ) as (_, patched_addCallback):
+            JobType.load(assignment)
+
+        patched_addCallback.assert_called_with(
+            JobType._jobtype_download_complete,
+            JobType._cache_key(assignment)
+        )
+
+
+class TestJobTypeCloseLogs(TestCase):
+    def test_close_logs(self):
+        jobtype = JobType(fake_assignment())
+
+        with patch.object(logpool, "close_log") as patched:
+            jobtype._close_logs()
+
+        patched.assert_called_with(jobtype.uuid)
+
+
+class TestJobTypeNode(TestCase):
+    def test_reraises_notimplemented(self):
+        def side_effect():
+            raise NotImplementedError
+
+        with nested(
+            patch.object(
+              system, "machine_architecture", side_effect=side_effect),
+            self.assertRaises(NotImplementedError)
+        ):
+            jobtype = JobType(fake_assignment())
+            jobtype.node()
+
+    def test_output(self):
+        jobtype = JobType(fake_assignment())
+
+        with nested(
+            patch.object(memory, "total_ram", return_value=1.1),
+            patch.object(memory, "free_ram", return_value=2.1),
+            patch.object(memory, "total_consumption", return_value=3.1)
+        ):
+            self.assertEqual(
+                jobtype.node(),
+                {
+                    "master_api": config.get("master-api"),
+                    "hostname": config["agent_hostname"],
+                    "agent_id": config["agent_id"],
+                    "id": config["agent_id"],
+                    "cpus": int(config["agent_cpus"]),
+                    "ram": int(config["agent_ram"]),
+                    "total_ram": 1,
+                    "free_ram": 2,
+                    "consumed_ram": 3,
+                    "admin": user.is_administrator(),
+                    "user": user.username(),
+                    "case_sensitive_files":
+                        system.filesystem_is_case_sensitive(),
+                    "case_sensitive_env":
+                        system.environment_is_case_sensitive(),
+                    "machine_architecture":
+                        system.machine_architecture(),
+                    "operating_system": system.operating_system()
+                }
+            )
+
+
+class TestJobTypeAssignments(TestCase):
+    def test_assignments(self):
+        assignment = fake_assignment()
+        jobtype = JobType(assignment)
+        self.assertEqual(jobtype.assignments(), assignment["tasks"])
+
+
+class TestJobTypeTempDir(TestCase):
+    def test_not_new_and_tempdir_already_set(self):
+        jobtype = JobType(fake_assignment())
+        jobtype._tempdir = "foobar"
+        self.assertEqual(jobtype.tempdir(), "foobar")
+
+    def test_creates_directory(self):
+        root_directory = tempfile.mkdtemp()
+        self.addCleanup(remove_directory, root_directory)
+        config["jobtype_tempdir_root"] = join(root_directory, "$JOBTYPE_UUID")
+        jobtype = JobType(fake_assignment())
+        self.assertTrue(isdir(jobtype.tempdir()))
+
+    def test_path_contains_jobtype_uuid(self):
+        root_directory = tempfile.mkdtemp()
+        self.addCleanup(remove_directory, root_directory)
+        config["jobtype_tempdir_root"] = join(root_directory, "$JOBTYPE_UUID")
+        jobtype = JobType(fake_assignment())
+        tempdir = jobtype.tempdir()
+        self.assertEqual(basename(dirname(tempdir)), str(jobtype.uuid))
+
+    def test_ignores_eexist(self):
+        root_directory = tempfile.mkdtemp()
+        self.addCleanup(remove_directory, root_directory)
+        config["jobtype_tempdir_root"] = join(root_directory, "$JOBTYPE_UUID")
+        jobtype = JobType(fake_assignment())
+        os.makedirs(config["jobtype_tempdir_root"].replace(
+            "$JOBTYPE_UUID", str(jobtype.uuid)))
+        jobtype.tempdir()
+
+    def test_makedirs_raises_other_errors(self):
+        root_directory = tempfile.mkdtemp()
+        self.addCleanup(remove_directory, root_directory)
+        config["jobtype_tempdir_root"] = join(root_directory, "$JOBTYPE_UUID")
+        jobtype = JobType(fake_assignment())
+
+        def side_effect(*args):
+            raise OSError("Foo", 4242)
+
+        with nested(
+            patch.object(os, "makedirs", side_effect=side_effect),
+            self.assertRaises(OSError)
+        ):
+            jobtype.tempdir()
+
+    def test_remove_on_finish_does_not_add_parent(self):
+        # It's important that we do not add the parent directory
+        # as the path to cleanup.  Otherwise we could end up
+        # cleaning up more than needed.
+        root_directory = tempfile.mkdtemp()
+        self.addCleanup(remove_directory, root_directory)
+        config["jobtype_tempdir_root"] = join(root_directory, "$JOBTYPE_UUID")
+        jobtype = JobType(fake_assignment())
+        jobtype.tempdir()
+
+        for value in jobtype._tempdirs:
+            self.assertNotEqual(value, dirname(config["jobtype_tempdir_root"]))
+
+    def test_remove_on_finish_adds_child_directory(self):
+        root_directory = tempfile.mkdtemp()
+        self.addCleanup(remove_directory, root_directory)
+        config["jobtype_tempdir_root"] = join(root_directory, "$JOBTYPE_UUID")
+        jobtype = JobType(fake_assignment())
+        tempdir = jobtype.tempdir()
+        self.assertIn(tempdir, jobtype._tempdirs)
+
+    def test_sets_tempdir(self):
+        jobtype = JobType(fake_assignment())
+        tempdir = jobtype.tempdir()
+        self.assertEqual(tempdir, jobtype._tempdir)
+
+    def test_new_tempdir_does_not_reset_default(self):
+        jobtype = JobType(fake_assignment())
+        tempdir1 = jobtype.tempdir()
+        tempdir2 = jobtype.tempdir(new=True)
+        self.assertNotEqual(tempdir1, tempdir2)
+        self.assertEqual(jobtype._tempdir, tempdir1)
+
+
+
+
+