Big step in the reorg.

hendrix/contrib/resources/static.py

@@ -1,5 +1,6 @@
 import os
-from hendrix.resources import DjangoStaticResource
+
+from hendrix.resources.resources import DjangoStaticResource
 from django.conf import settings
 from django.contrib.staticfiles import finders
 

hendrix/deploy/base.py

@@ -1,20 +1,19 @@
-import chalk
 import importlib
 import os
 import time
-
 import pickle
-
 from os import environ
 from socket import AF_INET
 
+import chalk
+from twisted.internet import reactor
+
 from hendrix import defaults
+
 from hendrix.options import options as hx_options
 from hendrix.resources import get_additional_resources
-from hendrix.services import get_additional_services, HendrixService
+from hendrix.resources.services import get_additional_services, HendrixService, TCPServer
 from hendrix.utils import get_pid
-from twisted.application.internet import TCPServer, SSLServer
-from twisted.internet import reactor
 
 
 class HendrixDeploy(object):
@@ -59,7 +58,8 @@ def __init__(self, action='start', options={}, reactor=reactor):
             django = importlib.import_module('django')
             if django.VERSION[:2] >= (1, 7):
                 installed_apps = getattr(settings, "INSTALLED_APPS")
-                django.apps.apps.populate(installed_apps)
+                from django.apps import apps
+                apps.populate(installed_apps)
             wsgi_dot_path = getattr(settings, 'WSGI_APPLICATION', None)
             self.application = HendrixDeploy.importWSGI(wsgi_dot_path)
 
@@ -123,6 +123,7 @@ def addHendrix(self):
         )
 
     def catalogServers(self, hendrix):
+        from hendrix.contrib.ssl import SSLServer
         "collects a list of service names serving on TCP or SSL"
         for service in hendrix.services:
             if isinstance(service, (TCPServer, SSLServer)):

hendrix/contrib/async/crosstown_traffic.py → hendrix/experience/crosstown_traffic.py

@@ -1,5 +1,6 @@
 from twisted.internet.threads import deferToThread, deferToThreadPool
 from twisted.internet import reactor
+from twisted.python.threadpool import ThreadPool
 
 import threading
 
@@ -8,12 +9,22 @@
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
+
+class ThreadHasNoResponse(RuntimeError):
+    pass
+
+
 def get_response_for_thread(thread=None):
 
     if not thread:
         thread = threading.current_thread()
 
-    response = thread.response_object
+    try:
+        response = thread.response_object
+    except AttributeError:
+        raise ThreadHasNoResponse('thread %s has no associated response object.' % thread)
+
+
     return response
 
 
@@ -25,30 +36,42 @@ def get_tasks_to_follow_current_response(thread=None):
 class ThroughToYou(object):
 
     def __init__(self,
+                 reactor=reactor,
                  same_thread=False,
                  no_go_status_codes=['5xx', '4xx'],
-                 reactor=reactor,
+                 fail_without_response=False
                  ):
+        self.reactor = reactor
         self.same_thread = same_thread
         self.no_go_status_codes = no_go_status_codes
-        self.reactor = reactor
+        self.fail_without_response = fail_without_response
+
         self.no_go = False
         self.status_code = None
 
-    def __call__(self, crosstown_task):
+    def __call__(self, crosstown_task=None):
         self.crosstown_task = crosstown_task
-        self.response = get_response_for_thread()
 
-        if not self.no_go:
-            logger.info("Adding '%s' to crosstown_traffic for %s" % (crosstown_task.__name__, self.response))
-            self.response.crosstown_tasks.append(self)
+        try:
+            self.response = get_response_for_thread()
+            logger.info("Adding '%s' to crosstown_traffic for %s" % (str(crosstown_task), self.response))
+            if not self.no_go:
+                self.response.crosstown_tasks.append(self)
+        except ThreadHasNoResponse:
+            if self.fail_without_response:
+                raise
+            else:
+                logger.info("thread %s has no response; running crosstown task now.  To supress this behavior, set fail_without_response == True." % threading.current_thread())
+                self.run()
+        return self.run
 
-    def run(self, threadpool):
-        # See if status code is a go
-        self.check_status_code_against_no_go_list()
+    def run(self, threadpool=None):
         if self.no_go:
             return
 
+        if not threadpool:
+            threadpool = reactor.threadpool or ThreadPool()
+
         if self.same_thread:
             self.crosstown_task()
         else:
@@ -93,4 +116,5 @@ def __call__(self, *args, **kwargs):
         decorator = ThroughToYou(*args, **kwargs)
         return decorator
 
+
 follow_response = FollowResponse()

hendrix/resources.py → hendrix/resources/__init__.py

@@ -1,18 +1,15 @@
 import sys
 import importlib
-import inspect
-from hendrix.utils import responseInColor
-from hendrix.contrib.async import crosstown_traffic
+import logging
+import threading
 
 from twisted.web import resource, static
 from twisted.web.server import NOT_DONE_YET
 from twisted.web.wsgi import WSGIResource, _WSGIResponse
-
-import logging
 import chalk
-from twisted.internet.threads import deferToThread
-import threading
-import  uuid
+
+from hendrix.utils import responseInColor
+
 
 logger = logging.getLogger(__name__)
 
@@ -33,12 +30,17 @@ def run(self, *args, **kwargs):
         self.request.setHeader('server', 'hendrix/Twisted')
         ran = super(HendrixWSGIResponse, self).run(*args, **kwargs)
         self.follow_response_tasks()
+        del self.thread.response_object
         return ran
 
     def follow_response_tasks(self):
 
         for task in self.crosstown_tasks:
-            logger.info("Processing crosstown task: '%s'" % task)
+            logger.info("Processing crosstown task: '%s'" % task.crosstown_task)
+
+            # Set no-go if status code is bad.
+            task.check_status_code_against_no_go_list()
+
             task.run(self.threadpool)
 
 class LoudWSGIResponse(HendrixWSGIResponse):

hendrix/services.py → hendrix/resources/services.py

@@ -1,11 +1,13 @@
 import importlib
-from .resources import HendrixResource
+import logging
+
 from twisted.application import internet, service
 from twisted.internet import reactor
 from twisted.python.threadpool import ThreadPool
 from twisted.web import server
 
-import logging
+from hendrix.resources import HendrixResource
+
 
 logger = logging.getLogger(__name__)
 

hendrix/test/resources.py

@@ -1,14 +1,15 @@
+#! python
+
 from multiprocessing import Queue
 import threading
 import logging
 
-from hendrix.contrib.async import crosstown_traffic
+from hendrix.experience import crosstown_traffic
 
-logger = logging.getLogger(__name__)
 
+logger = logging.getLogger(__name__)
 
-from hendrix.contrib.async.crosstown_traffic import get_response_for_thread,\
-    get_tasks_to_follow_current_response
+from hendrix.experience.crosstown_traffic import get_response_for_thread,   get_tasks_to_follow_current_response
 
 
 class TestNameSpace(object):

hendrix/test/test_crosstown_traffic.py

@@ -10,7 +10,8 @@
 from twisted.python.threadpool import ThreadPool
 from twisted.web.test.requesthelper import DummyRequest
 
-from hendrix.contrib.async import crosstown_traffic
+from hendrix.experience import crosstown_traffic
+from hendrix.experience.crosstown_traffic import ThreadHasNoResponse
 from hendrix.resources import HendrixWSGIResource
 from hendrix.test.resources import TestNameSpace, application as wsgi_application,\
     nameSpace
@@ -145,6 +146,12 @@ class PostResponseTest(TestCase):
     def setUp(self):
         nameSpace = TestNameSpace()
 
+    def tearDown(self):
+        try:
+            del threading.current_thread().response_object
+        except AttributeError:
+            pass
+
     def test_postiive_decorator_coherence(self):
         self.pass_flag = False
 
@@ -172,8 +179,8 @@ class FakeResponse(object):
             crosstown_tasks = []
             status = "418 I'm a teapot.  Seriously."
 
-        through_to_you = crosstown_traffic.follow_response(same_thread=True)
         threading.current_thread().response_object = FakeResponse()
+        through_to_you = crosstown_traffic.follow_response(same_thread=True)
 
         through_to_you.no_go = True  # If no_go is True...
         through_to_you(append_me_to_pass)  # and we call it...
@@ -185,6 +192,30 @@ class FakeResponse(object):
                          append_me_to_pass
                          )  # We will have added the function.
 
+    def test_with_no_request(self):
+
+        self.has_run = False
+
+        def append_me_to_pass():
+            self.has_run = True
+
+        through_to_you = crosstown_traffic.follow_response(same_thread=True)
+        through_to_you(append_me_to_pass)
+
+        self.assertTrue(self.has_run)
+
+    def test_fail_without_response(self):
+        '''
+        Same test as above, but with fail_without_response, we get an error.
+        '''
+        self.has_run = False
+
+        def append_me_to_pass():
+            self.has_run = True
+
+        through_to_you = crosstown_traffic.follow_response(same_thread=True, fail_without_response=True)
+
+        self.assertRaises(ThreadHasNoResponse, through_to_you, append_me_to_pass)
 
     def test_contemporaneous_requests(self):
 

hendrix/test/test_resources.py

@@ -1,10 +1,11 @@
-import mock
 import unittest
 
-from hendrix.resources import HendrixResource, NamedResource, WSGIResource
 from twisted.web.resource import getChildForRequest, NoResource
 from twisted.web.test.requesthelper import DummyRequest
 
+import mock
+from hendrix.resources import HendrixResource, NamedResource, WSGIResource
+
 
 class TestHendrixResource(unittest.TestCase):
 

hendrix/test/test_testing_utils.py

@@ -0,0 +1,49 @@
+from hendrix.experience import crosstown_traffic
+from hendrix.utils.test_utils import AsyncTestMixin
+from twisted.trial.unittest import TestCase
+
+
+
+class TestMixinAssertions(TestCase):
+
+    def test_assert_num_tasks(self):
+
+        a_mixin = AsyncTestMixin()
+        a_mixin.assertNumCrosstownTasks(0)
+
+        def some_task():
+            pass
+
+        through_to_you = crosstown_traffic.follow_response()
+        through_to_you(some_task)
+
+        a_mixin.assertNumCrosstownTasks(1)
+
+    def test_next_task(self):
+
+        a_mixin = AsyncTestMixin()
+        a_mixin.assertNumCrosstownTasks(0)
+
+        def some_task():
+            pass
+
+        through_to_you = crosstown_traffic.follow_response()
+        through_to_you(some_task)
+
+        self.assertIs(some_task, a_mixin.next_task())
+
+    def test_no_more_tasks(self):
+
+        a_mixin = AsyncTestMixin()
+        a_mixin.assertNumCrosstownTasks(0)
+
+        def some_task():
+            pass
+
+        through_to_you = crosstown_traffic.follow_response()
+        through_to_you(some_task)
+
+        same_task = a_mixin.next_task()
+
+        # That will be the only (and thus last) task.
+        self.assertRaises(StopIteration, a_mixin.next_task)

hendrix/ux.py

@@ -13,7 +13,16 @@
 import traceback
 from .options import HendrixOptionParser, cleanOptions
 from hendrix.contrib import SettingsError
-from hendrix.deploy import base, ssl, cache, hybrid
+from hendrix.deploy import base, cache
+
+import logging
+logger = logging.getLogger(__name__)
+
+try:
+    from hendrix.deploy import ssl, hybrid
+except ImportError:
+    logger.warning("hendrix SSL is not available.")
+
 from watchdog.observers import Observer
 from watchdog.events import FileSystemEventHandler
 

runtests.py

@@ -0,0 +1,13 @@
+#! python
+
+import os, sys
+
+# begin chdir armor
+sys.path[:] = map(os.path.abspath, sys.path)
+# end chdir armor
+
+sys.path.insert(0, os.path.abspath(os.getcwd()))
+sys.argv.append("hendrix/test")
+
+from twisted.scripts.trial import run
+run()

setup.py

@@ -69,5 +69,5 @@ def mkdir_p(path):
         (share_path, ['hendrix/utils/templates/init.d.j2', ]),
     ],
     install_requires=readlines('requirements'),
-    extras_require={'ssl': ['pyopenssl', ]}
+            extras_require={'ssl': ['pyopenssl', ]}
 )