Permalink
Browse files

Change behaviour of code download

Now, if a module already exists and is importable, code will not be
downloaded. If it isn't available, it will download.
  • Loading branch information...
1 parent 3a9b200 commit e8d987b999e9349ee60fce449e1ff58e0e16d2fd @oldpatricka oldpatricka committed Oct 30, 2012
Showing with 68 additions and 19 deletions.
  1. +25 −5 pidantic/nosetests/pyon_tests.py
  2. +1 −1 pidantic/pyon/pidpyon.py
  3. +42 −13 pidantic/pyon/pyon.py
@@ -1,6 +1,7 @@
import os
from uuid import uuid4
from tempfile import mkstemp
+from importlib import import_module
import pidantic.pyon.pyon as pyon
from pidantic.pyon.persistence import PyonDB, PyonDataObject, PyonProcDataObject
@@ -68,13 +69,30 @@ def test_load_module(self):
testdir = os.path.dirname(__file__)
testfile = os.path.join(testdir, "module_to_download.py")
- module_name = self.pyon.load_module(testfile, "modname")
+ module_name = self.pyon.load_module("modname", testfile)
- mod = __import__(module_name)
+ mod = import_module(module_name)
tp = mod.TestProcess()
assert tp.gettrue()
+ # Test module that should already exist
+ module_name = self.pyon.load_module("os.path", testfile)
+ print module_name
+ mod = import_module(module_name)
+ print dir(mod)
+ assert hasattr(mod, 'join')
+
+ # Test module that doesn't exist, no file
+ raised = False
+ try:
+ module_name = self.pyon.load_module("somefakemodule")
+ except ImportError:
+ raised = True
+ assert raised, "ImportError wasn't raised on nonexistant module"
+
+
+
def test_run(self):
testdir = os.path.dirname(__file__)
testfile = os.path.join(testdir, "module_to_download.py")
@@ -87,13 +105,15 @@ def test_run(self):
config_yaml = '{"some": "config"}'
config = {"some": "config"}
name = "myprocname"
+ pyon_process_id = "mypyonproc"
process = self.pyon.create_process_db(directory=directory,
module=module, module_uri=module_uri, cls=cls,
- config=config_yaml, pyon_name=name)
- pyon_id = self.pyon.run_process(process, async=False)
+ config=config_yaml, pyon_name=name, pyon_process_id=pyon_process_id)
+ self.pyon.run_process(process, async=False)
- pyon_proc = self.pyon_container.proc_manager[pyon_id]
+ print self.pyon_container.proc_manager.keys()
+ pyon_proc = self.pyon_container.proc_manager[pyon_process_id]
assert self.pyon.getState()
View
@@ -2,7 +2,7 @@
import logging
import threading
-from pidantic.pyon.pyon import Pyon, FAILED_PROCESS
+from pidantic.pyon.pyon import Pyon
from pidantic.state_machine import PIDanticState, PIDanticEvents
from pidantic.ui import PidanticFactory
View
@@ -7,17 +7,22 @@
from urllib import urlretrieve
from uuid import uuid4
from gevent import spawn
+from importlib import import_module
+
-from eeagent.util import unmake_id
from pidantic.pyon.persistence import PyonDataObject, PyonProcDataObject
from pidantic.pidantic_exceptions import PIDanticUsageException, PIDanticExecutionException
+try:
+ from interface.objects import ProcessStateEnum
+except ImportError:
+ ProcessStateEnum = object()
+
+CACHE_DOWNLOADS = True
-from interface.objects import ProcessStateEnum
class FakeIonService(object):
id = None
-FAILED_PROCESS = "failed process"
def proc_manager_lock(func):
def call(self, *args, **kwargs):
@@ -99,8 +104,9 @@ def create_process_db(self, **kwargs):
def run_process(self, process_object, async=True, state_change_callback=None):
if process_object.module_uri is not None:
- module_file = self.download_module(process_object.module_uri)
- module = self.load_module(module_file, process_object.module)
+ module = self.load_module(process_object.module,
+ module_uri=process_object.module_uri)
+
process_object.module = module
self._pyon_db.db_commit()
@@ -130,20 +136,43 @@ def _run_process(self, process_object, state_change_callback=None):
def download_module(self, module_uri):
try:
module_file, _ = urlretrieve(module_uri)
- except:
+ except Exception:
msg = "Unable to download code module %s" % module_uri
self._log.exception(msg)
raise PIDanticExecutionException(msg)
return module_file
- def load_module(self, module_file, module):
- module_name = self._unique_module_name(module)
+ def load_module(self, module, module_uri=None):
+ """load_module
+
+ Attempts to load module by name, if it is not available,
+ loads it by url
+ """
+ # Check to see that module isn't availble in python path
try:
- load_source(module_name, module_file)
- except:
- msg = "Unable to load code module %s" % module
- self._log.exception(msg)
- raise PIDanticExecutionException(msg)
+ import_module(module)
+ except ImportError, e:
+ if module_uri is None:
+ raise e
+ else:
+ return module
+
+ if not CACHE_DOWNLOADS:
+ module_name = self._unique_module_name(module)
+ else:
+ module_name = self._sanitize_module_name(module)
+
+ # Check to see whether we've already downloaded, otherwise, download
+ try:
+ import_module(module_name)
+ except ImportError:
+ module_file = self.download_module(module_uri)
+ try:
+ load_source(module_name, module_file)
+ except Exception:
+ msg = "Unable to load code module %s" % module
+ self._log.exception(msg)
+ raise PIDanticExecutionException(msg)
return module_name

0 comments on commit e8d987b

Please sign in to comment.