From 6480bfc877a7e9275fb8a6f94ff2fa558388a31c Mon Sep 17 00:00:00 2001 From: Jocelyn Boullier Date: Sun, 13 Jan 2013 15:21:48 +0100 Subject: [PATCH 1/3] Shell autoreloading functionality from @Bpless ported to Flask-Script. --- flask_script/commands.py | 236 +++++++++++++++++++++++++++++++++++---- 1 file changed, 216 insertions(+), 20 deletions(-) diff --git a/flask_script/commands.py b/flask_script/commands.py index fc9e0eb..bed780d 100644 --- a/flask_script/commands.py +++ b/flask_script/commands.py @@ -1,17 +1,59 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import from __future__ import with_statement +import atexit +import inspect import os import code import warnings +import logging + import argparse from flask import _request_ctx_stack from .cli import prompt, prompt_pass, prompt_bool, prompt_choices +try: + import importlib +except ImportError: + importlib = None + +_listener_enabled = False +FileSystemEventHandler = object + +if importlib is not None: + try: + from watchdog.events import FileSystemEventHandler + except ImportError: + pass + + try: + from watchdog.observers import Observer + + _listener_enabled = True + except ImportError: + pass + else: + # Threading object which listens for file system changes + # Keep a reference to this object in the global scope + # So that we can join it on program exit + # Failing to do so raises ugly Threading exceptions + # We may choose to just suppress stderr instead + # Since waiting for thread to end seems to take time (~1sec) + observer_thread = Observer() + + def kill_observer_thread(): + # Clean up the filesystem listener thread on program exit + # This seems to add roughly one second to shut down, so it's not ideal + if observer_thread.is_alive(): + observer_thread.stop() + observer_thread.join() + + # Register functions to be called when Python program ends normally + atexit.register(kill_observer_thread) class InvalidCommand(Exception): pass @@ -45,9 +87,9 @@ def __init__(self, *options, **kwargs): self.required = kwargs.pop("required", None) if ((self.title or self.description) and - (self.required or self.exclusive)): + (self.required or self.exclusive)): raise TypeError("title and/or description cannot be used with " - "required and/or exclusive.") + "required and/or exclusive.") super(Group, self).__init__(**kwargs) @@ -120,13 +162,13 @@ def create_parser(self, prog): if isinstance(option, Group): if option.exclusive: group = parser.add_mutually_exclusive_group( - required=option.required, - ) + required=option.required, + ) else: group = parser.add_argument_group( - title=option.title, - description=option.description, - ) + title=option.title, + description=option.description, + ) for opt in option.get_options(): group.add_argument(*opt.args, **opt.kwargs) else: @@ -191,6 +233,10 @@ class Shell(Command): :param use_ipython: use IPython shell if available, ignore if not. The IPython shell can be turned off in command line by passing the **--no-ipython** flag. + :param use_reloader: auto-reload the project if watchdog is available + by watching the root_path from app. + The auto-reload can be turned off in command + line by passing the **-no-reload** flag. """ banner = '' @@ -198,11 +244,11 @@ class Shell(Command): description = 'Runs a Python shell inside Flask application context.' def __init__(self, banner=None, make_context=None, use_ipython=True, - use_bpython=True): - + use_bpython=True, use_reloader=True): self.banner = banner or self.banner self.use_ipython = use_ipython self.use_bpython = use_bpython + self.use_reloader = use_reloader if make_context is None: make_context = lambda: dict(app=_request_ctx_stack.top.app) @@ -212,13 +258,17 @@ def __init__(self, banner=None, make_context=None, use_ipython=True, def get_options(self): return ( Option('--no-ipython', - action="store_true", - dest='no_ipython', - default=not(self.use_ipython)), + action="store_true", + dest='no_ipython', + default=not (self.use_ipython)), Option('--no-bpython', - action="store_true", - dest='no_bpython', - default=not(self.use_bpython)) + action="store_true", + dest='no_bpython', + default=not (self.use_bpython)), + Option('--no-reload', + action="store_true", + dest='no_reloader', + default=not (self.use_reloader)) ) def get_context(self): @@ -227,7 +277,7 @@ def get_context(self): """ return self.make_context() - def run(self, no_ipython, no_bpython): + def run(self, no_ipython, no_bpython, no_reloader): """ Runs the shell. If no_bpython is False or use_bpython is True, then a BPython shell is run (if installed). Else, if no_ipython is False or @@ -235,17 +285,40 @@ def run(self, no_ipython, no_bpython): """ context = self.get_context() + + if not no_reloader: + if not _listener_enabled: + print("If you want to use the auto-reloading functionality, you need to install watchdog " + "(you can do it using pip)") + else: + autoreload_path = _request_ctx_stack.top.app.root_path + + if not autoreload_path: + raise InvalidCommand("To use the auto-reload, the root_path variable of app must be available.") + else: + print("Using this path as project root: {path}".format(path=autoreload_path)) + + def listen_for_changes(shell_globals, project_root): + # Begin thread which listens for file system changes via Watchdog library + event_handler = ReloaderEventHandler(project_root=project_root, shell_globals=shell_globals) + observer_thread.schedule(event_handler, path=project_root, recursive=True) + observer_thread.start() + + listen_for_changes(context, autoreload_path) + if not no_bpython: # Try BPython try: from bpython import embed - embed(banner=self.banner, locals_=self.get_context()) + + embed(banner=self.banner, locals_=context) return except ImportError: # Try IPython if not no_ipython: try: import IPython + try: sh = IPython.Shell.IPShellEmbed(banner=self.banner) except AttributeError: @@ -259,6 +332,129 @@ def run(self, no_ipython, no_bpython): code.interact(self.banner, local=context) +class ReloaderEventHandler(FileSystemEventHandler): + """ + Listen for changes to modules within the Flask project + On change, reload the module in the Python Shell + + Almost everything here come from + https://github.com/Bpless/django-extensions/commit/0078b4a513b0035b97820c0788ba990d20f80d48#L2R142 + Thanks to him ! + """ + + def __init__(self, *args, **kwargs): + self.project_root = kwargs.pop('project_root', None) + self.shell_globals = kwargs.pop('shell_globals', None) + self.model_globals = kwargs.pop('model_globals', None) + super(ReloaderEventHandler, self).__init__(*args, **kwargs) + + def dispatch(self, event): + event_path = event.src_path + path, file_extension = os.path.splitext(event_path) + if all([ + file_extension == '.py', + 'shell_plus' not in path, + self.project_root in path + ]): + return super(ReloaderEventHandler, self).dispatch(event) + + def on_created(self, event): + super(ReloaderEventHandler, self).on_created(event) + self._force_reload(event) + + def on_modified(self, event): + """ + Called by dispatch on modification of file in the Flask project + """ + super(ReloaderEventHandler, self).on_modified(event) + self._force_reload(event) + + def _force_reload(self, event): + """ + Reload the altered module + """ + cleaned_path = self._clean_path(event.src_path) + path_components = cleaned_path.split(os.path.sep) + self._reload_module(path_components) + + def _clean_path(self, path): + """Remove the leading project path""" + project_root = self.project_root if self.project_root.endswith('/') else "{}/".format(self.project_root) + path_from_project_root = path.replace(project_root, '') + # Remove trailing ".py" from module for importing purposes + return os.path.splitext(path_from_project_root)[0] + + def _reload_module(self, path_components): + """ + Wrapper for __builtin__ reload() function + In addition to reloading the module, + we reset the associated classes in the global scope of the shell. + + Consequently, we don't have to manaully reimport (i.e. 'from app import MyClass') + Instead, MyClass will have changed for us automagically + + More interestingly, we also dynamically update the classes + of existing object instances in the global scope with `_update_class_instances`. + + ## In a Shell session + obj = MyKLS() + obj.getchar() --> 'a' + + ## While still in the Shell, + ### We change the function definition of getchar() in the filesytem to return 'b' + ### In our Shell, we will see that + + obj.getchar() --> 'b' + + This behavior is very experimental and possibly dangerous but powerful + Cuts down time and frustration during pdb debugging + """ + + # We attempt to import the module from the project root + # This SHOULD be agnostic of app/project structure + while True: + try: + module = importlib.import_module('.'.join(path_components)) + except ImportError: + path_components.pop(0) + if not path_components: + return + else: + break + + reload(module) + # Reload objects into the global scope + # This has the potential to cause namespace collisions + # The upside is that we don't have to reimport (i.e. from module import ObjName) + for attr in dir(module): + if ( + not (attr.startswith('__') and attr.endswith('__')) + and self.shell_globals.get(attr) + ): + self.shell_globals[attr] = getattr(module, attr) + self._update_class_instances(module, attr) + + + def _update_class_instances(self, module, attr): + """ + Reset the __class__ of all instances whoses + class has been reloaded into the shell + + This allows us to do CRAZY things such as + effectively manipulate an instance's source code + while inside a debugger + """ + module_obj = getattr(module, attr) + if inspect.isclass(module_obj): + for obj in self.shell_globals.values(): + # hasattr condition attempts to handle old style classes + # The class __str__ check may not be ideal but it seems to work + # The one exception being if you changes the __str__ method + # of the reloaded object. That edge case is not handled + if hasattr(obj, '__class__') and str(obj.__class__) == str(module_obj): + obj.__class__ = module_obj + + class Server(Command): """ Runs the Flask development server i.e. app.run() @@ -283,7 +479,6 @@ class Server(Command): def __init__(self, host='127.0.0.1', port=5000, use_debugger=True, use_reloader=True, threaded=False, processes=1, passthrough_errors=False, **options): - self.port = port self.host = host self.use_debugger = use_debugger @@ -294,7 +489,6 @@ def __init__(self, host='127.0.0.1', port=5000, use_debugger=True, self.passthrough_errors = passthrough_errors def get_options(self): - options = ( Option('-t', '--host', dest='host', @@ -365,6 +559,7 @@ def handle(self, app, host, port, use_debugger, use_reloader, class Clean(Command): "Remove *.pyc and *.pyo files recursively starting at current directory" + def run(self): for dirpath, dirnames, filenames in os.walk('.'): for filename in filenames: @@ -378,6 +573,7 @@ class ShowUrls(Command): """ Displays all of the url matching routes for the project. """ + def __init__(self, order='rule'): self.order = order @@ -387,7 +583,7 @@ def get_options(self): dest='order', default=self.order, help='Property on Rule to order by (default: %s)' % self.order, - ), + ), return options From c0f03be67eb552cafa87714c69350f2b7e076a09 Mon Sep 17 00:00:00 2001 From: Jocelyn Boullier Date: Tue, 22 Jan 2013 18:01:22 +0100 Subject: [PATCH 2/3] Remove model_global which is not needed anymore. --- flask_script/commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flask_script/commands.py b/flask_script/commands.py index bed780d..d5634ed 100644 --- a/flask_script/commands.py +++ b/flask_script/commands.py @@ -345,7 +345,6 @@ class ReloaderEventHandler(FileSystemEventHandler): def __init__(self, *args, **kwargs): self.project_root = kwargs.pop('project_root', None) self.shell_globals = kwargs.pop('shell_globals', None) - self.model_globals = kwargs.pop('model_globals', None) super(ReloaderEventHandler, self).__init__(*args, **kwargs) def dispatch(self, event): From e89adf8f220d1e2c1a8b2b2eec543085ae704aea Mon Sep 17 00:00:00 2001 From: Jocelyn Boullier Date: Sat, 9 Mar 2013 16:54:28 +0100 Subject: [PATCH 3/3] Temp commit --- .gitignore | 3 ++- flask_script/commands.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 62ba12a..60d4cf3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ dist/ *.swp *.tox *.zip -*.sublime-* \ No newline at end of file +*.sublime-* +.idea diff --git a/flask_script/commands.py b/flask_script/commands.py index d5634ed..a7c8abf 100644 --- a/flask_script/commands.py +++ b/flask_script/commands.py @@ -55,6 +55,7 @@ def kill_observer_thread(): # Register functions to be called when Python program ends normally atexit.register(kill_observer_thread) + class InvalidCommand(Exception): pass @@ -433,7 +434,6 @@ def _reload_module(self, path_components): self.shell_globals[attr] = getattr(module, attr) self._update_class_instances(module, attr) - def _update_class_instances(self, module, attr): """ Reset the __class__ of all instances whoses