Skip to content
Browse files

Changes, bug fixes, more builtins

  • Loading branch information...
1 parent 4c86d16 commit 972e9d77afde84c8293cf8362d855ba16de3c854 Louis Sobel committed
Showing with 229 additions and 164 deletions.
  1. +23 −15 drapache/dbapiio.py
  2. +20 −11 drapache/dbapijinja.py
  3. +4 −2 drapache/dbapiserver.py
  4. +155 −94 drapache/dbpybuiltins.py
  5. +8 −33 drapache/dbpyexecute.py
  6. +18 −7 drapache/httpserver.py
  7. +1 −2 drapache/index_generator.py
View
38 drapache/dbapiio.py
@@ -36,6 +36,11 @@ def write_error(self,*args,**kwargs):
class LiveDropboxFile(StringIO.StringIO):
+ """
+ An in-memory file object representing a dropbox file
+ It is 'live' in the sense that it once it is closed, all changes made are
+ reflected to dropbox. So it's not really live, but under sufficient pre-conditions
+ (locking the file) it will be. A leaky abstraction I guess."""
def __init__(self,path,client,download=True):
@@ -50,22 +55,25 @@ def __init__(self,path,client,download=True):
else:
StringIO.StringIO.__init__(self)
+
+ def is_open(self):
+ return self.__open
+
def _update(self,client):
if self.__open:
self.seek(0)
client.put_file(self.path,self,overwrite=True)
- def close(self,locker=None):
+ def _close(self,locker):
if self.__open:
- if locker is not None:
- try:
- self._update(locker.client)
- finally:
- locker.release(self.path)
- self.__open = False
+ try:
+ self._update(locker.client)
+ finally:
+ locker.release(self.path)
+ self.__open = False
else:
pass
#this allows for mutliple accidental callings .close()
@@ -75,11 +83,11 @@ class WritableDropboxFile(LiveDropboxFile):
def __init__(self,path,client,download=True,mode='append'):
#mode is either write or append
- #if append, we have to download if download is true
+ #if append, we have to download if download is true, which generally will
+ #be set by the caller to True only if the file exists
#so we only have to download the file if download is true and the mode is append
-
do_download = (download and mode == 'append')
- LiveDropboxFile.__init__(self,path,client,do_download,mode)
+ LiveDropboxFile.__init__(self,path,client,do_download)
if mode == 'append':
self.seek(0,2)
@@ -95,7 +103,7 @@ def _update(self,client):
def write(self,what):
- if not self.__open:
+ if not self.is_open():
raise IOError('Cannot write to a closed file')
if self.mode == 'append':
@@ -107,7 +115,7 @@ def writeline(self,line):
def writelines(self,sequence):
- if not self.__open:
+ if not self.is_open():
raise IOError('Cannot write to a closed file!')
if self.mode == 'append':
@@ -134,7 +142,7 @@ def _update(self,client):
self.seek(0)
try:
- json.dump(self.json_object,self)
+ json.dump(self.json_object,self,indent=4)
self.truncate()
except TypeError:
self.seek(0)
@@ -223,7 +231,7 @@ def lock(self,path,timeout=None):
flag_list.sort()
if not flag_list:
- raise AssertionError('There should not be an empty list of flags at this point')
+ raise IOError('There should not be an empty list of flags at this point')
first_uid = flag_list[0][1]
@@ -261,7 +269,7 @@ def lock(self,path,timeout=None):
def close_all(self,):
for file_h in self.open_files:
- file_h.close(locker=self)
+ file_h._close(self)
def register_open_file(self,file_h):
self.open_files.append(file_h)
View
31 drapache/dbapijinja.py
@@ -5,33 +5,42 @@
import os
import dropbox
+class TemplateNotFound(Exception):
+ pass
+
class DropboxLoader(jinja2.BaseLoader):
def __init__(self,client,search_root):
self.client = client
- self.search_root = search_root
-
-
- def get_source(self,environment,template):
-
- template_path = os.path.join(self.search_root,template)
+ self.search_root = search_root
+ def get_source(self,environment,path):
+
+
+ template_path = self.search_root + path
+
try:
f = self.client.get_file(template_path).read()
except dropbox.rest.ErrorResponse as e:
- raise jinja2.TemplateNotFound(template)
+ if e.status == 404:
+ raise jinja2.TemplateNotFound(template_path)
+ else:
+ raise IOError("Error connecting to dropbox to download template")
return f,template_path,True
-
-def render_dropbox_template(client,template,data,search_root):
+def render_dropbox_template(client,template_path,data):
+
+ search_root,path = template_path.rsplit('/',1)
+ search_root += '/'
+
env = jinja2.Environment(loader=DropboxLoader(client,search_root))
try:
- template = env.get_template(template)
+ template = env.get_template(path)
output = template.render(**data)
except jinja2.TemplateNotFound as e:
- output = "Template %s not found; looking in %s" % (template,search_root)
+ raise TemplateNotFound()
return output
View
6 drapache/dbapiserver.py
@@ -97,6 +97,9 @@ def _serve_static(self,file_meta):
"""
path = file_meta['path']
f = self.client.get_file(path).read()
+ if f.startswith('#DBPYEXECUTE'):
+ param_dict = dict(client=self.client,request=self.request)
+ return dbpyexecute.execute(f,**param_dict)
headers = {'Content-type':self._get_content_type(file_meta)}
return ResponseObject(200,f,headers)
@@ -108,11 +111,10 @@ def _serve_python(self,file_meta):
#allows these files to be shared without getting executed
headers = {'Content-type':'text/plain'}
return ResponseObject(200,f,headers)
-
param_dict = dict(client=self.client,request=self.request)
-
return dbpyexecute.execute(f,**param_dict)
+
def _find_and_serve_index(self,directory_meta,path):
"""
View
249 drapache/dbpybuiltins.py
@@ -20,19 +20,14 @@
import os.path
import markdown
+import json
import StringIO
from functools import wraps
-#### for some reason, we need to preload strptime.
-#### this is a weird bug
-#### without this block, I was getting "cannot unmarshal code objects in restricted execution mode"
-try:
- time.strptime('dfdf')
-except:
+class UserDieException(Exception):
pass
-
@@ -49,127 +44,148 @@ def get_builtins(**kwargs):
request = kwargs['request']
response = kwargs['response']
+
get_params = request.get_params or {}
+ post_params = request.post_params or {}
+
request_folder = request.folder
session = kwargs['session']
sandbox = kwargs['sandbox']
+ #goo... but i need state that is mutable
+ #by any privileged function... this dict becomes visible to all privileged
+ #function. this allows the nesting of privilegeds
+ in_sandbox = {'is':True}
+
+
+ built_in_hash = {'GETPARAMS':get_params,'POSTPARAMS':post_params,'SESSION':None}
+
- built_in_hash = {'GETPARAMS':get_params,'SESSION':None}
def register(function):
-
-
+ built_in_hash[function.func_name] = function
+ return function
+
+ def privileged(function):
+ """
+ A decorator that replaces the given function with
+ one that first takes the current frame out of the sandbox
+ and then executes the function, finally replaces the protections of the sandbox
+
+ There are some hacks that cater to the way that pysandbox (which is awesome) was written
+
+ And a dictionary was defined (in_sandbox) at the same scope os the function itself... this acts as
+ a global flag whether or not sandbox is currently enabled. This allows the nesting of privileged functions
+ """
def outer_wrapper(*args,**kwargs):
retval = None
-
-
+
+ unrolled = False
try:
#before I disable protections and restore privileged builtins,
#i need to change the frame that I am acting on to the current one
#instead of whatever frame enable was called in
- #find the builrin protection and set its frame
- for p in reversed(sandbox.protections):
- if p.__class__.__name__ == 'CleanupBuiltins':
- p.frame = sys._getframe()
- p.disable(sandbox)
+ #find the builtin protection and set its frame
+ if in_sandbox['is']:
+ for p in reversed(sandbox.protections):
+ if p.__class__.__name__ == 'CleanupBuiltins':
+ p.frame = sys._getframe()
+ p.disable(sandbox)
+ unrolled = True
+ in_sandbox['is'] = False
- sys.stderr.write('RIGHTHERE'+str(open)+'\n')
retval = function(*args,**kwargs)
finally:
#redo the protection
- sys.stderr.write('rolling with modules:%s\n'%str(len(sys.modules)))
#enable for the builtin protection grabs the frame 2 up from enable
- #i want it to enable the protections with outer_wrapper, which is now privileged
+ #i want it to enable the protections in the outer_wrapper frame, which is now privileged
#this ensures that privileged builtins are restored in the next disable
- #so instead of this acting the register frame, I wrap it in a function
- #so it acts on outer_wrapper
- def enable_protections():
- for p in sandbox.protections:
- p.enable(sandbox)
- enable_protections()
+ #so instead of this acting on the 'privileged' frame, I wrap it in a function
+ #to push it one place lower in the stack frame so it acts on outer_wrapper
+ if unrolled:
+ def enable_protections():
+ for p in sandbox.protections:
+ p.enable(sandbox)
+ enable_protections()
+ in_sandbox['is'] = True
return retval
-
- built_in_hash[function.func_name] = outer_wrapper
- return function
-
- def register2():
-
- def reg(function):
- register(function)
- bar = built_in_hash[function.func_name]
- def doit():
- bar()
- built_in_hash[function.func_name] = doit
- return doit
-
- return reg
+ #hack to make privileged functions compatible with register
+ outer_wrapper.func_name = function.func_name
+ return outer_wrapper
- def register_with_postop(postop):
+ def privileged_with_callback(callback,before=False):
+ """
+ A decorator factory that returns a decorator that wraps the function
+ by privileging it, and composing it with the unprivileged callback
- #postop will happen in a protected environment again
+ if before is True (false by default) the callback function will actually get executed *before* the privileged one
+ """
- def register2(function):
+ def outer_decorator(function):
- def outer_wrapper(*args,**kwargs):
-
- retval = None
- exception_occured = False
- try:
- old_modules = sandbox.protections[1].modules_dict
- old_builtins = sandbox.protections[1].builtin_dict
- for p in reversed(sandbox.protections):
- p.disable(sandbox)
-
- retval = function(*args,**kwargs)
-
- sys.stderr.write('just called function itself\n')
-
- finally:
- #redo the protection
- sys.stderr.write('unrolling (postop) with modules:%s\n'%str(len(sys.modules)))
- for p in sandbox.protections:
- p.enable(sandbox)
-
- #we only get here if there isn't an exception
- return postop(retval)
-
-
+ function_p = privileged(function)
- built_in_hash[function.func_name] = outer_wrapper
-
- return function
+ if before:
+ def outer_wrapper(*args,**kwargs):
+ return function_p(callback(*args,**kwargs))
+ else:
+ def outer_wrapper(*args,**kwargs):
+ return callback(function_p(*args,**kwargs))
+
+ #hack to make privileged functions compatible with register
+ outer_wrapper.func_name = function.func_name
+ return outer_wrapper
- return register2
+ return outer_decorator
-
+ ####### Template stuff
+ @privileged
+ def _render_template_to_string(path,with_data):
+ return dbapijinja.render_dropbox_template(client,path,with_data)
+
@register
def render_template(path,with_data=None,search_root="/_templates"):
"""
renders the given template
"""
- print render_template_to_string(path,with_data,search_root)
+ print render_template_to_string(path,with_data)
@register
- def render_template_to_string(path,with_data=None,search_root="/_templates"):
+ def render_template_to_string(path,with_data=None):
"""
Renders the template like render_template, but returns it as a a string instead
of printing it.
"""
- return dbapijinja.render_dropbox_template(client,path,with_data,search_root)
+ search_hierarchy = [request_folder,request_folder+'_templates/','/_templates/']
+
+ if path.startswith('/'):
+ return _render_template_to_string(path,with_data)
+
+ else:
+ for prefix in search_hierarchy:
+ check_path = prefix + path
+ try:
+ return _render_template_to_string(check_path,with_data)
+ except dbapijinja.TemplateNotFound:
+ pass
+ raise dbapijinja.TemplateNotFound()
+
+
+ ################### file io stuff
@register
+ @privileged
def _get_lock(path,timeout):
try:
file_exists = locker.lock(path,timeout)
@@ -178,14 +194,16 @@ def _get_lock(path,timeout):
raise IOError("Timeout waiting to open %s for writing or appending'%path")
return file_exists
-
+
@register
+ @privileged
def _release_lock(path):
#throws an IOError if it doesn't work
locker.release(path)
@register
+ @privileged
def open_file(path,to='read',timeout=None,allow_download=True):
"""
loads a file from the users dropbox and returns a string with the contents
@@ -260,24 +278,39 @@ def open_json_list(path,from_data=None,timeout=None):
out = open_json(path,from_data=from_data,timeout=timeout,default=list)
if not isinstance(out,list):
raise ValueError("Object opened by open_json_list is not a list!")
-
+
+ @register
+ @privileged
+ def close_file(file_handle):
+ file_handle._close(locker)
+
@register
+ @privileged
+ def close_json(inner_dict):
+ for open_file_h in locker.open_files:
+ if hasattr(open_file_h,'json_object'):
+ if open_file_h.json_object is inner_dict:
+ open_file_h._close(locker)
+
+ @register
+ @privileged
def save_json(path,json_object,timeout=None):
json_file = open_file(path,to='json',timeout=timeout,allow_download=False)
json_file.json_object = json_object
- json_file.close(loader)
+ close_file(json_file)
@register
def write_file(path,string,timeout=None):
text_file = open_file(path,to='write',timeout=timeout,allow_download=False)
text_file.write(string)
- text_file.close(loader)
+ close_file(text_file)
@register
def read_file(path):
return open_file(path).read()
@register
+ @privileged
def load_json(path):
"""
loads a json file and returns it
@@ -287,19 +320,41 @@ def load_json(path):
return json.load(open_file(path))
except ValueError:
raise ValueError('Unable to parse json file')
+
+ @register
+ @privileged
+ def delete_file(path):
+ if not path.startswith('/'):
+ path = request_folder + path
+
+ try:
+ client.file_delete(path)
+ except dropbox.rest.ErrorResponse:
+ raise IOError("Unable to delete file %s"%path)
+
+
+ ############ session stuff
@register
+ @privileged
def start_session():
session.start()
built_in_hash['SESSION'] = session.inner_dict
@register
+ @privileged
def destroy_session():
session.destroy()
+
+ ########## http stuff
@register
def set_response_header(key,value):
response.set_header(key,value)
+
+ @register
+ def get_request_header(key):
+ return request.headers.get(key)
@register
def set_response_status(status):
@@ -313,12 +368,15 @@ def redirect(where,immediately=True):
if immediately:
die("redirecting")
-
+
+ ############ text stuff
@register
+ @privileged
def markdown_to_html(markdown_string):
return markdown.markdown(markdown_string)
@register
+ @privileged
def pretty_print(thingy,pre=True):
"""
Pretty prints the given thingy
@@ -329,42 +387,45 @@ def pretty_print(thingy,pre=True):
print "</pre>"
+ ############ import stuff
- def dropbox_import_postop(imports):
- #hm... unfortunately, if any of the imports mutate builtins, then they call suffer
- #so should I call it for each?
+ def dropbox_import_callback(imports):
+ #hm... unfortunately, if any of the imports mutate the built_in_hash, they can
+ #affect everyones builtins
+ #so should I recursively create a new one for each for each?
#thats the reasoning behind it. I'd love if I didn't have to
for module_string,module in imports:
builtins = get_builtins(**kwargs)
exec module_string in builtins,module.__dict__
-
- @register_with_postop(dropbox_import_postop)
+ @register
+ @privileged_with_callback(dropbox_import_callback)
def dropbox_import(*module_paths):
#look first in the path given by folder search
#then look in a '/_scripts' folder? or similarly named?
#not right now
- #NO PACKAGE SUPPORT... SIMPLE FILES ONLY
- items = []
+ #NO PACKAGE SUPPORT... SIMPLE FILES ONLY FOR NOW
+ imports = []
for module_path in module_paths:
filestring = read_file(module_path)
module_name = os.path.basename(module_path).split('.',1)[0]
- sys.stderr.write('making module iwth name %s\n'%module_name)
out_module = imp.new_module(module_name)
built_in_hash[module_name] = out_module
- items.append( (filestring,out_module) )
+ imports.append( (filestring,out_module) )
- return items
+ return imports
-
+ ##### other stuff
@register
- def die(message=""):
+ def die(message="",report=True):
"""
Raises an Exception
"""
- raise Exception(message)
+ if report:
+ print message
+ raise UserDieException(message)
View
41 drapache/dbpyexecute.py
@@ -82,8 +82,6 @@ def __init__(self,sandbox,builtins,locker,session,response,code,timeout):
def run(self):
- traceback.print_stack()
-
try:
#enable for the builtin protection grabs the frame 2 up from enable
@@ -99,6 +97,9 @@ def enable_protections():
exec self.code in self.builtins
+ except dbpybuiltins.UserDieException:
+ pass
+
except Exception as e:
self.error = e
@@ -112,36 +113,8 @@ def enable_protections():
if protection.__class__.__name__ == 'CleanupBuiltins':
protection.frame = sys._getframe()
protection.disable(self.sandbox)
-
- sys.stderr.write('unrolled %d many modules\n'%len(sys.modules))
-
- t = memoryview('curl')
- sys.stderr.write("here, the error is %s\n"%(str(self.error)))
-
- builtins_set = set(__builtins__.keys())
- frame_set = set(sys._getframe().f_builtins.keys())
-
- difference = builtins_set ^ frame_set
-
- sys.stderr.write('the difference\n%s\n'%str(difference))
-
- sys.stderr.write("FLIJL"+str(__builtins__)+'\n')
- sys.stderr.write(str(len(sys._getframe().f_builtins)))
-
- sys.stderr.write('climbing up the frames:\n')
-
- ok = True
- level = 0
- while ok:
- try:
- f = sys._getframe(level)
- sys.stderr.write('%d\n'%len(f.f_builtins.keys()))
- except ValueError:
- ok = False
- level += 1
-
#finishing up
try:
self.locker.close_all()
@@ -165,6 +138,7 @@ def get_sandbox():
sandbox_config.enable("time")
sandbox_config.enable("math")
sandbox_config.enable("exit")
+ sandbox_config.enable("stderr")
sandbox_config.timeout = None
@@ -178,7 +152,7 @@ def execute(filestring,**kwargs):
PRINT_EXCEPTIONS = True
- EXEC_TIMEOUT = 15
+ EXEC_TIMEOUT = 25
DEBUG = True
response = ResponseObject(None,"")
@@ -250,8 +224,9 @@ def execute(filestring,**kwargs):
if response.status is None:
response.status = 200
-
- response.set_header('Content-Type','text/html')
+
+ if not 'Content-Type' in response.headers:
+ response.set_header('Content-Type','text/html')
return response
View
25 drapache/httpserver.py
@@ -8,6 +8,7 @@
import os
import sys
import urlparse
+import cgi
import util
@@ -33,8 +34,13 @@ class DropboxHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
#print self.request.settimeout(5)
def do_GET(self):
+ return self.serve(method='get')
-
+ def do_POST(self):
+ return self.serve(method='post')
+
+ def serve(self,method=None):
+
try:
request = util.RequestObject()
@@ -90,8 +96,18 @@ def do_GET(self):
subdomain_client = self.server.get_dropbox_client(subdomain_token)
- file_server = dbapiserver.FileServer(subdomain_client,request)
+ #post param checking
+ if method == 'post':
+ request_length = int(self.headers.get('Content-length',0))
+ foo = self.rfile.read(request_length)
+ post_params = urlparse.parse_qs(foo)
+ request.post_params = post_params
+ else:
+ request.post_params = None
+
+
+ file_server = dbapiserver.FileServer(subdomain_client,request)
response = file_server.serve(path)
if response.error:
@@ -99,10 +115,6 @@ def do_GET(self):
return None
else:
- #debug
- sys.stderr.write('memorview:%s\n'%str(memoryview))
- sys.stderr.write('dir:%s\n'%str(len(sys._getframe().f_builtins)))
- sys.stderr.write('inside of my HTTPSERVER, here are by builtins:%d\n'%len(__builtins__.keys()))
self.send_response(response.status)
for h,v in response.headers.items():
self.send_header(h,v)
@@ -111,7 +123,6 @@ def do_GET(self):
return None
except Exception as e:
- sys.stderr.write('Error in do get:\n')
traceback.print_exc()
self.send_error(500,str(e))
View
3 drapache/index_generator.py
@@ -20,11 +20,10 @@ def get_index_file(file_list,folder_path,client):
file_name = file_name + '/'
files.append(file_name)
- dropbox_env = jinja2.Environment(loader=dbapijinja.DropboxLoader(client,'/_templates'))
+ dropbox_env = jinja2.Environment(loader=dbapijinja.DropboxLoader(client,'/_templates/'))
try:
custom_index_template = dropbox_env.get_template('index.html')
- print 'hh'
return custom_index_template.render(files=files,path=folder_path)
except jinja2.TemplateNotFound:

0 comments on commit 972e9d7

Please sign in to comment.
Something went wrong with that request. Please try again.