Permalink
Browse files

tests passing with cumulus against postgres

  • Loading branch information...
1 parent fdf26ea commit 398444ef1cef482a9049fe05b630838c53af5d08 BuzzTroll committed Oct 4, 2010
@@ -1,14 +1,13 @@
import sqlite3
import pynimbusauthz
-from pynimbusauthz.cmd_opts import cbOpts
-from pynimbusauthz.db import DB
import uuid
class AuthzException(Exception):
e_types = {}
e_types['CLI_PARAMETER'] = 32
e_types['FILE_EXISTS'] = 33
e_types['USER_EXISTS'] = 34
+ e_types['DB_ERROR'] = 35
e_types['UNKNOWN'] = 254
@@ -1,6 +1,9 @@
import sqlite3
import os
import itertools
+import urlparse
+import pynimbusauthz
+from pynimbusauthz.authz_exception import AuthzException
def make_test_database(db_str=":memory:"):
f = open(os.environ['CUMULUS_AUTHZ_DDL'], "r")
@@ -21,25 +24,42 @@ def make_test_database(db_str=":memory:"):
# a simple wrapper around readonly
class DB(object):
def __init__(self, con_str=None, con=None):
+ self.replace_char = None
if con_str != None:
- self.con = sqlite3.connect(con_str)
+ url = urlparse.urlparse(con_str)
+ if url.scheme == "sqlite" or url.scheme == '':
+ self.con = sqlite3.connect(url.path)
+ elif url.scheme == "psycopg2":
+ import psycopg2
+
+ self.replace_char = '%s'
+ db = url.path[1:]
+ self.con = psycopg2.connect(user=url.username, password=url.password, host=url.hostname, database=db, port=url.port)
+ else:
+ raise AuthzException('DB_ERROR', "unsupport db url %s" % (con_str))
else:
self.con = con
#self.con.isolation_level = "EXCLUSIVE"
def _run_no_fetch(self, s, data):
+ if self.replace_char:
+ s = s.replace('?', self.replace_char)
c = self.con.cursor()
c.execute(s, data)
c.close()
def _run_fetch_iterator(self, s, data, convert_func, args=None):
+ if self.replace_char:
+ s = s.replace('?', self.replace_char)
c = self.con.cursor()
c.execute(s, data)
new_it = itertools.imap(lambda r: convert_func(self, r, args), c)
return new_it
def _run_fetch_all(self, s, data):
+ if self.replace_char:
+ s = s.replace('?', self.replace_char)
c = self.con.cursor()
c.execute(s, data)
r = c.fetchall()
@@ -48,6 +68,8 @@ def _run_fetch_all(self, s, data):
def _run_fetch_one(self, s, data):
+ if self.replace_char:
+ s = s.replace('?', self.replace_char)
c = self.con.cursor()
c.execute(s, data)
r = c.fetchone()
@@ -54,6 +54,7 @@ def __init__(self, db_obj, row):
self.object_size = row[File.cols['object_size']]
ctm = row[File.cols['creation_time']]
if ctm != None:
+ ctm = str(ctm)
ndx = ctm.rfind(".")
if ndx > 0:
ctm = ctm[:ndx]
@@ -186,7 +187,8 @@ def get_all_children(self, limit=None, match_str=None, clause=None):
s = s + " parent_id = ?"
data.append(self.id)
if match_str != None:
- s = s + " and name LIKE '" + match_str + "'"
+ s = s + " and name LIKE ? "
+ data.append(match_str)
if clause != None:
s = s + clause
@@ -201,10 +203,11 @@ def find_files(db_obj, pattern, object_type, parent=None):
ot = pynimbusauthz.object_types[object_type]
s = "SELECT " + File.get_select_str() + """
FROM objects
- WHERE name LIKE '%%%s%%' and object_type = %d""" % (pattern, ot)
- data = []
+ WHERE name LIKE ? and object_type = ?"""
+ data = [pattern, ot,]
if parent != None:
- s = s + " and parent_id = " + parent.get_id()
+ s = s + " and parent_id = ?"
+ data.append(parent.get_id())
c = db_obj._run_fetch_iterator(s, data, _convert_alias_row_to_File)
return c
@@ -49,12 +49,18 @@ def destroy(self):
# remove everything associated with a user
# including files if the are completely orphaned
def destroy_brutally(self):
+ s = "DELETE FROM object_quota where user_id = ?"
+ data = (self.uuid,)
+ self.db_obj._run_no_fetch(s, data)
s = "DELETE FROM user_alias where user_id = ?"
data = (self.uuid,)
self.db_obj._run_no_fetch(s, data)
s = "DELETE FROM object_acl where user_id = ?"
data = (self.uuid,)
self.db_obj._run_no_fetch(s, data)
+ s = "delete from objects where parent_id in (select id from objects where owner_id = ?)"
+ data = (self.uuid,)
+ self.db_obj._run_no_fetch(s, data)
s = "DELETE FROM objects where owner_id = ?"
data = (self.uuid,)
self.db_obj._run_no_fetch(s, data)
@@ -172,8 +178,8 @@ def find_user(db_obj, pattern):
# return all matching alias
def find_alias(db_obj, alias_name, alias_type=None):
s = "SELECT " + UserAlias.get_select_str()
- s = s + " FROM user_alias WHERE alias_name LIKE '%s'" % (alias_name)
- data = []
+ s = s + " FROM user_alias WHERE alias_name LIKE ? "
+ data = [alias_name,]
if alias_type != None:
at = pynimbusauthz.alias_types[alias_type]
s = s + "and alias_type = " + str(at)
@@ -242,8 +248,8 @@ def get_user_by_friendly(db_obj, friendly_name):
get_user_by_friendly = staticmethod(get_user_by_friendly)
def find_user_by_friendly(db_obj, friendly_pattern):
- s = "select id from users_canonical where friendly_name LIKE '%s'" % (friendly_pattern)
- data = []
+ s = "select id from users_canonical where friendly_name LIKE ? "
+ data = [friendly_pattern,]
c = db_obj._run_fetch_iterator(s, data, _convert_user_row_to_User)
return c
find_user_by_friendly = staticmethod(find_user_by_friendly)
@@ -343,8 +349,8 @@ def find_alias_by_friendly(db_obj, fn, type=pynimbusauthz.alias_type_s3):
def find_all_alias_by_friendly(db_obj, fn, type=pynimbusauthz.alias_type_s3):
at = pynimbusauthz.alias_types[type]
- s = "select "+ UserAlias.get_select_str()+" from user_alias where friendly_name LIKE '%s' and alias_type = ?" % (fn)
- data = (at,)
+ s = "select "+ UserAlias.get_select_str()+" from user_alias where friendly_name LIKE ? and alias_type = ?"
+ data = (fn, at,)
c = db_obj._run_fetch_iterator(s, data, _convert_alias_row_to_UserAlias)
return c
find_all_alias_by_friendly = staticmethod(find_all_alias_by_friendly)
@@ -0,0 +1,155 @@
+-- access types
+-- ============
+-- This is a small table consiting of just 5 rows (ripped off from s3 doc):
+-- r: read object
+-- w: write object
+-- x: execute object (?)
+-- R: read acl about object
+-- W: write acl about object (full control except changing owner)
+create table access_types(
+ mod char PRIMARY KEY,
+ description varchar(64)
+);
+
+insert into access_types(mod, description) values ('r', 'read data');
+insert into access_types(mod, description) values ('w', 'write data');
+insert into access_types(mod, description) values ('R', 'read ACL');
+insert into access_types(mod, description) values ('W', 'write ACL');
+
+
+-- object_types
+-- ===========
+-- Objects are like files. Each has a 'type' which is an access method
+-- presented to the user. This will be needed if we are to expose both
+-- a GridFTP interface and a s3 interface.
+--
+-- gridftp
+-- hdfs
+-- s3 bucket
+-- s3 key
+create table object_types(
+ id SERIAL PRIMARY KEY,
+ name varchar(64) UNIQUE NOT NULL
+);
+insert into object_types(name) values ('s3');
+insert into object_types(name) values ('gridftp');
+insert into object_types(name) values ('hdfs');
+
+-- users_canonical
+-- ==============
+-- This table is the canonical user description. A user may have many
+-- IDs and credentials and such that references this single ID
+create table users_canonical(
+ id char(36) PRIMARY KEY,
+ friendly_name varchar(64) UNIQUE NOT NULL
+);
+insert into users_canonical(id, friendly_name) values ('CumulusAuthenticatedUser', 'CumulusAuthenticatedUser');
+insert into users_canonical(id, friendly_name) values ('CumulusPublicUser', 'CumulusPublicUser');
+
+-- insert into users_canonical(id, friendly_name) values ('CumulusPublicUser', 'CumulusPublicUser');
+-- insert into users_canonical(id, friendly_name) values ('CumulusAuthenticatedUser', 'CumulusAuthenticatedUser');
+-- user_alias_types
+-- ================
+-- vairous types of user identifications mechanisms:
+-- auth_tokens: S3
+-- x509: gsi
+-- ssh: public key
+-- unix: password hash
+create table user_alias_types(
+ id SERIAL PRIMARY KEY,
+ name varchar(64) UNIQUE NOT NULL
+);
+insert into user_alias_types(name) values ('s3');
+insert into user_alias_types(name) values ('x509');
+insert into user_alias_types(name) values ('ssh');
+insert into user_alias_types(name) values ('unix');
+
+-- user_alias
+-- ==========
+-- this table references the canonical user. it allows us to have many
+-- means of identifying a single user. For example auth tokens, ssh, gsi.
+-- The format of the type_data is defined by the alias_type
+--
+-- i think we can be this generic but i am not 100% sure
+create table user_alias(
+ id SERIAL PRIMARY KEY,
+ user_id char(36) REFERENCES users_canonical(id) NOT NULL,
+ alias_name varchar(256) NOT NULL,
+ friendly_name varchar(256) NOT NULL,
+ alias_type INTEGER REFERENCES user_alias_types(id) NOT NULL,
+ alias_type_data varchar(1024),
+ UNIQUE(alias_name, alias_type),
+ UNIQUE(friendly_name, alias_type)
+);
+
+insert into user_alias(user_id, alias_name, friendly_name, alias_type) values ('CumulusAuthenticatedUser', 'CumulusAuthenticatedUser', 'CumulusAuthenticatedUser', 1);
+insert into user_alias(user_id, alias_name, friendly_name, alias_type) values ('CumulusPublicUser', 'CumulusPublicUser', 'CumulusPublicUser', 1);
+-- the actual data.
+-- this can be a file, a dhfs file key, or a gridftp url (?)
+-- it is names speced by the url spec
+--
+-- this is broken out because there could be many objects that
+-- reference the same physical data.
+--
+-- For example, we provide 2 access mechanisms to a single VM:
+-- GridFTP and s3. The s3 object might present its clients
+-- with a different path to the physical data than the GridFTP
+-- server does
+-- create table physical_data(
+--- id INTEGER PRIMARY KEY AUTOINCREMENT,
+-- data_key varchar(1024)
+--);
+-- ditching this for now. it seems over engineered.
+
+
+-- parent id may only be useful for s3
+-- data key is some sort of reference to where it actually is
+-- name is its name in the given object_type names space
+--
+-- for any given object_type the name an parent id must be null
+-- for s3:
+-- if the object is a bucket it will have a NULL parent
+-- and thus all buckets will be unique over s3 space
+-- if it is a key the key will be unique in the bucket
+-- this should meet s3 requirements
+--
+-- if it is a gridftp file, name can just be a full path
+-- with a null parent_id. this will ensure a unique full path
+-- to a file which should be consistant with unix file systems
+create table objects(
+ id SERIAL PRIMARY KEY,
+ name varchar(1024) NOT NULL,
+ friendly_name varchar(1024),
+ owner_id char(36) REFERENCES users_canonical(id) ON DELETE CASCADE NOT NULL,
+ data_key varchar(1024) NOT NULL,
+ object_type INTEGER REFERENCES object_types(id) NOT NULL,
+ parent_id INTEGER REFERENCES objects(id) ON DELETE CASCADE DEFAULT NULL,
+
+ md5sum CHAR(32),
+ object_size INTEGER DEFAULT 0,
+ creation_time TIMESTAMP,
+ UNIQUE(object_type, name, parent_id)
+);
+
+-- object_acl
+-- ==========
+-- This is a join table for descovering acl permissions associated with
+-- a file
+create table object_acl(
+ id SERIAL PRIMARY KEY,
+ user_id char(36) REFERENCES users_canonical(id) ON DELETE CASCADE NOT NULL,
+ object_id INTEGER REFERENCES objects(id) ON DELETE CASCADE NOT NULL,
+ access_type_id CHAR REFERENCES access_types(mod) NOT NULL,
+ unique(user_id, object_id, access_type_id)
+);
+
+
+create table object_quota(
+ id SERIAL PRIMARY KEY,
+ user_id char(36) REFERENCES users_canonical(id) ON DELETE CASCADE NOT NULL,
+ object_type INTEGER REFERENCES object_types(id) NOT NULL,
+ quota INTEGER NOT NULL,
+ UNIQUE(user_id, object_type)
+);
+
+
@@ -13,7 +13,8 @@ block_size=524288
[security]
type=authz
security_dir=@INSTALLDIR@/posixauth
-authzdb=@AUTHZDB@
+authzdb=sqlite:///@AUTHZDB@
+#authzdb=psycopg2://bresnaha:nimbus@localhost:5432/cumulus
[log]

0 comments on commit 398444e

Please sign in to comment.