diff --git a/AppController/lib/haproxy.rb b/AppController/lib/haproxy.rb index 3d722cabdf..aa023dea91 100644 --- a/AppController/lib/haproxy.rb +++ b/AppController/lib/haproxy.rb @@ -171,22 +171,22 @@ def self.write_app_config(app_name, app_number, num_of_servers, ip) # Updates the HAProxy config file for this App Engine application to # point to all the ports currently the application. In contrast with # write_app_config, these ports can be non-contiguous. - # TODO(cgb): Lots of copypasta here with write_app_config - eliminate it. - def self.update_app_config(app_name, app_number, ports, public_ip) + # TODO(cgb): Lots of copy/paste here with write_app_config - eliminate it. + def self.update_app_config(app_name, app_number, ports, private_ip) # Add a prefix to the app name to avoid collisions with non-GAE apps full_app_name = "gae_#{app_name}" index = 0 servers = [] ports.each { |port| - server = HAProxy.server_config(full_app_name, index, public_ip, port) + server = HAProxy.server_config(full_app_name, index, private_ip, port) index += 1 servers << server } listen_port = HAProxy.app_listen_port(app_number) config = "# Create a load balancer for the app #{app_name} \n" - config << "listen #{full_app_name} #{public_ip}:#{listen_port} \n" + config << "listen #{full_app_name} #{private_ip}:#{listen_port} \n" config << servers.join("\n") config_path = File.join(SITES_ENABLED_PATH, diff --git a/AppController/lib/pbserver.rb b/AppController/lib/pbserver.rb index 34e8ce1402..4b79087bfa 100644 --- a/AppController/lib/pbserver.rb +++ b/AppController/lib/pbserver.rb @@ -47,6 +47,9 @@ module PbServer # within AppScale. DBS_WITH_NATIVE_PBSERVER = ["mysql"] + # A list of databases that use the version 2 of database query support. + # The second version has secondary index support. + DBS_WITH_V2_PBSERVER = ['cassandra', 'hypertable', "hbase"] # The name that nginx should use as the identifier for the PBServer when it # we write its configuration files. @@ -64,14 +67,26 @@ def self.start(master_ip, db_local_ip, my_ip, table, zklocations) "MASTER_IP" => master_ip, "LOCAL_DB_IP" => db_local_ip } - - ports.each { |port| - start_cmd = "/usr/bin/python2.6 #{pbserver} -p #{port} " + + + if DBS_WITH_V2_PBSERVER.include?(table) + ports.each { |port| + start_cmd = "/usr/bin/python2.6 #{pbserver} -p #{port} " + + "--no_encryption --type #{table} -z \'#{zklocations}\' " + # stop command doesn't work, relies on terminate.rb + stop_cmd = "pkill -9 datastore_server" + GodInterface.start(:pbserver, start_cmd, stop_cmd, port, env_vars) + } + else + ports.each { |port| + start_cmd = "/usr/bin/python2.6 #{pbserver} -p #{port} " + + "--no_encryption --type #{table} -z \'#{zklocations}\' " "--no_encryption --type #{table} -z \'#{zklocations}\' " + "-s #{HelperFunctions.get_secret()} -a #{my_ip} --key" - stop_cmd = "pkill -9 appscale_server" - GodInterface.start(:pbserver, start_cmd, stop_cmd, port, env_vars) - } + # stop command doesn work, relies on terminate.rb + stop_cmd = "pkill -9 appscale_server" + GodInterface.start(:pbserver, start_cmd, stop_cmd, port, env_vars) + } + end end @@ -117,6 +132,8 @@ def self.is_running(my_ip) def self.get_executable_name(table) if DBS_WITH_NATIVE_PBSERVER.include?(table) return "#{APPSCALE_HOME}/AppDB/appscale_server_#{table}.py" + elsif DBS_WITH_V2_PBSERVER.include?(table) + return "#{APPSCALE_HOME}/AppDB/datastore_server.py" else return "#{APPSCALE_HOME}/AppDB/appscale_server.py" end diff --git a/AppController/terminate.rb b/AppController/terminate.rb index 42303191ab..e428100c49 100644 --- a/AppController/terminate.rb +++ b/AppController/terminate.rb @@ -113,7 +113,7 @@ ["memcached", "nginx", "haproxy", "collectd", "collectdmon", - "soap_server", "appscale_server", + "soap_server", "appscale_server", "datastore_server", "AppLoadBalancer", "AppMonitoring", # AppServer "dev_appserver", "DevAppServerMain", diff --git a/AppDB/appscale_datastore.py b/AppDB/appscale_datastore.py index 435d658655..36c80c75a2 100644 --- a/AppDB/appscale_datastore.py +++ b/AppDB/appscale_datastore.py @@ -5,24 +5,21 @@ # Author: Gaurav Kumar Mehta # Author: NOMURA Yoshihide # See LICENSE file -import threading +import imp +import os import sys -import string, cgi +import string import socket -import os +import threading import types -import imp + import appscale_logger from dbconstants import * -#from helper_functions import ThreadedLogger -#LOG_DIR = "%s/AppDB/logs" % APPSCALE_HOME -#LOG_FILENAME = LOG_DIR + "/appscale_datastore.log" -#app_datastore_logger = ThreadedLogger(LOG_FILENAME) -#app_datastore_logger.turnLoggingOn() app_datastore_logger = appscale_logger.getLogger("appscale_datastore") DB_ERROR = "DB_ERROR:" + ERROR_CODES = [DB_ERROR] DATASTORE_DIR= "%s/AppDB" % APPSCALE_HOME @@ -38,8 +35,10 @@ def getDatastore(cls, d_type): d_mod = imp.load_source(d_name, mod_path) datastore = d_mod.DatastoreProxy(app_datastore_logger) else: - app_datastore_logger.error("Fail to use datastore: %s. Please check the datastore type." % d_type) - raise Exception("Fail to use datastore: %s" % d_type) + app_datastore_logger.error("Fail to use datastore: %s. Please " + \ + "check the datastore type." % d_type) + raise Exception("Datastore was not found in %d directory. " + \ + "Fail to use datastore: %s" %(DATASTORE_DIR, d_type)) return datastore @classmethod diff --git a/AppDB/appscale_datastore_batch.py b/AppDB/appscale_datastore_batch.py new file mode 100644 index 0000000000..222c5cc785 --- /dev/null +++ b/AppDB/appscale_datastore_batch.py @@ -0,0 +1,54 @@ +#!/usr/bin/python +# Programmer: Navraj Chohan +# See LICENSE file + +import imp +import os +import socket +import string +import sys +import threading +import types + +import appscale_logger +import dbconstants + +app_datastore_logger = appscale_logger.getLogger("appscale_datastore_batch") + +DATASTORE_DIR= "%s/AppDB" % dbconstants.APPSCALE_HOME + +class DatastoreFactory: + + @classmethod + def getDatastore(cls, d_type): + """ Returns a reference for the datastore. Validates where + the _interface.py is and adds that path to + the system path. + + Args: + d_type: The name of the datastore (ex: cassandra) + """ + + datastore = None + mod_path = DATASTORE_DIR + "/" + d_type + "/" + d_type + "_interface.py" + + if os.path.exists(mod_path): + sys.path.append(DATASTORE_DIR + "/" + d_type) + d_mod = imp.load_source(d_type+"_interface.py", mod_path) + datastore = d_mod.DatastoreProxy(app_datastore_logger) + else: + app_datastore_logger.error("Fail to use datastore: %s. Please \ + check the datastore type." % d_type) + raise Exception("Fail to use datastore: %s" % d_type) + + return datastore + + @classmethod + def valid_datastores(cls): + """ Returns a list of directories where the datastore code is + + Returns: Directory list + """ + + dblist = os.listdir(DATASTORE_DIR) + return dblist diff --git a/AppDB/appscale_server.py b/AppDB/appscale_server.py index aae11aa41a..76a33884b0 100644 --- a/AppDB/appscale_server.py +++ b/AppDB/appscale_server.py @@ -956,8 +956,6 @@ def put_request(self, app_id, http_request_data): start_time = time.time() putreq_pb = datastore_pb.PutRequest(http_request_data) - #print "RECEIVED PUT_REQUEST %s" % putreq_pb - #logger.debug("RECEIVED PUT_REQUEST %s" % putreq_pb) putresp_pb = datastore_pb.PutResponse( ) txn = None root_key = None @@ -1039,7 +1037,6 @@ def put_request(self, app_id, http_request_data): # then the global counter is used # gen unique id only wants to know if a child exist uid = generate_unique_id(app_id, root_key, child_key) - #print "UID:" + str(uid) if uid <= 0: return(putresp_pb.Encode(), datastore_pb.Error.INTERNAL_ERROR, @@ -1078,7 +1075,6 @@ def put_request(self, app_id, http_request_data): 'No group entity or root key.') try: locktime = time.time() - print root_key gotLock = zoo_keeper.acquireLock( app_id, txn.handle(), root_key) if PROFILE: appscale_log.write("ACQUIRELOCK %d %f\n"%(txn.handle(), time.time() - locktime)) except zk.ZKTransactionException, zkex: @@ -1092,7 +1088,6 @@ def put_request(self, app_id, http_request_data): ####################################### # insert key table_name = getTableName(app_id, kind, namespace) - #print "Put Using table name:",table_name # Notify Users/Apps table if a new class is being added if table_name not in tableHashTable: # This is the first time this pbserver has seen this table @@ -1171,7 +1166,6 @@ def put_request(self, app_id, http_request_data): JOURNAL_SCHEMA, [e.Encode()]) entPut = putThread() - #print "Row key:" + str(row_key) handle = 0 if is_trans_on: handle = txn.handle() @@ -1220,7 +1214,6 @@ def get_request(self, app_id, http_request_data): global app_datastore getreq_pb = datastore_pb.GetRequest(http_request_data) #logger.debug("GET_REQUEST: %s" % getreq_pb) - #print "GET_REQUEST: %s" % getreq_pb getresp_pb = datastore_pb.GetResponse() is_trans_on = True @@ -1262,9 +1255,6 @@ def get_request(self, app_id, http_request_data): zoo_keeper = zoo_keeper_stub table_name = getTableName(app_id, kind, namespace) row_key = getRowKey(app_id,key.path().element_list()) - #print "get row key:" + str(row_key) - #print "table_name:" + str(table_name) - #print "schema:" + str(ENTITY_TABLE_SCHEMA) r = app_datastore.get_entity( table_name, row_key, ENTITY_TABLE_SCHEMA ) err = r[0] if err not in ERROR_CODES or len(r) != 3: @@ -1482,35 +1472,21 @@ def optimized_put_request(self, app_id, http_request_data): def void_proto(self, app_id, http_request_data): resp_pb = api_base_pb.VoidProto() - print "Got void" - #logger.debug("VOID_RESPONSE: %s to void" % resp_pb) return (resp_pb.Encode(), 0, "" ) def str_proto(self, app_id, http_request_data): str_pb = api_base_pb.StringProto( http_request_data ) composite_pb = datastore_pb.CompositeIndices() - print "Got a string proto" - print str_pb - #logger.debug("String proto received: %s"%str_pb) - #logger.debug("CompositeIndex response to string: %s" % composite_pb) return (composite_pb.Encode(), 0, "" ) def int64_proto(self, app_id, http_request_data): int64_pb = api_base_pb.Integer64Proto( http_request_data ) resp_pb = api_base_pb.VoidProto() - print "Got a int 64" - print int64_pb - #logger.debug("Int64 proto received: %s"%int64_pb) - #logger.debug("VOID_RESPONSE to int64: %s" % resp_pb) return (resp_pb.Encode(), 0, "") def compositeindex_proto(self, app_id, http_request_data): compindex_pb = entity_pb.CompositeIndex( http_request_data) resp_pb = api_base_pb.VoidProto() - print "Got Composite Index" - #print compindex_pb - #logger.debug("CompositeIndex proto recieved: %s"%str(compindex_pb)) - #logger.debug("VOID_RESPONSE to composite index: %s" % resp_pb) return (resp_pb.Encode(), 0, "") # Returns 0 on success, 1 on failure @@ -1526,28 +1502,22 @@ def create_index_tables(self, app_id): return 1 """ table_name = "__" + app_id + "__" + "single_prop_asc" - print "Building table: " + table_name columns = ["reference"] returned = app_datastore.create_table( table_name, columns ) err,res = returned if err not in ERROR_CODES: - #logger.debug("%s" % err) return 1 table_name = "__" + app_id + "__" + "single_prop_desc" - print "Building table: " + table_name returned = app_datastore.create_table( table_name, columns ) err,res = returned if err not in ERROR_CODES: - #logger.debug("%s" % err) return 1 table_name = "__" + app_id + "__" + "composite" - print "Building table: " + table_name returned = app_datastore.create_table( table_name, columns ) err,res = returned if err not in ERROR_CODES: - #logger.debug("%s" % err) return 1 return 0 diff --git a/AppDB/appscale_server_mysql.py b/AppDB/appscale_server_mysql.py index c81206fabe..fe6a37234c 100644 --- a/AppDB/appscale_server_mysql.py +++ b/AppDB/appscale_server_mysql.py @@ -1,26 +1,35 @@ #!/usr/bin/python +# See LICENSE file # # Author: -# Navraj Chohan (nchohan@cs.ucsb.edu) -# See LICENSE file +# Navraj Chohan (nlake44@gmail.com) import tornado.httpserver import tornado.ioloop import tornado.web -import sys -import socket +import array +import datetime +import getopt +import itertools +import md5 +import MySQLdb import os +import random +import SOAPpy +import socket +import sys +import threading +import time import types + +from SocketServer import BaseServer +from M2Crypto import SSL +import MySQLdb.constants.CR + import appscale_datastore -#import helper_functions -import SOAPpy -from dbconstants import * import appscale_logger -import md5 -import random -import getopt -import threading -import datetime +from dbconstants import * + from google.appengine.api import api_base_pb from google.appengine.api import datastore from google.appengine.api import datastore_errors @@ -33,72 +42,71 @@ from google.net.proto import ProtocolBuffer from google.appengine.datastore import entity_pb from google.appengine.ext.remote_api import remote_api_pb -from SocketServer import BaseServer -from M2Crypto import SSL from drop_privileges import * from zkappscale import zktransaction -import time -import array -import itertools from google.appengine.api import apiproxy_stub from google.appengine.api import apiproxy_stub_map from google.appengine.datastore import sortable_pb_encoder -import MySQLdb -import MySQLdb.constants.CR import __builtin__ buffer = __builtin__.buffer zoo_keeper = None -DEBUG = False +# Port used if server is using encryption DEFAULT_SSL_PORT = 8443 + +# Port used if no encryption is used by the server DEFAULT_PORT = 4080 + +# Whether encryption is on by default DEFAULT_ENCRYPTION = 1 + +# The SSL cert location CERT_LOCATION = "/etc/appscale/certs/mycert.pem" + +# The SSL private key location KEY_LOCATION = "/etc/appscale/certs/mykey.pem" + +# Where the secret key is located SECRET_LOCATION = "/etc/appscale/secret.key" -VALID_DATASTORES = [] -ERROR_CODES = [] + +# The length of a numerial key, padded until the key is this length ID_KEY_LENGTH = 64 -app_datastore = [] -logOn = False -logFilePtr = "" -tableHashTable = {} +# The accessor for the datastore +app_datastore = [] """MySQL-based stub for the Python datastore API. Entities are stored in a MySQL database in a similar fashion to the production datastore. Based on Nick Johnson's SQLite stub and Typhoonae's mysql stub """ - entity_pb.Reference.__hash__ = lambda self: hash(self.Encode()) datastore_pb.Query.__hash__ = lambda self: hash(self.Encode()) datastore_pb.Transaction.__hash__ = lambda self: hash(self.Encode()) datastore_pb.Cursor.__hash__ = lambda self: hash(self.Encode()) _DB_LOCATION = "127.0.0.1" + _USE_DATABASE = "appscale" + _DB_USER = "root" + _MAX_CONNECTIONS = 10 + _GC_TIME = 60 -_MAXIMUM_RESULTS = 1000 +_MAXIMUM_RESULTS = 1000 _MAX_QUERY_OFFSET = 1000 - _MAX_QUERY_COMPONENTS = 63 - _BATCH_SIZE = 20 - _MAX_ACTIONS_PER_TXN = 5 - _MAX_TIMEOUT = 5.0 - _OPERATOR_MAP = { datastore_pb.Query_Filter.LESS_THAN: '<', datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL: '<=', @@ -628,11 +636,11 @@ def __gc(self): self.__transDict_lock.release() last_gc_time = time.time() - def setupTransaction(self, app_id): + def setup_transaction(self, app_id): # New connection txn_id = zoo_keeper.getTransactionID(app_id) client = MySQLdb.connect(host=_DB_LOCATION, db=_USE_DATABASE, user=_DB_USER) - #client.autocommit(0) + #TODO is this lock a bottle neck, and is it really required? self.__transDict_lock.acquire() # entity group is set to None self.__transDict[txn_id] = client, None, time.time() @@ -941,30 +949,6 @@ def __ReleaseLockForEntityGroup(self, app_id, entity_group=''): cursor.execute("SELECT RELEASE_LOCK('%s');" % lock_str) self.__connection.commit() - def __getRootKey(app_id, ancestor_list): - key = app_id # mysql cannot have \ as the first char in the row key - a = ancestor_list[0] - key += "/" - - # append _ if the name is a number, prevents collisions of key names - if a.has_type(): - key += a.type() - else: - return None - - if a.has_id(): - zero_padded_id = ("0" * (ID_KEY_LENGTH - len(str(a.id())))) + str(a.id()) - key += ":" + zero_padded_id - elif a.has_name(): - if a.name().isdigit(): - key += ":__key__" + a.name() - else: - key += ":" + a.name() - else: - return None - - return key - @staticmethod def __ExtractEntityGroupFromKeys(app_id, keys): """Extracts entity group.""" @@ -1827,7 +1811,7 @@ def begin_transaction_request(self, app_id, http_request_data): transaction_pb = datastore_pb.Transaction() handle = 0 #print "Begin Trans Handle:",handle - handle = app_datastore.setupTransaction(app_id) + handle = app_datastore.setup_transaction(app_id) transaction_pb.set_app(app_id) transaction_pb.set_handle(handle) return (transaction_pb.Encode(), 0, "") @@ -1983,16 +1967,7 @@ def usage(): print "\t--no_encryption" def main(argv): global app_datastore - #global getKeyFromServer - #global tableServer - #global keySecret - global logOn global logFilePtr - #global optimizedQuery - #global soapServer - global ERROR_CODES - global VALID_DATASTORES - #global KEYBLOCKSIZE global zoo_keeper cert_file = CERT_LOCATION key_file = KEY_LOCATION @@ -2018,26 +1993,16 @@ def main(argv): if opt in ("-c", "--certificate"): cert_file = arg print "Using cert..." - #elif opt in ("-k", "--key" ): - # getKeyFromServer = True - # print "Using key server..." elif opt in ("-t", "--type"): db_type = arg print "Datastore type: ",db_type elif opt in ("-s", "--secret"): - #keySecret = arg print "Secret set..." elif opt in ("-l", "--log"): - logOn = True - logFile = arg - logFilePtr = open(logFile, "w") - logFilePtr.write("# type, app, start, end\n") + pass elif opt in ("-b", "--blocksize"): - #KEYBLOCKSIZE = arg - #print "Block size: ",KEYBLOCKSIZE pass elif opt in ("-a", "--soap"): - #soapServer = arg pass elif opt in ("-p", "--port"): port = int(arg) @@ -2047,12 +2012,9 @@ def main(argv): zoo_keeper_locations = arg app_datastore = DatastoreDistributed() - #tableServer = SOAPpy.SOAPProxy("https://" + soapServer + ":" + str(keyPort)) - - #global keyDictionaryLock + zoo_keeper = zktransaction.ZKTransaction(zoo_keeper_locations) - #keyDictionaryLock = threading.Lock() if port == DEFAULT_SSL_PORT and not isEncrypted: port = DEFAULT_PORT pb_application = tornado.web.Application([ diff --git a/AppDB/cassandra/cassandra_helper.rb b/AppDB/cassandra/cassandra_helper.rb index 9ab036838b..88d15ccdfc 100644 --- a/AppDB/cassandra/cassandra_helper.rb +++ b/AppDB/cassandra/cassandra_helper.rb @@ -1,7 +1,11 @@ +# Programmer: Navraj Chohan require 'djinn' require 'djinn_job_data' require 'helperfunctions' +# Whether to remove old data from a previous start +DROP_TABLES = true + def get_uaserver_ip() Djinn.get_nearest_db_ip end @@ -62,8 +66,11 @@ def start_db_master() Djinn.log_debug("Starting up Cassandra as master") Djinn.log_run("pkill ThriftBroker") - `rm -rf /var/appscale/cassandra*` - `rm /var/log/appscale/cassandra/system.log` + if DROP_TABLES + Djinn.log_run("rm -rf /var/appscale/cassandra*") + Djinn.log_run("rm /var/log/appscale/cassandra/system.log") + end + Djinn.log_run("#{APPSCALE_HOME}/AppDB/cassandra/cassandra/bin/cassandra start -p /var/appscale/appscale-cassandra.pid") HelperFunctions.sleep_until_port_is_open(HelperFunctions.local_ip, 9160) end @@ -74,10 +81,12 @@ def start_db_slave() HelperFunctions.sleep_until_port_is_open(Djinn.get_db_master_ip, 9160) sleep(5) - `rm -rf /var/appscale/cassandra*` - `rm /var/log/appscale/cassandra/system.log` - `#{APPSCALE_HOME}/AppDB/cassandra/cassandra/bin/cassandra start -p /var/appscale/appscale-cassandra.pid` - #Djinn.log_run("#{APPSCALE_HOME}/AppDB/cassandra/cassandra/bin/cassandra start -p /var/appscale/appscale-cassandra.pid") + if DROP_TABLES + Djinn.log_run("rm -rf /var/appscale/cassandra*") + Djinn.log_run("rm /var/log/appscale/cassandra/system.log") + end + Djinn.log_run("#{APPSCALE_HOME}/AppDB/cassandra/cassandra/bin/cassandra start -p /var/appscale/appscale-cassandra.pid") + Djinn.log_run("#{APPSCALE_HOME}/AppDB/cassandra/cassandra/bin/cassandra start -p /var/appscale/appscale-cassandra.pid") HelperFunctions.sleep_until_port_is_open(HelperFunctions.local_ip, 9160) end @@ -88,7 +97,9 @@ def stop_db_master def stop_db_slave Djinn.log_debug("Stopping Cassandra slave") - Djinn.log_run("#{APPSCALE_HOME}/AppDB/cassandra/cassandra/bin/nodetool decommission -h #{HelperFunctions.local_ip} -p 6666") + if DROP_TABLES + Djinn.log_run("#{APPSCALE_HOME}/AppDB/cassandra/cassandra/bin/nodetool decommission -h #{HelperFunctions.local_ip} -p 6666") + end Djinn.log_run("cat /var/appscale/appscale-cassandra.pid | xargs kill -9") end diff --git a/AppDB/cassandra/cassandra_interface.py b/AppDB/cassandra/cassandra_interface.py new file mode 100644 index 0000000000..931d89bec4 --- /dev/null +++ b/AppDB/cassandra/cassandra_interface.py @@ -0,0 +1,277 @@ +# Programmer: Navraj Chohan + +""" + Cassandra Interface for AppScale +""" +import base64 +import os +import string +import sys +import time + +from thrift_cass.Cassandra import Client +from thrift_cass.ttypes import * +from thrift import Thrift +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol + +import appscale_logger +import helper_functions +import pycassa + +from dbconstants import * +from dbinterface_batch import * +from pycassa.system_manager import * + +# This is the default cassandra connection port +CASS_DEFAULT_PORT = 9160 + +# Data consistency models available with cassandra +CONSISTENCY_ONE = pycassa.cassandra.ttypes.ConsistencyLevel.ONE +CONSISTENCY_QUORUM = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM +CONSISTENCY_ALL = pycassa.cassandra.ttypes.ConsistencyLevel.ALL + +# The keyspace used for all tables +KEYSPACE = "Keyspace1" + +# The standard column family used for tables +STANDARD_COL_FAM = "Standard1" + +class DatastoreProxy(AppDBInterface): + """ + Cassandra implementation of the AppDBInterface + """ + def __init__(self, logger = appscale_logger.getLogger("datastore-cassandra")): + """ + Constructor + + Args: + logger: An object that logging messages can be sent to + """ + + self.host = helper_functions.read_file(APPSCALE_HOME + \ + '/.appscale/my_private_ip') + self.port = CASS_DEFAULT_PORT + self.pool = pycassa.ConnectionPool(keyspace=KEYSPACE, + server_list=[self.host+":"+str(self.port)], + prefill=False) + self.logger = logger + + def batch_get_entity(self, table_name, row_keys, column_names): + """ + Takes in batches of keys and retrieves their cooresponding rows. + + Args: + table_name: The table to access + row_keys: A list of keys to access + column_names: A list of columns to access + Returns: + A dictionary of rows and columns/values of those rows. The format + looks like such: {key:{column_name:value,...}} + """ + + if not isinstance(table_name, str): raise TypeError("Expected a str") + if not isinstance(column_names, list): raise TypeError("Expected a list") + if not isinstance(row_keys, list): raise TypeError("Expected a list") + + ret_val = {} + client = self.pool.get() + path = ColumnPath(table_name) + slice_predicate = SlicePredicate(column_names=column_names) + results = client.multiget_slice(row_keys, + path, + slice_predicate, + CONSISTENCY_QUORUM) + + for row in row_keys: + col_dic = {} + for columns in results[row]: + col_dic[columns.column.name] = columns.column.value + ret_val[row] = col_dic + + if client: + self.pool.return_conn(client) + return ret_val + + def batch_put_entity(self, table_name, row_keys, column_names, cell_values): + """ + Allows callers to store multiple rows with a single call. A row can + have multiple columns and values with them. We refer to each row as + an entity. + + Args: + table_name: The table to mutate + row_keys: A list of keys to store on + column_names: A list of columns to mutate + cell_values: A dict of key/value pairs + Raises: + TypeError: when bad arguments are given + """ + + if not isinstance(table_name, str): raise TypeError("Expected a str") + if not isinstance(column_names, list): raise TypeError("Expected a list") + if not isinstance(row_keys, list): raise TypeError("Expected a list") + if not isinstance(cell_values, dict): raise TypeError("Expected a dic") + + cf = pycassa.ColumnFamily(self.pool,table_name) + multi_map = {} + for key in row_keys: + cols = {} + for cname in column_names: + cols[cname] = cell_values[key][cname] + multi_map[key] = cols + cf.batch_insert(multi_map) + + def batch_delete(self, table_name, row_keys, column_names=[]): + """ + Remove a set of rows cooresponding to a set of keys. + + Args: + table_name: Table to delete rows from + row_keys: A list of keys to remove + column_names: Not used + Raises: + AppScaleDBConnectionError: when unable to execute deletes + TypeError: when given bad argument types + """ + + if not isinstance(table_name, str): raise TypeError("Expected a str") + if not isinstance(row_keys, list): raise TypeError("Expected a list") + + path = ColumnPath(table_name) + try: + cf = pycassa.ColumnFamily(self.pool,table_name) + b = cf.batch() + for key in row_keys: + b.remove(key) + b.send() + except Exception, ex: + raise AppScaleDBConnectionError("Exception %s" % str(ex)) + + def delete_table(self, table_name): + """ + Drops a given table (aka column family in Cassandra) + + Args: + table_name: A string name of the table to drop + Rasies: + TypeError: when given bad argument types + """ + + if not isinstance(table_name, str): raise TypeError("Expected a str") + + sysman = pycassa.system_manager.SystemManager(self.host + ":" + str(CASS_DEFAULT_PORT)) + sysman.drop_column_family(KEYSPACE, table_name) + + def create_table(self, table_name, column_names): + """ + Creates a table as a column family + + Args: + table_name: The column family name + column_names: Not used but here to match the interface + Raises: + TypeError: when given bad argument types + """ + + if not isinstance(table_name, str): raise TypeError("Expected a str") + if not isinstance(column_names, list): raise TypeError("Expected a list") + + sysman = pycassa.system_manager.SystemManager(self.host + ":" + str(CASS_DEFAULT_PORT)) + try: + sysman.create_column_family(KEYSPACE, + table_name, + comparator_type=UTF8_TYPE) + except InvalidRequestException, e: + pass + + def range_query(self, + table_name, + column_names, + start_key, + end_key, + limit, + offset=0, + start_inclusive=True, + end_inclusive=True, + keys_only=False): + """ + Gets a dense range ordered by keys. Returns an ordered list of + a dictionary of [key:{column1:value1, column2:value2},...] + or a list of keys if keys only. + + Args: + table_name: Name of table to access + column_names: Columns which get returned within the key range + start_key: String for which the query starts at + end_key: String for which the query ends at + limit: Maximum number of results to return + offset: Cuts off these many from the results [offset:] + start_inclusive: Boolean if results should include the start_key + end_inclusive: Boolean if results should include the end_key + keys_only: Boolean if to only keys and not values + Raises: + TypeError: when bad arguments are given + Returns: + An ordered list of dictionaries of key=>columns/values + """ + + if not isinstance(table_name, str): raise TypeError("Expected a str") + if not isinstance(column_names, list): raise TypeError("Expected a list") + if not isinstance(start_key, str): raise TypeError("Expected a str") + if not isinstance(end_key, str): raise TypeError("Expected a str") + if not isinstance(limit, int) and not isinstance(limit, long): + raise TypeError("Expected an int or long") + if not isinstance(offset, int): raise TypeError("Expected an int") + + # We add extra rows in case we exclude the start/end keys + # This makes sure the limit is upheld correctly + row_count = limit + if not start_inclusive: + row_count += 1 + if not end_inclusive: + row_count += 1 + + results = [] + + cf = pycassa.ColumnFamily(self.pool,table_name) + keyslices = cf.get_range(columns=column_names, + start=start_key, + finish=end_key, + row_count=row_count, + read_consistency_level=CONSISTENCY_QUORUM) + + for key in keyslices: + if keys_only: + results.append(key[0]) + else: + columns = key[1] + col_mapping = {} + for column in columns.items(): + col_name = str(column[0]) + col_val = column[1] + col_mapping[col_name] = col_val + + k = key[0] + v = col_mapping + item = {k:v} + results.append(item) + + + if not start_inclusive and len(results) > 0: + if start_key in results[0]: + results = results[1:] + + if not end_inclusive and len(results) > 0: + if end_key in results[-1]: + results = results[:-1] + + if len(results) > limit: + results = results[:limit] + + if offset != 0 and offset <= len(results): + results = results[offset:] + + return results + diff --git a/AppDB/cassandra/prime_cassandra.py b/AppDB/cassandra/prime_cassandra.py index 0c0ab3f549..acc83459bb 100755 --- a/AppDB/cassandra/prime_cassandra.py +++ b/AppDB/cassandra/prime_cassandra.py @@ -1,50 +1,111 @@ #!/usr/bin/env python -import sys, time +import pycassa +import sys +import time +import dbconstants +import helper_functions import py_cassandra -from dbconstants import * -import pycassa -from pycassa.system_manager import * -CASS_PORT = 9160 + +from cassandra import cassandra_interface + +from pycassa import system_manager + def create_keyspaces(replication): - print "Creating Key Spaces" - f = open(APPSCALE_HOME + '/.appscale/my_private_ip', 'r') - host = f.read() - sys = SystemManager(host + ":" + str(CASS_PORT)) + """ + Creates keyspace which AppScale uses for storing application + and user data + + Args: + replication: Replication factor for Cassandra + Raises: + AppScaleBadArg: When args are bad + """ + if int(replication) <= 0: + raise dbconstants.AppScaleBadArg("Replication must be greater than zero") + + print "Creating Cassandra Key Spaces" + + # Set this to False to keep data from a previous deployment. Setting it + # it to True will remove previous tables. + _DROP_TABLES = True + + # TODO use shared library to get constants + host = helper_functions.read_file('/etc/appscale/my_private_ip') + + sysman = system_manager.SystemManager(host + ":" +\ + str(cassandra_interface.CASS_DEFAULT_PORT)) + + if _DROP_TABLES: + try: + sysman.drop_keyspace(cassandra_interface.KEYSPACE) + except pycassa.cassandra.ttypes.InvalidRequestException, e: + pass try: - sys.drop_keyspace('Keyspace1') - except pycassa.cassandra.ttypes.InvalidRequestException, e: - pass - - sys.create_keyspace('Keyspace1', pycassa.SIMPLE_STRATEGY, {'replication_factor':str(replication)}) - sys.create_column_family('Keyspace1', 'Standard1', #column_type="Standard", - comparator_type=UTF8_TYPE) - sys.create_column_family('Keyspace1', 'Standard2', #column_type="Standard", - comparator_type=UTF8_TYPE) - sys.create_column_family('Keyspace1', 'StandardByTime1', #column_type="Standard", - comparator_type=TIME_UUID_TYPE) - sys.create_column_family('Keyspace1', 'StandardByTime2', #column_type="Standard", - comparator_type=TIME_UUID_TYPE) - #sys.create_column_family('Keyspace1', 'Super1', column_type="Super", - # comparator_type=UTF8_TYPE) - #sys.create_column_family('Keyspace1', 'Super2', column_type="Super", - # comparator_type=UTF8_TYPE) - sys.close() - print "SUCCESS" + sysman.create_keyspace(cassandra_interface.KEYSPACE, + pycassa.SIMPLE_STRATEGY, + {'replication_factor':str(replication)}) + + # This column family is for testing for functional testing + sysman.create_column_family(cassandra_interface.KEYSPACE, + cassandra_interface.STANDARD_COL_FAM, + comparator_type=system_manager.UTF8_TYPE) + + for table_name in dbconstants.INITIAL_TABLES: + sysman.create_column_family(cassandra_interface.KEYSPACE, + table_name, + comparator_type=system_manager.UTF8_TYPE) + + sysman.close() + # TODO: Figure out the exact exceptions we're trying to catch in the + # case where we are doing data persistance + except Exception, e: + sysman.close() + # TODO: Figure out the exact exceptions we're trying to catch in the + print "Received an exception of type " + str(e.__class__) +\ + " with message: " + str(e) + if _DROP_TABLES: + raise e + + print "CASSANDRA SETUP SUCCESSFUL" + return True def prime_cassandra(replication): + """ + Create required tables for AppScale + + Args: + replication: Replication factor of data + Raises: + AppScaleBadArg if replication factor is not greater than 0 + Cassandra specific exceptions upon failure + Returns: + 0 on success, 1 on failure. Passed up as process exit value. + """ + if int(replication) <= 0: + raise AppScaleBadArg("Replication must be greater than zero") + create_keyspaces(int(replication)) - print "prime cassandra database" - db = py_cassandra.DatastoreProxy() - #print db.get("__keys_") - db.create_table(USERS_TABLE, USERS_SCHEMA) - db.create_table(APPS_TABLE, APPS_SCHEMA) - if len(db.get_schema(USERS_TABLE)) > 1 and len(db.get_schema(APPS_TABLE)) > 1: + print "Prime Cassandra database" + try: + db = py_cassandra.DatastoreProxy() + db.create_table(dbconstants.USERS_TABLE, dbconstants.USERS_SCHEMA) + db.create_table(dbconstants.APPS_TABLE, dbconstants.APPS_SCHEMA) + # TODO: Figure out the exact exceptions we're trying to catch in the + # case where we are doing data persistance + except Exception, e: + print "Received an exception of type " + str(e.__class__) +\ + " with message: " + str(e) + if _DROP_TABLES: + raise e + + if len(db.get_schema(dbconstants.USERS_TABLE)) > 1 and \ + len(db.get_schema(dbconstants.APPS_TABLE)) > 1: print "CREATE TABLE SUCCESS FOR USER AND APPS" - print db.get_schema(USERS_TABLE) - print db.get_schema(APPS_TABLE) + print "USERS:",db.get_schema(dbconstants.USERS_TABLE) + print "APPS:",db.get_schema(dbconstants.APPS_TABLE) return 0 else: print "FAILED TO CREATE TABLE FOR USER AND APPS" diff --git a/AppDB/cassandra/test_cassandra.py b/AppDB/cassandra/test_cassandra.py deleted file mode 100644 index c63ce2d3f6..0000000000 --- a/AppDB/cassandra/test_cassandra.py +++ /dev/null @@ -1,89 +0,0 @@ -import py_cassandra - -py_cassandra = py_cassandra.DatastoreProxy() - -columns = ["a","b","c"] -data = ["1","2","3"] -table_name = "hello" -key = "1" -print "key= " + key -print "columns= " + str(columns) -print "data= " + str(data) -print "table= " + table_name -#print py_cassandra.put_entity("__hi__", key, columns, data) -#print py_cassandra.put_entity("__hah__", key, columns, data) -#exit(0) -print py_cassandra.put_entity(table_name, key, columns, data) -ret = py_cassandra.get_entity(table_name, key, columns) -print "doing a put then get" -print ret -if ret[1:] != data: - print "ERROR doing a put then get. Data does not match" - print "returned: " + str(ret) - print "expected: " + str(data) - exit(1) -else: - print "Success" - -ret = py_cassandra.get_schema("hello") -print ret -print "checking schema:" -print ret -if ret[1:] != columns: - print "ERROR in recieved schema" - print "returned: " + str(ret) - print "expected: " + str(columns) - -ret = py_cassandra.delete_row(table_name, key) -print "Deleting the key %s"%key -print ret - -ret = py_cassandra.get_entity(table_name, key, columns) -print "Trying to get deleted key:" -print ret -print "doing a put with key %s"%key -print py_cassandra.put_entity("hello", "1", ["a","b","c"], ["1","2","3"]) -print "doing a get table" -print py_cassandra.get_table("hello", ["a","b","c"]) -py_cassandra.put_entity("hello", "2", ["a","b","c"], ["4","5","6"]) -print "doing get table:" -print py_cassandra.get_table("hello", ["a","b","c"]) -py_cassandra.put_entity("hello", "3", ["a","b","c"], ["1","2","3"]) -py_cassandra.get_table("hello", ["a","b","c"]) - -print "TRYING TO REPLACE KEY 3" -py_cassandra.put_entity("hello", "3", ["a","b","c"], ["1","2","3"]) -print "TRYING TO REPLACE KEY 3" -py_cassandra.get_table("hello", ["a","b","c"]) -print "TRYING TO REPLACE KEY 3" -ret = py_cassandra.delete_row("hello", "1") -print "TRYING TO REPLACE KEY 3" -ret = py_cassandra.delete_row("hello", "2") -print "TRYING TO REPLACE KEY 3" -ret = py_cassandra.delete_row("hello", "3") -print "TRYING TO REPLACE KEY 3" -py_cassandra.get_table("hello", ["a","b","c"]) -print "Deleting table:" -print py_cassandra.delete_table("hello") -print "deleting twice:" -print py_cassandra.delete_table("hello") - -table_name = u"testing_query" -print py_cassandra.delete_table(table_name) -column_names = [u"c1"] -limit = 1000 -offset = 0 -key = 0 -startrow = u"000" -endrow = u"100" -data = u"xxx" -for ii in range(0, 101): - key = str(ii) - key = ("0" * (3 - len(key))) + key - key = unicode(key) - print "Adding key " + key - print py_cassandra.put_entity(table_name, key, column_names, [data + key]) -inclusive = 1 -notJustKeys = 0 -print "SUCCESS" - diff --git a/AppDB/datastore_perf.py b/AppDB/datastore_perf.py index 626117a121..1a357b4bed 100644 --- a/AppDB/datastore_perf.py +++ b/AppDB/datastore_perf.py @@ -60,11 +60,11 @@ def getStDev(points, average=None): def createRandomList(number_of_columns, column_name_len): columns = [] for ii in range(0, number_of_columns): - columns += [hf.randomString(column_name_len)] + columns += [hf.random_string(column_name_len)] return columns columns = createRandomList(NUM_COLUMNS, 10) data = createRandomList(NUM_COLUMNS, 100) -table_name = hf.randomString(10) +table_name = hf.random_string(10) NUM_ACC = 10001 print "table= " + table_name #print "columns= " + str(columns) diff --git a/AppDB/datastore_server.py b/AppDB/datastore_server.py new file mode 100644 index 0000000000..5cd01a287b --- /dev/null +++ b/AppDB/datastore_server.py @@ -0,0 +1,1942 @@ +#!/usr/bin/python +# Programmer: Navraj Chohan +# See LICENSE file +# + +import __builtin__ +import datetime +import getopt +import itertools +import md5 +import os +import random +import SOAPpy +import sys +import socket +import time +import threading +import types + +import tornado.httpserver +import tornado.ioloop +import tornado.web + +import appscale_logger +import appscale_datastore +import appscale_datastore_batch +import dbconstants +import helper_functions + +from google.appengine.api import api_base_pb +from google.appengine.api import datastore +from google.appengine.api import datastore_errors +from google.appengine.api import datastore_types +from google.appengine.api import users + +from google.appengine.datastore import cassandra_stub_util +from google.appengine.datastore import datastore_pb +from google.appengine.datastore import datastore_index +from google.appengine.datastore import datastore_stub_util +from google.appengine.datastore import entity_pb +from google.appengine.datastore import sortable_pb_encoder + +from google.appengine.runtime import apiproxy_errors +from google.appengine.ext.remote_api import remote_api_pb +from google.net.proto import ProtocolBuffer + +from drop_privileges import * +from SocketServer import BaseServer +from M2Crypto import SSL + +# Buffer type used for key storage in the datastore +buffer = __builtin__.buffer + +# Global for accessing the datastore. An instance of DatastoreDistributed. +datastore_access = None + +entity_pb.Reference.__hash__ = lambda self: hash(self.Encode()) +datastore_pb.Query.__hash__ = lambda self: hash(self.Encode()) + +# The datastores supported for this version of the AppScale datastore +VALID_DATASTORES = ['cassandra', 'hbase', 'hypertable'] + +# Port this service binds to if using SSL +DEFAULT_SSL_PORT = 8443 + +# Port this service binds to (unencrypted and hence better performance) +DEFAULT_PORT = 4080 + +# IDs are acquired in block sizes of this +BLOCK_SIZE = 10000 + +class DatastoreDistributed(): + """ AppScale persistent layer for the datastore API. It is the + replacement for the AppServers to persist their data into + a distributed datastore instead of a flat file. + """ + # Max number of results for a query + _MAXIMUM_RESULTS = 1000000 + + # The number of entries looked at when doing a composite query + # It will keep looking at this size window when getting the result + _MAX_COMPOSITE_WINDOW = 1000 + + # Maximum amount of filter and orderings allowed within a query + _MAX_QUERY_COMPONENTS = 63 + + + # For enabling and disabling range inclusivity + _ENABLE_INCLUSIVITY = True + _DISABLE_INCLUSIVITY = False + + # Delimiter between app names and namespace and the rest of an entity key + _NAMESPACE_SEPARATOR = '/' + + # This is the terminating string for range queries + _TERM_STRING = chr(255) * 500 + + + def __init__(self, datastore_batch): + """ + Constructor. + + Args: + datastore_batch: a reference to the batch datastore interface + """ + # Each entry contains a tuple (last_accessed_timestamp, namespace) + # The key is the / + self.__namespaces = [] + + # Each entry contains a triple (last_accessed_timestamp, start, end) + # The key is the // + # IDs are inclusive + self.__id_map = {} + + # Each entry contains a tuple (last_accessed_timestamp, index_name) + # The key is the /// + self.__indexes = {} + + #TODO: Use locks for operations that should be atomic, i.e., we are + # updating global shared state + # lock for namespace and indexes during periodic garbage collection + self.__lock = threading.Lock() + + # datastore accessor used by this class to do datastore operations + self.datastore_batch = datastore_batch + + @staticmethod + def get_entity_kind(key_path): + """ Returns the Kind of the Entity. A Kind is like a type or a + particular class of entity. + + Args: + key_path: the key path of entity + Returns: + kind of the entity + """ + + if isinstance(key_path, entity_pb.EntityProto): + key_path = key_path.key() + return key_path.path().element_list()[-1].type() + + def get_entity_key(self, prefix, pb): + """ Returns the key for the entity table + + Args: + prefix: app name and namespace string + example-- 'guestbook/mynamespace' + pb: protocol buffer for which we will encode the index name + Returns: + Key for entity table + """ + return buffer(prefix + self._NAMESPACE_SEPARATOR) + self.__encode_index_pb(pb) + + def get_kind_key(self, prefix, key_path): + """ Returns a key for the kind table + + Args: + prefix: app name and namespace string + key_path: key path to build row key with + Returns: + Row key for kind table + """ + path = [] + # reverse of index paths because child kind must come first + all_reversed = key_path.element_list()[::-1] + for e in all_reversed: + if e.has_name(): + key_id = e.name() + elif e.has_id(): + # make sure ids are ordered lexigraphically by making sure they + # are of set size i.e. 2 > 0003 but 0002 < 0003 + key_id = str(e.id()).zfill(10) + path.append('%s:%s' % (e.type(), key_id)) + encoded_path = '!'.join(path) + encoded_path += '!' + + return prefix + self._NAMESPACE_SEPARATOR + encoded_path + + @staticmethod + def __encode_index_pb(pb): + """ Returns an encoded buffer + + Args: + pb: The protocol buffer to encode + Returns: + encoded pb + """ + + def _encode_path(pb): + """ Takes a protocol buffer and returns the encoded path """ + + path = [] + for e in pb.element_list(): + if e.has_name(): + key_id = e.name() + elif e.has_id(): + key_id = str(e.id()).zfill(10) + path.append('%s:%s' % (e.type(), key_id)) + val = '!'.join(path) + val += '!' + return val + + if isinstance(pb, entity_pb.PropertyValue) and pb.has_uservalue(): + userval = entity_pb.PropertyValue() + userval.mutable_uservalue().set_email(pb.uservalue().email()) + userval.mutable_uservalue().set_auth_domain(pb.uservalue().auth_domain()) + userval.mutable_uservalue().set_gaiaid(0) + pb = userval + + encoder = sortable_pb_encoder.Encoder() + pb.Output(encoder) + + if isinstance(pb, entity_pb.PropertyValue): + return buffer(encoder.buffer().tostring()) + elif isinstance(pb, entity_pb.Path): + return buffer(_encode_path(pb)) + + def validate_app_id(self, app_id): + """ Verify that this is the stub for app_id. + + Args: + app_id: An application ID. + Raises: + AppScaleBadArg: if name is not set + """ + + if not app_id: + raise dbconstants.AppScaleBadArg("Application name must be set") + + def validate_key(self, key): + """ Validate this key by checking to see if it has a name or id. + + Args: + key: entity_pb.Reference + Raises: + datastore_errors.BadRequestError: if the key is invalid + TypeError: if key is not of entity_pb.Reference + """ + + if not isinstance(key, entity_pb.Reference): + raise TypeError("Expected type Reference") + + self.validate_app_id(key.app()) + + for elem in key.path().element_list(): + if elem.has_id() and elem.has_name(): + raise datastore_errors.BadRequestError( + 'each key path element should have id or name but not both: %r' + % key) + + def get_index_key(self, app_id, name_space, kind, index_name): + """ Returns key string for storing namespaces. + Args: + app_id: The app ID. + name_space: The per-app namespace name. + kind: The per-app kind name. + index_name: The per-app index name. + Returns: + Key string for storing namespaces + """ + + return app_id + "/" + name_space + "/" + kind + "/" + index_name + + def configure_namespace(self, prefix, app_id, name_space): + """ Stores a key for the given namespace. + + Args: + prefix: The namespace prefix to configure. + app_id: The app ID. + name_space: The per-app namespace name. + """ + + vals = {} + row_key = prefix + vals[row_key] = {"namespaces":name_space} + self.datastore_batch.batch_put_entity(dbconstants.APP_NAMESPACE_TABLE, + [row_key], + dbconstants.APP_NAMESPACE_SCHEMA, + vals) + + + def get_table_prefix(self, data): + """ Returns the namespace prefix for a query. + + Args: + data: An Entity, Key or Query PB, or an (app_id, ns) tuple. + Returns: + A valid table prefix + """ + if isinstance(data, entity_pb.EntityProto): + data = data.key() + + if not isinstance(data, tuple): + data = (data.app(), data.name_space()) + + prefix = ('%s/%s' % data).replace('"', '""') + + if data not in self.__namespaces: + self.configure_namespace(prefix, *data) + self.__namespaces.append(data) + + return prefix + + def get_index_key_from_params(self, params): + """Returns the index key from params + Args: + params: a list of strings to be concatenated to form the key made of: + prefix, kind, property name, and path + Returns: + a string + Raises: + ValueError: if params are not of the correct cardinality + """ + if len(params) != 5 and len(params) != 4: raise ValueError + + if params[-1] == None: + # strip off the last None item + key = '/'.join(params[:-1]) + '/' + else: + key = '/'.join(params) + return key + + def get_index_kv_from_tuple(self, tuple_list, reverse=False): + """ Returns keys/value of indexes for a set of entities + + Args: + tuple_list: A list of tuples of prefix and pb entities + reverse: if these keys are for the descending table + Returns: + A list of keys and values of indexes + """ + def row_generator(entities,rev): + all_rows = [] + for prefix, e in entities: + for p in e.property_list(): + val = str(self.__encode_index_pb(p.value())) + # Remove the first binary character for lexigraphical ordering + val = str(val[1:]) + + if rev: + val = helper_functions.reverse_lex(val) + + params = [prefix, + self.get_entity_kind(e), + p.name(), + val, + str(self.__encode_index_pb(e.key().path()))] + + index_key = self.get_index_key_from_params(params) + p_vals = [index_key, + buffer(prefix + '/') + \ + self.__encode_index_pb(e.key().path())] + all_rows.append(p_vals) + return tuple(ii for ii in all_rows) + return row_generator(tuple_list, reverse) + + def delete_index_entries(self, entities): + """ Deletes the entities in the DB + + Args: + entities: A list of entities for which their + indexes are to be deleted + """ + + if len(entities) == 0: return + + entities_tuple = sorted((self.get_table_prefix(x), x) for x in entities) + asc_index_keys = self.get_index_kv_from_tuple(entities_tuple, + reverse=False) + desc_index_keys = self.get_index_kv_from_tuple(entities_tuple, + reverse=True) + # Remove the value, just get keys + asc_index_keys = [x[0] for x in asc_index_keys] + desc_index_keys = [x[0] for x in desc_index_keys] + # TODO Consider doing these in parallel with threads + self.datastore_batch.batch_delete(dbconstants.ASC_PROPERTY_TABLE, + asc_index_keys, + column_names=dbconstants.PROPERTY_SCHEMA) + self.datastore_batch.batch_delete(dbconstants.DSC_PROPERTY_TABLE, + desc_index_keys, + column_names=dbconstants.PROPERTY_SCHEMA) + + def insert_entities(self, entities): + """Inserts or updates entities in the DB. + Args: + entities: A list of entities to store. + """ + + def row_generator(entities): + for prefix, e in entities: + yield (self.get_entity_key(prefix, e.key().path()), + buffer(e.Encode())) + + def kind_row_generator(entities): + for prefix, e in entities: + # yield a tuple of kind key and a reference to entity table + yield (self.get_kind_key(prefix, e.key().path()), + self.get_entity_key(prefix, e.key().path())) + row_values = {} + row_keys = [] + + kind_row_keys = [] + kind_row_values = {} + + entities = sorted((self.get_table_prefix(x), x) for x in entities) + for prefix, group in itertools.groupby(entities, lambda x: x[0]): + group_rows = tuple(row_generator(group)) + new_row_keys = [str(ii[0]) for ii in group_rows] + row_keys += new_row_keys + for ii in group_rows: + row_values[str(ii[0])] = {dbconstants.APP_ENTITY_SCHEMA[0]:str(ii[1]), #ent + dbconstants.APP_ENTITY_SCHEMA[1]:"0"} #txnid + + for prefix, group in itertools.groupby(entities, lambda x: x[0]): + kind_group_rows = tuple(kind_row_generator(group)) + new_kind_keys = [str(ii[0]) for ii in kind_group_rows] + kind_row_keys += new_kind_keys + + for ii in kind_group_rows: + kind_row_values[str(ii[0])] = {dbconstants.APP_KIND_SCHEMA[0]:str(ii[1])} + + + # TODO do these in || + self.datastore_batch.batch_put_entity(dbconstants.APP_ENTITY_TABLE, + row_keys, + dbconstants.APP_ENTITY_SCHEMA, + row_values) + + self.datastore_batch.batch_put_entity(dbconstants.APP_KIND_TABLE, + kind_row_keys, + dbconstants.APP_KIND_SCHEMA, + kind_row_values) + + def insert_index_entries(self, entities): + """ Inserts index entries for the supplied entities. + + Args: + entities: A list of tuples of prefix and entities + to create index entries for. + """ + + entities = sorted((self.get_table_prefix(x), x) for x in entities) + asc_index_keys = self.get_index_kv_from_tuple(entities, reverse=False) + desc_index_keys = self.get_index_kv_from_tuple(entities, reverse=True) + + row_keys = [] + rev_row_keys = [] + row_values = {} + rev_row_values = {} + + for prefix, group in itertools.groupby(entities, lambda x: x[0]): + group_rows = self.get_index_kv_from_tuple(group,False) + row_keys = [str(ii[0]) for ii in group_rows] + for ii in group_rows: + row_values[str(ii[0])] = {'reference':str(ii[1])} + + for prefix, group in itertools.groupby(entities, lambda x: x[0]): + rev_group_rows = self.get_index_kv_from_tuple(group,True) + rev_row_keys = [str(ii[0]) for ii in rev_group_rows] + for ii in rev_group_rows: + rev_row_values[str(ii[0])] = {'reference':str(ii[1])} + + # TODO these in parallel + self.datastore_batch.batch_put_entity(dbconstants.ASC_PROPERTY_TABLE, + row_keys, + dbconstants.PROPERTY_SCHEMA, + row_values) + + self.datastore_batch.batch_put_entity(dbconstants.DSC_PROPERTY_TABLE, + rev_row_keys, + dbconstants.PROPERTY_SCHEMA, + rev_row_values) + + def acquire_id_block_from_db(self, prefix): + """ Gets a block of keys from the DB + + Args: + prefix: A table namespace prefix + Returns: + next id available + """ + + res = self.datastore_batch.batch_get_entity(dbconstants.APP_ID_TABLE, + [prefix], + dbconstants.APP_ID_SCHEMA) + if dbconstants.APP_ID_SCHEMA[0] in res[prefix]: + return int(res[prefix][dbconstants.APP_ID_SCHEMA[0]]) + return 0 + + def increment_id_in_db(self, prefix): + """ Updates the counter for a prefix to the DB + + Args: + prefix: A table namespace prefix + Returns: + next_block id + """ + # TODO getting and updating a block needs to be transactional + current_block = self.acquire_id_block_from_db(prefix) + next_block = current_block + 1 + cell_values = {prefix:{dbconstants.APP_ID_SCHEMA[0]:str(next_block)}} + + res = self.datastore_batch.batch_put_entity(dbconstants.APP_ID_TABLE, + [prefix], + dbconstants.APP_ID_SCHEMA, + cell_values) + return next_block * BLOCK_SIZE + + def allocate_ids(self, prefix, size): + """ Allocates IDs. + + Args: + prefix: A table namespace prefix. + size: Number of IDs to allocate. + Returns: + start and end ids: The beginning of a range of size IDs + Raises: + ValueError: if size is less than or equal to 0 + """ + # TODO make this thread safe because we're accessing global variables + if size <= 0: raise ValueError + + next_id, end_id = self.__id_map.get(prefix, (0, 0)) + if next_id == end_id or (end_id and (next_id + size > end_id)): + # Acquire a new block of ids, throw out the old ones + next_id = self.increment_id_in_db(prefix) + end_id = next_id + BLOCK_SIZE - 1 + + start = next_id + end = next_id + size - 1 + + self.__id_map[prefix] = (next_id + size, end_id) + return start, end + + def put_entities(self, entities): + """ Updates indexes of existing entities, inserts new entities and + indexes for them + Args: + entities: list of entities + """ + sorted_entities= sorted((self.get_table_prefix(x), x) for x in entities) + for prefix, group in itertools.groupby(sorted_entities, lambda x: x[0]): + keys = [e.key() for e in entities] + self.delete_entities(keys) + self.insert_entities(entities) + self.insert_index_entries(entities) + + def delete_entities(self, keys): + """ Deletes the entities and the indexes associated with them. + Args: + keys: list of keys to be deleted + """ + def row_generator(key_list): + for prefix, k in key_list: + yield (self.get_entity_key(prefix, k.path()), + buffer(k.Encode())) + + def kind_row_generator(key_list): + for prefix, k in key_list: + # yield a tuple of kind key and a reference to entity table + yield (self.get_kind_key(prefix, k.path()), + self.get_entity_key(prefix, k.path())) + + row_keys = [] + kind_keys = [] + + entities = sorted((self.get_table_prefix(x), x) for x in keys) + + for prefix, group in itertools.groupby(entities, lambda x: x[0]): + group_rows = tuple(row_generator(group)) + new_row_keys = [str(ii[0]) for ii in group_rows] + row_keys += new_row_keys + + for prefix, group in itertools.groupby(entities, lambda x: x[0]): + group_rows = tuple(kind_row_generator(group)) + new_row_keys = [str(ii[0]) for ii in group_rows] + kind_keys += new_row_keys + + # Must fetch the entities to get the keys of indexes before deleting + ret = self.datastore_batch.batch_get_entity(dbconstants.APP_ENTITY_TABLE, + row_keys, + dbconstants.APP_ENTITY_SCHEMA) + + #TODO do these in || + self.datastore_batch.batch_delete(dbconstants.APP_ENTITY_TABLE, + row_keys, + column_names=dbconstants.APP_ENTITY_SCHEMA) + + self.datastore_batch.batch_delete(dbconstants.APP_KIND_TABLE, + kind_keys, + column_names=dbconstants.APP_KIND_SCHEMA) + + entities = [] + for row_key in ret: + # Entities may not exist if this is the first put + if 'entity' in ret[row_key]: + ent = entity_pb.EntityProto() + ent.ParseFromString(ret[row_key]['entity']) + entities.append(ent) + + self.delete_index_entries(entities) + + def _dynamic_put(self, app_id, put_request, put_response): + """ Stores and entity and its indexes in the datastore + + Args: + app_id: Application ID + put_request: Request with entities to store + put_response: The response sent back to the app server + """ + + entities = put_request.entity_list() + keys = [e.key() for e in entities] + for entity in entities: + self.validate_key(entity.key()) + + for prop in itertools.chain(entity.property_list(), + entity.raw_property_list()): + if prop.value().has_uservalue(): + uid = md5.new(prop.value().uservalue().email().lower()).digest() + uid = '1' + ''.join(['%02d' % ord(x) for x in uid])[:20] + prop.mutable_value().mutable_uservalue().set_obfuscated_gaiaid(uid) + + last_path = entity.key().path().element_list()[-1] + if last_path.id() == 0 and not last_path.has_name(): + + id_, ignored = self.allocate_ids(self.get_table_prefix(entity.key()), 1) + last_path.set_id(id_) + + group = entity.mutable_entity_group() + root = entity.key().path().element(0) + group.add_element().CopyFrom(root) + + + self.put_entities(entities) + put_response.key_list().extend([e.key() for e in entities]) + + def fetch_keys(self, key_list): + """ Given a list of keys fetch the entities. + + Args: + key_list: A list of keys to fetch + Returns: + A tuple of entities from the datastore and key list + """ + row_keys = [] + for key in key_list: + self.validate_app_id(key.app()) + index_key = str(self.__encode_index_pb(key.path())) + prefix = self.get_table_prefix(key) + row_keys.append(prefix + '/' + index_key) + result = self.datastore_batch.batch_get_entity(dbconstants.APP_ENTITY_TABLE, + row_keys, + dbconstants.APP_ENTITY_SCHEMA) + return (result, row_keys) + + def _dynamic_get(self, get_request, get_response): + """ Fetch keys from the datastore. + + Args: + get_request: Request with list of keys + get_response: Response to application server + """ + + keys = get_request.key_list() + results, row_keys = self.fetch_keys(keys) + for r in row_keys: + if r in results and 'entity' in results[r]: + group = get_response.add_entity() + group.mutable_entity().CopyFrom( + entity_pb.EntityProto(results[r]['entity'])) + + def _dynamic_delete(self, delete_request, delete_response): + """ Deletes a set of rows. + + Args: + delete_request: Request with a list of keys + delete_response: Response to application server + """ + keys = delete_request.key_list() + self.delete_entities(delete_request.key_list()) + + def generate_filter_info(self, filters, query): + """Transform a list of filters into a more usable form. + + Args: + filters: A list of filter PBs. + query: The query to generate filter info for. + Returns: + A dict mapping property names to lists of (op, value) tuples. + """ + + def reference_property_to_reference(refprop): + ref = entity_pb.Reference() + ref.set_app(refprop.app()) + if refprop.has_name_space(): + ref.set_name_space(refprop.name_space()) + for pathelem in refprop.pathelement_list(): + ref.mutable_path().add_element().CopyFrom(pathelem) + return ref + + filter_info = {} + for filt in filters: + prop = filt.property(0) + value = prop.value() + if prop.name() == '__key__': + value = reference_property_to_reference(value.referencevalue()) + value = value.path() + filter_info.setdefault(prop.name(), []).append((filt.op(), + self.__encode_index_pb(value))) + return filter_info + + def generate_order_info(self, orders): + """Transform a list of orders into a more usable form which + is a tuple of properties and ordering directions. + + Args: + orders: A list of order PBs. + Returns: + A list of (property, direction) tuples. + """ + orders = [(order.property(), order.direction()) for order in orders] + if orders and orders[-1] == ('__key__', datastore_pb.Query_Order.ASCENDING): + orders.pop() + return orders + + def __get_start_key(self, prefix, prop_name, order, last_result): + """ Builds the start key for cursor query + + Args: + prop_name: property name of the filter + order: sort order + last_result: last result encoded in cursor + """ + e = last_result + start_key = None + if not prop_name and not order: + return str(prefix + '/' + str(self.__encode_index_pb(e.key().path()))) + + if e.property_list(): + plist = e.property_list() + else: + rkey = prefix + '/' + str(self.__encode_index_pb(e.key().path())) + ret = self.datastore_batch.batch_get_entity(dbconstants.APP_ENTITY_TABLE, + [rkey], + dbconstants.APP_ENTITY_SCHEMA) + if 'entity' in ret[rkey]: + ent = entity_pb.EntityProto(ret[rkey]['entity']) + plist = ent.property_list() + + for p in plist: + if p.name() == prop_name: + break + + val = str(self.__encode_index_pb(p.value())) + # remove first binary char for correct lexigraphical ordering + val = str(val[1:]) + + if order == datastore_pb.Query_Order.DESCENDING: + val = helper_functions.reverse_lex(val) + params = [prefix, + self.get_entity_kind(e), + p.name(), + val, + str(self.__encode_index_pb(e.key().path()))] + + return self.get_index_key_from_params(params) + + def __fetch_entities(self, refs): + """ Given the results from a table scan, get the references + + Args: + refs: key/value pairs where the values contain a reference to + the entitiy table + Returns: + Entities retrieved from entity table + """ + if len(refs) == 0: + return [] + keys = [item.keys()[0] for item in refs] + rowkeys = [] + for index, ent in enumerate(refs): + key = keys[index] + ent = ent[key]['reference'] + rowkeys.append(ent) + + result = self.datastore_batch.batch_get_entity(dbconstants.APP_ENTITY_TABLE, + rowkeys, + dbconstants.APP_ENTITY_SCHEMA) + entities = [] + keys = result.keys() + for key in rowkeys: + if 'entity' in result[key]: + entities.append(result[key]['entity']) + + return entities + + def __extract_entities(self, kv): + """ Given a result from a range query on the Entity table return a + list of encoded entities + Args: + kv: Key and values from a range query on the entity table + Returns: + The extracted entities + """ + keys = [item.keys()[0] for item in kv] + results = [] + for index, entity in enumerate(kv): + key = keys[index] + entity = entity[key]['entity'] + results.append(entity) + + return results + + def __AncestorQuery(self, query, filter_info, order_info): + """ Performs ancestor queries which is where you select + entities based on a particular root entitiy. + + Args: + query: The query to run + filter_info: tuple with filter operators and values + order_info: tuple with property name and the sort order + Returns: + start and end row keys + """ + ancestor = query.ancestor() + prefix = self.get_table_prefix(query) + path = buffer(prefix + '/') + self.__encode_index_pb(ancestor.path()) + + startrow = path + endrow = path + self._TERM_STRING + + end_inclusive = self._ENABLE_INCLUSIVITY + start_inclusive = self._ENABLE_INCLUSIVITY + + if '__key__' in filter_info: + op = filter_info['__key__'][0][0] + __key__ = str(filter_info['__key__'][0][1]) + if op and op == datastore_pb.Query_Filter.EQUAL: + startrow = prefix + '/' + __key__ + endrow = prefix + '/' + __key__ + elif op and op == datastore_pb.Query_Filter.GREATER_THAN: + start_inclusive = self._DISABLE_INCLUSIVITY + startrow = prefix + '/' + __key__ + elif op and op == datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: + startrow = prefix + '/' + __key__ + elif op and op == datastore_pb.Query_Filter.LESS_THAN: + endrow = prefix + '/' + __key__ + end_inclusive = self._DISABLE_INCLUSIVITY + elif op and op == datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL: + endrow = prefix + '/' + __key__ + + column_names = ['reference'] + if not order_info: + order = None + prop_name = None + + if query.has_compiled_cursor() and query.compiled_cursor().position_size(): + cursor = cassandra_stub_util.ListCursor(query) + last_result = cursor._GetLastResult() + startrow = self.__get_start_key(prefix, prop_name, order, last_result) + start_inclusive = self._DISABLE_INCLUSIVITY + + limit = query.limit() or self._MAXIMUM_RESULTS + + offset = query.offset() + + result = self.datastore_batch.range_query(dbconstants.APP_ENTITY_TABLE, + dbconstants.APP_ENTITY_SCHEMA, + startrow, + endrow, + limit, + offset=offset, + start_inclusive=start_inclusive, + end_inclusive=end_inclusive) + return self.__extract_entities(result) + + def __KindlessQuery(self, query, filter_info, order_info): + """ Performs kindless queries where queries are performed + on the entity table and go across kinds. + + Args: + query: The query to run + filter_info: tuple with filter operators and values + order_info: tuple with property name and the sort order + Returns: + Entities that match the query + """ + prefix = self.get_table_prefix(query) + + __key__ = str(filter_info['__key__'][0][1]) + op = filter_info['__key__'][0][0] + + end_inclusive = self._ENABLE_INCLUSIVITY + start_inclusive = self._ENABLE_INCLUSIVITY + + startrow = prefix + '/' + __key__ + self._TERM_STRING + endrow = prefix + '/' + self._TERM_STRING + if op and op == datastore_pb.Query_Filter.EQUAL: + startrow = prefix + '/' + __key__ + endrow = prefix + '/' + __key__ + elif op and op == datastore_pb.Query_Filter.GREATER_THAN: + start_inclusive = self._DISABLE_INCLUSIVITY + startrow = prefix + '/' + __key__ + endrow = prefix + '/' + self._TERM_STRING + end_inclusive = self._DISABLE_INCLUSIVITY + elif op and op == datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: + startrow = prefix + '/' + __key__ + endrow = prefix + '/' + self._TERM_STRING + end_inclusive = self._DISABLE_INCLUSIVITY + elif op and op == datastore_pb.Query_Filter.LESS_THAN: + startrow = prefix + '/' + endrow = prefix + '/' + __key__ + end_inclusive = self._DISABLE_INCLUSIVITY + elif op and op == datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL: + startrow = prefix + '/' + endrow = prefix + '/' + __key__ + + if not order_info: + order = None + prop_name = None + + if query.has_compiled_cursor() and query.compiled_cursor().position_size(): + cursor = cassandra_stub_util.ListCursor(query) + last_result = cursor._GetLastResult() + startrow = self.__get_start_key(prefix, prop_name, order, last_result) + start_inclusive = self._DISABLE_INCLUSIVITY + + limit = query.limit() or self._MAXIMUM_RESULTS + offset = query.offset() + + result = self.datastore_batch.range_query(dbconstants.APP_ENTITY_TABLE, + dbconstants.APP_ENTITY_SCHEMA, + startrow, + endrow, + limit, + offset=offset, + start_inclusive=start_inclusive, + end_inclusive=end_inclusive) + + return self.__extract_entities(result) + + def kind_query_range(self, query, filter_info, order_info): + """ Gets start and end keys for kind queries + + Args: + query: The query to run + filter_info: tuple with filter operators and values + order_info: tuple with property name and the sort order + Returns: + Entities that match the query + """ + prefix = self.get_table_prefix(query) + startrow = prefix + '/' + query.kind() + ':' + endrow = prefix + '/' + query.kind() + ':' + self._TERM_STRING + return startrow, endrow + + def __kind_query(self, query, filter_info, order_info): + """ Performs kind only queries, kind and ancestor, and ancestor queries + https://developers.google.com/appengine/docs/python/datastore/queries + Args: + query: The query to run + filter_info: tuple with filter operators and values + order_info: tuple with property name and the sort order + Returns: + An ordered list of entities matching the query + """ + + # Detect quickly if this is a kind query or not + for fi in filter_info: + if fi != "__key__": + return + + if order_info: + if len(order_info) > 0: return None + elif query.has_ancestor(): + return self.__AncestorQuery(query, filter_info, order_info) + elif not query.has_kind(): + return self.__KindlessQuery(query, filter_info, order_info) + + startrow, endrow = self.kind_query_range(query, + filter_info, + order_info) + + if startrow == None: + return None + + end_inclusive = self._ENABLE_INCLUSIVITY + start_inclusive = self._ENABLE_INCLUSIVITY + if not order_info: + order = None + prop_name = None + + if query.has_compiled_cursor() and query.compiled_cursor().position_size(): + cursor = cassandra_stub_util.ListCursor(query) + last_result = cursor._GetLastResult() + prefix = self.get_table_prefix(query) + startrow = self.__get_start_key(prefix, prop_name, order, last_result) + start_inclusive = self._DISABLE_INCLUSIVITY + + limit = query.limit() or self._MAXIMUM_RESULTS + + offset = query.offset() + + result = self.datastore_batch.range_query(dbconstants.APP_KIND_TABLE, + dbconstants.APP_KIND_SCHEMA, + startrow, + endrow, + limit, + offset=offset, + start_inclusive=start_inclusive, + end_inclusive=end_inclusive) + return self.__fetch_entities(result) + + def __single_property_query(self, query, filter_info, order_info): + """Performs queries satisfiable by the Single_Property tables + Args: + query: The query to run + filter_info: tuple with filter operators and values + order_info: tuple with property name and the sort order + Returns: + List of entities retrieved from the given query + """ + property_names = set(filter_info.keys()) + property_names.update(x[0] for x in order_info) + property_names.discard('__key__') + if len(property_names) != 1: + return None + + property_name = property_names.pop() + filter_ops = filter_info.get(property_name, []) + if len([1 for o, _ in filter_ops + if o == datastore_pb.Query_Filter.EQUAL]) > 1: + return None + + if len(order_info) > 1 or (order_info and order_info[0][0] == '__key__'): + return None + + if query.has_ancestor(): + return None + + if not query.has_kind(): + return None + + if order_info: + if order_info[0][0] == property_name: + direction = order_info[0][1] + else: + direction = datastore_pb.Query_Order.ASCENDING + + prefix = self.get_table_prefix(query) + + offset = query.offset() + + limit = query.limit() or self._MAXIMUM_RESULTS + + kind = query.kind() + + if query.has_compiled_cursor() and query.compiled_cursor().position_size(): + cursor = cassandra_stub_util.ListCursor(query) + last_result = cursor._GetLastResult() + startrow = self.__get_start_key(prefix, + property_name, + direction, + last_result) + else: + startrow = None + references = self.__apply_filters(filter_ops, + order_info, + property_name, + query.kind(), + prefix, + limit, + offset, + startrow) + return self.__fetch_entities(references) + + + def __apply_filters(self, + filter_ops, + order_info, + property_name, + kind, + prefix, + limit, + offset, + startrow): + """ + Applies property filters in the query. + Args: + filter_ops: Tuple with property filter operator and value + order_info: Tuple with property name and sort order + kind: Kind of the entity + prefix: Prefix for the table + limit: Number of results + offset: Number of results to skip + startrow: Start key for the range scan + Results: + Returns a list of entity keys + Raises: + NotImplementedError: For unsupported queries. + AppScaleMisconfiguredQuery: Bad filters or orderings + """ + end_inclusive = self._ENABLE_INCLUSIVITY + start_inclusive = self._ENABLE_INCLUSIVITY + + endrow = None + column_names = dbconstants.PROPERTY_SCHEMA + + if order_info: + if order_info[0][0] == property_name: + direction = order_info[0][1] + else: + direction = datastore_pb.Query_Order.ASCENDING + + if direction == datastore_pb.Query_Order.ASCENDING: + table_name = dbconstants.ASC_PROPERTY_TABLE + else: + table_name = dbconstants.DSC_PROPERTY_TABLE + + if startrow: start_inclusive = self._DISABLE_INCLUSIVITY + + # This query is returning based on order on a specfic property name + # The start key (if not already supplied) depends on the property + # name and does not take into consideration its value. The end key + # is based on the terminating string. + if len(filter_ops) == 0 and (order_info and len(order_info) == 1): + end_inclusive = self._ENABLE_INCLUSIVITY + start_inclusive = self._ENABLE_INCLUSIVITY + + if not startrow: + params = [prefix, kind, property_name, None] + startrow = self.get_index_key_from_params(params) + + params = [prefix, kind, property_name, self._TERM_STRING, None] + endrow = self.get_index_key_from_params(params) + + return self.datastore_batch.range_query(table_name, + column_names, + startrow, + endrow, + limit, + offset=offset, + start_inclusive=start_inclusive, + end_inclusive=end_inclusive) + + #TODO byte stuff value for '/' character? Since it's an escape + # character and we might have issues if data contains this char + # This query has a value it bases the query on for a property name + # The difference between operators is what the end and start key are + if len(filter_ops) == 1: + oper = filter_ops[0][0] + value = str(filter_ops[0][1]) + + # Strip off the first char of encoding + value = str(value[1:]) + + if direction == datastore_pb.Query_Order.DESCENDING: + value = helper_functions.reverse_lex(value) + + if oper == datastore_pb.Query_Filter.EQUAL: + start_value = value + end_value = value + self._TERM_STRING + elif oper == datastore_pb.Query_Filter.LESS_THAN: + start_value = None + end_value = value + '/' + if direction == datastore_pb.Query_Order.DESCENDING: + start_value = value + self._TERM_STRING + end_value = self._TERM_STRING + elif oper == datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL: + start_value = None + end_value = value + '/' + self._TERM_STRING + if direction == datastore_pb.Query_Order.DESCENDING: + start_value = value + '/' + end_value = self._TERM_STRING + elif oper == datastore_pb.Query_Filter.GREATER_THAN: + start_value = value + self._TERM_STRING + end_value = self._TERM_STRING + if direction == datastore_pb.Query_Order.DESCENDING: + start_value = None + end_value = value + '/' + elif oper == datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: + start_value = value + '/' + end_value = self._TERM_STRING + if direction == datastore_pb.Query_Order.DESCENDING: + start_value = None + end_value = value + '/' + self._TERM_STRING + elif oper == datastore_pb.Query_Filter.IN: + raise NotImplementedError("IN queries are not implemented") + elif oper == datastore_pb.Query_Filter.EXIST: + raise NotImplementedError("EXIST queries are not implemented") + else: + raise NotImplementedError("Unknown query of operation %d"%oper) + + if not startrow: + params = [prefix, kind, property_name, start_value] + startrow = self.get_index_key_from_params(params) + start_inclusive = self._DISABLE_INCLUSIVITY + params = [prefix, kind, property_name, end_value] + endrow = self.get_index_key_from_params(params) + + ret = self.datastore_batch.range_query(table_name, + column_names, + startrow, + endrow, + limit, + offset=offset, + start_inclusive=start_inclusive, + end_inclusive=end_inclusive) + + return ret + + # Here we have two filters and so we set the start and end key to + # get the given value within those ranges. + if len(filter_ops) > 1: + if filter_ops[0][0] == datastore_pb.Query_Filter.GREATER_THAN or \ + filter_ops[0][0] == datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: + oper1 = filter_ops[0][0] + oper2 = filter_ops[1][0] + value1 = str(filter_ops[0][1]) + value1 = str(value1[1:]) + value2 = str(filter_ops[1][1]) + value2 = str(value2[1:]) + else: + oper1 = filter_ops[1][0] + oper2 = filter_ops[0][0] + value1 = str(filter_ops[1][1]) + value1 = str(value1[1:]) + value2 = str(filter_ops[0][1]) + value2 = str(value2[1:]) + + if direction == datastore_pb.Query_Order.ASCENDING: + table_name = dbconstants.ASC_PROPERTY_TABLE + # The first operator will always be either > or >= + if startrow: + start_inclusive = self._DISABLE_INCLUSIVITY + elif oper1 == datastore_pb.Query_Filter.GREATER_THAN: + params = [prefix, kind, property_name, value1 + '/' + self._TERM_STRING] + startrow = self.get_index_key_from_params(params) + elif oper1 == datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: + params = [prefix, kind, property_name, value1 + '/'] + startrow = self.get_index_key_from_params(params) + else: + raise dbconstants.AppScaleMisconfiguredQuery("Bad filter ordering") + + # The second operator will be either < or <= + if oper2 == datastore_pb.Query_Filter.LESS_THAN: + params = [prefix, kind, property_name, value2 + '/'] + endrow = self.get_index_key_from_params(params) + end_inclusive = self._DISABLE_INCLUSIVITY + elif oper2 == datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL: + params = [prefix, kind, property_name, value2 + '/' + self._TERM_STRING] + endrow = self.get_index_key_from_params(params) + end_inclusive = self._ENABLE_INCLUSIVITY + else: + raise dbconstants.AppScaleMisconfiguredQuery("Bad filter ordering") + + if direction == datastore_pb.Query_Order.DESCENDING: + table_name = dbconstants.DSC_PROPERTY_TABLE + value1 = helper_functions.reverse_lex(value1) + value2 = helper_functions.reverse_lex(value2) + + if oper1 == datastore_pb.Query_Filter.GREATER_THAN: + params = [prefix, kind, property_name, value1 + '/'] + endrow = self.get_index_key_from_params(params) + end_inclusive = self._DISABLE_INCLUSIVITY + elif oper1 == datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: + params = [prefix, kind, property_name, value1 + '/' + self._TERM_STRING] + endrow = self.get_index_key_from_params(params) + end_inclusive = self._ENABLE_INCLUSIVITY + + if startrow: + start_inclusive = self._DISABLE_INCLUSIVITY + elif oper2 == datastore_pb.Query_Filter.LESS_THAN: + params = [prefix, kind, property_name, value2 + '/' + self._TERM_STRING] + startrow = self.get_index_key_from_params(params) + elif oper2 == datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL: + params = [prefix, kind, property_name, value2 + '/'] + startrow = self.get_index_key_from_params(params) + + return self.datastore_batch.range_query(table_name, + column_names, + startrow, + endrow, + limit, + offset=offset, + start_inclusive=start_inclusive, + end_inclusive=end_inclusive) + + return [] + + def __composite_query(self, query, filter_info, order_info): + """Performs Composite queries which is a combination of + multiple properties to query on. + Args: + query: The query to run + filter_info: tuple with filter operators and values + order_info: tuple with property name and the sort order + Returns: + List of entities retrieved from the given query + """ + if order_info and order_info[0][0] == '__key__': + return None + + if query.has_ancestor(): + return None + + if not query.has_kind(): + return None + + order_names = [x[0] for x in order_info] + + def set_prop_names(filt_info): + pnames = set(filt_info.keys()) + pnames.update(x[0] for x in order_info) + pnames.discard('__key__') + pnames = list(pnames) + + pname = None + for p in filt_info.keys(): + f = filt_info[p] + if f[0][0] != datastore_pb.Query_Filter.EQUAL: + pname = p + return pname, pnames + + property_name, property_names = set_prop_names(filter_info) + + if len(property_names) <= 1: + return None + + if not property_name: + property_name = property_names.pop() + + filter_ops = filter_info.get(property_name, []) + order_ops = [] + + for i in order_info: + if i[0] == property_name: + order_ops = [i] + break + if order_ops: + if order_ops[0][0] == property_name: + direction = order_ops[0][1] + else: + direction = datastore_pb.Query_Order.ASCENDING + + count = self._MAX_COMPOSITE_WINDOW + kind = query.kind() + + limit = query.limit() or self._MAXIMUM_RESULTS + + offset = query.offset() + prefix = self.get_table_prefix(query) + + if query.has_compiled_cursor() and query.compiled_cursor().position_size(): + cursor = cassandra_stub_util.ListCursor(query) + last_result = cursor._GetLastResult() + startrow = self.__get_start_key(prefix, + property_name, + direction, + last_result) + else: + startrow = None + result = [] + pre_filter = [] + # We loop and collect enough to fill the limit or until there are + # no more matching entities. The first filter is what we apply + # direct to the datastore, followed by in memory filters + # Research is required on figuring out what is the + # best filter to apply via range queries. + while len(result) < (limit+offset): + temp_res = self.__apply_filters(filter_ops, + order_ops, + property_name, + kind, + prefix, + count, + 0, + startrow) + if not temp_res: break + + ent_res = self.__fetch_entities(temp_res) + + # Create a copy from which we filter out + filtered_entities = ent_res[:] + + # Apply in-memory filters for each property + for ent in ent_res: + e = entity_pb.EntityProto(ent) + prop_list = e.property_list() + for prop in property_names: + temp_filt = filter_info.get(prop,[]) + + cur_prop = None + for each in prop_list: + if each.name() == prop: + cur_prop = each + break + + # Filter each property by the given value, only handling EQUAL + if not prop: + filtered_entities.remove(ent) + elif len(temp_filt) == 1: + oper = temp_filt[0][0] + value = str(temp_filt[0][1]) + if oper == datastore_pb.Query_Filter.EQUAL: + if cur_prop and str(self.__encode_index_pb(cur_prop.value())) != value: + if ent in filtered_entities: filtered_entities.remove(ent) + + result += filtered_entities + startrow = temp_res[-1].keys()[0] + + if len(order_info) > 1: + result = self.__order_composite_results(result, order_info) + + if result: result = result[offset:] + return result + + def __order_composite_results(self, result, order_info): + """ Takes results and applies ordering based on properties and + whether it should be ascending or decending. + Args: + result: unordered results + order_info: given ordering of properties + Returns: + A list of ordered entities + """ + # We can not fully filter past one filter without getting + # the entire table to make sure results are in the correct order. + # Composites must be implemented the correct way with specialized + # indexes to get the correct result. + # The effect is that entities at the edge of each batch have a high + # chance of being out of order with our current implementation. + + # Put all the values appended based on order info into a dictionary, + # The key being the values appended and the value being the index + if not result: return [] + vals = {} + for e in result: + key = "/" + e = entity_pb.EntityProto(e) + prop_list = e.property_list() + for ii in order_info: + ord_prop = ii[0] + ord_dir = ii[1] + for each in prop_list: + if each.name() == ord_prop: + if ord_dir == datastore_pb.Query_Order.DESCENDING: + key = str(key+ '/' + helper_functions.reverse_lex(str(each.value()))) + else: + key = str(key + '/' + str(each.value())) + break + vals[key] = e + keys = sorted(vals.keys()) + sorted_vals = [vals[ii] for ii in keys] + result = [e.Encode() for e in sorted_vals] + return result + + # These are the three different types of queries attempted. Queries + # can be identified by their filters and orderings + _QUERY_STRATEGIES = [ + __single_property_query, + __kind_query, + __composite_query, + ] + + + def __get_query_results(self, query): + """Applies the strategy for the provided query. + + Args: + query: A datastore_pb.Query protocol buffer. + Returns: + Result set + """ + if query.has_transaction() and not query.has_ancestor(): + raise apiproxy_errors.ApplicationError( + datastore_pb.Error.BAD_REQUEST, + 'Only ancestor queries are allowed inside transactions.') + + num_components = len(query.filter_list()) + len(query.order_list()) + if query.has_ancestor(): + num_components += 1 + if num_components > self._MAX_QUERY_COMPONENTS: + raise apiproxy_errors.ApplicationError( + datastore_pb.Error.BAD_REQUEST, + ('query is too large. may not have more than %s filters' + ' + sort orders ancestor total' % self._MAX_QUERY_COMPONENTS)) + + app_id = query.app() + self.validate_app_id(app_id) + filters, orders = datastore_index.Normalize(query.filter_list(), + query.order_list()) + filter_info = self.generate_filter_info(filters, query) + order_info = self.generate_order_info(orders) + for strategy in DatastoreDistributed._QUERY_STRATEGIES: + results = strategy(self, query, filter_info, order_info) + if results: + break + + # TODO keys only queries. + # They work but pass the entire entity back to the AppServer + if query.has_keys_only(): + pass + return results + + def _dynamic_run_query(self, app_id, query, query_result): + """Populates the query result and use that query result to + encode a cursor + Args: + app_id: The application ID + query: The query to run + query_result: The response given to the application server + """ + result = self.__get_query_results(query) + count = 0 + if result: + for index,ii in enumerate(result): + result[index] = entity_pb.EntityProto(ii) + count = len(result) + + cur = cassandra_stub_util.QueryCursor(query, result) + cur.PopulateQueryResult(count, query.offset(), query_result) + + def setup_transaction(self, app_id) : + """ Gets a transaction ID for a new transaction """ + _MAX_RAND = 1000000 # arbitary large number + return random.randint(1, _MAX_RAND) + +logger = appscale_logger.getLogger("pb_server") + +class MainHandler(tornado.web.RequestHandler): + """ + Defines what to do when the webserver receives different types of + HTTP requests. + """ + + ############## + # OTHER TYPE # + ############## + def unknown_request(self, app_id, http_request_data, pb_type): + """ Function which handles unknown protocol buffers + Args: + app_id: Name of the application + http_request_data: Stores the protocol buffer request from the AppServer + Raises: + Raises exception. + """ + raise NotImplementedError("Unknown request of operation %s"%pb_type) + + ######################### + # POST Request Handling # + ######################### + + @tornado.web.asynchronous + def post( self ): + """ Function which handles POST requests. Data of the request is + the request from the AppServer in an encoded protocol buffer + format. + """ + request = self.request + http_request_data = request.body + pb_type = request.headers['protocolbuffertype'] + app_data = request.headers['appdata'] + app_data = app_data.split(':') + + if len(app_data) == 4: + app_id, user_email, nick_name, auth_domain = app_data + os.environ['AUTH_DOMAIN'] = auth_domain + os.environ['USER_EMAIL'] = user_email + os.environ['USER_NICKNAME'] = nick_name + os.environ['APPLICATION_ID'] = app_id + elif len(app_data) == 1: + app_id = app_data[0] + os.environ['APPLICATION_ID'] = app_id + else: + return + + if pb_type == "Request": + self.remote_request(app_id, http_request_data) + else: + self.unknown_request(app_id, http_request_data, pb_type) + self.finish() + + ######################### + # GET Request Handling # + ######################### + + @tornado.web.asynchronous + def get(self): + """ Handles get request for the web server. Returns that it is currently + up in json. + """ + self.write("{'status':'up'}") + self.finish() + + def remote_request(self, app_id, http_request_data): + """ Receives a remote request to which it should give the correct + response. The http_request_data holds an encoded protocol buffer + of a certain type. Each type has a particular response type. + + Args: + app_id: The application ID that is sending this request + http_request_data: Encoded protocol buffer + """ + apirequest = remote_api_pb.Request() + apirequest.ParseFromString(http_request_data) + apiresponse = remote_api_pb.Response() + response = None + errcode = 0 + errdetail = "" + apperror_pb = None + + if not apirequest.has_method(): + errcode = datastore_pb.Error.BAD_REQUEST + errdetail = "Method was not set in request" + apirequest.set_method("NOT_FOUND") + if not apirequest.has_request(): + errcode = datastore_pb.Error.BAD_REQUEST + errdetail = "Request missing in call" + apirequest.set_method("NOT_FOUND") + apirequest.clear_request() + method = apirequest.method() + http_request_data = apirequest.request() + + if method == "Put": + response, errcode, errdetail = self.put_request(app_id, + http_request_data) + elif method == "Get": + response, errcode, errdetail = self.get_request(app_id, + http_request_data) + elif method == "Delete": + response, errcode, errdetail = self.delete_request(app_id, + http_request_data) + elif method == "RunQuery": + response, errcode, errdetail = self.run_query(app_id, + http_request_data) + elif method == "BeginTransaction": + response, errcode, errdetail = self.begin_transaction_request(app_id, + http_request_data) + elif method == "Commit": + response, errcode, errdetail = self.commit_transaction_request(app_id, + http_request_data) + elif method == "Rollback": + response, errcode, errdetail = self.rollback_transaction_request(app_id, + http_request_data) + elif method == "CreateIndex": + errcode = 0 + errdetail = "" + response = api_base_pb.Integer64Proto() + response.set_value(0) + response = response.Encode() + + elif method == "GetIndices": + response = datastore_pb.CompositeIndices().Encode() + errcode = 0 + errdetail = "" + + elif method == "UpdateIndex": + response = api_base_pb.VoidProto().Encode() + errcode = 0 + errdetail = "" + + elif method == "DeleteIndex": + response = api_base_pb.VoidProto().Encode() + errcode = 0 + errdetail = "" + + else: + errcode = datastore_pb.Error.BAD_REQUEST + errdetail = "Unknown datastore message" + + apiresponse.set_response(response) + if errcode != 0: + apperror_pb = apiresponse.mutable_application_error() + apperror_pb.set_code(errcode) + apperror_pb.set_detail(errdetail) + + if errcode != 0: + logger.debug("Reply: %s\nerrcode: %s\nerror details: %s" %\ + (str(method), str(errcode), str(errdetail))) + + self.write(apiresponse.Encode() ) + + def begin_transaction_request(self, app_id, http_request_data): + """ Handles the intial request to start a transaction. Replies with + a unique identifier to handle this transaction in future requests. + + Args: + app_id: The application ID requesting the transaction + http_request_data: The encoded request + Returns: + An encoded transaction protocol buffer with a unique handler + """ + global datastore_access + transaction_pb = datastore_pb.Transaction() + handle = datastore_access.setup_transaction(app_id) + transaction_pb.set_app(app_id) + transaction_pb.set_handle(handle) + return (transaction_pb.Encode(), 0, "") + + def commit_transaction_request(self, app_id, http_request_data): + """ Handles the commit phase of a transaction + + Args: + app_id: The application ID requesting the transaction commit + http_request_data: The encoded request + Returns: + An encoded protocol buffer commit response + """ + commitres_pb = datastore_pb.CommitResponse() + # TODO implement transactions + """ + transaction_pb = datastore_pb.Transaction(http_request_data) + txn_id = transaction_pb.handle() + try: + self._Dynamic_Commit(app_id, transaction_pb, commitres_pb) + except: + return (commitres_pb.Encode(), + datastore_pb.Error.PERMISSION_DENIED, + "Unable to commit for this transaction") + """ + return (commitres_pb.Encode(), 0, "") + + def rollback_transaction_request(self, app_id, http_request_data): + """ Handles the rollback phase of a transaction + + Args: + app_id: The application ID requesting the rollback + http_request_data: The encoded request + Returns: + An encoded protocol buffer void response + """ + # TODO implement transactions + """ + transaction_pb = datastore_pb.Transaction(http_request_data) + handle = transaction_pb.handle() + try: + self.datastore_access._Dynamic_Rollback(app_id, transaction_pb, None) + except: + return(api_base_pb.VoidProto().Encode(), + datastore_pb.Error.PERMISSION_DENIED, + "Unable to rollback for this transaction") + """ + return (api_base_pb.VoidProto().Encode(), 0, "") + + def run_query(self, app_id, http_request_data): + """ High level function for running queries + Args: + app_id: Name of the application + http_request_data: Stores the protocol buffer request from the AppServer + Returns: + Returns an encoded query response + """ + + global datastore_access + query = datastore_pb.Query(http_request_data) + + # Pack Results into a clone of QueryResult # + clone_qr_pb = datastore_pb.QueryResult() + datastore_access._dynamic_run_query(app_id, query, clone_qr_pb) + return (clone_qr_pb.Encode(), 0, "") + + + def put_request(self, app_id, http_request_data): + """ High level function for doing puts + Args: + app_id: Name of the application + http_request_data: Stores the protocol buffer request from the AppServer + Returns: + Returns an encoded put response + """ + + global datastore_access + putreq_pb = datastore_pb.PutRequest(http_request_data) + putresp_pb = datastore_pb.PutResponse( ) + datastore_access._dynamic_put(app_id, putreq_pb, putresp_pb) + return (putresp_pb.Encode(), 0, "") + + def get_request(self, app_id, http_request_data): + """ High level function for doing gets + Args: + app_id: Name of the application + http_request_data: Stores the protocol buffer request from the AppServer + Returns: + Returns an encoded get response + """ + + global datastore_access + getreq_pb = datastore_pb.GetRequest(http_request_data) + getresp_pb = datastore_pb.GetResponse() + datastore_access._dynamic_get(getreq_pb, getresp_pb) + return (getresp_pb.Encode(), 0, "") + + def delete_request(self, app_id, http_request_data): + """ High level function for doing deletes + Args: + app_id: Name of the application + http_request_data: Stores the protocol buffer request from the AppServer + Returns: + Returns an encoded delete response + """ + + global datastore_access + delreq_pb = datastore_pb.DeleteRequest( http_request_data ) + delresp_pb = api_base_pb.VoidProto() + datastore_access._dynamic_delete(delreq_pb, delresp_pb) + return (delresp_pb.Encode(), 0, "") + + def void_proto(self, app_id, http_request_data): + """ Function which handles void protocol buffers + Args: + app_id: Name of the application + http_request_data: Stores the protocol buffer request from the AppServer + Returns: + Default message for void protocol buffers + """ + + resp_pb = api_base_pb.VoidProto() + logger.debug("VOID_RESPONSE: %s to void" % resp_pb) + return (resp_pb.Encode(), 0, "" ) + + def str_proto(self, app_id, http_request_data): + """ Function which handles string protocol buffers + Args: + app_id: Name of the application + http_request_data: Stores the protocol buffer request from the AppServer + Returns: + Default message for string protocol buffers which is a composite + response + """ + + str_pb = api_base_pb.StringProto( http_request_data ) + composite_pb = datastore_pb.CompositeIndices() + logger.debug("String proto received: %s"%str_pb) + logger.debug("CompositeIndex response to string: %s" % composite_pb) + return (composite_pb.Encode(), 0, "" ) + + def int64_proto(self, app_id, http_request_data): + """ Function which handles integer protocol buffers. Application + server expects a void protocol as an acknowledgement. + + Args: + app_id: Name of the application + http_request_data: Stores the protocol buffer request from the AppServer + Returns: + void protocol buffer + """ + + int64_pb = api_base_pb.Integer64Proto( http_request_data ) + resp_pb = api_base_pb.VoidProto() + logger.debug("Int64 proto received: %s"%int64_pb) + logger.debug("VOID_RESPONSE to int64: %s" % resp_pb) + return (resp_pb.Encode(), 0, "") + + def compositeindex_proto(self, app_id, http_request_data): + """ Function which handles composite index protocol buffers. + + Args: + app_id: Name of the application + http_request_data: Stores the protocol buffer request from the AppServer + Returns: + Default message for string protocol buffers which is a void protocol + buffer + """ + + compindex_pb = entity_pb.CompositeIndex( http_request_data) + resp_pb = api_base_pb.VoidProto() + logger.debug("CompositeIndex proto recieved: %s"%str(compindex_pb)) + logger.debug("VOID_RESPONSE to composite index: %s" % resp_pb) + return (resp_pb.Encode(), 0, "") + +def usage(): + print "AppScale Server" + print + print "Options:" + print "\t--type=<" + ','.join(VALID_DATASTORES) + ">" + print "\t--no_encryption" + print "\t--port" + print "\t--zoo_keeper " + +pb_application = tornado.web.Application([ + (r"/*", MainHandler), +]) + +def main(argv): + global datastore_access + zoo_keeper_locations = "" + + db_type = "cassandra" + port = DEFAULT_SSL_PORT + isEncrypted = True + + try: + opts, args = getopt.getopt( argv, "t:p:n:z:", + ["type=", + "port", + "no_encryption", + "zoo_keeper"] ) + except getopt.GetoptError: + usage() + sys.exit(1) + + for opt, arg in opts: + if opt in ("-t", "--type"): + db_type = arg + print "Datastore type: ",db_type + elif opt in ("-p", "--port"): + port = int(arg) + elif opt in ("-n", "--no_encryption"): + isEncrypted = False + elif opt in ("-z", "--zoo_keeper"): + zoo_keeper_locations = arg + + if db_type not in VALID_DATASTORES: + print "This datastore is not supported for this version of the AppScale\ + datastore API:" + db_type + exit(1) + + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(db_type) + datastore_access = DatastoreDistributed(datastore_batch) + + if port == DEFAULT_SSL_PORT and not isEncrypted: + port = DEFAULT_PORT + + server = tornado.httpserver.HTTPServer(pb_application) + server.listen(port) + + while 1: + try: + # Start Server # + tornado.ioloop.IOLoop.instance().start() + except SSL.SSLError: + # This happens when connections timeout, there is a just a bad + # SSL connection such as it does not use SSL when expected. + pass + except KeyboardInterrupt: + print "Server interrupted by user, terminating..." + exit(1) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/AppDB/dbconstants.py b/AppDB/dbconstants.py index fb9b68dcc4..ffe6b1be7d 100644 --- a/AppDB/dbconstants.py +++ b/AppDB/dbconstants.py @@ -1,4 +1,7 @@ -# Constants +# Programmer: Navraj Chohan +""" + Datastore Constants +""" import os APPSCALE_HOME=os.environ.get("APPSCALE_HOME") @@ -8,16 +11,57 @@ ERROR_DEFAULT = "DB_ERROR:" NONEXISTANT_TRANSACTION = "0" -# DB schema - +# Table names USERS_TABLE = "USERS__" APPS_TABLE = "APPS__" JOURNAL_TABLE = "JOURNAL__" + +ASC_PROPERTY_TABLE = "ASC_PROPERTY__" +DSC_PROPERTY_TABLE = "DSC_PROPERTY__" +APP_INDEX_TABLE = "APP_INDEXES__" +APP_NAMESPACE_TABLE = "APP_NAMESPACES__" +APP_ID_TABLE = "APP_IDS__" +APP_ENTITY_TABLE = "ENTITIES__" +APP_KIND_TABLE = "KINDS__" + +INITIAL_TABLES = [ASC_PROPERTY_TABLE, + DSC_PROPERTY_TABLE, + APP_INDEX_TABLE, + APP_NAMESPACE_TABLE, + APP_ID_TABLE, + APP_ENTITY_TABLE, + APP_KIND_TABLE] + +########################################### +# DB schemas for version 1 of the datastore +########################################### JOURNAL_SCHEMA = [ "Encoded_Entity"] + ENTITY_TABLE_SCHEMA = [ "Encoded_Entity", - "Txn_Num" ] + "Txn_Num"] + +########################################### +# DB schema for version 2 of the datastore +########################################### + +# The schema of the table which holds the encoded entities +APP_ENTITY_SCHEMA = [ + "entity", + "txnID"] + +# Index tables store references are to entity table +PROPERTY_SCHEMA = [ + "reference" ] +APP_INDEX_SCHEMA = [ + "indexes" ] +APP_NAMESPACE_SCHEMA = [ + "namespaces" ] +APP_ID_SCHEMA = [ + "next_id" ] +APP_KIND_SCHEMA = [ + "reference" ] USERS_SCHEMA = [ "email", @@ -74,3 +118,31 @@ ) ENGINE=ndbcluster; """] +############################### +# Generic Datastore Exceptions +############################### +class AppScaleDBConnectionError(Exception): + """ Tossed when there is a bad connection + """ + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class AppScaleMisconfiguredQuery(Exception): + """ Tossed when a query is misconfigured + """ + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class AppScaleBadArg(Exception): + """ Bad Argument given for a function + """ + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + + diff --git a/AppDB/dbinterface.py b/AppDB/dbinterface.py index 67e901c8dd..e5783b8686 100644 --- a/AppDB/dbinterface.py +++ b/AppDB/dbinterface.py @@ -32,7 +32,7 @@ def commit(self, txnid): raise NotImplementedError("commit is not implemented in %s." % self.__class__) def rollback(self, txnid): raise NotImplementedError("rollback is not implemented in %s." % self.__class__) - def setupTransaction(self, txnid): + def setup_transaction(self, txnid): raise NotImplementedError("rollback is not implemented in %s." % self.__class__) diff --git a/AppDB/dbinterface_batch.py b/AppDB/dbinterface_batch.py new file mode 100644 index 0000000000..05b8009d5b --- /dev/null +++ b/AppDB/dbinterface_batch.py @@ -0,0 +1,154 @@ +# Programmer: Navraj Chohan + +""" + AppScale Datastore Interface +""" +import os + +class AppDBInterface: + def batch_get_entity(self, table_name, row_key, column_names): + """ + Takes in batches of keys and retrieves their cooresponding rows. + + Args: + table_name: The table to access + row_keys: A list of keys to access + column_names: A list of columns to access + Returns: + A dictionary of rows and columns/values of those rows. The format + looks like such: {key:{column_name:value,...}} + """ + + raise NotImplementedError("get_entity is not implemented in %s." % self.__class__) + def batch_put_entity(self, table_name, row_key, column_names, cell_values): + """ + Allows callers to store multiple rows with a single call. A row can + have multiple columns and values with them. We refer to each row as + an entity. + + Args: + table_name: The table to mutate + row_keys: A list of keys to store on + column_names: A list of columns to mutate + cell_values: A dict of key/value pairs + Raises: + TypeError: when bad arguments are given + """ + + raise NotImplementedError("put_entity is not implemented in %s." % self.__class__) + + def batch_delete(self, table_name, row_keys, column_names=[]): + """ + Remove a set of rows cooresponding to a set of keys. + + Args: + table_name: Table to delete rows from + row_keys: A list of keys to remove + column_names: Not used + Raises: + AppScaleDBConnectionError: when unable to execute deletes + TypeError: when given bad argument types + """ + raise NotImplementedError("delete_row is not implemented in %s." % self.__class__) + + def delete_table(self, table_name): + """ + Drops a given table. + + Args: + table_name: A string name of the table to drop + Rasies: + TypeError: when given bad argument types + """ + raise NotImplementedError("delete_table is not implemented in %s." % self.__class__) + + def range_query(self, + table_name, + column_names, + start_key, + end_key, + limit, + offset=0, + start_inclusive=True, + end_inclusive=True, + keys_only=False): + """ + Gets a dense range ordered by keys. Returns an ordered list of + a dictionary of [key:{column1:value1, column2:value2},...] + or a list of keys if keys only. + + Args: + table_name: Name of table to access + column_names: Columns which get returned within the key range + start_key: String for which the query starts at + end_key: String for which the query ends at + limit: Maximum number of results to return + offset: Cuts off these many from the results [offset:] + start_inclusive: Boolean if results should include the start_key + end_inclusive: Boolean if results should include the end_key + keys_only: Boolean if to only keys and not values + Raises: + TypeError: when bad arguments are given + Returns: + An ordered list of dictionaries of key=>columns/values + """ + raise NotImplementedError("range_query is not implemented in %s." % self.__class__) + + def create_table(self,table_name, column_names): + """ + Creates a table given a schema (column_names). + + Args: + table_name: The name of the table to create + column_names: Not used but here to match the interface + Raises: + TypeError: when given bad argument types + """ + raise NotImplementedError("create_table is not implemented in %s." % self.__class__) + + def get_local_ip(self): + """ Gets the local IP of the current node. + + Returns: + The local IP + """ + + try: + local_ip = self.__local_ip + except AttributeError: + local_ip = None + + if local_ip is None: + local_ip = os.environ.get("LOCAL_DB_IP") + + if local_ip is None: + raise Exception("Env var LOCAL_DB_IP was not set.") + else: + self.__local_ip = local_ip + + return self.__local_ip + + def get_master_ip(self): + """ Gets the master database IP of the current AppScale deployment. + + Returns: + The master DB IP + """ + + try: + master_ip = self.__master_ip + except AttributeError: + master_ip = None + + if master_ip is None: + master_ip = os.environ.get("MASTER_IP") + + if master_ip is None: + raise Exception("Env var MASTER_IP was not set.") + else: + self.__master_ip = master_ip + + return self.__master_ip + + + diff --git a/AppDB/delete_all_record.py b/AppDB/delete_all_record.py index b161b49b93..937e3df25f 100755 --- a/AppDB/delete_all_record.py +++ b/AppDB/delete_all_record.py @@ -1,69 +1,59 @@ #!/usr/bin/env python -# -# delete all application record for testing. -# -# Author: Soo Hwan Park (suwanny@gmail.com) -# - -import sys, os - -APPSCALE_HOME = os.environ.get("APPSCALE_HOME") -if APPSCALE_HOME: - pass -else: - APPSCALE_HOME = "/etc/appscale" - os.environ["APPSCALE_HOME"] = APPSCALE_HOME - print "APPSCALE_HOME env var not set. Setting to " + APPSCALE_HOME - -sys.path.append("/usr/lib/python2.6/site-packages") -sys.path.append("/etc/appscale/AppDB") -sys.path.append("/etc/appscale/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/src/py/ThriftClient/gen-py") - -from appscale_datastore import Datastore - -USER_ID="a@a.a" -entityDict = {"guestbook":"Greeting", } -DEBUG=False +# Programmer: Navraj Chohan + +""" + Delete all application record for testing. +""" + + +import os +import sys +from dbconstants import * +import appscale_datastore_batch + +_MAX_ENTITIES = 1000000 +def get_entities(table, schema, db): + """ Gets entities from a table. + + Args: + table: Name of the table + schema: The schema of table to get from + db: The database accessor + Returns: + The entire table up to _MAX_ENTITIES + """ + return db.range_query(table, schema, "", "", _MAX_ENTITIES) + +def delete_all(entities, table, db): + """ Delets all entities in a table. + + Args: + table: The table to delete from + db: The database accessor + """ + for ii in entities: + db.batch_delete(table, ii.keys()) def main(argv): - DB_TYPE="hbase" + DB_TYPE="cassandra" if len(argv) < 2: - print "usage: ./delete_app_recode.py db_type table_name" + print "usage: ./delete_app_recode.py db_type" else: DB_TYPE = argv[1] - db = Datastore(DB_TYPE) + db = appscale_datastore_batch.DatastoreFactory.getDatastore(DB_TYPE) + entities = get_entities(APP_ENTITY_TABLE, APP_ENTITY_SCHEMA, db) + delete_all(entities, APP_ENTITY_TABLE, db) - app_schema = db.get_schema("APPS__")[1:] - user_schema = db.get_schema("USERS__")[1:] - #print "APPS", app_schema - #print "USER", user_schema - - app = db.get_entity("USERS__", USER_ID, ['applications'])[1] - if DEBUG: print "application:", app - - #app_table = app + "___" + entityDict[app] - #if DEBUG: print "app table:", app_table + entities = get_entities(ASC_PROPERTY_TABLE, PROPERTY_SCHEMA, db) + delete_all(entities, ASC_PROPERTY_TABLE, db) - version, classes, count = db.get_entity("APPS__", app, ['version', 'classes', 'num_entries'])[1:] - version = int(version) - count = int(count) - print "version:", version, ", classes:", classes, ", num_entries:", count + entities = get_entities(DSC_PROPERTY_TABLE, PROPERTY_SCHEMA, db) + delete_all(entities, DSC_PROPERTY_TABLE, db) - if DEBUG: print "delete all rows" - - for i in range(version + 1): - app_table = app + "___" + classes + "___" + str(i) - print "table name:", app_table - for i in range(count + 1): - db.delete_row(app_table, str(i)) - if DEBUG: print "delete table: %s, key: %s" % (app_table, str(i)) - print "delete_all_table is completed" + entities = get_entities(APP_KIND_TABLE, APP_KIND_SCHEMA, db) + delete_all(entities, APP_KIND_TABLE, db) - #for i in range(count + 1): - # db.delete_row(app_table, str(i)) - #print "deleted entrites:", count - if __name__ == "__main__": try: main(sys.argv) diff --git a/AppDB/hbase/Hbase-remote b/AppDB/hbase/Hbase-remote index 4580e25951..28a436418d 100755 --- a/AppDB/hbase/Hbase-remote +++ b/AppDB/hbase/Hbase-remote @@ -38,6 +38,10 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print ' getRowWithColumns(Text tableName, Text row, columns)' print ' getRowTs(Text tableName, Text row, i64 timestamp)' print ' getRowWithColumnsTs(Text tableName, Text row, columns, i64 timestamp)' + print ' getRows(Text tableName, rows)' + print ' getRowsWithColumns(Text tableName, rows, columns)' + print ' getRowsTs(Text tableName, rows, i64 timestamp)' + print ' getRowsWithColumnsTs(Text tableName, rows, columns, i64 timestamp)' print ' void mutateRow(Text tableName, Text row, mutations)' print ' void mutateRowTs(Text tableName, Text row, mutations, i64 timestamp)' print ' void mutateRows(Text tableName, rowBatches)' @@ -49,6 +53,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print ' void deleteAllRowTs(Text tableName, Text row, i64 timestamp)' print ' ScannerID scannerOpen(Text tableName, Text startRow, columns)' print ' ScannerID scannerOpenWithStop(Text tableName, Text startRow, Text stopRow, columns)' + print ' ScannerID scannerOpenWithPrefix(Text tableName, Text startAndPrefix, columns)' print ' ScannerID scannerOpenTs(Text tableName, Text startRow, columns, i64 timestamp)' print ' ScannerID scannerOpenWithStopTs(Text tableName, Text startRow, Text stopRow, columns, i64 timestamp)' print ' scannerGet(ScannerID id)' @@ -66,20 +71,22 @@ http = False argi = 1 if sys.argv[argi] == '-h': - parts = sys.argv[argi+1].split(':') + parts = sys.argv[argi+1].split(':') host = parts[0] port = int(parts[1]) argi += 2 if sys.argv[argi] == '-u': url = urlparse(sys.argv[argi+1]) - parts = url[1].split(':') + parts = url[1].split(':') host = parts[0] if len(parts) > 1: port = int(parts[1]) else: port = 80 uri = url[2] + if url[4]: + uri += '?%s' % url[4] http = True argi += 2 @@ -204,6 +211,30 @@ elif cmd == 'getRowWithColumnsTs': sys.exit(1) pp.pprint(client.getRowWithColumnsTs(eval(args[0]),eval(args[1]),eval(args[2]),eval(args[3]),)) +elif cmd == 'getRows': + if len(args) != 2: + print 'getRows requires 2 args' + sys.exit(1) + pp.pprint(client.getRows(eval(args[0]),eval(args[1]),)) + +elif cmd == 'getRowsWithColumns': + if len(args) != 3: + print 'getRowsWithColumns requires 3 args' + sys.exit(1) + pp.pprint(client.getRowsWithColumns(eval(args[0]),eval(args[1]),eval(args[2]),)) + +elif cmd == 'getRowsTs': + if len(args) != 3: + print 'getRowsTs requires 3 args' + sys.exit(1) + pp.pprint(client.getRowsTs(eval(args[0]),eval(args[1]),eval(args[2]),)) + +elif cmd == 'getRowsWithColumnsTs': + if len(args) != 4: + print 'getRowsWithColumnsTs requires 4 args' + sys.exit(1) + pp.pprint(client.getRowsWithColumnsTs(eval(args[0]),eval(args[1]),eval(args[2]),eval(args[3]),)) + elif cmd == 'mutateRow': if len(args) != 3: print 'mutateRow requires 3 args' @@ -270,6 +301,12 @@ elif cmd == 'scannerOpenWithStop': sys.exit(1) pp.pprint(client.scannerOpenWithStop(eval(args[0]),eval(args[1]),eval(args[2]),eval(args[3]),)) +elif cmd == 'scannerOpenWithPrefix': + if len(args) != 3: + print 'scannerOpenWithPrefix requires 3 args' + sys.exit(1) + pp.pprint(client.scannerOpenWithPrefix(eval(args[0]),eval(args[1]),eval(args[2]),)) + elif cmd == 'scannerOpenTs': if len(args) != 4: print 'scannerOpenTs requires 4 args' @@ -300,4 +337,8 @@ elif cmd == 'scannerClose': sys.exit(1) pp.pprint(client.scannerClose(eval(args[0]),)) +else: + print 'Unrecognized method %s' % cmd + sys.exit(1) + transport.close() diff --git a/AppDB/hbase/Hbase.py b/AppDB/hbase/Hbase.py index f3dd901c6b..d0bb4ea457 100644 --- a/AppDB/hbase/Hbase.py +++ b/AppDB/hbase/Hbase.py @@ -8,7 +8,7 @@ from ttypes import * from thrift.Thrift import TProcessor from thrift.transport import TTransport -from thrift.protocol import TBinaryProtocol +from thrift.protocol import TBinaryProtocol, TProtocol try: from thrift.protocol import fastbinary except: @@ -19,10 +19,9 @@ class Iface: def enableTable(self, tableName): """ Brings a table on-line (enables it) - @param tableName name of the table - + Parameters: - - tableName + - tableName: name of the table """ pass @@ -30,20 +29,18 @@ def disableTable(self, tableName): """ Disables a table (takes it off-line) If it is being served, the master will tell the servers to stop serving it. - @param tableName name of the table - + Parameters: - - tableName + - tableName: name of the table """ pass def isTableEnabled(self, tableName): """ - @param tableName name of table to check @return true if table is on-line - + Parameters: - - tableName + - tableName: name of the table to check """ pass @@ -64,29 +61,30 @@ def majorCompact(self, tableNameOrRegionName): def getTableNames(self, ): """ List all the userspace tables. - @return - returns a list of names + + @return returns a list of names """ pass def getColumnDescriptors(self, tableName): """ List all the column families assoicated with a table. - @param tableName table name + @return list of column family descriptors - + Parameters: - - tableName + - tableName: table name """ pass def getTableRegions(self, tableName): """ List the regions associated with a table. - @param tableName table name + @return list of region descriptors - + Parameters: - - tableName + - tableName: table name """ pass @@ -94,30 +92,28 @@ def createTable(self, tableName, columnFamilies): """ Create a table with the specified column families. The name field for each ColumnDescriptor must be set and must end in a - colon (:). All other fields are optional and will get default + colon (:). All other fields are optional and will get default values if not explicitly specified. - - @param tableName name of table to create - @param columnFamilies list of column family descriptors - + @throws IllegalArgument if an input parameter is invalid + @throws AlreadyExists if the table name already exists - + Parameters: - - tableName - - columnFamilies + - tableName: name of table to create + - columnFamilies: list of column family descriptors """ pass def deleteTable(self, tableName): """ Deletes a table - @param tableName name of table to delete + @throws IOError if table doesn't exist on server or there was some other problem - + Parameters: - - tableName + - tableName: name of table to delete """ pass @@ -125,16 +121,13 @@ def get(self, tableName, row, column): """ Get a single TCell for the specified table, row, and column at the latest timestamp. Returns an empty list if no such value exists. - - @param tableName name of table - @param row row key - @param column column name + @return value for specified row/column - + Parameters: - - tableName - - row - - column + - tableName: name of table + - row: row key + - column: column name """ pass @@ -142,18 +135,14 @@ def getVer(self, tableName, row, column, numVersions): """ Get the specified number of versions for the specified table, row, and column. - - @param tableName name of table - @param row row key - @param column column name - @param numVersions number of versions to retrieve + @return list of cells for specified row/column - + Parameters: - - tableName - - row - - column - - numVersions + - tableName: name of table + - row: row key + - column: column name + - numVersions: number of versions to retrieve """ pass @@ -162,20 +151,15 @@ def getVerTs(self, tableName, row, column, timestamp, numVersions): Get the specified number of versions for the specified table, row, and column. Only versions less than or equal to the specified timestamp will be returned. - - @param tableName name of table - @param row row key - @param column column name - @param timestamp timestamp - @param numVersions number of versions to retrieve + @return list of cells for specified row/column - + Parameters: - - tableName - - row - - column - - timestamp - - numVersions + - tableName: name of table + - row: row key + - column: column name + - timestamp: timestamp + - numVersions: number of versions to retrieve """ pass @@ -183,14 +167,12 @@ def getRow(self, tableName, row): """ Get all the data for the specified table and row at the latest timestamp. Returns an empty list if the row does not exist. - - @param tableName name of table - @param row row key + @return TRowResult containing the row and map of columns to TCells - + Parameters: - - tableName - - row + - tableName: name of table + - row: row key """ pass @@ -198,16 +180,13 @@ def getRowWithColumns(self, tableName, row, columns): """ Get the specified columns for the specified table and row at the latest timestamp. Returns an empty list if the row does not exist. - - @param tableName name of table - @param row row key - @param columns List of columns to return, null for all columns + @return TRowResult containing the row and map of columns to TCells - + Parameters: - - tableName - - row - - columns + - tableName: name of table + - row: row key + - columns: List of columns to return, null for all columns """ pass @@ -215,16 +194,13 @@ def getRowTs(self, tableName, row, timestamp): """ Get all the data for the specified table and row at the specified timestamp. Returns an empty list if the row does not exist. - - @param tableName of table - @param row row key - @param timestamp timestamp + @return TRowResult containing the row and map of columns to TCells - + Parameters: - - tableName - - row - - timestamp + - tableName: name of the table + - row: row key + - timestamp: timestamp """ pass @@ -232,16 +208,69 @@ def getRowWithColumnsTs(self, tableName, row, columns, timestamp): """ Get the specified columns for the specified table and row at the specified timestamp. Returns an empty list if the row does not exist. - - @param tableName name of table - @param row row key - @param columns List of columns to return, null for all columns + @return TRowResult containing the row and map of columns to TCells - + + Parameters: + - tableName: name of table + - row: row key + - columns: List of columns to return, null for all columns + - timestamp + """ + pass + + def getRows(self, tableName, rows): + """ + Get all the data for the specified table and rows at the latest + timestamp. Returns an empty list if no rows exist. + + @return TRowResult containing the rows and map of columns to TCells + + Parameters: + - tableName: name of table + - rows: row keys + """ + pass + + def getRowsWithColumns(self, tableName, rows, columns): + """ + Get the specified columns for the specified table and rows at the latest + timestamp. Returns an empty list if no rows exist. + + @return TRowResult containing the rows and map of columns to TCells + + Parameters: + - tableName: name of table + - rows: row keys + - columns: List of columns to return, null for all columns + """ + pass + + def getRowsTs(self, tableName, rows, timestamp): + """ + Get all the data for the specified table and rows at the specified + timestamp. Returns an empty list if no rows exist. + + @return TRowResult containing the rows and map of columns to TCells + + Parameters: + - tableName: name of the table + - rows: row keys + - timestamp: timestamp + """ + pass + + def getRowsWithColumnsTs(self, tableName, rows, columns, timestamp): + """ + Get the specified columns for the specified table and rows at the specified + timestamp. Returns an empty list if no rows exist. + + @return TRowResult containing the rows and map of columns to TCells + Parameters: - - tableName - - row - - columns + - tableName: name of table + - rows: row keys + - columns: List of columns to return, null for all columns - timestamp """ pass @@ -252,15 +281,11 @@ def mutateRow(self, tableName, row, mutations): single transaction. If an exception is thrown, then the transaction is aborted. Default current timestamp is used, and all entries will have an identical timestamp. - - @param tableName name of table - @param row row key - @param mutations list of mutation commands - + Parameters: - - tableName - - row - - mutations + - tableName: name of table + - row: row key + - mutations: list of mutation commands """ pass @@ -270,17 +295,12 @@ def mutateRowTs(self, tableName, row, mutations, timestamp): single transaction. If an exception is thrown, then the transaction is aborted. The specified timestamp is used, and all entries will have an identical timestamp. - - @param tableName name of table - @param row row key - @param mutations list of mutation commands - @param timestamp timestamp - + Parameters: - - tableName - - row - - mutations - - timestamp + - tableName: name of table + - row: row key + - mutations: list of mutation commands + - timestamp: timestamp """ pass @@ -290,13 +310,10 @@ def mutateRows(self, tableName, rowBatches): in a single transaction. If an exception is thrown, then the transaction is aborted. Default current timestamp is used, and all entries will have an identical timestamp. - - @param tableName name of table - @param rowBatches list of row batches - + Parameters: - - tableName - - rowBatches + - tableName: name of table + - rowBatches: list of row batches """ pass @@ -306,46 +323,34 @@ def mutateRowsTs(self, tableName, rowBatches, timestamp): in a single transaction. If an exception is thrown, then the transaction is aborted. The specified timestamp is used, and all entries will have an identical timestamp. - - @param tableName name of table - @param rowBatches list of row batches - @param timestamp timestamp - + Parameters: - - tableName - - rowBatches - - timestamp + - tableName: name of table + - rowBatches: list of row batches + - timestamp: timestamp """ pass def atomicIncrement(self, tableName, row, column, value): """ Atomically increment the column value specified. Returns the next value post increment. - @param tableName name of table - @param row row to increment - @param column name of column - @param value amount to increment by - + Parameters: - - tableName - - row - - column - - value + - tableName: name of table + - row: row to increment + - column: name of column + - value: amount to increment by """ pass def deleteAll(self, tableName, row, column): """ Delete all cells that match the passed row and column. - - @param tableName name of table - @param row Row to update - @param column name of column whose value is to be deleted - + Parameters: - - tableName - - row - - column + - tableName: name of table + - row: Row to update + - column: name of column whose value is to be deleted """ pass @@ -353,30 +358,22 @@ def deleteAllTs(self, tableName, row, column, timestamp): """ Delete all cells that match the passed row and column and whose timestamp is equal-to or older than the passed timestamp. - - @param tableName name of table - @param row Row to update - @param column name of column whose value is to be deleted - @param timestamp timestamp - + Parameters: - - tableName - - row - - column - - timestamp + - tableName: name of table + - row: Row to update + - column: name of column whose value is to be deleted + - timestamp: timestamp """ pass def deleteAllRow(self, tableName, row): """ Completely delete the row's cells. - - @param tableName name of table - @param row key of the row to be completely deleted. - + Parameters: - - tableName - - row + - tableName: name of table + - row: key of the row to be completely deleted. """ pass @@ -384,15 +381,11 @@ def deleteAllRowTs(self, tableName, row, timestamp): """ Completely delete the row's cells marked with a timestamp equal-to or older than the passed timestamp. - - @param tableName name of table - @param row key of the row to be completely deleted. - @param timestamp timestamp - + Parameters: - - tableName - - row - - timestamp + - tableName: name of table + - row: key of the row to be completely deleted. + - timestamp: timestamp """ pass @@ -400,20 +393,16 @@ def scannerOpen(self, tableName, startRow, columns): """ Get a scanner on the current table starting at the specified row and ending at the last row in the table. Return the specified columns. - - @param columns columns to scan. If column name is a column family, all - columns of the specified column family are returned. Its also possible - to pass a regex in the column qualifier. - @param tableName name of table - @param startRow starting row in table to scan. send "" (empty string) to - start at the first row. - + @return scanner id to be used with other scanner procedures - + Parameters: - - tableName - - startRow - - columns + - tableName: name of table + - startRow: Starting row in table to scan. + Send "" (empty string) to start at the first row. + - columns: columns to scan. If column name is a column family, all + columns of the specified column family are returned. It's also possible + to pass a regex in the column qualifier. """ pass @@ -422,23 +411,32 @@ def scannerOpenWithStop(self, tableName, startRow, stopRow, columns): Get a scanner on the current table starting and stopping at the specified rows. ending at the last row in the table. Return the specified columns. - - @param columns columns to scan. If column name is a column family, all - columns of the specified column family are returned. Its also possible - to pass a regex in the column qualifier. - @param tableName name of table - @param startRow starting row in table to scan. send "" (empty string) to - start at the first row. - @param stopRow row to stop scanning on. This row is *not* included in the - scanner's results - + @return scanner id to be used with other scanner procedures - + + Parameters: + - tableName: name of table + - startRow: Starting row in table to scan. + Send "" (empty string) to start at the first row. + - stopRow: row to stop scanning on. This row is *not* included in the + scanner's results + - columns: columns to scan. If column name is a column family, all + columns of the specified column family are returned. It's also possible + to pass a regex in the column qualifier. + """ + pass + + def scannerOpenWithPrefix(self, tableName, startAndPrefix, columns): + """ + Open a scanner for a given prefix. That is all rows will have the specified + prefix. No other rows will be returned. + + @return scanner id to use with other scanner calls + Parameters: - - tableName - - startRow - - stopRow - - columns + - tableName: name of table + - startAndPrefix: the prefix (and thus start row) of the keys you want + - columns: the columns you want returned """ pass @@ -447,22 +445,17 @@ def scannerOpenTs(self, tableName, startRow, columns, timestamp): Get a scanner on the current table starting at the specified row and ending at the last row in the table. Return the specified columns. Only values with the specified timestamp are returned. - - @param columns columns to scan. If column name is a column family, all - columns of the specified column family are returned. Its also possible - to pass a regex in the column qualifier. - @param tableName name of table - @param startRow starting row in table to scan. send "" (empty string) to - start at the first row. - @param timestamp timestamp - + @return scanner id to be used with other scanner procedures - + Parameters: - - tableName - - startRow - - columns - - timestamp + - tableName: name of table + - startRow: Starting row in table to scan. + Send "" (empty string) to start at the first row. + - columns: columns to scan. If column name is a column family, all + columns of the specified column family are returned. It's also possible + to pass a regex in the column qualifier. + - timestamp: timestamp """ pass @@ -472,25 +465,19 @@ def scannerOpenWithStopTs(self, tableName, startRow, stopRow, columns, timestamp specified rows. ending at the last row in the table. Return the specified columns. Only values with the specified timestamp are returned. - - @param columns columns to scan. If column name is a column family, all - columns of the specified column family are returned. Its also possible - to pass a regex in the column qualifier. - @param tableName name of table - @param startRow starting row in table to scan. send "" (empty string) to - start at the first row. - @param stopRow row to stop scanning on. This row is *not* included - in the scanner's results - @param timestamp timestamp - + @return scanner id to be used with other scanner procedures - + Parameters: - - tableName - - startRow - - stopRow - - columns - - timestamp + - tableName: name of table + - startRow: Starting row in table to scan. + Send "" (empty string) to start at the first row. + - stopRow: row to stop scanning on. This row is *not* included in the + scanner's results + - columns: columns to scan. If column name is a column family, all + columns of the specified column family are returned. It's also possible + to pass a regex in the column qualifier. + - timestamp: timestamp """ pass @@ -500,14 +487,15 @@ def scannerGet(self, id): row in the table. When there are no more rows in the table, or a key greater-than-or-equal-to the scanner's specified stopRow is reached, an empty list is returned. - - @param id id of a scanner returned by scannerOpen + @return a TRowResult containing the current row and a map of the columns to TCells. + @throws IllegalArgument if ScannerID is invalid + @throws NotFound when the scanner reaches the end - + Parameters: - - id + - id: id of a scanner returned by scannerOpen """ pass @@ -517,28 +505,27 @@ def scannerGetList(self, id, nbRows): rows and advances to the next row in the table. When there are no more rows in the table, or a key greater-than-or-equal-to the scanner's specified stopRow is reached, an empty list is returned. - - @param id id of a scanner returned by scannerOpen - @param nbRows number of results to regturn + @return a TRowResult containing the current row and a map of the columns to TCells. + @throws IllegalArgument if ScannerID is invalid + @throws NotFound when the scanner reaches the end - + Parameters: - - id - - nbRows + - id: id of a scanner returned by scannerOpen + - nbRows: number of results to return """ pass def scannerClose(self, id): """ Closes the server-state associated with an open scanner. - - @param id id of a scanner returned by scannerOpen + @throws IllegalArgument if ScannerID is invalid - + Parameters: - - id + - id: id of a scanner returned by scannerOpen """ pass @@ -553,10 +540,9 @@ def __init__(self, iprot, oprot=None): def enableTable(self, tableName): """ Brings a table on-line (enables it) - @param tableName name of the table - + Parameters: - - tableName + - tableName: name of the table """ self.send_enableTable(tableName) self.recv_enableTable() @@ -587,10 +573,9 @@ def disableTable(self, tableName): """ Disables a table (takes it off-line) If it is being served, the master will tell the servers to stop serving it. - @param tableName name of the table - + Parameters: - - tableName + - tableName: name of the table """ self.send_disableTable(tableName) self.recv_disableTable() @@ -619,11 +604,10 @@ def recv_disableTable(self, ): def isTableEnabled(self, tableName): """ - @param tableName name of table to check @return true if table is on-line - + Parameters: - - tableName + - tableName: name of the table to check """ self.send_isTableEnabled(tableName) return self.recv_isTableEnabled() @@ -715,7 +699,8 @@ def recv_majorCompact(self, ): def getTableNames(self, ): """ List all the userspace tables. - @return - returns a list of names + + @return returns a list of names """ self.send_getTableNames() return self.recv_getTableNames() @@ -746,11 +731,11 @@ def recv_getTableNames(self, ): def getColumnDescriptors(self, tableName): """ List all the column families assoicated with a table. - @param tableName table name + @return list of column family descriptors - + Parameters: - - tableName + - tableName: table name """ self.send_getColumnDescriptors(tableName) return self.recv_getColumnDescriptors() @@ -782,11 +767,11 @@ def recv_getColumnDescriptors(self, ): def getTableRegions(self, tableName): """ List the regions associated with a table. - @param tableName table name + @return list of region descriptors - + Parameters: - - tableName + - tableName: table name """ self.send_getTableRegions(tableName) return self.recv_getTableRegions() @@ -819,18 +804,16 @@ def createTable(self, tableName, columnFamilies): """ Create a table with the specified column families. The name field for each ColumnDescriptor must be set and must end in a - colon (:). All other fields are optional and will get default + colon (:). All other fields are optional and will get default values if not explicitly specified. - - @param tableName name of table to create - @param columnFamilies list of column family descriptors - + @throws IllegalArgument if an input parameter is invalid + @throws AlreadyExists if the table name already exists - + Parameters: - - tableName - - columnFamilies + - tableName: name of table to create + - columnFamilies: list of column family descriptors """ self.send_createTable(tableName, columnFamilies) self.recv_createTable() @@ -865,12 +848,12 @@ def recv_createTable(self, ): def deleteTable(self, tableName): """ Deletes a table - @param tableName name of table to delete + @throws IOError if table doesn't exist on server or there was some other problem - + Parameters: - - tableName + - tableName: name of table to delete """ self.send_deleteTable(tableName) self.recv_deleteTable() @@ -901,16 +884,13 @@ def get(self, tableName, row, column): """ Get a single TCell for the specified table, row, and column at the latest timestamp. Returns an empty list if no such value exists. - - @param tableName name of table - @param row row key - @param column column name + @return value for specified row/column - + Parameters: - - tableName - - row - - column + - tableName: name of table + - row: row key + - column: column name """ self.send_get(tableName, row, column) return self.recv_get() @@ -945,18 +925,14 @@ def getVer(self, tableName, row, column, numVersions): """ Get the specified number of versions for the specified table, row, and column. - - @param tableName name of table - @param row row key - @param column column name - @param numVersions number of versions to retrieve + @return list of cells for specified row/column - + Parameters: - - tableName - - row - - column - - numVersions + - tableName: name of table + - row: row key + - column: column name + - numVersions: number of versions to retrieve """ self.send_getVer(tableName, row, column, numVersions) return self.recv_getVer() @@ -993,20 +969,15 @@ def getVerTs(self, tableName, row, column, timestamp, numVersions): Get the specified number of versions for the specified table, row, and column. Only versions less than or equal to the specified timestamp will be returned. - - @param tableName name of table - @param row row key - @param column column name - @param timestamp timestamp - @param numVersions number of versions to retrieve + @return list of cells for specified row/column - + Parameters: - - tableName - - row - - column - - timestamp - - numVersions + - tableName: name of table + - row: row key + - column: column name + - timestamp: timestamp + - numVersions: number of versions to retrieve """ self.send_getVerTs(tableName, row, column, timestamp, numVersions) return self.recv_getVerTs() @@ -1043,14 +1014,12 @@ def getRow(self, tableName, row): """ Get all the data for the specified table and row at the latest timestamp. Returns an empty list if the row does not exist. - - @param tableName name of table - @param row row key + @return TRowResult containing the row and map of columns to TCells - + Parameters: - - tableName - - row + - tableName: name of table + - row: row key """ self.send_getRow(tableName, row) return self.recv_getRow() @@ -1084,16 +1053,13 @@ def getRowWithColumns(self, tableName, row, columns): """ Get the specified columns for the specified table and row at the latest timestamp. Returns an empty list if the row does not exist. - - @param tableName name of table - @param row row key - @param columns List of columns to return, null for all columns + @return TRowResult containing the row and map of columns to TCells - + Parameters: - - tableName - - row - - columns + - tableName: name of table + - row: row key + - columns: List of columns to return, null for all columns """ self.send_getRowWithColumns(tableName, row, columns) return self.recv_getRowWithColumns() @@ -1128,16 +1094,13 @@ def getRowTs(self, tableName, row, timestamp): """ Get all the data for the specified table and row at the specified timestamp. Returns an empty list if the row does not exist. - - @param tableName of table - @param row row key - @param timestamp timestamp + @return TRowResult containing the row and map of columns to TCells - + Parameters: - - tableName - - row - - timestamp + - tableName: name of the table + - row: row key + - timestamp: timestamp """ self.send_getRowTs(tableName, row, timestamp) return self.recv_getRowTs() @@ -1172,16 +1135,13 @@ def getRowWithColumnsTs(self, tableName, row, columns, timestamp): """ Get the specified columns for the specified table and row at the specified timestamp. Returns an empty list if the row does not exist. - - @param tableName name of table - @param row row key - @param columns List of columns to return, null for all columns + @return TRowResult containing the row and map of columns to TCells - + Parameters: - - tableName - - row - - columns + - tableName: name of table + - row: row key + - columns: List of columns to return, null for all columns - timestamp """ self.send_getRowWithColumnsTs(tableName, row, columns, timestamp) @@ -1214,21 +1174,181 @@ def recv_getRowWithColumnsTs(self, ): raise result.io raise TApplicationException(TApplicationException.MISSING_RESULT, "getRowWithColumnsTs failed: unknown result"); + def getRows(self, tableName, rows): + """ + Get all the data for the specified table and rows at the latest + timestamp. Returns an empty list if no rows exist. + + @return TRowResult containing the rows and map of columns to TCells + + Parameters: + - tableName: name of table + - rows: row keys + """ + self.send_getRows(tableName, rows) + return self.recv_getRows() + + def send_getRows(self, tableName, rows): + self._oprot.writeMessageBegin('getRows', TMessageType.CALL, self._seqid) + args = getRows_args() + args.tableName = tableName + args.rows = rows + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getRows(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = getRows_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.io != None: + raise result.io + raise TApplicationException(TApplicationException.MISSING_RESULT, "getRows failed: unknown result"); + + def getRowsWithColumns(self, tableName, rows, columns): + """ + Get the specified columns for the specified table and rows at the latest + timestamp. Returns an empty list if no rows exist. + + @return TRowResult containing the rows and map of columns to TCells + + Parameters: + - tableName: name of table + - rows: row keys + - columns: List of columns to return, null for all columns + """ + self.send_getRowsWithColumns(tableName, rows, columns) + return self.recv_getRowsWithColumns() + + def send_getRowsWithColumns(self, tableName, rows, columns): + self._oprot.writeMessageBegin('getRowsWithColumns', TMessageType.CALL, self._seqid) + args = getRowsWithColumns_args() + args.tableName = tableName + args.rows = rows + args.columns = columns + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getRowsWithColumns(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = getRowsWithColumns_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.io != None: + raise result.io + raise TApplicationException(TApplicationException.MISSING_RESULT, "getRowsWithColumns failed: unknown result"); + + def getRowsTs(self, tableName, rows, timestamp): + """ + Get all the data for the specified table and rows at the specified + timestamp. Returns an empty list if no rows exist. + + @return TRowResult containing the rows and map of columns to TCells + + Parameters: + - tableName: name of the table + - rows: row keys + - timestamp: timestamp + """ + self.send_getRowsTs(tableName, rows, timestamp) + return self.recv_getRowsTs() + + def send_getRowsTs(self, tableName, rows, timestamp): + self._oprot.writeMessageBegin('getRowsTs', TMessageType.CALL, self._seqid) + args = getRowsTs_args() + args.tableName = tableName + args.rows = rows + args.timestamp = timestamp + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getRowsTs(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = getRowsTs_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.io != None: + raise result.io + raise TApplicationException(TApplicationException.MISSING_RESULT, "getRowsTs failed: unknown result"); + + def getRowsWithColumnsTs(self, tableName, rows, columns, timestamp): + """ + Get the specified columns for the specified table and rows at the specified + timestamp. Returns an empty list if no rows exist. + + @return TRowResult containing the rows and map of columns to TCells + + Parameters: + - tableName: name of table + - rows: row keys + - columns: List of columns to return, null for all columns + - timestamp + """ + self.send_getRowsWithColumnsTs(tableName, rows, columns, timestamp) + return self.recv_getRowsWithColumnsTs() + + def send_getRowsWithColumnsTs(self, tableName, rows, columns, timestamp): + self._oprot.writeMessageBegin('getRowsWithColumnsTs', TMessageType.CALL, self._seqid) + args = getRowsWithColumnsTs_args() + args.tableName = tableName + args.rows = rows + args.columns = columns + args.timestamp = timestamp + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getRowsWithColumnsTs(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = getRowsWithColumnsTs_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.io != None: + raise result.io + raise TApplicationException(TApplicationException.MISSING_RESULT, "getRowsWithColumnsTs failed: unknown result"); + def mutateRow(self, tableName, row, mutations): """ Apply a series of mutations (updates/deletes) to a row in a single transaction. If an exception is thrown, then the transaction is aborted. Default current timestamp is used, and all entries will have an identical timestamp. - - @param tableName name of table - @param row row key - @param mutations list of mutation commands - + Parameters: - - tableName - - row - - mutations + - tableName: name of table + - row: row key + - mutations: list of mutation commands """ self.send_mutateRow(tableName, row, mutations) self.recv_mutateRow() @@ -1265,17 +1385,12 @@ def mutateRowTs(self, tableName, row, mutations, timestamp): single transaction. If an exception is thrown, then the transaction is aborted. The specified timestamp is used, and all entries will have an identical timestamp. - - @param tableName name of table - @param row row key - @param mutations list of mutation commands - @param timestamp timestamp - + Parameters: - - tableName - - row - - mutations - - timestamp + - tableName: name of table + - row: row key + - mutations: list of mutation commands + - timestamp: timestamp """ self.send_mutateRowTs(tableName, row, mutations, timestamp) self.recv_mutateRowTs() @@ -1313,13 +1428,10 @@ def mutateRows(self, tableName, rowBatches): in a single transaction. If an exception is thrown, then the transaction is aborted. Default current timestamp is used, and all entries will have an identical timestamp. - - @param tableName name of table - @param rowBatches list of row batches - + Parameters: - - tableName - - rowBatches + - tableName: name of table + - rowBatches: list of row batches """ self.send_mutateRows(tableName, rowBatches) self.recv_mutateRows() @@ -1355,15 +1467,11 @@ def mutateRowsTs(self, tableName, rowBatches, timestamp): in a single transaction. If an exception is thrown, then the transaction is aborted. The specified timestamp is used, and all entries will have an identical timestamp. - - @param tableName name of table - @param rowBatches list of row batches - @param timestamp timestamp - + Parameters: - - tableName - - rowBatches - - timestamp + - tableName: name of table + - rowBatches: list of row batches + - timestamp: timestamp """ self.send_mutateRowsTs(tableName, rowBatches, timestamp) self.recv_mutateRowsTs() @@ -1397,16 +1505,12 @@ def recv_mutateRowsTs(self, ): def atomicIncrement(self, tableName, row, column, value): """ Atomically increment the column value specified. Returns the next value post increment. - @param tableName name of table - @param row row to increment - @param column name of column - @param value amount to increment by - + Parameters: - - tableName - - row - - column - - value + - tableName: name of table + - row: row to increment + - column: name of column + - value: amount to increment by """ self.send_atomicIncrement(tableName, row, column, value) return self.recv_atomicIncrement() @@ -1443,15 +1547,11 @@ def recv_atomicIncrement(self, ): def deleteAll(self, tableName, row, column): """ Delete all cells that match the passed row and column. - - @param tableName name of table - @param row Row to update - @param column name of column whose value is to be deleted - + Parameters: - - tableName - - row - - column + - tableName: name of table + - row: Row to update + - column: name of column whose value is to be deleted """ self.send_deleteAll(tableName, row, column) self.recv_deleteAll() @@ -1484,17 +1584,12 @@ def deleteAllTs(self, tableName, row, column, timestamp): """ Delete all cells that match the passed row and column and whose timestamp is equal-to or older than the passed timestamp. - - @param tableName name of table - @param row Row to update - @param column name of column whose value is to be deleted - @param timestamp timestamp - + Parameters: - - tableName - - row - - column - - timestamp + - tableName: name of table + - row: Row to update + - column: name of column whose value is to be deleted + - timestamp: timestamp """ self.send_deleteAllTs(tableName, row, column, timestamp) self.recv_deleteAllTs() @@ -1527,13 +1622,10 @@ def recv_deleteAllTs(self, ): def deleteAllRow(self, tableName, row): """ Completely delete the row's cells. - - @param tableName name of table - @param row key of the row to be completely deleted. - + Parameters: - - tableName - - row + - tableName: name of table + - row: key of the row to be completely deleted. """ self.send_deleteAllRow(tableName, row) self.recv_deleteAllRow() @@ -1565,15 +1657,11 @@ def deleteAllRowTs(self, tableName, row, timestamp): """ Completely delete the row's cells marked with a timestamp equal-to or older than the passed timestamp. - - @param tableName name of table - @param row key of the row to be completely deleted. - @param timestamp timestamp - + Parameters: - - tableName - - row - - timestamp + - tableName: name of table + - row: key of the row to be completely deleted. + - timestamp: timestamp """ self.send_deleteAllRowTs(tableName, row, timestamp) self.recv_deleteAllRowTs() @@ -1606,20 +1694,16 @@ def scannerOpen(self, tableName, startRow, columns): """ Get a scanner on the current table starting at the specified row and ending at the last row in the table. Return the specified columns. - - @param columns columns to scan. If column name is a column family, all - columns of the specified column family are returned. Its also possible - to pass a regex in the column qualifier. - @param tableName name of table - @param startRow starting row in table to scan. send "" (empty string) to - start at the first row. - + @return scanner id to be used with other scanner procedures - + Parameters: - - tableName - - startRow - - columns + - tableName: name of table + - startRow: Starting row in table to scan. + Send "" (empty string) to start at the first row. + - columns: columns to scan. If column name is a column family, all + columns of the specified column family are returned. It's also possible + to pass a regex in the column qualifier. """ self.send_scannerOpen(tableName, startRow, columns) return self.recv_scannerOpen() @@ -1655,23 +1739,18 @@ def scannerOpenWithStop(self, tableName, startRow, stopRow, columns): Get a scanner on the current table starting and stopping at the specified rows. ending at the last row in the table. Return the specified columns. - - @param columns columns to scan. If column name is a column family, all - columns of the specified column family are returned. Its also possible - to pass a regex in the column qualifier. - @param tableName name of table - @param startRow starting row in table to scan. send "" (empty string) to - start at the first row. - @param stopRow row to stop scanning on. This row is *not* included in the - scanner's results - + @return scanner id to be used with other scanner procedures - + Parameters: - - tableName - - startRow - - stopRow - - columns + - tableName: name of table + - startRow: Starting row in table to scan. + Send "" (empty string) to start at the first row. + - stopRow: row to stop scanning on. This row is *not* included in the + scanner's results + - columns: columns to scan. If column name is a column family, all + columns of the specified column family are returned. It's also possible + to pass a regex in the column qualifier. """ self.send_scannerOpenWithStop(tableName, startRow, stopRow, columns) return self.recv_scannerOpenWithStop() @@ -1703,27 +1782,63 @@ def recv_scannerOpenWithStop(self, ): raise result.io raise TApplicationException(TApplicationException.MISSING_RESULT, "scannerOpenWithStop failed: unknown result"); + def scannerOpenWithPrefix(self, tableName, startAndPrefix, columns): + """ + Open a scanner for a given prefix. That is all rows will have the specified + prefix. No other rows will be returned. + + @return scanner id to use with other scanner calls + + Parameters: + - tableName: name of table + - startAndPrefix: the prefix (and thus start row) of the keys you want + - columns: the columns you want returned + """ + self.send_scannerOpenWithPrefix(tableName, startAndPrefix, columns) + return self.recv_scannerOpenWithPrefix() + + def send_scannerOpenWithPrefix(self, tableName, startAndPrefix, columns): + self._oprot.writeMessageBegin('scannerOpenWithPrefix', TMessageType.CALL, self._seqid) + args = scannerOpenWithPrefix_args() + args.tableName = tableName + args.startAndPrefix = startAndPrefix + args.columns = columns + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_scannerOpenWithPrefix(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = scannerOpenWithPrefix_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.io != None: + raise result.io + raise TApplicationException(TApplicationException.MISSING_RESULT, "scannerOpenWithPrefix failed: unknown result"); + def scannerOpenTs(self, tableName, startRow, columns, timestamp): """ Get a scanner on the current table starting at the specified row and ending at the last row in the table. Return the specified columns. Only values with the specified timestamp are returned. - - @param columns columns to scan. If column name is a column family, all - columns of the specified column family are returned. Its also possible - to pass a regex in the column qualifier. - @param tableName name of table - @param startRow starting row in table to scan. send "" (empty string) to - start at the first row. - @param timestamp timestamp - + @return scanner id to be used with other scanner procedures - + Parameters: - - tableName - - startRow - - columns - - timestamp + - tableName: name of table + - startRow: Starting row in table to scan. + Send "" (empty string) to start at the first row. + - columns: columns to scan. If column name is a column family, all + columns of the specified column family are returned. It's also possible + to pass a regex in the column qualifier. + - timestamp: timestamp """ self.send_scannerOpenTs(tableName, startRow, columns, timestamp) return self.recv_scannerOpenTs() @@ -1761,25 +1876,19 @@ def scannerOpenWithStopTs(self, tableName, startRow, stopRow, columns, timestamp specified rows. ending at the last row in the table. Return the specified columns. Only values with the specified timestamp are returned. - - @param columns columns to scan. If column name is a column family, all - columns of the specified column family are returned. Its also possible - to pass a regex in the column qualifier. - @param tableName name of table - @param startRow starting row in table to scan. send "" (empty string) to - start at the first row. - @param stopRow row to stop scanning on. This row is *not* included - in the scanner's results - @param timestamp timestamp - + @return scanner id to be used with other scanner procedures - + Parameters: - - tableName - - startRow - - stopRow - - columns - - timestamp + - tableName: name of table + - startRow: Starting row in table to scan. + Send "" (empty string) to start at the first row. + - stopRow: row to stop scanning on. This row is *not* included in the + scanner's results + - columns: columns to scan. If column name is a column family, all + columns of the specified column family are returned. It's also possible + to pass a regex in the column qualifier. + - timestamp: timestamp """ self.send_scannerOpenWithStopTs(tableName, startRow, stopRow, columns, timestamp) return self.recv_scannerOpenWithStopTs() @@ -1818,14 +1927,15 @@ def scannerGet(self, id): row in the table. When there are no more rows in the table, or a key greater-than-or-equal-to the scanner's specified stopRow is reached, an empty list is returned. - - @param id id of a scanner returned by scannerOpen + @return a TRowResult containing the current row and a map of the columns to TCells. + @throws IllegalArgument if ScannerID is invalid + @throws NotFound when the scanner reaches the end - + Parameters: - - id + - id: id of a scanner returned by scannerOpen """ self.send_scannerGet(id) return self.recv_scannerGet() @@ -1862,16 +1972,16 @@ def scannerGetList(self, id, nbRows): rows and advances to the next row in the table. When there are no more rows in the table, or a key greater-than-or-equal-to the scanner's specified stopRow is reached, an empty list is returned. - - @param id id of a scanner returned by scannerOpen - @param nbRows number of results to regturn + @return a TRowResult containing the current row and a map of the columns to TCells. + @throws IllegalArgument if ScannerID is invalid + @throws NotFound when the scanner reaches the end - + Parameters: - - id - - nbRows + - id: id of a scanner returned by scannerOpen + - nbRows: number of results to return """ self.send_scannerGetList(id, nbRows) return self.recv_scannerGetList() @@ -1906,12 +2016,11 @@ def recv_scannerGetList(self, ): def scannerClose(self, id): """ Closes the server-state associated with an open scanner. - - @param id id of a scanner returned by scannerOpen + @throws IllegalArgument if ScannerID is invalid - + Parameters: - - id + - id: id of a scanner returned by scannerOpen """ self.send_scannerClose(id) self.recv_scannerClose() @@ -1962,6 +2071,10 @@ def __init__(self, handler): self._processMap["getRowWithColumns"] = Processor.process_getRowWithColumns self._processMap["getRowTs"] = Processor.process_getRowTs self._processMap["getRowWithColumnsTs"] = Processor.process_getRowWithColumnsTs + self._processMap["getRows"] = Processor.process_getRows + self._processMap["getRowsWithColumns"] = Processor.process_getRowsWithColumns + self._processMap["getRowsTs"] = Processor.process_getRowsTs + self._processMap["getRowsWithColumnsTs"] = Processor.process_getRowsWithColumnsTs self._processMap["mutateRow"] = Processor.process_mutateRow self._processMap["mutateRowTs"] = Processor.process_mutateRowTs self._processMap["mutateRows"] = Processor.process_mutateRows @@ -1973,6 +2086,7 @@ def __init__(self, handler): self._processMap["deleteAllRowTs"] = Processor.process_deleteAllRowTs self._processMap["scannerOpen"] = Processor.process_scannerOpen self._processMap["scannerOpenWithStop"] = Processor.process_scannerOpenWithStop + self._processMap["scannerOpenWithPrefix"] = Processor.process_scannerOpenWithPrefix self._processMap["scannerOpenTs"] = Processor.process_scannerOpenTs self._processMap["scannerOpenWithStopTs"] = Processor.process_scannerOpenWithStopTs self._processMap["scannerGet"] = Processor.process_scannerGet @@ -2236,6 +2350,62 @@ def process_getRowWithColumnsTs(self, seqid, iprot, oprot): oprot.writeMessageEnd() oprot.trans.flush() + def process_getRows(self, seqid, iprot, oprot): + args = getRows_args() + args.read(iprot) + iprot.readMessageEnd() + result = getRows_result() + try: + result.success = self._handler.getRows(args.tableName, args.rows) + except IOError, io: + result.io = io + oprot.writeMessageBegin("getRows", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_getRowsWithColumns(self, seqid, iprot, oprot): + args = getRowsWithColumns_args() + args.read(iprot) + iprot.readMessageEnd() + result = getRowsWithColumns_result() + try: + result.success = self._handler.getRowsWithColumns(args.tableName, args.rows, args.columns) + except IOError, io: + result.io = io + oprot.writeMessageBegin("getRowsWithColumns", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_getRowsTs(self, seqid, iprot, oprot): + args = getRowsTs_args() + args.read(iprot) + iprot.readMessageEnd() + result = getRowsTs_result() + try: + result.success = self._handler.getRowsTs(args.tableName, args.rows, args.timestamp) + except IOError, io: + result.io = io + oprot.writeMessageBegin("getRowsTs", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_getRowsWithColumnsTs(self, seqid, iprot, oprot): + args = getRowsWithColumnsTs_args() + args.read(iprot) + iprot.readMessageEnd() + result = getRowsWithColumnsTs_result() + try: + result.success = self._handler.getRowsWithColumnsTs(args.tableName, args.rows, args.columns, args.timestamp) + except IOError, io: + result.io = io + oprot.writeMessageBegin("getRowsWithColumnsTs", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_mutateRow(self, seqid, iprot, oprot): args = mutateRow_args() args.read(iprot) @@ -2400,6 +2570,20 @@ def process_scannerOpenWithStop(self, seqid, iprot, oprot): oprot.writeMessageEnd() oprot.trans.flush() + def process_scannerOpenWithPrefix(self, seqid, iprot, oprot): + args = scannerOpenWithPrefix_args() + args.read(iprot) + iprot.readMessageEnd() + result = scannerOpenWithPrefix_result() + try: + result.success = self._handler.scannerOpenWithPrefix(args.tableName, args.startAndPrefix, args.columns) + except IOError, io: + result.io = io + oprot.writeMessageBegin("scannerOpenWithPrefix", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_scannerOpenTs(self, seqid, iprot, oprot): args = scannerOpenTs_args() args.read(iprot) @@ -2482,7 +2666,7 @@ def process_scannerClose(self, seqid, iprot, oprot): class enableTable_args: """ Attributes: - - tableName + - tableName: name of the table """ thrift_spec = ( @@ -2523,6 +2707,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -2580,6 +2767,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -2595,7 +2785,7 @@ def __ne__(self, other): class disableTable_args: """ Attributes: - - tableName + - tableName: name of the table """ thrift_spec = ( @@ -2636,6 +2826,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -2693,6 +2886,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -2708,7 +2904,7 @@ def __ne__(self, other): class isTableEnabled_args: """ Attributes: - - tableName + - tableName: name of the table to check """ thrift_spec = ( @@ -2749,6 +2945,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -2817,6 +3016,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -2873,6 +3075,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -2930,6 +3135,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -2986,6 +3194,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3043,6 +3254,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3081,6 +3295,9 @@ def write(self, oprot): oprot.writeStructBegin('getTableNames_args') oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3157,6 +3374,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3172,7 +3392,7 @@ def __ne__(self, other): class getColumnDescriptors_args: """ Attributes: - - tableName + - tableName: table name """ thrift_spec = ( @@ -3213,6 +3433,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3292,6 +3515,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3307,7 +3533,7 @@ def __ne__(self, other): class getTableRegions_args: """ Attributes: - - tableName + - tableName: table name """ thrift_spec = ( @@ -3348,6 +3574,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3425,6 +3654,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3440,8 +3672,8 @@ def __ne__(self, other): class createTable_args: """ Attributes: - - tableName - - columnFamilies + - tableName: name of table to create + - columnFamilies: list of column family descriptors """ thrift_spec = ( @@ -3502,6 +3734,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3585,6 +3820,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3600,7 +3838,7 @@ def __ne__(self, other): class deleteTable_args: """ Attributes: - - tableName + - tableName: name of table to delete """ thrift_spec = ( @@ -3641,6 +3879,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3698,6 +3939,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3713,9 +3957,9 @@ def __ne__(self, other): class get_args: """ Attributes: - - tableName - - row - - column + - tableName: name of table + - row: row key + - column: column name """ thrift_spec = ( @@ -3778,6 +4022,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3855,6 +4102,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -3870,10 +4120,10 @@ def __ne__(self, other): class getVer_args: """ Attributes: - - tableName - - row - - column - - numVersions + - tableName: name of table + - row: row key + - column: column name + - numVersions: number of versions to retrieve """ thrift_spec = ( @@ -3947,6 +4197,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -4024,6 +4277,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -4039,11 +4295,11 @@ def __ne__(self, other): class getVerTs_args: """ Attributes: - - tableName - - row - - column - - timestamp - - numVersions + - tableName: name of table + - row: row key + - column: column name + - timestamp: timestamp + - numVersions: number of versions to retrieve """ thrift_spec = ( @@ -4128,6 +4384,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -4205,6 +4464,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -4220,8 +4482,8 @@ def __ne__(self, other): class getRow_args: """ Attributes: - - tableName - - row + - tableName: name of table + - row: row key """ thrift_spec = ( @@ -4273,6 +4535,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -4350,6 +4615,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -4365,9 +4633,9 @@ def __ne__(self, other): class getRowWithColumns_args: """ Attributes: - - tableName - - row - - columns + - tableName: name of table + - row: row key + - columns: List of columns to return, null for all columns """ thrift_spec = ( @@ -4438,6 +4706,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -4515,6 +4786,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -4530,9 +4804,9 @@ def __ne__(self, other): class getRowTs_args: """ Attributes: - - tableName - - row - - timestamp + - tableName: name of the table + - row: row key + - timestamp: timestamp """ thrift_spec = ( @@ -4595,6 +4869,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -4672,6 +4949,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -4687,9 +4967,9 @@ def __ne__(self, other): class getRowWithColumnsTs_args: """ Attributes: - - tableName - - row - - columns + - tableName: name of table + - row: row key + - columns: List of columns to return, null for all columns - timestamp """ @@ -4772,6 +5052,709 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class getRowWithColumnsTs_result: + """ + Attributes: + - success + - io + """ + + thrift_spec = ( + (0, TType.LIST, 'success', (TType.STRUCT,(TRowResult, TRowResult.thrift_spec)), None, ), # 0 + (1, TType.STRUCT, 'io', (IOError, IOError.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, io=None,): + self.success = success + self.io = io + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype105, _size102) = iprot.readListBegin() + for _i106 in xrange(_size102): + _elem107 = TRowResult() + _elem107.read(iprot) + self.success.append(_elem107) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.io = IOError() + self.io.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('getRowWithColumnsTs_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter108 in self.success: + iter108.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.io != None: + oprot.writeFieldBegin('io', TType.STRUCT, 1) + self.io.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class getRows_args: + """ + Attributes: + - tableName: name of table + - rows: row keys + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'tableName', None, None, ), # 1 + (2, TType.LIST, 'rows', (TType.STRING,None), None, ), # 2 + ) + + def __init__(self, tableName=None, rows=None,): + self.tableName = tableName + self.rows = rows + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.tableName = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.rows = [] + (_etype112, _size109) = iprot.readListBegin() + for _i113 in xrange(_size109): + _elem114 = iprot.readString(); + self.rows.append(_elem114) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('getRows_args') + if self.tableName != None: + oprot.writeFieldBegin('tableName', TType.STRING, 1) + oprot.writeString(self.tableName) + oprot.writeFieldEnd() + if self.rows != None: + oprot.writeFieldBegin('rows', TType.LIST, 2) + oprot.writeListBegin(TType.STRING, len(self.rows)) + for iter115 in self.rows: + oprot.writeString(iter115) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class getRows_result: + """ + Attributes: + - success + - io + """ + + thrift_spec = ( + (0, TType.LIST, 'success', (TType.STRUCT,(TRowResult, TRowResult.thrift_spec)), None, ), # 0 + (1, TType.STRUCT, 'io', (IOError, IOError.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, io=None,): + self.success = success + self.io = io + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype119, _size116) = iprot.readListBegin() + for _i120 in xrange(_size116): + _elem121 = TRowResult() + _elem121.read(iprot) + self.success.append(_elem121) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.io = IOError() + self.io.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('getRows_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter122 in self.success: + iter122.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.io != None: + oprot.writeFieldBegin('io', TType.STRUCT, 1) + self.io.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class getRowsWithColumns_args: + """ + Attributes: + - tableName: name of table + - rows: row keys + - columns: List of columns to return, null for all columns + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'tableName', None, None, ), # 1 + (2, TType.LIST, 'rows', (TType.STRING,None), None, ), # 2 + (3, TType.LIST, 'columns', (TType.STRING,None), None, ), # 3 + ) + + def __init__(self, tableName=None, rows=None, columns=None,): + self.tableName = tableName + self.rows = rows + self.columns = columns + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.tableName = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.rows = [] + (_etype126, _size123) = iprot.readListBegin() + for _i127 in xrange(_size123): + _elem128 = iprot.readString(); + self.rows.append(_elem128) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.columns = [] + (_etype132, _size129) = iprot.readListBegin() + for _i133 in xrange(_size129): + _elem134 = iprot.readString(); + self.columns.append(_elem134) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('getRowsWithColumns_args') + if self.tableName != None: + oprot.writeFieldBegin('tableName', TType.STRING, 1) + oprot.writeString(self.tableName) + oprot.writeFieldEnd() + if self.rows != None: + oprot.writeFieldBegin('rows', TType.LIST, 2) + oprot.writeListBegin(TType.STRING, len(self.rows)) + for iter135 in self.rows: + oprot.writeString(iter135) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.columns != None: + oprot.writeFieldBegin('columns', TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.columns)) + for iter136 in self.columns: + oprot.writeString(iter136) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class getRowsWithColumns_result: + """ + Attributes: + - success + - io + """ + + thrift_spec = ( + (0, TType.LIST, 'success', (TType.STRUCT,(TRowResult, TRowResult.thrift_spec)), None, ), # 0 + (1, TType.STRUCT, 'io', (IOError, IOError.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, io=None,): + self.success = success + self.io = io + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype140, _size137) = iprot.readListBegin() + for _i141 in xrange(_size137): + _elem142 = TRowResult() + _elem142.read(iprot) + self.success.append(_elem142) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.io = IOError() + self.io.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('getRowsWithColumns_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter143 in self.success: + iter143.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.io != None: + oprot.writeFieldBegin('io', TType.STRUCT, 1) + self.io.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class getRowsTs_args: + """ + Attributes: + - tableName: name of the table + - rows: row keys + - timestamp: timestamp + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'tableName', None, None, ), # 1 + (2, TType.LIST, 'rows', (TType.STRING,None), None, ), # 2 + (3, TType.I64, 'timestamp', None, None, ), # 3 + ) + + def __init__(self, tableName=None, rows=None, timestamp=None,): + self.tableName = tableName + self.rows = rows + self.timestamp = timestamp + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.tableName = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.rows = [] + (_etype147, _size144) = iprot.readListBegin() + for _i148 in xrange(_size144): + _elem149 = iprot.readString(); + self.rows.append(_elem149) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.timestamp = iprot.readI64(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('getRowsTs_args') + if self.tableName != None: + oprot.writeFieldBegin('tableName', TType.STRING, 1) + oprot.writeString(self.tableName) + oprot.writeFieldEnd() + if self.rows != None: + oprot.writeFieldBegin('rows', TType.LIST, 2) + oprot.writeListBegin(TType.STRING, len(self.rows)) + for iter150 in self.rows: + oprot.writeString(iter150) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.timestamp != None: + oprot.writeFieldBegin('timestamp', TType.I64, 3) + oprot.writeI64(self.timestamp) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class getRowsTs_result: + """ + Attributes: + - success + - io + """ + + thrift_spec = ( + (0, TType.LIST, 'success', (TType.STRUCT,(TRowResult, TRowResult.thrift_spec)), None, ), # 0 + (1, TType.STRUCT, 'io', (IOError, IOError.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, io=None,): + self.success = success + self.io = io + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype154, _size151) = iprot.readListBegin() + for _i155 in xrange(_size151): + _elem156 = TRowResult() + _elem156.read(iprot) + self.success.append(_elem156) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.io = IOError() + self.io.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('getRowsTs_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter157 in self.success: + iter157.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.io != None: + oprot.writeFieldBegin('io', TType.STRUCT, 1) + self.io.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class getRowsWithColumnsTs_args: + """ + Attributes: + - tableName: name of table + - rows: row keys + - columns: List of columns to return, null for all columns + - timestamp + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'tableName', None, None, ), # 1 + (2, TType.LIST, 'rows', (TType.STRING,None), None, ), # 2 + (3, TType.LIST, 'columns', (TType.STRING,None), None, ), # 3 + (4, TType.I64, 'timestamp', None, None, ), # 4 + ) + + def __init__(self, tableName=None, rows=None, columns=None, timestamp=None,): + self.tableName = tableName + self.rows = rows + self.columns = columns + self.timestamp = timestamp + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.tableName = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.rows = [] + (_etype161, _size158) = iprot.readListBegin() + for _i162 in xrange(_size158): + _elem163 = iprot.readString(); + self.rows.append(_elem163) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.columns = [] + (_etype167, _size164) = iprot.readListBegin() + for _i168 in xrange(_size164): + _elem169 = iprot.readString(); + self.columns.append(_elem169) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.timestamp = iprot.readI64(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('getRowsWithColumnsTs_args') + if self.tableName != None: + oprot.writeFieldBegin('tableName', TType.STRING, 1) + oprot.writeString(self.tableName) + oprot.writeFieldEnd() + if self.rows != None: + oprot.writeFieldBegin('rows', TType.LIST, 2) + oprot.writeListBegin(TType.STRING, len(self.rows)) + for iter170 in self.rows: + oprot.writeString(iter170) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.columns != None: + oprot.writeFieldBegin('columns', TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.columns)) + for iter171 in self.columns: + oprot.writeString(iter171) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.timestamp != None: + oprot.writeFieldBegin('timestamp', TType.I64, 4) + oprot.writeI64(self.timestamp) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -4784,7 +5767,7 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) -class getRowWithColumnsTs_result: +class getRowsWithColumnsTs_result: """ Attributes: - success @@ -4812,11 +5795,11 @@ def read(self, iprot): if fid == 0: if ftype == TType.LIST: self.success = [] - (_etype105, _size102) = iprot.readListBegin() - for _i106 in xrange(_size102): - _elem107 = TRowResult() - _elem107.read(iprot) - self.success.append(_elem107) + (_etype175, _size172) = iprot.readListBegin() + for _i176 in xrange(_size172): + _elem177 = TRowResult() + _elem177.read(iprot) + self.success.append(_elem177) iprot.readListEnd() else: iprot.skip(ftype) @@ -4835,12 +5818,12 @@ def write(self, oprot): if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return - oprot.writeStructBegin('getRowWithColumnsTs_result') + oprot.writeStructBegin('getRowsWithColumnsTs_result') if self.success != None: oprot.writeFieldBegin('success', TType.LIST, 0) oprot.writeListBegin(TType.STRUCT, len(self.success)) - for iter108 in self.success: - iter108.write(oprot) + for iter178 in self.success: + iter178.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.io != None: @@ -4849,6 +5832,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -4864,9 +5850,9 @@ def __ne__(self, other): class mutateRow_args: """ Attributes: - - tableName - - row - - mutations + - tableName: name of table + - row: row key + - mutations: list of mutation commands """ thrift_spec = ( @@ -4903,11 +5889,11 @@ def read(self, iprot): elif fid == 3: if ftype == TType.LIST: self.mutations = [] - (_etype112, _size109) = iprot.readListBegin() - for _i113 in xrange(_size109): - _elem114 = Mutation() - _elem114.read(iprot) - self.mutations.append(_elem114) + (_etype182, _size179) = iprot.readListBegin() + for _i183 in xrange(_size179): + _elem184 = Mutation() + _elem184.read(iprot) + self.mutations.append(_elem184) iprot.readListEnd() else: iprot.skip(ftype) @@ -4932,12 +5918,15 @@ def write(self, oprot): if self.mutations != None: oprot.writeFieldBegin('mutations', TType.LIST, 3) oprot.writeListBegin(TType.STRUCT, len(self.mutations)) - for iter115 in self.mutations: - iter115.write(oprot) + for iter185 in self.mutations: + iter185.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5008,6 +5997,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5023,10 +6015,10 @@ def __ne__(self, other): class mutateRowTs_args: """ Attributes: - - tableName - - row - - mutations - - timestamp + - tableName: name of table + - row: row key + - mutations: list of mutation commands + - timestamp: timestamp """ thrift_spec = ( @@ -5065,11 +6057,11 @@ def read(self, iprot): elif fid == 3: if ftype == TType.LIST: self.mutations = [] - (_etype119, _size116) = iprot.readListBegin() - for _i120 in xrange(_size116): - _elem121 = Mutation() - _elem121.read(iprot) - self.mutations.append(_elem121) + (_etype189, _size186) = iprot.readListBegin() + for _i190 in xrange(_size186): + _elem191 = Mutation() + _elem191.read(iprot) + self.mutations.append(_elem191) iprot.readListEnd() else: iprot.skip(ftype) @@ -5099,8 +6091,8 @@ def write(self, oprot): if self.mutations != None: oprot.writeFieldBegin('mutations', TType.LIST, 3) oprot.writeListBegin(TType.STRUCT, len(self.mutations)) - for iter122 in self.mutations: - iter122.write(oprot) + for iter192 in self.mutations: + iter192.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.timestamp != None: @@ -5109,6 +6101,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5179,6 +6174,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5194,8 +6192,8 @@ def __ne__(self, other): class mutateRows_args: """ Attributes: - - tableName - - rowBatches + - tableName: name of table + - rowBatches: list of row batches """ thrift_spec = ( @@ -5225,11 +6223,11 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.rowBatches = [] - (_etype126, _size123) = iprot.readListBegin() - for _i127 in xrange(_size123): - _elem128 = BatchMutation() - _elem128.read(iprot) - self.rowBatches.append(_elem128) + (_etype196, _size193) = iprot.readListBegin() + for _i197 in xrange(_size193): + _elem198 = BatchMutation() + _elem198.read(iprot) + self.rowBatches.append(_elem198) iprot.readListEnd() else: iprot.skip(ftype) @@ -5250,12 +6248,15 @@ def write(self, oprot): if self.rowBatches != None: oprot.writeFieldBegin('rowBatches', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.rowBatches)) - for iter129 in self.rowBatches: - iter129.write(oprot) + for iter199 in self.rowBatches: + iter199.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5326,6 +6327,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5341,9 +6345,9 @@ def __ne__(self, other): class mutateRowsTs_args: """ Attributes: - - tableName - - rowBatches - - timestamp + - tableName: name of table + - rowBatches: list of row batches + - timestamp: timestamp """ thrift_spec = ( @@ -5375,11 +6379,11 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.rowBatches = [] - (_etype133, _size130) = iprot.readListBegin() - for _i134 in xrange(_size130): - _elem135 = BatchMutation() - _elem135.read(iprot) - self.rowBatches.append(_elem135) + (_etype203, _size200) = iprot.readListBegin() + for _i204 in xrange(_size200): + _elem205 = BatchMutation() + _elem205.read(iprot) + self.rowBatches.append(_elem205) iprot.readListEnd() else: iprot.skip(ftype) @@ -5405,8 +6409,8 @@ def write(self, oprot): if self.rowBatches != None: oprot.writeFieldBegin('rowBatches', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.rowBatches)) - for iter136 in self.rowBatches: - iter136.write(oprot) + for iter206 in self.rowBatches: + iter206.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.timestamp != None: @@ -5415,6 +6419,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5485,6 +6492,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5500,10 +6510,10 @@ def __ne__(self, other): class atomicIncrement_args: """ Attributes: - - tableName - - row - - column - - value + - tableName: name of table + - row: row to increment + - column: name of column + - value: amount to increment by """ thrift_spec = ( @@ -5577,6 +6587,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5658,6 +6671,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5673,9 +6689,9 @@ def __ne__(self, other): class deleteAll_args: """ Attributes: - - tableName - - row - - column + - tableName: name of table + - row: Row to update + - column: name of column whose value is to be deleted """ thrift_spec = ( @@ -5738,6 +6754,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5795,6 +6814,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5810,10 +6832,10 @@ def __ne__(self, other): class deleteAllTs_args: """ Attributes: - - tableName - - row - - column - - timestamp + - tableName: name of table + - row: Row to update + - column: name of column whose value is to be deleted + - timestamp: timestamp """ thrift_spec = ( @@ -5887,6 +6909,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5944,6 +6969,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -5959,8 +6987,8 @@ def __ne__(self, other): class deleteAllRow_args: """ Attributes: - - tableName - - row + - tableName: name of table + - row: key of the row to be completely deleted. """ thrift_spec = ( @@ -6012,6 +7040,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -6069,6 +7100,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -6084,9 +7118,9 @@ def __ne__(self, other): class deleteAllRowTs_args: """ Attributes: - - tableName - - row - - timestamp + - tableName: name of table + - row: key of the row to be completely deleted. + - timestamp: timestamp """ thrift_spec = ( @@ -6149,6 +7183,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -6206,6 +7243,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -6221,9 +7261,12 @@ def __ne__(self, other): class scannerOpen_args: """ Attributes: - - tableName - - startRow - - columns + - tableName: name of table + - startRow: Starting row in table to scan. + Send "" (empty string) to start at the first row. + - columns: columns to scan. If column name is a column family, all + columns of the specified column family are returned. It's also possible + to pass a regex in the column qualifier. """ thrift_spec = ( @@ -6260,10 +7303,10 @@ def read(self, iprot): elif fid == 3: if ftype == TType.LIST: self.columns = [] - (_etype140, _size137) = iprot.readListBegin() - for _i141 in xrange(_size137): - _elem142 = iprot.readString(); - self.columns.append(_elem142) + (_etype210, _size207) = iprot.readListBegin() + for _i211 in xrange(_size207): + _elem212 = iprot.readString(); + self.columns.append(_elem212) iprot.readListEnd() else: iprot.skip(ftype) @@ -6288,12 +7331,15 @@ def write(self, oprot): if self.columns != None: oprot.writeFieldBegin('columns', TType.LIST, 3) oprot.writeListBegin(TType.STRING, len(self.columns)) - for iter143 in self.columns: - oprot.writeString(iter143) + for iter213 in self.columns: + oprot.writeString(iter213) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -6362,6 +7408,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -6377,10 +7426,14 @@ def __ne__(self, other): class scannerOpenWithStop_args: """ Attributes: - - tableName - - startRow - - stopRow - - columns + - tableName: name of table + - startRow: Starting row in table to scan. + Send "" (empty string) to start at the first row. + - stopRow: row to stop scanning on. This row is *not* included in the + scanner's results + - columns: columns to scan. If column name is a column family, all + columns of the specified column family are returned. It's also possible + to pass a regex in the column qualifier. """ thrift_spec = ( @@ -6424,10 +7477,10 @@ def read(self, iprot): elif fid == 4: if ftype == TType.LIST: self.columns = [] - (_etype147, _size144) = iprot.readListBegin() - for _i148 in xrange(_size144): - _elem149 = iprot.readString(); - self.columns.append(_elem149) + (_etype217, _size214) = iprot.readListBegin() + for _i218 in xrange(_size214): + _elem219 = iprot.readString(); + self.columns.append(_elem219) iprot.readListEnd() else: iprot.skip(ftype) @@ -6456,12 +7509,15 @@ def write(self, oprot): if self.columns != None: oprot.writeFieldBegin('columns', TType.LIST, 4) oprot.writeListBegin(TType.STRING, len(self.columns)) - for iter150 in self.columns: - oprot.writeString(iter150) + for iter220 in self.columns: + oprot.writeString(iter220) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -6530,6 +7586,171 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class scannerOpenWithPrefix_args: + """ + Attributes: + - tableName: name of table + - startAndPrefix: the prefix (and thus start row) of the keys you want + - columns: the columns you want returned + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'tableName', None, None, ), # 1 + (2, TType.STRING, 'startAndPrefix', None, None, ), # 2 + (3, TType.LIST, 'columns', (TType.STRING,None), None, ), # 3 + ) + + def __init__(self, tableName=None, startAndPrefix=None, columns=None,): + self.tableName = tableName + self.startAndPrefix = startAndPrefix + self.columns = columns + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.tableName = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.startAndPrefix = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.columns = [] + (_etype224, _size221) = iprot.readListBegin() + for _i225 in xrange(_size221): + _elem226 = iprot.readString(); + self.columns.append(_elem226) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('scannerOpenWithPrefix_args') + if self.tableName != None: + oprot.writeFieldBegin('tableName', TType.STRING, 1) + oprot.writeString(self.tableName) + oprot.writeFieldEnd() + if self.startAndPrefix != None: + oprot.writeFieldBegin('startAndPrefix', TType.STRING, 2) + oprot.writeString(self.startAndPrefix) + oprot.writeFieldEnd() + if self.columns != None: + oprot.writeFieldBegin('columns', TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.columns)) + for iter227 in self.columns: + oprot.writeString(iter227) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class scannerOpenWithPrefix_result: + """ + Attributes: + - success + - io + """ + + thrift_spec = ( + (0, TType.I32, 'success', None, None, ), # 0 + (1, TType.STRUCT, 'io', (IOError, IOError.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, io=None,): + self.success = success + self.io = io + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I32: + self.success = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.io = IOError() + self.io.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('scannerOpenWithPrefix_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.I32, 0) + oprot.writeI32(self.success) + oprot.writeFieldEnd() + if self.io != None: + oprot.writeFieldBegin('io', TType.STRUCT, 1) + self.io.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -6545,10 +7766,13 @@ def __ne__(self, other): class scannerOpenTs_args: """ Attributes: - - tableName - - startRow - - columns - - timestamp + - tableName: name of table + - startRow: Starting row in table to scan. + Send "" (empty string) to start at the first row. + - columns: columns to scan. If column name is a column family, all + columns of the specified column family are returned. It's also possible + to pass a regex in the column qualifier. + - timestamp: timestamp """ thrift_spec = ( @@ -6587,10 +7811,10 @@ def read(self, iprot): elif fid == 3: if ftype == TType.LIST: self.columns = [] - (_etype154, _size151) = iprot.readListBegin() - for _i155 in xrange(_size151): - _elem156 = iprot.readString(); - self.columns.append(_elem156) + (_etype231, _size228) = iprot.readListBegin() + for _i232 in xrange(_size228): + _elem233 = iprot.readString(); + self.columns.append(_elem233) iprot.readListEnd() else: iprot.skip(ftype) @@ -6620,8 +7844,8 @@ def write(self, oprot): if self.columns != None: oprot.writeFieldBegin('columns', TType.LIST, 3) oprot.writeListBegin(TType.STRING, len(self.columns)) - for iter157 in self.columns: - oprot.writeString(iter157) + for iter234 in self.columns: + oprot.writeString(iter234) oprot.writeListEnd() oprot.writeFieldEnd() if self.timestamp != None: @@ -6630,6 +7854,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -6698,6 +7925,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -6713,11 +7943,15 @@ def __ne__(self, other): class scannerOpenWithStopTs_args: """ Attributes: - - tableName - - startRow - - stopRow - - columns - - timestamp + - tableName: name of table + - startRow: Starting row in table to scan. + Send "" (empty string) to start at the first row. + - stopRow: row to stop scanning on. This row is *not* included in the + scanner's results + - columns: columns to scan. If column name is a column family, all + columns of the specified column family are returned. It's also possible + to pass a regex in the column qualifier. + - timestamp: timestamp """ thrift_spec = ( @@ -6763,10 +7997,10 @@ def read(self, iprot): elif fid == 4: if ftype == TType.LIST: self.columns = [] - (_etype161, _size158) = iprot.readListBegin() - for _i162 in xrange(_size158): - _elem163 = iprot.readString(); - self.columns.append(_elem163) + (_etype238, _size235) = iprot.readListBegin() + for _i239 in xrange(_size235): + _elem240 = iprot.readString(); + self.columns.append(_elem240) iprot.readListEnd() else: iprot.skip(ftype) @@ -6800,8 +8034,8 @@ def write(self, oprot): if self.columns != None: oprot.writeFieldBegin('columns', TType.LIST, 4) oprot.writeListBegin(TType.STRING, len(self.columns)) - for iter164 in self.columns: - oprot.writeString(iter164) + for iter241 in self.columns: + oprot.writeString(iter241) oprot.writeListEnd() oprot.writeFieldEnd() if self.timestamp != None: @@ -6810,6 +8044,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -6878,6 +8115,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -6893,7 +8133,7 @@ def __ne__(self, other): class scannerGet_args: """ Attributes: - - id + - id: id of a scanner returned by scannerOpen """ thrift_spec = ( @@ -6934,6 +8174,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -6977,11 +8220,11 @@ def read(self, iprot): if fid == 0: if ftype == TType.LIST: self.success = [] - (_etype168, _size165) = iprot.readListBegin() - for _i169 in xrange(_size165): - _elem170 = TRowResult() - _elem170.read(iprot) - self.success.append(_elem170) + (_etype245, _size242) = iprot.readListBegin() + for _i246 in xrange(_size242): + _elem247 = TRowResult() + _elem247.read(iprot) + self.success.append(_elem247) iprot.readListEnd() else: iprot.skip(ftype) @@ -7010,8 +8253,8 @@ def write(self, oprot): if self.success != None: oprot.writeFieldBegin('success', TType.LIST, 0) oprot.writeListBegin(TType.STRUCT, len(self.success)) - for iter171 in self.success: - iter171.write(oprot) + for iter248 in self.success: + iter248.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.io != None: @@ -7024,6 +8267,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -7039,8 +8285,8 @@ def __ne__(self, other): class scannerGetList_args: """ Attributes: - - id - - nbRows + - id: id of a scanner returned by scannerOpen + - nbRows: number of results to return """ thrift_spec = ( @@ -7092,6 +8338,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -7135,11 +8384,11 @@ def read(self, iprot): if fid == 0: if ftype == TType.LIST: self.success = [] - (_etype175, _size172) = iprot.readListBegin() - for _i176 in xrange(_size172): - _elem177 = TRowResult() - _elem177.read(iprot) - self.success.append(_elem177) + (_etype252, _size249) = iprot.readListBegin() + for _i253 in xrange(_size249): + _elem254 = TRowResult() + _elem254.read(iprot) + self.success.append(_elem254) iprot.readListEnd() else: iprot.skip(ftype) @@ -7168,8 +8417,8 @@ def write(self, oprot): if self.success != None: oprot.writeFieldBegin('success', TType.LIST, 0) oprot.writeListBegin(TType.STRUCT, len(self.success)) - for iter178 in self.success: - iter178.write(oprot) + for iter255 in self.success: + iter255.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.io != None: @@ -7182,6 +8431,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -7197,7 +8449,7 @@ def __ne__(self, other): class scannerClose_args: """ Attributes: - - id + - id: id of a scanner returned by scannerOpen """ thrift_spec = ( @@ -7238,6 +8490,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -7308,6 +8563,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -7319,5 +8577,3 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - diff --git a/AppDB/hbase/hbase_interface.py b/AppDB/hbase/hbase_interface.py new file mode 100644 index 0000000000..e36168483c --- /dev/null +++ b/AppDB/hbase/hbase_interface.py @@ -0,0 +1,316 @@ +# Author: Navraj Chohan + +import os + +import appscale_logger +import Hbase +import helper_functions +import threading +import ttypes + +from thrift import Thrift +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol +from dbinterface import * + +# Thrift port to connect to HBase +THRIFT_PORT = 9090 + +class DatastoreProxy(AppDBInterface): + """ + The AppScale DB API class implementation for HBase + """ + def __init__(self, logger = appscale_logger.getLogger("datastore-hbase")): + """ + Constructor + Args: + logger: Used for logging + """ + self.lock = threading.Lock() + self.logger = logger + self.connection = self.create_connection() + + def batch_get_entity(self, table_name, row_keys, column_names): + """Allows access to multiple rows with a single call + + Args: + table_name: The table to access + row_keys: A list of keys to access + column_names: A list of columns to access + Returns: + A dictionary of {key:{column_name:value,...}} + Raise: + TypeError: Raised when given bad types for args. + """ + + if not isinstance(table_name, str): raise TypeError("Expected str") + if not isinstance(column_names, list): raise TypeError("Expected list") + if not isinstance(row_keys, list): raise TypeError("Expected list") + + result = {} + rows = [] + column_list = [] + client = self.__init_connection() + + for ii in column_names: + column_list.append(ii + ":") + rows = client.getRowsWithColumns(table_name, row_keys, column_list) + + for row in rows: + result[row.row] = {} + for col in column_names: + if (col+":") in row.columns: + result[row.row][col] = row.columns[col + ":"].value + + for row in row_keys: + if row not in result: + result[row] = {} + self.__release_lock() + return result + + def batch_put_entity(self, table_name, row_keys, column_names, cell_values): + """Allows callers to store multiple rows with a single call. + + Args: + table_name: The table to mutate + row_keys: A list of keys to store on + column_names: A list of columns to mutate + cell_values: A dict of key/value pairs + Returns: + Nothing + Raises: + TypeError: Raised when given bad types of for args. + """ + + if not isinstance(table_name, str): raise TypeError("Expected str") + if not isinstance(column_names, list): raise TypeError("Expected list") + if not isinstance(row_keys, list): raise TypeError("Expected list") + if not isinstance(cell_values, dict): raise TypeError("Expected dict") + + + all_mutations = [] + for row in row_keys: + batch_mutation = ttypes.BatchMutation() + mutations = [] + for col in column_names: + m = ttypes.Mutation() + m.column = col + ":" + m.value = cell_values[row][col] + mutations.append(m) + batch_mutation.mutations = mutations + batch_mutation.row = row + all_mutations.append(batch_mutation) + + client = self.__init_connection() + client.mutateRows(table_name, all_mutations) + self.__release_lock() + + def batch_delete(self, table_name, row_keys, column_names=[]): + """ Remove a batch of rows. + + Args: + table_name: Table to delete rows from + row_keys: A list of keys to remove + column_names: A list of column names + Returns: + Nothing + Raises: + TypeError: Raised when given bad types of for args. + """ + + if not isinstance(table_name, str): raise TypeError("Expected str") + if not isinstance(row_keys, list): raise TypeError("Expected list") + + + all_mutations = [] + for row in row_keys: + batch_mutation = ttypes.BatchMutation() + mutations = [] + for col in column_names: + m = ttypes.Mutation(isDelete=True) + m.column = col + ":" + mutations.append(m) + batch_mutation.mutations = mutations + batch_mutation.row = row + all_mutations.append(batch_mutation) + client = self.__init_connection() + client.mutateRows(table_name, all_mutations) + self.__release_lock() + + def delete_table(self, table_name): + """ Drops a given table + + Args: + table_name: The table to drop + Returns: + Nothing + Raises: + TypeError: Raised when given bad types of for args. + """ + if not isinstance(table_name, str): raise TypeError("Excepted str") + + client = self.__init_connection() + try: + client.disableTable(table_name) + client.deleteTable(table_name) + except ttypes.IOError, io: # table not found + pass + self.__release_lock() + + def create_table(self, table_name, column_names): + """ Creates a table as a column family. + + Args: + table_name: The column family name + column_names: not used + Returns: + Nothing + Raises: + TypeError: Raised when given bad types of for args. + """ + + if not isinstance(table_name, str): raise TypeError("Expected str") + if not isinstance(column_names, list): raise TypeError("Expected list") + + columnlist = [] + for ii in column_names: + col = ttypes.ColumnDescriptor() + col.name = ii + ":" + col.maxVersions = 1 + columnlist.append(col) + client = self.__init_connection() + client.createTable(table_name, columnlist) + self.__release_lock() + + def range_query(self, + table_name, + column_names, + start_key, + end_key, + limit, + offset=0, + start_inclusive=True, + end_inclusive=True, + keys_only=False): + """ Gets a dense range ordered by keys. Returns an ordered list of + dictionary of [key:{column1:value1, column2:value2},...] + or a list of keys if keys only. + + Args: + table_name: Table to access + column_names: Columns which get returned within the key range + start_key: String key starting the range query + end_key: String key ending the range query + limit: Maximum number of results to return + offset: Number to cut off from the results [offset:] + start_inclusive: Boolean if results should include the start_key + end_inclusive: Boolean if results should include the end_key + keys_only: Boolean if only returns keys and not values + Raises: + TypeError: Raised when given bad types of for args. + Returns: + Dictionary of the results + """ + if not isinstance(table_name, str): raise TypeError("Expected str") + if not isinstance(column_names, list): raise TypeError("Expected list") + if not isinstance(start_key, str): raise TypeError("Expected str") + if not isinstance(end_key, str): raise TypeError("Expected str") + if not isinstance(limit, int) and not isinstance(limit, long): + raise TypeError("Expected int or long") + if not isinstance(offset, int): raise TypeError + + results = [] + + # We add extra rows in case we exclude the start/end keys + # This makes sure the limit is upheld correctly + row_count = limit + if not start_inclusive: + row_count += 1 + if not end_inclusive: + row_count += 1 + + col_names = [] + for col in column_names: + col_names.append(col + ":") + + client = self.__init_connection() + scanner = client.scannerOpenWithStop( + table_name, start_key, end_key, col_names) + + rowresult = client.scannerGetList(scanner, row_count) + while rowresult: + row_count -= len(rowresult) + for row in rowresult: + item = {} + col_dict = {} + for c in column_names: + col_dict[c] = row.columns[c+":"].value + item[row.row] = col_dict + results.append(item) + if row_count <= 0: + break + rowresult = client.scannerGetList(scanner, row_count) + client.scannerClose(scanner) + self.__release_lock() + + # The end key is not included in the scanner. Get the last key if + # needed + if row_count != 0 and end_inclusive: + item = self.batch_get_entity(table_name, [end_key], column_names) + if item[end_key]: + results.append(item) + + if not start_inclusive and len(results) > 0: + if start_key in results[0]: + results = results[1:] + + if not end_inclusive and len(results) > 0: + if end_key in results[-1]: + results = results[:-1] + + if len(results) > limit: + results = results[:limit] + + if offset != 0 and offset <= len(results): + results = results[offset:] + + return results + + ######################## + # Private methods + ######################## + def create_connection(self): + """ Creates a connection to HBase's Thrift to the local node. + + Returns: + An HBase client object + """ + host = helper_functions.read_file('/etc/appscale/my_private_ip') + t = TSocket.TSocket(host, THRIFT_PORT) + t = TTransport.TBufferedTransport(t) + p = TBinaryProtocol.TBinaryProtocol(t) + c = Hbase.Client(p) + t.open() + return c + + def __init_connection(self): + """ + Provides a locking wrapper around a connection to make it threadsafe. + Blocks until the lock is available. + Returns: + An HBase connection + """ + self.lock.acquire() + if self.connection: + return self.connection + else: + self.connection = self.create_connection() + return self.connection + + def __release_lock(self): + """ + Releases the connection lock. + """ + self.lock.release() + diff --git a/AppDB/hbase/prime_hbase.py b/AppDB/hbase/prime_hbase.py index 948bb41a56..8f5f120127 100644 --- a/AppDB/hbase/prime_hbase.py +++ b/AppDB/hbase/prime_hbase.py @@ -23,15 +23,38 @@ from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol import py_hbase +import hbase_interface from dbconstants import * -def create_table(tablename, columns): +def create_table(table_name, columns): + """ Calls HBase to create a table + + Args: + table_name: Table to create + columns: columns for the table + Returns: + A list of current tables + """ client = py_hbase.DatastoreProxy() - return client.create_table(tablename, columns) + return client.create_table(table_name, columns) +def create_app_tables(): + """ Creates application tables for AppScale + """ + db = hbase_interface.DatastoreProxy() + db.create_table(ASC_PROPERTY_TABLE, PROPERTY_SCHEMA) + db.create_table(DSC_PROPERTY_TABLE, PROPERTY_SCHEMA) + db.create_table(APP_INDEX_TABLE, APP_INDEX_SCHEMA) + db.create_table(APP_NAMESPACE_TABLE, APP_NAMESPACE_SCHEMA) + db.create_table(APP_ID_TABLE, APP_ID_SCHEMA) + db.create_table(APP_ENTITY_TABLE, APP_ENTITY_SCHEMA) + db.create_table(APP_KIND_TABLE, APP_KIND_SCHEMA) def prime_hbase(): + """ Creates tables required for AppScale + """ print "prime hbase database" + create_app_tables() create_table(USERS_TABLE, USERS_SCHEMA) result = create_table(APPS_TABLE, APPS_SCHEMA) if (USERS_TABLE in result) and (APPS_TABLE in result): diff --git a/AppDB/hbase/test_hbase.py b/AppDB/hbase/test_hbase.py deleted file mode 100644 index 0ecc49510d..0000000000 --- a/AppDB/hbase/test_hbase.py +++ /dev/null @@ -1,130 +0,0 @@ -import py_hbase - -py_hbase = py_hbase.DatastoreProxy() - -columns = ["a","b","c"] -data = ["1","2","3"] -table_name = "hello" -key = "1" -print "key= " + key -print "columns= " + str(columns) -print "data= " + str(data) -print "table= " + table_name - -print py_hbase.put_entity(table_name, key, columns, data) -ret = py_hbase.get_entity(table_name, key, columns) -print "doing a put then get" -print ret -if ret[1:] != data: - print "ERROR doing a put then get. Data does not match" - print "returned: " + str(ret) - print "expected: " + str(data) - exit(1) -else: - print "Success" - -ret = py_hbase.get_schema("hello") -print ret -print "checking schema:" -print ret -if ret[1:] != columns: - print "ERROR in recieved schema" - print "returned: " + str(ret) - print "expected: " + str(columns) - -ret = py_hbase.delete_row(table_name, key) -print "Deleting the key %s"%key -print ret - -ret = py_hbase.get_entity(table_name, key, columns) -print "Trying to get deleted key:" -print ret -print "doing a put with key %s"%key -print py_hbase.put_entity("hello", "1", ["a","b","c"], ["1","2","3"]) -print "doing a get table" -print py_hbase.get_table("hello", ["a","b","c"]) -py_hbase.put_entity("hello", "2", ["a","b","c"], ["4","5","6"]) -print "doing get table:" -print py_hbase.get_table("hello", ["a","b","c"]) -py_hbase.put_entity("hello", "3", ["a","b","c"], ["1","2","3"]) -py_hbase.get_table("hello", ["a","b","c"]) - -print "TRYING TO REPLACE KEY 3" -py_hbase.put_entity("hello", "3", ["a","b","c"], ["1","2","3"]) -print "TRYING TO REPLACE KEY 3" -py_hbase.get_table("hello", ["a","b","c"]) -print "TRYING TO REPLACE KEY 3" -py_hbase.get_row_count("hello") -print "TRYING TO REPLACE KEY 3" -ret = py_hbase.delete_row("hello", "1") -print "TRYING TO REPLACE KEY 3" -ret = py_hbase.delete_row("hello", "2") -print "TRYING TO REPLACE KEY 3" -ret = py_hbase.delete_row("hello", "3") -print "TRYING TO REPLACE KEY 3" -py_hbase.get_table("hello", ["a","b","c"]) -print "Deleting table:" -print py_hbase.delete_table("hello") -print "deleting twice:" -print py_hbase.delete_table("hello") - -table_name = u"testing_query" -print py_hbase.delete_table(table_name) -column_names = [u"c1"] -limit = 1000 -offset = 0 -key = 0 -startrow = u"000" -endrow = u"100" -data = u"xxx" -for ii in range(0, 101): - key = str(ii) - key = ("0" * (3 - len(key))) + key - key = unicode(key) - print "Adding key " + key - print py_hbase.put_entity(table_name, key, column_names, [data + key]) -inclusive = 1 -notJustKeys = 0 -print "from table %s get columns %s with limits %s and offset %s, with start row %s and endrow %s"%(table_name, column_names, limit, offset, startrow, endrow) - -results = py_hbase.run_query(table_name, column_names, limit, offset, startrow, endrow, notJustKeys, inclusive, inclusive) -results = results[1:] -if len(results) != 101: - print "ERORR: AAA Bad number of cells returned with no limit query. 101 versus %d"%len(results) - -limit = 50 -results = py_hbase.run_query(table_name, column_names, limit, offset, startrow, endrow, notJustKeys, inclusive, inclusive) -results = results[1:] -if len(results) != 50: - print "ERORR: BBB Bad number of cells returned with 50 limit query. 50 versus %d"%len(results) -limit = 25 -offset = 25 -results = py_hbase.run_query(table_name, column_names, limit, offset, startrow, endrow, notJustKeys, inclusive, inclusive) -results = results[1:] -if len(results) != 25: - print "ERORR: CCC Bad number of cells returned with 50 limit query. 50 versus %d"%len(results) -first_key = u"xxx049" -print "Number of results:" -print len(results) -if results[24] != first_key: - print "ERORR: DDD Bad first key returned for 25 limit query with 25 offset. %s vs %s"%(first_key, results[24]) - -getOnlyKeys = 1 -results = py_hbase.run_query(table_name, column_names, limit, offset, startrow, endrow, getOnlyKeys, inclusive, inclusive) - -#print results -startrow = "001" -endrow = "003" -limit = 1000 -offset = 0 -exclusive = 0 -results = py_hbase.run_query(table_name, column_names, limit, offset, startrow, endrow, getOnlyKeys, exclusive, exclusive) -results = results[1:] -if len(results) != 1: - print "ERORR: EEE Bad number of cells returned with 0 limit query. 1 versus %d"%len(results) -last_key = u"002" -print "Number of results:" -print len(results) -if results[len(results) - 1] != last_key: - print "ERORR: FFF Bad first key returned for 0 limit query with 0 offset. %s vs %s"%(last_key, results[len(results) - 1]) - diff --git a/AppDB/hbase/ttypes.py b/AppDB/hbase/ttypes.py index d9a20b6d01..af075654dc 100644 --- a/AppDB/hbase/ttypes.py +++ b/AppDB/hbase/ttypes.py @@ -7,20 +7,21 @@ from thrift.Thrift import * from thrift.transport import TTransport -from thrift.protocol import TBinaryProtocol +from thrift.protocol import TBinaryProtocol, TProtocol try: from thrift.protocol import fastbinary except: fastbinary = None + class TCell: """ TCell - Used to transport a cell value (byte[]) and the timestamp it was stored with together as a result for get and getRow methods. This promotes the timestamp of a cell to a first-class value, making it easy to take note of temporal data. Cell is used all the way from HStore up to HTable. - + Attributes: - value - timestamp @@ -75,6 +76,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -92,13 +96,12 @@ class ColumnDescriptor: An HColumnDescriptor contains information about a column family such as the number of versions, compression settings, etc. It is used as input when creating a table or adding a column. - + Attributes: - name - maxVersions - compression - inMemory - - maxValueLength - bloomFilterType - bloomFilterVectorSize - bloomFilterNbHashes @@ -112,20 +115,18 @@ class ColumnDescriptor: (2, TType.I32, 'maxVersions', None, 3, ), # 2 (3, TType.STRING, 'compression', None, "NONE", ), # 3 (4, TType.BOOL, 'inMemory', None, False, ), # 4 - (5, TType.I32, 'maxValueLength', None, 2147483647, ), # 5 - (6, TType.STRING, 'bloomFilterType', None, "NONE", ), # 6 - (7, TType.I32, 'bloomFilterVectorSize', None, 0, ), # 7 - (8, TType.I32, 'bloomFilterNbHashes', None, 0, ), # 8 - (9, TType.BOOL, 'blockCacheEnabled', None, False, ), # 9 - (10, TType.I32, 'timeToLive', None, -1, ), # 10 + (5, TType.STRING, 'bloomFilterType', None, "NONE", ), # 5 + (6, TType.I32, 'bloomFilterVectorSize', None, 0, ), # 6 + (7, TType.I32, 'bloomFilterNbHashes', None, 0, ), # 7 + (8, TType.BOOL, 'blockCacheEnabled', None, False, ), # 8 + (9, TType.I32, 'timeToLive', None, -1, ), # 9 ) - def __init__(self, name=None, maxVersions=thrift_spec[2][4], compression=thrift_spec[3][4], inMemory=thrift_spec[4][4], maxValueLength=thrift_spec[5][4], bloomFilterType=thrift_spec[6][4], bloomFilterVectorSize=thrift_spec[7][4], bloomFilterNbHashes=thrift_spec[8][4], blockCacheEnabled=thrift_spec[9][4], timeToLive=thrift_spec[10][4],): + def __init__(self, name=None, maxVersions=thrift_spec[2][4], compression=thrift_spec[3][4], inMemory=thrift_spec[4][4], bloomFilterType=thrift_spec[5][4], bloomFilterVectorSize=thrift_spec[6][4], bloomFilterNbHashes=thrift_spec[7][4], blockCacheEnabled=thrift_spec[8][4], timeToLive=thrift_spec[9][4],): self.name = name self.maxVersions = maxVersions self.compression = compression self.inMemory = inMemory - self.maxValueLength = maxValueLength self.bloomFilterType = bloomFilterType self.bloomFilterVectorSize = bloomFilterVectorSize self.bloomFilterNbHashes = bloomFilterNbHashes @@ -162,31 +163,26 @@ def read(self, iprot): else: iprot.skip(ftype) elif fid == 5: - if ftype == TType.I32: - self.maxValueLength = iprot.readI32(); - else: - iprot.skip(ftype) - elif fid == 6: if ftype == TType.STRING: self.bloomFilterType = iprot.readString(); else: iprot.skip(ftype) - elif fid == 7: + elif fid == 6: if ftype == TType.I32: self.bloomFilterVectorSize = iprot.readI32(); else: iprot.skip(ftype) - elif fid == 8: + elif fid == 7: if ftype == TType.I32: self.bloomFilterNbHashes = iprot.readI32(); else: iprot.skip(ftype) - elif fid == 9: + elif fid == 8: if ftype == TType.BOOL: self.blockCacheEnabled = iprot.readBool(); else: iprot.skip(ftype) - elif fid == 10: + elif fid == 9: if ftype == TType.I32: self.timeToLive = iprot.readI32(); else: @@ -217,32 +213,31 @@ def write(self, oprot): oprot.writeFieldBegin('inMemory', TType.BOOL, 4) oprot.writeBool(self.inMemory) oprot.writeFieldEnd() - if self.maxValueLength != None: - oprot.writeFieldBegin('maxValueLength', TType.I32, 5) - oprot.writeI32(self.maxValueLength) - oprot.writeFieldEnd() if self.bloomFilterType != None: - oprot.writeFieldBegin('bloomFilterType', TType.STRING, 6) + oprot.writeFieldBegin('bloomFilterType', TType.STRING, 5) oprot.writeString(self.bloomFilterType) oprot.writeFieldEnd() if self.bloomFilterVectorSize != None: - oprot.writeFieldBegin('bloomFilterVectorSize', TType.I32, 7) + oprot.writeFieldBegin('bloomFilterVectorSize', TType.I32, 6) oprot.writeI32(self.bloomFilterVectorSize) oprot.writeFieldEnd() if self.bloomFilterNbHashes != None: - oprot.writeFieldBegin('bloomFilterNbHashes', TType.I32, 8) + oprot.writeFieldBegin('bloomFilterNbHashes', TType.I32, 7) oprot.writeI32(self.bloomFilterNbHashes) oprot.writeFieldEnd() if self.blockCacheEnabled != None: - oprot.writeFieldBegin('blockCacheEnabled', TType.BOOL, 9) + oprot.writeFieldBegin('blockCacheEnabled', TType.BOOL, 8) oprot.writeBool(self.blockCacheEnabled) oprot.writeFieldEnd() if self.timeToLive != None: - oprot.writeFieldBegin('timeToLive', TType.I32, 10) + oprot.writeFieldBegin('timeToLive', TType.I32, 9) oprot.writeI32(self.timeToLive) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -258,7 +253,7 @@ def __ne__(self, other): class TRegionInfo: """ A TRegionInfo contains information about an HTable region. - + Attributes: - startKey - endKey @@ -349,6 +344,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -364,7 +362,7 @@ def __ne__(self, other): class Mutation: """ A Mutation object is used to either update or delete a column-value. - + Attributes: - isDelete - column @@ -431,6 +429,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -446,7 +447,7 @@ def __ne__(self, other): class BatchMutation: """ A BatchMutation object is used to apply a number of Mutations to a single row. - + Attributes: - row - mutations @@ -510,6 +511,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -525,7 +529,7 @@ def __ne__(self, other): class TRowResult: """ Holds row name and then a map of columns to cells. - + Attributes: - row - columns @@ -591,6 +595,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __repr__(self): L = ['%s=%r' % (key, value) @@ -608,7 +615,7 @@ class IOError(Exception): An IOError exception signals that an error occurred communicating to the Hbase master or an Hbase region server. Also used to return more general Hbase error conditions. - + Attributes: - message """ @@ -651,6 +658,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __str__(self): return repr(self) @@ -670,7 +680,7 @@ class IllegalArgument(Exception): """ An IllegalArgument exception indicates an illegal or invalid argument was passed into a procedure. - + Attributes: - message """ @@ -713,6 +723,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __str__(self): return repr(self) @@ -732,7 +745,7 @@ class AlreadyExists(Exception): """ An AlreadyExists exceptions signals that a table with the specified name already exists - + Attributes: - message """ @@ -775,6 +788,9 @@ def write(self, oprot): oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() + def validate(self): + return + def __str__(self): return repr(self) @@ -789,4 +805,3 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - diff --git a/AppDB/helper_functions.py b/AppDB/helper_functions.py index 9e3b98e02b..ed837b05f6 100644 --- a/AppDB/helper_functions.py +++ b/AppDB/helper_functions.py @@ -1,32 +1,52 @@ +# Programmer: Navraj Chohan """ -Author: Navraj Chohan -Random functions/classes which are used by datastores +Helpful functions and classes which are used by AppDB """ +import hashlib import logging import logging.handlers +import inspect import os import os.path import random -import hashlib -""" -strings must be in unicode to reverse the string -strings are returned in unicode and may not able -able to be converted to a regular string -""" -def reverseLex(ustring): - newstr = u"" +import time + +def read_file(file_name): + """ Opens and reads a file. Helpful for mocking out builtin + functions. + Args: + file_name: path to file to read + Returns: + Contents of file + """ + fp = open(file_name, 'r') + contents = fp.read() + fp.close() + return contents + +def reverse_lex(ustring): + """ Strings must be in unicode to reverse the string + strings are returned in unicode and may not able + able to be converted to a regular string + + Args: + ustring: String to reverse + """ + newstr = "" for ii in ustring: ordinance = ord(ii) new_byte = 255 - ordinance - char = unichr(new_byte) + char = chr(new_byte) newstr += char return newstr -""" -Cetain datastores are unable to store keys with unichars of 128 or more -this function reflects on 127 and less. -""" -def reverseLex128(ustring): +def reverse_lex_128(ustring): + """ Certain datastores are unable to store keys with unichars of + 128 or more this function reflects on 127 and less. + + Args: + ustring: String to reverse + """ newstr = u"" for ii in ustring: ordinance = ord(ii) @@ -38,8 +58,8 @@ def reverseLex128(ustring): class ThreadedLogger(): def __init__(self, filename): split_path = os.path.split(filename) - dir = split_path[0] - if not os.path.exists(dir): os.mkdir(dir, 0777) + directory = split_path[0] + if not os.path.exists(directory): os.mkdir(directory, 0777) self.log_logger = logging.getLogger(filename) self.log_logger.setLevel(logging.INFO) self.formatter = logging.Formatter("%(asctime)s %(module)s:%(lineno)-4d %(message)s") @@ -55,7 +75,14 @@ def debug(self, string): if self.loggingOn: self.log_logger.info(string) -def randomString(length): +def random_string(length): + """ Returns a string of a given length. + + Args: + length: The length of the random string which is returned. + Returns: + A random string. + """ s = hashlib.sha256() ret = "a" while len(ret) < length: @@ -63,7 +90,12 @@ def randomString(length): ret += s.hexdigest() return ret[0:length] -import inspect def lineno(): - """Returns the current line number in our program.""" - return inspect.currentframe().f_back.f_lineno + """ Returns the current line number in our program. + + Returns: + The current line number in our program. + """ + return inspect.currentframe().f_back.f_lineno + + diff --git a/AppDB/hypertable/hypertable_interface.py b/AppDB/hypertable/hypertable_interface.py new file mode 100644 index 0000000000..6588da7a92 --- /dev/null +++ b/AppDB/hypertable/hypertable_interface.py @@ -0,0 +1,393 @@ +# Programmer: Navraj Chohan + +""" + Hypertable Interface for AppScale +""" +import os +import time + +import appscale_logger +import helper_functions + +import hyperthrift.gen.ttypes as ttypes + +from dbinterface_batch import * +from dbconstants import * +from hypertable import thriftclient + + +from xml.sax import make_parser +from xml.sax import parseString +from xml.sax.handler import feature_namespaces +from xml.sax import ContentHandler +from xml.sax import saxutils +from xml.sax.handler import ContentHandler + +# The port hypertable's thrift uses on the local machine +THRIFT_PORT = 38080 + +# AppScale default namespace for Hypertable +NS = "/appscale" + +# XML tags used for parsing Hypertable results +ROOT_TAG_BEGIN="" +ROOT_TAG_END="" +ACCGRP_TAG_BEGIN='' +ACCGRP_TAG_END="" +COLFMLY_TAG_BEGIN="" +COLFMLY_TAG_END="" +NAME_TAG_TEXT = "Name" +NAME_TAG_BEGIN="<"+NAME_TAG_TEXT+">" +NAME_TAG_END="" + + +class XmlSchemaParser(ContentHandler): + def __init__(self, tag_name): + self.tag_name = tag_name + self.isName = 0 + self.attributes = [] + + def clear_attributes(self): + self.attributes = [] + + def startElement(self, name, attrs): + if name == self.tag_name: + self.isName = 1 + + def endElement(self, name): + if name == self.tag_name: + self.isName = 0 + + def characters(self, ch): + if self.isName == 1: + self.attributes.append(ch) + +class DatastoreProxy(AppDBInterface): + """ Note: Hypertable will truncate any bytes after the terminating char + and hence requires encoding/decoding functions. Yet, the encoding and + decoding functions must keep lexigraphical ordering for range queries + to work properly. + """ + def __init__(self, logger = appscale_logger.getLogger("datastore-hypertable")): + """ Constructor + + Args: + logger: Object where log messages are sent + """ + self.logger = logger + self.host = helper_functions.read_file( + APPSCALE_HOME + '/.appscale/my_private_ip') + self.conn = thriftclient.ThriftClient(self.host, THRIFT_PORT) + self.ns = self.conn.namespace_open(NS) + + def batch_get_entity(self, table_name, row_keys, column_names): + """Allows access to multiple rows with a single call + + Args: + table_name: The table to access + row_keys: A list of keys to access + column_names: A list of columns to access + Raises: + TypeError: Bad argument types + Returns: + A dictionary of {key:{column_name:value,...}} + """ + + if not isinstance(table_name, str): raise TypeError("Expected str") + if not isinstance(column_names, list): raise TypeError("Expected list") + if not isinstance(row_keys, list): raise TypeError("Expected list") + + + row_keys = [self.__encode(row) for row in row_keys] + + ret = {} + row_intervals = [] + cell_intervals = None + include_deletes = False + for row in row_keys: + row_intervals.append(ttypes.RowInterval(row, True, row, True)) + + scan_spec = ttypes.ScanSpec(row_intervals, + cell_intervals, + include_deletes, + 1, 0, 0, + None, + column_names) + + res = self.conn.get_cells(self.ns, table_name, scan_spec) + for cell in res: + if self.__decode(cell.key.row) in ret: + # update the dictionary + col_dict = ret[self.__decode(cell.key.row)] + else: + # first time seen + col_dict = {} + col_dict[cell.key.column_family] = cell.value + ret[self.__decode(cell.key.row)] = col_dict + + # If nothing was returned for any cell, put in empty values + for row in row_keys: + if self.__decode(row) not in ret: + col_dict = {} + ret[self.__decode(row)] = col_dict + + return ret + + def batch_put_entity(self, table_name, row_keys, column_names, cell_values): + """Allows callers to store multiple rows with a single call. + + Args: + table_name: The table to mutate + row_keys: A list of keys to store on + column_names: A list of columns to mutate + cell_values: A dict of key/value pairs + Raises: + TypeError: Bad argument types + Returns: + Nothing + """ + + if not isinstance(table_name, str): raise TypeError("Expected str") + if not isinstance(column_names, list): raise TypeError("Expected list") + if not isinstance(row_keys, list): raise TypeError("Expected list") + if not isinstance(cell_values, dict): raise TypeError("Expected dict") + + __INSERT = 255 + cell_list = [] + + mutator = self.conn.mutator_open(self.ns, table_name, 0, 0) + + for key in row_keys: + for col in column_names: + cell = ttypes.Cell() + ttypekey = ttypes.Key(row=self.__encode(key), + column_family=col, + flag=__INSERT) + cell.key = ttypekey + cell.value = cell_values[key][col] + cell_list.append(cell) + + self.conn.mutator_set_cells(mutator, cell_list) + self.conn.mutator_close(mutator) + + def batch_delete(self, table_name, row_keys, column_names=[]): + """Remove a set of keys + + Args: + table_name: Table to delete rows from + row_keys: A list of keys to remove + column_names: Not used + Raises: + AppScaleDBConnectionError when unable to execute deletes + TypeError: Bad argument types + Returns: + Nothing + """ + + if not isinstance(table_name, str): raise TypeError("Expected str") + if not isinstance(row_keys, list): raise TypeError("Expected list") + + row_keys = [self.__encode(row) for row in row_keys] + __DELETE_ROW = 0 + cell_list = [] + + mutator = self.conn.mutator_open(self.ns, table_name, 0, 0) + + for key in row_keys: + cell = ttypes.Cell() + ttypekey = ttypes.Key(row=key, flag=__DELETE_ROW) + cell.key = ttypekey + cell_list.append(cell) + + self.conn.mutator_set_cells(mutator, cell_list) + self.conn.mutator_close(mutator) + + + def delete_table(self, table_name): + """ Drops a given column family + + Args: + table_name: The column family name + Raises: + TypeError: Bad argument types + Returns: + Nothing + """ + if not isinstance(table_name, str): raise TypeError("Expected str") + + self.conn.drop_table(self.ns, table_name, 1) + return + + + def create_table(self, table_name, column_names): + """ Creates a table as a column family + + Args: + table_name: The column family name + column_names: Not used + Raises: + TypeError: Bad argument types + Returns: + Nothing + """ + + if not isinstance(table_name, str): raise TypeError("Expected str") + if not isinstance(column_names, list): raise TypeError("Expected list") + + table_schema_xml = self.__construct_schema_xml(column_names) + self.conn.create_table(self.ns,table_name,table_schema_xml) + return + + def range_query(self, + table_name, + column_names, + start_key, + end_key, + limit, + offset=0, + start_inclusive=True, + end_inclusive=True, + keys_only=False): + """ Gets a dense range ordered by keys. Returns an ordered list of + a dictionary of [key:{column1:value1, column2:value2},...] + or a list of keys if keys only. + + Args: + table_name: Name of table to access + column_names: Columns which get returned within the key range + start_key: String for which the query starts at + end_key: String for which the query ends at + limit: Maximum number of results to return + offset: Number to cut off from the results [offset:] + start_inclusive: Boolean if results should include the start_key + end_inclusive: Boolean if results should include the end_key + keys_only: Boolean if to only keys and not values + Raises: + TypeError: Bad argument types + Return: + List of ordered results. + """ + if not isinstance(table_name, str): raise TypeError("Expected str") + if not isinstance(column_names, list): raise TypeError("Expected list") + if not isinstance(start_key, str): raise TypeError("Expected str") + if not isinstance(end_key, str): raise TypeError("Expected str") + if not isinstance(limit, int) and not isinstance(limit, long): + raise TypeError("Expected int or long") + if not isinstance(offset, int): raise TypeError("Expected int") + + start_key = self.__encode(start_key) + end_key = self.__encode(end_key) + + # We add two extra rows in case we exclude the start/end keys + # This makes sure the limit is upheld correctly, where we have + # to remove the first and last key + row_count = limit + if not start_inclusive: + row_count += 1 + if not end_inclusive: + row_count += 1 + + row_intervals = [] + row_intervals.append(ttypes.RowInterval(start_key, + start_inclusive, + end_key, + end_inclusive)) + + cell_intervals = None + include_deletes = False + + scan_spec = ttypes.ScanSpec(row_intervals, + cell_intervals, + include_deletes, + 1, # max revisions + row_count, + 0, + None, + column_names) + res = self.conn.get_cells(self.ns, table_name, scan_spec) + + results = [] + last_row = None + for cell in res: + # the current list element needs to be updated + if cell.key.row == last_row: + if not keys_only: + row_dict = results[-1] + col_dict = row_dict[self.__decode(cell.key.row)] + col_dict[cell.key.column_family] = cell.value + results[-1] = {self.__decode(cell.key.row):col_dict} + # add a new list element for this item + else: + last_row = cell.key.row + if keys_only: + results.append(self.__decode(cell.key.row)) + else: + col_dict = {} + col_dict[cell.key.column_family] = cell.value + results.append({self.__decode(cell.key.row):col_dict}) + + if not start_inclusive and len(results) > 0: + if start_key in results[0]: + results = results[1:] + + if not end_inclusive and len(results) > 0: + if end_key in results[-1]: + results = results[:-1] + + if len(results) > limit: + results = results[:limit] + + if offset != 0 and offset <= len(results): + results = results[offset:] + + return results + + ###################################################################### + # private methods + ###################################################################### + + def __construct_schema_xml(self, column_names): + """ For the column names of a table, this method returns + an xml string representing the columns, which can + then be used with hypertable's thrift api + Args: + column_names: A list of column names to construct xml + """ + + if not isinstance(column_names, list): raise TypeError("Expected list") + + schema_xml = ''.join([ROOT_TAG_BEGIN, ACCGRP_TAG_BEGIN]) + + for col_name in column_names: + schema_xml += ''.join([COLFMLY_TAG_BEGIN, + NAME_TAG_BEGIN, + col_name, + NAME_TAG_END, + COLFMLY_TAG_END]) + + schema_xml += ''.join([ACCGRP_TAG_END, ROOT_TAG_END]) + return schema_xml + + + def __encode(self, bytes_in): + """ Removes \x00 character with \x01 because hypertable truncates strings + with null chars. + + Args: + bytes_in: The string which will be encoded + Returns: + modified string with replaced chars + """ + return bytes_in.replace('\x00','\x01') + + def __decode(self, bytes_out): + """ Replaces \x01 character with \x00 because we swap out strings to + prevent truncating keys. + + Args: + bytes_out: The string which will be decoded + Returns: + Modified string with replaced chars + """ + return bytes_out.replace('\x01', '\x00') + diff --git a/AppDB/hypertable/hypertable_test.py b/AppDB/hypertable/hypertable_test.py deleted file mode 100644 index bb2ca38099..0000000000 --- a/AppDB/hypertable/hypertable_test.py +++ /dev/null @@ -1,47 +0,0 @@ -import py_hypertable -import string -import random -def GenPasswd2(length=8, chars=string.letters + string.digits): - return ''.join([random.choice(chars) for i in range(length)]) - -print py_hypertable.get_schema("APPS__") -print py_hypertable.get_schema("USERS__") -print "does apps table exist: (should)" -print py_hypertable.__table_exist("APPS__") -print "does qwert table exist: (should not)" -print py_hypertable.__table_exist("qwerty") - -table = "test_"+ GenPasswd2(10) -print "creating table " + table + " result and adding 2 rows:" -print py_hypertable.put_entity(table, "1", ["c1","c2", "c3"], ["a1","b2","c3"]) -print py_hypertable.put_entity(table, "2", ["c1","c2", "c3"], ["d4","e5","f6"]) -print "does this newly table exist:" -print py_hypertable.__table_exist(table) - -print "doing a get entity for row key 1:" -print py_hypertable.get_entity(table, "1", ["c1", "c2", "c3"]) -print "doing a get entity for row key 2:" -print py_hypertable.get_entity(table, "2", ["c1", "c2", "c3"]) -print "how many rows are in this table?" -print py_hypertable.get_row_count(table) -print "getting entire table:" -print py_hypertable.get_table(table, ["c1","c2","c3"]) - -print "what happens when trying to do a get on a table that doesnt exist:" -print py_hypertable.get_entity("qwerty", "1", ["a","b","c"]) - -print "query that table" -print py_hypertable.__query_table(table) - -print "delete row from table ",table -print py_hypertable.delete_row(table, "1") - -print "Doing a get on that which was deleted" -print py_hypertable.get_entity(table, "1", ["c1", "c2", "c3"]) -print "query the table" -print py_hypertable.__query_table(table) -print "doing a get entity:" -print py_hypertable.get_entity(table, "1", ["c1", "c2", "c3"]) -print "getting entire table:" -print py_hypertable.get_table(table, ["c1","c2","c3"]) - diff --git a/AppDB/hypertable/prime_hypertable.py b/AppDB/hypertable/prime_hypertable.py index 402d863ccb..8526ad9af5 100644 --- a/AppDB/hypertable/prime_hypertable.py +++ b/AppDB/hypertable/prime_hypertable.py @@ -7,6 +7,7 @@ import string import sys import py_hypertable +import hypertable_interface from dbconstants import * APPSCALE_HOME = os.environ.get("APPSCALE_HOME") @@ -16,6 +17,16 @@ print "APPSCALE_HOME env var not set" exit(1) +def create_app_tables(): + db = py_hypertable.DatastoreProxy() + db.create_table(ASC_PROPERTY_TABLE, PROPERTY_SCHEMA) + db.create_table(DSC_PROPERTY_TABLE, PROPERTY_SCHEMA) + db.create_table(APP_INDEX_TABLE, APP_INDEX_SCHEMA) + db.create_table(APP_NAMESPACE_TABLE, APP_NAMESPACE_SCHEMA) + db.create_table(APP_ID_TABLE, APP_ID_SCHEMA) + db.create_table(APP_ENTITY_TABLE, APP_ENTITY_SCHEMA) + db.create_table(APP_KIND_TABLE, APP_KIND_SCHEMA) + def prime_hypertable(): print "prime hypertable database" @@ -24,14 +35,16 @@ def prime_hypertable(): print client.create_table(USERS_TABLE,USERS_SCHEMA) print "Creating apps table" tables = client.create_table(APPS_TABLE,APPS_SCHEMA) - + + create_app_tables() + if USERS_TABLE in tables and APPS_TABLE in tables: print "CREATE TABLE SUCCESS FOR USER AND APPS" return 0 else: print "FAILED TO CREATE TABLE FOR USER AND APPS" return 1 + if __name__ == "__main__": sys.exit(prime_hypertable()) - diff --git a/AppDB/hypertable/py_hypertable.py b/AppDB/hypertable/py_hypertable.py index c0255790d1..b281fe2f4a 100644 --- a/AppDB/hypertable/py_hypertable.py +++ b/AppDB/hypertable/py_hypertable.py @@ -1,45 +1,51 @@ # Author: Navraj Chohan # Author: Kowshik Prakasam - -import sys +# TODO: deprecate this file and move towards updated DB interface import os +import string +import sys import time +import xml -import helper_functions from hypertable.thriftclient import * import hyperthrift.gen.ttypes as ttypes -#from hyperthrift.gen2 import * -import string -import cgi + from xml.sax import make_parser -import xml from xml.sax import parseString from xml.sax.handler import feature_namespaces from xml.sax import ContentHandler from xml.sax import saxutils from xml.sax.handler import ContentHandler -#import sqlalchemy.pool as pool -from dbinterface import * + import appscale_logger +from dbinterface import * +import helper_functions import threading THRIFT_PORT = 38080 + ERROR_HT = "DB_ERROR:" + DB_LOCATION = "localhost" + NS = "/appscale" ROOT_TAG_BEGIN="" + ROOT_TAG_END="" ACCGRP_TAG_BEGIN='' + ACCGRP_TAG_END="" COLFMLY_TAG_BEGIN="" + COLFMLY_TAG_END="" NAME_TAG_TEXT = "Name" NAME_TAG_BEGIN="<"+NAME_TAG_TEXT+">" + NAME_TAG_END="" PROFILING = False @@ -54,8 +60,6 @@ def debug(self, string): self.ht_logger.info(string) self.ht_lock.release() -#ht_logger = HTLogger(log_logger) - class XmlSchemaParser(ContentHandler): def __init__(self, tag_name): self.tag_name = tag_name @@ -78,12 +82,8 @@ def characters(self, ch): self.attributes.append(ch) class DatastoreProxy(AppDBInterface): - def __init__(self, logger = appscale_logger.getLogger("datastore-hypertable")): self.logger = logger - #self.pool = pool.QueuePool(self.__createConnection) - #self.conn = ThriftClient(self.get_local_ip(), THRIFT_PORT) - #self.ns = self.conn.open_namespace(NS) self.conn = None self.tableCache = [] self.lock = threading.Lock() @@ -99,22 +99,15 @@ def __initConnection(self, create_ns=False): print "Unable to create namepsace" print e self.ns = self.conn.namespace_open(NS) - # self.ns = self.conn.open_namespace(NS) - #if PROFILING: - # self.logger.debug("HT InitConnection: %s"%str(endtime - starttime)) return self.conn def __closeConnection(self, conn): - #if conn: self.lock.release() - #conn.close_namespace(self.ns) - #conn.close() # tag is the xml tag which holds the schema attributes def getListFromXMLSchema(self, table, tag): parser = make_parser() - #parser = setFeature(feature_namespaces, 0) dh = XmlSchemaParser(tag) dh.clear_attributes() parser.setContentHandler(dh) diff --git a/AppDB/hypertable/test.py b/AppDB/hypertable/test.py deleted file mode 100644 index d842f968de..0000000000 --- a/AppDB/hypertable/test.py +++ /dev/null @@ -1,23 +0,0 @@ -import sys -import os - -APPSCALE_HOME = os.environ.get("APPSCALE_HOME") -if APPSCALE_HOME: - pass -else: - print "APPSCALE_HOME env var not set" - exit(1) - -sys.path.append(APPSCALE_HOME + "AppDB/hypertable") -import py_hypertable -import xml_parser -import string, cgi -USER_TABLE = "USERS__" -HYPERTABLE_XML_TAG = "Name" - - -schema = xml_parser.getListFromXMLSchema(USER_TABLE, HYPERTABLE_XML_TAG) -print schema -#table = USER_TABLE -#schema = py_hypertable.get_schema(table) -#print schema diff --git a/AppDB/hypertable/test_hypertable.py b/AppDB/hypertable/test_hypertable.py deleted file mode 100644 index b00e60635b..0000000000 --- a/AppDB/hypertable/test_hypertable.py +++ /dev/null @@ -1,302 +0,0 @@ -import py_hypertable -import sys -import helper_functions -hf = helper_functions -columns = ["a","b","c"] -data = ["1"*300,"2"*300,"3"*300] -print data -import time -table_name = "hello" -key = "1" -print "key= " + key -print "columns= " + str(columns) -print "data= " + str(data) -print "table= " + table_name - -print py_hypertable.put_entity(table_name, key, columns, data) -ret = py_hypertable.get_entity(table_name, key, columns) -print "doing a put then get" -print ret -if ret[1:] != data: - print "ERROR doing a put then get. Data does not match" - print "returned: " + str(ret) - print "expected: " + str(data) - exit(1) -else: - print "Success" - -ret = py_hypertable.get_schema("hello") -print ret -print "checking schema:" -print ret -if ret[1:] != columns: - print "ERROR in recieved schema" - print "returned: " + str(ret) - print "expected: " + str(columns) - -ret = py_hypertable.__table_exist(table_name) -print "Does table we just created exist?" -print ret - -ret = py_hypertable.delete_row(table_name, key) -print "Deleting the key %s"%key -print ret - -ret = py_hypertable.get_entity(table_name, key, columns) -print "Trying to get deleted key:" -print ret -print "doing a put with key %s"%key -print py_hypertable.put_entity("hello", "1", ["a","b","c"], ["1","2","3"]) -print "doing a get table" -print py_hypertable.get_table("hello", ["a","b","c"]) -py_hypertable.put_entity("hello", "2", ["a","b","c"], ["4","5","6"]) -print "doing get table:" -print py_hypertable.get_table("hello", ["a","b","c"]) -py_hypertable.put_entity("hello", "3", ["a","b","c"], ["1","2","3"]) -py_hypertable.get_table("hello", ["a","b","c"]) - -print "TRYING TO REPLACE KEY 3" -py_hypertable.put_entity("hello", "3", ["a","b","c"], ["1","2","3"]) -py_hypertable.get_table("hello", ["a","b","c"]) -py_hypertable.get_row_count("hello") -ret = py_hypertable.delete_row("hello", "1") -ret = py_hypertable.delete_row("hello", "2") -ret = py_hypertable.delete_row("hello", "3") -py_hypertable.get_table("hello", ["a","b","c"]) -print "Deleting table:" -print py_hypertable.delete_table("hello") -print "deleting twice:" -print py_hypertable.delete_table("hello") - -table_name = u"testing_query" -print py_hypertable.delete_table(table_name) -column_names = [u"c1"] -limit = 1000 -offset = 0 -key = 0 -startrow = u"000" -endrow = u"100" -data = u"xxx" -totalstarttime = time.time() -for ii in range(0, 101): - key = str(ii) - key = ("0" * (3 - len(key))) + key - key = unicode(key) - print "Adding key " + key - start = time.time() - print py_hypertable.put_entity(table_name, key, column_names, [data + key]) - stop = time.time() - - print "For inserting 1 record: start time: %s, end time: %s, total time: %s"%(str(start), str(stop), str(stop - start)) -totalstoptime = time.time() -print "For inserting 100 records: start time: %s, end time: %s, total time: %s"%(str(totalstarttime), str(totalstoptime), str(totalstoptime - totalstarttime)) -inclusive = 1 -notJustKeys = 0 -print "from table %s get columns %s with limits %s and offset %s, with start row %s and endrow %s"%(table_name, column_names, limit, offset, startrow, endrow) - -results = py_hypertable.run_query(table_name, column_names, limit, offset, startrow, endrow, notJustKeys, inclusive, inclusive) -results = results[1:] -if len(results) != 101: - print "ERORR: AAA Bad number of cells returned with no limit query. 101 versus %d"%len(results) - -limit = 50 -results = py_hypertable.run_query(table_name, column_names, limit, offset, startrow, endrow, notJustKeys, inclusive, inclusive) -results = results[1:] -if len(results) != 50: - print "ERORR: BBB Bad number of cells returned with 50 limit query. 50 versus %d"%len(results) -limit = 25 -offset = 25 -results = py_hypertable.run_query(table_name, column_names, limit, offset, startrow, endrow, notJustKeys, inclusive, inclusive) -results = results[1:] -if len(results) != 25: - print "ERORR: CCC Bad number of cells returned with 50 limit query. 50 versus %d"%len(results) -first_key = u"xxx049" -print "Number of results:" -print len(results) -if results[24] != first_key: - print "ERORR: DDD Bad first key returned for 25 limit query with 25 offset. %s vs %s"%(first_key, results[24]) - -getOnlyKeys = 1 -results = py_hypertable.run_query(table_name, column_names, limit, offset, startrow, endrow, getOnlyKeys, inclusive, inclusive) - -#print results -startrow = "001" -endrow = "003" -limit = 1000 -offset = 0 -exclusive = 0 -results = py_hypertable.run_query(table_name, column_names, limit, offset, startrow, endrow, getOnlyKeys, exclusive, exclusive) -results = results[1:] -if len(results) != 1: - print "ERORR: EEE Bad number of cells returned with 0 limit query. 1 versus %d"%len(results) -last_key = u"002" -print "Number of results:" -print len(results) -if results[len(results) - 1] != last_key: - print "ERORR: FFF Bad first key returned for 0 limit query with 0 offset. %s vs %s"%(last_key, results[len(results) - 1]) - -NUM_COLUMNS = 10 -def err(test_num, code): - print "Failed for test at " + sys.argv[0] + ":" + str(test_num) \ - + " with a return of: " + str(code) - exit(1) - -def createRandomList(number_of_columns, column_name_len): - columns = [] - for ii in range(0, number_of_columns): - columns += [ "a" + hf.randomString(column_name_len)] - return columns - -columns = createRandomList(NUM_COLUMNS, 10) -data = createRandomList(NUM_COLUMNS, 100) -table_name = "a" + hf.randomString(10) -key = hf.randomString(10) -print "key= " + key -#print "columns= " + str(columns) -#print "data= " + str(data) -print "table= " + table_name -app_datastore = py_hypertable -#app_datastore = appscale_datastore.Datastore(datastore_type) -ERROR_CODES = ["HT_ERROR:"] -#################### -# Put on a new table -#################### -print columns -ret = app_datastore.put_entity(table_name, key, columns, data) -if ret[0] not in ERROR_CODES or ret[1] != "0": - err(hf.lineno(),ret) -#################### -# Get on all columns -#################### -print columns -ret = app_datastore.get_entity(table_name, key, columns) -if ret[0] not in ERROR_CODES or ret[1:] != data: - err(hf.lineno(),ret) -########################################### -# Get on a random column and check the data -########################################### -import random -for ii in range(0, NUM_COLUMNS): - rand = int((random.random() * 1000 ) % NUM_COLUMNS) - ret = app_datastore.get_entity(table_name, key, [columns[rand]]) - if ret[0] not in ERROR_CODES or ret[1] != data[rand]: - err(hf.lineno(),ret) -################################################# -# Get random data from two columns and check data -################################################# -for ii in range(0, NUM_COLUMNS): - rand = int((random.random() * 1000) % NUM_COLUMNS) - rand2 = int((random.random() * 1000) % NUM_COLUMNS) - ret = app_datastore.get_entity(table_name, key, \ - [columns[rand],columns[rand2]]) - if ret[0] not in ERROR_CODES or ret[1] != data[rand] \ - or ret[2] != data[rand2]: - err(hf.lineno(),ret) -##################################### -# Get and a delete on invalid row key -##################################### -invalid_key = "a" + hf.randomString(10) -ret = app_datastore.get_entity(table_name, invalid_key, \ - columns) -if ret[0] in ERROR_CODES: - err(hf.lineno(),ret) -ret = app_datastore.delete_row(table_name, invalid_key) -if ret[0] not in ERROR_CODES: - err(hf.lineno(),ret) -########################### -# Get just the first column -########################### -ret = app_datastore.get_entity(table_name, key, [columns[0]]) -if ret[0] not in ERROR_CODES or ret[1] != data[0]: - print ret - err(hf.lineno(),ret) -########################### -# Put on new row -########################### -data2 = createRandomList(NUM_COLUMNS, 100) -key2 = hf.randomString(10) -ret = app_datastore.put_entity(table_name, key2, columns, data2) -if ret[0] not in ERROR_CODES or ret[1] != "0": - err(hf.lineno(),ret) -############################ -# Get on just added row -############################ -ret = app_datastore.get_entity(table_name, key2, columns) -if ret[0] not in ERROR_CODES or ret[1:] != data2 or ret[1:] == data: - err(hf.lineno(),ret) -######################################### -# Delete the new row once, and then again -######################################### -ret = app_datastore.delete_row(table_name, key2) -if ret[0] not in ERROR_CODES: - err(hf.lineno(),ret) -ret = app_datastore.delete_row(table_name, key2) -if ret[0] not in ERROR_CODES: - err(hf.lineno(),ret) -################################################# -# Get and a delete on a table that does not exist -################################################# -invalid_table = hf.randomString(10) -ret = app_datastore.delete_row(invalid_table, key) -if ret[0] in ERROR_CODES: - err(hf.lineno(), ret) -ret = app_datastore.get_entity(invalid_table, key, columns) -if ret[0] in ERROR_CODES: - err(hf.lineno(), ret) -###################### -# Delete a table twice -###################### -ret = app_datastore.delete_table(table_name) -if ret[0] not in ERROR_CODES: - err(hf.lineno(), ret) -ret = app_datastore.delete_table(table_name) -if ret[0] in ERROR_CODES: - err(hf.lineno(), ret) -##################### -# Put on a same table -##################### -ret = app_datastore.put_entity(table_name, key, columns, data) -if ret[0] not in ERROR_CODES or ret[1] != "0": - err(hf.lineno(),ret, ret) -#################### -# Get on all columns -#################### -ret = app_datastore.get_entity(table_name, key, columns) -if ret[0] not in ERROR_CODES or ret[1:] != data: - err(hf.lineno(),ret) -########################## -# Put on same row new data -########################## -data = createRandomList(NUM_COLUMNS, 10000) -ret = app_datastore.put_entity(table_name, key, columns, data) -if ret[0] not in ERROR_CODES or ret[1] != "0": - err(hf.lineno(),ret, ret) -#################### -# Get on all columns -#################### -ret = app_datastore.get_entity(table_name, key, columns) -if ret[0] not in ERROR_CODES or ret[1:] != data: - err(hf.lineno(),ret) -#################################### -# Do a put on first and last columns -#################################### -data1 = hf.randomString(10) -data2 = hf.randomString(10) -ret = app_datastore.put_entity(table_name, key, [columns[0], \ -columns[NUM_COLUMNS - 1]], [data1, data2]) -if ret[0] not in ERROR_CODES or ret[1] != "0": - err(hf.lineno(), ret) -ret = app_datastore.get_entity(table_name, key, [columns[0], \ -columns[NUM_COLUMNS - 1]]) -if ret[0] not in ERROR_CODES or ret[1] != data1 or ret[2] != data2: - err(hf.lineno(), ret) -####################################################### -# Get schema on a table that exist, and one that doesnt -####################################################### -ret = app_datastore.get_schema(table_name) -if ret[0] not in ERROR_CODES or ret[1:] != columns: - err(hf.lineno(), ret) -ret = app_datastore.get_schema(invalid_table) -if ret[0] in ERROR_CODES: - err(hf.lineno(), ret) -print "SUCCESS" diff --git a/AppDB/hypertable/test_hypertable2.py b/AppDB/hypertable/test_hypertable2.py deleted file mode 100644 index 6ad758bac5..0000000000 --- a/AppDB/hypertable/test_hypertable2.py +++ /dev/null @@ -1,66 +0,0 @@ -import py_hypertable -py_hypertable = py_hypertable.DatastoreProxy() -columns = ["a","b","c"] -data = ["1","2","3"] -table_name = "hello" -key = "1" -print "key= " + key -print "columns= " + str(columns) -print "data= " + str(data) -print "table= " + table_name -print "PUT" -print py_hypertable.put_entity(table_name, key, columns, data) -print "GET" -ret = py_hypertable.get_entity(table_name, key, columns) -print "doing a put then get" -print ret -if ret[1:] != data: - print "ERROR doing a put then get. Data does not match" - print "returned: " + str(ret) - print "expected: " + str(data) - exit(1) -else: - print "Success" - -ret = py_hypertable.get_schema("hello") -print ret -print "checking schema:" -print ret -if ret[1:] != columns: - print "ERROR in recieved schema" - print "returned: " + str(ret) - print "expected: " + str(columns) - -#ret = py_hypertable.__table_exist(table_name) -#print "Does table we just created exist?" -#print ret - -ret = py_hypertable.delete_row(table_name, key) -print "Deleting the key %s"%key -print ret - -ret = py_hypertable.get_entity(table_name, key, columns) -print "Trying to get deleted key:" -print ret -print "doing a put with key %s"%key -print py_hypertable.put_entity("hello", "1", ["a","b","c"], ["1","2","3"]) -print "doing a get table" -print py_hypertable.get_table("hello", ["a","b","c"]) -py_hypertable.put_entity("hello", "2", ["a","b","c"], ["4","5","6"]) -print "doing get table:" -print py_hypertable.get_table("hello", ["a","b","c"]) -py_hypertable.put_entity("hello", "3", ["a","b","c"], ["1","2","3"]) -py_hypertable.get_table("hello", ["a","b","c"]) - -print "TRYING TO REPLACE KEY 3" -py_hypertable.put_entity("hello", "3", ["a","b","c"], ["1","2","3"]) -py_hypertable.get_table("hello", ["a","b","c"]) -py_hypertable.get_row_count("hello") -ret = py_hypertable.delete_row("hello", "1") -ret = py_hypertable.delete_row("hello", "2") -ret = py_hypertable.delete_row("hello", "3") -py_hypertable.get_table("hello", ["a","b","c"]) -print "Deleting table:" -print py_hypertable.delete_table("hello") -print "deleting twice:" -print py_hypertable.delete_table("hello") diff --git a/AppDB/test/bigbinary b/AppDB/test/bigbinary new file mode 100644 index 0000000000..6c5effe7ef Binary files /dev/null and b/AppDB/test/bigbinary differ diff --git a/AppDB/test/functional/test_cassandra.py b/AppDB/test/functional/test_cassandra.py new file mode 100644 index 0000000000..f2a855c589 --- /dev/null +++ b/AppDB/test/functional/test_cassandra.py @@ -0,0 +1,197 @@ +import os +import sys +import unittest + +from dbconstants import * + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../cassandra")) +import cassandra_interface + +# Prereq: Cassandra must be running first +TEST1_TABLE = "TestTable1" +TEST1_TABLE_SCHEMA = ['c1','c2','c3'] + +TEST2_TABLE = "TestTable2" +TEST2_TABLE_SCHEMA = ['c1','c2','c3'] + +TEST3_TABLE = "TestTable3" +TEST3_TABLE_SCHEMA = ['c1','c2','c3'] + +TEST4_TABLE = "TestTable4" +TEST4_TABLE_SCHEMA = ['c1','c2','c3'] + +TEST5_TABLE = "TestTable5" +TEST5_TABLE_SCHEMA = ['c1','c2','c3'] + + +class PutTestCase(unittest.TestCase): + def setUp(self): + self.cass = cassandra_interface.DatastoreProxy() + self.cass.create_table(TEST1_TABLE, TEST1_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c'] + cell_values = {'a':{'c1':'1','c2':'2','c3':'3'}, + 'b':{'c1':'4','c2':'5','c3':'6'}, + 'c':{'c1':'7','c2':'8','c3':'9'}} + self.cass.batch_put_entity(TEST1_TABLE, row_key, TEST1_TABLE_SCHEMA, + cell_values) + assert self.cass.batch_get_entity(TEST1_TABLE, row_key, TEST1_TABLE_SCHEMA) == cell_values + + def tearDown(self): + self.cass.delete_table(TEST1_TABLE) + +class DeleteTestCase(unittest.TestCase): + def setUp(self): + self.cass = cassandra_interface.DatastoreProxy() + self.cass.create_table(TEST2_TABLE, TEST2_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c'] + cell_values = {'a':{'c1':'1','c2':'2','c3':'3'}, + 'b':{'c1':'4','c2':'5','c3':'6'}, + 'c':{'c1':'7','c2':'8','c3':'9'}} + self.cass.batch_put_entity(TEST2_TABLE, row_key, TEST2_TABLE_SCHEMA, + cell_values) + + assert self.cass.batch_get_entity(TEST2_TABLE, + row_key, + TEST2_TABLE_SCHEMA) == cell_values + self.cass.batch_delete(TEST2_TABLE, ['a','b']) + assert self.cass.batch_get_entity(TEST2_TABLE, + row_key, + TEST2_TABLE_SCHEMA) == \ + {'a':{}, 'b':{}, 'c':{'c1':'7','c2':'8','c3':'9'}} + + def tearDown(self): + self.cass.delete_table(TEST2_TABLE) + +class GetOnNonExistantKey(unittest.TestCase): + def setUp(self): + self.cass = cassandra_interface.DatastoreProxy() + self.cass.create_table(TEST3_TABLE, TEST3_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c'] + cell_values = {'a':{}, 'b':{}, 'c':{}} + ret = self.cass.batch_get_entity(TEST3_TABLE, row_key, TEST3_TABLE_SCHEMA) + assert ret == cell_values + + def tearDown(self): + self.cass.delete_table(TEST3_TABLE) + +class PutOverwriteTestCase(unittest.TestCase): + def setUp(self): + self.cass = cassandra_interface.DatastoreProxy() + self.cass.create_table(TEST4_TABLE, TEST4_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c'] + cell_values = {'a':{'c1':'1','c2':'2','c3':'3'}, + 'b':{'c1':'4','c2':'5','c3':'6'}, + 'c':{'c1':'7','c2':'8','c3':'9'}} + self.cass.batch_put_entity(TEST4_TABLE, row_key, TEST4_TABLE_SCHEMA, + cell_values) + cell_values = {'a':{'c1':'10','c2':'20','c3':'30'}, + 'b':{'c1':'40','c2':'50','c3':'60'}, + 'c':{'c1':'70','c2':'80','c3':'90'}} + self.cass.batch_put_entity(TEST4_TABLE, row_key, TEST4_TABLE_SCHEMA, + cell_values) + assert self.cass.batch_get_entity(TEST4_TABLE, row_key, TEST4_TABLE_SCHEMA) == cell_values + + def tearDown(self): + self.cass.delete_table(TEST4_TABLE) + +class RangeTestCase(unittest.TestCase): + def setUp(self): + self.cass = cassandra_interface.DatastoreProxy() + self.cass.create_table(TEST5_TABLE, TEST5_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c','d','e','f'] + cell_values = {'a':{'c1':'1','c2':'2','c3':'3'}, + 'b':{'c1':'4','c2':'5','c3':'6'}, + 'c':{'c1':'7','c2':'8','c3':'9'}, + 'd':{'c1':'10','c2':'11','c3':'12'}, + 'e':{'c1':'13','c2':'14','c3':'15'}, + 'f':{'c1':'16','c2':'17','c3':'18'}} + expected = [{'a':{'c1':'1','c2':'2','c3':'3'}}, + {'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}, + {'d':{'c1':'10','c2':'11','c3':'12'}}, + {'e':{'c1':'13','c2':'14','c3':'15'}}, + {'f':{'c1':'16','c2':'17','c3':'18'}}] + + self.cass.batch_put_entity(TEST5_TABLE, row_key, TEST5_TABLE_SCHEMA, + cell_values) + column_names = TEST5_TABLE_SCHEMA + limit = 10 + offset = 0 + startrow = 'a' + endrow = 'f' + start_in = True + end_in = True + assert self.cass.range_query(TEST5_TABLE, column_names, + startrow, endrow, + limit, offset, start_in, end_in) == expected + + keys_only = True + expected = ['a','b','c','d','e','f'] + assert self.cass.range_query(TEST5_TABLE, column_names, + startrow, endrow, + limit, offset, start_in, + end_in, keys_only) == expected + + keys_only = False + start_in = False + expected = [{'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}, + {'d':{'c1':'10','c2':'11','c3':'12'}}, + {'e':{'c1':'13','c2':'14','c3':'15'}}, + {'f':{'c1':'16','c2':'17','c3':'18'}}] + assert self.cass.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, + offset, start_in, end_in) == expected + + expected = [{'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}, + {'d':{'c1':'10','c2':'11','c3':'12'}}, + {'e':{'c1':'13','c2':'14','c3':'15'}}] + + end_in = False + assert self.cass.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, + offset, start_in, end_in) == expected + + expected = [{'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}, + {'d':{'c1':'10','c2':'11','c3':'12'}}, + {'e':{'c1':'13','c2':'14','c3':'15'}}] + start_in = True + end_in = True + startrow = 'b' + endrow = 'e' + assert self.cass.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, offset, + start_in, end_in) == expected + + expected = [{'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}] + limit = 2 + assert self.cass.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, + offset, start_in, end_in) == expected + + expected = [{'c':{'c1':'7','c2':'8','c3':'9'}}] + offset = 1 + assert self.cass.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, + offset, start_in, end_in) == expected + + def tearDown(self): + self.cass.delete_table(TEST5_TABLE) + + +if __name__ == "__main__": + unittest.main() + diff --git a/AppDB/test/test_datastore.py b/AppDB/test/functional/test_datastore.py similarity index 100% rename from AppDB/test/test_datastore.py rename to AppDB/test/functional/test_datastore.py diff --git a/AppDB/test/test_datastore_memcachedb.py b/AppDB/test/functional/test_datastore_memcachedb.py similarity index 100% rename from AppDB/test/test_datastore_memcachedb.py rename to AppDB/test/functional/test_datastore_memcachedb.py diff --git a/AppDB/test/functional/test_datastore_server.py b/AppDB/test/functional/test_datastore_server.py new file mode 100644 index 0000000000..30980408e7 --- /dev/null +++ b/AppDB/test/functional/test_datastore_server.py @@ -0,0 +1,493 @@ +# Programmer: Navraj Chohan +import os +import time +import unittest + +import appscale_datastore_batch +import datastore_server + +from dbconstants import * + +from google.appengine.api import datastore +from google.appengine.api import datastore_types +from google.appengine.datastore import datastore_index +from google.appengine.datastore import entity_pb +from google.appengine.datastore import datastore_query + +# The separator of the namespace +NAMESPACE_SEP = '/' + +# Default DB to use +DB = "hbase" + +class ValidateIDCase(unittest.TestCase): + def setUp(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) + def runTest(self): + self.app_datastore.ValidateAppId("hi") + def tearDown(self): + pass + +class GetIndexKeyCase(unittest.TestCase): + def setUp(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) + def runTest(self): + assert self.app_datastore.GetIndexKey("hi","bye","nye","guy") \ + == "hi/bye/nye/guy" + def tearDown(self): + pass + +class GetPrefixCase(unittest.TestCase): + def setUp(self): + global datastore_batch + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) + def runTest(self): + assert self.app_datastore.GetTablePrefix(('hi1','bye1')) == "hi1/bye1" + def tearDown(self): + key = self.app_datastore.GetTablePrefix(("hi1", "bye1")) + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + datastore_batch.batch_delete(APP_NAMESPACE_TABLE, [key], column_names=APP_NAMESPACE_SCHEMA) + + +class ConfigureNamespaceCase(unittest.TestCase): + def setUp(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) + def runTest(self): + assert self.app_datastore.GetTablePrefix(("hi","bye")) == "hi/bye" + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + assert datastore_batch.batch_get_entity(APP_NAMESPACE_TABLE, ["hi/bye"], + ['namespaces']) == {'hi/bye':{'namespaces':'bye'}} + def tearDown(self): + key = self.app_datastore.GetTablePrefix(("hi", "bye")) + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + datastore_batch.batch_delete(APP_NAMESPACE_TABLE, [key], APP_NAMESPACE_SCHEMA) + assert datastore_batch.batch_get_entity(APP_NAMESPACE_TABLE, ["hi/bye"], + ['namespaces']) == {'hi/bye':{}} + +class InsertEntityCase(unittest.TestCase): + def setUp(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) + def runTest(self): + entities = [] + for ii in range(0,10): + entity = datastore.Entity("TestKind", + _app="test", + name=str(ii), + namespace='') + # have properties with different values bye same property names + entity.update({'aaa': "1111_" + str(ii), + 'bbb': "2222", + 'ccc': "3"*ii}) + entities.append(entity.ToPb()) + + self.keys = ['test//TestKind:0!', + 'test//TestKind:1!', + 'test//TestKind:2!', + 'test//TestKind:3!', + 'test//TestKind:4!', + 'test//TestKind:5!', + 'test//TestKind:6!', + 'test//TestKind:7!', + 'test//TestKind:8!', + 'test//TestKind:9!'] + + self.app_datastore.InsertEntities(entities) + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + # Verify an entity has been stored + ret = datastore_batch.batch_get_entity(APP_ENTITY_TABLE, self.keys, + APP_ENTITY_SCHEMA) + assert 'aaa' in ret['test//TestKind:0!']['entity'] + assert 'bbb' in ret['test//TestKind:0!']['entity'] + assert 'ccc' in ret['test//TestKind:0!']['entity'] + assert 'aaa' in ret['test//TestKind:9!']['entity'] + assert 'bbb' in ret['test//TestKind:9!']['entity'] + assert 'ccc' in ret['test//TestKind:9!']['entity'] + + + def tearDown(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + datastore_batch.batch_delete(APP_ENTITY_TABLE, self.keys, APP_ENTITY_SCHEMA) + datastore_batch.batch_delete(APP_KIND_TABLE, self.keys, APP_KIND_SCHEMA) + # Verify an entity has been deleted + ret = datastore_batch.batch_get_entity(APP_ENTITY_TABLE, self.keys, + APP_ENTITY_SCHEMA) + assert 'entity' not in ret['test//TestKind:0!'] + +class InsertEntityGroupCase(unittest.TestCase): + def setUp(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) + self.entities = [] + prev = None + for ii in range(0,4): + entity = datastore.Entity("TestKind", + _app="test", + name=str(ii), + namespace='b', + parent = prev) + prev = entity + # have properties with different values bye same property names + entity.update({'aaa': "1111_" + str(ii), + 'bbb': "2222", + 'ccc': "3"*ii}) + self.entities.append(entity.ToPb()) + self.keys = ['test/b/TestKind:0!', + 'test/b/TestKind:0!TestKind:1!', + 'test/b/TestKind:0!TestKind:1!TestKind:2!', + 'test/b/TestKind:0!TestKind:1!TestKind:2!TestKind:3!'] + self.kkeys = ['test/b/TestKind:0!', + 'test/b/TestKind:1!TestKind:0!', + 'test/b/TestKind:2!TestKind:1!TestKind:0!', + 'test/b/TestKind:3!TestKind:2!TestKind:1!TestKind:0!'] + def runTest(self): + self.app_datastore.InsertEntities(self.entities) + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + ret = datastore_batch.batch_get_entity(APP_ENTITY_TABLE, self.keys, + APP_ENTITY_SCHEMA) + + assert 'aaa' in ret['test/b/TestKind:0!']['entity'] + assert 'bbb' in ret['test/b/TestKind:0!']['entity'] + assert 'ccc' in ret['test/b/TestKind:0!']['entity'] + assert 'aaa' in ret['test/b/TestKind:0!TestKind:1!']['entity'] + assert 'bbb' in ret['test/b/TestKind:0!TestKind:1!']['entity'] + assert 'ccc' in ret['test/b/TestKind:0!TestKind:1!']['entity'] + + def tearDown(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + ret = datastore_batch.batch_delete(APP_ENTITY_TABLE, self.keys, APP_ENTITY_SCHEMA) + ret = datastore_batch.batch_delete(APP_KIND_TABLE, self.kkeys, APP_KIND_SCHEMA) + # Verify an entity has been deleted + ret = datastore_batch.batch_get_entity(APP_ENTITY_TABLE, self.keys, + APP_ENTITY_SCHEMA) + assert 'entity' not in ret['test/b/TestKind:0!'] + +class InsertEntityIndexCase(unittest.TestCase): + def setUp(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) + self.entities = [] + for ii in range(0,3): + entity = datastore.Entity("TestKind", + _app="test", + name=str(ii), + namespace='a') + # have properties with different values bye same property names + entity.update({'aaa': "1111_" + str(ii), 'bbb': "2222"}) + + self.entities.append(entity.ToPb()) + + self.keys = ['test/a/TestKind/aaa/1111_2\x00/TestKind:2!', + 'test/a/TestKind/bbb/2222\x00/TestKind:2!', + 'test/a/TestKind/aaa/1111_0\x00/TestKind:0!', + 'test/a/TestKind/bbb/2222\x00/TestKind:0!', + 'test/a/TestKind/aaa/1111_1\x00/TestKind:1!', + 'test/a/TestKind/bbb/2222\x00/TestKind:1!'] + self.rkeys = ['test/a/TestKind/aaa/\xce\xce\xce\xce\xa0\xcd\xff/TestKind:2!', + 'test/a/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:2!', + 'test/a/TestKind/aaa/\xce\xce\xce\xce\xa0\xcc\xff/TestKind:3', + 'test/a/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:3!', + 'test/a/TestKind/aaa/\xce\xce\xce\xce\xa0\xcf\xff/TestKind:0!', + 'test/a/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!', + 'test/a/TestKind/aaa/\xce\xce\xce\xce\xa0\xce\xff/TestKind:1!', + 'test/a/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:1!'] + + def runTest(self): + self.app_datastore.InsertIndexEntries(self.entities) + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + # Verify an entity has been stored + ret = datastore_batch.batch_get_entity(ASC_PROPERTY_TABLE, self.keys, + PROPERTY_SCHEMA) + assert 'test/a/TestKind:2!' in \ + ret['test/a/TestKind/aaa/1111_2\x00/TestKind:2!']['reference'] + + ret = datastore_batch.batch_get_entity(DSC_PROPERTY_TABLE, self.rkeys, + PROPERTY_SCHEMA) + assert 'test/a/TestKind:2!' in \ + ret['test/a/TestKind/aaa/\xce\xce\xce\xce\xa0\xcd\xff/TestKind:2!']\ + ['reference'] + + def tearDown(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + ret = datastore_batch.batch_delete(ASC_PROPERTY_TABLE, self.keys, PROPERTY_SCHEMA) + ret = datastore_batch.batch_delete(DSC_PROPERTY_TABLE, self.rkeys, PROPERTY_SCHEMA) + # Verify an entity has been deleted + ret = datastore_batch.batch_get_entity(ASC_PROPERTY_TABLE, self.keys, + PROPERTY_SCHEMA) + assert 'reference' not in \ + ret['test/a/TestKind/aaa/1111_2\x00/TestKind:2!'] + + ret = datastore_batch.batch_get_entity(DSC_PROPERTY_TABLE, self.rkeys, + PROPERTY_SCHEMA) + assert 'reference' not in \ + ret['test/a/TestKind/aaa/\xce\xce\xce\xce\xa0\xcd\xff/TestKind:2!'] + + +class InsertGroupEntityIndexCase(unittest.TestCase): + def setUp(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) + self.entities = [] + prev = None + for ii in range(0,3): + entity = datastore.Entity("TestKind", + _app="test", + name=str(ii), + parent = prev, + namespace='c') + prev = entity + # have properties with different values bye same property names + entity.update({'aaa': "1111_" + str(ii), + 'bbb': "2222"}) + self.entities.append(entity.ToPb()) + self.keys = ['test/c/TestKind/aaa/1111_0\x00/TestKind:0!', + 'test/c/TestKind/bbb/2222\x00/TestKind:0!', + 'test/c/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!', + 'test/c/TestKind/bbb/2222\x00/TestKind:0!TestKind:1!', + 'test/c/TestKind/aaa/1111_2\x00/TestKind:0!TestKind:1!TestKind:2!', + 'test/c/TestKind/bbb/2222\x00/TestKind:0!TestKind:1!TestKind:2!'] + self.rkeys = ['test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xcf\xff/TestKind:0!', + 'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!', + 'test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xce\xff/TestKind:0!TestKind:1!', + 'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!TestKind:1!', + 'test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xcd\xff/TestKind:0!TestKind:1!TestKind:2!', + 'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!TestKind:1!TestKind:2!'] + + def runTest(self): + self.app_datastore.InsertIndexEntries(self.entities) + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + # Verify an entity has been stored + ret = datastore_batch.batch_get_entity(ASC_PROPERTY_TABLE, self.keys, + PROPERTY_SCHEMA) + assert 'test/c/TestKind:0!TestKind:1!' in \ + ret['test/c/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!']['reference'] + + def tearDown(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + ret = datastore_batch.batch_delete(ASC_PROPERTY_TABLE, self.keys, PROPERTY_SCHEMA) + ret = datastore_batch.batch_delete(DSC_PROPERTY_TABLE, self.rkeys, PROPERTY_SCHEMA) + # Verify an entity has been deleted + ret = datastore_batch.batch_get_entity(ASC_PROPERTY_TABLE, self.keys, + PROPERTY_SCHEMA) + assert 'reference' not in \ + ret['test/c/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!'] + +class AllocateIDsCase(unittest.TestCase): + def setUp(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + datastore_batch.batch_delete(APP_ID_TABLE, ["a/a"], APP_ID_SCHEMA) + self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) + def runTest(self): + s, e = self.app_datastore.AllocateIds("a/a", 1000) + assert s == 10000 and e == 10999 + for ii in range (0,20): + s, e = self.app_datastore.AllocateIds("a/a", 500) + assert s == 21000 and e == 21499 + + def tearDown(self): + pass + +class InsertAndDeleteIndexesCase(unittest.TestCase): + def setUp(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) + self.entities = [] + prev = None + for ii in range(0,3): + entity = datastore.Entity("TestKind", + _app="test", + name=str(ii), + parent = prev, + namespace='d') + prev = entity + # have properties with different values bye same property names + entity.update({'aaa': "1111_" + str(ii), + 'bbb': "2222"}) + self.entities.append(entity.ToPb()) + self.keys = ['test/d/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!'] + + def runTest(self): + self.app_datastore.InsertIndexEntries(self.entities) + ret = datastore_batch.batch_get_entity(ASC_PROPERTY_TABLE, self.keys, + PROPERTY_SCHEMA) + assert 'test/d/TestKind:0!TestKind:1!' in \ + ret['test/d/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!']['reference'] + + self.app_datastore.DeleteIndexEntries(self.entities) + ret = datastore_batch.batch_get_entity(ASC_PROPERTY_TABLE, self.keys, + PROPERTY_SCHEMA) + assert 'reference' not in \ + ret['test/d/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!'] + + def tearDown(self): + pass + +class PutCase(unittest.TestCase): + def setUp(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) + self.entities = [] + prev = None + for ii in range(0,3): + entity = datastore.Entity("TestKind", + _app="test", + name = str(ii), + parent = prev, + namespace='e') + prev = entity + # have properties with different values bye same property names + entity.update({'aaa': "1111_" + str(ii), + 'bbb': "2222"}) + self.entities.append(entity.ToPb()) + + self.entities2 = [] + prev = None + for ii in range(0,3): + entity = datastore.Entity("TestKind", + _app="test", + name = str(ii), + parent = prev, + namespace='e') + prev = entity + # have properties with different values bye same property names + entity.update({'aaa': "x111_" + str(ii), + 'bbb': "x222"}) + self.entities2.append(entity.ToPb()) + + tuples = sorted((self.app_datastore.GetTablePrefix(x), x) for x in self.entities) + + # keys should be the same for entities and entities2 + self.keys = self.app_datastore.GetIndexKVFromTuple(tuples, reverse=False) + self.keys = [x[0] for x in self.keys] + tuples = sorted((self.app_datastore.GetTablePrefix(x), x) for x in self.entities2) + + # keys should be the same for entities and entities2 + self.keys2 = self.app_datastore.GetIndexKVFromTuple(tuples, reverse=False) + self.keys2 = [x[0] for x in self.keys2] + + def runTest(self): + self.app_datastore.PutEntities(self.entities) + ret = datastore_batch.batch_get_entity(ASC_PROPERTY_TABLE, self.keys, + PROPERTY_SCHEMA) + assert 'test/e/TestKind:0!TestKind:1' in \ + ret['test/e/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!']['reference'] + # overwrite test + self.app_datastore.PutEntities(self.entities2) + ret = datastore_batch.batch_get_entity(ASC_PROPERTY_TABLE, self.keys2, + PROPERTY_SCHEMA) + assert 'test/e/TestKind:0!TestKind:1' in \ + ret['test/e/TestKind/aaa/x111_1\x00/TestKind:0!TestKind:1!']['reference'] + + def tearDown(self): + keys = [e.key() for e in self.entities] + self.app_datastore.DeleteEntities(keys) + +class GetCase(unittest.TestCase): + def setUp(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) + self.entities = [] + self.keys = [] + prev = None + for ii in range(0,3): + entity = datastore.Entity("TestKind", + _app="test", + name = str(ii), + parent = prev, + namespace='e') + prev = entity + # have properties with different values bye same property names + entity.update({'aaa': "1111_" + str(ii), + 'bbb': "2222"}) + self.entities.append(entity.ToPb()) + self.keys = [e.key() for e in self.entities] + self.app_datastore.PutEntities(self.entities) + def runTest(self): + results, keys = self.app_datastore.FetchKeys(self.keys) + self.app_datastore.DeleteEntities(self.keys) + results, keys = self.app_datastore.FetchKeys(self.keys) + for ii in results: + if 'entity' in ii: raise + + def tearDown(self): + pass + +class KindQueryCase(unittest.TestCase): + def setUp(self): + datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) + self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) + self.entities = [] + self.keys = [] + self.entities2 = [] + self.keys2 = [] + + prev = None + for ii in range(0,3): + entity = datastore.Entity(kind="ATestKind", + _app="test", + name = str(ii), + parent = prev, + namespace='f') + prev = entity + # have properties with different values bye same property names + entity.update({'aaa': "1111_" + str(ii), + 'bbb': "2222"}) + self.entities.append(entity.ToPb()) + self.keys = [e.key() for e in self.entities] + self.app_datastore.PutEntities(self.entities) + + prev = None + for ii in range(0,3): + entity = datastore.Entity(kind="BTestKind", + _app="test", + name = str(ii), + parent = prev, + namespace='f') + prev = entity + # have properties with different values bye same property names + entity.update({'aaa': "1111_" + str(ii), + 'bbb': "2222"}) + self.entities2.append(entity.ToPb()) + self.keys2 = [e.key() for e in self.entities2] + self.app_datastore.PutEntities(self.entities2) + + def runTest(self): + def testKind(kind): + q = datastore.Query(kind=kind, _app="test", namespace='f') + q = q._ToPb() + + result = self.app_datastore.KindQuery(q, [], []) + + for ii in result: + item = entity_pb.EntityProto(ii) + for ii in item.entity_group().element_list(): + assert kind == ii.type() + testKind("ATestKind") + testKind("BTestKind") + + q = datastore.Query(kind="ATestKind", _app="test", namespace='f') + q = q._ToPb() + q.set_limit(1) + + result = self.app_datastore.KindQuery(q, [], []) + last_item = entity_pb.EntityProto(result[0]) + last_item = last_item.key() + + q = datastore.Query(kind="ATestKind", _app="test", namespace='f') + q = q._ToPb() + f = q.add_filter() + #entity_pb.Property(last_item) + #filt = datastore_query.PropertyFilter(">", last_item) + #print filt + + def tearDown(self): + self.app_datastore.DeleteEntities(self.keys) + self.app_datastore.DeleteEntities(self.keys2) + + +if __name__ == "__main__": + unittest.main() diff --git a/AppDB/datastore_tester.py b/AppDB/test/functional/test_datastore_v1.py similarity index 99% rename from AppDB/datastore_tester.py rename to AppDB/test/functional/test_datastore_v1.py index 015a357ac1..65c16e8d83 100644 --- a/AppDB/datastore_tester.py +++ b/AppDB/test/functional/test_datastore_v1.py @@ -43,8 +43,6 @@ def createRandomList(number_of_columns, column_name_len): table_name = hf.randomString(10) key = hf.randomString(10) print "key= " + key -#print "columns= " + str(columns) -#print "data= " + str(data) print "table= " + table_name app_datastore = appscale_datastore.DatastoreFactory.getDatastore(datastore_type) ERROR_CODES = appscale_datastore.DatastoreFactory.error_codes() diff --git a/AppDB/test/functional/test_hbase_v1.py b/AppDB/test/functional/test_hbase_v1.py new file mode 100644 index 0000000000..0efc1f1fb3 --- /dev/null +++ b/AppDB/test/functional/test_hbase_v1.py @@ -0,0 +1,198 @@ +import os +import unittest + +import hbase_interface +from dbconstants import * + +# Prereq: HBase must be running first +TEST1_TABLE = "TestTable1" +TEST1_TABLE_SCHEMA = ['c1','c2','c3'] + +TEST2_TABLE = "TestTable2" +TEST2_TABLE_SCHEMA = ['c1','c2','c3'] + +TEST3_TABLE = "TestTable3" +TEST3_TABLE_SCHEMA = ['c1','c2','c3'] + +TEST4_TABLE = "TestTable4" +TEST4_TABLE_SCHEMA = ['c1','c2','c3'] + +TEST5_TABLE = "TestTable5" +TEST5_TABLE_SCHEMA = ['c1','c2','c3'] + +class PutTestCase(unittest.TestCase): + def setUp(self): + self.dbstore = hbase_interface.DatastoreProxy() + self.dbstore.delete_table(TEST1_TABLE) + self.dbstore.create_table(TEST1_TABLE, TEST1_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c'] + cell_values = {'a':{'c1':'1','c2':'2','c3':'3'}, + 'b':{'c1':'4','c2':'5','c3':'6'}, + 'c':{'c1':'7','c2':'8','c3':'9'}} + self.dbstore.batch_put_entity(TEST1_TABLE, row_key, TEST1_TABLE_SCHEMA, + cell_values) + assert self.dbstore.batch_get_entity(TEST1_TABLE, row_key, TEST1_TABLE_SCHEMA) == cell_values + + def tearDown(self): + self.dbstore.delete_table(TEST1_TABLE) + +class DeleteTestCase(unittest.TestCase): + def setUp(self): + self.dbstore = hbase_interface.DatastoreProxy() + self.dbstore.delete_table(TEST2_TABLE) + self.dbstore.create_table(TEST2_TABLE, TEST2_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c'] + cell_values = {'a':{'c1':'1','c2':'2','c3':'3'}, + 'b':{'c1':'4','c2':'5','c3':'6'}, + 'c':{'c1':'7','c2':'8','c3':'9'}} + self.dbstore.batch_put_entity(TEST2_TABLE, row_key, TEST2_TABLE_SCHEMA, + cell_values) + + assert self.dbstore.batch_get_entity(TEST2_TABLE, + row_key, + TEST2_TABLE_SCHEMA) == cell_values + self.dbstore.batch_delete(TEST2_TABLE, ['a','b'], column_names=TEST2_TABLE_SCHEMA) + assert self.dbstore.batch_get_entity(TEST2_TABLE, + row_key, + TEST2_TABLE_SCHEMA) == \ + {'a':{}, 'b':{}, 'c':{'c1':'7','c2':'8','c3':'9'}} + + def tearDown(self): + self.dbstore.delete_table(TEST2_TABLE) + +class GetOnNonExistentKey(unittest.TestCase): + def setUp(self): + self.dbstore = hbase_interface.DatastoreProxy() + self.dbstore.delete_table(TEST3_TABLE) + self.dbstore.create_table(TEST3_TABLE, TEST3_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c'] + cell_values = {'a':{}, 'b':{}, 'c':{}} + ret = self.dbstore.batch_get_entity(TEST3_TABLE, row_key, TEST3_TABLE_SCHEMA) + assert ret == cell_values + + def tearDown(self): + self.dbstore.delete_table(TEST3_TABLE) + +class PutOverwriteTestCase(unittest.TestCase): + def setUp(self): + self.dbstore = hbase_interface.DatastoreProxy() + self.dbstore.delete_table(TEST4_TABLE) + self.dbstore.create_table(TEST4_TABLE, TEST4_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c'] + cell_values = {'a':{'c1':'1','c2':'2','c3':'3'}, + 'b':{'c1':'4','c2':'5','c3':'6'}, + 'c':{'c1':'7','c2':'8','c3':'9'}} + self.dbstore.batch_put_entity(TEST4_TABLE, row_key, TEST4_TABLE_SCHEMA, + cell_values) + cell_values = {'a':{'c1':'10','c2':'20','c3':'30'}, + 'b':{'c1':'40','c2':'50','c3':'60'}, + 'c':{'c1':'70','c2':'80','c3':'90'}} + self.dbstore.batch_put_entity(TEST4_TABLE, row_key, TEST4_TABLE_SCHEMA, + cell_values) + + assert self.dbstore.batch_get_entity(TEST4_TABLE, row_key, TEST4_TABLE_SCHEMA) == cell_values + + def tearDown(self): + self.dbstore.delete_table(TEST4_TABLE) + +class RangeTestCase(unittest.TestCase): + def setUp(self): + self.dbstore = hbase_interface.DatastoreProxy() + self.dbstore.delete_table(TEST5_TABLE) + self.dbstore.create_table(TEST5_TABLE, TEST5_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c','d','e','f'] + cell_values = {'a':{'c1':'1','c2':'2','c3':'3'}, + 'b':{'c1':'4','c2':'5','c3':'6'}, + 'c':{'c1':'7','c2':'8','c3':'9'}, + 'd':{'c1':'10','c2':'11','c3':'12'}, + 'e':{'c1':'13','c2':'14','c3':'15'}, + 'f':{'c1':'16','c2':'17','c3':'18'}} + expected = [{'a':{'c1':'1','c2':'2','c3':'3'}}, + {'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}, + {'d':{'c1':'10','c2':'11','c3':'12'}}, + {'e':{'c1':'13','c2':'14','c3':'15'}}, + {'f':{'c1':'16','c2':'17','c3':'18'}}] + + self.dbstore.batch_put_entity(TEST5_TABLE, row_key, TEST5_TABLE_SCHEMA, + cell_values) + column_names = TEST5_TABLE_SCHEMA + limit = 10 + offset = 0 + startrow = 'a' + endrow = 'f' + start_in = True + end_in = True + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, + limit, offset, start_in, end_in) == expected + return + keys_only = True + expected = ['a','b','c','d','e','f'] + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, + limit, offset, start_in, + end_in, keys_only) == expected + + keys_only = False + start_in = False + expected = [{'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}, + {'d':{'c1':'10','c2':'11','c3':'12'}}, + {'e':{'c1':'13','c2':'14','c3':'15'}}, + {'f':{'c1':'16','c2':'17','c3':'18'}}] + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, + offset, start_in, end_in) == expected + + expected = [{'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}, + {'d':{'c1':'10','c2':'11','c3':'12'}}, + {'e':{'c1':'13','c2':'14','c3':'15'}}] + + end_in = False + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, + offset, start_in, end_in) == expected + + expected = [{'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}, + {'d':{'c1':'10','c2':'11','c3':'12'}}, + {'e':{'c1':'13','c2':'14','c3':'15'}}] + start_in = True + end_in = True + startrow = 'b' + endrow = 'e' + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, offset, + start_in, end_in) == expected + + expected = [{'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}] + limit = 2 + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, + offset, start_in, end_in) == expected + + expected = [{'c':{'c1':'7','c2':'8','c3':'9'}}] + offset = 1 + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, + offset, start_in, end_in) == expected + + def tearDown(self): + self.dbstore.delete_table(TEST5_TABLE) + +if __name__ == "__main__": + unittest.main() + diff --git a/AppDB/test/functional/test_hypertable_v1.py b/AppDB/test/functional/test_hypertable_v1.py new file mode 100644 index 0000000000..772af6e949 --- /dev/null +++ b/AppDB/test/functional/test_hypertable_v1.py @@ -0,0 +1,201 @@ +import os +import unittest + +import hypertable_interface +from dbconstants import * + +# Prereq: Hypertable must be running first +TEST1_TABLE = "TestTable1" +TEST1_TABLE_SCHEMA = ['c1','c2','c3'] + +TEST2_TABLE = "TestTable2" +TEST2_TABLE_SCHEMA = ['c1','c2','c3'] + +TEST3_TABLE = "TestTable3" +TEST3_TABLE_SCHEMA = ['c1','c2','c3'] + +TEST4_TABLE = "TestTable4" +TEST4_TABLE_SCHEMA = ['c1','c2','c3'] + +TEST5_TABLE = "TestTable5" +TEST5_TABLE_SCHEMA = ['c1','c2','c3'] + + +class PutTestCase(unittest.TestCase): + def setUp(self): + self.dbstore = hypertable_interface.DatastoreProxy() + self.dbstore.delete_table(TEST1_TABLE) + self.dbstore.create_table(TEST1_TABLE, TEST1_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c'] + cell_values = {'a':{'c1':'1','c2':'2','c3':'3'}, + 'b':{'c1':'4','c2':'5','c3':'6'}, + 'c':{'c1':'7','c2':'8','c3':'9'}} + self.dbstore.batch_put_entity(TEST1_TABLE, row_key, TEST1_TABLE_SCHEMA, + cell_values) + + assert self.dbstore.batch_get_entity(TEST1_TABLE, row_key, TEST1_TABLE_SCHEMA) == cell_values + + def tearDown(self): + self.dbstore.delete_table(TEST1_TABLE) + +class DeleteTestCase(unittest.TestCase): + def setUp(self): + self.dbstore = hypertable_interface.DatastoreProxy() + self.dbstore.delete_table(TEST2_TABLE) + self.dbstore.create_table(TEST2_TABLE, TEST2_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c'] + cell_values = {'a':{'c1':'1','c2':'2','c3':'3'}, + 'b':{'c1':'4','c2':'5','c3':'6'}, + 'c':{'c1':'7','c2':'8','c3':'9'}} + self.dbstore.batch_put_entity(TEST2_TABLE, row_key, TEST2_TABLE_SCHEMA, + cell_values) + + assert self.dbstore.batch_get_entity(TEST2_TABLE, + row_key, + TEST2_TABLE_SCHEMA) == cell_values + self.dbstore.batch_delete(TEST2_TABLE, ['a','b']) + assert self.dbstore.batch_get_entity(TEST2_TABLE, + row_key, + TEST2_TABLE_SCHEMA) == \ + {'a':{}, 'b':{}, 'c':{'c1':'7','c2':'8','c3':'9'}} + + def tearDown(self): + self.dbstore.delete_table(TEST2_TABLE) + +class GetOnNonExistantKey(unittest.TestCase): + def setUp(self): + self.dbstore = hypertable_interface.DatastoreProxy() + self.dbstore.delete_table(TEST3_TABLE) + self.dbstore.create_table(TEST3_TABLE, TEST3_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c'] + cell_values = {'a':{}, 'b':{}, 'c':{}} + ret = self.dbstore.batch_get_entity(TEST3_TABLE, row_key, TEST3_TABLE_SCHEMA) + assert ret == cell_values + + def tearDown(self): + self.dbstore.delete_table(TEST3_TABLE) + +class PutOverwriteTestCase(unittest.TestCase): + def setUp(self): + self.dbstore = hypertable_interface.DatastoreProxy() + self.dbstore.delete_table(TEST4_TABLE) + self.dbstore.create_table(TEST4_TABLE, TEST4_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c'] + cell_values = {'a':{'c1':'1','c2':'2','c3':'3'}, + 'b':{'c1':'4','c2':'5','c3':'6'}, + 'c':{'c1':'7','c2':'8','c3':'9'}} + self.dbstore.batch_put_entity(TEST4_TABLE, row_key, TEST4_TABLE_SCHEMA, + cell_values) + cell_values = {'a':{'c1':'10','c2':'20','c3':'30'}, + 'b':{'c1':'40','c2':'50','c3':'60'}, + 'c':{'c1':'70','c2':'80','c3':'90'}} + self.dbstore.batch_put_entity(TEST4_TABLE, row_key, TEST4_TABLE_SCHEMA, + cell_values) + + assert self.dbstore.batch_get_entity(TEST4_TABLE, row_key, TEST4_TABLE_SCHEMA) == cell_values + + def tearDown(self): + self.dbstore.delete_table(TEST4_TABLE) + +class RangeTestCase(unittest.TestCase): + def setUp(self): + self.dbstore = hypertable_interface.DatastoreProxy() + self.dbstore.delete_table(TEST5_TABLE) + self.dbstore.create_table(TEST5_TABLE, TEST5_TABLE_SCHEMA) + + def runTest(self): + row_key = ['a','b','c','d','e','f'] + cell_values = {'a':{'c1':'1','c2':'2','c3':'3'}, + 'b':{'c1':'4','c2':'5','c3':'6'}, + 'c':{'c1':'7','c2':'8','c3':'9'}, + 'd':{'c1':'10','c2':'11','c3':'12'}, + 'e':{'c1':'13','c2':'14','c3':'15'}, + 'f':{'c1':'16','c2':'17','c3':'18'}} + expected = [{'a':{'c1':'1','c2':'2','c3':'3'}}, + {'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}, + {'d':{'c1':'10','c2':'11','c3':'12'}}, + {'e':{'c1':'13','c2':'14','c3':'15'}}, + {'f':{'c1':'16','c2':'17','c3':'18'}}] + + self.dbstore.batch_put_entity(TEST5_TABLE, row_key, TEST5_TABLE_SCHEMA, + cell_values) + column_names = TEST5_TABLE_SCHEMA + limit = 10 + offset = 0 + startrow = 'a' + endrow = 'f' + start_in = True + end_in = True + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, + limit, offset, start_in, end_in) == expected + + keys_only = True + expected = ['a','b','c','d','e','f'] + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, + limit, offset, start_in, + end_in, keys_only) == expected + + keys_only = False + start_in = False + expected = [{'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}, + {'d':{'c1':'10','c2':'11','c3':'12'}}, + {'e':{'c1':'13','c2':'14','c3':'15'}}, + {'f':{'c1':'16','c2':'17','c3':'18'}}] + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, + offset, start_in, end_in) == expected + + expected = [{'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}, + {'d':{'c1':'10','c2':'11','c3':'12'}}, + {'e':{'c1':'13','c2':'14','c3':'15'}}] + + end_in = False + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, + offset, start_in, end_in) == expected + + expected = [{'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}, + {'d':{'c1':'10','c2':'11','c3':'12'}}, + {'e':{'c1':'13','c2':'14','c3':'15'}}] + start_in = True + end_in = True + startrow = 'b' + endrow = 'e' + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, offset, + start_in, end_in) == expected + + expected = [{'b':{'c1':'4','c2':'5','c3':'6'}}, + {'c':{'c1':'7','c2':'8','c3':'9'}}] + limit = 2 + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, + offset, start_in, end_in) == expected + + expected = [{'c':{'c1':'7','c2':'8','c3':'9'}}] + offset = 1 + assert self.dbstore.range_query(TEST5_TABLE, column_names, + startrow, endrow, limit, + offset, start_in, end_in) == expected + + def tearDown(self): + self.dbstore.delete_table(TEST5_TABLE) + + +if __name__ == "__main__": + unittest.main() + diff --git a/AppDB/test/test_memcache_mutex.py b/AppDB/test/functional/test_memcache_mutex.py similarity index 100% rename from AppDB/test/test_memcache_mutex.py rename to AppDB/test/functional/test_memcache_mutex.py diff --git a/AppDB/migration_integrationtest.py b/AppDB/test/functional/test_migration.py similarity index 100% rename from AppDB/migration_integrationtest.py rename to AppDB/test/functional/test_migration.py diff --git a/AppDB/migration_unittest.py b/AppDB/test/functional/test_migration2.py similarity index 100% rename from AppDB/migration_unittest.py rename to AppDB/test/functional/test_migration2.py diff --git a/AppDB/soap_tester.py b/AppDB/test/functional/test_soap_server.py similarity index 99% rename from AppDB/soap_tester.py rename to AppDB/test/functional/test_soap_server.py index 99015c1857..758757694d 100644 --- a/AppDB/soap_tester.py +++ b/AppDB/test/functional/test_soap_server.py @@ -54,7 +54,6 @@ #Navraj Chohan app_location = "localhost" bindport = DEFAULT_SSL_PORT -super_secret = "" encrypt = True appname = "" username = "" diff --git a/AppDB/test/test_zktransaction.py b/AppDB/test/functional/test_zktransaction.py similarity index 100% rename from AppDB/test/test_zktransaction.py rename to AppDB/test/functional/test_zktransaction.py diff --git a/AppDB/test/unit/test_cassandra_interface.py b/AppDB/test/unit/test_cassandra_interface.py new file mode 100644 index 0000000000..8102b43852 --- /dev/null +++ b/AppDB/test/unit/test_cassandra_interface.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# Programmer: Navraj Chohan + +import pycassa +import os +import sys +import unittest + +from flexmock import flexmock + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../cassandra")) +import cassandra_interface + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../")) +import helper_functions + +class FakeCassClient(): + """ Fake cassandra client class for mocking """ + def __init__(self): + return + def multiget_slice(self, rk, path, slice_predicate, consistency): + return {} + +class FakePool(): + """ Fake cassandra connection pool for mocking """ + def get(self): + return FakeCassClient() + def return_conn(self, client): + return + +class FakeColumnFamily(): + """ Fake column family class for mocking """ + def __init__(self): + return + def batch_insert(self, multi_map): + return + def get_range(self, start='', finish='', columns='', row_count='', + read_consistency_level=''): + return {} + +class FakeSystemManager(): + """ Fake system manager class for mocking """ + def __init__(self): + return + def drop_column_family(self, keyspace, table_name): + return + +class TestCassandra(unittest.TestCase): + def testConstructor(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(pycassa).should_receive("ConnectionPool") \ + .and_return(FakePool()) + + db = cassandra_interface.DatastoreProxy() + + def testGet(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(pycassa).should_receive("ConnectionPool") \ + .and_return(FakePool()) + + db = cassandra_interface.DatastoreProxy() + + # Make sure no exception is thrown + assert {} == db.batch_get_entity('table', [], []) + + def testPut(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(pycassa) \ + .should_receive("ColumnFamily") \ + .and_return(FakeColumnFamily()) + + db = cassandra_interface.DatastoreProxy() + + # Make sure no exception is thrown + assert None == db.batch_put_entity('table', [], [], {}) + + def testDeleteTable(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(pycassa.system_manager) \ + .should_receive("SystemManager") \ + .and_return(FakeSystemManager()) + + db = cassandra_interface.DatastoreProxy() + + # Make sure no exception is thrown + db.delete_table('table') + + def testRangeQuery(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(pycassa) \ + .should_receive("ColumnFamily") \ + .and_return(FakeColumnFamily()) + + db = cassandra_interface.DatastoreProxy() + + assert [] == db.range_query("table", [], "start", "end", 0) + +if __name__ == "__main__": + unittest.main() diff --git a/AppDB/test/unit/test_cassandra_prime.py b/AppDB/test/unit/test_cassandra_prime.py new file mode 100644 index 0000000000..2ba5aaa70f --- /dev/null +++ b/AppDB/test/unit/test_cassandra_prime.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# Programmer: Navraj Chohan + +import dbconstants +import pycassa +import os +import sys +import unittest + +from flexmock import flexmock + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../cassandra")) +import prime_cassandra + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../")) +import helper_functions + +class FakeCassClient(): + """ Fake cassandra client class for mocking """ + def __init__(self): + return + def multiget_slice(self, rk, path, slice_predicate, consistency): + return {} + +class FakePool(): + """ Fake cassandra connection pool for mocking """ + def get(self): + return FakeCassClient() + def return_conn(self, client): + return + +class FakeColumnFamily(): + """ Fake column family class for mocking """ + def __init__(self): + return + def batch_insert(self, multi_map): + return + def get_range(self, start='', finish='', columns='', row_count='', + read_consistency_level=''): + return {} + +class FakeSystemManager(): + """ Fake system manager class for mocking """ + def __init__(self): + return + def drop_column_family(self, keyspace, table_name): + return + def drop_keyspace(self, keyspace): + return + def create_keyspace(self, keyspace, strategy, rep_factor): + return + def create_column_family(self, keysapce, col_fam, + comparator_type=pycassa.system_manager.UTF8_TYPE): + return + def close(self): + return + +class TestCassandraPrimer(unittest.TestCase): + def test_primer(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(pycassa.system_manager).should_receive("SystemManager") \ + .and_return(FakeSystemManager()) + + assert prime_cassandra.create_keyspaces(1) + + def test_bad_arg(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(pycassa.system_manager).should_receive("SystemManager") \ + .and_return(FakeSystemManager()) + + #prime_cassandra.create_keyspaces(-1) + self.assertRaises(dbconstants.AppScaleBadArg, + prime_cassandra.create_keyspaces, -1) + +if __name__ == "__main__": + unittest.main() diff --git a/AppDB/test/unit/test_datastore_server.py b/AppDB/test/unit/test_datastore_server.py new file mode 100644 index 0000000000..83dd4f6283 --- /dev/null +++ b/AppDB/test/unit/test_datastore_server.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python +# Programmer: Navraj Chohan + +import os +import sys +import unittest +from flexmock import flexmock + +from google.appengine.ext import db +from google.appengine.datastore import entity_pb + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../")) +from appscale_datastore_batch import DatastoreFactory +from datastore_server import DatastoreDistributed +from datastore_server import BLOCK_SIZE +from dbconstants import * + +class Item(db.Model): + name = db.StringProperty(required = True) + +class TestDatastoreServer(unittest.TestCase): + """ + A set of test cases for the datastore server (datastore server v2) + """ + def test_get_entity_kind(self): + dd = DatastoreDistributed(None) + item = Item(name="Bob", _app="hello") + key = db.model_to_protobuf(item) + assert dd.get_entity_kind(key) =="Item" + + def test_kind_key(self): + dd = DatastoreDistributed(None) + item = Item(name="Dyan", _app="hello") + key = db.model_to_protobuf(item) + assert dd.get_kind_key("howdy", key.key().path()) == "howdy/Item:0000000000!" + + item1 = Item(key_name="Bob", name="Bob", _app="hello") + key = db.model_to_protobuf(item1) + assert dd.get_kind_key("howdy", key.key().path()) == "howdy/Item:Bob!" + + item2 = Item(key_name="Frank", name="Frank", _app="hello", parent = item1) + key = db.model_to_protobuf(item2) + assert dd.get_kind_key("howdy", key.key().path()) == \ + "howdy/Item:Frank!Item:Bob!" + + def test_get_entity_key(self): + dd = DatastoreDistributed(None) + item = Item(key_name="Bob", name="Bob", _app="hello") + key = db.model_to_protobuf(item) + assert dd.get_entity_key("howdy", key.key().path()) == "howdy/Item:Bob!" + + def test_validate_key(self): + dd = DatastoreDistributed(None) + item = Item(key_name="Bob", name="Bob", _app="hello") + key = db.model_to_protobuf(item) + dd.validate_key(key.key()) + + def test_get_index_key(self): + dd = DatastoreDistributed(None) + dd.get_index_key("a","b","c","d") == "a/b/c/d" + + def test_configure_namespace(self): + db_batch = flexmock() + db_batch.should_receive("batch_put_entity").and_return(None) + dd = DatastoreDistributed(db_batch) + assert dd.configure_namespace("howdy", "hello", "ns") == True + + def test_configure_namespace(self): + db_batch = flexmock() + db_batch.should_receive("batch_put_entity").and_return(None) + dd = DatastoreDistributed(db_batch) + item = Item(key_name="Bob", name="Bob", _app="hello") + key = db.model_to_protobuf(item) + assert dd.get_table_prefix(key) == "hello/" + + def test_get_index_key_from_params(self): + dd = DatastoreDistributed(None) + params = ['a','b','c','d','e'] + assert dd.get_index_key_from_params(params) == "a/b/c/d/e" + + def test_get_index_kv_from_tuple(self): + dd = DatastoreDistributed(None) + item1 = Item(key_name="Bob", name="Bob", _app="hello") + item2 = Item(key_name="Sally", name="Sally", _app="hello") + key1 = db.model_to_protobuf(item1) + key2 = db.model_to_protobuf(item2) + tuples_list = [("a/b",key1),("a/b",key2)] + assert dd.get_index_kv_from_tuple(tuples_list) == (['a/b/Item/name/Bob\x00/Item:Bob!', 'a/b/Item:Bob!'], ['a/b/Item/name/Sally\x00/Item:Sally!', 'a/b/Item:Sally!']) + + def test_delete_index_entries(self): + db_batch = flexmock() + db_batch.should_receive("batch_delete").and_return(None) + db_batch.should_receive("batch_put_entity").and_return(None) + dd = DatastoreDistributed(db_batch) + item1 = Item(key_name="Bob", name="Bob", _app="hello") + item2 = Item(key_name="Sally", name="Sally", _app="hello") + key1 = db.model_to_protobuf(item1) + key2 = db.model_to_protobuf(item2) + dd.delete_index_entries([key1,key2]) + + def test_insert_entities(self): + db_batch = flexmock() + db_batch.should_receive("batch_put_entity").and_return(None) + dd = DatastoreDistributed(db_batch) + item1 = Item(key_name="Bob", name="Bob", _app="hello") + item2 = Item(key_name="Sally", name="Sally", _app="hello") + key1 = db.model_to_protobuf(item1) + key2 = db.model_to_protobuf(item2) + dd.insert_entities([key1,key2]) + + def test_insert_index_entries(self): + db_batch = flexmock() + db_batch.should_receive("batch_put_entity").and_return(None) + dd = DatastoreDistributed(db_batch) + item1 = Item(key_name="Bob", name="Bob", _app="hello") + item2 = Item(key_name="Sally", name="Sally", _app="hello") + key1 = db.model_to_protobuf(item1) + key2 = db.model_to_protobuf(item2) + dd.insert_index_entries([key1,key2]) + + def test_acquire_id_block_from_db(self): + PREFIX = "x" + db_batch = flexmock() + db_batch.should_receive("batch_get_entity").and_return({PREFIX:{APP_ID_SCHEMA[0]:"1"}}) + dd = DatastoreDistributed(db_batch) + assert dd.acquire_id_block_from_db(PREFIX) == 1 + + PREFIX = "x" + db_batch = flexmock() + db_batch.should_receive("batch_get_entity").and_return({PREFIX:{}}) + dd = DatastoreDistributed(db_batch) + assert dd.acquire_id_block_from_db(PREFIX) == 0 + + def test_increment_id_in_db(self): + PREFIX = "x" + db_batch = flexmock() + db_batch.should_receive("batch_get_entity").and_return({PREFIX:{APP_ID_SCHEMA[0]:"0"}}) + db_batch.should_receive("batch_put_entity").and_return(None) + dd = DatastoreDistributed(db_batch) + assert dd.increment_id_in_db(PREFIX) == BLOCK_SIZE + + PREFIX = "x" + db_batch = flexmock() + db_batch.should_receive("batch_get_entity").and_return({PREFIX:{APP_ID_SCHEMA[0]:"1"}}) + db_batch.should_receive("batch_put_entity").and_return(None) + dd = DatastoreDistributed(db_batch) + assert dd.increment_id_in_db(PREFIX) == 2 * BLOCK_SIZE + + def test_allocate_ids(self): + PREFIX = "x" + BATCH_SIZE = 1000 + db_batch = flexmock() + db_batch.should_receive("batch_get_entity").and_return({PREFIX:{APP_ID_SCHEMA[0]:"1"}}) + db_batch.should_receive("batch_put_entity").and_return(None) + dd = DatastoreDistributed(db_batch) + assert dd.allocate_ids(PREFIX, BATCH_SIZE) == (20000, 20999) + + def test_put_entities(self): + item1 = Item(key_name="Bob", name="Bob", _app="hello") + item2 = Item(key_name="Sally", name="Sally", _app="hello") + key1 = db.model_to_protobuf(item1) + key2 = db.model_to_protobuf(item2) + + db_batch = flexmock() + db_batch.should_receive("batch_delete").and_return(None) + db_batch.should_receive("batch_put_entity").and_return(None) + db_batch.should_receive("batch_get_entity").and_return({"key1":{},"key2":{}}) + dd = DatastoreDistributed(db_batch) + dd.put_entities([key1, key2]) + + db_batch = flexmock() + db_batch.should_receive("batch_delete").and_return(None) + db_batch.should_receive("batch_put_entity").and_return(None) + db_batch.should_receive("batch_get_entity").and_return({"key1":{"entity":key1.Encode()},"key2":{"entity":key2.Encode()}}) + dd = DatastoreDistributed(db_batch) + dd.put_entities([key1, key2]) + + def testFetchKeys(self): + item1 = Item(key_name="Bob", name="Bob", _app="hello") + item2 = Item(key_name="Sally", name="Sally", _app="hello") + key1 = db.model_to_protobuf(item1) + key2 = db.model_to_protobuf(item2) + + db_batch = flexmock() + db_batch.should_receive("batch_delete").and_return(None) + db_batch.should_receive("batch_put_entity").and_return(None) + db_batch.should_receive("batch_get_entity").and_return(['aaa']) + dd = DatastoreDistributed(db_batch) + assert dd.fetch_keys([key1.key(), key2.key()]) == (['aaa'], ['hello//Item:Bob!', 'hello//Item:Sally!']) + +if __name__ == "__main__": + unittest.main() diff --git a/AppDB/test/unit/test_hbase_interface.py b/AppDB/test/unit/test_hbase_interface.py new file mode 100644 index 0000000000..9985bd8684 --- /dev/null +++ b/AppDB/test/unit/test_hbase_interface.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# Programmer: Navraj Chohan + +import os +import sys +import unittest + +from flexmock import flexmock + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../hbase/")) +import hbase_interface + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../")) +import helper_functions + +class FakeHBaseClient(): + """ Fake hbase client class for mocking """ + def __init__(self): + return + def getRowsWithColumns(self, table_name, row_keys, column_list): + return "NS" + def mutateRows(self, table_name, all_mutations): + return [] + def disableTable(self, table_name): + return + def deleteTable(self, table_name): + return + def createTable(self, table_name): + return + def scannerOpenWithStop(self, table_name, start_key, end_key, col_names): + return [] + def scannerGetList(self, scanner, rowcount): + return [] + def scannerClose(self, scanner): + return + +class TestHBase(unittest.TestCase): + def testConstructor(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(hbase_interface.DatastoreProxy).should_receive("create_connection") \ + .and_return(FakeHBaseClient()) + + db = hbase_interface.DatastoreProxy() + + def testGet(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(hbase_interface.DatastoreProxy).should_receive("create_connection") \ + .and_return(FakeHBaseClient()) + + db = hbase_interface.DatastoreProxy() + + # Make sure no exception is thrown + assert {} == db.batch_get_entity('table', [], []) + + def testPut(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(hbase_interface.DatastoreProxy).should_receive("create_connection") \ + .and_return(FakeHBaseClient()) + + db = hbase_interface.DatastoreProxy() + + # Make sure no exception is thrown + assert None == db.batch_put_entity('table', [], [], {}) + + def testDeleteTable(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(hbase_interface.DatastoreProxy).should_receive("create_connection") \ + .and_return(FakeHBaseClient()) + + db = hbase_interface.DatastoreProxy() + + # Make sure no exception is thrown + assert None == db.delete_table('table') + + def testDelete(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(hbase_interface.DatastoreProxy).should_receive("create_connection") \ + .and_return(FakeHBaseClient()) + + db = hbase_interface.DatastoreProxy() + + # Make sure no exception is thrown + assert None == db.batch_delete('table', []) + + def testRangeQuery(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(hbase_interface.DatastoreProxy).should_receive("create_connection") \ + .and_return(FakeHBaseClient()) + + db = hbase_interface.DatastoreProxy() + assert [] == db.range_query("table", [], "start", "end", 0) + +if __name__ == "__main__": + unittest.main() diff --git a/AppDB/test/unit/test_hbase_prime.py b/AppDB/test/unit/test_hbase_prime.py new file mode 100644 index 0000000000..aa868f68d1 --- /dev/null +++ b/AppDB/test/unit/test_hbase_prime.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# Programmer: Navraj Chohan + +import os +import sys +import unittest + +from flexmock import flexmock + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../hbase/")) +import py_hbase +import prime_hbase + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../")) +import helper_functions + +class FakeHBaseClient(): + """ Fake hbase client class for mocking """ + def __init__(self): + return + def getRowsWithColumns(self, table_name, row_keys, column_list): + return "NS" + def mutateRows(self, table_name, all_mutations): + return [] + def disableTable(self, table_name): + return + def deleteTable(self, table_name): + return + def create_table(self, table_name, columns): + return + def scannerOpenWithStop(self, table_name, start_key, end_key, col_names): + return [] + def scannerGetList(self, scanner, rowcount): + return [] + def scannerClose(self, scanner): + return + +class TestHBasePrimer(unittest.TestCase): + def testPrimer(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(py_hbase).should_receive("DatastoreProxy") \ + .and_return(FakeHBaseClient()) + + prime_hbase.create_table('table', ['a','b','c']) +if __name__ == "__main__": + unittest.main() diff --git a/AppDB/test/unit/test_hypertable_interface.py b/AppDB/test/unit/test_hypertable_interface.py new file mode 100644 index 0000000000..a9446a46a0 --- /dev/null +++ b/AppDB/test/unit/test_hypertable_interface.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# Programmer: Navraj Chohan + +import os +import sys +import unittest + +from flexmock import flexmock + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../hypertable/")) +from hypertable import thriftclient +import hypertable_interface + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../")) +import helper_functions + +class FakeHypertableClient(): + """ Fake hypertable client class for mocking """ + def __init__(self): + return + def namespace_open(self, NS): + return "NS" + def get_cells(self, ns, table_name, scane_spec): + return [] + def drop_table(self, ns, table_name, x): + return None + def mutator_open(self, ns, table_name, x, y): + return None + def mutator_set_cells(self, mutator, cell_list): + return None + def mutator_close(self, mutator): + return None + +class TestHypertable(unittest.TestCase): + def testConstructor(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(thriftclient).should_receive("ThriftClient") \ + .and_return(FakeHypertableClient()) + + db = hypertable_interface.DatastoreProxy() + + def testGet(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(thriftclient).should_receive("ThriftClient") \ + .and_return(FakeHypertableClient()) + + db = hypertable_interface.DatastoreProxy() + + # Make sure no exception is thrown + assert {} == db.batch_get_entity('table', [], []) + + def testPut(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(thriftclient).should_receive("ThriftClient") \ + .and_return(FakeHypertableClient()) + + db = hypertable_interface.DatastoreProxy() + + # Make sure no exception is thrown + assert None == db.batch_put_entity('table', [], [], {}) + + def testDeleteTable(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(thriftclient).should_receive("ThriftClient") \ + .and_return(FakeHypertableClient()) + + db = hypertable_interface.DatastoreProxy() + + # Make sure no exception is thrown + assert None == db.delete_table('table') + + def testDelete(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(thriftclient).should_receive("ThriftClient") \ + .and_return(FakeHypertableClient()) + + db = hypertable_interface.DatastoreProxy() + + # Make sure no exception is thrown + assert None == db.batch_delete('table', []) + + def testRangeQuery(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(thriftclient).should_receive("ThriftClient") \ + .and_return(FakeHypertableClient()) + + db = hypertable_interface.DatastoreProxy() + assert [] == db.range_query("table", [], "start", "end", 0) + +if __name__ == "__main__": + unittest.main() diff --git a/AppDB/test/unit/test_hypertable_prime.py b/AppDB/test/unit/test_hypertable_prime.py new file mode 100644 index 0000000000..1bb026dd22 --- /dev/null +++ b/AppDB/test/unit/test_hypertable_prime.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +# Programmer: Navraj Chohan + +import os +import sys +import unittest + +from flexmock import flexmock + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../hypertable/")) +from hypertable import thriftclient +import hypertable_interface +import py_hypertable +import prime_hypertable + +sys.path.append(os.path.join(os.path.dirname(__file__), "../../")) +import helper_functions + +class FakeHypertableClient(): + """ Fake hypertable client class for mocking """ + def __init__(self): + return + def namespace_open(self, NS): + return "NS" + def get_cells(self, ns, table_name, scane_spec): + return [] + def drop_table(self, ns, table_name, x): + return None + def mutator_open(self, ns, table_name, x, y): + return None + def mutator_set_cells(self, mutator, cell_list): + return None + def mutator_close(self, mutator): + return None + def create_table(self, table, columns): + return None + +class TestHypertable(unittest.TestCase): + def testConstructor(self): + flexmock(helper_functions) \ + .should_receive('read_file') \ + .and_return('127.0.0.1') + + flexmock(py_hypertable).should_receive("DatastoreProxy") \ + .and_return(FakeHypertableClient()) + + prime_hypertable.create_app_tables() + +if __name__ == "__main__": + unittest.main() diff --git a/AppDB/view_all_records.py b/AppDB/view_all_records.py new file mode 100755 index 0000000000..044a4d2b97 --- /dev/null +++ b/AppDB/view_all_records.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# Programmer: Navraj Chohan +""" + View all application entities. +""" + +import os +import sys + +from dbconstants import * +import appscale_datastore_batch + +_MAX_ENTITIES = 1000000 +def get_entities(table, schema, db): + """ Gets entities from a table. + + Args: + table: Name of the table + schema: The schema of table to get from + db: The database accessor + Returns: + The entire table up to _MAX_ENTITIES + """ + return db.range_query(table, schema, "", "", _MAX_ENTITIES) + +def view_all(entities, table, db): + """ View all entities for a table + + Args: + entities: Shows all entities in a list + table: The table these entities are from + db: database accessor + """ + print + print "TABLE:",table + for ii in entities: + print ii + print + +def main(argv): + """ Main + """ + DB_TYPE="cassandra" + if len(argv) < 2: + print "usage: ./view_all_records.py db_type" + else: + DB_TYPE = argv[1] + + db = appscale_datastore_batch.DatastoreFactory.getDatastore(DB_TYPE) + entities = get_entities(APP_NAMESPACE_TABLE, APP_NAMESPACE_SCHEMA, db) + view_all(entities, APP_NAMESPACE_TABLE, db) + + entities = get_entities(APP_ENTITY_TABLE, APP_ENTITY_SCHEMA, db) + view_all(entities, APP_ENTITY_TABLE, db) + + entities = get_entities(ASC_PROPERTY_TABLE, PROPERTY_SCHEMA, db) + view_all(entities, ASC_PROPERTY_TABLE, db) + + entities = get_entities(DSC_PROPERTY_TABLE, PROPERTY_SCHEMA, db) + view_all(entities, DSC_PROPERTY_TABLE, db) + + entities = get_entities(APP_KIND_TABLE, APP_KIND_SCHEMA, db) + view_all(entities, APP_KIND_TABLE, db) + +if __name__ == "__main__": + try: + main(sys.argv) + except: + raise + diff --git a/AppServer/google/appengine/api/datastore.py b/AppServer/google/appengine/api/datastore.py index e3f48d6cf8..4aaac5ea04 100644 --- a/AppServer/google/appengine/api/datastore.py +++ b/AppServer/google/appengine/api/datastore.py @@ -589,7 +589,7 @@ def local_extra_hook(entities): if multiple: result = entities else: - if entities[0] is None: + if not entities or entities[0] is None: raise datastore_errors.EntityNotFoundError() result = entities[0] if extra_hook: diff --git a/AppServer/google/appengine/api/datastore_distributed.py b/AppServer/google/appengine/api/datastore_distributed.py index de8a69160e..eff5c5fc9e 100755 --- a/AppServer/google/appengine/api/datastore_distributed.py +++ b/AppServer/google/appengine/api/datastore_distributed.py @@ -362,13 +362,6 @@ def _Dynamic_RunQuery(self, query, query_result): results = query_response.result_list() results = [datastore.Entity._FromPb(r) for r in results] - if query.has_ancestor(): - ancestor_path = query.ancestor().path().element_list() - def is_descendant(entity): - path = entity.key()._Key__reference.path().element_list() - return path[:len(ancestor_path)] == ancestor_path - results = filter(is_descendant, results) - operators = {datastore_pb.Query_Filter.LESS_THAN: '<', datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL: '<=', datastore_pb.Query_Filter.GREATER_THAN: '>', @@ -393,20 +386,6 @@ def has_prop_indexed(entity, prop): return True return False - for filt in filters: - assert filt.op() != datastore_pb.Query_Filter.IN - - prop = filt.property(0).name().decode('utf-8') - op = operators[filt.op()] - - filter_val_list = [datastore_types.FromPropertyPb(filter_prop) - for filter_prop in filt.property_list()] - - - for order in orders: - prop = order.property().decode('utf-8') - results = [entity for entity in results if has_prop_indexed(entity, prop)] - def order_compare_entities(a, b): """ Return a negative, zero or positive number depending on whether entity a is considered smaller than, equal to, or larger than b, @@ -467,17 +446,11 @@ def order_compare_properties(x, y): else: return cmp(x_type, y_type) - results.sort(order_compare_entities) - clone = datastore_pb.Query() clone.CopyFrom(query) clone.clear_hint() clone.clear_limit() clone.clear_offset() - #if clone in self.__query_history: - # self.__query_history[clone] += 1 - #else: - # self.__query_history[clone] = 1 results = [r._ToPb() for r in results] for result in results: @@ -488,7 +461,7 @@ def order_compare_properties(x, y): cursor = datastore_stub_util.ListCursor(query, results, order_compare_entities_pb) - self.__queries[cursor.cursor] = cursor + self.__queries = cursor if query.has_count(): count = query.count() @@ -510,15 +483,14 @@ def _Dynamic_Next(self, next_request, query_result): self.__ValidateAppId(next_request.cursor().app()) cursor_handle = next_request.cursor().cursor() - - try: - cursor = self.__queries[cursor_handle] - except KeyError: + + cursor = self.__queries + if cursor.cursor != cursor_handle: raise apiproxy_errors.ApplicationError( - datastore_pb.Error.BAD_REQUEST, 'Cursor %d not found' % cursor_handle) + datastore_pb.Error.BAD_REQUEST, 'Cursor %d not found' % cursor_handle) assert cursor.app == next_request.cursor().app() - + logging.info(str(cursor)) count = _BATCH_SIZE if next_request.has_count(): count = next_request.count() diff --git a/AppServer/google/appengine/api/labs/taskqueue/__init__.py b/AppServer/google/appengine/api/labs/taskqueue/__init__.py deleted file mode 100644 index 1e8fb50ba9..0000000000 --- a/AppServer/google/appengine/api/labs/taskqueue/__init__.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2007 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - - - -"""Task Queue API module (labs compatibility).""" - - - - - - - -import os -import warnings - -from taskqueue import * - - - - -__all__ = [ - - 'BadTaskStateError', 'BadTransactionState', 'BadTransactionStateError', - 'DatastoreError', 'DuplicateTaskNameError', 'Error', 'InternalError', - 'InvalidQueueError', 'InvalidQueueNameError', 'InvalidTaskError', - 'InvalidTaskNameError', 'InvalidUrlError', 'PermissionDeniedError', - 'TaskAlreadyExistsError', 'TaskTooLargeError', 'TombstonedTaskError', - 'TooManyTasksError', 'TransientError', 'UnknownQueueError', - - 'MAX_QUEUE_NAME_LENGTH', 'MAX_TASK_NAME_LENGTH', 'MAX_TASK_SIZE_BYTES', - 'MAX_URL_LENGTH', - - 'Queue', 'Task', 'add'] - - - -if os.environ.get('DATACENTER', None) is None: - warnings.warn('google.appengine.api.labs.taskqueue is deprecated, please use ' - 'google.appengine.api.taskqueue', DeprecationWarning, - stacklevel=2) diff --git a/AppServer/google/appengine/api/labs/taskqueue/__init__.py b/AppServer/google/appengine/api/labs/taskqueue/__init__.py new file mode 120000 index 0000000000..95c83cc039 --- /dev/null +++ b/AppServer/google/appengine/api/labs/taskqueue/__init__.py @@ -0,0 +1 @@ +../../taskqueue/__init__.py \ No newline at end of file diff --git a/AppServer/google/appengine/api/labs/taskqueue/taskqueue.py b/AppServer/google/appengine/api/labs/taskqueue/taskqueue.py deleted file mode 100644 index 1dcb06184e..0000000000 --- a/AppServer/google/appengine/api/labs/taskqueue/taskqueue.py +++ /dev/null @@ -1,1042 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2007 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - - - -"""Task Queue API. - -Enables an application to queue background work for itself. Work is done through -webhooks that process tasks pushed from a queue. Tasks will execute in -best-effort order of ETA. Webhooks that fail will cause tasks to be retried at a -later time. Multiple queues may exist with independent throttling controls. - -Webhook URLs may be specified directly for Tasks, or the default URL scheme -may be used, which will translate Task names into URLs relative to a Queue's -base path. A default queue is also provided for simple usage. -""" - - - - - - - - -__all__ = [ - - 'BadTaskStateError', 'BadTransactionState', 'BadTransactionStateError', - 'DatastoreError', 'DuplicateTaskNameError', 'Error', 'InternalError', - 'InvalidQueueError', 'InvalidQueueNameError', 'InvalidTaskError', - 'InvalidTaskNameError', 'InvalidUrlError', 'PermissionDeniedError', - 'TaskAlreadyExistsError', 'TaskTooLargeError', 'TombstonedTaskError', - 'TooManyTasksError', 'TransientError', 'UnknownQueueError', - - 'MAX_QUEUE_NAME_LENGTH', 'MAX_TASK_NAME_LENGTH', 'MAX_TASK_SIZE_BYTES', - 'MAX_URL_LENGTH', - - 'Queue', 'Task', 'TaskRetryOptions', 'add'] - - -import calendar -import datetime -import math -import os -import re -import time -import urllib -import urlparse - -from google.appengine.api import apiproxy_stub_map -from google.appengine.api import namespace_manager -from google.appengine.api import urlfetch -from google.appengine.api.labs.taskqueue import taskqueue_service_pb -from google.appengine.runtime import apiproxy_errors - - -class Error(Exception): - """Base-class for exceptions in this module.""" - - -class UnknownQueueError(Error): - """The queue specified is unknown.""" - - -class TransientError(Error): - """There was a transient error while accessing the queue. - - Please Try again later. - """ - - -class InternalError(Error): - """There was an internal error while accessing this queue. - - If this problem continues, please contact the App Engine team through - our support forum with a description of your problem. - """ - - -class InvalidTaskError(Error): - """The task's parameters, headers, or method is invalid.""" - - -class InvalidTaskNameError(InvalidTaskError): - """The task's name is invalid.""" - - -class TaskTooLargeError(InvalidTaskError): - """The task is too large with its headers and payload.""" - - -class TaskAlreadyExistsError(InvalidTaskError): - """Task already exists. It has not yet run.""" - - -class TombstonedTaskError(InvalidTaskError): - """Task has been tombstoned.""" - - -class InvalidUrlError(InvalidTaskError): - """The task's relative URL is invalid.""" - - -class BadTaskStateError(Error): - """The task is in the wrong state for the requested operation.""" - - -class InvalidQueueError(Error): - """The Queue's configuration is invalid.""" - - -class InvalidQueueNameError(InvalidQueueError): - """The Queue's name is invalid.""" - - -class _RelativeUrlError(Error): - """The relative URL supplied is invalid.""" - - -class PermissionDeniedError(Error): - """The requested operation is not allowed for this app.""" - - -class DuplicateTaskNameError(Error): - """The add arguments contain tasks with identical names.""" - - -class TooManyTasksError(Error): - """Too many tasks were present in a single function call.""" - - -class DatastoreError(Error): - """There was a datastore error while accessing the queue.""" - - -class BadTransactionStateError(Error): - """The state of the current transaction does not permit this operation.""" - - -class InvalidTaskRetryOptionsError(Error): - """The task retry configuration is invalid.""" - - - -BadTransactionState = BadTransactionStateError - -MAX_QUEUE_NAME_LENGTH = 100 - -MAX_TASK_NAME_LENGTH = 500 - -MAX_TASK_SIZE_BYTES = 10 * (2 ** 10) - -MAX_TASKS_PER_ADD = 100 - - -MAX_URL_LENGTH = 2083 - -_DEFAULT_QUEUE = 'default' - -_DEFAULT_QUEUE_PATH = '/_ah/queue' - -_METHOD_MAP = { - 'GET': taskqueue_service_pb.TaskQueueAddRequest.GET, - 'POST': taskqueue_service_pb.TaskQueueAddRequest.POST, - 'HEAD': taskqueue_service_pb.TaskQueueAddRequest.HEAD, - 'PUT': taskqueue_service_pb.TaskQueueAddRequest.PUT, - 'DELETE': taskqueue_service_pb.TaskQueueAddRequest.DELETE, -} - -_NON_POST_METHODS = frozenset(['GET', 'HEAD', 'PUT', 'DELETE']) - -_BODY_METHODS = frozenset(['POST', 'PUT']) - -_TASK_NAME_PATTERN = r'^[a-zA-Z0-9-]{1,%s}$' % MAX_TASK_NAME_LENGTH - -_TASK_NAME_RE = re.compile(_TASK_NAME_PATTERN) - -_QUEUE_NAME_PATTERN = r'^[a-zA-Z0-9-]{1,%s}$' % MAX_QUEUE_NAME_LENGTH - -_QUEUE_NAME_RE = re.compile(_QUEUE_NAME_PATTERN) - -_ERROR_MAPPING = { - taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_QUEUE: UnknownQueueError, - taskqueue_service_pb.TaskQueueServiceError.TRANSIENT_ERROR: - TransientError, - taskqueue_service_pb.TaskQueueServiceError.INTERNAL_ERROR: InternalError, - taskqueue_service_pb.TaskQueueServiceError.TASK_TOO_LARGE: - TaskTooLargeError, - taskqueue_service_pb.TaskQueueServiceError.INVALID_TASK_NAME: - InvalidTaskNameError, - taskqueue_service_pb.TaskQueueServiceError.INVALID_QUEUE_NAME: - InvalidQueueNameError, - taskqueue_service_pb.TaskQueueServiceError.INVALID_URL: InvalidUrlError, - taskqueue_service_pb.TaskQueueServiceError.INVALID_QUEUE_RATE: - InvalidQueueError, - taskqueue_service_pb.TaskQueueServiceError.PERMISSION_DENIED: - PermissionDeniedError, - taskqueue_service_pb.TaskQueueServiceError.TASK_ALREADY_EXISTS: - TaskAlreadyExistsError, - taskqueue_service_pb.TaskQueueServiceError.TOMBSTONED_TASK: - TombstonedTaskError, - taskqueue_service_pb.TaskQueueServiceError.INVALID_ETA: InvalidTaskError, - taskqueue_service_pb.TaskQueueServiceError.INVALID_REQUEST: Error, - taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_TASK: Error, - taskqueue_service_pb.TaskQueueServiceError.TOMBSTONED_QUEUE: Error, - taskqueue_service_pb.TaskQueueServiceError.DUPLICATE_TASK_NAME: - DuplicateTaskNameError, - - taskqueue_service_pb.TaskQueueServiceError.TOO_MANY_TASKS: - TooManyTasksError, - -} - - - - - - - -_PRESERVE_ENVIRONMENT_HEADERS = ( - ('X-AppEngine-Default-Namespace', 'HTTP_X_APPENGINE_DEFAULT_NAMESPACE'),) - - - -class _UTCTimeZone(datetime.tzinfo): - """UTC timezone.""" - - ZERO = datetime.timedelta(0) - - def utcoffset(self, dt): - return self.ZERO - - def dst(self, dt): - return self.ZERO - - def tzname(self, dt): - return 'UTC' - - -_UTC = _UTCTimeZone() - - -def _parse_relative_url(relative_url): - """Parses a relative URL and splits it into its path and query string. - - Args: - relative_url: The relative URL, starting with a '/'. - - Returns: - Tuple (path, query) where: - path: The path in the relative URL. - query: The query string in the URL without the '?' character. - - Raises: - _RelativeUrlError if the relative_url is invalid for whatever reason - """ - if not relative_url: - raise _RelativeUrlError('Relative URL is empty') - (scheme, netloc, path, query, fragment) = urlparse.urlsplit(relative_url) - if scheme or netloc: - raise _RelativeUrlError('Relative URL may not have a scheme or location') - if fragment: - raise _RelativeUrlError('Relative URL may not specify a fragment') - if not path or path[0] != '/': - raise _RelativeUrlError('Relative URL path must start with "/"') - return path, query - - -def _flatten_params(params): - """Converts a dictionary of parameters to a list of parameters. - - Any unicode strings in keys or values will be encoded as UTF-8. - - Args: - params: Dictionary mapping parameter keys to values. Values will be - converted to a string and added to the list as tuple (key, value). If - a values is iterable and not a string, each contained value will be - added as a separate (key, value) tuple. - - Returns: - List of (key, value) tuples. - """ - def get_string(value): - if isinstance(value, unicode): - return unicode(value).encode('utf-8') - else: - - - - - return str(value) - - param_list = [] - for key, value in params.iteritems(): - key = get_string(key) - if isinstance(value, basestring): - param_list.append((key, get_string(value))) - else: - try: - iterator = iter(value) - except TypeError: - param_list.append((key, str(value))) - else: - param_list.extend((key, get_string(v)) for v in iterator) - - return param_list - - -class TaskRetryOptions(object): - """The options used to decide when a failed Task will be retried.""" - - __CONSTRUCTOR_KWARGS = frozenset( - ['min_backoff_seconds', 'max_backoff_seconds', - 'task_age_limit', 'max_doublings', 'task_retry_limit']) - - def __init__(self, **kwargs): - """Initializer. - - Args: - min_backoff_seconds: The minimum number of seconds to wait before retrying - a task after failure. (optional) - max_backoff_seconds: The maximum number of seconds to wait before retrying - a task after failure. (optional) - task_age_limit: The number of seconds after creation afterwhich a failed - task will no longer be retried. The given value will be rounded up to - the nearest integer. If task_retry_limit is also specified then the task - will be retried until both limits are reached. (optional) - max_doublings: The maximum number of times that the interval between - failed task retries will be doubled before the increase becomes - constant. The constant will be: - 2**(max_doublings - 1) * min_backoff_seconds. (optional) - task_retry_limit: The maximum number of times to retry a failed task - before giving up. If task_age_limit is specified then the task will be - retried until both limits are reached. (optional) - - Raises: - InvalidTaskRetryOptionsError if any of the parameters are invalid. - """ - args_diff = set(kwargs.iterkeys()) - self.__CONSTRUCTOR_KWARGS - if args_diff: - raise TypeError('Invalid arguments: %s' % ', '.join(args_diff)) - - self.__min_backoff_seconds = kwargs.get('min_backoff_seconds') - if (self.__min_backoff_seconds is not None and - self.__min_backoff_seconds < 0): - raise InvalidTaskRetryOptionsError( - 'The minimum retry interval cannot be negative') - - self.__max_backoff_seconds = kwargs.get('max_backoff_seconds') - if (self.__max_backoff_seconds is not None and - self.__max_backoff_seconds < 0): - raise InvalidTaskRetryOptionsError( - 'The maximum retry interval cannot be negative') - - if (self.__min_backoff_seconds is not None and - self.__max_backoff_seconds is not None and - self.__max_backoff_seconds < self.__min_backoff_seconds): - raise InvalidTaskRetryOptionsError( - 'The maximum retry interval cannot be less than the ' - 'minimum retry interval') - - self.__max_doublings = kwargs.get('max_doublings') - if self.__max_doublings is not None and self.__max_doublings < 0: - raise InvalidTaskRetryOptionsError( - 'The maximum number of retry interval doublings cannot be negative') - - self.__task_retry_limit = kwargs.get('task_retry_limit') - if self.__task_retry_limit is not None and self.__task_retry_limit < 0: - raise InvalidTaskRetryOptionsError( - 'The maximum number of retries cannot be negative') - - self.__task_age_limit = kwargs.get('task_age_limit') - if self.__task_age_limit is not None: - if self.__task_age_limit < 0: - raise InvalidTaskRetryOptionsError( - 'The expiry countdown cannot be negative') - self.__task_age_limit = int(math.ceil(self.__task_age_limit)) - - @property - def min_backoff_seconds(self): - """The minimum number of seconds to wait before retrying a task.""" - return self.__min_backoff_seconds - - @property - def max_backoff_seconds(self): - """The maximum number of seconds to wait before retrying a task.""" - return self.__max_backoff_seconds - - @property - def task_age_limit(self): - """The number of seconds afterwhich a failed task will not be retried.""" - return self.__task_age_limit - - @property - def max_doublings(self): - """The number of times that the retry interval will be doubled.""" - return self.__max_doublings - - @property - def task_retry_limit(self): - """The number of times that a failed task will be retried.""" - return self.__task_retry_limit - - -class Task(object): - """Represents a single Task on a queue.""" - - - __CONSTRUCTOR_KWARGS = frozenset([ - 'countdown', 'eta', 'headers', 'method', 'name', 'params', - 'retry_options', 'url']) - - - __eta_posix = None - - def __init__(self, payload=None, **kwargs): - """Initializer. - - All parameters are optional. - - Args: - payload: The payload data for this Task that will be delivered to the - webhook as the HTTP request body. This is only allowed for POST and PUT - methods. - countdown: Time in seconds into the future that this Task should execute. - Defaults to zero. - eta: Absolute time when the Task should execute. May not be specified - if 'countdown' is also supplied. This may be timezone-aware or - timezone-naive. - headers: Dictionary of headers to pass to the webhook. Values in the - dictionary may be iterable to indicate repeated header fields. - method: Method to use when accessing the webhook. Defaults to 'POST'. - name: Name to give the Task; if not specified, a name will be - auto-generated when added to a queue and assigned to this object. Must - match the _TASK_NAME_PATTERN regular expression. - params: Dictionary of parameters to use for this Task. For POST requests - these params will be encoded as 'application/x-www-form-urlencoded' and - set to the payload. For all other methods, the parameters will be - converted to a query string. May not be specified if the URL already - contains a query string. - url: Relative URL where the webhook that should handle this task is - located for this application. May have a query string unless this is - a POST method. - retry_options: TaskRetryOptions used to control when the task will be - retried if it fails. - - Raises: - InvalidTaskError if any of the parameters are invalid; - InvalidTaskNameError if the task name is invalid; InvalidUrlError if - the task URL is invalid or too long; TaskTooLargeError if the task with - its payload is too large. - """ - args_diff = set(kwargs.iterkeys()) - self.__CONSTRUCTOR_KWARGS - if args_diff: - raise TypeError('Invalid arguments: %s' % ', '.join(args_diff)) - - self.__name = kwargs.get('name') - if self.__name and not _TASK_NAME_RE.match(self.__name): - raise InvalidTaskNameError( - 'Task name does not match expression "%s"; found %s' % - (_TASK_NAME_PATTERN, self.__name)) - - self.__default_url, self.__relative_url, query = Task.__determine_url( - kwargs.get('url', '')) - self.__headers = urlfetch._CaselessDict() - self.__headers.update(kwargs.get('headers', {})) - self.__method = kwargs.get('method', 'POST').upper() - self.__payload = None - params = kwargs.get('params', {}) - - - for header_name, environ_name in _PRESERVE_ENVIRONMENT_HEADERS: - value = os.environ.get(environ_name) - if value is not None: - self.__headers.setdefault(header_name, value) - - self.__headers.setdefault('X-AppEngine-Current-Namespace', - namespace_manager.get_namespace()) - if query and params: - raise InvalidTaskError('Query string and parameters both present; ' - 'only one of these may be supplied') - - if self.__method == 'POST': - if payload and params: - raise InvalidTaskError('Message body and parameters both present for ' - 'POST method; only one of these may be supplied') - elif query: - raise InvalidTaskError('POST method may not have a query string; ' - 'use the "params" keyword argument instead') - elif params: - self.__payload = Task.__encode_params(params) - self.__headers.setdefault( - 'content-type', 'application/x-www-form-urlencoded') - elif payload is not None: - self.__payload = Task.__convert_payload(payload, self.__headers) - elif self.__method in _NON_POST_METHODS: - if payload and self.__method not in _BODY_METHODS: - raise InvalidTaskError('Payload may only be specified for methods %s' % - ', '.join(_BODY_METHODS)) - if payload: - self.__payload = Task.__convert_payload(payload, self.__headers) - if params: - query = Task.__encode_params(params) - if query: - self.__relative_url = '%s?%s' % (self.__relative_url, query) - else: - raise InvalidTaskError('Invalid method: %s' % self.__method) - - self.__headers_list = _flatten_params(self.__headers) - self.__eta_posix = Task.__determine_eta_posix( - kwargs.get('eta'), kwargs.get('countdown')) - self.__eta = None - self.__retry_options = kwargs.get('retry_options') - self.__enqueued = False - - if self.size > MAX_TASK_SIZE_BYTES: - raise TaskTooLargeError('Task size must be less than %d; found %d' % - (MAX_TASK_SIZE_BYTES, self.size)) - - @staticmethod - def __determine_url(relative_url): - """Determines the URL of a task given a relative URL and a name. - - Args: - relative_url: The relative URL for the Task. - - Returns: - Tuple (default_url, relative_url, query) where: - default_url: True if this Task is using the default URL scheme; - False otherwise. - relative_url: String containing the relative URL for this Task. - query: The query string for this task. - - Raises: - InvalidUrlError if the relative_url is invalid. - """ - if not relative_url: - default_url, query = True, '' - else: - default_url = False - try: - relative_url, query = _parse_relative_url(relative_url) - except _RelativeUrlError, e: - raise InvalidUrlError(e) - - if len(relative_url) > MAX_URL_LENGTH: - raise InvalidUrlError( - 'Task URL must be less than %d characters; found %d' % - (MAX_URL_LENGTH, len(relative_url))) - - return (default_url, relative_url, query) - - @staticmethod - def __determine_eta_posix(eta=None, countdown=None, current_time=time.time): - """Determines the ETA for a task. - - If 'eta' and 'countdown' are both None, the current time will be used. - Otherwise, only one of them may be specified. - - Args: - eta: A datetime.datetime specifying the absolute ETA or None; - this may be timezone-aware or timezone-naive. - countdown: Count in seconds into the future from the present time that - the ETA should be assigned to. - - Returns: - A float giving a POSIX timestamp containing the ETA. - - Raises: - InvalidTaskError if the parameters are invalid. - """ - if eta is not None and countdown is not None: - raise InvalidTaskError('May not use a countdown and ETA together') - elif eta is not None: - if not isinstance(eta, datetime.datetime): - raise InvalidTaskError('ETA must be a datetime.datetime instance') - elif eta.tzinfo is None: - - return time.mktime(eta.timetuple()) + eta.microsecond*1e-6 - else: - - return calendar.timegm(eta.utctimetuple()) + eta.microsecond*1e-6 - elif countdown is not None: - try: - countdown = float(countdown) - except ValueError: - raise InvalidTaskError('Countdown must be a number') - except OverflowError: - raise InvalidTaskError('Countdown out of range') - else: - return current_time() + countdown - else: - return current_time() - - @staticmethod - def __encode_params(params): - """URL-encodes a list of parameters. - - Args: - params: Dictionary of parameters, possibly with iterable values. - - Returns: - URL-encoded version of the params, ready to be added to a query string or - POST body. - """ - return urllib.urlencode(_flatten_params(params)) - - @staticmethod - def __convert_payload(payload, headers): - """Converts a Task payload into UTF-8 and sets headers if necessary. - - Args: - payload: The payload data to convert. - headers: Dictionary of headers. - - Returns: - The payload as a non-unicode string. - - Raises: - InvalidTaskError if the payload is not a string or unicode instance. - """ - if isinstance(payload, unicode): - headers.setdefault('content-type', 'text/plain; charset=utf-8') - payload = payload.encode('utf-8') - elif not isinstance(payload, str): - raise InvalidTaskError( - 'Task payloads must be strings; invalid payload: %r' % payload) - return payload - - @property - def on_queue_url(self): - """Returns True if this Task will run on the queue's URL.""" - return self.__default_url - - @property - def eta_posix(self): - """Returns a POSIX timestamp giving when this Task will execute.""" - if self.__eta_posix is None and self.__eta is not None: - - self.__eta_posix = Task.__determine_eta_posix(self.__eta) - return self.__eta_posix - - @property - def eta(self): - """Returns a datetime when this Task will execute.""" - if self.__eta is None and self.__eta_posix is not None: - self.__eta = datetime.datetime.fromtimestamp(self.__eta_posix, _UTC) - return self.__eta - - @property - def headers(self): - """Returns a copy of the headers for this Task.""" - return self.__headers.copy() - - @property - def method(self): - """Returns the method to use for this Task.""" - return self.__method - - @property - def name(self): - """Returns the name of this Task. - - Will be None if using auto-assigned Task names and this Task has not yet - been added to a Queue. - """ - return self.__name - - @property - def payload(self): - """Returns the payload for this task, which may be None.""" - return self.__payload - - @property - def size(self): - """Returns the size of this task in bytes.""" - HEADER_SEPERATOR = len(': \r\n') - header_size = sum((len(key) + len(value) + HEADER_SEPERATOR) - for key, value in self.__headers_list) - return (len(self.__method) + len(self.__payload or '') + - len(self.__relative_url) + header_size) - - @property - def url(self): - """Returns the relative URL for this Task.""" - return self.__relative_url - - @property - def retry_options(self): - """Returns the TaskRetryOptions for this task, which may be None.""" - return self.__retry_options - - @property - def was_enqueued(self): - """Returns True if this Task has been enqueued. - - Note: This will not check if this task already exists in the queue. - """ - return self.__enqueued - - def add(self, queue_name=_DEFAULT_QUEUE, transactional=False): - """Adds this Task to a queue. See Queue.add.""" - return Queue(queue_name).add(self, transactional=transactional) - - -class Queue(object): - """Represents a Queue.""" - - def __init__(self, name=_DEFAULT_QUEUE): - """Initializer. - - Args: - name: Name of this queue. If not supplied, defaults to the default queue. - - Raises: - InvalidQueueNameError if the queue name is invalid. - """ - - - if not _QUEUE_NAME_RE.match(name): - raise InvalidQueueNameError( - 'Queue name does not match pattern "%s"; found %s' % - (_QUEUE_NAME_PATTERN, name)) - self.__name = name - self.__url = '%s/%s' % (_DEFAULT_QUEUE_PATH, self.__name) - - - - - - self._app = None - - def purge(self): - """Removes all the tasks in this Queue. - - This function takes constant time to purge a Queue and some delay may apply - before the call is effective. - - Raises: - UnknownQueueError if the Queue does not exist on server side. - """ - request = taskqueue_service_pb.TaskQueuePurgeQueueRequest() - response = taskqueue_service_pb.TaskQueuePurgeQueueResponse() - - request.set_queue_name(self.__name) - if self._app: - request.set_app_id(self._app) - - try: - apiproxy_stub_map.MakeSyncCall('taskqueue', - 'PurgeQueue', - request, - response) - except apiproxy_errors.ApplicationError, e: - raise self.__TranslateError(e.application_error, e.error_detail) - - def add(self, task, transactional=False): - """Adds a Task or list of Tasks to this Queue. - - If a list of more than one Tasks is given, a raised exception does not - guarantee that no tasks were added to the queue (unless transactional is set - to True). To determine which tasks were successfully added when an exception - is raised, check the Task.was_enqueued property. - - Args: - task: A Task instance or a list of Task instances that will added to the - queue. - transactional: If False adds the Task(s) to a queue irrespectively to the - enclosing transaction success or failure. An exception is raised if True - and called outside of a transaction. (optional) - - Returns: - The Task or list of tasks that was supplied to this method. - - Raises: - BadTaskStateError: if the Task(s) has already been added to a queue. - BadTransactionStateError: if the transactional argument is true but this - call is being made outside of the context of a transaction. - Error-subclass on application errors. - """ - try: - tasks = list(iter(task)) - except TypeError: - tasks = [task] - multiple = False - else: - multiple = True - - self.__AddTasks(tasks, transactional) - - if multiple: - return tasks - else: - assert len(tasks) == 1 - return tasks[0] - - def __AddTasks(self, tasks, transactional): - """Internal implementation of .add() where tasks must be a list.""" - - if len(tasks) > MAX_TASKS_PER_ADD: - raise TooManyTasksError( - 'No more than %d tasks can be added in a single add call' % - MAX_TASKS_PER_ADD) - - request = taskqueue_service_pb.TaskQueueBulkAddRequest() - response = taskqueue_service_pb.TaskQueueBulkAddResponse() - - task_names = set() - for task in tasks: - if task.name: - if task.name in task_names: - raise DuplicateTaskNameError( - 'The task name %r is used more than once in the request' % - task.name) - task_names.add(task.name) - - self.__FillAddRequest(task, request.add_add_request(), transactional) - - try: - apiproxy_stub_map.MakeSyncCall('taskqueue', 'BulkAdd', request, response) - except apiproxy_errors.ApplicationError, e: - raise self.__TranslateError(e.application_error, e.error_detail) - - assert response.taskresult_size() == len(tasks), ( - 'expected %d results from BulkAdd(), got %d' % ( - len(tasks), response.taskresult_size())) - - exception = None - for task, task_result in zip(tasks, response.taskresult_list()): - if task_result.result() == taskqueue_service_pb.TaskQueueServiceError.OK: - if task_result.has_chosen_task_name(): - task._Task__name = task_result.chosen_task_name() - task._Task__enqueued = True - elif (task_result.result() == - taskqueue_service_pb.TaskQueueServiceError.SKIPPED): - pass - elif exception is None: - exception = self.__TranslateError(task_result.result()) - - if exception is not None: - raise exception - - return tasks - - def __FillTaskQueueRetryParameters(self, - retry_options, - retry_retry_parameters): - """Populates a TaskQueueRetryParameters with data from a TaskRetryOptions. - - Args: - retry_options: The TaskRetryOptions instance to use as a source for the - data to be added to retry_retry_parameters. - retry_retry_parameters: A taskqueue_service_pb.TaskQueueRetryParameters - to populate. - """ - if retry_options.min_backoff_seconds is not None: - retry_retry_parameters.set_min_backoff_sec( - retry_options.min_backoff_seconds) - - if retry_options.max_backoff_seconds is not None: - retry_retry_parameters.set_max_backoff_sec( - retry_options.max_backoff_seconds) - - if retry_options.task_retry_limit is not None: - retry_retry_parameters.set_retry_limit(retry_options.task_retry_limit) - - if retry_options.task_age_limit is not None: - retry_retry_parameters.set_age_limit_sec(retry_options.task_age_limit) - - if retry_options.max_doublings is not None: - retry_retry_parameters.set_max_doublings(retry_options.max_doublings) - - def __FillAddRequest(self, task, task_request, transactional): - """Populates a TaskQueueAddRequest with the data from a Task instance. - - Args: - task: The Task instance to use as a source for the data to be added to - task_request. - task_request: The taskqueue_service_pb.TaskQueueAddRequest to populate. - transactional: If true then populates the task_request.transaction message - with information from the enclosing transaction (if any). - - Raises: - BadTaskStateError: If the task was already added to a Queue. - BadTransactionStateError: If the transactional argument is True and there - is no enclosing transaction. - InvalidTaskNameError: If the transactional argument is True and the task - is named. - """ - if task.was_enqueued: - raise BadTaskStateError('Task has already been enqueued') - - adjusted_url = task.url - if task.on_queue_url: - adjusted_url = self.__url + task.url - - - - - - - - - task_request.set_queue_name(self.__name) - task_request.set_eta_usec(long(task.eta_posix * 1e6)) - task_request.set_method(_METHOD_MAP.get(task.method)) - task_request.set_url(adjusted_url) - - if task.name: - task_request.set_task_name(task.name) - else: - task_request.set_task_name('') - - if task.payload: - task_request.set_body(task.payload) - for key, value in _flatten_params(task.headers): - header = task_request.add_header() - header.set_key(key) - header.set_value(value) - - if task.retry_options: - self.__FillTaskQueueRetryParameters( - task.retry_options, task_request.mutable_retry_parameters()) - - if self._app: - task_request.set_app_id(self._app) - - - - if transactional: - from google.appengine.api import datastore - if not datastore._MaybeSetupTransaction(task_request, []): - raise BadTransactionStateError( - 'Transactional adds are not allowed outside of transactions') - - if task_request.has_transaction() and task.name: - raise InvalidTaskNameError( - 'Task bound to a transaction cannot be named.') - - @property - def name(self): - """Returns the name of this queue.""" - return self.__name - - @staticmethod - def __TranslateError(error, detail=''): - """Translates a TaskQueueServiceError into an exception. - - Args: - error: Value from TaskQueueServiceError enum. - detail: A human-readable description of the error. - - Returns: - The corresponding Exception sub-class for that error code. - """ - if (error >= taskqueue_service_pb.TaskQueueServiceError.DATASTORE_ERROR - and isinstance(error, int)): - from google.appengine.api import datastore - datastore_exception = datastore._DatastoreExceptionFromErrorCodeAndDetail( - error - taskqueue_service_pb.TaskQueueServiceError.DATASTORE_ERROR, - detail) - - class JointException(datastore_exception.__class__, DatastoreError): - """There was a datastore error while accessing the queue.""" - __msg = (u'taskqueue.DatastoreError caused by: %s %s' % - (datastore_exception.__class__, detail)) - def __str__(self): - return JointException.__msg - - return JointException() - else: - exception_class = _ERROR_MAPPING.get(error, None) - if exception_class: - return exception_class(detail) - else: - return Error('Application error %s: %s' % (error, detail)) - - - -def add(*args, **kwargs): - """Convenience method will create a Task and add it to a queue. - - All parameters are optional. - - Args: - name: Name to give the Task; if not specified, a name will be - auto-generated when added to a queue and assigned to this object. Must - match the _TASK_NAME_PATTERN regular expression. - queue_name: Name of this queue. If not supplied, defaults to - the default queue. - url: Relative URL where the webhook that should handle this task is - located for this application. May have a query string unless this is - a POST method. - method: Method to use when accessing the webhook. Defaults to 'POST'. - headers: Dictionary of headers to pass to the webhook. Values in the - dictionary may be iterable to indicate repeated header fields. - payload: The payload data for this Task that will be delivered to the - webhook as the HTTP request body. This is only allowed for POST and PUT - methods. - params: Dictionary of parameters to use for this Task. For POST requests - these params will be encoded as 'application/x-www-form-urlencoded' and - set to the payload. For all other methods, the parameters will be - converted to a query string. May not be specified if the URL already - contains a query string. - transactional: If False adds the Task(s) to a queue irrespectively to the - enclosing transaction success or failure. An exception is raised if True - and called outside of a transaction. (optional) - countdown: Time in seconds into the future that this Task should execute. - Defaults to zero. - eta: Absolute time when the Task should execute. May not be specified - if 'countdown' is also supplied. This may be timezone-aware or - timezone-naive. - retry_options: TaskRetryOptions used to control when the task will be - retried if it fails. - - Returns: - The Task that was added to the queue. - - Raises: - InvalidTaskError if any of the parameters are invalid; - InvalidTaskNameError if the task name is invalid; InvalidUrlError if - the task URL is invalid or too long; TaskTooLargeError if the task with - its payload is too large. - """ - transactional = kwargs.pop('transactional', False) - queue_name = kwargs.pop('queue_name', _DEFAULT_QUEUE) - return Task(*args, **kwargs).add( - queue_name=queue_name, transactional=transactional) diff --git a/AppServer/google/appengine/api/labs/taskqueue/taskqueue.py b/AppServer/google/appengine/api/labs/taskqueue/taskqueue.py new file mode 120000 index 0000000000..f46ed7658b --- /dev/null +++ b/AppServer/google/appengine/api/labs/taskqueue/taskqueue.py @@ -0,0 +1 @@ +../../taskqueue/taskqueue.py \ No newline at end of file diff --git a/AppServer/google/appengine/api/labs/taskqueue/taskqueue_rabbitmq.py b/AppServer/google/appengine/api/labs/taskqueue/taskqueue_rabbitmq.py new file mode 120000 index 0000000000..9e92aed442 --- /dev/null +++ b/AppServer/google/appengine/api/labs/taskqueue/taskqueue_rabbitmq.py @@ -0,0 +1 @@ +../../taskqueue/taskqueue_rabbitmq.py \ No newline at end of file diff --git a/AppServer/google/appengine/api/labs/taskqueue/taskqueue_service_pb.py b/AppServer/google/appengine/api/labs/taskqueue/taskqueue_service_pb.py deleted file mode 100644 index 803f3731ac..0000000000 --- a/AppServer/google/appengine/api/labs/taskqueue/taskqueue_service_pb.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2007 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - - - -"""A shim to access the new taskqueue_service_pb module. - -This contains all of the protocol messages as of 1.4.2. -""" - - -from google.appengine.api.taskqueue import taskqueue_service_pb - - -TaskQueueServiceError = taskqueue_service_pb.TaskQueueServiceError -TaskQueueRetryParameters = taskqueue_service_pb.TaskQueueRetryParameters -TaskQueueAddRequest_Header = taskqueue_service_pb.TaskQueueAddRequest_Header -TaskQueueAddRequest_CronTimetable = taskqueue_service_pb.TaskQueueAddRequest_CronTimetable -TaskQueueAddRequest = taskqueue_service_pb.TaskQueueAddRequest -TaskQueueAddResponse = taskqueue_service_pb.TaskQueueAddResponse -TaskQueueBulkAddRequest = taskqueue_service_pb.TaskQueueBulkAddRequest -TaskQueueBulkAddResponse_TaskResult = taskqueue_service_pb.TaskQueueBulkAddResponse_TaskResult -TaskQueueBulkAddResponse = taskqueue_service_pb.TaskQueueBulkAddResponse -TaskQueueDeleteRequest = taskqueue_service_pb.TaskQueueDeleteRequest -TaskQueueDeleteResponse = taskqueue_service_pb.TaskQueueDeleteResponse -TaskQueueForceRunRequest = taskqueue_service_pb.TaskQueueForceRunRequest -TaskQueueForceRunResponse = taskqueue_service_pb.TaskQueueForceRunResponse -TaskQueueUpdateQueueRequest = taskqueue_service_pb.TaskQueueUpdateQueueRequest -TaskQueueUpdateQueueResponse = taskqueue_service_pb.TaskQueueUpdateQueueResponse -TaskQueueFetchQueuesRequest = taskqueue_service_pb.TaskQueueFetchQueuesRequest -TaskQueueFetchQueuesResponse_Queue = taskqueue_service_pb.TaskQueueFetchQueuesResponse_Queue -TaskQueueFetchQueuesResponse = taskqueue_service_pb.TaskQueueFetchQueuesResponse -TaskQueueFetchQueueStatsRequest = taskqueue_service_pb.TaskQueueFetchQueueStatsRequest -TaskQueueScannerQueueInfo = taskqueue_service_pb.TaskQueueScannerQueueInfo -TaskQueueFetchQueueStatsResponse_QueueStats = taskqueue_service_pb.TaskQueueFetchQueueStatsResponse_QueueStats -TaskQueueFetchQueueStatsResponse = taskqueue_service_pb.TaskQueueFetchQueueStatsResponse -TaskQueuePauseQueueRequest = taskqueue_service_pb.TaskQueuePauseQueueRequest -TaskQueuePauseQueueResponse = taskqueue_service_pb.TaskQueuePauseQueueResponse -TaskQueuePurgeQueueRequest = taskqueue_service_pb.TaskQueuePurgeQueueRequest -TaskQueuePurgeQueueResponse = taskqueue_service_pb.TaskQueuePurgeQueueResponse -TaskQueueDeleteQueueRequest = taskqueue_service_pb.TaskQueueDeleteQueueRequest -TaskQueueDeleteQueueResponse = taskqueue_service_pb.TaskQueueDeleteQueueResponse -TaskQueueDeleteGroupRequest = taskqueue_service_pb.TaskQueueDeleteGroupRequest -TaskQueueDeleteGroupResponse = taskqueue_service_pb.TaskQueueDeleteGroupResponse -TaskQueueQueryTasksRequest = taskqueue_service_pb.TaskQueueQueryTasksRequest -TaskQueueQueryTasksResponse_TaskHeader = taskqueue_service_pb.TaskQueueQueryTasksResponse_TaskHeader -TaskQueueQueryTasksResponse_TaskCronTimetable = taskqueue_service_pb.TaskQueueQueryTasksResponse_TaskCronTimetable -TaskQueueQueryTasksResponse_TaskRunLog = taskqueue_service_pb.TaskQueueQueryTasksResponse_TaskRunLog -TaskQueueQueryTasksResponse_Task = taskqueue_service_pb.TaskQueueQueryTasksResponse_Task -TaskQueueQueryTasksResponse = taskqueue_service_pb.TaskQueueQueryTasksResponse -TaskQueueUpdateStorageLimitRequest = taskqueue_service_pb.TaskQueueUpdateStorageLimitRequest -TaskQueueUpdateStorageLimitResponse = taskqueue_service_pb.TaskQueueUpdateStorageLimitResponse diff --git a/AppServer/google/appengine/api/labs/taskqueue/taskqueue_service_pb.py b/AppServer/google/appengine/api/labs/taskqueue/taskqueue_service_pb.py new file mode 120000 index 0000000000..f566b4a186 --- /dev/null +++ b/AppServer/google/appengine/api/labs/taskqueue/taskqueue_service_pb.py @@ -0,0 +1 @@ +../../taskqueue/taskqueue_service_pb.py \ No newline at end of file diff --git a/AppServer/google/appengine/api/labs/taskqueue/taskqueue_stub.py b/AppServer/google/appengine/api/labs/taskqueue/taskqueue_stub.py deleted file mode 100644 index 820054e52a..0000000000 --- a/AppServer/google/appengine/api/labs/taskqueue/taskqueue_stub.py +++ /dev/null @@ -1,1101 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2007 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - - - -"""Stub version of the Task Queue API. - -This stub stores tasks and runs them via dev_appserver's AddEvent capability. -It also validates the tasks by checking their queue name against the queue.yaml. - -As well as implementing Task Queue API functions, the stub exposes various other -functions that are used by the dev_appserver's admin console to display the -application's queues and tasks. -""" - - - - - - - - - - -import StringIO -import base64 -import bisect -import datetime -import logging -import os -import random -import string -import time - - -from google.appengine.api import api_base_pb -from google.appengine.api import apiproxy_stub -from google.appengine.api import apiproxy_stub_map -from google.appengine.api import queueinfo -from google.appengine.api.labs.taskqueue import taskqueue_service_pb -from google.appengine.runtime import apiproxy_errors - - - - -DEFAULT_RATE = '5.00/s' - - - - - -DEFAULT_BUCKET_SIZE = 5 - - -MAX_ETA_DELTA_DAYS = 30 - - -admin_console_dummy_tasks = {} - - - -BUILT_IN_HEADERS = set(['x-appengine-queuename', - 'x-appengine-taskname', - 'x-appengine-taskretrycount', - 'x-appengine-development-payload', - 'content-length']) - - - -DEFAULT_QUEUE_NAME = 'default' - - -CRON_QUEUE_NAME = '__cron' - - -class _DummyTaskStore(object): - """A class that encapsulates a sorted store of tasks. - - Used for testing the admin console. - """ - - def __init__(self): - """Constructor.""" - - self._sorted_by_name = [] - - self._sorted_by_eta = [] - - def _InsertTask(self, task): - """Insert a task into the dummy store, keeps lists sorted. - - Args: - task: the new task. - """ - eta = task.eta_usec() - name = task.task_name() - bisect.insort_left(self._sorted_by_eta, (eta, name, task)) - bisect.insort_left(self._sorted_by_name, (name, task)) - - def Lookup(self, maximum, name=None, eta=None): - """Lookup a number of sorted tasks from the store. - - If 'eta' is specified, the tasks are looked up in a list sorted by 'eta', - then 'name'. Otherwise they are sorted by 'name'. We need to be able to - sort by 'eta' and 'name' because tasks can have identical eta. If you had - 20 tasks with the same ETA, you wouldn't be able to page past them, since - the 'next eta' would give the first one again. Names are unique, though. - - Args: - maximum: the maximum number of tasks to return. - name: a task name to start with. - eta: an eta to start with. - - Returns: - A list of up to 'maximum' tasks. - - Raises: - ValueError: if the task store gets corrupted. - """ - if eta is None: - - pos = bisect.bisect_left(self._sorted_by_name, (name,)) - - tasks = (x[1] for x in self._sorted_by_name[pos:pos + maximum]) - return list(tasks) - if name is None: - raise ValueError('must supply name or eta') - - pos = bisect.bisect_left(self._sorted_by_eta, (eta, name)) - - tasks = (x[2] for x in self._sorted_by_eta[pos:pos + maximum]) - return list(tasks) - - def Count(self): - """Returns the number of tasks in the store.""" - return len(self._sorted_by_name) - - def Oldest(self): - """Returns the oldest eta in the store, or None if no tasks.""" - if self._sorted_by_eta: - return self._sorted_by_eta[0][0] - return None - - def Add(self, request): - """Inserts a new task into the store. - - Args: - request: A taskqueue_service_pb.TaskQueueAddRequest. - - Raises: - apiproxy_errors.ApplicationError: If a task with the same name is already - in the store. - """ - - pos = bisect.bisect_left(self._sorted_by_name, (request.task_name(),)) - if (pos < len(self._sorted_by_name) and - self._sorted_by_name[pos][0] == request.task_name()): - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.TASK_ALREADY_EXISTS) - - now = datetime.datetime.utcnow() - now_sec = time.mktime(now.timetuple()) - task = taskqueue_service_pb.TaskQueueQueryTasksResponse_Task() - task.set_task_name(request.task_name()) - task.set_eta_usec(request.eta_usec()) - task.set_creation_time_usec(now_sec * 1e6) - task.set_url(request.url()) - task.set_method(request.method()) - for keyvalue in task.header_list(): - header = task.add_header() - header.set_key(keyvalue.key()) - header.set_value(keyvalue.value()) - if request.has_description(): - task.set_description(request.description()) - if request.has_body(): - task.set_body(request.body()) - if request.has_crontimetable(): - task.mutable_crontimetable().set_schedule( - request.crontimetable().schedule()) - task.mutable_crontimetable().set_timezone( - request.crontimetable().timezone()) - self._InsertTask(task) - - def Delete(self, name): - """Deletes a task from the store by name. - - Args: - name: the name of the task to delete. - - Returns: - TaskQueueServiceError.UNKNOWN_TASK: if the task is unknown. - TaskQueueServiceError.INTERNAL_ERROR: if the store is corrupted. - TaskQueueServiceError.OK: otherwise. - """ - pos = bisect.bisect_left(self._sorted_by_name, (name,)) - if pos >= len(self._sorted_by_name): - - return taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_TASK - if self._sorted_by_name[pos][1].task_name() != name: - logging.info('looking for task name %s, got task name %s', name, - self._sorted_by_name[pos][1].task_name()) - return taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_TASK - old_task = self._sorted_by_name.pop(pos)[1] - - eta = old_task.eta_usec() - pos = bisect.bisect_left(self._sorted_by_eta, (eta, name, None)) - if self._sorted_by_eta[pos][2] is not old_task: - logging.error('task store corrupted') - return taskqueue_service_pb.TaskQueueServiceError.INTERNAL_ERRROR - self._sorted_by_eta.pop(pos) - return taskqueue_service_pb.TaskQueueServiceError.OK - - def Populate(self, num_tasks): - """Populates the store with a number of tasks. - - Args: - num_tasks: the number of tasks to insert. - """ - now = datetime.datetime.utcnow() - now_sec = time.mktime(now.timetuple()) - - def RandomTask(): - """Creates a new task and randomly populates values.""" - task = taskqueue_service_pb.TaskQueueQueryTasksResponse_Task() - task.set_task_name(''.join(random.choice(string.ascii_lowercase) - for x in range(20))) - - task.set_eta_usec(int(now_sec * 1e6) + random.randint(-10e6, 600e6)) - - - - task.set_creation_time_usec(min(now_sec * 1e6, task.eta_usec()) - - random.randint(0, 2e7)) - - task.set_url(random.choice(['/a', '/b', '/c', '/d'])) - if random.random() < 0.2: - task.set_method( - taskqueue_service_pb.TaskQueueQueryTasksResponse_Task.POST) - task.set_body('A' * 2000) - else: - task.set_method( - taskqueue_service_pb.TaskQueueQueryTasksResponse_Task.GET) - task.set_retry_count(max(0, random.randint(-10, 5))) - if random.random() < 0.3: - random_headers = [('nexus', 'one'), - ('foo', 'bar'), - ('content-type', 'text/plain'), - ('from', 'user@email.com')] - for _ in xrange(random.randint(1, 4)): - elem = random.randint(0, len(random_headers)-1) - key, value = random_headers.pop(elem) - header_proto = task.add_header() - header_proto.set_key(key) - header_proto.set_value(value) - return task - - for _ in range(num_tasks): - self._InsertTask(RandomTask()) - - -def _ParseQueueYaml(unused_self, root_path): - """Loads the queue.yaml file and parses it. - - Args: - unused_self: Allows this function to be bound to a class member. Not used. - root_path: Directory containing queue.yaml. Not used. - - Returns: - None if queue.yaml doesn't exist, otherwise a queueinfo.QueueEntry object - populated from the queue.yaml. - """ - if root_path is None: - return None - for queueyaml in ('queue.yaml', 'queue.yml'): - try: - fh = open(os.path.join(root_path, queueyaml), 'r') - except IOError: - continue - try: - queue_info = queueinfo.LoadSingleQueue(fh) - return queue_info - finally: - fh.close() - return None - - -def _CompareTasksByEta(a, b): - """Python sort comparator for tasks by estimated time of arrival (ETA). - - Args: - a: A taskqueue_service_pb.TaskQueueAddRequest. - b: A taskqueue_service_pb.TaskQueueAddRequest. - - Returns: - Standard 1/0/-1 comparison result. - """ - if a.eta_usec() > b.eta_usec(): - return 1 - if a.eta_usec() < b.eta_usec(): - return -1 - return 0 - - - -def _FormatEta(eta_usec): - """Formats a task ETA as a date string in UTC.""" - eta = datetime.datetime.fromtimestamp(eta_usec/1000000) - return eta.strftime('%Y/%m/%d %H:%M:%S') - - -def _EtaDelta(eta_usec): - """Formats a task ETA as a relative time string.""" - eta = datetime.datetime.fromtimestamp(eta_usec/1000000) - now = datetime.datetime.utcnow() - if eta > now: - return str(eta - now) + ' from now' - else: - return str(now - eta) + ' ago' - - -class TaskQueueServiceStub(apiproxy_stub.APIProxyStub): - """Python only task queue service stub. - - This stub executes tasks when enabled by using the dev_appserver's AddEvent - capability. When task running is disabled this stub will store tasks for - display on a console, where the user may manually execute the tasks. - """ - - - queue_yaml_parser = _ParseQueueYaml - - def __init__(self, - service_name='taskqueue', - root_path=None, - auto_task_running=False, - task_retry_seconds=30, - _all_queues_valid=False): - """Constructor. - - Args: - service_name: Service name expected for all calls. - root_path: Root path to the directory of the application which may contain - a queue.yaml file. If None, then it's assumed no queue.yaml file is - available. - auto_task_running: When True, the dev_appserver should automatically - run tasks after they are enqueued. - task_retry_seconds: How long to wait between task executions after a - task fails. - """ - super(TaskQueueServiceStub, self).__init__(service_name) - - self._taskqueues = {} - self._next_task_id = 1 - self._root_path = root_path - - - - self._all_queues_valid = _all_queues_valid - - - - - - - self._add_event = None - self._auto_task_running = auto_task_running - self._task_retry_seconds = task_retry_seconds - - - - - - - - self._app_queues = {} - - class _QueueDetails(taskqueue_service_pb.TaskQueueUpdateQueueRequest): - def __init__(self, paused=False): - self.paused = paused - - def _ChooseTaskName(self): - """Returns a string containing a unique task name.""" - self._next_task_id += 1 - return 'task%d' % (self._next_task_id - 1) - - def _VerifyTaskQueueAddRequest(self, request): - """Checks that a TaskQueueAddRequest is valid. - - Checks that a TaskQueueAddRequest specifies a valid eta and a valid queue. - - Args: - request: The taskqueue_service_pb.TaskQueueAddRequest to validate. - - Returns: - A taskqueue_service_pb.TaskQueueServiceError indicating any problems with - the request or taskqueue_service_pb.TaskQueueServiceError.OK if it is - valid. - """ - if request.eta_usec() < 0: - return taskqueue_service_pb.TaskQueueServiceError.INVALID_ETA - - eta = datetime.datetime.utcfromtimestamp(request.eta_usec() / 1e6) - max_eta = (datetime.datetime.utcnow() + - datetime.timedelta(days=MAX_ETA_DELTA_DAYS)) - if eta > max_eta: - return taskqueue_service_pb.TaskQueueServiceError.INVALID_ETA - - return taskqueue_service_pb.TaskQueueServiceError.OK - - def _Dynamic_Add(self, request, response): - bulk_request = taskqueue_service_pb.TaskQueueBulkAddRequest() - bulk_response = taskqueue_service_pb.TaskQueueBulkAddResponse() - - bulk_request.add_add_request().CopyFrom(request) - self._Dynamic_BulkAdd(bulk_request, bulk_response) - - assert bulk_response.taskresult_size() == 1 - result = bulk_response.taskresult(0).result() - - if result != taskqueue_service_pb.TaskQueueServiceError.OK: - raise apiproxy_errors.ApplicationError(result) - elif bulk_response.taskresult(0).has_chosen_task_name(): - response.set_chosen_task_name( - bulk_response.taskresult(0).chosen_task_name()) - - def _Dynamic_BulkAdd(self, request, response): - """Add many tasks to a queue using a single request. - - Args: - request: The taskqueue_service_pb.TaskQueueBulkAddRequest. See - taskqueue_service.proto. - response: The taskqueue_service_pb.TaskQueueBulkAddResponse. See - taskqueue_service.proto. - """ - - - - - - - - - - - assert request.add_request_size(), 'taskqueue should prevent empty requests' - app_id = None - if request.add_request(0).has_app_id(): - app_id = request.add_request(0).app_id() - - if not self._IsValidQueue(request.add_request(0).queue_name(), app_id): - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_QUEUE) - - error_found = False - task_results_with_chosen_names = [] - - - for add_request in request.add_request_list(): - task_result = response.add_taskresult() - error = self._VerifyTaskQueueAddRequest(add_request) - if error == taskqueue_service_pb.TaskQueueServiceError.OK: - if not add_request.task_name(): - chosen_name = self._ChooseTaskName() - add_request.set_task_name(chosen_name) - task_results_with_chosen_names.append(task_result) - - - - task_result.set_result( - taskqueue_service_pb.TaskQueueServiceError.SKIPPED) - else: - error_found = True - task_result.set_result(error) - - if error_found: - return - - - if request.add_request(0).has_transaction(): - self._TransactionalBulkAdd(request) - elif request.add_request(0).has_app_id(): - - - self._DummyTaskStoreBulkAdd(request, response) - else: - self._NonTransactionalBulkAdd(request, response) - - - for add_request, task_result in zip(request.add_request_list(), - response.taskresult_list()): - if (task_result.result() == - taskqueue_service_pb.TaskQueueServiceError.SKIPPED): - task_result.set_result(taskqueue_service_pb.TaskQueueServiceError.OK) - if task_result in task_results_with_chosen_names: - task_result.set_chosen_task_name(add_request.task_name()) - - def _TransactionalBulkAdd(self, request): - """Uses datastore.AddActions to associate tasks with a transaction. - - Args: - request: The taskqueue_service_pb.TaskQueueBulkAddRequest containing the - tasks to add. N.B. all tasks in the request have been validated and - assigned unique names. - """ - try: - apiproxy_stub_map.MakeSyncCall( - 'datastore_v3', 'AddActions', request, api_base_pb.VoidProto()) - except apiproxy_errors.ApplicationError, e: - raise apiproxy_errors.ApplicationError( - e.application_error + - taskqueue_service_pb.TaskQueueServiceError.DATASTORE_ERROR, - e.error_detail) - - def _DummyTaskStoreBulkAdd(self, request, response): - """Adds tasks to the appropriate DummyTaskStore. - - Args: - request: The taskqueue_service_pb.TaskQueueBulkAddRequest containing the - tasks to add. N.B. all tasks in the request have been validated and - those with empty names have been assigned unique names. - response: The taskqueue_service_pb.TaskQueueBulkAddResponse to populate - with the results. N.B. the chosen_task_name field in the response will - not be filled-in. - """ - store = self.GetDummyTaskStore(request.add_request(0).app_id(), - request.add_request(0).queue_name()) - for add_request, task_result in zip(request.add_request_list(), - response.taskresult_list()): - try: - store.Add(add_request) - except apiproxy_errors.ApplicationError, e: - task_result.set_result(e.application_error) - else: - task_result.set_result(taskqueue_service_pb.TaskQueueServiceError.OK) - - def _NonTransactionalBulkAdd(self, request, response): - """Adds tasks to the appropriate list in in self._taskqueues. - - Args: - request: The taskqueue_service_pb.TaskQueueBulkAddRequest containing the - tasks to add. N.B. all tasks in the request have been validated and - those with empty names have been assigned unique names. - response: The taskqueue_service_pb.TaskQueueBulkAddResponse to populate - with the results. N.B. the chosen_task_name field in the response will - not be filled-in. - """ - existing_tasks = self._taskqueues.setdefault( - request.add_request(0).queue_name(), []) - existing_task_names = set(task.task_name() for task in existing_tasks) - - - def DefineCallback(queue_name, task_name): - return lambda: self._RunTask(queue_name, task_name) - - for add_request, task_result in zip(request.add_request_list(), - response.taskresult_list()): - if add_request.task_name() in existing_task_names: - task_result.set_result( - taskqueue_service_pb.TaskQueueServiceError.TASK_ALREADY_EXISTS) - else: - existing_tasks.append(add_request) - - - if self._add_event and self._auto_task_running: - self._add_event( - add_request.eta_usec() / 1000000.0, - DefineCallback(add_request.queue_name(), add_request.task_name())) - - existing_tasks.sort(_CompareTasksByEta) - - def _IsValidQueue(self, queue_name, app_id): - """Determines whether a queue is valid, i.e. tasks can be added to it. - - Valid queues are the 'default' queue, plus any queues in the queue.yaml - file. - - Args: - queue_name: the name of the queue to validate. - app_id: the app_id. Can be None. - - Returns: - True iff queue is valid. - """ - if self._all_queues_valid: - return True - if queue_name == DEFAULT_QUEUE_NAME or queue_name == CRON_QUEUE_NAME: - return True - queue_info = self.queue_yaml_parser(self._root_path) - if queue_info and queue_info.queue: - for entry in queue_info.queue: - if entry.name == queue_name: - return True - - if app_id is not None: - queues = self._app_queues.get(app_id, {}) - return queues.get(queue_name, None) is not None - return False - - def _RunTask(self, queue_name, task_name): - """Returns a fake request for running a task in the dev_appserver. - - Args: - queue_name: The queue the task is in. - task_name: The name of the task to run. - - Returns: - None if this task no longer exists or tuple (connection, addrinfo) of - a fake connection and address information used to run this task. The - task will be deleted after it runs or re-enqueued in the future on - failure. - """ - task_list = self.GetTasks(queue_name) - for task in task_list: - if task['name'] == task_name: - break - else: - return None - - class FakeConnection(object): - def __init__(self, input_buffer): - self.rfile = StringIO.StringIO(input_buffer) - self.wfile = StringIO.StringIO() - self.wfile_close = self.wfile.close - self.wfile.close = self.connection_done - - def connection_done(myself): - result = myself.wfile.getvalue() - myself.wfile_close() - - - first_line, rest = (result.split('\n', 1) + ['', ''])[:2] - version, code, rest = (first_line.split(' ', 2) + ['', '500', ''])[:3] - - - try: - code = int(code) - except ValueError: - code = 500 - - if 200 <= int(code) <= 299: - self.DeleteTask(queue_name, task_name) - return - - logging.warning('Task named "%s" on queue "%s" failed with code %s; ' - 'will retry in %d seconds', - task_name, queue_name, code, self._task_retry_seconds) - self._add_event( - time.time() + self._task_retry_seconds, - lambda: self._RunTask(queue_name, task_name)) - - def close(self): - pass - - def makefile(self, mode, buffsize): - if mode.startswith('w'): - return self.wfile - else: - return self.rfile - - payload = StringIO.StringIO() - payload.write('%s %s HTTP/1.1\r\n' % (task['method'], task['url'])) - for key, value in task['headers']: - payload.write('%s: %s\r\n' % (key, value)) - payload.write('\r\n') - payload.write(task['body']) - - return FakeConnection(payload.getvalue()), ('0.1.0.2', 80) - - def GetQueues(self): - """Gets all the applications's queues. - - Returns: - A list of dictionaries, where each dictionary contains one queue's - attributes. E.g.: - [{'name': 'some-queue', - 'max_rate': '1/s', - 'bucket_size': 5, - 'oldest_task': '2009/02/02 05:37:42', - 'eta_delta': '0:00:06.342511 ago', - 'tasks_in_queue': 12}, ...] - The list of queues always includes the default queue. - """ - queues = [] - queue_info = self.queue_yaml_parser(self._root_path) - has_default = False - if queue_info and queue_info.queue: - for entry in queue_info.queue: - if entry.name == DEFAULT_QUEUE_NAME: - has_default = True - queue = {} - queues.append(queue) - queue['name'] = entry.name - queue['max_rate'] = entry.rate - if entry.bucket_size: - queue['bucket_size'] = entry.bucket_size - else: - queue['bucket_size'] = DEFAULT_BUCKET_SIZE - - tasks = self._taskqueues.setdefault(entry.name, []) - if tasks: - queue['oldest_task'] = _FormatEta(tasks[0].eta_usec()) - queue['eta_delta'] = _EtaDelta(tasks[0].eta_usec()) - else: - queue['oldest_task'] = '' - queue['tasks_in_queue'] = len(tasks) - - - if not has_default: - queue = {} - queues.append(queue) - queue['name'] = DEFAULT_QUEUE_NAME - queue['max_rate'] = DEFAULT_RATE - queue['bucket_size'] = DEFAULT_BUCKET_SIZE - - tasks = self._taskqueues.get(DEFAULT_QUEUE_NAME, []) - if tasks: - queue['oldest_task'] = _FormatEta(tasks[0].eta_usec()) - queue['eta_delta'] = _EtaDelta(tasks[0].eta_usec()) - else: - queue['oldest_task'] = '' - queue['tasks_in_queue'] = len(tasks) - return queues - - def GetTasks(self, queue_name): - """Gets a queue's tasks. - - Args: - queue_name: Queue's name to return tasks for. - - Returns: - A list of dictionaries, where each dictionary contains one task's - attributes. E.g. - [{'name': 'task-123', - 'queue_name': 'default', - 'url': '/update', - 'method': 'GET', - 'eta': '2009/02/02 05:37:42', - 'eta_delta': '0:00:06.342511 ago', - 'body': '', - 'headers': [('user-header', 'some-value') - ('X-AppEngine-QueueName': 'update-queue'), - ('X-AppEngine-TaskName': 'task-123'), - ('X-AppEngine-TaskRetryCount': '0'), - ('X-AppEngine-Development-Payload': '1'), - ('Content-Length': 0), - ('Content-Type': 'application/octet-stream')] - - Raises: - ValueError: A task request contains an unknown HTTP method type. - """ - tasks = self._taskqueues.get(queue_name, []) - result_tasks = [] - for task_request in tasks: - task = {} - result_tasks.append(task) - task['name'] = task_request.task_name() - task['queue_name'] = queue_name - task['url'] = task_request.url() - method = task_request.method() - if method == taskqueue_service_pb.TaskQueueAddRequest.GET: - task['method'] = 'GET' - elif method == taskqueue_service_pb.TaskQueueAddRequest.POST: - task['method'] = 'POST' - elif method == taskqueue_service_pb.TaskQueueAddRequest.HEAD: - task['method'] = 'HEAD' - elif method == taskqueue_service_pb.TaskQueueAddRequest.PUT: - task['method'] = 'PUT' - elif method == taskqueue_service_pb.TaskQueueAddRequest.DELETE: - task['method'] = 'DELETE' - else: - raise ValueError('Unexpected method: %d' % method) - - task['eta'] = _FormatEta(task_request.eta_usec()) - task['eta_delta'] = _EtaDelta(task_request.eta_usec()) - task['body'] = base64.b64encode(task_request.body()) - - - - headers = [(header.key(), header.value()) - for header in task_request.header_list() - if header.key().lower() not in BUILT_IN_HEADERS] - - - headers.append(('X-AppEngine-QueueName', queue_name)) - headers.append(('X-AppEngine-TaskName', task['name'])) - headers.append(('X-AppEngine-TaskRetryCount', '0')) - headers.append(('X-AppEngine-Development-Payload', '1')) - headers.append(('Content-Length', len(task['body']))) - if 'content-type' not in frozenset(key.lower() for key, _ in headers): - headers.append(('Content-Type', 'application/octet-stream')) - task['headers'] = headers - - return result_tasks - - def DeleteTask(self, queue_name, task_name): - """Deletes a task from a queue. - - Args: - queue_name: the name of the queue to delete the task from. - task_name: the name of the task to delete. - """ - tasks = self._taskqueues.get(queue_name, []) - for task in tasks: - if task.task_name() == task_name: - tasks.remove(task) - return - - def FlushQueue(self, queue_name): - """Removes all tasks from a queue. - - Args: - queue_name: the name of the queue to remove tasks from. - """ - self._taskqueues[queue_name] = [] - - def _Dynamic_UpdateQueue(self, request, unused_response): - """Local implementation of the UpdateQueue RPC in TaskQueueService. - - Must adhere to the '_Dynamic_' naming convention for stubbing to work. - See taskqueue_service.proto for a full description of the RPC. - - Args: - request: A taskqueue_service_pb.TaskQueueUpdateQueueRequest. - unused_response: A taskqueue_service_pb.TaskQueueUpdateQueueResponse. - Not used. - """ - queues = self._app_queues.setdefault(request.app_id(), {}) - if request.queue_name() in queues and queues[request.queue_name()] is None: - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.TOMBSTONED_QUEUE) - - defensive_copy = self._QueueDetails() - defensive_copy.CopyFrom(request) - - queues[request.queue_name()] = defensive_copy - - def _Dynamic_FetchQueues(self, request, response): - """Local implementation of the FetchQueues RPC in TaskQueueService. - - Must adhere to the '_Dynamic_' naming convention for stubbing to work. - See taskqueue_service.proto for a full description of the RPC. - - Args: - request: A taskqueue_service_pb.TaskQueueFetchQueuesRequest. - response: A taskqueue_service_pb.TaskQueueFetchQueuesResponse. - """ - queues = self._app_queues.get(request.app_id(), {}) - for unused_key, queue in sorted(queues.items()): - if request.max_rows() == response.queue_size(): - break - - if queue is None: - continue - - response_queue = response.add_queue() - response_queue.set_queue_name(queue.queue_name()) - response_queue.set_bucket_refill_per_second( - queue.bucket_refill_per_second()) - response_queue.set_bucket_capacity(queue.bucket_capacity()) - response_queue.set_user_specified_rate(queue.user_specified_rate()) - if queue.has_max_concurrent_requests(): - response_queue.set_max_concurrent_requests( - queue.max_concurrent_requests()) - response_queue.set_paused(queue.paused) - - def _Dynamic_FetchQueueStats(self, request, response): - """Local 'random' implementation of the TaskQueueService.FetchQueueStats. - - This implementation loads some stats from the dummy store, - the rest with random numbers. - Must adhere to the '_Dynamic_' naming convention for stubbing to work. - See taskqueue_service.proto for a full description of the RPC. - - Args: - request: A taskqueue_service_pb.TaskQueueFetchQueueStatsRequest. - response: A taskqueue_service_pb.TaskQueueFetchQueueStatsResponse. - """ - for queue in request.queue_name_list(): - store = self.GetDummyTaskStore(request.app_id(), queue) - stats = response.add_queuestats() - stats.set_num_tasks(store.Count()) - if stats.num_tasks() == 0: - stats.set_oldest_eta_usec(-1) - else: - stats.set_oldest_eta_usec(store.Oldest()) - - - if random.randint(0, 9) > 0: - scanner_info = stats.mutable_scanner_info() - scanner_info.set_executed_last_minute(random.randint(0, 10)) - scanner_info.set_executed_last_hour(scanner_info.executed_last_minute() - + random.randint(0, 100)) - scanner_info.set_sampling_duration_seconds(random.random() * 10000.0) - scanner_info.set_requests_in_flight(random.randint(0, 10)) - - def GetDummyTaskStore(self, app_id, queue_name): - """Get the dummy task store for this app_id/queue_name pair. - - Creates an entry and populates it, if there's not already an entry. - - Args: - app_id: the app_id. - queue_name: the queue_name. - - Returns: - the existing or the new dummy store. - """ - task_store_key = (app_id, queue_name) - if task_store_key not in admin_console_dummy_tasks: - store = _DummyTaskStore() - if not self._all_queues_valid and queue_name != CRON_QUEUE_NAME: - store.Populate(random.randint(10, 100)) - admin_console_dummy_tasks[task_store_key] = store - else: - store = admin_console_dummy_tasks[task_store_key] - return store - - def _Dynamic_QueryTasks(self, request, response): - """Local implementation of the TaskQueueService.QueryTasks RPC. - - Uses the dummy store, creating tasks if this is the first time the - queue has been seen. - - Args: - request: A taskqueue_service_pb.TaskQueueQueryTasksRequest. - response: A taskqueue_service_pb.TaskQueueQueryTasksResponse. - """ - store = self.GetDummyTaskStore(request.app_id(), request.queue_name()) - - if request.has_start_eta_usec(): - tasks = store.Lookup(request.max_rows(), name=request.start_task_name(), - eta=request.start_eta_usec()) - else: - tasks = store.Lookup(request.max_rows(), name=request.start_task_name()) - for task in tasks: - response.add_task().MergeFrom(task) - - def _Dynamic_Delete(self, request, response): - """Local delete implementation of TaskQueueService.Delete. - - Deletes tasks from the dummy store. A 1/20 chance of a transient error. - - Args: - request: A taskqueue_service_pb.TaskQueueDeleteRequest. - response: A taskqueue_service_pb.TaskQueueDeleteResponse. - """ - task_store_key = (request.app_id(), request.queue_name()) - if task_store_key not in admin_console_dummy_tasks: - for _ in request.task_name_list(): - response.add_result( - taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_QUEUE) - return - - store = admin_console_dummy_tasks[task_store_key] - for taskname in request.task_name_list(): - if random.random() <= 0.05: - response.add_result( - taskqueue_service_pb.TaskQueueServiceError.TRANSIENT_ERROR) - else: - response.add_result(store.Delete(taskname)) - - def _Dynamic_ForceRun(self, request, response): - """Local force run implementation of TaskQueueService.ForceRun. - - Forces running of a task in a queue. This is a no-op here. - This will fail randomly for testing. - - Args: - request: A taskqueue_service_pb.TaskQueueForceRunRequest. - response: A taskqueue_service_pb.TaskQueueForceRunResponse. - """ - if random.random() <= 0.05: - response.set_result( - taskqueue_service_pb.TaskQueueServiceError.TRANSIENT_ERROR) - elif random.random() <= 0.052: - response.set_result( - taskqueue_service_pb.TaskQueueServiceError.INTERNAL_ERROR) - else: - response.set_result( - taskqueue_service_pb.TaskQueueServiceError.OK) - - def _Dynamic_DeleteQueue(self, request, response): - """Local delete implementation of TaskQueueService.DeleteQueue. - - Args: - request: A taskqueue_service_pb.TaskQueueDeleteQueueRequest. - response: A taskqueue_service_pb.TaskQueueDeleteQueueResponse. - """ - if not request.queue_name(): - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.INVALID_QUEUE_NAME) - - queues = self._app_queues.get(request.app_id(), {}) - if request.queue_name() not in queues: - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_QUEUE) - elif queues[request.queue_name()] is None: - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.TOMBSTONED_QUEUE) - - queues[request.queue_name()] = None - - def _Dynamic_PauseQueue(self, request, response): - """Local pause implementation of TaskQueueService.PauseQueue. - - Args: - request: A taskqueue_service_pb.TaskQueuePauseQueueRequest. - response: A taskqueue_service_pb.TaskQueuePauseQueueResponse. - """ - if not request.queue_name(): - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.INVALID_QUEUE_NAME) - - queues = self._app_queues.get(request.app_id(), {}) - if request.queue_name() != DEFAULT_QUEUE_NAME: - if request.queue_name() not in queues: - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_QUEUE) - elif queues[request.queue_name()] is None: - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.TOMBSTONED_QUEUE) - - queues[request.queue_name()].paused = request.pause() - - def _Dynamic_PurgeQueue(self, request, response): - """Local purge implementation of TaskQueueService.PurgeQueue. - - Args: - request: A taskqueue_service_pb.TaskQueuePurgeQueueRequest. - response: A taskqueue_service_pb.TaskQueuePurgeQueueResponse. - """ - if not request.queue_name(): - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.INVALID_QUEUE_NAME) - - if request.has_app_id(): - queues = self._app_queues.get(request.app_id(), {}) - if request.queue_name() != DEFAULT_QUEUE_NAME: - if request.queue_name() not in queues: - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_QUEUE) - elif queues[request.queue_name()] is None: - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.TOMBSTONED_QUEUE) - - store = self.GetDummyTaskStore(request.app_id(), request.queue_name()) - for task in store.Lookup(store.Count()): - store.Delete(task.task_name()) - elif (not self._IsValidQueue(request.queue_name(), None) - and not request.queue_name() in self._taskqueues): - - - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_QUEUE) - - self.FlushQueue(request.queue_name()) - - def _Dynamic_DeleteGroup(self, request, response): - """Local delete implementation of TaskQueueService.DeleteGroup. - - Args: - request: A taskqueue_service_pb.TaskQueueDeleteGroupRequest. - response: A taskqueue_service_pb.TaskQueueDeleteGroupResponse. - """ - queues = self._app_queues.get(request.app_id(), {}) - - - for queue in queues.iterkeys(): - store = self.GetDummyTaskStore(request.app_id(), queue) - for task in store.Lookup(store.Count()): - store.Delete(task.task_name()) - self.FlushQueue(queue) - - self._app_queues[request.app_id()] = {} - - def _Dynamic_UpdateStorageLimit(self, request, response): - """Local implementation of TaskQueueService.UpdateStorageLimit. - Args: - request: A taskqueue_service_pb.TaskQueueUpdateStorageLimitRequest. - response: A taskqueue_service_pb.TaskQueueUpdateStorageLimitResponse. - """ - if request.limit() < 0 or request.limit() > 1000 * (1024 ** 4): - raise apiproxy_errors.ApplicationError( - taskqueue_service_pb.TaskQueueServiceError.INVALID_REQUEST) - - response.set_new_limit(request.limit()) diff --git a/AppServer/google/appengine/api/labs/taskqueue/taskqueue_stub.py b/AppServer/google/appengine/api/labs/taskqueue/taskqueue_stub.py new file mode 120000 index 0000000000..71fa65c6b7 --- /dev/null +++ b/AppServer/google/appengine/api/labs/taskqueue/taskqueue_stub.py @@ -0,0 +1 @@ +../../taskqueue/taskqueue_stub.py \ No newline at end of file diff --git a/AppServer/google/appengine/datastore/cassandra_stub_util.py b/AppServer/google/appengine/datastore/cassandra_stub_util.py new file mode 100755 index 0000000000..e93e8c4ae5 --- /dev/null +++ b/AppServer/google/appengine/datastore/cassandra_stub_util.py @@ -0,0 +1,890 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + + +"""Utility functions shared between the file and sqlite datastore stubs.""" + + + + + +try: + import hashlib + _MD5_FUNC = hashlib.md5 +except ImportError: + import md5 + _MD5_FUNC = md5.new + +import struct +import threading + +from google.appengine.api import datastore_types +from google.appengine.api.datastore_errors import BadRequestError +from google.appengine.datastore import datastore_index +from google.appengine.datastore import datastore_pb +from google.appengine.runtime import apiproxy_errors +from google.appengine.datastore import entity_pb + + + + + +_MAXIMUM_RESULTS = 1000 + + + + + +_MAX_QUERY_OFFSET = 1000 + + + + +_CURSOR_CONCAT_STR = '!CURSOR!' + + +_PROPERTY_TYPE_NAMES = { + 0: 'NULL', + entity_pb.PropertyValue.kint64Value: 'INT64', + entity_pb.PropertyValue.kbooleanValue: 'BOOLEAN', + entity_pb.PropertyValue.kstringValue: 'STRING', + entity_pb.PropertyValue.kdoubleValue: 'DOUBLE', + entity_pb.PropertyValue.kPointValueGroup: 'POINT', + entity_pb.PropertyValue.kUserValueGroup: 'USER', + entity_pb.PropertyValue.kReferenceValueGroup: 'REFERENCE' + } + + +_SCATTER_PROPORTION = 32768 + +def _GetScatterProperty(entity_proto): + """Gets the scatter property for an object. + + For ease of implementation, this is not synchronized with the actual + value on the App Engine server, but should work equally well. + + Note: This property may change, either here or in production. No client + other than the mapper framework should rely on it directly. + + Returns: + The PropertyValue of the scatter property or None if this entity should not + have a scatter property. + """ + hash_obj = _MD5_FUNC() + for element in entity_proto.key().path().element_list(): + if element.has_name(): + hash_obj.update(element.name()) + elif element.has_id(): + hash_obj.update(str(element.id())) + hash_bytes = hash_obj.digest()[0:2] + (hash_int,) = struct.unpack('H', hash_bytes) + + if hash_int >= _SCATTER_PROPORTION: + return None + + scatter_property = entity_pb.Property() + scatter_property.set_name('__scatter__') + scatter_property.set_meaning(entity_pb.Property.BYTESTRING) + scatter_property.set_multiple(False) + property_value = scatter_property.mutable_value() + property_value.set_stringvalue(hash_bytes) + return scatter_property + + + + + +_SPECIAL_PROPERTY_MAP = { + '__scatter__' : (False, True, _GetScatterProperty) + } + +def GetInvisibleSpecialPropertyNames(): + """Gets the names of all non user-visible special properties.""" + invisible_names = [] + for name, value in _SPECIAL_PROPERTY_MAP.items(): + is_visible, is_stored, property_func = value + if not is_visible: + invisible_names.append(name) + return invisible_names + +def _PrepareSpecialProperties(entity_proto, is_load): + """Computes special properties for loading or storing. + Strips other special properties.""" + for i in xrange(entity_proto.property_size() - 1, -1, -1): + if _SPECIAL_PROPERTY_MAP.has_key(entity_proto.property(i).name()): + del entity_proto.property_list()[i] + + for is_visible, is_stored, property_func in _SPECIAL_PROPERTY_MAP.values(): + if is_load: + should_process = is_visible + else: + should_process = is_stored + + if should_process: + special_property = property_func(entity_proto) + if special_property: + entity_proto.property_list().append(special_property) + + +def PrepareSpecialPropertiesForStore(entity_proto): + """Computes special properties for storing. + Strips other special properties.""" + _PrepareSpecialProperties(entity_proto, False) + + +def PrepareSpecialPropertiesForLoad(entity_proto): + """Computes special properties that are user-visible. + Strips other special properties.""" + _PrepareSpecialProperties(entity_proto, True) + + +def ValidateQuery(query, filters, orders, max_query_components): + """Validate a datastore query with normalized filters, orders. + + Raises an ApplicationError when any of the following conditions are violated: + - transactional queries have an ancestor + - queries that are not too large + (sum of filters, orders, ancestor <= max_query_components) + - ancestor (if any) app and namespace match query app and namespace + - kindless queries only filter on __key__ and only sort on __key__ ascending + - multiple inequality (<, <=, >, >=) filters all applied to the same property + - filters on __key__ compare to a reference in the same app and namespace as + the query + - if an inequality filter on prop X is used, the first order (if any) must + be on X + + Args: + query: query to validate + filters: normalized (by datastore_index.Normalize) filters from query + orders: normalized (by datastore_index.Normalize) orders from query + max_query_components: limit on query complexity + """ + + def BadRequest(message): + raise apiproxy_errors.ApplicationError( + datastore_pb.Error.BAD_REQUEST, message) + + key_prop_name = datastore_types._KEY_SPECIAL_PROPERTY + unapplied_log_timestamp_us_name = ( + datastore_types._UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY) + + if query.has_transaction(): + + if not query.has_ancestor(): + BadRequest('Only ancestor queries are allowed inside transactions.') + + + num_components = len(filters) + len(orders) + if query.has_ancestor(): + num_components += 1 + if num_components > max_query_components: + BadRequest('query is too large. may not have more than %s filters' + ' + sort orders ancestor total' % max_query_components) + + + if query.has_ancestor(): + ancestor = query.ancestor() + if query.app() != ancestor.app(): + BadRequest('query app is %s but ancestor app is %s' % + (query.app(), ancestor.app())) + if query.name_space() != ancestor.name_space(): + BadRequest('query namespace is %s but ancestor namespace is %s' % + (query.name_space(), ancestor.name_space())) + + + + ineq_prop_name = None + for filter in filters: + if filter.property_size() != 1: + BadRequest('Filter has %d properties, expected 1' % + filter.property_size()) + + prop = filter.property(0) + prop_name = prop.name().decode('utf-8') + + if prop_name == key_prop_name: + + + + if not prop.value().has_referencevalue(): + BadRequest('%s filter value must be a Key' % key_prop_name) + ref_val = prop.value().referencevalue() + if ref_val.app() != query.app(): + BadRequest('%s filter app is %s but query app is %s' % + (key_prop_name, ref_val.app(), query.app())) + if ref_val.name_space() != query.name_space(): + BadRequest('%s filter namespace is %s but query namespace is %s' % + (key_prop_name, ref_val.name_space(), query.name_space())) + + if (filter.op() in datastore_index.INEQUALITY_OPERATORS and + prop_name != unapplied_log_timestamp_us_name): + if ineq_prop_name is None: + ineq_prop_name = prop_name + elif ineq_prop_name != prop_name: + BadRequest(('Only one inequality filter per query is supported. ' + 'Encountered both %s and %s') % (ineq_prop_name, prop_name)) + + if ineq_prop_name is not None and orders: + + first_order_prop = orders[0].property().decode('utf-8') + if first_order_prop != ineq_prop_name: + BadRequest('The first sort property must be the same as the property ' + 'to which the inequality filter is applied. In your query ' + 'the first sort property is %s but the inequality filter ' + 'is on %s' % (first_order_prop, ineq_prop_name)) + + if not query.has_kind(): + + for filter in filters: + prop_name = filter.property(0).name().decode('utf-8') + if (prop_name != key_prop_name and + prop_name != unapplied_log_timestamp_us_name): + BadRequest('kind is required for non-__key__ filters') + for order in orders: + prop_name = order.property().decode('utf-8') + if not (prop_name == key_prop_name and + order.direction() is datastore_pb.Query_Order.ASCENDING): + BadRequest('kind is required for all orders except __key__ ascending') + + +class ValueRange(object): + """A range of values defined by its two extremes (inclusive or exclusive).""" + + def __init__(self): + """Constructor. + + Creates an unlimited range. + """ + self.__start = self.__end = None + self.__start_inclusive = self.__end_inclusive = False + + def Update(self, rel_op, limit): + """Filter the range by 'rel_op limit'. + + Args: + rel_op: relational operator from datastore_pb.Query_Filter. + limit: the value to limit the range by. + """ + + if rel_op == datastore_pb.Query_Filter.LESS_THAN: + if self.__end is None or limit <= self.__end: + self.__end = limit + self.__end_inclusive = False + elif (rel_op == datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL or + rel_op == datastore_pb.Query_Filter.EQUAL): + if self.__end is None or limit < self.__end: + self.__end = limit + self.__end_inclusive = True + + if rel_op == datastore_pb.Query_Filter.GREATER_THAN: + if self.__start is None or limit >= self.__start: + self.__start = limit + self.__start_inclusive = False + elif (rel_op == datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL or + rel_op == datastore_pb.Query_Filter.EQUAL): + if self.__start is None or limit > self.__start: + self.__start = limit + self.__start_inclusive = True + + def Contains(self, value): + """Check if the range contains a specific value. + + Args: + value: the value to check. + Returns: + True iff value is contained in this range. + """ + if self.__start is not None: + if self.__start_inclusive and value < self.__start: return False + if not self.__start_inclusive and value <= self.__start: return False + if self.__end is not None: + if self.__end_inclusive and value > self.__end: return False + if not self.__end_inclusive and value >= self.__end: return False + return True + + def Remap(self, mapper): + """Transforms the range extremes with a function. + + The function mapper must preserve order, i.e. + x rel_op y iff mapper(x) rel_op y + + Args: + mapper: function to apply to the range extremes. + """ + self.__start = self.__start and mapper(self.__start) + self.__end = self.__end and mapper(self.__end) + + def MapExtremes(self, mapper): + """Evaluate a function on the range extremes. + + Args: + mapper: function to apply to the range extremes. + Returns: + (x, y) where x = None if the range has no start, + mapper(start, start_inclusive, False) otherwise + y = None if the range has no end, + mapper(end, end_inclusive, True) otherwise + """ + return ( + self.__start and mapper(self.__start, self.__start_inclusive, False), + self.__end and mapper(self.__end, self.__end_inclusive, True)) + + +def ParseKeyFilteredQuery(filters, orders): + """Parse queries which only allow filters and ascending-orders on __key__. + + Raises exceptions for illegal queries. + Args: + filters: the normalized filters of a query. + orders: the normalized orders of a query. + Returns: + The key range (a ValueRange over datastore_types.Key) requested in the + query. + """ + + remaining_filters = [] + key_range = ValueRange() + key_prop = datastore_types._KEY_SPECIAL_PROPERTY + for f in filters: + op = f.op() + if not (f.property_size() == 1 and + f.property(0).name() == key_prop and + not (op == datastore_pb.Query_Filter.IN or + op == datastore_pb.Query_Filter.EXISTS)): + remaining_filters.append(f) + continue + + val = f.property(0).value() + if not val.has_referencevalue(): + raise BadRequestError('__key__ kind must be compared to a key') + limit = datastore_types.FromReferenceProperty(val) + key_range.Update(op, limit) + + + remaining_orders = [] + for o in orders: + if not (o.direction() == datastore_pb.Query_Order.ASCENDING and + o.property() == datastore_types._KEY_SPECIAL_PROPERTY): + remaining_orders.append(o) + else: + break + + + + if remaining_filters: + raise BadRequestError( + 'Only comparison filters on ' + key_prop + ' supported') + if remaining_orders: + raise BadRequestError('Only ascending order on ' + key_prop + ' supported') + + return key_range + + +def ParseKindQuery(query, filters, orders): + """Parse __kind__ (schema) queries. + + Raises exceptions for illegal queries. + Args: + query: A Query PB. + filters: the normalized filters from query. + orders: the normalized orders from query. + Returns: + The kind range (a ValueRange over string) requested in the query. + """ + + if query.has_ancestor(): + raise BadRequestError('ancestor queries on __kind__ not allowed') + + key_range = ParseKeyFilteredQuery(filters, orders) + key_range.Remap(_KindKeyToString) + + return key_range + + +def _KindKeyToString(key): + """Extract kind name from __kind__ key. + + Raises an ApplicationError if the key is not of the form '__kind__'/name. + + Args: + key: a key for a __kind__ instance. + Returns: + kind specified by key. + """ + key_path = key.to_path() + if (len(key_path) == 2 and key_path[0] == '__kind__' and + isinstance(key_path[1], basestring)): + return key_path[1] + raise BadRequestError('invalid Key for __kind__ table') + + +def ParseNamespaceQuery(query, filters, orders): + """Parse __namespace__ queries. + + Raises exceptions for illegal queries. + Args: + query: A Query PB. + filters: the normalized filters from query. + orders: the normalized orders from query. + Returns: + The kind range (a ValueRange over string) requested in the query. + """ + + if query.has_ancestor(): + raise BadRequestError('ancestor queries on __namespace__ not allowed') + + key_range = ParseKeyFilteredQuery(filters, orders) + key_range.Remap(_NamespaceKeyToString) + + return key_range + + +def _NamespaceKeyToString(key): + """Extract namespace name from __namespace__ key. + + Raises an ApplicationError if the key is not of the form '__namespace__'/name + or '__namespace__'/_EMPTY_NAMESPACE_ID. + + Args: + key: a key for a __namespace__ instance. + Returns: + namespace specified by key. + """ + key_path = key.to_path() + if len(key_path) == 2 and key_path[0] == '__namespace__': + if key_path[1] == datastore_types._EMPTY_NAMESPACE_ID: + return '' + if isinstance(key_path[1], basestring): + return key_path[1] + raise BadRequestError('invalid Key for __namespace__ table') + + +def ParsePropertyQuery(query, filters, orders): + """Parse __property__ queries. + + Raises exceptions for illegal queries. + Args: + query: A Query PB. + filters: the normalized filters from query. + orders: the normalized orders from query. + Returns: + The kind range (a ValueRange over (kind, property) pairs) requested + in the query. + """ + + if query.has_transaction(): + raise BadRequestError('transactional queries on __property__ not allowed') + + key_range = ParseKeyFilteredQuery(filters, orders) + key_range.Remap(lambda x: _PropertyKeyToString(x, '')) + + if query.has_ancestor(): + ancestor = datastore_types.Key._FromPb(query.ancestor()) + ancestor_kind, ancestor_property = _PropertyKeyToString(ancestor, None) + + + if ancestor_property is not None: + key_range.Update(datastore_pb.Query_Filter.EQUAL, + (ancestor_kind, ancestor_property)) + else: + + key_range.Update(datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL, + (ancestor_kind, '')) + key_range.Update(datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL, + (ancestor_kind + '\0', '')) + query.clear_ancestor() + + return key_range + +def _PropertyKeyToString(key, default_property): + """Extract property name from __property__ key. + + Raises an ApplicationError if the key is not of the form + '__kind__'/kind, '__property__'/property or '__kind__'/kind + + Args: + key: a key for a __property__ instance. + default_property: property value to return when key only has a kind. + Returns: + kind, property if key = '__kind__'/kind, '__property__'/property + kind, default_property if key = '__kind__'/kind + """ + key_path = key.to_path() + if (len(key_path) == 2 and + key_path[0] == '__kind__' and isinstance(key_path[1], basestring)): + return (key_path[1], default_property) + if (len(key_path) == 4 and + key_path[0] == '__kind__' and isinstance(key_path[1], basestring) and + key_path[2] == '__property__' and isinstance(key_path[3], basestring)): + return (key_path[1], key_path[3]) + + raise BadRequestError('invalid Key for __property__ table') + + +def SynthesizeUserId(email): + """Return a synthetic user ID from an email address. + + Note that this is not the same user ID found in the production system. + + Args: + email: An email address. + + Returns: + A string userid derived from the email address. + """ + + user_id_digest = _MD5_FUNC(email.lower()).digest() + user_id = '1' + ''.join(['%02d' % ord(x) for x in user_id_digest])[:20] + return user_id + + +def FillUsersInQuery(filters): + """Fill in a synthetic user ID for all user properties in a set of filters. + + Args: + filters: The normalized filters from query. + """ + for filter in filters: + for property in filter.property_list(): + FillUser(property) + + +def FillUser(property): + """Fill in a synthetic user ID for a user properties. + + Args: + property: A Property which may have a user value. + """ + if property.value().has_uservalue(): + uid = SynthesizeUserId(property.value().uservalue().email()) + if uid: + property.mutable_value().mutable_uservalue().set_obfuscated_gaiaid(uid) + + +class BaseCursor(object): + """A base query cursor over a list of entities. + + Public properties: + cursor: the integer cursor + app: the app for which this cursor was created + + Class attributes: + _next_cursor: the next cursor to allocate + _next_cursor_lock: protects _next_cursor + """ + _next_cursor = 1 + _next_cursor_lock = threading.Lock() + + def __init__(self, app): + """Constructor. + + Args: + app: The app this cursor is being created for. + """ + self.app = app + self.cursor = self._AcquireCursorID() + + def PopulateCursor(self, query_result): + if query_result.more_results(): + cursor = query_result.mutable_cursor() + cursor.set_app(self.app) + cursor.set_cursor(self.cursor) + + @classmethod + def _AcquireCursorID(cls): + """Acquires the next cursor id in a thread safe manner.""" + cls._next_cursor_lock.acquire() + try: + cursor_id = cls._next_cursor + cls._next_cursor += 1 + finally: + cls._next_cursor_lock.release() + return cursor_id + + +class QueryCursor(object): + """Encapsulates a database cursor and provides methods to fetch results.""" + + def __init__(self, query, results): + """Constructor. + + Args: + query: A Query PB. + db_cursor: An MySQL cursor returning n+2 columns. The first 2 columns + must be the path of the entity and the entity itself, while the + remaining columns must be the sort columns for the query. + """ + self.__results = results + self.__query = query + self.app = query.app() + #self.__cursor = db_cursor + #self.__num_results = 0 + #if db_cursor: + # self.__num_results = db_cursor.rowcount + #self.__seen = set() + + #self.__position = ('', '') + + #self.__next_result = (None, None) + + if query.has_limit(): + self.limit = query.limit() + query.offset() + else: + self.limit = None + + def _MinimalQueryInfo(self, query): + """Extract the minimal set of information for query matching. + + Args: + query: datastore_pb.Query instance from which to extract info. + + Returns: + datastore_pb.Query instance suitable for matching against when + validating cursors. + """ + query_info = datastore_pb.Query() + query_info.set_app(query.app()) + + for filter in query.filter_list(): + query_info.filter_list().append(filter) + for order in query.order_list(): + query_info.order_list().append(order) + + if query.has_ancestor(): + query_info.mutable_ancestor().CopyFrom(query.ancestor()) + + for attr in ('kind', 'name_space', 'search_query'): + query_has_attr = getattr(query, 'has_%s' % attr) + query_attr = getattr(query, attr) + query_info_set_attr = getattr(query_info, 'set_%s' % attr) + if query_has_attr(): + query_info_set_attr(query_attr()) + + return query_info + + def _MinimalEntityInfo(self, entity_proto, query): + """Extract the minimal set of information that preserves entity order. + + Args: + entity_proto: datastore_pb.EntityProto instance from which to extract + information + query: datastore_pb.Query instance for which ordering must be preserved. + + Returns: + datastore_pb.EntityProto instance suitable for matching against a list of + results when finding cursor positions. + """ + entity_info = datastore_pb.EntityProto() + order_names = [o.property() for o in query.order_list()] + entity_info.mutable_key().MergeFrom(entity_proto.key()) + entity_info.mutable_entity_group().MergeFrom(entity_proto.entity_group()) + for prop in entity_proto.property_list(): + if order_names: + if prop.name() in order_names: + entity_info.add_property().MergeFrom(prop) + else: + entity_info.add_property().MergeFrom(prop) + return entity_info + + + def _EncodeCompiledCursor(self, compiled_cursor): + """Converts the current state of the cursor into a compiled_cursor. + + Args: + query: the datastore_pb.Query this cursor is related to + compiled_cursor: an empty datstore_pb.CompiledCursor + """ + if self.__results: + last_result = self.__results[-1] + else: + last_result = None + position = compiled_cursor.add_position() + query_info = self._MinimalQueryInfo(self.__query) + entity_info = self._MinimalEntityInfo(last_result, self.__query) + start_key = _CURSOR_CONCAT_STR.join(( + query_info.Encode(), + entity_info.Encode())) + position.set_start_key(str(start_key)) + position.set_start_inclusive(False) + + def PopulateQueryResult(self, count, offset, result): + """Populates a QueryResult PB with results from the cursor. + + Args: + count: The number of results to retrieve. + offset: The number of results to skip. + result: out: A query_result PB. + """ + limited_offset = min(offset, _MAX_QUERY_OFFSET) + if limited_offset: + result.set_skipped_results(limited_offset) + + if offset == limited_offset: + if count > _MAXIMUM_RESULTS: + count = _MAXIMUM_RESULTS + + result_list = result.result_list() + if self.__results: + result_list.extend(self.__results) + else: + result_list = [] + result.set_keys_only(self.__query.keys_only()) + result.set_more_results(offset < count) + if self.__results: + self._EncodeCompiledCursor(result.mutable_compiled_cursor()) + + +class ListCursor(BaseCursor): + """A query cursor over a list of entities. + + Public properties: + keys_only: whether the query is keys_only + """ + + def __init__(self, query): + """Constructor. + + Args: + query: the query request proto + # the query results, in order, such that results[self.offset+1] is + # the next result + results: list of datastore_pb.EntityProto + order_compare_entities: a __cmp__ function for datastore_pb.EntityProto + that follows sort order as specified by the query + """ + super(ListCursor, self).__init__(query.app()) + + if query.has_compiled_cursor() and query.compiled_cursor().position_list(): + (self.__last_result, inclusive) = self._DecodeCompiledCursor( + query, query.compiled_cursor()) + self.__query = query + self.__offset = 0 + self.__count = query.limit() + + + self.keys_only = query.keys_only() + + def _GetLastResult(self): + return self.__last_result + + @staticmethod + def _GetCursorOffset(results, cursor_entity, inclusive, compare): + """Converts a cursor entity into a offset into the result set even if the + cursor_entity no longer exists. + + Args: + results: the query's results (sequence of datastore_pb.EntityProto) + cursor_entity: the datastore_pb.EntityProto from the compiled query + inclusive: boolean that specifies if to offset past the cursor_entity + compare: a function that takes two datastore_pb.EntityProto and compares + them. + Returns: + the integer offset + """ + lo = 0 + hi = len(results) + if inclusive: + + while lo < hi: + mid = (lo + hi) // 2 + if compare(results[mid], cursor_entity) < 0: + lo = mid + 1 + else: + hi = mid + else: + + while lo < hi: + mid = (lo + hi) // 2 + if compare(cursor_entity, results[mid]) < 0: + hi = mid + else: + lo = mid + 1 + return lo + + def _ValidateQuery(self, query, query_info): + """Ensure that the given query matches the query_info. + + Args: + query: datastore_pb.Query instance we are chacking + query_info: datastore_pb.Query instance we want to match + + Raises BadRequestError on failure. + """ + error_msg = 'Cursor does not match query: %s' + if query_info.filter_list() != query.filter_list(): + raise BadRequestError(error_msg % 'filters do not match') + if query_info.order_list() != query.order_list(): + raise BadRequestError(error_msg % 'orders do not match') + + + for attr in ('ancestor', 'kind', 'name_space', 'search_query'): + query_info_has_attr = getattr(query_info, 'has_%s' % attr) + query_info_attr = getattr(query_info, attr) + query_has_attr = getattr(query, 'has_%s' % attr) + query_attr = getattr(query, attr) + if query_info_has_attr(): + if not query_has_attr() or query_info_attr() != query_attr(): + raise BadRequestError(error_msg % ('%s does not match' % attr)) + elif query_has_attr(): + raise BadRequestError(error_msg % ('%s does not match' % attr)) + + + def _DecodeCompiledCursor(self, query, compiled_cursor): + """Converts a compiled_cursor into a cursor_entity. + + Returns: + (cursor_entity, inclusive): a datastore_pb.EntityProto and if it should + be included in the result set. + """ + assert len(compiled_cursor.position_list()) == 1 + + position = compiled_cursor.position(0) + entity_as_pb = datastore_pb.EntityProto() + (query_info_encoded, entity_encoded) = position.start_key().split(_CURSOR_CONCAT_STR, 1) + query_info_pb = datastore_pb.Query() + query_info_pb.ParseFromString(query_info_encoded) + self._ValidateQuery(query, query_info_pb) + + entity_as_pb.ParseFromString(entity_encoded) + return (entity_as_pb, position.start_inclusive()) + + + def Count(self): + """Counts results, up to the query's limit. + + Note this method does not deduplicate results, so the query it was generated + from should have the 'distinct' clause applied. + + Returns: + int: Result count. + """ + return self.__count + + +def CompareEntityPbByKey(a, b): + """Compare two entity protobuf's by key. + + Args: + a: datastore_pb.EntityProto to compare + b: datastore_pb.EntityProto to compare + Returns: + <0 if a's key is before b's, =0 if they are the same key, and >0 otherwise. + """ + return cmp(datastore_types.Key._FromPb(a.key()), + datastore_types.Key._FromPb(b.key())) diff --git a/AppServer/google/appengine/tools/dev_appserver.py b/AppServer/google/appengine/tools/dev_appserver.py index b6d1d7fe68..b2265d28f1 100644 --- a/AppServer/google/appengine/tools/dev_appserver.py +++ b/AppServer/google/appengine/tools/dev_appserver.py @@ -372,6 +372,7 @@ class AppServerRequest(object): infile: File-like object with input data from the request. force_admin: Allow request admin-only URLs to proceed regardless of whether user is logged in or is an admin. + secret_hash: Security for task queue paths """ ATTRIBUTES = ['relative_url', diff --git a/appscale_rhel b/appscale_rhel deleted file mode 100755 index 078ad6016c..0000000000 --- a/appscale_rhel +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# -# appscale: Starts and stops appscale programs -# -# chkconfig: 345 56 50 -# -# description: Starts and stops programs for appscale -# -# -# processname: appscale -PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" -prog="appscale" - -RETVAL=0 - -start() { - echo -n $"Starting $prog: " - echo "Starting God" - god -c /root/appscale/AppLoadBalancer/config/global.god - echo "0" > /proc/sys/net/ipv4/icmp_echo_ignore_broadcasts - #ruby /root/appscale/AppController/changeRootPwd.rb -} -stop() { - echo -n $"Stopping $prog: " - echo "Stopping God" - pkill -9 god -} -restart() { - stop - start -} - -# See how we were called. -case "$1" in - start) - start - ;; - stop) - stop - ;; - status) - echo "no status to report" - ;; - restart|reload) - restart - ;; - condrestart) - ;; - *) - echo $"Usage: $0 {start|stop|status|restart|reload|condrestart}" - exit 1 -esac - -exit $? diff --git a/appscale_scratch_install.sh b/appscale_scratch_install.sh deleted file mode 100644 index 094c544b97..0000000000 --- a/appscale_scratch_install.sh +++ /dev/null @@ -1,502 +0,0 @@ -cp /etc/apt/sources.list /etc/apt/sources.list.pre-appscale -echo "deb http://us.archive.ubuntu.com/ubuntu/ jaunty main restricted" > /etc/apt/sources.list -echo "deb-src http://us.archive.ubuntu.com/ubuntu/ jaunty main restricted" >> /etc/apt/sources.list - -echo "deb http://us.archive.ubuntu.com/ubuntu/ jaunty-updates main restricted" >> /etc/apt/sources.list -echo "deb-src http://us.archive.ubuntu.com/ubuntu/ jaunty-updates main restricted" >> /etc/apt/sources.list - -echo "deb http://us.archive.ubuntu.com/ubuntu/ jaunty universe" >> /etc/apt/sources.list -echo "deb-src http://us.archive.ubuntu.com/ubuntu/ jaunty universe" >> /etc/apt/soure.list -echo "deb http://us.archive.ubuntu.com/ubuntu/ jaunty-updates universe" >> /etc/apt/sources.list -echo "deb-src http://us.archive.ubuntu.com/ubuntu/ jaunty-updates universe" >> /etc/apt/sources.list - -echo "deb http://us.archive.ubuntu.com/ubuntu/ jaunty multiverse" >> /etc/apt/sources.list -echo "deb-src http://us.archive.ubuntu.com/ubuntu/ jaunty multiverse" >> /etc/apt/sources.list -echo "deb http://us.archive.ubuntu.com/ubuntu/ jaunty-updates multiverse" >> /etc/apt/sources.list -echo "deb-src http://us.archive.ubuntu.com/ubuntu/ jaunty-updates multiverse" >> /etc/apt/sources.list - -echo "deb http://security.ubuntu.com/ubuntu jaunty-security main restricted" >> /etc/apt/sources.list -echo "deb-src http://security.ubuntu.com/ubuntu jaunty-security main restricted" >> /etc/apt/sources.list -echo "deb http://security.ubuntu.com/ubuntu jaunty-security universe" >> /etc/apt/sources.list -echo "deb-src http://security.ubuntu.com/ubuntu jaunty-security universe" >> /etc/apt/sources.list -echo "deb http://security.ubuntu.com/ubuntu jaunty-security multiverse" >> /etc/apt/sources.list -echo "deb-src http://security.ubuntu.com/ubuntu jaunty-security multiverse" >> /etc/apt/sources.list - -apt-get update -apt-get -y upgrade -apt-get -y dist-upgrade - -apt-get -y install xterm ssh openssl libssl-dev sun-java6-jdk python2.6-dev python-m2crypto python-soappy make gcc g++ ntp python-yaml cvs wget autoconf automake libtool bison flex zlib1g zlib1g-dev libzlib-ruby ruby1.8 libzip-ruby1.8 libboost-dev ruby1.8-dev libopenssl-ruby libopenssl-ruby1.8 libruby1.8 irb1.8 rdoc iptables - -apt-get -y install cmake libboost-filesystem-dev libboost-serialization-dev libboost-thread-dev libboost-program-options-dev libboost-iostreams-dev libboost-python-dev liblog4cpp5-dev libexpat1-dev libreadline5-dev libncurses5-dev libevent-dev build-essential automake pkg-config doxygen graphviz rsync lsof - -apt-get -y install make tcl-dev libbz2-dev libreadline-dev libgdbm-dev python-tk byacc tk8.4-dev sun-java5-jdk ant libdb4.7++-dev nginx python-imaging haproxy - -apt-get -y remove gij ecj - -cp /etc/hosts /etc/hosts.orig -echo "127.0.0.1 localhost localhost.localdomain"> /etc/hosts -echo "::1 ip6-localhost ip6-loopback">> /etc/hosts -echo "fe00::0 ip6-localnet">> /etc/hosts -echo "ff00::0 ip6-mcastprefix" >> /etc/hosts -echo "ff02::1 ip6-allnodes" >> /etc/hosts -echo "ff02::2 ip6-allrouters" >>/etc/hosts -echo "ff02::3 ip6-allhosts" >>/etc/hosts - -update-alternatives --install /bin/sh sh /bin/dash 1 -update-alternatives --install /bin/sh sh /bin/bash 1 -update-alternatives --config sh - -echo "source /root/appscale/appscale.env" >> /root/.bashrc -echo "export APPSCALE_HOME=/root/appscale" >> /root/.bashrc -echo "export PYTHON_EGG_CACHE=/tmp/.python_eggs" >> /root/.bashrc -echo "export PYTHONPATH=/usr/local/python-2.6/lib/python26.zip:/usr/local/python-2.6/lib/python2.6:/usr/local/python-2.6/lib/python2.6/plat-linux2:/usr/local/python-2.6/lib/python2.6/lib-tk:/usr/local/python-2.6/lib/python2.6/lib-dynload:/usr/local/python-2.6/lib/python2.6/site-packages:/usr/lib/python2.6/site-packages:/var/lib/python-support:/usr/share/python-support/python-yaml:/usr/share/python-support:/usr/share/python-support/python-m2crypto:/usr/share/python-support/python-soappy:/root/appscale/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/src/py/ThriftClient/gen-py:/usr/lib/python-support/python-m2crypto/python2.6/M2Crypto/" >> /root/.bashrc - -export APPSCALE_HOME=/root/appscale -export PYTHONPATH=/usr/local/python-2.6/lib/python25.zip:/usr/local/python-2.6/lib/python2.6:/usr/local/python-2.6/lib/python2.6/plat-linux2:/usr/local/python-2.6/lib/python2.6/lib-tk:/usr/local/python-2.6/lib/python2.6/lib-dynload:/usr/local/python-2.6/lib/python2.6/site-packages:/usr/lib/python2.6/site-packages:/var/lib/python-support:/usr/share/python-support/python-yaml:/usr/share/python-support:/usr/share/python-support/python-m2crypto:/usr/share/python-support/python-soappy:/root/appscale/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/src/py/ThriftClient/gen-py:/usr/lib/python-support/python-m2crypto/python2.6/M2Crypto/ - -ln /usr/bin/ruby1.8 /usr/bin/ruby - -apt-get -y install bzr -bzr branch lp:~appscale-maintainers/appscale/appscale -if [ $? != 0 ]; then echo "UNABLE TO DOWNLOAD APPSCALE. CHECK BZR." ; exit; fi - -#wget http://appscale.googlecode.com/files/appscale-1.1.tar.gz -#if [ $? != 0 ]; then echo "UNABLE TO DOWNLOAD APPSCALE TAR BALL. CHECK LINK." ; exit; fi -#tar zxvf appscale-1.1.tar.gz -#rm appscale-1.1.tar.gz - -cd ${APPSCALE_HOME} -mkdir downloads -cd ${APPSCALE_HOME}/downloads -wget http://appscale.cs.ucsb.edu/appscale_files/thrift.tgz -O ${APPSCALE_HOME}/downloads/thrift.tgz -if [ $? != 0 ]; then echo "UNABLE TO DOWNLOAD THRIFT. CHECK LINK." ; exit; fi - -tar zxfv thrift.tgz -cd ${APPSCALE_HOME}/downloads/thrift -./bootstrap.sh -CONFIG_SHELL=/bin/bash; export CONFIG_SHELL -$CONFIG_SHELL ./configure --without-csharp --disable-gen-java --without-java -make -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE THRIFT. " ; exit; fi -make install -cd ${APPSCALE_HOME}/downloads/thrift/lib/py -make install -cd ${APPSCALE_HOME}/downloads/thrift/lib/rb -make install -cd ${APPSCALE_HOME}/downloads/thrift/lib/perl -perl Makefile.PL -make install -echo "include /usr/local/lib" >> /etc/ld.so.conf - -cd ${APPSCALE_HOME}/downloads -wget http://appscale.cs.ucsb.edu/appscale_files/thrift-hypertable.tar.gz -O ${APPSCALE_HOME}/downloads/thrift-hypertable.tar.gz -if [ $? != 0 ]; then echo "UNABLE TO DOWNLOAD THRIFT-HYPERTABLE. CHECK LINK." ; exit; fi - -tar zxfv thrift-hypertable.tar.gz -cd ${APPSCALE_HOME}/downloads/thrift-hypertable -./bootstrap.sh -CONFIG_SHELL=/bin/bash; export CONFIG_SHELL -$CONFIG_SHELL ./configure -make -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE THRIFT HYPERTABLE. " ; exit; fi -make install -cd ${APPSCALE_HOME}/downloads/thrift-hypertable/lib/py -make install -cd ${APPSCALE_HOME}/downloads/thrift-hypertable/lib/rb -make install -cd ${APPSCALE_HOME}/downloads/thrift-hypertable/lib/perl -perl Makefile.PL -make install - -cd ${APPSCALE_HOME}/downloads -wget http://appscale.cs.ucsb.edu/appscale_files/rubygems-1.3.1.tgz -O ${APPSCALE_HOME}/downloads/rubygems-1.3.1.tgz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD RUBY GEMS. CHECK LINK." ; exit; fi - -tar zxvf rubygems-*.tgz -cd ${APPSCALE_HOME}/downloads/rubygems-1.3.1 -ruby setup.rb -ln -sf /usr/bin/gem1.8 /usr/bin/gem - -gem update -gem install god redgreen - -gem install -v=2.3.4 rails -gem install mongrel mongrel_cluster - -cd ${APPSCALE_HOME} -cp AppLoadBalancer/config/load-balancer.conf /etc/nginx/sites-enabled/ -cp AppLoadBalancer/config/haproxy.cfg /etc/haproxy/ - -cd ${APPSCALE_HOME}/downloads -wget http://appscale.cs.ucsb.edu/appscale_files/hyperic-sigar-1.6.0.tar.gz -O ${APPSCALE_HOME}/downloads/hyperic-sigar-1.6.0.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD HYPERIC SIGAR. CHECK LINK." ; exit; fi - -tar -xzvf hyperic-sigar-1.6.0.tar.gz -cp ${APPSCALE_HOME}/downloads/hyperic-sigar-1.6.0/sigar-bin/include/*.h /usr/local/include -# 64 BIT -cp ${APPSCALE_HOME}/downloads/hyperic-sigar-1.6.0/sigar-bin/lib/libsigar-amd64-linux.so hyperic-sigar-1.6.0/sigar-bin/lib/libsigar-ia64-linux.so /usr/local/lib/ -# 32 BIT (comment the above line) -#cp hyperic-sigar-1.6.0/sigar-bin/lib/libsigar-x86-linux.so /usr/local/lib/ -ldconfig - -gem install -y capistrano - -echo "export PATH=\${PATH}:/var/lib/gems/1.8/bin" >> /root/.bashrc -export PATH=${PATH}:/var/lib/gems/1.8/bin - -cd ${APPSCALE_HOME}/AppDB -rm -rf hadoop-0.20.0 -wget http://appscale.cs.ucsb.edu/appscale_files/hadoop-0.20.0.tar.gz -O ${APPSCALE_HOME}/AppDB/hadoop-0.20.0.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD HADOOP 0.20.0. CHECK LINK." ; exit; fi - -tar xvzf hadoop-0.20.0.tar.gz -cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0 -echo "export APPSCALE_HOME=/root/appscale" >> ./conf/hadoop-env.sh -echo "export JAVA_HOME=/usr/lib/jvm/java-6-sun" >> ./conf/hadoop-env.sh - -echo "export HADOOP_HOME=${APPSCALE_HOME}/AppDB/hadoop-0.20.0" >> ./conf/hadoop-env.sh - -echo "export HBASE_HOME=${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.3" >> ./conf/hadoop-env.sh - -echo "export HADOOP_CLASSPATH=${HBASE_HOME}:${HBASE_HOME}/hbase/hbase-0.20.3.jar:${HBASE_HOME}/hbase/hbase-0.20.3-test.jar:${HBASE_HOME}/conf:${HBASE_HOME}/build/classes:${HADOOP_HOME}/build/classes" >> ./conf/hadoop-env.sh - -echo "export CLASSPATH=${CLASSPATH}:${APPSCALE_HOME}/AppDB/hadoop-0.20.0/build/classes" >> ./conf/hadoop-env.sh - -echo "export HADOOP_HEAPSIZE=2000" >> ./conf/hadoop-env.sh - -cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0 -wget http://issues.apache.org/jira/secure/attachment/12405513/hadoop-includes-2.patch -O ${APPSCALE_HOME}/AppDB/hadoop-0.20.0/hadoop-includes-2.patch -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO HADOOP PATH. CHECK LINK." ; exit; fi -patch -p0 < hadoop-includes-2.patch -cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0/src/c++/utils -sh configure - -make - -sed -i "s/CXXFLAGS = -g/CXXFLAGS = -fPIC -g/g" ./Makefile - -make clean -make - -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE HADOOP UTIL. " ; exit; fi - -make install -cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0/src/c++/pipes -sh configure -make -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE HADOOP PIPES. " ; exit; fi - -make install -echo "${APPSCALE_HOME}/AppDB/hadoop-0.20.0/src/c++/install/lib" | tee -a /etc/ld.so.conf.d/hadoop.conf -ldconfig - - -cd ${APPSCALE_HOME}/AppDB/hypertable -rm -rf src build 0.9.2.5 -mkdir -p src build -cd ${APPSCALE_HOME}/AppDB/hypertable/src -wget http://appscale.cs.ucsb.edu/appscale_files/hypertable-0.9.2.5-alpha-src.tar.gz -O ${APPSCALE_HOME}/AppDB/hypertable/src/hypertable-0.9.2.5-alpha-src.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD HYPERTABLE 0.9.2.5. CHECK LINK." ; exit; fi - -tar xvzf hypertable-0.9.2.5-alpha-src.tar.gz - -export HYPERTABLE=${APPSCALE_HOME}/AppDB/hypertable -export HADOOP=${APPSCALE_HOME}/AppDB/hadoop-0.20.0 -cd ${APPSCALE_HOME}/AppDB/hypertable/build -cmake -DHADOOP_INCLUDE_PATH=${HADOOP}/src/c++/install/include/ -DHADOOP_LIB_PATH=${HADOOP}/src/c++/install/lib/ -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=${HYPERTABLE} -DJAVA_INCLUDE_PATH=/usr/lib/jvm/java-6-sun/include/ -DJAVA_INCLUDE_PATH2=/usr/lib/jvm/java-6-sun/include/linux/ ../src/hypertable-0.9.2.5-alpha - -cp ${APPSCALE_HOME}/AppDB/hypertable/hypertablefix/TableRangeMap.cc ${APPSCALE_HOME}/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/contrib/cc/MapReduce/TableRangeMap.cc -cp ${APPSCALE_HOME}/AppDB/hypertable/hypertablefix/TableReader.cc ${APPSCALE_HOME}/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/contrib/cc/MapReduce/TableReader.cc - -make -j 4 -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE HYPERTABLE 0.9.2.5. " ; exit; fi - - -make install -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE INSTALL HYPERTABLE 0.9.2.5. " ; exit; fi - -echo "${HYPERTABLE}/0.9.2.5/lib" | tee -a /etc/ld.so.conf.d/hypertable.conf -ldconfig - - -cd ${APPSCALE_HOME}/AppDB -rm -fdr ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.3 -cd ${APPSCALE_HOME}/AppDB/hbase -wget http://appscale.cs.ucsb.edu/appscale_files/hbase-0.20.3.tar.gz -O ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.3.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD HBASE 0.20.3. CHECK LINK." ; exit; fi - -tar zxvf hbase-0.20.3.tar.gz -echo "export APPSCALE_HOME=/root/appscale" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.3/conf/hbase-env.sh -echo "export JAVA_HOME=/usr/lib/jvm/java-6-sun" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.3/conf/hbase-env.sh - -echo "export HBASE_CLASSPATH=${APPSCALE_HOME}/AppDB/hadoop-0.20.0/build/classes" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.3/conf/hbase-env.sh - -echo "export HBASE_HEAPSIZE=2000" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.3/conf/hbase-env.sh -echo "export HBASE_MANAGES_ZK=FALSE" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.3/conf/hbase-env.sh - -cp ${APPSCALE_HOME}/AppDB/hbase/hbasefix/DistributedFileSystem.java ${APPSCALE_HOME}/AppDB/hadoop-0.20.0/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java -cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0 -ant jar -cp build/hadoop-0.20.1-dev-core.jar ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.3/lib/hadoop-0.20.0-core.jar - -cp /root/appscale/appscale /etc/init.d/ -chmod ugo+x /etc/init.d/appscale -update-rc.d appscale defaults - -cp /root/appscale/haproxy /etc/init.d/ -chmod ugo+x /etc/init.d/haproxy -update-rc.d haproxy defaults - - -#install the tools so that auto deployment over ec2 is possible -#TODO: add the same for the faster euca tools -sh ${APPSCALE_HOME}/ec2_scratch_install.sh - -echo "source /root/appscale/appscale.env" >> /root/.bashrc -echo "export PATH=/usr/local/ec2-api-tools/bin/:\$PATH" >> /root/.bashrc - -echo "export EC2_HOME=/usr/local/ec2-api-tools" >> /root/.bashrc - -echo "export EC2_PRIVATE_KEY=/root/appscale/.appscale/certs/mykey.pem " >> /root/.bashrc - -echo "export EC2_CERT=/root/appscale/.appscale/certs/mycert.pem " >> /root/.bashrc - -echo "export JAVA_HOME=/usr/lib/jvm/java-6-sun" >> /root/.bashrc - -source /root/.bashrc -mkdir -p ${APPSCALE_HOME}/logs - -# Install Cassandra database - version 0.50.0 -echo "Install Cassandra 0.50.0 for AppScale" -mkdir -p ${APPSCALE_HOME}/AppDB/cassandra -cd ${APPSCALE_HOME}/AppDB/cassandra -wget http://appscale.cs.ucsb.edu/appscale_files/apache-cassandra-incubating-0.5.0-src.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD CASSANDRA. CHECK LINK." ; exit; fi -tar xzvf apache-cassandra-incubating-0.5.0-src.tar.gz -mv apache-cassandra-incubating-0.5.0-src cassandra -rm -f apache-cassandra-incubating-0.5.0-src.tar.gz -cd ${APPSCALE_HOME}/AppDB/cassandra/cassandra -ant -mkdir /var/cassandra -chmod +x bin/cassandra -cp ${APPSCALE_HOME}/.appscale/cassie_templates/cassandra.in.sh ${APPSCALE_HOME}/AppDB/cassandra/cassandra/bin - -# Voldemort pre-req - protocol buffers -cd ${APPSCALE_HOME} -wget http://appscale.cs.ucsb.edu/appscale_files/protobuf-2.3.0.tar.gz -tar zxvf protobuf-2.3.0.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD PROTOBUF. CHECK LINK." ; exit; fi -tar zxvf protobuf-2.3.0.tar.gz -rm protobuf-2.3.0.tar.gz -cd protobuf-2.3.0 -./configure -make -make check -make install -cd python -python setup.py install - -# Install Voldemort database - 0.80 -mkdir -p ${APPSCALE_HOME}/AppDB/voldemort -cd ${APPSCALE_HOME}/AppDB/voldemort -wget http://appscale.cs.ucsb.edu/appscale_files/voldemort-0.80.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD VOLDEMORT. CHECK LINK." ; exit; fi -tar xzvf voldemort-0.80.tar.gz -mv voldemort-0.80 voldemort -rm voldemort-0.80.tar.gz -cd ${APPSCALE_HOME}/AppDB/voldemort/voldemort -ant jar -mkdir /var/voldemort -mkdir -p ${APPSCALE_HOME}/AppDB/voldemort/voldemort/config/appscale/config -chmod +x bin/voldemort-server.sh - -# MySQL 64 bit -mkdir -p ${APPSCALE_HOME}/AppDB/mysql -cd ${APPSCALE_HOME}/AppDB/mysql -wget http://appscale.cs.ucsb.edu/appscale_files/mysql-cluster-gpl-6.3.20-linux-x86_64-glibc23.tar.gz -O ${APPSCALE_HOME}/AppDB/mysql/mysql-cluster-gpl-6.3.20-linux-x86_64-glibc23.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD MYSQL. CHECK LINK." ; exit; fi -mv mysql-cluster-gpl-6.3.20-linux-x86_64-glibc23.tar.gz mysql.tar.gz - -# FOR 32 BIT COMMENT THE ABOVE THREE LINES AND UNCOMMENT -# THE FOLLOWING THREE LINES -###wget http://appscale.cs.ucsb.edu/appscale_files/mysql-cluster-gpl-6.3.20-linux-i686-glibc23.tar.gz -###if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD MYSQL. CHECK LINK." ; exit; fi -###mv mysql-cluster-gpl-6.3.20-linux-i686-glibc23.tar.gz mysql.tar.gz - -# MongoDB 64bit - version 1.2.2 -cd ${APPSCALE_HOME}/AppDB/ -wget http://appscale.cs.ucsb.edu/appscale_files/mongodb-linux-x86_64-1.2.2.tgz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD MONGO. CHECK LINK." ; exit; fi -tar zxvf mongodb-linux-x86_64-1.2.2.tgz -mv mongodb-linux-x86_64-1.2.2 mongodb -chmod +x mongodb/bin/mongo mongodb/bin/mongod -rm mongodb-linux-x86_64-1.2.2.tgz - -cd ${APPSCALE_HOME} -wget http://pypi.python.org/packages/2.6/s/setuptools/setuptools-0.6c9-py2.6.egg#md5=ca37b1ff16fa2ede6e19383e7b59245a -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD SETUPTOOLS. CHECK LINK." ; exit; fi -tar zxvf mongodb-linux-x86_64-1.0.0.tgz -sh setuptools-0.6c9-py2.6.egg -rm setuptools-0.6c9-py2.6.egg -easy_install pymongo - -# install euca2ools dependencies -cd ${APPSCALE_HOME} -apt-get -y install python-dev swig libssl-dev -wget http://kings.cs.ucsb.edu/appscale_files/euca2ools-1.0-src-deps.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD EUCA TOOLS. CHECK LINK." ; exit; fi -tar zvxf euca2ools-1.0-src-deps.tar.gz -cd euca2ools-1.0-src-deps -tar zxvf boto-1.8d.tar.gz -cd boto-1.8d -sudo python setup.py install - -# install euca2ools -cd ${APPSCALE_HOME} -wget http://kings.cs.ucsb.edu/appscale_files/euca2ools-1.0.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD EUCA TOOLS. CHECK LINK." ; exit; fi -tar zxvf euca2ools-1.0.tar.gz -cd euca2ools-1.0 -make -cd -rm -rf euca2ools-1.0* - -# install appscale-tools -cd ${APPSCALE_HOME} -wget http://kings.cs.ucsb.edu/appscale_files/appscale-tools-1.3.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD APPSCALE TOOLS. CHECK LINK." ; exit; fi -tar zxvf appscale-tools-1.3.tar.gz -mv appscale-tools /usr/local/ - -# install memcached -# The version being installed from source seg faults upon connection -# Just use the version in the repo until a solution is found --jkupferman -apt-get -y install memcached - -#cd ${APPSCALE_HOME} -#wget http://kings.cs.ucsb.edu/appscale_files/memcached-1.4.2.tar.gz -#if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD MEMCACHE. CHECK LINK." ; exit; fi -#tar zxvf memcached-1.4.2.tar.gz -#cd memcached-1.4.2 -#./configure -#make -#make install -#cd .. -#rm -rf memcached-1.4.2* - -# install python lib for memcached -apt-get -y install python-memcache - -# this isn't working right now - will return to this with raj -#cd ${APPSCALE_HOME} -#wget http://kings.cs.ucsb.edu/appscale_files/python-memcached-1.44.tar.gz -#tar zxvf python-memcached-1.44.tar.gz -#cd python-memcached-1.44 -#python setup.py install -#cd .. -#rm -rf python-memcached-1.44* - -# install memcachedb - -# first install berkeley db -cd ${APPSCALE_HOME} -wget http://kings.cs.ucsb.edu/appscale_files/db-4.8.24.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD BERK DB. CHECK LINK." ; exit; fi -tar zxvf db-4.8.24.tar.gz -cd db-4.8.24 -cd build_unix/ -../dist/configure -make -make install -cd ${APPSCALE_HOME} -rm -rf db-4.8.24* - -# next, install libevent -cd ${APPSCALE_HOME} -wget http://kings.cs.ucsb.edu/appscale_files/libevent-1.4.12-stable.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD LIB EVENT. CHECK LINK." ; exit; fi -tar zxvf libevent-1.4.12-stable.tar.gz -cd libevent-1.4.12-stable -./configure -make -make install -echo "include /usr/local/lib" >> /etc/ld.so.conf -echo "include /usr/local/BerkeleyDB.4.7/lib" >> /etc/ld.so.conf -ldconfig -cd ${APPSCALE_HOME} -rm -rf libevent-1.4.12-stable* - -# now we can install memcachedb 1.2.1-beta -cd ${APPSCALE_HOME}/AppDB/memcachedb -wget http://kings.cs.ucsb.edu/appscale_files/memcachedb.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD MEMCAHCEDB. CHECK LINK." ; exit; fi -tar zxvf memcachedb.tar.gz -cd memcachedb -./configure --enable-threads --with-libevent=/usr/local/lib -make -make install -cd ${APPSCALE_HOME}/AppDB/memcachedb -rm memcachedb.tar.gz - -apt-get -y install curl screen tcpdump -apt-get -y remove ant mysql-server mysql-common mysql-client -apt-get -y install python-mysqldb -apt-get -y install python2.5 - -# install TimesTen depending -apt-get -y install unixodbc-dev unzip -cd ${APPSCALE_HOME}/AppDB/timesten -wget http://pyodbc.googlecode.com/files/pyodbc-2.1.6.zip -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD TIMESTEN. CHECK LINK." ; exit; fi -unzip pyodbc-2.1.6.zip -cd pyodbc-2.1.6 -python setup.py install -cat >> /root/.bashrc <> ~/.bashrc - -# install x10 support -cd /usr/local -wget http://appscale.cs.ucsb.edu/appscale_files/x10-2.0.1_linux_x86_64.tgz -tar zxvf x10-2.0.1_linux_x86_64.tgz -rm x10-2.0.1_linux_x86_64.tgz -echo 'export PATH=/usr/local/x10/bin:$PATH' >> ~/.bashrc - -###################### -# Prime Python eggs -###################### -python2.6 ${APPSCALE_HOME}/AppDB/appscale_server.py -pkill python - -# finally -# cgb: control-c here if building on ec2 -# otherwise it messes up the ssh key that ec2 puts on and you can't log in -ssh-keygen -cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys - diff --git a/appscale_scratch_install_hardy.sh b/appscale_scratch_install_hardy.sh deleted file mode 100644 index 950ed9c4f0..0000000000 --- a/appscale_scratch_install_hardy.sh +++ /dev/null @@ -1,435 +0,0 @@ -#!/bin/bash -x - -echo "hardy installation for appscale" - -############################################################################## -# functions # -############################################################################## - -function check_hardy { - RELEASE=`lsb_release -a | grep Codename | awk '{print $2}'` - if [ "$RELEASE" != "hardy" ]; then - echo "this is not hardy" - exit; - fi -} - -function upgrade_distribution { - echo "deb http://us.archive.ubuntu.com/ubuntu/ hardy main restricted" >> /etc/apt/sources.list - echo "deb-src http://us.archive.ubuntu.com/ubuntu/ hardy main restricted" >> /etc/apt/sources.list - - echo "deb http://us.archive.ubuntu.com/ubuntu/ hardy-updates main restricted" >> /etc/apt/sources.list - echo "deb-src http://us.archive.ubuntu.com/ubuntu/ hardy-updates main restricted" >> /etc/apt/sources.list - - echo "deb http://us.archive.ubuntu.com/ubuntu/ hardy universe" >> /etc/apt/sources.list - echo "deb-src http://us.archive.ubuntu.com/ubuntu/ hardy universe" >> /etc/apt/soure.list - echo "deb http://us.archive.ubuntu.com/ubuntu/ hardy-updates universe" >> /etc/apt/sources.list - echo "deb-src http://us.archive.ubuntu.com/ubuntu/ hardy-updates universe" >> /etc/apt/sources.list - - echo "deb http://us.archive.ubuntu.com/ubuntu/ hardy multiverse" >> /etc/apt/sources.list - echo "deb-src http://us.archive.ubuntu.com/ubuntu/ hardy multiverse" >> /etc/apt/sources.list - echo "deb http://us.archive.ubuntu.com/ubuntu/ hardy-updates multiverse" >> /etc/apt/sources.list - echo "deb-src http://us.archive.ubuntu.com/ubuntu/ hardy-updates multiverse" >> /etc/apt/sources.list - - echo "deb http://security.ubuntu.com/ubuntu hardy-security main restricted" >> /etc/apt/sources.list - echo "deb-src http://security.ubuntu.com/ubuntu hardy-security main restricted" >> /etc/apt/sources.list - echo "deb http://security.ubuntu.com/ubuntu hardy-security universe" >> /etc/apt/sources.list - echo "deb-src http://security.ubuntu.com/ubuntu hardy-security universe" >> /etc/apt/sources.list - echo "deb http://security.ubuntu.com/ubuntu hardy-security multiverse" >> /etc/apt/sources.list - echo "deb-src http://security.ubuntu.com/ubuntu hardy-security multiverse" >> /etc/apt/sources.list - - apt-get update - apt-get -y upgrade - apt-get update && apt-get upgrade - apt-get dist-upgrade -} - -function make_etc_hosts { - echo "127.0.0.1 localhost"> /etc/hosts - echo "::1 ip6-localhost ip6-loopback">> /etc/hosts - echo "fe00::0 ip6-localnet">> /etc/hosts - echo "ff00::0 ip6-mcastprefix" >> /etc/hosts - echo "ff02::1 ip6-allnodes" >> /etc/hosts - echo "ff02::2 ip6-allrouters" >>/etc/hosts - echo "ff02::3 ip6-allhosts" >>/etc/hosts -} - -function install_programming_tools { - apt-get -y install ssh openssl libssl-dev - apt-get -y install sun-java6-jdk - apt-get -y install python-m2crypto python-soappy python-yaml - apt-get -y install python-mysqldb - apt-get -y install make gcc g++ ntp cvs wget autoconf automake libtool bison flex - apt-get -y install zlib1g zlib1g-dev libzlib-ruby - apt-get -y install ruby1.8 libzip-ruby1.8 libboost-dev ruby1.8-dev libopenssl-ruby libopenssl-ruby1.8 libruby1.8 irb1.8 - apt-get -y install rdoc iptables - ln /usr/bin/ruby1.8 /usr/bin/ruby - echo "export JAVA_HOME=/usr/lib/jvm/java-6-sun" >> /root/.bashrc -} - -function install_python2_5 { - apt-get -y install python2.5 - apt-get -y install python-mysqldb python-pycurl - ln -s /usr/bin/python2.5 /usr/bin/python2.6 -} - -function install_python2_6 { - mkdir -p /root/pythonAppScale - cd /root/pythonAppScale - wget http://appscale.cs.ucsb.edu/appscale_files/Python-2.6.tgz - if [ $? = 1 ]; then echo "FAILURE: UNABLE TO DOWNLOAD PYTHON2.6. CHECK LINK." ; exit; fi - tar zxvf Python-2.6.tgz - cd /root/pythonAppScale/Python-2.6/ - ./configure - make - make install - cd ~ -} - -function install_programming_tools2 { - apt-get -y install cmake - apt-get -y install libboost-filesystem-dev libboost-serialization-dev libboost-thread-dev - apt-get -y install libboost-program-options-dev libboost-iostreams-dev libboost-python-dev - apt-get -y install liblog4cpp5-dev libexpat1-dev libreadline5-dev - apt-get -y install libncurses5-dev libevent-dev - apt-get -y install build-essential automake pkg-config doxygen graphviz rsync lsof - apt-get -y install make tcl-dev libbz2-dev libreadline-dev libgdbm-dev python-tk byacc tk8.4-dev - apt-get -y install sun-java5-jdk ant - apt-get -y install libdb4.6++-dev # for hardy .., for jaunty use libdb4.7++-dev. -} - -function remove_java_tools { - apt-get -y remove gij ecj -} - -function update_alternatives { - update-alternatives --install /bin/sh sh /bin/dash 1 - update-alternatives --install /bin/sh sh /bin/bash 1 - update-alternatives --config sh -} - -function make_root_dir { - export APPSCALE_HOME=/root/appscale - #echo "source /root/appscale/appscale.env" >> /root/.bashrc - echo "export APPSCALE_HOME=/root/appscale" >> /root/.bashrc - mkdir ${APPSCALE_HOME} -} - -function set_pythonpath_for_python2_6 { - echo "export PYTHONPATH=/usr/local/python-2.6/lib/python26.zip:/usr/local/python-2.6/lib/python2.6:/usr/local/python-2.6/lib/python2.6/plat-linux2:/usr/local/python-2.6/lib/python2.6/lib-tk:/usr/local/python-2.6/lib/python2.6/lib-dynload:/usr/local/python-2.6/lib/python2.6/site-packages:/usr/lib/python2.6/site-packages:/var/lib/python-support:/usr/share/python-support/python-yaml:/usr/share/python-support:/usr/share/python-support/python-m2crypto:/usr/share/python-support/python-soappy:/root/appscale/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/src/py/ThriftClient/gen-py:/usr/lib/python-support/python-m2crypto/python2.6/M2Crypto/" >> /root/.bashrc - source /root/.bashrc -} - -function set_pythonpath_for_python2_5 { - echo "export PYTHONPATH=/usr/local/python-2.5/lib/python25.zip:/usr/local/python-2.5/lib/python2.5:/usr/local/python-2.5/lib/python2.5/plat-linux2:/usr/local/python-2.5/lib/python2.5/lib-tk:/usr/local/python-2.5/lib/python2.5/lib-dynload:/usr/local/python-2.5/lib/python2.5/site-packages:/usr/lib/python2.5/site-packages:/var/lib/python-support:/usr/share/python-support/python-yaml:/usr/share/python-support:/usr/share/python-support/python-m2crypto:/usr/share/python-support/python-soappy:/root/appscale/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/src/py/ThriftClient/gen-py:/usr/lib/python-support/python-m2crypto/python2.5/M2Crypto/" >> /root/.bashrc - source /root/.bashrc -} - -function download_appscale_code { - apt-get -y install bzr - cd /root - bzr branch lp:~appscale-maintainers/appscale/appscale -} - -function install_thrift { - cd ${APPSCALE_HOME} - mkdir downloads - cd ${APPSCALE_HOME}/downloads - wget http://appscale.cs.ucsb.edu/appscale_files/thrift.tgz -O ${APPSCALE_HOME}/downloads/thrift.tgz - if [ $? = 1 ]; then echo "UNABLE TO DOWNLOAD THRIFT. CHECK LINK." ; exit; fi - - tar zxfv thrift.tgz - cd ${APPSCALE_HOME}/downloads/thrift - ./bootstrap.sh - CONFIG_SHELL=/bin/bash; export CONFIG_SHELL - $CONFIG_SHELL ./configure --without-csharp --disable-gen-java --without-java - make - make install - cd ${APPSCALE_HOME}/downloads/thrift/lib/py - make install - cd ${APPSCALE_HOME}/downloads/thrift/lib/rb - make install - cd ${APPSCALE_HOME}/downloads/thrift/lib/perl - perl Makefile.PL - make install - echo "include /usr/local/lib" >> /etc/ld.so.conf -} - -function install_thrift_hypertable { - cd ${APPSCALE_HOME}/downloads - wget http://appscale.cs.ucsb.edu/appscale_files/thrift-hypertable.tar.gz -O ${APPSCALE_HOME}/downloads/thrift-hypertable.tar.gz - if [ $? = 1 ]; then echo "UNABLE TO DOWNLOAD THRIFT-HYPERTABLE. CHECK LINK." ; exit; fi - - tar zxfv thrift-hypertable.tar.gz - cd ${APPSCALE_HOME}/downloads/thrift-hypertable - ./bootstrap.sh - CONFIG_SHELL=/bin/bash; export CONFIG_SHELL - $CONFIG_SHELL ./configure - make - make install - cd ${APPSCALE_HOME}/downloads/thrift-hypertable/lib/py - make install - cd ${APPSCALE_HOME}/downloads/thrift-hypertable/lib/rb - make install - cd ${APPSCALE_HOME}/downloads/thrift-hypertable/lib/perl - perl Makefile.PL - make install -} - -function install_ruby_gem { - cd ${APPSCALE_HOME}/downloads - wget http://appscale.cs.ucsb.edu/appscale_files/rubygems-1.3.1.tgz -O ${APPSCALE_HOME}/downloads/rubygems-1.3.1.tgz - if [ $? = 1 ]; then echo "FAILURE: UNABLE TO DOWNLOAD RUBY GEMS. CHECK LINK." ; exit; fi - - tar zxvf rubygems-*.tgz - cd ${APPSCALE_HOME}/downloads/rubygems-1.3.1 - ruby setup.rb - ln -sf /usr/bin/gem1.8 /usr/bin/gem - echo "export PATH=\${PATH}:/var/lib/gems/1.8/bin" >> /root/.bashrc - export PATH=${PATH}:/var/lib/gems/1.8/bin -} - -function install_god { - gem update - gem install god -} - -function install_rails { - gem install -v=2.0.2 rails - apt-get -y install nginx haproxy - gem install mongrel mongrel_cluster - - cd ${APPSCALE_HOME} - cp AppLoadBalancer/config/haproxy.cfg /etc/haproxy/ - cp AppLoadBalancer/config/nginx.conf /etc/nginx/ - cp AppLoadBalancer/config/load-balancer.conf /etc/nginx/sites-enabled/ - - cp /root/appscale/haproxy /etc/init.d/ - cd /etc/init.d - chmod ugo+x haproxy - update-rc.d haproxy defaults -} - -function install_sigar { - cd ${APPSCALE_HOME}/downloads - wget http://appscale.cs.ucsb.edu/appscale_files/hyperic-sigar-1.6.0.tar.gz -O ${APPSCALE_HOME}/downloads/hyperic-sigar-1.6.0.tar.gz - if [ $? = 1 ]; then echo "FAILURE: UNABLE TO DOWNLOADi HYPERIC SIGAR. CHECK LINK." ; exit; fi - - tar -xzvf hyperic-sigar-1.6.0.tar.gz - cp ${APPSCALE_HOME}/downloads/hyperic-sigar-1.6.0/sigar-bin/include/*.h /usr/local/include - # 64 BIT - cp ${APPSCALE_HOME}/downloads/hyperic-sigar-1.6.0/sigar-bin/lib/libsigar-amd64-linux.so hyperic-sigar-1.6.0/sigar-bin/lib/libsigar-ia64-linux.so /usr/local/lib/ - # 32 BIT (comment the above line) - #cp hyperic-sigar-1.6.0/sigar-bin/lib/libsigar-x86-linux.so /usr/local/lib/ - ldconfig -} - -function install_capistrano { - gem install -y capistrano -} - -function install_hadoop { - cd ${APPSCALE_HOME}/AppDB - rm -rf hadoop-0.20.0 - wget http://appscale.cs.ucsb.edu/appscale_files/hadoop-0.20.0.tar.gz -O ${APPSCALE_HOME}/AppDB/hadoop-0.20.0.tar.gz - if [ $? = 1 ]; then echo "FAILURE: UNABLE TO DOWNLOAD HADOOP 0.20.0. CHECK LINK." ; exit; fi - - tar xvzf hadoop-0.20.0.tar.gz - cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0 - echo "export APPSCALE_HOME=/root/appscale" >> ./conf/hadoop-env.sh - echo "export JAVA_HOME=/usr/lib/jvm/java-6-sun" >> ./conf/hadoop-env.sh - echo "export HADOOP_HOME=${APPSCALE_HOME}/AppDB/hadoop-0.20.0" >> ./conf/hadoop-env.sh - echo "export HBASE_HOME=${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha" >> ./conf/hadoop-env.sh - echo "export HADOOP_CLASSPATH=${HBASE_HOME}:${HBASE_HOME}/hbase/hbase-0.20.0-alpha.jar:${HBASE_HOME}/hbase/hbase-0.20.0-alpha-test.jar:${HBASE_HOME}/conf:${HBASE_HOME}/build/classes:${HADOOP_HOME}/build/classes" >> ./conf/hadoop-env.sh - echo "export CLASSPATH=${CLASSPATH}:${APPSCALE_HOME}/AppDB/hadoop-0.20.0/build/classes" >> ./conf/hadoop-env.sh - echo "export HADOOP_HEAPSIZE=2000" >> ./conf/hadoop-env.sh - - cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0 - wget http://issues.apache.org/jira/secure/attachment/12405513/hadoop-includes-2.patch -O ${APPSCALE_HOME}/AppDB/hadoop-0.20.0/hadoop-includes-2.patch - patch -p0 < hadoop-includes-2.patch - cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0/src/c++/utils - sh configure - make - - sed -i "s/CXXFLAGS = -g/CXXFLAGS = -fPIC -g/g" ./Makefile - make clean - make - make install - cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0/src/c++/pipes - sh configure - make - make install - echo "${APPSCALE_HOME}/AppDB/hadoop-0.20.0/src/c++/install/lib" | tee -a /etc/ld.so.conf.d/hadoop.conf - ldconfig -} - -function install_hypertable { - cd ${APPSCALE_HOME}/AppDB/hypertable - rm -rf src build 0.9.2.5 - mkdir -p src build - cd ${APPSCALE_HOME}/AppDB/hypertable/src - wget http://appscale.cs.ucsb.edu/appscale_files/hypertable-0.9.2.5-alpha-src.tar.gz -O ${APPSCALE_HOME}/AppDB/hypertable/src/hypertable-0.9.2.5-alpha-src.tar.gz - if [ $? = 1 ]; then echo "FAILURE: UNABLE TO DOWNLOAD HYPERTABLE 0.9.2.5. CHECK LINK." ; exit; fi - - tar xvzf hypertable-0.9.2.5-alpha-src.tar.gz - export HYPERTABLE=${APPSCALE_HOME}/AppDB/hypertable - export HADOOP=${APPSCALE_HOME}/AppDB/hadoop-0.20.0 - cd ${APPSCALE_HOME}/AppDB/hypertable/build - cmake -DHADOOP_INCLUDE_PATH=${HADOOP}/src/c++/install/include/ -DHADOOP_LIB_PATH=${HADOOP}/src/c++/install/lib/ -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=${HYPERTABLE} -DJAVA_INCLUDE_PATH=/usr/lib/jvm/java-6-sun/include/ -DJAVA_INCLUDE_PATH2=/usr/lib/jvm/java-6-sun/include/linux/ ../src/hypertable-0.9.2.5-alpha - - cp ${APPSCALE_HOME}/AppDB/hypertable/hypertablefix/TableRangeMap.cc ${APPSCALE_HOME}/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/contrib/cc/MapReduce/TableRangeMap.cc - cp ${APPSCALE_HOME}/AppDB/hypertable/hypertablefix/TableReader.cc ${APPSCALE_HOME}/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/contrib/cc/MapReduce/TableReader.cc - - make -j 4 - make install - echo "${HYPERTABLE}/0.9.2.5/lib" | tee -a /etc/ld.so.conf.d/hypertable.conf - ldconfig -} - -function install_hbase { - cd ${APPSCALE_HOME}/AppDB - rm -fdr ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha - cd ${APPSCALE_HOME}/AppDB/hbase - wget http://appscale.cs.ucsb.edu/appscale_files/hbase-0.20.0-alpha.tar.gz -O ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha.tar.gz - if [ $? = 1 ]; then echo "FAILURE: UNABLE TO DOWNLOAD HBASE 0.20.0-alpha. CHECK LINK." ; exit; fi - - tar zxvf hbase-0.20.0-alpha.tar.gz - echo "export APPSCALE_HOME=/root/appscale" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha/conf/hbase-env.sh - echo "export JAVA_HOME=/usr/lib/jvm/java-6-sun" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha/conf/hbase-env.sh - echo "export HBASE_CLASSPATH=${APPSCALE_HOME}/AppDB/hadoop-0.20.0/build/classes" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha/conf/hbase-env.sh - echo "export HBASE_HEAPSIZE=2000" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha/conf/hbase-env.sh - echo "export HBASE_MANAGES_ZK=FALSE" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha/conf/hbase-env.sh - - cp ${APPSCALE_HOME}/AppDB/hbase/hbasefix/DistributedFileSystem.java ${APPSCALE_HOME}/AppDB/hadoop-0.20.0/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java - cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0 - ant jar - cp build/hadoop-0.20.1-dev-core.jar ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha/lib/hadoop-0.20.0-core.jar -} - -function install_init_appscale { - cp /root/appscale/appscale /etc/init.d/ - cd /etc/init.d - chmod ugo+x appscale - update-rc.d appscale defaults -} - -function install_ec2_tools { - #On the machine from which you will upload and manipulate AppScale/Eucalyptus: - sh ${APPSCALE_HOME}/ec2_scratch_install.sh - echo "source /root/appscale/appscale.env" >> /root/.bashrc - echo "export PATH=/usr/local/ec2-api-tools/bin/:\$PATH" >> /root/.bashrc - echo "export EC2_HOME=/usr/local/ec2-api-tools" >> /root/.bashrc - echo "export EC2_PRIVATE_KEY=/root/appscale/.appscale/certs/mykey.pem " >> /root/.bashrc - echo "export EC2_CERT=/root/appscale/.appscale/certs/mycert.pem " >> /root/.bashrc - echo "export JAVA_HOME=/usr/lib/jvm/java-6-sun" >> /root/.bashrc -} - -function install_cassandra { - source /root/.bashrc - local TARBALL=appscale_cassandra.06302009.tgz - mkdir -p ${APPSCALE_HOME}/AppDB/cassandra - cd ${APPSCALE_HOME}/AppDB/cassandra - wget http://appscale.cs.ucsb.edu/appscale_files/${TARBALL} -O ${APPSCALE_HOME}/AppDB/cassandra/${TARBALL} - if [ $? = 1 ]; then echo "FAILURE: UNABLE TO DOWNLOAD ${TARBALL}. CHECK LINK." ; exit; fi - tar xzvf ${TARBALL}; rm -f ${TARBALL} - cd ${APPSCALE_HOME}/AppDB/cassandra/cassandra - ant build - mkdir /var/cassandra - chmod +x bin/cassandra -} - -function install_voldemort { - local TARBALL=appscale_voldemort.06302009.tgz - mkdir -p ${APPSCALE_HOME}/AppDB/voldemort - cd ${APPSCALE_HOME}/AppDB/voldemort - wget http://appscale.cs.ucsb.edu/appscale_files/${TARBALL} -O ${APPSCALE_HOME}/AppDB/voldemort/${TARBALL} - if [ $? = 1 ]; then echo "FAILURE: UNABLE TO DOWNLOAD ${TARBALL}. CHECK LINK." ; exit; fi - tar xzvf ${TARBALL}; rm ${TARBALL} - cd ${APPSCALE_HOME}/AppDB/voldemort/voldemort - ant jar - mkdir /var/voldemort - chmod +x bin/voldemort -} - -function install_mysql { - # MySQL 64 bit - local TARBALL=mysql-cluster-gpl-6.3.20-linux-x86_64-glibc23.tar.gz - mkdir -p ${APPSCALE_HOME}/AppDB/mysql - cd ${APPSCALE_HOME}/AppDB/mysql - wget http://appscale.cs.ucsb.edu/appscale_files/${TARBALL} -O ${APPSCALE_HOME}/AppDB/mysql/${TARBALL} - if [ $? = 1 ]; then echo "FAILURE: UNABLE TO DOWNLOAD ${TARBALL}. CHECK LINK." ; exit; fi - mv ${TARBALL} mysql.tar.gz - - apt-get -y remove mysql-server mysql-common mysql-client -} - -function create_ssh_keys { - ssh-keygen - cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys -} - -function change_controller_for_hardy { - cd ${APPSCALE_HOME}/AppController - mkdir backup - FILES=`ls *.rb` - for file in ${FILES}; do - sed 's/python2.6/python2.5/g' ${file} > ${file}.new - mv ${file} backup - mv ${file}.new ${file} - done -} - -function change_python_path_in_djinnserver { - local DJINN_SERVER=${APPSCALE_HOME}/AppController/djinnServer.rb - sed 's/python2.6/python2.5/g' ${DJINN_SERVER} > /tmp/djinnServer.rb - mv ${DJINN_SERVER} ${DJINN_SERVER}.org - mv /tmp/djinnServer.rb ${DJINN_SERVER} -} - -############################################################################## - -# Start Install AppScale -echo "Start Install AppScale .. " - -export APPSCALE_HOME=/root/appscale - -check_hardy -upgrade_distribution -make_etc_hosts -install_programming_tools | tee install_programming_tools.log -##install_python2_6 | tee install_python2_6.log -install_python2_5 | tee install_python2_5.log -install_programming_tools2 | tee install_programming_tools2.log -remove_java_tools -update_alternatives - -download_appscale_code | tee download_appscale_code.log -make_root_dir - -##set_pythonpath_for_python2_6 -set_pythonpath_for_python2_5 - -install_thrift -install_thrift_hypertable - -install_ruby_gem -install_god -install_rails -install_sigar -install_capistrano - -install_init_appscale -install_ec2_tools - -# install datastores -install_hadoop | tee install_hadoop.log -install_hypertable | tee install_hypertable.log -install_hbase | tee install_hbase.log -install_cassandra | tee install_cassandra.log -install_voldemort | tee install_voldemort.log -install_mysql | tee install_mysql.log - -#change_controller_for_hardy -change_python_path_in_djinnserver -create_ssh_keys - diff --git a/debian/appscale_install.sh b/debian/appscale_install.sh index 50a417237b..0130c52339 100755 --- a/debian/appscale_install.sh +++ b/debian/appscale_install.sh @@ -24,6 +24,8 @@ case "$1" in installmonitoring installthrift_fromsource installtornado_fromsource + installflexmock + installnose installhadoop # if [ "$DIST" = "jaunty" -o "$DIST" = "karmic" ]; then installzookeeper diff --git a/debian/appscale_install_functions.sh b/debian/appscale_install_functions.sh index a3a6014ced..65c1088286 100644 --- a/debian/appscale_install_functions.sh +++ b/debian/appscale_install_functions.sh @@ -258,6 +258,11 @@ installtornado() fi } +installnose() +{ + easy_install nose || exit 1 +} + installflexmock() { easy_install flexmock || exit 1 diff --git a/ec2_scratch_install.sh b/ec2_scratch_install.sh deleted file mode 100644 index 70dfc07a7f..0000000000 --- a/ec2_scratch_install.sh +++ /dev/null @@ -1,23 +0,0 @@ -# EC2 Scratch Install Section -cd /root/ -mkdir temp -cd temp -wget http://appscale.cs.ucsb.edu/appscale_files/ec2-api-tools-1.3-30349.zip -if [ $? = 1 ]; then echo "FAILURE: UNABLE TO DOWNLOAD EC2 API TOOLS. -CHECK LINK." ; exit; fi -wget http://appscale.cs.ucsb.edu/appscale_files/ec2-ami-tools-1.3-26357.zip -if [ $? = 1 ]; then echo "FAILURE: UNABLE TO DOWNLOAD EC2 API TOOLS. -CHECK LINK." ; exit; fi - -apt-get -y install unzip -unzip ec2-api-tools-1.3-30349.zip -unzip ec2-ami-tools-1.3-26357.zip -apt-get -y remove unzip - -mv ec2-api-tools-1.3-30349 ec2-api-tools -mv ec2-ami-tools-1.3-26357/* ec2-api-tools/ # should say directory not empty for bin and lib, that's fine -mv ec2-ami-tools-1.3-26357/bin/* ec2-api-tools/bin/ -mv ec2-ami-tools-1.3-26357/lib/* ec2-api-tools/lib/ -mv ec2-api-tools /usr/local/ -rm -rf /root/temp - diff --git a/non_interactive_appscale_install_rhel_rc2.sh b/non_interactive_appscale_install_rhel_rc2.sh deleted file mode 100644 index a8028b5e2c..0000000000 --- a/non_interactive_appscale_install_rhel_rc2.sh +++ /dev/null @@ -1,706 +0,0 @@ -#!/bin/bash -function pause(){ -if [ "$1" = "-i" ] -then - echo "press any key" - read -p "$*" -fi -} -if [ "$1" = "-i" ] -then - echo "Interactive mode" -fi -echo "Starting Script" - -pause $1 -echo "export APPSCALE_HOME=/root/appscale" >> /root/.bashrc -export APPSCALE_HOME=/root/appscale -export PYTHON_EGG_CACHE=/tmp/.python_eggs -ls /root/ibm_yum.sh -if [ $? != 0 ]; then echo "FAILURE: You must have ibm_yum.sh in /root/ in order to run this build script. " ; exit; fi -bash ibm_yum.sh -y update -# install packages here -pause $1 -bash ibm_yum.sh -y install ruby -pause $1 -bash ibm_yum.sh -y install ant -pause $1 -bash ibm_yum.sh -y install ruby-devel -pause $1 -bash ibm_yum.sh -y install gcc-c++ -pause $1 -bash ibm_yum.sh -y install make -pause $1 -bash ibm_yum.sh -y install doxygen -pause $1 -bash ibm_yum.sh -y install rsync -pause $1 -bash ibm_yum.sh -y install libtool -pause $1 -bash ibm_yum.sh -y install boost -pause $1 -bash ibm_yum.sh -y install boost-devel -pause $1 -bash ibm_yum.sh -y install rdoc -pause $1 -#bash ibm_yum.sh -y install MySQL-python -#pause $1 -bash ibm_yum.sh -y install zlib-devel -pause $1 -#bash ibm_yum.sh -y install libevent-devel -#pause $1 -bash ibm_yum.sh -y remove libevent -pause $1 -bash ibm_yum.sh -y install bison -pause $1 -bash ibm_yum.sh -y install flex -pause $1 -bash ibm_yum.sh -y install java-1.6.0-sun-devel -pause $1 -bash ibm_yum.sh -y install junit -pause $1 -bash ibm_yum.sh -y install ant-junit -pause $1 -bash ibm_yum.sh -y install swig -pause $1 -bash ibm_yum.sh -y install openssl-devel -pause $1 -bash ibm_yum.sh -y install byacc -pause $1 -bash ibm_yum.sh -y install pcre-devel -pause $1 -rhn_register -################################# -# get BZR with python2.4 for yum -################################# -cd -rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm -yum -y install bzr -pause $1 -yum -y install libyaml -pause $1 -yum -y install libyaml-devel -pause $1 -yum -y install bzip2-devel -pause $1 -yum -y install cmake -pause $1 -yum -y install log4cpp-devel -pause $1 -yum -y install readline-devel -pause $1 -yum -y install ncurses-devel -pause $1 - -# Grab the appscale source, after installing python 2.6 this breaks -cd -bzr branch lp:~appscale-maintainers/appscale/appscale -if [ $? != 0 ]; then echo "UNABLE TO DOWNLOAD APPSCALE. CHECK BZR." ; exit; fi -disabling mysql and hypertable for now - -mkdir -p ${APPSCALE_HOME}/downloads - -pause $1 -##################### -# Install libevent -##################### -cd ${APPSCALE_HOME}/downloads -wget http://kings.cs.ucsb.edu/appscale_files/libevent-1.4.12-stable.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD LIBEVENT. CHECK LINK." ; exit; fi -tar zxvf libevent-1.4.12-stable.tar.gz -cd libevent-1.4.12-stable -./configure -make -make install - -pause $1 - -cd ${APPSCALE_HOME}/downloads -wget http://www.graphviz.org/pub/graphviz/stable/SOURCES/graphviz-2.26.0.tar.gz -tar zxvf graphviz-2.26.0.tar.gz -cd graphviz-2.26.0 -./configure -make -make install -pause $1 - -cd ${APPSCALE_HOME}/downloads -wget http://sysoev.ru/nginx/nginx-0.7.64.tar.gz -tar zxvf nginx-0.7.64.tar.gz -cd nginx-0.7.64 -./configure -make -make install -pause $1 - -######################## -# Install libyaml by rpm -# If yum install fails -######################## -#cd ${APPSCALE_HOME}/downloads -#wget http://packages.sw.be/libyaml/libyaml-0.0.1-1.el5.rf.x86_64.rpm -#rpm -i libyaml-0.0.1-1.el5.rf.x86_64.rpm - - -# need junit for ant -export CLASSPATH=/usr/share/java/junit.jar - -# ant for hbase -cd ${APPSCALE_HOME}/downloads -wget http://www.gtlib.gatech.edu/pub/apache/ant/source/apache-ant-1.7.1-src.tar.gz -tar zxvf apache-ant-1.7.1-src.tar.gz -export JAVA_HOME=/usr/lib/jvm/java-1.6.0-sun.x86_64 -cd apache-ant-1.7.1/ -./bootstrap.sh -sh build.sh -export ANT_HOME=/root/appscale/downloads/apache-ant-1.7.1/bootstrap/ -echo "export ANT_HOME=/root/appscale/downloads/apache-ant-1.7.1/bootstrap/" >> ~/.bashrc -############### -# Get Python2.6 -############### -cd ${APPSCALE_HOME}/downloads -wget http://appscale.cs.ucsb.edu/appscale_files/Python-2.6.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO GET PYTHON2.6. " ; exit; fi -tar zxvf Python-2.6.tar.gz -cd Python-2.6 -./configure -make -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE PYTHON2.6. " ; exit; fi -make install -rm -f /usr/bin/python -ln -s /usr/local/bin/python2.6 /usr/bin/python -pause $1 - -cd ${APPSCALE_HOME}/downloads -wget http://downloads.sourceforge.net/project/swig/swig/swig-1.3.40/swig-1.3.40.tar.gz?use_mirror=voxel -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD SWIG. CHECK LINK." ; exit; fi -tar zxvf swig-1.3.40.tar.gz -cd swig-1.3.40 -./configure -make -make install -cp -f /usr/local/bin/swig /usr/bin/swig -pause $1 - -cd ${APPSCALE_HOME}/downloads -wget http://pypi.python.org/packages/source/M/M2Crypto/M2Crypto-0.20.2.tar.gz#md5=6c24410410d6eb1920ea43f77a93613a -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD M2Crypto. CHECK LINK." ; exit; fi -tar zxvf M2Crypto-0.20.2.tar.gz -cd M2Crypto-0.20.2 -#sed -i 's/opensslconf_x86_64.h/openssl\/opensslconf_x86_64.h/g' /usr/include/openssl/opensslconf.h -./fedora_setup.sh build -./fedora_setup.sh install -export PYTHONPATH="/usr/local/lib/python2.6/site-packages/M2Crypto" -python -c "from M2Crypto import SSL; ctx = SSL.Context()" -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO USE M2Crypto. CHECK LINK." ; exit; fi -pause $1 - -cd ${APPSCALE_HOME}/downloads -wget http://pyyaml.org/download/pyyaml/PyYAML-3.09.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD PYYAML. CHECK LINK." ; exit; fi -tar zxvf PyYAML-3.09.tar.gz -cd PyYAML-3.09 -python setup.py install - -pause $1 - -# expat libraries for hypertable -cd ${APPSCALE_HOME}/downloads -wget http://downloads.sourceforge.net/project/expat/expat/2.0.1/expat-2.0.1.tar.gz?use_mirror=softlayer -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO GET EXPAT. " ; exit; fi -tar zxvf expat-2.0.1.tar.gz -cd expat-2.0.1/ -./configure -make -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE EXPAT. " ; exit; fi -make install - -pause $1 - -# Update /etc/hosts file here -echo "fe00::0 ip6-localnet">> /etc/hosts -echo "ff00::0 ip6-mcastprefix" >> /etc/hosts -echo "ff02::1 ip6-allnodes" >> /etc/hosts -echo "ff02::2 ip6-allrouters" >>/etc/hosts -echo "ff02::3 ip6-allhosts" >>/etc/hosts - -update-alternatives --install /bin/sh sh /bin/dash 1 -update-alternatives --install /bin/sh sh /bin/bash 1 -update-alternatives --config sh - -echo "source /root/appscale/appscale.env" >> /root/.bashrc -echo "export PYTHON_EGG_CACHE=/tmp/.python_eggs" >> /root/.bashrc -echo "export PYTHONPATH=/usr/lib/python2.6/site-packages:/root/appscale/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/src/py/ThriftClient/gen-py:/usr/local/lib/python2.6/site-packages:/usr/local/lib/python2.6:/usr/local/lib/python2.6/site-packages/M2Crypto:/usr/lib/python2.6/site-packages/thrift" >> /root/.bashrc -source ~/.bashrc -########### -# fpconst -########### -cd ${APPSCALE_HOME}/downloads/ -wget http://pypi.python.org/packages/source/f/fpconst/fpconst-0.7.2.tar.gz#md5=10ba9e04129af23108d24c22c3a698b1 -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO GET FPCONST. " ; exit; fi -tar zxvf fpconst-0.7.2.tar.gz -cd fpconst-0.7.2 -python setup.py install -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO INSTALL FPCONST. " ; exit; fi - -pause $1 -############################# -# modified version of SOAPpy -############################ -cd ${APPSCALE_HOME}/downloads/ -wget http://appscale.cs.ucsb.edu/appscale_files/SOAPpy-0.12.0.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO GET SOAP. " ; exit; fi -tar zxvf SOAPpy-0.12.0.tar.gz -cd SOAPpy-0.12.0 -python setup.py install -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE SOAP. " ; exit; fi - -pause $1 -###################### -# Setup tools -###################### -cd ${APPSCALE_HOME}/downloads/ -wget http://pypi.python.org/packages/source/s/setuptools/setuptools-0.6c11.tar.gz#md5=7df2a529a074f613b509fb44feefe74e -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO GET SETUPTOOLS. " ; exit; fi -tar xfvz setuptools-0.6c11.tar.gz -cd setuptools-0.6c11 -python setup.py build -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO BUILD SETUPTOOLS. " ; exit; fi -python setup.py install - -################ -# MySQL 64 bit -################ -mkdir -p ${APPSCALE_HOME}/AppDB/mysql -cd ${APPSCALE_HOME}/AppDB/mysql -wget http://appscale.cs.ucsb.edu/appscale_files/mysql-cluster-gpl-6.3.20-linux-x86_64-glibc23.tar.gz -O ${APPSCALE_HOME}/AppDB/mysql/mysql-cluster-gpl-6.3.20-linux-x86_64-glibc23.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD MYSQL. CHECK LINK." ; exit; fi -mv mysql-cluster-gpl-6.3.20-linux-x86_64-glibc23.tar.gz mysql.tar.gz - - -pause $1 -cd ${APPSCALE_HOME}/AppDB/mysql/ -tar zxvf mysql.tar.gz -cd ${APPSCALE_HOME}/downloads/ -# MySQL for python -wget http://downloads.sourceforge.net/project/mysql-python/mysql-python-test/1.2.3c1/MySQL-python-1.2.3c1.tar.gz?use_mirror=hivelocity -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO GET MYSQL-PYTHON. " ; exit; fi -tar zxvf MySQL-python-1.2.3c1.tar.gz -cd MySQL-python-1.2.3c1 -echo "mysql_config = /root/appscale/AppDB/mysql/mysql-cluster-gpl-6.3.20-linux-x86_64-glibc23/bin/mysql_config" >> site.cfg -python setup.py build -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO BUILD MYSQL-PYTHON. " ; exit; fi -python setup.py install -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/root/appscale/AppDB/mysql/mysql-cluster-gpl-6.3.20-linux-x86_64-glibc23/lib/ -echo "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/root/appscale/AppDB/mysql/mysql-cluster-gpl-6.3.20-linux-x86_64-glibc23/lib/" >> ~/.bashrc -################# -# Thrift -################# -cd ${APPSCALE_HOME}/downloads -wget http://appscale.cs.ucsb.edu/appscale_files/thrift.tgz -O ${APPSCALE_HOME}/downloads/thrift.tgz -if [ $? != 0 ]; then echo "UNABLE TO DOWNLOAD THRIFT. CHECK LINK." ; exit; fi -tar zxvf thrift.tgz -cd ${APPSCALE_HOME}/downloads/thrift -# move python to python2.4 in /usr/local/bin/ -# softlink python2.5 to python -./bootstrap.sh -CONFIG_SHELL=/bin/bash; export CONFIG_SHELL -$CONFIG_SHELL ./configure --without-csharp --disable-gen-java --without-java --without-ruby -make -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE THRIFT. " ; exit; fi -make install -cd ${APPSCALE_HOME}/downloads/thrift/lib/py -make install -cd ${APPSCALE_HOME}/downloads/thrift/lib/perl -perl Makefile.PL -make install -echo "include /usr/local/lib" >> /etc/ld.so.conf - -pause $1 -#rpm -Uvh ftp://download.fedora.redhat.com/pub/fedora/linux/releases/10/Everything/source/SRPMS/boost-1.34.1-17.fc10.src.rpm - -#wget http://downloads.sourceforge.net/project/boost/boost/1.41.0/boost_1_41_0.tar.gz?use_mirror=cdnetworks-us-2 - -cd ${APPSCALE_HOME}/downloads -wget http://appscale.cs.ucsb.edu/appscale_files/rubygems-1.3.1.tgz -O ${APPSCALE_HOME}/downloads/rubygems-1.3.1.tgz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD RUBY GEMS. CHECK LINK." ; exit; fi -tar zxvf rubygems-*.tgz -cd ${APPSCALE_HOME}/downloads/rubygems-1.3.1 -ruby setup.rb - -pause $1 -gem update -pause $1 -gem install god redgreen -pause $1 -gem install capistrano -gem install -v=2.3.4 rails -pause $1 -gem install mongrel mongrel_cluster -pause $1 -# this questionable since the file is not in BZR -cd ${APPSCALE_HOME} -mkdir -p /etc/nginx -mkdir -p /etc/nginx/sites-enabled/ -cp AppLoadBalancer/config/nginx.conf /etc/nginx/ -sed -i 's/user www-data;/user root;/g' /etc/nginx/nginx.conf -cp /usr/local/nginx/conf/mime.types /etc/nginx/ -cp AppLoadBalancer/config/load-balancer.conf /etc/nginx/sites-enabled/ -sed -i 's/false/off/g' /etc/nginx/sites-enabled/load-balancer.conf -mkdir -p /var/log/nginx/ -touch /var/log/nginx/error.log - -cp AppLoadBalancer/nginx /etc/init.d/nginx -chkconfig --add nginx -chkconfig nginx on -pause $1 - -echo "export PATH=\${PATH}:/var/lib/gems/1.8/bin" >> /root/.bashrc -export PATH=${PATH}:/var/lib/gems/1.8/bin - -#TODO -cp /root/appscale/appscale_rhel /etc/init.d/appscale -chmod ugo+x /etc/init.d/appscale -chkconfig --add appscale -chkconfig appscale on - -################### -# HADOOP -################### -cd ${APPSCALE_HOME}/AppDB -rm -rf hadoop-0.20.0 -wget http://appscale.cs.ucsb.edu/appscale_files/hadoop-0.20.0.tar.gz -O ${APPSCALE_HOME}/AppDB/hadoop-0.20.0.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD HADOOP 0.20.0. CHECK LINK." ; exit; fi - -tar xvzf hadoop-0.20.0.tar.gz -cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0 -echo "export APPSCALE_HOME=/root/appscale" >> ./conf/hadoop-env.sh -echo "export JAVA_HOME=/usr/lib/jvm/java-1.6.0-sun.x86_64" >> ./conf/hadoop-env.sh - -echo "export HADOOP_HOME=${APPSCALE_HOME}/AppDB/hadoop-0.20.0" >> ./conf/hadoop-env.sh - -echo "export HBASE_HOME=${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha" >> ./conf/hadoop-env.sh - -echo "export HADOOP_CLASSPATH=${HBASE_HOME}:${HBASE_HOME}/hbase/hbase-0.20.0-alpha.jar:${HBASE_HOME}/hbase/hbase-0.20.0-alpha-test.jar:${HBASE_HOME}/conf:${HBASE_HOME}/build/classes:${HADOOP_HOME}/build/classes" >> ./conf/hadoop-env.sh - -echo "export CLASSPATH=${CLASSPATH}:${APPSCALE_HOME}/AppDB/hadoop-0.20.0/build/classes" >> ./conf/hadoop-env.sh - -echo "export HADOOP_HEAPSIZE=2000" >> ./conf/hadoop-env.sh - -cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0 -wget http://issues.apache.org/jira/secure/attachment/12405513/hadoop-includes-2.patch -O ${APPSCALE_HOME}/AppDB/hadoop-0.20.0/hadoop-includes-2.patch -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO GET PATH. " ; exit; fi -patch -p0 < hadoop-includes-2.patch -cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0/src/c++/utils -sh configure - -make - -sed -i "s/CXXFLAGS = -g/CXXFLAGS = -fPIC -g/g" ./Makefile - -make clean -make - -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE HADOOP UTIL. " ; exit; fi - -make install -cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0/src/c++/pipes -sh configure -make -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE HADOOP PIPES. " ; exit; fi - -make install -echo "${APPSCALE_HOME}/AppDB/hadoop-0.20.0/src/c++/install/lib" | tee -a /etc/ld.so.conf.d/hadoop.conf -ldconfig -pause $1 -######### -# HBASE -######### -cd ${APPSCALE_HOME}/AppDB -rm -fdr ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha -cd ${APPSCALE_HOME}/AppDB/hbase -wget http://appscale.cs.ucsb.edu/appscale_files/hbase-0.20.0-alpha.tar.gz -O ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD HBASE 0.20.0-alpha. CHECK LINK." ; exit; fi - -tar zxvf hbase-0.20.0-alpha.tar.gz -echo "export APPSCALE_HOME=/root/appscale" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha/conf/hbase-env.sh -echo "export JAVA_HOME=/usr/lib/jvm/java-1.6.0-sun.x86_64" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha/conf/hbase-env.sh - -echo "export HBASE_CLASSPATH=${APPSCALE_HOME}/AppDB/hadoop-0.20.0/build/classes" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha/conf/hbase-env.sh - -echo "export HBASE_HEAPSIZE=2000" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha/conf/hbase-env.sh -echo "export HBASE_MANAGES_ZK=FALSE" >> ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha/conf/hbase-env.sh - -rm -f ${APPSCALE_HOME}/AppDB/hadoop-0.20.0/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java - -cp -f ${APPSCALE_HOME}/AppDB/hbase/hbasefix/DistributedFileSystem.java ${APPSCALE_HOME}/AppDB/hadoop-0.20.0/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java - -cd ${APPSCALE_HOME}/AppDB/hadoop-0.20.0 -/root/appscale/downloads/apache-ant-1.7.1/bootstrap/bin/ant jar -cp build/hadoop-0.20.1-dev-core.jar ${APPSCALE_HOME}/AppDB/hbase/hbase-0.20.0-alpha/lib/hadoop-0.20.0-core.jar - -pause $1 -############################ -# Install Cassandra database -############################ -echo "Install Cassandra for AppScale" -mkdir -p ${APPSCALE_HOME}/AppDB/cassandra -cd ${APPSCALE_HOME}/AppDB/cassandra -wget http://appscale.cs.ucsb.edu/appscale_files/appscale_cassandra.06302009.tgz -O ${APPSCALE_HOME}/AppDB/cassandra/appscale_cassandra.06302009.tgz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD CASSANDRA. CHECK LINK." ; exit; fi -tar xzvf appscale_cassandra.06302009.tgz -rm -f appscale_cassandra.06302009.tgz -cd ${APPSCALE_HOME}/AppDB/cassandra/cassandra -ant build -mkdir /var/cassandra -chmod +x bin/cassandra - -############################# -# Install Voldemort database -############################# -mkdir -p ${APPSCALE_HOME}/AppDB/voldemort -cd ${APPSCALE_HOME}/AppDB/voldemort -wget http://appscale.cs.ucsb.edu/appscale_files/appscale_voldemort.06302009.tgz -O ${APPSCALE_HOME}/AppDB/voldemort/appscale_voldemort.06302009.tgz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD VOLDEMORT. CHECK LINK." ; exit; fi -tar xzvf appscale_voldemort.06302009.tgz -rm -f appscale_voldemort.06302009.tgz -cd ${APPSCALE_HOME}/AppDB/voldemort/voldemort -ant jar -mkdir /var/voldemort -chmod +x bin/voldemort - -#sh ${APPSCALE_HOME}/ec2_scratch_install.sh - -echo "export PATH=/usr/local/ec2-api-tools/bin/:\$PATH" >> /root/.bashrc - -echo "export EC2_HOME=/usr/local/ec2-api-tools" >> /root/.bashrc - -echo "export EC2_PRIVATE_KEY=/root/appscale/.appscale/certs/mykey.pem " >> /root/.bashrc - -echo "export EC2_CERT=/root/appscale/.appscale/certs/mycert.pem " >> /root/.bashrc - -echo "export JAVA_HOME=/usr/lib/jvm/java-1.6.0-sun.x86_64" >> /root/.bashrc - -source /root/.bashrc -mkdir -p ${APPSCALE_HOME}/logs - -################# -# MongoDB 64bit -################# -cd ${APPSCALE_HOME}/AppDB/ -wget http://appscale.cs.ucsb.edu/appscale_files/mongodb-linux-x86_64-1.0.0.tgz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD MONGODB. CHECK LINK." ; exit; fi -tar zxvf mongodb-linux-x86_64-1.0.0.tgz -mv mongodb-linux-x86_64-1.0.0 mongodb -chmod +x mongodb/mongodb-linux-x86_64-1.0.0/bin/mongo mongodb//mongodb-linux-x86_64-1.0.0/bin/mongod -rm -f mongodb-linux-x86_64-1.0.0.tgz -easy_install -U setuptools -easy_install pymongo - -pause $1 -#################### -# install memcached -#################### -cd ${APPSCALE_HOME} -wget http://kings.cs.ucsb.edu/appscale_files/memcached-1.4.2.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD MEMCAHCED. CHECK LINK." ; exit; fi -tar zxvf memcached-1.4.2.tar.gz -cd memcached-1.4.2 -./configure -make -make install -cd .. -rm -rf memcached-1.4.2* - -# install python lib for memcached -#apt-get -y install python-memcached -wget ftp://ftp.tummy.com/pub/python-memcached/python-memcached-1.45.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD python memcached. CHECK LINK." ; exit; fi -tar zxvf python-memcached-1.45.tar.gz -cd python-memcached-1.45 -python setup.py build -python setup.py install -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO INSTALL python memcached. CHECK LINK." ; exit; fi - -pause $1 -# first install berkeley db -cd ${APPSCALE_HOME} -wget http://kings.cs.ucsb.edu/appscale_files/db-4.8.24.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD BERKDB. CHECK LINK." ; exit; fi -tar zxvf db-4.8.24.tar.gz -cd db-4.8.24 -cd build_unix/ -../dist/configure -make -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO BUILD BERKDB. " ; exit; fi -make install -cd ${APPSCALE_HOME} -rm -rf db-4.8.24* - -pause $1 -# first install berkeley db -cd ${APPSCALE_HOME} -wget http://kings.cs.ucsb.edu/appscale_files/db-4.7.25.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD BERKDB. CHECK LINK." ; exit; fi -tar zxvf db-4.7.25.tar.gz -cd db-4.7.25 -cd build_unix/ -../dist/configure --enable-cxx -make -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO BUILD BERKDB. " ; exit; fi -make install -cd ${APPSCALE_HOME} -rm -rf db-4.7.25* -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/BerkeleyDB.4.7/lib/:/usr/local/lib/ -echo "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/BerkeleyDB.4.7/lib:/usr/local/lib/" >> ~/.bashrc -echo "include /usr/local/lib" >> /etc/ld.so.conf -echo "include /usr/local/BerkeleyDB.4.7/lib" >> /etc/ld.so.conf -ldconfig - -pause $1 -# Install memcachedb -cd ${APPSCALE_HOME}/AppDB/memcachedb -wget http://kings.cs.ucsb.edu/appscale_files/memcachedb.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD MEMCACHEDB. CHECK LINK." ; exit; fi -tar zxvf memcachedb.tar.gz -cd memcachedb -./configure --enable-threads --with-libevent=/usr/local/lib -make -make install -pause $1 -cd ${APPSCALE_HOME}/AppDB/memcachedb -rm -f memcachedb.tar.gz -cd ${APPSCALE_HOME}/downloads/ -wget http://appscale.cs.ucsb.edu/appscale_files/Python-2.5.4.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD PYTHON2.5. CHECK LINK." ; exit; fi -tar zxvf Python-2.5.4.tar.gz -cd Python-2.5.4 -./configure -make -make install -pause $1 -# make the default 2.6 -cp -f /usr/local/bin/python2.6 /usr/local/bin/python - -################### -# HYPERTABLE THRIFT -################### -cd ${APPSCALE_HOME}/downloads -wget http://appscale.cs.ucsb.edu/appscale_files/thrift-hypertable.tar.gz -O ${APPSCALE_HOME}/downloads/thrift-hypertable.tar.gz -if [ $? != 0 ]; then echo "UNABLE TO DOWNLOAD THRIFT-HYPERTABLE. CHECK LINK." ; exit; fi - -tar zxfv thrift-hypertable.tar.gz -cd ${APPSCALE_HOME}/downloads/thrift-hypertable -./bootstrap.sh -CONFIG_SHELL=/bin/bash; export CONFIG_SHELL -$CONFIG_SHELL ./configure --without-java -make -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE THRIFT HYPERTABLE. " ; exit; fi -make install -cd ${APPSCALE_HOME}/downloads/thrift-hypertable/lib/py -make install -cd ${APPSCALE_HOME}/downloads/thrift-hypertable/lib/rb -make install -cd ${APPSCALE_HOME}/downloads/thrift-hypertable/lib/perl -perl Makefile.PL -make install - -######################## -# Hypertable -######################## -cd ${APPSCALE_HOME}/downloads -wget http://appscale.cs.ucsb.edu/appscale_files/hyperic-sigar-1.6.0.tar.gz -O ${APPSCALE_HOME}/downloads/hyperic-sigar-1.6.0.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD HYPERIC SIGAR. CHECK LINK." ; exit; fi - -tar -xzvf hyperic-sigar-1.6.0.tar.gz -cp ${APPSCALE_HOME}/downloads/hyperic-sigar-1.6.0/sigar-bin/include/*.h /usr/local/include -# 64 BIT -cp ${APPSCALE_HOME}/downloads/hyperic-sigar-1.6.0/sigar-bin/lib/libsigar-amd64-linux.so ${APPSCALE_HOME}/downloads/hyperic-sigar-1.6.0/sigar-bin/lib/libsigar-ia64-linux.so /usr/local/lib/ - -cd ${APPSCALE_HOME}/downloads - wget http://downloads.sourceforge.net/project/boost/boost/1.41.0/boost_1_41_0.tar.gz?use_mirror=softlayer -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD BOOST. CHECK LINK." ; exit; fi -tar zxvf boost_1_41_0.tar.gz -cd boost_1_41_0 -./bootstrap.sh -./bjam -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO BUILD BOOST. CHECK LINK." ; exit; fi -./bjam install -cd ${APPSCALE_HOME}/AppDB/hypertable -rm -rf src build 0.9.2.5 -mkdir -p src build -cd ${APPSCALE_HOME}/AppDB/hypertable/src -wget http://appscale.cs.ucsb.edu/appscale_files/hypertable-0.9.2.5-alpha-src.tar.gz -O ${APPSCALE_HOME}/AppDB/hypertable/src/hypertable-0.9.2.5-alpha-src.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD HYPERTABLE 0.9.2.5. CHECK LINK." ; exit; fi -tar xvzf hypertable-0.9.2.5-alpha-src.tar.gz - -export HYPERTABLE=${APPSCALE_HOME}/AppDB/hypertable -export HADOOP=${APPSCALE_HOME}/AppDB/hadoop-0.20.0 -cd ${APPSCALE_HOME}/AppDB/hypertable/build -cmake -DHADOOP_INCLUDE_PATH=${HADOOP}/src/c++/install/include/ -DHADOOP_LIB_PATH=${HADOOP}/src/c++/install/lib/ -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=${HYPERTABLE} -DJAVA_INCLUDE_PATH=${JAVA_HOME}/include/ -DJAVA_INCLUDE_PATH2=${JAVA_HOME}/include/linux/ ../src/hypertable-0.9.2.5-alpha - -rm -f ${APPSCALE_HOME}/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/contrib/cc/MapReduce/TableRangeMap.cc - -cp ${APPSCALE_HOME}/AppDB/hypertable/hypertablefix/TableRangeMap.cc ${APPSCALE_HOME}/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/contrib/cc/MapReduce/TableRangeMap.cc -rm -f ${APPSCALE_HOME}/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/contrib/cc/MapReduce/TableReader.cc - -cp ${APPSCALE_HOME}/AppDB/hypertable/hypertablefix/TableReader.cc ${APPSCALE_HOME}/AppDB/hypertable/src/hypertable-0.9.2.5-alpha/contrib/cc/MapReduce/TableReader.cc - -make -j 2 -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE HYPERTABLE 0.9.2.5. " ; exit; fi - -make install -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO MAKE INSTALL HYPERTABLE 0.9.2.5. " ; exit; fi - -echo "${HYPERTABLE}/0.9.2.5/lib" | tee -a /etc/ld.so.conf.d/hypertable.conf -ldconfig - -################## -# /hypertable -################# - -pause $1 - -ssh-keygen -cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys - -pause $1 -cp /usr/bin/ruby /usr/bin/ruby1.8 -#/usr/bin/ruby1.8 /usr/bin/god -c /root/appscale/AppLoadBalancer/config/global.god -# in case it was not installed before, seen issues where things are not installed -# the first time around -gem install god redgreen -gem install -v=2.3.4 rails -gem install mongrel mongrel_cluster -pause $1 -# Port 9000 is cslistener so replace it for RH -sed -i 's/grep 9000/grep cslistener/g' ${APPSCALE_HOME}/AppDB/wait_on_hadoop.py - -######### -# Pig -######### -cd ${APPSCALE_HOME}/downloads -wget http://apache.deathculture.net/hadoop/pig/pig-0.5.0/pig-0.5.0.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD PIG. CHECK LINK." ; exit; fi -tar zxvf pig-0.5.0.tar.gz -cd pig-0.5.0 -mkdir tmp -cp pig-0.5.0-core.jar tmp/ -cd tmp -jar xvf pig-0.5.0-core.jar -rm -rf pig-0.5.0-core.jar -/bin/cp -f ~/appscale/AppDB/hadoop-0.20.0/build/classes/org/apache/hadoop/hdfs/* ${APPSCALE_HOME}/downloads/pig-0.5.0/tmp/org/apache/hadoop/hdfs/ -jar cvf ../pig-0.5.0-core.jar ./* -rm -rf ./* -wget http://appscale.cs.ucsb.edu/appscale_files/pigtutorial.tar.gz -if [ $? != 0 ]; then echo "FAILURE: UNABLE TO DOWNLOAD PIGTUTORIAL. CHECK LINK." ; exit; fi -tar zxvf pigtutorial.tar.gz -export PIG_CLASSPATH=${APPSCALE_HOME}/downloads/pig-0.5.0/tmp/pigtmp/pig.jar:${APPSCALE_HOME}/AppDB/hadoop-0.20.0/conf -echo "export PIG_CLASSPATH=${APPSCALE_HOME}/downloads/pig-0.5.0/tmp/pig.jar:${APPSCALE_HOME}/AppDB/hadoop-0.20.0/conf" >> ~/.bashrc - -###################### -# Prime Python eggs -###################### -python2.6 ${APPSCALE_HOME}/AppDB/appscale_server.py -pkill python -