From 1bc509b98da92e88aaf0c38247e6783eeceb18eb Mon Sep 17 00:00:00 2001 From: Michael Pearson Date: Tue, 13 Jul 2010 10:29:52 -0700 Subject: [PATCH] added longtype + example, fixed a unit test fail on core during connect --- examples/long_columnfamily.php | 59 +++++++++++++++++++++++++ lib/ColumnContainer.class.php | 77 +++++++++++++++++++++++++-------- lib/Core.class.php | 72 ++++++++++++++++-------------- lib/SuperColumnFamily.class.php | 2 +- 4 files changed, 157 insertions(+), 53 deletions(-) create mode 100644 examples/long_columnfamily.php diff --git a/examples/long_columnfamily.php b/examples/long_columnfamily.php new file mode 100644 index 0000000..213ab98 --- /dev/null +++ b/examples/long_columnfamily.php @@ -0,0 +1,59 @@ + + * + * Script performs a save, plus two cross checks - loading directly from an + * anonymous columnfamily model, and populating a model from a slice. + * + */ + +require_once('../config.php'); + +PandraCore::connectSeededKeyspace('localhost'); + +$ks = 'Keyspace1'; +$cfName = 'StandardByLong1'; +$keyID = 'PandraTestLong1'; + +$cf = new PandraColumnFamily($keyID, $ks, $cfName, PandraColumnFamily::TYPE_LONG); + +$cf->addColumn(PandraCore::getTime())->setValue('numericly indexed!'); +echo 'Saving...
'; +print_r($cf->toJSON()); +$cf->save(); + +// load from model +echo '

Loading via CF container...
'; +$cfNew = new PandraColumnFamily($keyID, $ks, $cfName, PandraColumnFamily::TYPE_LONG); + +$cfNew->limit(5)->load(); +echo '
Loaded...
'; +print_r($cfNew->toJSON()); + +// get slice +echo '

Loading Slice...
'; +$result = PandraCore::getCFSlice($ks, + $keyID, + new cassandra_ColumnParent(array( + 'column_family' => $cfName + )), + new PandraSlicePredicate( + PandraSlicePredicate::TYPE_RANGE, + array('start' => '', + 'finish' => '', + 'count' => 5, + 'reversed' => true)) + ); + +var_dump($result); + +$cfNew = new PandraColumnFamily($keyID, $ks, $cfName, PandraColumnFamily::TYPE_LONG); + +$cfNew->populate($result); + +echo '
Imported...
'; +print_r($cfNew->toJSON()); + +?> \ No newline at end of file diff --git a/lib/ColumnContainer.class.php b/lib/ColumnContainer.class.php index d653446..283a369 100644 --- a/lib/ColumnContainer.class.php +++ b/lib/ColumnContainer.class.php @@ -31,6 +31,12 @@ abstract class PandraColumnContainer implements ArrayAccess, Iterator, Countable /* @var int 'LONG' container type @todo - not implemented! */ const TYPE_LONG = 3; + /* @var int 'string' flag for type conversion */ + const CONTEXT_STRING = 0; + + /* @var int 'binary' flag for type conversion */ + const CONTEXT_BIN = 1; + /* @var array complete list of errors for this object instance */ public $errors = array(); @@ -430,8 +436,18 @@ public function getReversed() { return $this->_reversed; } + private function _setStartFinish($value, $attrib = '_start') { + if (($this->_containerType == self::TYPE_UUID) ) { + $this->$attrib = $this->typeConvert($value, self::CONTEXT_BIN); + } elseif (($this->_containerType == self::TYPE_LONG)) { + $this->$attrib = $this->typeConvert($value, self::CONTEXT_BIN); + } else { + $this->$attrib = $value; + } + } + public function setStart($start) { - $this->_start = $this->typeConvert($start, UUID::UUID_BIN); + $this->_setStartFinish($start); } public function start($start) { @@ -444,7 +460,7 @@ public function getStart() { } public function setFinish($finish) { - $this->_finish = $this->typeConvert($finish, UUID::UUID_BIN); + $this->_setStartFinish($finish, '_finish'); } public function finish($finish) { @@ -462,33 +478,42 @@ public function getFinish() { * This stub can also potentially handle long and utf8 cf types * * @param string $columnName column name - * @param int $toFmt convert to type (UUID::UUID_BIN, UUID::UUID_STR) + * @param int $toFmt convert to type CONTEXT_BIN OR CONTEXT_STRING * @return mixed converted column name */ protected function typeConvert($columnName, $toFmt) { - if (($this->_containerType == self::TYPE_UUID) ) { - - $bin = UUID::isBinary($columnName); + $bin = UUID::isBinary($columnName); + if (($this->_containerType == self::TYPE_UUID) ) { // Save accidental double-conversions on binaries - if (($bin && $toFmt == UUID::UUID_BIN) || - (!$bin && $toFmt == UUID::UUID_STR)) { + if (($bin && $toFmt == self::CONTEXT_BIN) || + (!$bin && $toFmt == self::CONTEXT_STRING)) { return $columnName; } elseif (!$bin && !UUID::validUUID($columnName)) { throw new RuntimeException('Column Name ('.$columnName.') cannot be converted'); } - if ($toFmt == UUID::UUID_BIN) { + if ($toFmt == self::CONTEXT_BIN) { return UUID::toBin($columnName); - } elseif ($toFmt == UUID::UUID_STR) { + } elseif ($toFmt == self::CONTEXT_STRING) { return UUID::toStr($columnName); } + } else if ($this->_containerType == self::TYPE_LONG) { - $columnName = UUID::isBinary($columnName) ? - unpack('NN', $columnName) : - pack('NN', $columnName); - } + // Save accidental double-conversions on binaries + if (($bin && $toFmt == self::CONTEXT_BIN) || + (!$bin && $toFmt == self::CONTEXT_STRING)) { + return $columnName; + + // unpack the long + } elseif ($bin && $toFmt == self::CONTEXT_STRING) { + $columnName = array_pop(unpack('N', $columnName)); + // pack the long + } elseif (!$bin && $toFmt == self::CONTEXT_BIN) { + $columnName = pack('NN', $columnName, 0); + } + } return $columnName; } @@ -500,15 +525,29 @@ protected function typeConvert($columnName, $toFmt) { * @return PandraColumn reference to created column */ public function addColumn($columnName, $typeDef = array(), $callbackOnSave = NULL) { - if (!array_key_exists($columnName, $this->_columns)) { + + // can't use array_key_exists - it truncates floats on 32 bit systems + $foundKey = FALSE; + foreach ($this->_columns as $key => $value) { + $foundKey = ($key == $columnName); + if ($foundKey) break; + } + + if (!$foundKey) { $this->_columns[$columnName] = - new PandraColumn($this->typeConvert($columnName, UUID::UUID_BIN), $typeDef); + new PandraColumn($this->typeConvert($columnName, self::CONTEXT_BIN, $typeDef)); $this->_columns[$columnName]->setParent($this, FALSE); } // pre-save callback if (!empty($callbackOnSave)) $this->getColumn($columnName)->setCallback($callbackOnSave); + + // php sucks balls, lets lose our precision. + if (!PANDRA_64 && $this->_containerType == self::TYPE_LONG) { + $columnName = (int) $columnName; + } + return $this->getColumn($columnName); } @@ -518,7 +557,7 @@ public function addColumn($columnName, $typeDef = array(), $callbackOnSave = NUL */ public function addColumnObj(PandraColumn $columnObj) { if ($columnObj->getName() === NULL) throw new RuntimeException('Column has no name'); - $this->_columns[$this->typeConvert($columnObj->name, UUID::UUID_STR)] = $columnObj; + $this->_columns[$this->typeConvert($columnObj->name, self::CONTEXT_STRING)] = $columnObj; } /** @@ -682,7 +721,7 @@ public function populate($data, $colAutoCreate = NULL) { // Check depth, take first few keys as keyspace/columnfamily/key foreach ($data as $idx => $column) { if ($column instanceof cassandra_Column) { - $columnName = $this->typeConvert($column->name, UUID::UUID_STR); + $columnName = $this->typeConvert($column->name, self::CONTEXT_STRING); if ($this->getAutoCreate($colAutoCreate) ) { $this->_columns[$columnName] = PandraColumn::cast($column, $this); @@ -692,7 +731,7 @@ public function populate($data, $colAutoCreate = NULL) { // circular dependency? } elseif ($column instanceof cassandra_ColumnOrSuperColumn && !empty($column->column)) { - $columnName = $this->typeConvert($column->column->name, UUID::UUID_STR); + $columnName = $this->typeConvert($column->column->name, self::CONTEXT_STRING); if ($this->getAutoCreate($colAutoCreate) ) { $this->_columns[$columnName] = PandraColumn::cast($column->column, $this); diff --git a/lib/Core.class.php b/lib/Core.class.php index 461340b..807e4b2 100755 --- a/lib/Core.class.php +++ b/lib/Core.class.php @@ -52,9 +52,9 @@ class PandraCore { static private $writeMode = self::MODE_RANDOM; static private $_supportedReadConsistency = array( - cassandra_ConsistencyLevel::ONE, - cassandra_ConsistencyLevel::QUORUM, - cassandra_ConsistencyLevel::ALL + cassandra_ConsistencyLevel::ONE, + cassandra_ConsistencyLevel::QUORUM, + cassandra_ConsistencyLevel::ALL ); /* @var supported modes for this core version */ @@ -227,6 +227,8 @@ static public function connect($connectionID, $host, $keySpace = self::DEFAULT_P // Create Thrift transport and binary protocol cassandra client $transport = new TBufferedTransport(new TSocket($host, $port, PERSIST_CONNECTIONS, 'PandraCore::registerError'), 1024, 1024); + self::_authOpen($transport, $keySpace); + self::$_socketPool[$keySpace][$connectionID] = array( 'retries' => 0, 'transport' => $transport, @@ -244,7 +246,6 @@ function_exists("thrift_protocol_write_binary") ? } } catch (TException $te) { self::registerError('TException: '.$te->getMessage(), PandraLog::LOG_CRIT); - } return FALSE; @@ -316,8 +317,8 @@ static public function registerError($errorMsg, $priority = PandraLog::LOG_WARNI static public function authKeyspace($keySpace, $username, $password) { $auth = new cassandra_AuthenticationRequest(); $auth->credentials = array ( - "username" => $username, - "password" => $password + "username" => $username, + "password" => $password ); self::$_ksAuth[$keySpace] = $auth; } @@ -325,44 +326,49 @@ static public function authKeyspace($keySpace, $username, $password) { /** * Alias for connectBySeed (deprecated) */ - static public function auto($host, $keySpace = self::DEFAULT_POOL_NAME, $port = THRIFT_PORT_DEFAULT) { - return self::connectSeededKeyspace($host, $keySpace, $port); + static public function auto($hosts, $keySpace = self::DEFAULT_POOL_NAME, $port = THRIFT_PORT_DEFAULT) { + return self::connectSeededKeyspace($hosts, $keySpace, $port); } /** * Given a single host, attempts to find other nodes in the cluster and attaches them * to the pool * @todo build connections from token map - * @param string $host host name or IP of connecting node + * @param mixed $hosts host name or IP of connecting node (or array thereof) * @param string $keySpace name of the connection pool (cluster name - usually keyspace) * @param int $port TCP port of connecting node * @return bool connected ok */ - static public function connectSeededKeyspace($host, $keySpace = self::DEFAULT_POOL_NAME, $port = THRIFT_PORT_DEFAULT) { + static public function connectSeededKeyspace($hosts, $keySpace = self::DEFAULT_POOL_NAME, $port = THRIFT_PORT_DEFAULT) { - try { - // Create Thrift transport and binary protocol cassandra client - $transport = new TBufferedTransport(new TSocket($host, $port, PERSIST_CONNECTIONS, 'PandraCore::registerError'), 1024, 1024); - $transport->open(); - $client = new CassandraClient( - (function_exists("thrift_protocol_write_binary") ? - new TBinaryProtocolAccelerated($transport) : - new TBinaryProtocol($transport))); - - // @todo this has been deprecated by 'describe_ring' 0.6.3 - $tokenMap = $client->get_string_property('token map'); - $transport->close(); - unset($transport); unset($client); - $tokens = json_decode($tokenMap); - foreach ($tokens as $token => $host) { - if (!self::connect($token, $host, $keySpace)) { - return FALSE; + if (!is_array($hosts)) { + $hosts = (array) $hosts; + } + + foreach ($hosts as $host) { + try { + // Create Thrift transport and binary protocol cassandra client + $transport = new TBufferedTransport(new TSocket($host, $port, PERSIST_CONNECTIONS, 'PandraCore::registerError'), 1024, 1024); + $transport->open(); + $client = new CassandraClient( + (function_exists("thrift_protocol_write_binary") ? + new TBinaryProtocolAccelerated($transport) : + new TBinaryProtocol($transport))); + + // @todo this has been deprecated by 'describe_ring' 0.6.3 + $tokenMap = $client->get_string_property('token map'); + $transport->close(); + unset($transport); unset($client); + $tokens = json_decode($tokenMap); + foreach ($tokens as $token => $host) { + if (!self::connect($token, $host, $keySpace)) { + return FALSE; + } } + return TRUE; + } catch (TException $te) { + self::registerError( 'TException: '.$te->getMessage().' '.(isset($te->why) ? $te->why : '')); } - - return TRUE; - } catch (TException $te) { - self::registerError( 'TException: '.$te->getMessage().' '.(isset($te->why) ? $te->why : '')); } return FALSE; } @@ -425,7 +431,7 @@ static public function getAPCAvailable() { * @param bool $writeMode optional get the write mode client * @param string $keySpace optional keyspace where auth has been defined */ - static public function getClient($writeMode = FALSE, $keySpace = NULL) { + static public function getClient($writeMode = FALSE, $keySpace = self::DEFAULT_POOL_NAME) { // Catch trimmed nodes or a completely trimmed pool if (empty(self::$_activeNode) || empty(self::$_socketPool[self::$_activePool])) { @@ -463,7 +469,7 @@ static public function getClient($writeMode = FALSE, $keySpace = NULL) { // check connection is open try { - if (!self::$_socketPool[self::$_activePool][self::$_activeNode]['transport']->isOpen()) { + if (!self::$_socketPool[self::$_activePool][self::$_activeNode]['transport']->isOpen()) { self::_authOpen(self::$_socketPool[self::$_activePool][self::$_activeNode]['transport'], $keySpace); } return $conn; diff --git a/lib/SuperColumnFamily.class.php b/lib/SuperColumnFamily.class.php index 73526e9..b18e770 100644 --- a/lib/SuperColumnFamily.class.php +++ b/lib/SuperColumnFamily.class.php @@ -64,7 +64,7 @@ public function addSuper(PandraSuperColumn $scObj) { public function addColumn($superName, $containerType = NULL) { if (!array_key_exists($superName, $this->_columns)) { $this->_columns[$superName] = new PandraSuperColumn( - $this->typeConvert($superName, UUID::UUID_BIN), + $this->typeConvert($superName, self::CONTEXT_BIN), $this->getKeyID(), $this->getKeySpace(), $this,