Permalink
Browse files

Removed IRODS and bumped ion version

  • Loading branch information...
1 parent ed636a2 commit 461995b8ada28bf501858c45b9124923cfffcd21 David Stuebe committed Jan 24, 2011
Showing with 7 additions and 292 deletions.
  1. +4 −0 README.txt
  2. +1 −1 ion/core/_version.py
  3. +0 −231 ion/core/data/irodsstore.py
  4. +0 −47 ion/core/data/test/test_irodsstore.py
  5. +0 −1 requirements.txt
  6. +0 −10 res/config/ion.config
  7. +2 −2 setup.py
View
4 README.txt
@@ -131,6 +131,10 @@ To compile all code to see if there are Python compile errors anywhere:
---------------------------------------------------------------------------
Change log:
===========
+
+2011-1-24:
+- Removed IRODS as backend storage configureation option
+
2011-1-21:
- ION is now using google protocol buffer objects in the resource object model
- Bumped version number to 0.4.0 consistent with the setup.py file.
View
2 ion/core/_version.py
@@ -154,4 +154,4 @@ def _formatGitCommit(self):
return ' (Git:%s)' % commit
# VERSION !!! This is the main version !!!
-version = Version('ion', 0, 4, 0)
+version = Version('ion', 0, 4, 1)
View
231 ion/core/data/irodsstore.py
@@ -1,231 +0,0 @@
-#!/usr/bin/env python
-"""
-@file ion/data/backends/irodsstore.py
-@author Bing Zhu
-@brief Implementation of ion.data.store.IStore using PyIrods to interface a
- iRODS backend storage servers.
-
-@note Test cases for the iRODS backend are in ion.data.backends.test.test_irodsstore
-@note The Python iRODS library package can be installed with: easy_install --find-links http://ooici.net/packages pyrods-irods
-@
-
-"""
-
-import logging
-logging = logging.getLogger(__name__)
-
-from twisted.internet import defer
-
-from ion.core import ioninit
-from ion.data.store import IStore
-
-from irods import *
-from irods_error import *
-
-CONF = ioninit.config(__name__)
-
-
-class IrodsStoreError(Exception):
- """
- Exception class for IrodsStore
- """
-
-
-class IrodsStore(IStore):
- """
- @brief Store interface for interacting with the iRODS distributed storage system
- @note see http://www.irods.org
- @note The login info is stored in the ion config file.
- """
- def __init__(self, **kwargs):
- """
- @brief constructor to read iRODS info from args or the 'ion' config file
- """
- self.hostname = self._get_config('irodsHost')
- self.port_num = self._get_config('irodsPort')
- self.default_resc = self._get_config('irodsDefResource')
- self.obj_home = self._get_config('irodsOoiCollection')
- self.user_name = self._get_config('irodsUserName')
- self.user_passwd = self._get_config('irodsUserPasswd')
- self.zone = self._get_config('irodsZone')
- self.conn = None
-
- # can be overridden by args individually
- if kwargs:
- if kwargs.get('irodsHost', None):
- self.hostname = kwargs.get('irodsHost', None)
- if kwargs.get('irodsPort', None):
- self.port_num = kwargs.get('irodsPort', None)
- if kwargs.get('irodsDefResource', None):
- self.default_resc = kwargs.get('irodsDefResource', None)
- if kwargs.get('irodsOoiCollection', None):
- self.obj_home = kwargs.get('irodsOoiCollection', None)
- if kwargs.get('irodsUserName', None):
- self.user_name = kwargs.get('irodsUserName', None)
- if kwargs.get('irodsUserPasswd', None):
- self.user_passwd = kwargs.get('irodsUserPasswd', None)
- if kwargs.get('irodsZone', None):
- self.zone = kwargs.get('irodsZone', None)
-
- def _get_config(self, key):
- try:
- value = CONF[key]
- except KeyError:
- # TODO - do not catch generic exceptions! This should be a key error exception I believe
- value = None
- return value
-
- def connect_to_irods(self):
- """
- @brief creates a client connection with an iRODS server
- """
- self.conn, errMsg = rcConnect(self.hostname, int(self.port_num), self.user_name, self.zone)
- if not self.conn:
- logging.info('rcConnect() error: ' + str(errMsg))
- raise IrodsStoreError('rcConnect error: ' + str(errMsg))
-
- status = clientLoginWithPassword(self.conn, self.user_passwd)
- if status < 0:
- self.conn = None
- errName, subErrName = rodsErrorName(status)
- errMsg = str(status) + ':' + str(errName) + ' ' + str(subErrName)
- logging.info('rcConnect() error: ' + errMsg)
- raise IrodsStoreError('clientLoginWithPassword() error:' + errMsg)
-
- irods_info = 'irods connection succeeded: ' + self.hostname + '/' + self.port_num
- logging.info(irods_info)
-
- @classmethod
- def create_store(cls, **kwargs):
- """
- @brief factory method to create an create an instance of the irods store
- @param from optional kwargs
- @retval IStore instance or None
- """
- inst = cls(**kwargs)
-
- inst.kwargs = kwargs
-
- inst.connect_to_irods()
-
- return defer.succeed(inst)
-
- def disconnect_from_irods(self):
- """
- @brief close the TCP connection with the iRODS server
- """
- rcDisconnect(self.conn)
- self.conn = None
-
-
- def clear_store(self):
- """
- @brief Clean the iRODS collection and disonnect from iRODS
- """
- if not self.conn:
- return defer.succeed(None)
-
- collinp = collInp_t()
- collinp.setCollName(self.obj_home)
- collinp.addCondInputKeyVal(RECURSIVE_OPR__KW, '')
- collinp.addCondInputKeyVal(FORCE_FLAG_KW, '')
- rcRmColl(self.conn, collinp, 0)
-
- self.disconnect_from_irods()
-
- return defer.succeed(None)
-
- def _get_irods_fname_by_key(self, key):
- """
- @brief Return the irods file name based on the key value. This scheme is used for a single zone and assumes we store all data in a single collection.
- @param key The OOI unique key for an data object
- """
- fname = self.obj_home + '/' + key
- return fname
-
-
- def get(self, key):
- """
- @brief Return a RODS value (the content of the file) corresponding to a given key
- @param key The sha1 hash key from OOI repository
- @retval the content of the irods file, or None
- """
- logging.debug('reading value from iRODS for key %s' % (key))
-
- value = None
- self.fname = self._get_irods_fname_by_key(key)
-
- f = iRodsOpen(self.conn, self.fname, "r")
- if not f:
- logging.info('Failed to open file for read: ' + str(self.fname))
- #raise IrodsStoreError('Failed to open file for read: ' + self.fname)
- return defer.succeed(None)
- value = f.read()
- f.close()
- return defer.succeed(value)
-
-
- def put(self, key, value):
- """
- @brief Write a the data into iRODS.
- @param key
- @param value Corresponding value
- @note Value is composed into iRODS under 'irods_ooi_home' with filename of the key value.
- @retval None for success
- """
-
- logging.debug('writing data %s into iRODS' % (key))
-
- if not self.conn:
- return None
-
- #create the collection 'obj_home' if it does not exist.
- collinp = collInp_t()
- collinp.setCollName(self.obj_home)
- status = collinp.addCondInputKeyVal(RECURSIVE_OPR__KW, "")
- status = rcCollCreate(self.conn, collinp)
- if status < 0:
- errMsg = self._get_errmsg_by_status(status)
- logging.info('rcCollCreate() error: ' + str(errMsg))
- raise IrodsStoreError('rcCollCreate() error: ' + str(errMsg))
-
- fname = self._get_irods_fname_by_key(key)
- f = iRodsOpen(self.conn, fname, 'w', self.default_resc)
- if not f:
- logging.info('Failed to open file for write: ' + str(fname))
- raise IrodsStoreError('Failed to open file for write: ' + str(fname))
-
- f.write(value)
- f.close()
-
- return defer.succeed(None)
-
- def _get_errmsg_by_status(self, t):
- errName, subErrName = rodsErrorName(t)
- errMsg = str(t) + ':' + str(errName) + ' ' + str(subErrName)
- return errMsg
-
-
- def remove(self, key):
- """
- @brief delete an iRODS file by OOICI key
- @param key Key to delete
- @retval None for success of operation
- """
-
- logging.debug('deleting data %s from iRODS' % (key))
-
- self.fname = self._get_irods_fname_by_key(key)
- dataObjInp = dataObjInp_t()
- dataObjInp.getCondInput().addKeyVal(FORCE_FLAG_KW, "")
- dataObjInp.setObjPath(self.fname)
-
- t = rcDataObjUnlink(self.conn, dataObjInp)
-
- if t < 0:
- errMsg = self._get_errmsg_by_status(t)
- logging.info('rcDataObjUnlink() error: ' + str(errMsg))
- raise IrodsStoreError('rcDataObjUnlink() error: ' + str(errMsg))
-
- return defer.succeed(None)
-
View
47 ion/core/data/test/test_irodsstore.py
@@ -1,47 +0,0 @@
-#!/usr/bin/env python
-
-"""
-@file ion/data/backends/test/test_setdatastore.py
-@author Bing Zhu
-@test Service only test of the iRODS datastore
-"""
-
-
-from twisted.trial import unittest
-import ion.util.ionlog
-log = ion.util.ionlog.getLogger(__name__)
-from uuid import uuid4
-from ion.core.data.irodsstore import IrodsStore
-from twisted.internet import defer
-
-
-class TestIrodsDataStoreInterface(unittest.TestCase):
- def setUp(self):
- # This will use the iRODS config info in 'ion.config'
- # self.ds = IrodsStore.create_store()
-
- # here we use config through args
- irods_config = {'irodsHost': 'ec2-204-236-159-249.us-west-1.compute.amazonaws.com', \
- 'irodsPort':'1247', \
- 'irodsDefResource':'ooi-test-resc1', \
- 'irodsOoiCollection':'/ooi-test-cluster1/home/testuser/OOI', \
- 'irodsUserName':'testuser', \
- 'irodsUserPasswd':'test', \
- 'irodsZone':'ooi-test-cluster1'}
- ds = IrodsStore.create_store(**irods_config)
- self.ds = ds.result
-
- self.key = self._mkey()
- self.value = self._mkey()
-
- def tearDown(self):
- self.ds.clear_store()
-
- def _mkey(self):
- # Generate a pseudo-random string. handy, that.
- return str(uuid4())
-
- def test_irods(self):
- self.ds.put(self.key, self.value)
- value = self.ds.get(self.key)
-
View
1 requirements.txt
@@ -10,7 +10,6 @@ gviz-api.py==1.7.0
httplib2==0.6.0
msgpack-python==015final
nimboss
-pyrods-irods==2.4.3
simplejson==2.1.2
telephus==0.7-beta3.3
twotp==0.7
View
10 res/config/ion.config
@@ -53,16 +53,6 @@
'rpc_timeout':15,
},
-'ion.data.backends.irodsstore': {
- 'irodsHost': 'ec2-204-236-137-245.us-west-1.compute.amazonaws.com',
- 'irodsPort':'1247',
- 'irodsDefResource':'ooi-test-resc1',
- 'irodsOoiCollection':'/ooi-test-cluster1/home/testuser/OOI',
- 'irodsUserName':'testuser',
- 'irodsUserPasswd':'test',
- 'irodsZone':'ooi-test-cluster1',
-},
-
'ion.resources.description_utility':[
'ion.resources.cei_resource_descriptions',
View
4 setup.py
@@ -12,7 +12,7 @@
setupdict = {
'name' : 'ioncore',
- 'version' : '0.4.0', #VERSION,
+ 'version' : '0.4.1', #VERSION,
'description' : 'OOI ION Python Capability Container and Core Modules',
'url': 'http://www.oceanobservatories.org/spaces/display/CIDev/LCAARCH+Development+Project',
'download_url' : 'http://ooici.net/packages',
@@ -41,7 +41,7 @@
setupdict['install_requires'] = ['Twisted==10.2.0', 'carrot==0.10.11-txamqp', 'txamqp==0.3',
'simplejson==2.1.2', 'httplib2==0.6.0','msgpack-python==015final',
'gviz_api.py==1.7.0','nimboss','txrabbitmq==0.4', 'Telephus==0.7-beta3.3',
- 'pyrods-irods==2.4.3', 'M2Crypto==0.20.2', 'ionproto==0.2.3', 'protobuf==2.3.0-p1']
+ 'M2Crypto==0.20.2', 'ionproto==0.2.3', 'protobuf==2.3.0-p1']
setupdict['include_package_data'] = True

0 comments on commit 461995b

Please sign in to comment.