Resolved issue with use of base64.urlsafe_b64encode and urlsafe_b64de…

…code.

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All significant changes will be recorded in this document
 
+## 0.1.7 - Bug fix
+
+* Added
+    * Nothing
+
+* Changed
+    * Now safe to pass urlencoded strings as key names
+
+* Removed
+    * Nothing
 
 ## 0.1.5 - Python3 compatible
 

VERSION

@@ -1 +1 @@
-0.1.6
+0.1.7

agavedb/keyval.py

@@ -12,9 +12,14 @@
 from __future__ import print_function
 from __future__ import absolute_import
 
+from future.standard_library import install_aliases
+install_aliases()
+
 from builtins import object
 from past.builtins import basestring
 from agavepy.agave import Agave
+
+import base64
 import re
 import json
 import logging
@@ -23,7 +28,8 @@
 import os
 
 sys.path.insert(0, os.path.dirname(__file__))
-from . import uniqueid
+#from . import uniqueid
+import uniqueid
 
 _SEP = '/'
 _PREFIX = '_agkvs_v1'
@@ -100,7 +106,7 @@ def _namespace(self, keyname):
         e.g.) keyname => _agavedb/keyname#username
         """
         if self._key_is_valid(keyname):
-            _keyname = keyname
+            _keyname = base64.urlsafe_b64encode(keyname.encode()).decode()
             _keyname = _PREFIX + _SEP + _keyname + \
                 '#' + self._username()
             return _keyname
@@ -127,6 +133,15 @@ def _rev_namespace(self, fullkeyname, removeusername=True):
             if keyname.endswith(_suffix):
                 keyname = keyname[:(-1 * len(_suffix))]
 
+        keyname_tmp = ''
+        try:
+            keyname_tmp = base64.urlsafe_b64decode(keyname.encode())
+            keyname_tmp.decode('ascii')
+            keyname = keyname_tmp
+        except Exception:
+            keyname = str(keyname)
+            pass
+
         return keyname
 
     def _slugify(self, value, allow_unicode=False):
@@ -263,7 +278,8 @@ def _getall(self, namespace=False, sorted=True, uuids=False):
             if uuids:
                 all_keys.append(key_obj['uuid'])
             elif namespace:
-                all_keys.append(key_obj['name'])
+                all_keys.append(self._rev_namespace(key_obj['name'],
+                                removeusername=namespace))
             else:
                 all_keys.append(self._rev_namespace(key_obj['name']))
 
@@ -390,65 +406,7 @@ def main():
     ag = Agave.restore()
     kvs = AgaveKeyValStore(ag)
 
-    print("Test namespacing")
-    print(kvs._namespace("abc123"))
-    print(kvs._rev_namespace("_agkvs_v1/abc123#sd2eadm"))
-    print(kvs._rev_namespace("_agkvs_v1/abc123#sd2eadm", False))
-
-    print("Test simple set/get")
-    print(kvs.set('abc123', 'value34567'))
-    print(kvs.get('abc123'))
-    # non-existent key
-    print(kvs.get('XXXX'))
-
-    # set some more key/values
-    print(kvs.set('org.sd2e.reactors', 'DFGHJKJHGFDSASDFGH'))
-    print(kvs.set('abc345', 'value34567'))
-    print(kvs.set('abc456', 'value34567'))
-
-    print("Test key properties enforcement")
-    # min length
-    kvs.set('xyz', 'SDFGHJKJHGFDSDFGHJK')
-    # type
-    kvs.set(123, 456)
-    kvs.set(None, 456)
-
-    print("Test value type enforcement")
-    kvs.set('abc789', {'name': 'value'})
-    kvs.set('abc789', ('new york', 'los angeles'))
-    kvs.set('abc789', [1, 2, 3])
-    kvs.set('abc789', None)
-    kvs.set('abc789', kvs)
-
-    print("Test get all keys")
-    print(kvs.getall())
-
-    print("Test key remove")
-    kvs.rem('abc123')
-    # key abc123 should be done
     print(kvs.getall())
 
-    # print("Test remove all keys")
-    # kvs.deldb()
-    # empty list
-    print(kvs.getall())
-
-    print("Test adding string w number at beginning")
-    print(kvs.set('numstring', '6G5x4jqPBvBJy'))
-    print(kvs.get('numstring'))
-
-    print("Test adding numerical values")
-    print(kvs.set('numbers.Number.6', '6'))
-    print(kvs.get('numbers.Number.6'))
-    print(kvs.set('numbers.Number.6', 6))
-    print(kvs.get('numbers.Number.6'))
-
-    # print(kvs.set('numbers_Number_6', 6))
-    # print(kvs.get('numstring_Number_6'))
-
-    print("Generate Unique ID")
-    print(kvs.create_key_name())
-
-
 if __name__ == '__main__':
     main()

agavedb/tests/test_keyval.py

@@ -3,6 +3,10 @@
 
 # Usage: See bundled README.md
 
+from future.standard_library import install_aliases
+install_aliases()
+from urllib.parse import urlencode, quote
+
 from builtins import str
 
 import json
@@ -69,7 +73,7 @@ def agave(credentials):
                api_secret=credentials.get('apisecret'),
                token=credentials.get('token', None),
                refresh_token=credentials.get('refresh_token', None),
-               verify=credentials.get('verify_certs', True))
+               verify=True)
     return ag
 
 
@@ -118,7 +122,7 @@ def test_key_valid(keyvalstore, credentials, test_data):
 def test_namespace_fwd(keyvalstore, credentials):
     '''_namespace'''
     ns = keyvalstore._namespace('abc123')
-    expected = '_agkvs_v1/abc123#' + credentials['username']
+    expected = '_agkvs_v1/YWJjMTIz#' + credentials['username']
     assert ns == expected
 
 
@@ -191,3 +195,18 @@ def test_username(keyvalstore, credentials):
     '''verify that agavedb and test view of username is same'''
     assert credentials['username'] == keyvalstore._username()
 
+
+def test_keys_can_contain_urlchars(keyvalstore, credentials):
+    '''tests to address github.com/TACC/agavedb/issues/1'''
+    test_keys = ['manifest.json:INSERT:sailfish',
+                 u'manifest.json:INSERT:sailfish',
+                 quote('manifest.json:INSERT:sailfish'),
+                 quote(u'manifest.json:INSERT:sailfish')]
+    for key_name in test_keys:
+        key_valu = uuid.uuid4().hex
+        set_key_name = keyvalstore.set(key_name, key_valu)
+        assert set_key_name == key_name
+        get_key_valu = keyvalstore.get(key_name)
+        assert get_key_valu == key_valu
+        assert keyvalstore.rem(key_name) is True
+