add new import system for optional modules

requirements.txt

@@ -1,11 +1,7 @@
 Cython>=0.28.0
-datasketch>=1.2.1
 numpy>=1.12.0
 scipy>=1.1.0
-redis>=2.0.0
-happybase>=1.1.0
 matplotlib>=2.0.0
 pandas>=0.20.0
 dask>=0.19.2
 distributed>=1.23
-plyvel>=1.0.5

rltk/io/adapter/hbase_key_value_adapter.py

@@ -1,8 +1,10 @@
-import happybase
-
 from rltk.record import Record
 from rltk.io.adapter import KeyValueAdapter
 from rltk.io.serializer import Serializer, PickleSerializer
+from rltk.utils import module_importer
+
+
+happybase = module_importer('happybase', 'happybase>=1.1.0')
 
 
 class HBaseKeyValueAdapter(KeyValueAdapter):
@@ -34,7 +36,7 @@ class HBaseKeyValueAdapter(KeyValueAdapter):
     def __init__(self, host, table, serializer: Serializer = None, key_prefix: str = '', clean: bool = False, **kwargs):
         if not serializer:
             serializer = PickleSerializer()
-        self._conn = happybase.Connection(host=host, timeout=None, **kwargs)
+        self._conn = happybase().Connection(host=host, timeout=None, **kwargs)
         self._serializer = serializer
         self._key_prefix = key_prefix
         self._family_name = 'rltk'
@@ -52,6 +54,7 @@ def __init__(self, host, table, serializer: Serializer = None, key_prefix: str =
     parallel_safe = True
 
     def _encode_key(self, key):
+        happybase.Connection('asd')
         return '{prefix}{key}'.format(prefix=self._key_prefix, key=key).encode('utf-8')
 
     def _decode_key(self, key):

rltk/io/adapter/leveldb_key_set_adapter.py

@@ -1,8 +1,11 @@
 import os
-import plyvel
 
 from rltk.io.serializer import Serializer, PickleSerializer
 from rltk.io.adapter.key_set_adapter import KeySetAdapter
+from rltk.utils import module_importer
+
+
+plyvel = module_importer('plyvel', 'plyvel>=1.0.5')
 
 
 class LevelDbKeySetAdapter(KeySetAdapter):
@@ -36,7 +39,7 @@ def __init__(self, path: str, name: str, serializer: Serializer = None, clean: b
         if not self.__class__._db_instance:
             if not os.path.exists(path):
                 os.mkdir(path)
-            self.__class__._db_instance = plyvel.DB(path, create_if_missing=True, **kwargs)
+            self.__class__._db_instance = plyvel().DB(path, create_if_missing=True, **kwargs)
         self._db = self.__class__._db_instance
         self.__class__._db_ref_count += 1
 

rltk/io/adapter/redis_key_set_adapter.py

@@ -2,6 +2,10 @@
 
 from rltk.io.serializer import Serializer, PickleSerializer
 from rltk.io.adapter.key_set_adapter import KeySetAdapter
+from rltk.utils import module_importer
+
+
+redis = module_importer('redis', 'redis>=2.0.0')
 
 
 class RedisKeySetAdapter(KeySetAdapter):
@@ -20,7 +24,7 @@ class RedisKeySetAdapter(KeySetAdapter):
     def __init__(self, host, key_prefix: str = '', serializer: Serializer=None, clean: bool = False, **kwargs):
         if not serializer:
             serializer = PickleSerializer()
-        self._redis = redis.Redis(host=host, **kwargs)
+        self._redis = redis().Redis(host=host, **kwargs)
         self._serializer = serializer
         self._key_prefix = key_prefix
 

rltk/io/adapter/redis_key_value_adapter.py

@@ -3,6 +3,10 @@
 from rltk.record import Record
 from rltk.io.adapter import KeyValueAdapter
 from rltk.io.serializer import Serializer, PickleSerializer
+from rltk.utils import module_importer
+
+
+redis = module_importer('redis', 'redis>=2.0.0')
 
 
 class RedisKeyValueAdapter(KeyValueAdapter):
@@ -20,7 +24,7 @@ class RedisKeyValueAdapter(KeyValueAdapter):
     def __init__(self, host, serializer: Serializer=None, key_prefix: str = '', clean: bool = False, **kwargs):
         if not serializer:
             serializer = PickleSerializer()
-        self._redis = redis.Redis(host=host, **kwargs)
+        self._redis = redis().Redis(host=host, **kwargs)
         self._serializer = serializer
         self._key_prefix = key_prefix
 

rltk/utils.py

@@ -1,4 +1,5 @@
 import unicodedata
+import rltk.cli
 
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
@@ -56,3 +57,26 @@ def get_record_pairs(dataset1,
         for r1 in dataset1:
             for r2 in dataset2:
                 yield r1, r2
+
+
+def module_importer(module_names: str, dependencies: str, notes: str = None):
+    if isinstance(dependencies, str):
+        dependencies = [dependencies]
+
+    def module():
+        try:
+            return __import__(module_names)
+        except ImportError:
+            rltk.cli.prompt('Import Dependencies Error')
+
+            if len(dependencies) > 0:
+                rltk.cli.prompt('Please install dependencies:')
+                for d in dependencies:
+                    rltk.cli.prompt(d)
+
+            if notes:
+                rltk.cli.prompt(notes)
+
+            exit(500)
+
+    return module