Build patched redis-dump-load (#2277)

* Build patched redis-dump-load
* Fix build
* Add build rule

.gitmodules

@@ -62,3 +62,6 @@
 [submodule "platform/mellanox/hw-management/hw-mgmt"]
 	path = platform/mellanox/hw-management/hw-mgmt
 	url = https://github.com/Mellanox/hw-mgmt/
+[submodule "src/redis-dump-load"]
+	path = src/redis-dump-load
+	url = https://github.com/p/redis-dump-load.git

files/build_templates/sonic_debian_extension.j2

@@ -63,7 +63,7 @@ sudo mkdir -p $FILESYSTEM_ROOT/etc/sonic/
 sudo mkdir -p $FILESYSTEM_ROOT/var/cache/sonic/
 sudo mkdir -p $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/
 
-# Install dependencies for SONiC config engine 
+# Install dependencies for SONiC config engine
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install \
     python-dev     \
     python-lxml    \
@@ -78,7 +78,12 @@ sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME
 
 # Install Python client for Redis
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==2.10.6"
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install redis-dump-load
+
+# Install redis-dump-load Python 2 package
+REDIS_DUMP_LOAD_PY2_WHEEL_NAME=$(basename {{redis_dump_load_py2_wheel_path}})
+sudo cp {{redis_dump_load_py2_wheel_path}} $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $REDIS_DUMP_LOAD_PY2_WHEEL_NAME
+sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME
 
 # Install SwSS SDK Python 2 package
 SWSSSDK_PY2_WHEEL_NAME=$(basename {{swsssdk_py2_wheel_path}})
@@ -292,9 +297,9 @@ sudo LANG=C cp $SCRIPTS_DIR/syncd.sh $FILESYSTEM_ROOT/usr/local/bin/syncd.sh
 sudo cp $BUILD_TEMPLATES/snmp.timer $FILESYSTEM_ROOT/etc/systemd/system/
 sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable snmp.timer
 
-sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get remove -y python-dev 
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get remove -y python-dev
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get clean -y
-sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get autoremove -y 
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get autoremove -y
 
 {% for file in installer_extra_files.split(' ') -%}
 {% if file.strip() -%}

rules/redis-dump-load-py2.mk

@@ -0,0 +1,6 @@
+# redis_dump_load python2 wheel
+
+REDIS_DUMP_LOAD_PY2 = redis_dump_load-1.1-py2-none-any.whl
+$(REDIS_DUMP_LOAD_PY2)_SRC_PATH = $(SRC_PATH)/redis-dump-load
+$(REDIS_DUMP_LOAD_PY2)_PYTHON_VERSION = 2
+SONIC_PYTHON_WHEELS += $(REDIS_DUMP_LOAD_PY2)

slave.mk

@@ -479,7 +479,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
                 $(LIBNSS_TACPLUS)) \
         $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \
         $$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE)) \
-        $$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2))
+        $$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2)) \
+        $$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2))
 	$(HEADER)
 	# Pass initramfs and linux kernel explicitly. They are used for all platforms
 	export initramfs_tools="$(DEBS_PATH)/$(INITRAMFS_TOOLS)"
@@ -498,6 +499,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export config_engine_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE))"
 	export swsssdk_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY2))"
 	export platform_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2))"
+	export redis_dump_load_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2))"
 
 	$(foreach docker, $($*_DOCKERS),\
 		export docker_image="$(docker)"

src/redis-dump-load.patch/0001-Use-pipelines-when-dumping-52.patch

@@ -0,0 +1,154 @@
+From ed20dced07d8b2d140e2c1d79d506be0e12f339e Mon Sep 17 00:00:00 2001
+From: Oleg Pudeyev <oleg@bsdpower.com>
+Date: Sat, 28 Jan 2017 15:37:43 -0500
+Subject: [PATCH] Use pipelines when dumping, #52
+
+---
+ redisdl.py | 102 +++++++++++++++++++++++++++++++++++++++--------------
+ 1 file changed, 75 insertions(+), 27 deletions(-)
+
+diff --git a/redisdl.py b/redisdl.py
+index df2870b..1b6063b 100755
+--- a/redisdl.py
++++ b/redisdl.py
+@@ -141,18 +141,18 @@ def dumps(host='localhost', port=6379, password=None, db=0, pretty=False,
+ class BytesWriteWrapper(object):
+     def __init__(self, stream):
+         self.stream = stream
+-        
++
+     def write(self, str):
+         return self.stream.write(str.encode())
+
+ def dump(fp, host='localhost', port=6379, password=None, db=0, pretty=False,
+          unix_socket_path=None, encoding='utf-8', keys='*'):
+-    
++
+     try:
+         fp.write('')
+     except TypeError:
+         fp = BytesWriteWrapper(fp)
+-    
++
+     if pretty:
+         # hack to avoid implementing pretty printing
+         fp.write(dumps(host=host, port=port, password=password, db=db,
+@@ -276,28 +276,76 @@ def _read_key(key, r, pretty, encoding):
+     return (type, ttl, value)
+
+ def _reader(r, pretty, encoding, keys='*'):
+-    for encoded_key in r.keys(keys):
+-        key = encoded_key.decode(encoding)
+-        handled = False
+-        for i in range(10):
+-            try:
+-                type, ttl, value = _read_key(encoded_key, r, pretty, encoding)
+-                yield key, type, ttl, value
+-                handled = True
+-                break
+-            except KeyDeletedError:
+-                # do not dump the key
+-                handled = True
+-                break
+-            except redis.WatchError:
+-                # same logic as key type changed
+-                pass
+-            except KeyTypeChangedError:
+-                # retry reading type again
+-                pass
+-        if not handled:
+-            # ran out of retries
+-            raise ConcurrentModificationError('Key %s is being concurrently modified' % key)
++    encoded_keys = r.keys(keys)
++    i = 0
++    while i < len(encoded_keys):
++        for key, type, ttl, value in _read_keys(r, encoded_keys[i:i+10000],
++                pretty=pretty, encoding=encoding):
++            yield key, type, ttl, value
++        i += 10000
++
++def _read_keys(r, encoded_keys, pretty, encoding):
++    decoded_keys = [encoded_key.decode() for encoded_key in encoded_keys]
++    do_keys = decoded_keys
++    retries = 5
++    type_results = None
++    while len(do_keys) > 0 and retries > 0:
++        next_do_keys = []
++        next_type_results = []
++
++        if type_results is None:
++            # first pass, need to get the types.
++            # on subsequent passes we know the types
++            # because the previous pass retrieved them and
++            # found a type mismatch
++            p = r.pipeline()
++            for key in do_keys:
++                p.type(key)
++            type_results = p.execute()
++
++        p = r.pipeline()
++        for i in range(len(do_keys)):
++            key = decoded_keys[i]
++            type = type_results[i].decode('ascii')
++            if type == 'none':
++                # key was deleted by a concurrent operation on the data store.
++                # issue noops so that the number of results does not change
++                p.type(key)
++                p.type(key)
++                p.type(key)
++                continue
++            reader = readers.get(type)
++            if reader is None:
++                raise UnknownTypeError("Unknown key type: %s" % type)
++            reader.send_command(p, key)
++            r.pttl_or_ttl_pipeline(p, key)
++            p.type(key)
++        results = p.execute()
++
++        for i in range(len(do_keys)):
++            key = decoded_keys[i]
++            original_type = type_results[i]
++            if original_type == 'none':
++                # this is where we actually skip a key that was deleted
++                # by concurrent operations
++                continue
++            final_type = results[i*3+2].decode('ascii')
++            if original_type != final_type:
++                # type changed, will retry
++                next_do_keys.append(key)
++                # need to update expected type
++                next_type_results.append(final_type)
++                continue
++            reader = readers.get(original_type)
++            value = reader.handle_response(results[i*3], pretty, encoding)
++            ttl = r.decode_pttl_or_ttl_pipeline_value(results[i*3+1])
++            yield key, final_type, ttl, value
++        retries -= 1
++        do_keys = next_do_keys
++        type_results = next_type_results
++
++    if len(do_keys) > 0:
++        raise ConcurrentModificationError('Keys %s are being concurrently modified' % ', '.join(do_keys))
+
+ def _empty(r):
+     for key in r.keys():
+@@ -372,14 +420,14 @@ def ijson_top_level_items(file, local_streaming_backend):
+ class TextReadWrapper(object):
+     def __init__(self, fp):
+         self.fp = fp
+-        
++
+     def read(self, *args, **kwargs):
+         return self.fp.read(*args, **kwargs).decode()
+
+ class BytesReadWrapper(object):
+     def __init__(self, fp):
+         self.fp = fp
+-        
++
+     def read(self, *args, **kwargs):
+         return self.fp.read(*args, **kwargs).encode('utf-8')
+
+-- 
+2.18.0
+

src/redis-dump-load.patch/0002-Fix-setup.py-for-test-and-bdist_wheel.patch

@@ -0,0 +1,26 @@
+From c2c93fa3b702a4f2364383fd4ae69763068686d2 Mon Sep 17 00:00:00 2001
+From: Qi Luo <qiluo-msft@users.noreply.github.com>
+Date: Tue, 20 Nov 2018 03:21:31 +0000
+Subject: [PATCH] Fix setup.py for test and bdist_wheel
+
+Signed-off-by: Qi Luo <qiluo-msft@users.noreply.github.com>
+---
+ setup.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/setup.py b/setup.py
+index 8ccf31f..6457163 100644
+--- a/setup.py
++++ b/setup.py
+@@ -1,7 +1,7 @@
+ #!/usr/bin/env python
+
+ import os.path
+-from distutils.core import setup
++from setuptools import setup, find_packages
+
+ package_name = 'redis-dump-load'
+ package_version = '1.1'
+-- 
+2.18.0
+

src/redis-dump-load.patch/series

@@ -0,0 +1,2 @@
+0001-Use-pipelines-when-dumping-52.patch
+0002-Fix-setup.py-for-test-and-bdist_wheel.patch