Merge pull request #26 from wjo1212/master

support to compress data for get/put API with zip

aliyun/log/logclient.py

@@ -39,6 +39,7 @@
 from .version import API_VERSION, USER_AGENT
 import json
 import six
+import zlib
 
 CONNECTION_TIME_OUT = 20
 
@@ -264,8 +265,9 @@ def put_logs(self, request):
 
         compress_data = None
         if is_compress:
-            headers['x-log-compresstype'] = 'lz4'
-            compress_data = logservice_lz4.compress(body)
+            headers['x-log-compresstype'] = 'deflate'
+            compress_data = zlib.compress(body)
+            #compress_data = logservice_lz4.compress(body)
 
         params = {}
         logstore = request.get_logstore()
@@ -566,7 +568,7 @@ def pull_logs(self, project_name, logstore_name, shard_id, cursor, count=1000, e
         """
         headers = {}
         if compress:
-            headers['Accept-Encoding'] = 'lz4'
+            headers['Accept-Encoding'] = 'gzip'
         else:
             headers['Accept-Encoding'] = ''
 
@@ -580,10 +582,16 @@ def pull_logs(self, project_name, logstore_name, shard_id, cursor, count=1000, e
         if end_cursor:
             params['end_cursor'] = end_cursor
         (resp, header) = self._send("GET", project_name, None, resource, params, headers, "binary")
-        if compress:
+
+        compress_type = Util.h_v_td(header,'x-log-compresstype', '').lower()
+        if compress_type == 'lz4':
             raw_size = int(Util.h_v_t(header, 'x-log-bodyrawsize'))
             raw_data = logservice_lz4.uncompress(raw_size, resp)
             return PullLogResponse(raw_data, header)
+        elif compress_type in ('gzip', 'deflate'):
+            raw_size = int(Util.h_v_t(header, 'x-log-bodyrawsize'))
+            raw_data = zlib.decompress(resp)
+            return PullLogResponse(raw_data, header)
         else:
             return PullLogResponse(resp, header)
 

aliyun/log/putlogsrequest.py

@@ -25,11 +25,15 @@ class PutLogsRequest(LogRequest):
     :type logitems: list<LogItem>
     :param logitems: log data
 
-    :type logtags: list 
-    :param logtags: list of key:value tag pair , [(tag_key_1,tag_value_1) , (tag_key_2,tag_value_2)]
-
     :type hashKey: String
     :param hashKey: put data with set hash, the data will be send to shard whose range contains the hashKey
+
+    :type compress: bool
+    :param compress: if need to compress the logs
+
+    :type logtags: list
+    :param logtags: list of key:value tag pair , [(tag_key_1,tag_value_1) , (tag_key_2,tag_value_2)]
+
     """
 
     def __init__(self, project=None, logstore=None, topic=None, source=None, logitems=None, hashKey=None,

aliyun/log/version.py

@@ -1,4 +1,4 @@
-__version__ = '0.6.9'
+__version__ = '0.6.10'
 USER_AGENT = 'log-python-sdk-v-' + __version__
 API_VERSION = '0.6.0'
 

tests/sample.py

@@ -8,7 +8,7 @@
 import os
 
 
-def sample_put_logs(client, project, logstore):
+def sample_put_logs(client, project, logstore, compress=False):
     topic = 'TestTopic_2'
     source = ''
     contents = [
@@ -21,7 +21,7 @@ def sample_put_logs(client, project, logstore):
     logItem.set_contents(contents)
     for i in range(0, 1):
         logitemList.append(logItem)
-    request = PutLogsRequest(project, logstore, topic, source, logitemList)
+    request = PutLogsRequest(project, logstore, topic, source, logitemList, compress=compress)
 
     response = client.put_logs(request)
     response.log_print()
@@ -39,12 +39,12 @@ def sample_put_logs(client, project, logstore):
     res.log_print()
 
 # @log_enter_exit
-def sample_pull_logs(client, project, logstore):
+def sample_pull_logs(client, project, logstore, compress=False):
     res = client.get_cursor(project, logstore, 0, int(time.time() - 60))
     res.log_print()
     cursor = res.get_cursor()
 
-    res = client.pull_logs(project, logstore, 0, cursor, 1)
+    res = client.pull_logs(project, logstore, 0, cursor, 1, compress=compress)
     res.log_print()
 
 # @log_enter_exit
@@ -258,8 +258,10 @@ def main():
     sample_list_topics(client, project, logstore)
     time.sleep(40)
     sample_put_logs(client, project, logstore)
+    sample_put_logs(client, project, logstore, compress=True)
 
     sample_pull_logs(client, project, logstore)
+    sample_pull_logs(client, project, logstore, compress=True)
 
     time.sleep(40)
     sample_get_logs(client, project, logstore)