Merge pull request #1555 from wlach/1277499

Bug 1277499 - Handle non-gzipped files in logslice API

tests/webapp/api/test_logslice_api.py

@@ -9,10 +9,16 @@
                                      JobLog)
 
 
-@pytest.mark.parametrize('logname', ['buildbot_text', 'builds-4h'])
-@pytest.mark.parametrize('line_range', [(0, 10), (5, 8), (1, 11)])
+@pytest.mark.parametrize('logname, line_range, gzipped, num_loads', [
+    ('buildbot_text', (0, 10), True, 1),
+    ('builds-4h', (0, 10), True, 1),
+    ('builds-4h', (0, 10), False, 1),
+    ('builds-4h', (0, 10), True, 2),
+    ('builds-4h', (0, 10), False, 2),
+    ('builds-4h', (5, 8), True, 1),
+    ('builds-4h', (1, 11), True, 1)])
 def test_logslice_api(test_repository, webapp, activate_responses, logname,
-                      line_range):
+                      line_range, gzipped, num_loads):
     job = Job.objects.create(repository=test_repository,
                              guid="12345", project_specific_id=1)
     fake_log_url = 'http://www.fakelog.com/log.gz'
@@ -21,19 +27,24 @@ def test_logslice_api(test_repository, webapp, activate_responses, logname,
 
     lines = ['cheezburger %s' % i for i in range(10)]
 
-    # set up a gzipped file response
+    # set up a file response
+    text = "\n".join(lines) + '\n'
     content = BytesIO()
-    with gzip.GzipFile('none', 'w', fileobj=content) as gz:
-        gz.write("\n".join(lines) + '\n')
+    if gzipped:
+        with gzip.GzipFile('none', 'w', fileobj=content) as gz:
+            gz.write(text)
+    else:
+        content.write(text)
     content.seek(0)
     responses.add(responses.GET, fake_log_url,
                   body=content.read(),
                   content_type="text/plain;charset=utf-8", status=200)
 
     # now test it
-    resp = webapp.get(reverse('logslice-list',
-                              kwargs={"project": test_repository.name}) +
-                      '?start_line={}&end_line={}&job_id=1'.format(line_range[0],
-                                                                   line_range[1]))
-    assert resp.json == [{'index': i + line_range[0], 'text': l + '\n'} for (i, l) in
-                         enumerate(lines[line_range[0]:line_range[1]])]
+    for i in range(num_loads):
+        resp = webapp.get(reverse('logslice-list',
+                                  kwargs={"project": test_repository.name}) +
+                          '?start_line={}&end_line={}&job_id=1'.format(line_range[0],
+                                                                       line_range[1]))
+        assert resp.json == [{'index': i + line_range[0], 'text': l + '\n'} for (i, l) in
+                             enumerate(lines[line_range[0]:line_range[1]])]

treeherder/webapp/api/logslice.py

@@ -33,7 +33,7 @@ def list(self, request, project):
             log_names = ["buildbot_text", "builds-4h"]
         format = 'json' if log_name == 'mozlog_json' else 'text'
 
-        gz_file = None
+        file = None
 
         start_line = request.query_params.get("start_line")
         end_line = request.query_params.get("end_line")
@@ -63,16 +63,28 @@ def list(self, request, project):
             return Response("Job log does not exist", 404)
 
         try:
-            gz_file = filesystem.get(url)
-            if not gz_file:
+            file = filesystem.get(url)
+            if not file:
                 r = make_request(url)
-                gz_file = gzip.GzipFile(fileobj=BytesIO(r.content))
-                filesystem.set(url, gz_file.fileobj)
+                try:
+                    file = gzip.GzipFile(fileobj=BytesIO(r.content))
+                    # read 16 bytes, just to make sure the file is gzipped
+                    file.read(16)
+                    file.seek(0)
+                    filesystem.set(url, file.fileobj)
+                except IOError:
+                    # file is not gzipped, but we should still store / read
+                    # it as such, to save space
+                    file = BytesIO(r.content)
+                    gz_file_content = BytesIO()
+                    with gzip.GzipFile('none', 'w', fileobj=gz_file_content) as gz:
+                        gz.write(r.content)
+                    filesystem.set(url, gz_file_content)
             else:
-                gz_file = gzip.GzipFile(fileobj=gz_file)
+                file = gzip.GzipFile(fileobj=file)
 
             lines = []
-            for i, line in enumerate(gz_file):
+            for i, line in enumerate(file):
                 if i < start_line:
                     continue
                 elif i >= end_line:
@@ -86,5 +98,5 @@ def list(self, request, project):
             return Response(lines)
 
         finally:
-            if gz_file:
-                gz_file.close()
+            if file:
+                file.close()