Merge branch 'master' into parallelism-improvements

.travis.yml

@@ -3,6 +3,7 @@ cache:
   pip: true
   directories:
     - $HOME/.cache/sacad-build
+    - $HOME/.cache/sacad-test
 matrix:
   include:
     - python: "3.3"
@@ -22,8 +23,8 @@ install:
   - "sudo apt-get -V install optipng jpegoptim"
   - "if [ $WINDOWS -eq 1 ]; then sudo apt-get -V install wine p7zip-full; fi"
 script:
-  - "travis_retry coverage run --source=sacad setup.py test"
-  - "if [ $WINDOWS -eq 1 ]; then pushd win && make test && popd; fi"
+  - "TEST_DL_CACHE_DIR=$HOME/.cache/sacad-test travis_retry coverage run --source=sacad setup.py test"
+  - "if [ $WINDOWS -eq 1 ]; then pushd win && TEST_DL_CACHE_DIR=$HOME/.cache/sacad-test DOWNLOAD_DIR=$HOME/.cache/sacad-build make test && popd; fi"
 after_success:
   - "coveralls"
 before_deploy:

sacad/cover.py

@@ -1,6 +1,7 @@
 import concurrent.futures
 import contextlib
 import enum
+import imghdr
 import io
 import itertools
 import logging
@@ -48,7 +49,8 @@ class CoverSourceResult:
 
   """ Cover image returned by a source, candidate to be downloaded. """
 
-  MAX_FILE_METADATA_PEEK_SIZE = 2 ** 15
+  METADATA_PEEK_SIZE_INCREMENT = 2 ** 12
+  MAX_FILE_METADATA_PEEK_SIZE = 20 * 2 ** 12
   IMG_SIG_SIZE = 16
 
   def __init__(self, urls, size, format, *, thumbnail_url, source, source_quality, rank=None,
@@ -268,11 +270,9 @@ def updateImageMetadata(self):
               except KeyError:
                 pass
             if self.needMetadataUpdate():
-              for new_img_data in response.iter_content(chunk_size=2 ** 12):
+              for new_img_data in response.iter_content(chunk_size=__class__.METADATA_PEEK_SIZE_INCREMENT):
                 img_data.extend(new_img_data)
                 metadata = __class__.getImageMetadata(img_data)
-                if len(img_data) >= CoverSourceResult.MAX_FILE_METADATA_PEEK_SIZE:
-                  break
                 if metadata is not None:
                   format, width, height = metadata
                   if idx == idxs[-1][0]:
@@ -281,6 +281,12 @@ def updateImageMetadata(self):
                     # we may need to check size for other URLs
                     self.check_metadata &= ~CoverImageMetadata.FORMAT
                   break
+                if len(img_data) >= CoverSourceResult.MAX_FILE_METADATA_PEEK_SIZE:
+                  # we won't fetch image header any further, try to identify format
+                  format = __class__.getImageFormat(img_data)
+                  if format is not None:
+                    self.check_metadata &= ~CoverImageMetadata.FORMAT
+                  break
           if (self.check_metadata & CoverImageMetadata.FORMAT) != 0:
             # if we get here, file is probably not reachable, or not even an image
             logging.getLogger().debug("Unable to get file metadata from file or HTTP headers for URL '%s', "
@@ -488,6 +494,12 @@ def getImageMetadata(img_data):
     width, height = img.size
     return format, width, height
 
+  @staticmethod
+  def getImageFormat(img_data):
+    """ Identify an image format from its first bytes. """
+    r = imghdr.what(None, h=img_data)
+    return SUPPORTED_IMG_FORMATS.get(r, None)
+
   @staticmethod
   def preProcessForComparison(results, target_size, size_tolerance_prct):
     """ Process results to prepare them for future comparison and sorting. """

tests/__init__.py

@@ -2,6 +2,7 @@
 
 import contextlib
 import logging
+import math
 import socket
 import unittest
 import unittest.mock
@@ -75,7 +76,7 @@ def test_getImageUrlMetadata(self):
     """ Download the beginning of image files to guess their format and resolution. """
     refs = {"https://www.nuclearblast.de/static/articles/152/152118.jpg/1000x1000.jpg": (sacad.cover.CoverImageFormat.JPEG,
                                                                                          (700, 700),
-                                                                                         5),
+                                                                                         math.ceil(18000 / sacad.CoverSourceResult.METADATA_PEEK_SIZE_INCREMENT)),
             "http://img2-ak.lst.fm/i/u/55ad95c53e6043e3b150ba8a0a3b20a1.png": (sacad.cover.CoverImageFormat.PNG,
                                                                                (600, 600),
                                                                                1)}

tests/recurse_test.py

@@ -8,6 +8,7 @@
 import shutil
 import tempfile
 import unittest
+import urllib.parse
 
 import mutagen
 import requests
@@ -30,11 +31,21 @@ def redirect_stdout(s):
 
 
 def download(url, filepath):
+  cache_dir = os.getenv("TEST_DL_CACHE_DIR")
+  if cache_dir is not None:
+    os.makedirs(cache_dir, exist_ok=True)
+    cache_filepath = os.path.join(cache_dir,
+                                  os.path.basename(urllib.parse.urlsplit(url).path))
+    if os.path.isfile(cache_filepath):
+      shutil.copyfile(cache_filepath, filepath)
+      return
   with contextlib.closing(requests.get(url, stream=True)) as response:
     response.raise_for_status()
     with open(filepath, "wb") as f:
       for chunk in response.iter_content(2 ** 14):
         f.write(chunk)
+  if cache_dir is not None:
+    shutil.copyfile(filepath, cache_filepath)
 
 
 class TestRecursive(unittest.TestCase):

win/Makefile

@@ -6,7 +6,7 @@ CXFREEZE_VERSION     := 4.3.3
 LXML_VERSION         := 3.4.3
 
 # installers
-DOWNLOAD_DIR       := ${HOME}/.cache/sacad-build
+DOWNLOAD_DIR       ?= dl
 PYTHON_INSTALLER   := ${DOWNLOAD_DIR}/python-${PYTHON_VERSION}.msi
 CXFREEZE_INSTALLER := ${DOWNLOAD_DIR}/cx_Freeze-${CXFREEZE_VERSION}.win32-py${PYTHON_VERSION_MAJOR}.msi
 PIP_INSTALLER      := ${DOWNLOAD_DIR}/get-pip.py
@@ -65,6 +65,8 @@ sacad/sacad.exe: sacad ${CXFREEZE_INSTALLED} ${LXML_INSTALLED} .requirements
 	cd .. && ${WINE} ${PYTHON_INSTALLED} freeze.py build_exe -b win/sacad 2> /dev/null > /dev/null
 	touch $@
 
+sacad/sacad_r.exe: sacad/sacad.exe
+
 sacad/python${PYTHON_VERSION_SHORT}.dll: sacad ${PYTHON_INSTALLED}
 	cp $(dir ${PYTHON_INSTALLED})$(notdir $@) $@
 
@@ -96,7 +98,7 @@ ${TEST_LIB_DIR}:
 ${TEST_LIB_DIR}/sample.ogg: ${DOWNLOAD_DIR}/sample.ogg ${TEST_LIB_DIR}
 	cp $< $@
 
-${TEST_LIB_DIR}/test.png: ${TEST_LIB_DIR}/sample.ogg sacad/sacad.exe sacad/sacad_r.exe sacad/python${PYTHON_VERSION_SHORT}.dll sacad/sqlite3.dll
+${TEST_LIB_DIR}/test.png: ${TEST_LIB_DIR}/sample.ogg sacad/sacad_r.exe sacad/python${PYTHON_VERSION_SHORT}.dll sacad/sqlite3.dll
 	${WINE} sacad/sacad_r.exe $(dir $@) 300 $(notdir $@)