Merge pull request #531 from theatlantic/pr/compiler-compressor-unit-…

…tests

Add unit tests for the bundled compiler and compressor classes

.gitignore

@@ -13,6 +13,7 @@ docs/_build/
 coverage/
 tests/static/
 tests/assets/js/dummy.js
+tests/node_modules/
 .tox/
 .DS_Store
 .idea
@@ -21,3 +22,6 @@ tests/assets/js/dummy.js
 .pydevproject
 .ropeproject
 __pycache__
+npm-debug.log
+tests/npm-cache
+django-pipeline-*/

MANIFEST.in

@@ -3,5 +3,8 @@ recursive-include pipeline/jinja2 *.html *.jinja
 include AUTHORS LICENSE README.rst HISTORY.rst
 recursive-include tests *
 recursive-exclude tests *.pyc *.pyo
+recursive-exclude tests/node_modules *
+recursive-exclude tests/npm-cache *
+recursive-exclude tests/npm *
 include docs/Makefile docs/make.bat docs/conf.py
 recursive-include docs *.rst

pipeline/compilers/__init__.py

@@ -12,7 +12,7 @@
 
 from pipeline.conf import settings
 from pipeline.exceptions import CompilerError
-from pipeline.utils import to_class
+from pipeline.utils import to_class, set_std_streams_blocking
 
 
 class Compiler(object):
@@ -120,6 +120,7 @@ def execute_command(self, command, cwd=None, stdout_captured=None):
                                              stdout=stdout,
                                              stderr=subprocess.PIPE)
                 _, stderr = compiling.communicate()
+                set_std_streams_blocking()
 
             if compiling.returncode != 0:
                 stdout_captured = None  # Don't save erroneous result.

pipeline/compressors/__init__.py

@@ -14,7 +14,7 @@
 
 from pipeline.conf import settings
 from pipeline.exceptions import CompressorError
-from pipeline.utils import to_class, relpath
+from pipeline.utils import to_class, relpath, set_std_streams_blocking
 
 URL_DETECTOR = r"""url\((['"]){0,1}\s*(.*?)["']{0,1}\)"""
 URL_REPLACER = r"""url\(__EMBED__(.+?)(\?\d+)?\)"""
@@ -248,6 +248,7 @@ def execute_command(self, command, content):
         if content:
             content = smart_bytes(content)
         stdout, stderr = pipe.communicate(content)
+        set_std_streams_blocking()
         if stderr.strip() and pipe.returncode != 0:
             raise CompressorError(stderr)
         elif self.verbose:

pipeline/conf.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import os
 import collections
 import shlex
 
@@ -101,7 +102,7 @@ def __getitem__(self, key):
         value = self.settings[key]
         if key.endswith(("_BINARY", "_ARGUMENTS")):
             if isinstance(value, string_types):
-                return tuple(shlex.split(value))
+                return tuple(shlex.split(value, posix=(os.name == 'posix')))
             return tuple(value)
         return value
 

pipeline/utils.py

@@ -1,8 +1,16 @@
 from __future__ import unicode_literals
 
+try:
+    import fcntl
+except ImportError:
+    # windows
+    fcntl = None
+
 import importlib
 import mimetypes
 import posixpath
+import os
+import sys
 
 try:
     from urllib.parse import quote
@@ -54,3 +62,19 @@ def relpath(path, start=posixpath.curdir):
     if not rel_list:
         return posixpath.curdir
     return posixpath.join(*rel_list)
+
+
+def set_std_streams_blocking():
+    """
+    Set stdout and stderr to be blocking.
+
+    This is called after Popen.communicate() to revert stdout and stderr back
+    to be blocking (the default) in the event that the process to which they
+    were passed manipulated one or both file descriptors to be non-blocking.
+    """
+    if not fcntl:
+        return
+    for f in (sys.__stdout__, sys.__stderr__):
+        fileno = f.fileno()
+        flags = fcntl.fcntl(fileno, fcntl.F_GETFL)
+        fcntl.fcntl(fileno, fcntl.F_SETFL, flags & ~os.O_NONBLOCK)

tests/assets/compilers/coffee/expected.js

@@ -0,0 +1,12 @@
+(function() {
+  var cube, square;
+
+  square = function(x) {
+    return x * x;
+  };
+
+  cube = function(x) {
+    return square(x) * x;
+  };
+
+}).call(this);

tests/assets/compilers/coffee/input.coffee

@@ -0,0 +1,2 @@
+square = (x) -> x * x
+cube   = (x) -> square(x) * x

tests/assets/compilers/es6/expected.js

@@ -0,0 +1,27 @@
+"use strict";
+
+// Expression bodies
+var odds = evens.map(function (v) {
+  return v + 1;
+});
+var nums = evens.map(function (v, i) {
+  return v + i;
+});
+
+// Statement bodies
+nums.forEach(function (v) {
+  if (v % 5 === 0) fives.push(v);
+});
+
+// Lexical this
+var bob = {
+  _name: "Bob",
+  _friends: [],
+  printFriends: function printFriends() {
+    var _this = this;
+
+    this._friends.forEach(function (f) {
+      return console.log(_this._name + " knows " + f);
+    });
+  }
+};

tests/assets/compilers/es6/input.es6

@@ -0,0 +1,19 @@
+// Expression bodies
+var odds = evens.map(v => v + 1);
+var nums = evens.map((v, i) => v + i);
+
+// Statement bodies
+nums.forEach(v => {
+  if (v % 5 === 0)
+    fives.push(v);
+});
+
+// Lexical this
+var bob = {
+  _name: "Bob",
+  _friends: [],
+  printFriends() {
+    this._friends.forEach(f =>
+      console.log(this._name + " knows " + f));
+  }
+};

tests/assets/compilers/less/expected.css

@@ -0,0 +1,3 @@
+.a {
+  width: 1px;
+}

tests/assets/compilers/less/input.less

@@ -0,0 +1,5 @@
+@a: 1;
+
+.a {
+    width: (@a + 0px);
+}

tests/assets/compilers/livescript/expected.js

@@ -0,0 +1,6 @@
+(function(){
+  var times;
+  times = function(x, y){
+    return x * y;
+  };
+}).call(this);

tests/assets/compilers/livescript/input.ls

@@ -0,0 +1,2 @@
+times = (x, y) ->
+  x * y

tests/assets/compilers/scss/expected.css

@@ -0,0 +1,5 @@
+.a .b {
+  display: none; }
+
+.c .d {
+  display: block; }

tests/assets/compilers/scss/input.scss

@@ -0,0 +1,10 @@
+.a {
+    .b {
+        display: none;
+    }
+}
+.c {
+    .d {
+        display: block;
+    }
+}

tests/assets/compilers/stylus/expected.css

@@ -0,0 +1,3 @@
+.a {
+  color: #000;
+}

tests/assets/compilers/stylus/input.styl

@@ -0,0 +1,2 @@
+.a
+    color: black

tests/assets/compressors/closure.js

@@ -0,0 +1 @@
+(function(){(function(){window.concat=function(){console.log(arguments)}})();(function(){window.cat=function(){console.log("hello world")}})()}).call(this);

tests/assets/compressors/cssmin.css

@@ -0,0 +1 @@
+.concat{display:none}.concatenate{display:block}

tests/assets/compressors/csstidy.css

@@ -0,0 +1 @@
+.concat{display:none;}.concatenate{display:block;}

tests/assets/compressors/uglifyjs.js

@@ -0,0 +1 @@
+(function(){(function(){window.concat=function(){console.log(arguments)}})();(function(){window.cat=function(){console.log("hello world")}})()}).call(this);

tests/assets/compressors/yuglify.css

@@ -0,0 +1 @@
+.concat{display:none}.concatenate{display:block}

tests/assets/compressors/yuglify.js

@@ -0,0 +1 @@
+(function(){(function(){window.concat=function(){console.log(arguments)}})(),function(){window.cat=function(){console.log("hello world")}}()}).call(this);

tests/assets/compressors/yui.css

@@ -0,0 +1 @@
+.concat{display:none}.concatenate{display:block}

tests/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "django-pipeline-tests",
+  "private": true,
+  "version": "1.0.0",
+  "description": "Pipeline is an asset packaging library for Django.",
+  "author": "Timothée Peignier <timothee.peignier@tryphon.org>",
+  "license": "MIT",
+  "readmeFilename": "../README.rst",
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/jazzband/django-pipeline.git"
+  },
+  "dependencies": {
+    "babel-cli": "^6.4.5",
+    "babel-preset-es2015": "^6.3.13",
+    "coffee-script": "^1.10.0",
+    "less": "^2.5.3",
+    "livescript": "^1.4.0",
+    "node-sass": "^3.4.2",
+    "stylus": "^0.53.0",
+    "cssmin": "^0.4.3",
+    "google-closure-compiler": "^20151216.2.0",
+    "uglify-js": "^2.6.1",
+    "yuglify": "^0.1.4",
+    "yuicompressor": "^2.4.8"
+  }
+}

tests/scripts/npm_install.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+"""
+A cross-platform compatible `npm install` call, checking whether npm is
+in fact installed on the system first (and on windows, checking that the
+npm version is at least 3.0 because of a bug in 2.x with MAX_PATH)
+"""
+import distutils.spawn
+import os
+from pkg_resources import parse_version
+import re
+import subprocess
+import sys
+
+
+def main():
+    tests_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+    if os.name == 'nt':
+        try:
+            npm_paths = subprocess.check_output(['where', 'npm.cmd'])
+        except subprocess.CalledProcessError:
+            return
+        else:
+            npm_bin = re.split(r'\r?\n', npm_paths)[0]
+    else:
+        npm_bin = distutils.spawn.find_executable('npm')
+    if not npm_bin:
+        return
+    if os.name == 'nt':
+        os.environ.setdefault('APPDATA', '.')
+        npm_version = subprocess.check_output([npm_bin, '--version']).strip()
+        # Skip on windows if npm version is less than 3 because of
+        # MAX_PATH issues in version 2
+        if parse_version(npm_version) < parse_version('3.0'):
+            return
+    pipe = subprocess.Popen([npm_bin, 'install'],
+        cwd=tests_dir, stdout=sys.stdout, stderr=sys.stderr)
+    pipe.communicate()
+    sys.exit(pipe.returncode)
+
+
+if __name__ == '__main__':
+    main()

tests/settings.py

@@ -1,4 +1,6 @@
+import glob
 import os
+import distutils.spawn
 
 
 def local_path(path):
@@ -58,6 +60,8 @@ def local_path(path):
 
 PIPELINE = {
     'PIPELINE_ENABLED': True,
+    'JS_COMPRESSOR': None,
+    'CSS_COMPRESSOR': None,
     'STYLESHEETS': {
         'screen': {
             'source_filenames': (
@@ -118,6 +122,48 @@ def local_path(path):
     }
 }
 
+NODE_MODULES_PATH = local_path('node_modules')
+NODE_BIN_PATH = os.path.join(NODE_MODULES_PATH, '.bin')
+NODE_EXE_PATH = distutils.spawn.find_executable('node')
+JAVA_EXE_PATH = distutils.spawn.find_executable('java')
+CSSTIDY_EXE_PATH = distutils.spawn.find_executable('csstidy')
+HAS_NODE = os.path.exists(NODE_BIN_PATH) and NODE_EXE_PATH
+HAS_JAVA = bool(JAVA_EXE_PATH)
+HAS_CSSTIDY = bool(CSSTIDY_EXE_PATH)
+
+if HAS_NODE:
+    def node_exe_path(command):
+        exe_ext = '.cmd' if os.name == 'nt' else ''
+        return os.path.join(NODE_BIN_PATH, "%s%s" % (command, exe_ext))
+
+    PIPELINE.update({
+        'SASS_BINARY': node_exe_path('node-sass'),
+        'COFFEE_SCRIPT_BINARY': node_exe_path('coffee'),
+        'COFFEE_SCRIPT_ARGUMENTS': ['--no-header'],
+        'LESS_BINARY': node_exe_path('lessc'),
+        'BABEL_BINARY': node_exe_path('babel'),
+        'BABEL_ARGUMENTS': ['--presets', 'es2015'],
+        'STYLUS_BINARY': node_exe_path('stylus'),
+        'LIVE_SCRIPT_BINARY': node_exe_path('lsc'),
+        'LIVE_SCRIPT_ARGUMENTS': ['--no-header'],
+        'YUGLIFY_BINARY': node_exe_path('yuglify'),
+        'UGLIFYJS_BINARY': node_exe_path('uglifyjs'),
+        'CSSMIN_BINARY': node_exe_path('cssmin'),
+    })
+
+if HAS_NODE and HAS_JAVA:
+    PIPELINE.update({
+        'CLOSURE_BINARY': [
+            JAVA_EXE_PATH, '-jar',
+            os.path.join(NODE_MODULES_PATH, 'google-closure-compiler', 'compiler.jar')],
+        'YUI_BINARY': [
+            JAVA_EXE_PATH, '-jar',
+            glob.glob(os.path.join(NODE_MODULES_PATH, 'yuicompressor', 'build', '*.jar'))[0]]
+    })
+
+if HAS_CSSTIDY:
+    PIPELINE.update({'CSSTIDY_BINARY': CSSTIDY_EXE_PATH})
+
 TEMPLATE_DIRS = (
     local_path('templates'),
 )

tests/tests/__init__.py

@@ -1,4 +1,12 @@
 # -*- coding: utf-8 flake8: noqa -*-
+import os
+import sys
+
+
+if sys.platform.startswith('win'):
+    os.environ.setdefault('NUMBER_OF_PROCESSORS', '1')
+
+
 from .test_compiler import *
 from .test_compressor import *
 from .test_template import *