various fixes from testing with bugzilla

docs/features.rst

@@ -38,3 +38,9 @@ Supported Features and Limitations
 
 * Broken HTML or broken/invalid CSS isn't support and good results can
   not be guaranteed.
+
+
+**Things that don't work:**
+
+* link tags wrapped in IE-only style comments (e.g ``<!--[if lte IE
+  7]>``) is not supported.

mincss/processor.py

@@ -54,8 +54,8 @@ def _download(self, url):
                     raise DownloadError(
                         '%s -- %s ' % (url, response.getcode())
                     )
-            html = response.read()
-            return unicode(html, 'utf-8')
+            content = response.read()
+            return unicode(content, 'utf-8')
         except IOError:
             raise IOError(url)
 
@@ -116,7 +116,7 @@ def process_html(self, html, url):
         for link in CSSSelector('link')(page):
             if (
                 link.attrib.get('rel', '') == 'stylesheet' or
-                link.attrib['href'].lower().endswith('.css')
+                link.attrib['href'].lower().split('?')[0].endswith('.css')
             ):
                 link_url = self._make_absolute_url(url, link.attrib['href'])
                 key = (link_url, link.attrib['href'])

proxy/app.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python
+import codecs
 import datetime
 import os
 import functools
+import logging
 import hashlib
 import re
 import urllib
@@ -14,12 +16,10 @@
 from flask import Flask, abort, make_response, request
 app = Flask(__name__)
 
-try:
-    from mincss.processor import Processor
-except ImportError:
-    import sys
-    sys.path.insert(0, os.path.normpath('../'))
-    from mincss.processor import Processor
+import sys
+# do this to help development
+sys.path.insert(0, os.path.normpath('../'))
+from mincss.processor import Processor
 
 
 CACHE_DIR = os.path.join(
@@ -47,11 +47,19 @@ def proxy(path):
     if path == 'favicon.ico':
         abort(404)
     url = path
-    if not path.startswith('http://'):
+    if not path.count('://'):
         url = 'http://' + url
+
+    query = urlparse.urlparse(request.url).query
+    if query:
+        url += '?%s' % query
+    logging.info('Downloading %s' % url)
     html = download(url)
+
     p = Processor(debug=False)
-    p.process(url)
+    # since we've already download the HTML
+    p.process_html(html, url)
+    p.process()
 
     collect_stats = request.args.get('MINCSS_STATS', False)
     stats = []
@@ -73,6 +81,7 @@ def css_url_replacer(match, href=None):
             return bail
 
         if not filename.startswith('/'):
+
             filename = os.path.normpath(
                 os.path.join(
                     os.path.dirname(href),
@@ -86,7 +95,10 @@ def css_url_replacer(match, href=None):
     for i, each in enumerate(p.inlines):
         # this should be using CSSSelector instead
         new_inline = each.after
-        new_inline = css_url_regex.sub(css_url_replacer, new_inline)
+        new_inline = css_url_regex.sub(
+            functools.partial(css_url_replacer, href=url),
+            new_inline
+        )
         stats.append(
             ('inline %s' % (i + 1), each.before, each.after)
         )
@@ -103,11 +115,12 @@ def css_url_replacer(match, href=None):
     #root = tree if stripped.startswith(tree.docinfo.doctype) else page
 
     links = dict((x.href, x) for x in p.links)
+
     #all_lines = html.splitlines()
     for link in CSSSelector('link')(page):
         if (
             link.attrib.get('rel', '') == 'stylesheet' or
-            link.attrib['href'].lower().endswith('.css')
+            link.attrib['href'].lower().split('?')[0].endswith('.css')
         ):
             hash_ = hashlib.md5(url + link.attrib['href']).hexdigest()[:7]
             now = datetime.date.today()
@@ -118,7 +131,6 @@ def css_url_replacer(match, href=None):
                 str(now.day),
             )
             mkdir(destination_dir)
-
             new_css = links[link.attrib['href']].after
             stats.append((
                 link.attrib['href'],
@@ -133,7 +145,8 @@ def css_url_replacer(match, href=None):
                 new_css
             )
             destination = os.path.join(destination_dir, hash_ + '.css')
-            with open(destination, 'w') as f:
+
+            with codecs.open(destination, 'w', 'utf-8') as f:
                 f.write(new_css)
 
             link.attrib['href'] = (