From ff1cfab10452cbfd28c13303aa58af43d88f730c Mon Sep 17 00:00:00 2001 From: Brandon Savage Date: Thu, 20 Sep 2012 16:11:32 -0400 Subject: [PATCH] Third time is the charm. --- src/product_details/__init__.py | 113 +++++++++++ src/product_details/json/.gitignore | 2 + src/product_details/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../commands/update_product_details.py | 180 +++++++++++++++++ src/product_details/settings_defaults.py | 13 ++ .../version_compare/__init__.py | 184 ++++++++++++++++++ .../version_compare/decorators.py | 21 ++ src/product_details/version_compare/tests.py | 107 ++++++++++ src/product_details/version_compare/utils.py | 17 ++ 10 files changed, 637 insertions(+) create mode 100644 src/product_details/__init__.py create mode 100644 src/product_details/json/.gitignore create mode 100644 src/product_details/management/__init__.py create mode 100644 src/product_details/management/commands/__init__.py create mode 100644 src/product_details/management/commands/update_product_details.py create mode 100644 src/product_details/settings_defaults.py create mode 100644 src/product_details/version_compare/__init__.py create mode 100644 src/product_details/version_compare/decorators.py create mode 100644 src/product_details/version_compare/tests.py create mode 100644 src/product_details/version_compare/utils.py diff --git a/src/product_details/__init__.py b/src/product_details/__init__.py new file mode 100644 index 0000000..a7bf40d --- /dev/null +++ b/src/product_details/__init__.py @@ -0,0 +1,113 @@ +""" +When this module is imported, we load all the .json files and insert them as +module attributes using locals(). It's a magical and wonderful process. +""" +import codecs +import collections +import datetime +import json +import logging +import os + +# During `pip install`, we need this to pass even without Django present. +try: + from django.conf import settings +except ImportError: + settings = None + +from product_details import settings_defaults + + +class MissingJSONData(IOError): + pass + + +VERSION = (0, 5) +__version__ = '.'.join(map(str, VERSION)) +__all__ = ['VERSION', '__version__', 'product_details', 'version_compare'] + +log = logging.getLogger('product_details') +log.setLevel(logging.WARNING) + + +def settings_fallback(key): + """Grab user-defined settings, or fall back to default.""" + try: + return getattr(settings, key) + except (AttributeError, ImportError): + return getattr(settings_defaults, key) + + +class ProductDetails(object): + """ + Main product details class. Implements the JSON files' content as + attributes, e.g.: product_details.firefox_version_history . + """ + json_data = {} + + def __init__(self): + """Load JSON files and keep them in memory.""" + + json_dir = settings_fallback('PROD_DETAILS_DIR') + + for filename in os.listdir(json_dir): + if filename.endswith('.json'): + name = os.path.splitext(filename)[0] + path = os.path.join(json_dir, filename) + self.json_data[name] = json.load(open(path)) + + def __getattr__(self, key): + """Catch-all for access to JSON files.""" + try: + return self.json_data[key] + except KeyError: + log.warn('Requested product details file %s not found!' % key) + return collections.defaultdict(lambda: None) + + @property + def last_update(self): + """Return the last-updated date, if it exists.""" + + json_dir = settings_fallback('PROD_DETAILS_DIR') + fmt = '%a, %d %b %Y %H:%M:%S %Z' + dates = [] + for directory in (json_dir, os.path.join(json_dir, 'regions')): + file = os.path.join(directory, '.last_update') + try: + with open(file) as f: + d = f.read() + except IOError: + d = '' + + try: + dates.append(datetime.datetime.strptime(d, fmt)) + except ValueError: + dates.append(None) + + if None in dates: + return None + # For backwards compat., just return the date of the parent. + return dates[0] + + def get_regions(self, locale): + """Loads regions json file into memory, but only as needed.""" + lookup = [locale, 'en-US'] + if '-' in locale: + fallback, _, _ = locale.partition('-') + lookup.insert(1, fallback) + for l in lookup: + key = 'regions/%s' % l + path = os.path.join(settings_fallback('PROD_DETAILS_DIR'), + 'regions', '%s.json' % l) + if self.json_data.get(key): + return self.json_data.get(key) + if os.path.exists(path): + with codecs.open(path, encoding='utf8') as fd: + self.json_data[key] = json.load(fd) + return self.json_data[key] + + raise MissingJSONData('Unable to load region data for %s or en-US' % + locale) + + +product_details = ProductDetails() diff --git a/src/product_details/json/.gitignore b/src/product_details/json/.gitignore new file mode 100644 index 0000000..5cf10c4 --- /dev/null +++ b/src/product_details/json/.gitignore @@ -0,0 +1,2 @@ +*.json +.last_update diff --git a/src/product_details/management/__init__.py b/src/product_details/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/product_details/management/commands/__init__.py b/src/product_details/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/product_details/management/commands/update_product_details.py b/src/product_details/management/commands/update_product_details.py new file mode 100644 index 0000000..3766ec3 --- /dev/null +++ b/src/product_details/management/commands/update_product_details.py @@ -0,0 +1,180 @@ +import httplib +import json +import logging +from optparse import make_option +import os +import re +import shutil +import tempfile +import urllib2 +from urlparse import urljoin + +from django.core.management.base import NoArgsCommand, CommandError + +from product_details import settings_fallback + + +log = logging.getLogger('prod_details') +log.addHandler(logging.StreamHandler()) +log.setLevel(settings_fallback('LOG_LEVEL')) + + +class Command(NoArgsCommand): + help = 'Update Mozilla product details off SVN.' + requires_model_validation = False + option_list = NoArgsCommand.option_list + ( + make_option('-f', '--force', action='store_true', dest='force', + default=False, help=( + 'Download product details even if they have not been ' + 'updated since the last fetch.')), + make_option('-q', '--quiet', action='store_true', dest='quiet', + default=False, help=( + 'If no error occurs, swallow all output.')), + ) + + def __init__(self, *args, **kwargs): + # some settings + self.PROD_DETAILS_DIR = settings_fallback('PROD_DETAILS_DIR') + self.PROD_DETAILS_URL = settings_fallback('PROD_DETAILS_URL') + + super(Command, self).__init__(*args, **kwargs) + + def handle_noargs(self, **options): + self.options = options + + # Should we be quiet? + if self.options['quiet']: + log.setLevel(logging.WARNING) + + # Determine last update timestamp and check if we need to update again. + if self.options['force']: + log.info('Product details update forced.') + + self.download_directory(self.PROD_DETAILS_URL, self.PROD_DETAILS_DIR) + self.download_directory(urljoin(self.PROD_DETAILS_URL, 'regions/'), + os.path.join(self.PROD_DETAILS_DIR, 'regions/')) + + log.debug('Product Details update run complete.') + + def download_directory(self, src, dest): + # Grab list of JSON files from server. + log.debug('Grabbing list of JSON files from the server from %s' % src) + if not os.path.exists(dest): + os.makedirs(dest) + + json_files = self.get_file_list(src, dest) + if not json_files: + return + + # Grab all modified JSON files from server and replace them locally. + had_errors = False + for json_file in json_files: + if not self.download_json_file(src, dest, json_file): + had_errors = True + + if had_errors: + log.warn('Update run had errors, not storing "last updated" ' + 'timestamp.') + else: + # Save Last-Modified timestamp to detect updates against next time. + log.debug('Writing last-updated timestamp (%s).' % ( + self.last_mod_response)) + with open(os.path.join(dest, '.last_update'), + 'w') as timestamp_file: + timestamp_file.write(self.last_mod_response) + + def get_file_list(self, src, dest): + """ + Get list of files to be updated from the server. + + If no files have been modified, returns an empty list. + """ + # If not forced: Read last updated timestamp + self.last_update_local = None + headers = {} + + if not self.options['force']: + try: + self.last_update_local = open(os.path.join(dest, '.last_update')).read() + headers = {'If-Modified-Since': self.last_update_local} + log.debug('Found last update timestamp: %s' % ( + self.last_update_local)) + except (IOError, ValueError): + log.info('No last update timestamp found.') + + # Retrieve file list if modified since last update + try: + resp = urllib2.urlopen(urllib2.Request(src, headers=headers)) + except urllib2.URLError, e: + if e.code == httplib.NOT_MODIFIED: + log.info('Product Details were up to date.') + return [] + else: + raise CommandError('Could not retrieve file list: %s' % e) + + + # Remember Last-Modified header. + self.last_mod_response = resp.info()['Last-Modified'] + + json_files = set(re.findall(r'href="([^"]+.json)"', resp.read())) + return json_files + + def download_json_file(self, src, dest, json_file): + """ + Downloads a JSON file off the server, checks its validity, then drops + it into the target dir. + + Returns True on success, False otherwise. + """ + log.info('Updating %s from server' % json_file) + + if not self.options['force']: + headers = {'If-Modified-Since': self.last_update_local} + else: + headers = {} + + # Grab JSON data if modified + try: + resp = urllib2.urlopen(urllib2.Request( + urljoin(src, json_file), headers=headers)) + except urllib2.URLError, e: + if e.code == httplib.NOT_MODIFIED: + log.debug('%s was not modified.' % json_file) + return True + else: + log.warn('Error retrieving %s: %s' % (json_file, e)) + return False + + json_data = resp.read() + + # Empty results are fishy + if not json_data: + log.warn('JSON source for %s was empty. Cowardly denying to ' + 'import empty data.' % json_file) + return False + + # Try parsing the file, import if it's valid JSON. + try: + parsed = json.loads(json_data) + except ValueError: + log.warn('Could not parse JSON data from %s. Skipping.' % ( + json_file)) + return False + + # Write JSON data to HD. + log.debug('Writing new copy of %s to %s.' % ( + json_file, dest)) + tf = tempfile.NamedTemporaryFile(delete=False) + tf.write(urllib2.urlopen( + urljoin(src, json_file)).read()) + tf.close() + + # lchmod is available on BSD-based Unixes only. + if hasattr(os, 'lchmod'): + os.lchmod(tf.name, 0644) + else: + os.chmod(tf.name, 0644) + + shutil.move(tf.name, os.path.join(dest, json_file)) + + return True diff --git a/src/product_details/settings_defaults.py b/src/product_details/settings_defaults.py new file mode 100644 index 0000000..ef4d8b3 --- /dev/null +++ b/src/product_details/settings_defaults.py @@ -0,0 +1,13 @@ +import logging +import os + + +# URL to clone product_details JSON files from. +# Include trailing slash. +PROD_DETAILS_URL = 'http://svn.mozilla.org/libs/product-details/json/' + +# Target dir to drop JSON files into (must be writable) +PROD_DETAILS_DIR = os.path.join(os.path.dirname(__file__), 'json') + +# log level. +LOG_LEVEL = logging.INFO diff --git a/src/product_details/version_compare/__init__.py b/src/product_details/version_compare/__init__.py new file mode 100644 index 0000000..40eb9e0 --- /dev/null +++ b/src/product_details/version_compare/__init__.py @@ -0,0 +1,184 @@ +"""Version comparison module for Mozilla-style application versions.""" +import re + +from product_details.version_compare.decorators import memoize +from product_details.version_compare.utils import uniquifier + + +# Regex for parsing well-formed version numbers. +_version_re = re.compile( + r"""(?P\d+) # major (x in x.y) + \.(?P\d+) # minor1 (y in x.y) + \.?(?P\d+)? # minor2 (z in x.y.z) + \.?(?P\d+)? # minor3 (w in x.y.z.w) + (?P[a|b]?) # alpha/beta + (?P\d*) # alpha/beta version + (?P
pre)?       # pre release
+        (?P\d)?    # pre release version""",
+    re.VERBOSE)
+
+
+class Version(object):
+    """An object representing a version."""
+    _version = None
+    _version_int = None
+    _version_dict = None
+
+    def __init__(self, version):
+        """Version constructor."""
+        try:
+            # Parse version.
+            assert version
+            self._version = version
+            self._version_int = version_int(version)
+            assert version_int != 0
+            self._version_dict = version_dict(version)
+
+            # Make parsed data available as properties.
+            for key, val in self._version_dict.items():
+                setattr(self, key, val)
+
+        except AssertionError, e:
+            raise ValueError('Error parsing version: %s' % e)
+
+    def __str__(self):
+        return str(self._version)
+
+    def __cmp__(self, other):
+        """Compare two versions."""
+        assert isinstance(other, Version)
+        return cmp(self._version_int, other._version_int)
+
+    @property
+    def is_beta(self):
+        """
+        Is this a beta version?
+
+        Nightlies, while containing "b2" etc., are not betas.
+        """
+        return self.alpha == 'b' and not self.is_nightly
+
+    @property
+    def is_nightly(self):
+        return self.pre == 'pre'
+
+    @property
+    def is_release(self):
+        return not (self.is_beta or self.is_nightly)
+
+    @property
+    def simplified(self):
+        return simplify_version(self._version)
+
+
+def version_list(releases, key=None, reverse=True, hide_below='0.0',
+                 filter=lambda v: True):
+    """
+    Build a sorted list of simplified versions.
+
+    ``releases`` is expected to be a dictionary like:
+        {'1.0': '2000-01-01'}
+
+    hide_below is the minimum version to be included in the list.
+    filter is a function that maps Version objects to "include? True/False".
+    """
+    if not key:
+        key = lambda x: x[1]  # Default: Sort by release date.
+
+    lowest = Version(hide_below)
+    versions = []
+    for v, released in sorted(releases.items(), key=key, reverse=reverse):
+        ver = Version(v)
+        if ver < lowest or not filter(ver):
+            continue
+        versions.append(ver.simplified)
+    return uniquifier(versions)
+
+
+def dict_from_int(version_int):
+    """Converts a version integer into a dictionary with major/minor/...
+    info."""
+    d = {}
+    rem = version_int
+    (rem, d['pre_ver']) = divmod(rem, 100)
+    (rem, d['pre']) = divmod(rem, 10)
+    (rem, d['alpha_ver']) = divmod(rem, 100)
+    (rem, d['alpha']) = divmod(rem, 10)
+    (rem, d['minor3']) = divmod(rem, 100)
+    (rem, d['minor2']) = divmod(rem, 100)
+    (rem, d['minor1']) = divmod(rem, 100)
+    (rem, d['major']) = divmod(rem, 100)
+    d['pre'] = None if d['pre'] else 'pre'
+    d['alpha'] = {0: 'a', 1: 'b'}.get(d['alpha'])
+
+    return d
+
+
+@memoize
+def version_dict(version):
+    """Turn a version string into a dict with major/minor/... info."""
+    match = _version_re.match(version or '')
+    letters = 'alpha pre'.split()
+    numbers = 'major minor1 minor2 minor3 alpha_ver pre_ver'.split()
+    if match:
+        d = match.groupdict()
+        for letter in letters:
+            d[letter] = d[letter] if d[letter] else None
+        for num in numbers:
+            d[num] = int(d[num]) if d[num] else None
+    else:
+        d = dict((k, None) for k in numbers)
+        d.update((k, None) for k in letters)
+    return d
+
+
+@memoize
+def version_int(version):
+    version_data = version_dict(str(version))
+
+    d = {}
+    for key in ['alpha_ver', 'major', 'minor1', 'minor2', 'minor3',
+                'pre_ver']:
+        d[key] = version_data.get(key) or 0
+    atrans = {'a': 0, 'b': 1}
+    d['alpha'] = atrans.get(version_data['alpha'], 2)
+    d['pre'] = 0 if version_data['pre'] else 1
+
+    v = "%d%02d%02d%02d%d%02d%d%02d" % (d['major'], d['minor1'],
+            d['minor2'], d['minor3'], d['alpha'], d['alpha_ver'], d['pre'],
+            d['pre_ver'])
+    return int(v)
+
+
+def simplify_version(version):
+    """
+    Strips cruft (like build1, which won't show up in a UA string) from a
+    version number by parsing and rebuilding it.
+    """
+    v = dict_from_int(version_int(version))
+    # major and minor1 always exist
+    pieces = [v['major'], v['minor1']]
+    suffixes = []
+
+    # minors 2 and 3 are optional
+    if v['minor2']:
+        pieces.append(v['minor2'])
+    if v['minor3']:
+        pieces.append(v['minor3'])
+
+    # if this is a real beta, attach the version
+    if v['alpha'] and v['alpha_ver']:
+        suffixes += [v['alpha'], v['alpha_ver']]
+
+    # attach pre
+    if v['pre'] and v['alpha_ver']:
+        suffixes.append(v['pre'])
+        if v['pre_ver']:
+            suffixes.append(v['pre_ver'])
+
+    # stringify
+    pieces = map(str, pieces)
+    suffixes = map(str, suffixes)
+
+    # build version number
+    return '.'.join(pieces) + ''.join(suffixes)
diff --git a/src/product_details/version_compare/decorators.py b/src/product_details/version_compare/decorators.py
new file mode 100644
index 0000000..7db39e9
--- /dev/null
+++ b/src/product_details/version_compare/decorators.py
@@ -0,0 +1,21 @@
+import cPickle
+import functools
+
+
+def memoize(fctn):
+    """
+    Memoizing decorator, courtesy of:
+    http://pko.ch/2008/08/22/memoization-in-python-easier-than-what-it-should-be/
+    """
+    memory = {}
+
+    @functools.wraps(fctn)
+    def memo(*args,**kwargs):
+        haxh = cPickle.dumps((args, sorted(kwargs.iteritems())))
+        if haxh not in memory:
+            memory[haxh] = fctn(*args,**kwargs)
+        return memory[haxh]
+
+    if memo.__doc__:
+        memo.__doc__ = "\n".join([memo.__doc__,"This function is memoized."])
+    return memo
diff --git a/src/product_details/version_compare/tests.py b/src/product_details/version_compare/tests.py
new file mode 100644
index 0000000..90b93d1
--- /dev/null
+++ b/src/product_details/version_compare/tests.py
@@ -0,0 +1,107 @@
+"""
+Most of these tests are directly migrated from mozilla-central's reference
+implementation.
+"""
+import copy
+
+from nose.tools import eq_
+
+from product_details.version_compare import (
+    Version, version_list, version_dict, version_int)
+
+
+# Versions to test listed in ascending order, none can be equal.
+# TODO Add support for asterisks.
+COMPARISONS = (
+    "0.9",
+    "0.9.1",
+    "1.0pre1",
+    "1.0pre2",
+    "1.0",
+    "1.1pre",
+    "1.1pre1a",
+    "1.1pre1",
+    "1.1pre10a",
+    "1.1pre10",
+    "1.1",
+    "1.1.0.1",
+    "1.1.1",
+    #"1.1.*",
+    #"1.*",
+    "2.0",
+    "2.1",
+    "3.0.-1",
+    "3.0",
+)
+
+# Every version in this list means the same version number.
+# TODO add support for + signs.
+EQUALITY = (
+  "1.1pre",
+  "1.1pre0",
+  #"1.0+",
+)
+
+
+def test_version_compare():
+    """Test version comparison code, for parity with mozilla-central."""
+    numlist = enumerate(map(lambda v: Version(v), COMPARISONS))
+    for i, v1 in numlist:
+        for j, v2 in numlist:
+            if i < j:
+                assert v1 < v2, '%s is not less than %s' % (v1, v2)
+            elif i > j:
+                assert v1 > v2, '%s is not greater than %s' % (v1, v2)
+            else:
+                eq_(v1, v2)
+
+    equal_vers = map(lambda v: Version(v), EQUALITY)
+    for v1 in equal_vers:
+        for v2 in equal_vers:
+            eq_(v1, v2)
+
+
+def test_simplify_version():
+    """Make sure version simplification works."""
+    versions = {
+        '4.0b1': '4.0b1',
+        '3.6': '3.6',
+        '3.6.4b1': '3.6.4b1',
+        '3.6.4build1': '3.6.4',
+        '3.6.4build17': '3.6.4',
+    }
+    for v in versions:
+        ver = Version(v)
+        eq_(ver.simplified, versions[v])
+
+
+def test_dict_vs_int():
+    """
+    version_dict and _int can use each other's data but must not overwrite
+    it.
+    """
+    version_string = '4.0b8pre'
+    dict1 = copy.copy(version_dict(version_string))
+    int1 = version_int(version_string)
+    dict2 = version_dict(version_string)
+    int2 = version_int(version_string)
+    eq_(dict1, dict2)
+    eq_(int1, int2)
+
+
+def test_version_list():
+    """Test if version lists are generated properly."""
+    my_versions = {
+        '4.0b2build8': '2010-12-06',
+        '3.0': '2010-12-01',
+        '4.0b1': '2010-11-24',
+        '4.0b2build7': '2010-12-05',
+    }
+    expected = ('4.0b2', '4.0b1')
+
+    test_list = version_list(my_versions, hide_below='4.0b1')
+
+    # Check if the generated version list is the same as we expect.
+    eq_(len(expected), len(test_list))
+    for n, v in enumerate(test_list):
+        eq_(v, expected[n])
diff --git a/src/product_details/version_compare/utils.py b/src/product_details/version_compare/utils.py
new file mode 100644
index 0000000..f8e1d4c
--- /dev/null
+++ b/src/product_details/version_compare/utils.py
@@ -0,0 +1,17 @@
+def uniquifier(seq, key=None):
+    """
+    Make a unique list from a sequence. Optional key argument is a callable
+    that transforms an item to its key.
+
+    Borrowed in part from http://www.peterbe.com/plog/uniqifiers-benchmark
+    """
+    if key is None:
+        key = lambda x: x
+    def finder(seq):
+        seen = {}
+        for item in seq:
+            marker = key(item)
+            if marker not in seen:
+                seen[marker] = True
+                yield item
+    return list(finder(seq))