Merge branch 'release/0.33.1'

CHANGES.rst

@@ -2,6 +2,13 @@
 Changes
 -------
 
+0.33.1
+~~~~~~
+
+- bugfix: BibTeX files containing @ within field values could not get parsed correctly.
+- feature: The functionality to add google books info gained a cleanup command.
+
+
 0.33
 ~~~~
 

clld/__init__.py

@@ -5,7 +5,7 @@
 from clld import interfaces
 
 
-__version__ = "0.33"
+__version__ = "0.33.1"
 
 
 class Resource(namedtuple('Resource', 'name model interface with_index with_rdfdump')):

clld/lib/bibtex.py

@@ -554,8 +554,8 @@ def from_file(cls, bibFile, encoding='utf8', lowercase=False):
         else:
             content = ''
 
-        return cls((Record.from_string(m.group(), lowercase=lowercase)
-                    for m in re.finditer('@[^@]*', content)))
+        return cls((Record.from_string('@' + m, lowercase=lowercase)
+                    for m in re.split('^\s*@', content, 0, re.MULTILINE)))
 
     def __len__(self):
         return len(self.records)

clld/scripts/freeze.py

@@ -333,7 +333,7 @@ def unfreeze_func(args, engine=None):
     engine = engine or DBSession.get_bind()
     data_dir = path(mkdtemp())
 
-    with ZipFile(args.data_file('..', 'data.zip')) as fp:
+    with ZipFile(args.module_dir.joinpath('data.zip')) as fp:
         fp.extractall(data_dir)
 
     for table in Base.metadata.sorted_tables:

clld/scripts/util.py

@@ -5,7 +5,6 @@
 from distutils.util import strtobool
 from collections import defaultdict
 import argparse
-import json
 import logging
 from functools import partial
 
@@ -22,7 +21,7 @@
 from clld.db.meta import VersionedDBSession, DBSession, Base
 from clld.db.models import common
 from clld.db.util import page_query
-from clld.util import slug
+from clld.util import slug, jsonload, jsondump
 from clld.interfaces import IDownload
 from clld.lib import bibtex
 
@@ -251,7 +250,7 @@ def create_downloads(**kw):  # pragma: no cover
 
 def gbs(**kw):  # pragma: no cover
     add_args = [
-        (("command",), dict(help="download|verify|update")),
+        (("command",), dict(help="download|verify|update|cleanup")),
         (("--api-key",), dict(default=kw.get('key', os.environ.get('GBS_API_KEY')))),
     ]
 
@@ -271,6 +270,16 @@ def words(s):
     count = 0
     api_url = "https://www.googleapis.com/books/v1/volumes?"
 
+    if command == 'cleanup':
+        for fname in args.data_file('gbs').files('*.json'):
+            try:
+                data = jsonload(fname)
+                if data.get('totalItems') == 0:
+                    os.remove(fname)
+            except ValueError:
+                os.remove(fname)
+        return
+
     if not sources:
         sources = DBSession.query(common.Source)\
             .order_by(common.Source.id)\
@@ -287,12 +296,11 @@ def words(s):
 
         if command in ['verify', 'update']:
             if filepath.exists():
-                with open(filepath) as fp:
-                    try:
-                        data = json.load(fp)
-                    except ValueError:
-                        log.warn('no JSON object found in: %s' % filepath)
-                        continue
+                try:
+                    data = jsonload(filepath)
+                except ValueError:
+                    log.warn('no JSON object found in: %s' % filepath)
+                    continue
                 if not data['totalItems']:
                     continue
                 item = data['items'][0]
@@ -330,8 +338,7 @@ def words(s):
                 log.info(source.publisher)
                 if not confirm('Are the records the same?'):
                     log.warn('---- removing ----')
-                    with open(filepath, 'w') as fp:
-                        json.dump({"totalItems": 0}, fp)
+                    jsondump({"totalItems": 0}, filepath)
         elif command == 'update':
             source.google_book_search_id = item['id']
             source.update_jsondata(gbs=item)

clld/tests/test.bib

@@ -1,4 +1,4 @@
 @article{id,
-    title = {the title},
+    title = {the title with an @book},
     year = {1234}
 }

clld/tests/test_lib_bibtex.py

@@ -142,5 +142,6 @@ def test_Database(self):
         self.assertEqual(len(db), 0)
         db = Database.from_file(TESTS_DIR.joinpath('test.bib'))
         self.assertEqual(len(db), 1)
+        assert '@' in db[0]['title']
         assert [r for r in db]
         self.assertRaises(NotImplementedError, db.format, 'txt')

clld/tests/test_scripts_freeze.py

@@ -22,6 +22,7 @@ def test_freeze(self):
 
         class Args(object):
             env = self.env
+            module_dir = tmp
 
             def data_file(self, *comps):
                 return tmp.joinpath('data', *comps)

docs/conf.py

@@ -51,10 +51,10 @@
 # built documents.
 #
 # The full version, including alpha/beta/rc tags.
-release = '0.33'
+release = '0.33.1'
 
 # The short X.Y version.
-version = '0.33'
+version = '0.33.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

setup.py

@@ -81,7 +81,7 @@
 ]
 
 setup(name='clld',
-      version='0.33',
+      version='0.33.1',
       description=(
           'Python library supporting the development of cross-linguistic databases'),
       long_description=README + '\n\n' + CHANGES,

tools/unfreeze.sh

@@ -1,5 +1,12 @@
 #!/bin/bash
-cd ~/venvs
+#
+# get zip-URL from DOI:
+# http://dx.doi.org/10.5281/zenodo.19782 ->  http://zenodo.org/record/19782
+# curl http://zenodo.org/record/19782 | grep "\.zip"
+#
+# ./tools/unfreeze.sh csd "https://zenodo.org/record/19782/files/csd-v1.0.zip"
+#
+cd /tmp
 virtualenv --no-site-packages $1
 cd $1
 . bin/activate
@@ -26,4 +33,3 @@ pip install nosexcover
 pip install mock
 cp sqlite.ini development.ini
 nosetests
-