Merge pull request #14 from sciunto-org/newdb

Switch to a more versatile db with json

.gitignore

@@ -1,7 +1,7 @@
 # Project
 .backups
 MANIFEST
-example/db.db
+example/db.json
 
 
 # Byte-compiled / optimized / DLL files

CHANGELOG

@@ -2,6 +2,10 @@
 =================
 
 * add a tool to extract filepath from includegraphics in tex files
+* add a tool to automake upon changes in src
+* switch to a json database
+* add a function make_pdf to build pdf only
+* exports happen only if necessary
 
 0.1.2  2016/01/10
 =================

example/Makefile

@@ -5,6 +5,9 @@ POACONF=po4a.cfg
 all:
 	python scifig.py -d $(DIR)
 
+pdf:
+	python scifig.py --pdf -d $(DIR)
+
 i18n:
 	$(POA) -k0 $(POACONF)
 

example/automake.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Author: Francois Boulogne
+# License:
+
+import subprocess
+import os
+import time
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+
+class ChangeHandler(FileSystemEventHandler):
+    """React to modified source files."""
+    def on_modified(self, event):
+        extensions = ('.plt', '.tikz', '.tikzsnippet')
+        if os.path.splitext(event.src_path)[-1].lower() in extensions:
+            subprocess.call("make pdf", shell=True)
+
+
+def main():
+    handler = ChangeHandler()
+    observer = Observer()
+    observer.schedule(handler, 'src', recursive=True)
+    observer.start()
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        observer.stop()
+    observer.join()
+
+
+if __name__ == '__main__':
+    main()

example/scifig.py

@@ -8,7 +8,7 @@
 import shutil
 import argparse
 
-from libscifig import detector
+from libscifig import detector, database
 
 
 def list_figdirs(src='src'):
@@ -30,35 +30,39 @@ def clean_up(path):
     """
     Clean up all compiled files.
     """
+    db = os.path.join(path, 'db.json')
+    logging.debug('Clean up %s' % db)
+    try:
+        os.remove(db)
+    except FileNotFoundError:
+        pass
     build = os.path.join(path, 'build')
-    db = os.path.join(path, 'db.db')
     logging.debug('Clean up %s' % build)
     if os.path.exists(build):
         shutil.rmtree(build)
-    from libscifig.database import erase_db
-    erase_db(db)
 
 
-def main(workingdir, dest='/tmp'):
+def main(workingdir, dest='/tmp', pdf_only=False):
     make_build_dir(os.path.join(workingdir, 'build'))
     tasks = []
     for directory in list_figdirs(os.path.join(workingdir, 'src')):
         tasks.extend(detector.detect_task(directory, workingdir))
 
-    for task in tasks:
-        if task.check():
-            logging.info('Build %s' % task.get_name())
-            task.make()
-            logging.info('Export %s' % task.get_name())
-            task.export(dst=dest)
-        else:
-            logging.info('Nothing to do for %s' % task.get_name())
+    db_path = os.path.join(workingdir, 'db.json')
+    with database.DataBase(db_path) as db:
+        for task in tasks:
+            if pdf_only:
+                task.make_pdf(db)
+            else:
+                task.make(db)
+            task.export(db, dst=dest)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='', epilog='')
-    #parser.add_argument('conf', help='Configuration file', metavar='CONF')
     parser.add_argument('-c', '--clean', action='store_true',
                         default=False, help='Clean')
+    parser.add_argument('--pdf', action='store_true',
+                        default=False, help='PDF only')
     parser.add_argument('-d', '--dest', metavar='DEST',
                         default='/tmp', help='destination')
     parser.add_argument('-w', '--workingdir', metavar='WORKINGDIR',
@@ -90,9 +94,10 @@ def main(workingdir, dest='/tmp'):
     steam_handler.setFormatter(color_formatter)
     logger.addHandler(steam_handler)
 
-    #logger.debug(args)
     if args.clean:
         logger.info('Cleaning...')
         clean_up(args.workingdir)
+    elif args.pdf:
+        main(args.workingdir, args.dest, pdf_only=True)
     else:
-        main(args.workingdir, args.dest)
+        main(args.workingdir, args.dest, pdf_only=False)

libscifig/checksum.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Author: Francois Boulogne
+# License:
+
+import hashlib
+import logging
+
+
+def calculate_checksum(filepath):
+    """
+    Calculate the checksum of a file.
+
+    :param filepath: file path
+    :returns: string
+    """
+    hasher = hashlib.md5()
+    with open(filepath, 'rb') as afile:
+        buf = afile.read()
+        hasher.update(buf)
+    return hasher.hexdigest()
+
+
+def is_different(cur_hashes, db_hashes):
+    """
+    Check if at least one item changed.
+
+    :param cur_hashes: current hashes
+    :param db_hashes: previous hashes
+
+    :returns: boolean
+    """
+    for dep, md5 in cur_hashes.items():
+        try:
+            value = db_hashes[dep]
+        except KeyError:
+            return True
+        if value is None or value != md5:
+            logging.debug('value of %s is None or does not match, modif is True', dep)
+            return True
+    return False

libscifig/database.py

@@ -1,82 +1,54 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+# Author: Francois Boulogne
+# License:
 
+import json
 import logging
-import os.path
-import hashlib
-try:
-    import cPickle as pickle
-except:
-    import pickle
 
-def _calculate_checksum(filepath):
-    hasher = hashlib.md5()
-    with open(filepath, 'rb') as afile:
-        buf = afile.read()
-        hasher.update(buf)
-    return hasher.hexdigest()
-
-
-def check_modification(name, dependencies, db_path):
-    """
-    Check if at least one dependency changed.
-
-    :param name: name of the figure
-    :param dependencies: list of dependencies
-    :param db_path: path of the database
-    :returns: boolean
-    """
-    logging.debug('Check modification for %s', name)
-    if not os.path.isfile(db_path):
-        logging.debug('No db, modif is True')
-        return True
-    cur_signature = {}
-    for dep in dependencies:
-        cur_signature[dep] = _calculate_checksum(dep)
-    with open(db_path, 'rb') as fh:
-        db = pickle.load(fh)
-        db = db.get(name)
-        if db is None:
-            logging.debug('name unknown in db, modif is True')
-            return True
-        for dep, md5 in cur_signature.items():
-            value = db.get(dep)
-            if value is None or value != md5:
-                logging.debug('value of %s is None or does not match, modif is True', dep)
-                return True
-    return False
-
-
-def store_checksum(name, dependencies, db_path):
-    """
-    Store the checksum in the db.
-
-    :param name: name of the figure
-    :param dependencies: list of dependencies
-    :param db_path: path of the database
-    """
-    logging.debug('Store checksums in db')
-    # Calculate md5 sums
-    cur_signature = {}
-    for dep in dependencies:
-        cur_signature[dep] = _calculate_checksum(dep)
-    try:
-        with open(db_path, 'rb') as fh:
-            db = pickle.load(fh)
-    except FileNotFoundError:
-        db = {}
-    # Merge dict
-    db[name] = cur_signature
-    with open(db_path, 'wb') as fh:
-        pickle.dump(db, fh)
-
-
-def erase_db(db_path):
+class DataBase():
     """
-    Erase a database.
+    Custom class to manipulate a json database.
 
-    :param db_path: path of the database
+    :param db_path: filepath of the database.
     """
-    logging.debug('Erase db')
-    with open(db_path, 'wb') as fh:
-        pickle.dump({}, fh)
+    def __init__(self, db_path):
+        self.path = db_path
+
+    def __enter__(self):
+        try:
+            with open(self.path, 'r') as f:
+                self.data = json.load(f)
+        except FileNotFoundError:
+            self.data = {}
+        return self
+
+    def __exit__(self, type, value, traceback):
+        with open(self.path, 'w') as f:
+            json.dump(self.data, f)
+
+    def set(self, name, obj, content):
+        """
+        Set a content to a tree (name--object).
+
+        :param name: ID of the element, like filepath
+        :param obj: Content type (like deps, targets...)
+        :param content: Content to store, a dict.
+        """
+        if not name in self.data.keys():
+            self.data[name] = {obj: content}
+        else:
+            self.data[name][obj] = content
+
+    def get(self, name, obj):
+        """
+        Get a content from a tree (name--object).
+
+        :param name: ID of the element, like filepath
+        :param obj: Content type (like deps, targets...)
+        """
+        try:
+            d = self.data[name][obj]
+        except KeyError:
+            d = {}
+        return d

libscifig/detector.py

@@ -64,7 +64,6 @@ def detect_task(directory, root_path):
     """
     plt_files = glob.glob(os.path.join(directory, '*.plt'))
     tikz_files = glob.glob(os.path.join(directory, '*.tikz'))
-    db_path = os.path.join(root_path, 'db.db')
     tasks = []
     for plt_file in plt_files:
         data = detect_datafile(plt_file, root_path)
@@ -74,12 +73,10 @@ def detect_task(directory, root_path):
                                  tikzsnippet=snippet,
                                  tikzsnippet1=snippet1,
                                  tikzsnippet2=snippet2,
-                                 db=db_path,
                                  ))
     for tikz_file in tikz_files:
         data = detect_datafile(tikz_file, root_path)
         tasks.append(TikzTask(tikz_file,
                               datafiles=data,
-                              db=db_path,
                               ))
     return tasks