From ba3306a787c50679299ce52b4068b0c6683591b1 Mon Sep 17 00:00:00 2001 From: Simon Conseil Date: Sat, 30 Aug 2014 15:50:10 +0200 Subject: [PATCH 1/9] Prepare next version. --- docs/changelog.rst | 5 +++++ sigal/pkgmeta.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6c8d3234..6f396d23 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,11 @@ Changelog =========== +Version 0.9.dev +~~~~~~~~~~~~~~~ + +Released on 2014-xx-xx. + Version 0.8.0 ~~~~~~~~~~~~~ diff --git a/sigal/pkgmeta.py b/sigal/pkgmeta.py index cce04084..657745d5 100644 --- a/sigal/pkgmeta.py +++ b/sigal/pkgmeta.py @@ -22,7 +22,7 @@ __title__ = 'sigal' __author__ = 'Simon Conseil' -__version__ = '0.8.0' +__version__ = '0.9.0-dev' __license__ = 'MIT' __url__ = 'https://github.com/saimn/sigal' __all__ = ['__title__', '__author__', '__version__', '__license__', '__url__'] From 15091deeb55a4dc3da3ccbc6703c2c8de27c67e3 Mon Sep 17 00:00:00 2001 From: Simon Conseil Date: Thu, 4 Sep 2014 22:33:19 +0200 Subject: [PATCH 2/9] Catch cPickle error and add a message about serialization error with the settings file. --- sigal/gallery.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sigal/gallery.py b/sigal/gallery.py index 948015dd..136b07fe 100644 --- a/sigal/gallery.py +++ b/sigal/gallery.py @@ -23,6 +23,7 @@ from __future__ import absolute_import, print_function +import cPickle import fnmatch import logging import multiprocessing @@ -540,6 +541,13 @@ def build(self, force=False): except KeyboardInterrupt: self.pool.terminate() sys.exit('Interrupted') + except cPickle.PicklingError: + self.logger.critical( + "Failed to process files with the multiprocessing feature." + " This can be caused by some module import or object " + "defined in the settings file, which can't be serialized.", + exc_info=True) + sys.exit('Abort') print('') From 52ff2b947dc26b906af4d8a547efa0ad1618d8a9 Mon Sep 17 00:00:00 2001 From: Simon Conseil Date: Thu, 4 Sep 2014 22:49:01 +0200 Subject: [PATCH 3/9] Add warnings about imports in the settings file, to avoid breaking the multiprocessing feature. --- docs/plugins.rst | 17 +++++++++++++---- sigal/templates/sigal.conf.py | 6 ++++++ tests/sample/sigal.conf.py | 3 +-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 95b7c128..06d6b2bf 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -7,15 +7,24 @@ How to use plugins Plugins must be specified with the ``plugins`` setting: +.. code-block:: python + + plugins = ['sigal.plugins.adjust', 'sigal.plugins.copyright'] + +You can either specify the name of the module which contains the plugin, or +import the plugin before adding it to the list: + .. code-block:: python from sigal.plugins import copyright plugins = ['sigal.plugins.adjust', copyright] -You can either specify the name of the module which contains the plugin, or -import the plugin before adding it to the list. The ``plugin_paths`` setting -can be used to specify paths to search for plugins (if they are not in the -python path). +.. note:: Using an import like this will break the multiprocessing feature, + because the settings dict must be serializable. So in most cases you should + prefer the first option. + +The ``plugin_paths`` setting can be used to specify paths to search for plugins +(if they are not in the python path). Write a new plugin ------------------ diff --git a/sigal/templates/sigal.conf.py b/sigal/templates/sigal.conf.py index 26dd48c0..0d27dff8 100644 --- a/sigal/templates/sigal.conf.py +++ b/sigal/templates/sigal.conf.py @@ -138,6 +138,12 @@ # Plugins # -------- +# List of plugins to use. The values must be a path than can be imported. +# Another option is to import the plugin and put the module in the list, but +# this will break with the multiprocessing feature (the settings dict obtained +# from this file must be serializable). +# plugins = ['sigal.plugins.adjust', 'sigal.plugins.copyright'] + # Add a copyright text on the image (default: '') # copyright = "© An example copyright message" diff --git a/tests/sample/sigal.conf.py b/tests/sample/sigal.conf.py index ec6a44a0..e693f83e 100644 --- a/tests/sample/sigal.conf.py +++ b/tests/sample/sigal.conf.py @@ -8,8 +8,7 @@ links = [('Example link', 'http://example.org'), ('Another link', 'http://example.org')] -from sigal.plugins import copyright -plugins = ['sigal.plugins.adjust', copyright] +plugins = ['sigal.plugins.adjust', 'sigal.plugins.copyright'] copyright = u"© An example copyright message" adjust_options = {'color': 0.0, 'brightness': 1.0, 'contrast': 1.0, 'sharpness': 0.0} From 2d1be134e7d8eb7162c4887d38d02231b99cd21d Mon Sep 17 00:00:00 2001 From: Simon Conseil Date: Thu, 4 Sep 2014 23:15:45 +0200 Subject: [PATCH 4/9] cPickle no more exists on py3. --- sigal/gallery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sigal/gallery.py b/sigal/gallery.py index 136b07fe..eb584d0a 100644 --- a/sigal/gallery.py +++ b/sigal/gallery.py @@ -23,11 +23,11 @@ from __future__ import absolute_import, print_function -import cPickle import fnmatch import logging import multiprocessing import os +import pickle import sys import zipfile @@ -541,7 +541,7 @@ def build(self, force=False): except KeyboardInterrupt: self.pool.terminate() sys.exit('Interrupted') - except cPickle.PicklingError: + except pickle.PicklingError: self.logger.critical( "Failed to process files with the multiprocessing feature." " This can be caused by some module import or object " From 9e92415a5046ecd3624ad155ba6b69d076eec080 Mon Sep 17 00:00:00 2001 From: Vikram Shirgur Date: Mon, 1 Sep 2014 00:37:33 -0700 Subject: [PATCH 5/9] Allow destination to be read from option sepcified in config file. Write error to log when destination doesn't exist and config is not available. Add self to AUTHORS. Remove logging from serve command due to issues with running test_cli. Add test-case for 'serve' command. Add docs for the serve command. --- AUTHORS | 1 + docs/getting_started.rst | 21 +++++++++++++++++- sigal/__init__.py | 47 +++++++++++++++++++++++++--------------- tests/test_cli.py | 14 ++++++++++++ 4 files changed, 64 insertions(+), 19 deletions(-) diff --git a/AUTHORS b/AUTHORS index 14a185f5..60b9c420 100644 --- a/AUTHORS +++ b/AUTHORS @@ -3,4 +3,5 @@ alphabetical order): - Christophe-Marie Duquesne - Matthias Vogelgesang +- Vikram Shirgur - Yuce Tekol diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 0c655564..83d51c40 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -34,7 +34,7 @@ Serve ``index.html`` to the urls to allow browsing without a server. -Help of the ``sigal build`` command +Help on the ``sigal build`` command ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :: @@ -73,3 +73,22 @@ Optional arguments: ``-n NCPU, --ncpu NCPU`` Number of cpu to use (default: all) + +Help on the ``sigal serve`` command +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + $ sigal serve [-c CONFIG] [-p PORT] [destination] + +Optional arguments: + +``-c CONFIG, --config CONFIG`` + Configuration file (default: ``sigal.conf.py`` in the current working + directory) + +``-p PORT, --port PORT`` + Port number to start the server on (default: 8000) + +``destination`` + Destination directory where the output of build is located (default: _build) diff --git a/sigal/__init__.py b/sigal/__init__.py index b9e8e5fe..03763d03 100644 --- a/sigal/__init__.py +++ b/sigal/__init__.py @@ -169,24 +169,35 @@ def init_plugins(settings): @main.command() -@argument('path', default='_build') +@argument('destination', default='_build') @option('-p', '--port', help="Port to use", default=8000) -def serve(path, port): +@option('-c', '--config', default=_DEFAULT_CONFIG_FILE, show_default=True, + help='Configuration file') +def serve(destination, port, config): """Run a simple web server.""" - - if os.path.exists(path): - os.chdir(path) - Handler = server.SimpleHTTPRequestHandler - httpd = socketserver.TCPServer(("", port), Handler, False) - print(" * Running on http://127.0.0.1:{}/".format(port)) - - try: - httpd.allow_reuse_address = True - httpd.server_bind() - httpd.server_activate() - httpd.serve_forever() - except KeyboardInterrupt: - print('\nAll done!') + if os.path.exists(destination): + pass + elif os.path.exists(config): + settings = read_settings(config) + destination = settings.get('destination') + if not os.path.exists(destination): + sys.stderr.write("The '{}' directory doesn't exist, maybe try building first?\n".format(destination)) + sys.exit(1) else: - sys.stderr.write("The '%s' directory doesn't exist.\n" % path) - sys.exit(1) + sys.stderr.write("The {destination} directory doesn't exist and the config file ({config}) could not be read.".format(destination=destination, config=config)) + sys.exit(2) + + print('DESTINATION : {}'.format(destination)) + os.chdir(destination) + Handler = server.SimpleHTTPRequestHandler + httpd = socketserver.TCPServer(("", port), Handler, False) + print(" * Running on http://127.0.0.1:{}/".format(port)) + + try: + httpd.allow_reuse_address = True + httpd.server_bind() + httpd.server_activate() + httpd.serve_forever() + except KeyboardInterrupt: + print('\nAll done!') + diff --git a/tests/test_cli.py b/tests/test_cli.py index a6862804..86f0912f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,6 +4,7 @@ from click.testing import CliRunner from sigal import init +from sigal import serve def test_init(tmpdir): @@ -18,3 +19,16 @@ def test_init(tmpdir): assert result.exit_code == 1 assert result.output == ("Found an existing config file, will abort to " "keep it safe.\n") + +def test_serve(tmpdir): + config_file = str(tmpdir.join('sigal.conf.py')) + runner = CliRunner() + result = runner.invoke(init, [config_file]) + assert result.exit_code == 0 + + result = runner.invoke(serve) + assert result.exit_code == 2 + + result = runner.invoke(serve, ['-c', config_file]) + assert result.exit_code == 1 + From d90091071b8912fca0821e9c1fc73acd9a739b31 Mon Sep 17 00:00:00 2001 From: Vikram Shirgur Date: Mon, 1 Sep 2014 18:48:48 -0700 Subject: [PATCH 6/9] Truncate long lines to < 80 chars. --- sigal/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sigal/__init__.py b/sigal/__init__.py index 03763d03..61bdaa68 100644 --- a/sigal/__init__.py +++ b/sigal/__init__.py @@ -171,8 +171,8 @@ def init_plugins(settings): @main.command() @argument('destination', default='_build') @option('-p', '--port', help="Port to use", default=8000) -@option('-c', '--config', default=_DEFAULT_CONFIG_FILE, show_default=True, - help='Configuration file') +@option('-c', '--config', default=_DEFAULT_CONFIG_FILE, + show_default=True, help='Configuration file') def serve(destination, port, config): """Run a simple web server.""" if os.path.exists(destination): @@ -181,10 +181,15 @@ def serve(destination, port, config): settings = read_settings(config) destination = settings.get('destination') if not os.path.exists(destination): - sys.stderr.write("The '{}' directory doesn't exist, maybe try building first?\n".format(destination)) + sys.stderr.write("The '{}' directory doesn't exist, " + "maybe try building first?" + "\n".format(destination)) sys.exit(1) else: - sys.stderr.write("The {destination} directory doesn't exist and the config file ({config}) could not be read.".format(destination=destination, config=config)) + sys.stderr.write("The {destination} directory doesn't exist " + "and the config file ({config}) could not be " + "read." + "\n".format(destination=destination, config=config)) sys.exit(2) print('DESTINATION : {}'.format(destination)) From fa7f9d9528c0a0a5a287daeeddda864d3167ba4e Mon Sep 17 00:00:00 2001 From: Simon Conseil Date: Sun, 7 Sep 2014 23:09:53 +0200 Subject: [PATCH 7/9] Check that the file exists before removing. (fix #110) --- sigal/video.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sigal/video.py b/sigal/video.py index 65ecd4e8..166f8b8e 100644 --- a/sigal/video.py +++ b/sigal/video.py @@ -44,7 +44,8 @@ def check_subprocess(cmd, source, outname): returncode, stdout, stderr = call_subprocess(cmd) except KeyboardInterrupt: logger.debug('Process terminated, removing file %s', outname) - os.remove(outname) + if os.path.isfile(outname): + os.remove(outname) raise if returncode: @@ -52,7 +53,8 @@ def check_subprocess(cmd, source, outname): logger.debug('STDOUT:\n %s', stdout) logger.debug('STDERR:\n %s', stderr) logger.debug('Process failed, removing file %s', outname) - os.remove(outname) + if os.path.isfile(outname): + os.remove(outname) def video_size(source): From 038b10d77286dd9a0b18923e181ee9b893128ce9 Mon Sep 17 00:00:00 2001 From: Simon Conseil Date: Sun, 7 Sep 2014 23:16:09 +0200 Subject: [PATCH 8/9] Use the correct filename for original videos (fix #111). --- sigal/gallery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sigal/gallery.py b/sigal/gallery.py index eb584d0a..a8b15d47 100644 --- a/sigal/gallery.py +++ b/sigal/gallery.py @@ -567,7 +567,7 @@ def process_dir(self, album, force=False): self.stats[f.type + '_skipped'] += 1 else: if self.settings['keep_orig']: - copy(f.src_path, join(album.orig_path, f.filename), + copy(f.src_path, join(album.orig_path, f.src_filename), symlink=self.settings['orig_link']) self.stats[f.type] += 1 From 4c3a35c66fab228a54d7a97219c40b375d8f95e2 Mon Sep 17 00:00:00 2001 From: Simon Conseil Date: Sun, 7 Sep 2014 23:58:39 +0200 Subject: [PATCH 9/9] Change where the original copy is done and add a test for this. --- sigal/gallery.py | 19 +++++++------------ tests/test_gallery.py | 11 +++++++---- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/sigal/gallery.py b/sigal/gallery.py index a8b15d47..b78a1aff 100644 --- a/sigal/gallery.py +++ b/sigal/gallery.py @@ -92,10 +92,15 @@ def __unicode__(self): @property def big(self): """Path to the original image, if ``keep_orig`` is set (relative to the - album directory). + album directory). Copy the file if needed. """ if self.settings['keep_orig']: - return os.path.join(self.settings['orig_dir'], self.src_filename) + s = self.settings + orig_path = join(s['destination'], self.path, s['orig_dir']) + check_or_create_dir(orig_path) + copy(self.src_path, join(orig_path, self.src_filename), + symlink=s['orig_link']) + return url_from_path(join(s['orig_dir'], self.src_filename)) else: return None @@ -185,7 +190,6 @@ def __init__(self, path, settings, dirnames, filenames, gallery): self.name = path.split(os.path.sep)[-1] self.gallery = gallery self.settings = settings - self.orig_path = None self._thumbnail = None if path == '.': @@ -278,10 +282,6 @@ def create_output_directories(self): check_or_create_dir(join(self.dst_path, self.settings['thumb_dir'])) - if self.medias and self.settings['keep_orig']: - self.orig_path = join(self.dst_path, self.settings['orig_dir']) - check_or_create_dir(self.orig_path) - @property def images(self): """List of images (:class:`~sigal.gallery.Image`).""" @@ -560,16 +560,11 @@ def build(self, force=False): def process_dir(self, album, force=False): """Process a list of images in a directory.""" - for f in album: if isfile(f.dst_path) and not force: self.logger.info("%s exists - skipping", f.filename) self.stats[f.type + '_skipped'] += 1 else: - if self.settings['keep_orig']: - copy(f.src_path, join(album.orig_path, f.src_filename), - symlink=self.settings['orig_link']) - self.stats[f.type] += 1 yield f.type, f.src_path, album.dst_path, self.settings diff --git a/tests/test_gallery.py b/tests/test_gallery.py index 71d3df79..3fe3e11b 100644 --- a/tests/test_gallery.py +++ b/tests/test_gallery.py @@ -78,18 +78,21 @@ def test_media(settings): assert str(m) == file_path -def test_media_orig(settings): +def test_media_orig(settings, tmpdir): settings['keep_orig'] = False m = Media('11.jpg', 'dir1/test1', settings) assert m.big is None settings['keep_orig'] = True + settings['destination'] = str(tmpdir) + m = Image('11.jpg', 'dir1/test1', settings) assert m.big == 'original/11.jpg' - m = Video('file.ogv', 'video', settings) - assert m.filename == 'file.webm' - assert m.big == 'original/file.ogv' + m = Video('stallman software-freedom-day-low.ogv', 'video', settings) + assert m.filename == 'stallman software-freedom-day-low.webm' + assert m.big == 'original/stallman software-freedom-day-low.ogv' + assert os.path.isfile(join(settings['destination'], m.path, m.big)) def test_image(settings, tmpdir):