Skip to content
Permalink
Browse files

merged master

  • Loading branch information
ralsina committed Jan 29, 2016
2 parents 304b9e5 + a25b5ea commit 82fb86d64d46e9e56fa662582368ae596249416c
Showing with 144 additions and 12 deletions.
  1. +2 −1 CHANGES.txt
  2. +26 −0 docs/extending.txt
  3. +7 −0 nikola/nikola.py
  4. +26 −11 nikola/plugins/command/deploy.py
  5. +83 −0 nikola/state.py
@@ -10,7 +10,8 @@ Features
if ``GITHUB_COMMIT_SOURCE`` is set to True (Issue #2186)
* Hugo-like shortcodes (Issue #1707)
* New Galician translation
* New PRESERVE_EXIF_DATA to copy EXIF when resizing images (Issue #2204)
* New PRESERVE_EXIF_DATA option to copy EXIF when resizing images (Issue #2204)
* New facilities for data persistence and data caching (Issues #2209 and #2009)

Bugfixes
--------
@@ -592,3 +592,29 @@ So, for example::
Will cause a call like this::

foo_handler("bar", "beep", baz="bat", data="Some text", site=whatever)

State and Cache
===============

Sometimes your plugins will need to cache things to speed up further actions. Here are the conventions for that:

* If it's a file, put it somewhere in ```self.site.config['CACHE_FOLDER']``` (defaults to ```cache/```.
* If it's a value, use ```self.site.cache.set(key, value)``` to set it and ```self.site.cache.get(key)``` to get it.
The key should be a string, the value should be json-encodable (so, be careful with datetime objects)

The values and files you store there can **and will** be deleted sometimes by the user. They should always be
things you can reconstruct without lossage. They are throwaways.

On the other hand, sometimes you want to save something that is **not** a throwaway. These are things that may
change the output, so the user should not delete them. We call that **state**. To save state:

* If it's a file, put it somewhere in the working directory. Try not to do that please.
* If it's a value, use ```self.site.state.set(key, value)``` to set it and ```self.state.cache.get(key)``` to get it.
The key should be a string, the value should be json-encodable (so, be careful with datetime objects)

The ```cache``` and ```state``` objects are rather simplistic, and that's intentional. They have no default values: if
the key is not there, you will get ```None``` and like it. They are meant to be both threadsafe, but hey, who can
guarantee that sort of thing?

There are no sections, and no access protection, so let's not use it to store passwords and such. Use responsibly.

@@ -57,6 +57,7 @@
from blinker import signal

from .post import Post # NOQA
from .state import Persistor
from . import DEBUG, utils, shortcodes
from .plugin_categories import (
Command,
@@ -814,6 +815,12 @@ def __init__(self, **config):

self._set_global_context()

# Set persistent state facility
self.state = Persistor(os.path.join('state_data.json'))

# Set cache facility
self.cache = Persistor(os.path.join(self.config['CACHE_FOLDER'], 'cache_data.json'))

def init_plugins(self, commands_only=False):
"""Load plugins as needed."""
self.plugin_manager = PluginManager(categories_filter={
@@ -30,14 +30,15 @@
import io
from datetime import datetime
from dateutil.tz import gettz
import dateutil
import os
import subprocess
import time

from blinker import signal

from nikola.plugin_categories import Command
from nikola.utils import get_logger, remove_file, unicode_str, makedirs, STDERR_HANDLER
from nikola.utils import get_logger, remove_file, makedirs, STDERR_HANDLER


class CommandDeploy(Command):
@@ -55,6 +56,29 @@ def _execute(self, command, args):
self.logger = get_logger('deploy', STDERR_HANDLER)
# Get last successful deploy date
timestamp_path = os.path.join(self.site.config['CACHE_FOLDER'], 'lastdeploy')

# Get last-deploy from persistent state
last_deploy = self.site.state.get('last_deploy')
if last_deploy is None:
# If there is a last-deploy saved, move it to the new state persistence thing
# FIXME: remove in Nikola 8
if os.path.isfile(timestamp_path):
try:
with io.open(timestamp_path, 'r', encoding='utf8') as inf:
last_deploy = dateutil.parser.parse(inf.read())
clean = False
except (IOError, Exception) as e:
self.logger.debug("Problem when reading `{0}`: {1}".format(timestamp_path, e))
last_deploy = datetime(1970, 1, 1)
clean = True
os.unlink(timestamp_path) # Remove because from now on it's in state
else: # Just a default
last_deploy = datetime(1970, 1, 1)
clean = True
else:
last_deploy = dateutil.parser.parse(last_deploy)
clean = False

if self.site.config['COMMENT_SYSTEM_ID'] == 'nikolademo':
self.logger.warn("\nWARNING WARNING WARNING WARNING\n"
"You are deploying using the nikolademo Disqus account.\n"
@@ -102,22 +126,13 @@ def _execute(self, command, args):
return e.returncode

self.logger.info("Successful deployment")
try:
with io.open(timestamp_path, 'r', encoding='utf8') as inf:
last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f")
clean = False
except (IOError, Exception) as e:
self.logger.debug("Problem when reading `{0}`: {1}".format(timestamp_path, e))
last_deploy = datetime(1970, 1, 1)
clean = True

new_deploy = datetime.utcnow()
self._emit_deploy_event(last_deploy, new_deploy, clean, undeployed_posts)

makedirs(self.site.config['CACHE_FOLDER'])
# Store timestamp of successful deployment
with io.open(timestamp_path, 'w+', encoding='utf8') as outf:
outf.write(unicode_str(new_deploy.isoformat()))
self.site.state.set('last_deploy', new_deploy.isoformat())

def _emit_deploy_event(self, last_deploy, new_deploy, clean=False, undeployed=None):
"""Emit events for all timeline entries newer than last deploy.
@@ -0,0 +1,83 @@
# -*- coding: utf-8 -*-

# Copyright © 2012-2016 Roberto Alsina and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Persistent state implementation."""

import json
import os
import shutil
import tempfile
import threading

from . import utils


class Persistor():
"""Persist stuff in a place.
This is an intentionally dumb implementation. It is *not* meant to be
fast, or useful for arbitrarily large data. Use lightly.
Intentionally it has no namespaces, sections, etc. Use as a
responsible adult.
"""

def __init__(self, path):
"""Where do you want it persisted."""
self._path = path
utils.makedirs(os.path.dirname(path))
self._local = threading.local()
self._local.data = {}

def get(self, key):
"""Get data stored in key."""
self._read()
return self._local.data.get(key)

def set(self, key, value):
"""Store value in key."""
self._read()
self._local.data[key] = value
self._save()

def delete(self, key):
"""Delete key and the value it contains."""
self._read()
if key in self._local.data:
self._local.data.pop(key)
self._save()

def _read(self):
if os.path.isfile(self._path):
with open(self._path) as inf:
self._local.data = json.load(inf)

def _save(self):
dname = os.path.dirname(self._path)
with tempfile.NamedTemporaryFile(dir=dname, delete=False) as outf:
tname = outf.name
json.dump(self._local.data, outf, sort_keys=True, indent=2)
shutil.move(tname, self._path)

0 comments on commit 82fb86d

Please sign in to comment.
You can’t perform that action at this time.