Skip to content
Permalink
Browse files

Merge branch 'master' into new-auto

  • Loading branch information
ralsina committed Jun 2, 2015
2 parents 364fea9 + 627b57e commit d2867aedee13ef6159e8bb26f618cf6497fe3481
@@ -4,6 +4,7 @@ New in master
Features
--------

* New --get-path option for ``nikola install_theme`` (Issue #1762)
* New `nikola rst2html` command (Issue #1710)
* New `nikola status` command (Issue #1740)
* Support [code] in wordpress importers (Issue #1186)
@@ -14,6 +15,7 @@ Features
Bugfixes
--------

* Extract ``nikola check`` target list from actual task list instead of parsing (Issue #1758)
* Treat special-purpose “draft” tag case-insensitive
* Avoid some rebuild loops (Issue #1747)
* Better error if two posts/pages output conflict (Issue #1749)

Large diffs are not rendered by default.

@@ -25,6 +25,7 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import print_function
from collections import defaultdict
import os
import re
import sys
@@ -34,6 +35,7 @@
except ImportError:
from urllib.parse import unquote, urlparse, urljoin, urldefrag # NOQA

from doit.loader import generate_tasks
import lxml.html
try:
import requests
@@ -44,33 +46,28 @@
from nikola.utils import get_logger, req_missing


def _call_nikola_list(l, site, arguments):
class NotReallyAStream(object):
"""A massive hack."""
out = []
def _call_nikola_list(site):
files = []
deps = defaultdict(list)
for task in generate_tasks('render_site', site.gen_tasks('render_site', "Task", '')):
files.extend(task.targets)
for target in task.targets:
deps[target].extend(task.file_dep)
for task in generate_tasks('post_render', site.gen_tasks('render_site', "LateTask", '')):
files.extend(task.targets)
for target in task.targets:
deps[target].extend(task.file_dep)
return files, deps

def write(self, t):
self.out.append(t)

oldstream = l.outstream
newstream = NotReallyAStream()
try:
l.outstream = newstream
l.parse_execute(arguments)
return newstream.out
finally:
l.outstream = oldstream


def real_scan_files(l, site):
def real_scan_files(site):
task_fnames = set([])
real_fnames = set([])
output_folder = site.config['OUTPUT_FOLDER']
# First check that all targets are generated in the right places
for task in _call_nikola_list(l, site, ["--all"]):
task = task.strip()
if output_folder in task and ':' in task:
fname = task.split(':', 1)[-1]
for fname in _call_nikola_list(site)[0]:
fname = fname.strip()
if fname.startswith(output_folder):
task_fnames.add(fname)
# And now check that there are no non-target files
for root, dirs, files in os.walk(output_folder, followlinks=True):
@@ -154,7 +151,6 @@ class CommandCheck(Command):
def _execute(self, options, args):
"""Check the generated site."""
self.logger = get_logger('check', self.site.loghandlers)
self.l = self._doitargs['cmds'].get_plugin('list')(config=self.config, **self._doitargs)

if not options['links'] and not options['files'] and not options['clean']:
print(self.help())
@@ -175,21 +171,25 @@ def _execute(self, options, args):
existing_targets = set([])
checked_remote_targets = {}

def analyze(self, task, find_sources=False, check_remote=False):
def analyze(self, fname, find_sources=False, check_remote=False):
rv = False
self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']]
base_url = urlparse(self.site.config['BASE_URL'])
self.existing_targets.add(self.site.config['SITE_URL'])
self.existing_targets.add(self.site.config['BASE_URL'])
url_type = self.site.config['URL_TYPE']

deps = {}
if find_sources:
deps = _call_nikola_list(self.site)[1]

if check_remote and requests is None:
req_missing(['requests'], 'check remote links')

if url_type in ('absolute', 'full_path'):
url_netloc_to_root = urlparse(self.site.config['BASE_URL']).path
try:
filename = task.split(":")[-1]
filename = fname

if filename.startswith(self.site.config['CACHE_FOLDER']):
# Do not look at links in the cache, which are not parsed by
@@ -200,7 +200,7 @@ def analyze(self, task, find_sources=False, check_remote=False):

d = lxml.html.fromstring(open(filename, 'rb').read())
for l in d.iterlinks():
target = l[0].attrib[l[1]]
target = l[2]
if target == "#":
continue
target, _ = urldefrag(target)
@@ -236,8 +236,12 @@ def analyze(self, task, find_sources=False, check_remote=False):
continue

if url_type == 'rel_path':
target_filename = os.path.abspath(
os.path.join(os.path.dirname(filename), unquote(target)))
if target.startswith('/'):
target_filename = os.path.abspath(
os.path.join(os.path.dirname(filename), unquote(target.lstrip('/'))))
else: # Relative path
target_filename = os.path.abspath(
os.path.join(os.path.dirname(filename), unquote(target)))

elif url_type in ('full_path', 'absolute'):
if url_type == 'absolute':
@@ -262,7 +266,7 @@ def analyze(self, task, find_sources=False, check_remote=False):
self.logger.warn("Broken link in {0}: {1}".format(filename, target))
if find_sources:
self.logger.warn("Possible sources:")
self.logger.warn("\n".join(_call_nikola_list(self.l, self.site, ["--deps", task])))
self.logger.warn("\n".join(deps[filename]))
self.logger.warn("===============================\n")
except Exception as exc:
self.logger.error("Error with: {0} {1}".format(filename, exc))
@@ -273,14 +277,11 @@ def scan_links(self, find_sources=False, check_remote=False):
self.logger.info("===============\n")
self.logger.notice("{0} mode".format(self.site.config['URL_TYPE']))
failure = False
for task in _call_nikola_list(self.l, self.site, ["--all"]):
task = task.strip()
if task.split(':')[0] in (
'render_tags', 'render_archive',
'render_galleries', 'render_indexes',
'render_pages', 'render_posts',
'render_site') and '.html' in task:
if self.analyze(task, find_sources, check_remote):
# Maybe we should just examine all HTML files
output_folder = self.site.config['OUTPUT_FOLDER']
for fname in _call_nikola_list(self.site)[0]:
if fname.startswith(output_folder) and '.html' == fname[-5:]:
if self.analyze(fname, find_sources, check_remote):
failure = True
if not failure:
self.logger.info("All links checked.")
@@ -290,7 +291,7 @@ def scan_files(self):
failure = False
self.logger.info("Checking Files:")
self.logger.info("===============\n")
only_on_output, only_on_input = real_scan_files(self.l, self.site)
only_on_output, only_on_input = real_scan_files(self.site)

# Ignore folders
only_on_output = [p for p in only_on_output if not os.path.isdir(p)]
@@ -312,7 +313,7 @@ def scan_files(self):
return failure

def clean_files(self):
only_on_output, _ = real_scan_files(self.l, self.site)
only_on_output, _ = real_scan_files(self.site)
for f in only_on_output:
os.unlink(f)
return True
@@ -86,8 +86,7 @@ def _execute(self, command, args):
sys.exit(build)

# Clean non-target files
l = self._doitargs['cmds'].get_plugin('list')(config=self.config, **self._doitargs)
only_on_output, _ = real_scan_files(l, self.site)
only_on_output, _ = real_scan_files(self.site)
for f in only_on_output:
os.unlink(f)

@@ -88,6 +88,14 @@ class CommandInstallTheme(Command):
"https://themes.getnikola.com/v7/themes.json)",
'default': 'https://themes.getnikola.com/v7/themes.json'
},
{
'name': 'getpath',
'short': 'g',
'long': 'get-path',
'type': bool,
'default': False,
'help': "Print the path for installed theme",
},
]

def _execute(self, options, args):
@@ -102,6 +110,14 @@ def _execute(self, options, args):
else:
name = None

if options['getpath'] and name:
path = utils.get_theme_path(name)
if path:
print(path)
else:
print('not installed')
exit(0)

if name is None and not listing:
LOGGER.error("This command needs either a theme name or the -l option.")
return False
@@ -41,6 +41,5 @@ class CommandOrphans(Command):
Output contains filenames only (it is passable to `xargs rm` or the like)."""

def _execute(self, options, args):
l = self._doitargs['cmds'].get_plugin('list')(config=self.config, **self._doitargs)
orphans = real_scan_files(l, self.site)[0]
orphans = real_scan_files(self.site)[0]
print('\n'.join([p for p in orphans if not os.path.isdir(p)]))
@@ -40,8 +40,34 @@ class CommandDeploy(Command):
doc_purpose = "display site status"
doc_description = "Show information about the posts and site deployment."
logger = None

def _execute(self, command, args):
cmd_options = [
{
'name': 'list_drafts',
'short': 'd',
'long': 'list-drafts',
'type': bool,
'default': False,
'help': 'List all drafts',
},
{
'name': 'list_modified',
'short': 'm',
'long': 'list-modified',
'type': bool,
'default': False,
'help': 'List all modified files since last deployment',
},
{
'name': 'list_scheduled',
'short': 's',
'long': 'list-scheduled',
'type': bool,
'default': False,
'help': 'List all scheduled posts',
},
]

def _execute(self, options, args):

self.site.scan_posts()

@@ -58,38 +84,47 @@ def _execute(self, command, args):

if last_deploy:

fmod_since_deployment = 0
fmod_since_deployment = []
for root, dirs, files in os.walk(self.site.config["OUTPUT_FOLDER"], followlinks=True):
if not dirs and not files:
continue
for fname in files:
fpath = os.path.join(root, fname)
fmodtime = datetime.fromtimestamp(os.stat(fpath).st_mtime)
if fmodtime.replace(tzinfo=tzlocal()) > last_deploy.replace(tzinfo=gettz("UTC")).astimezone(tz=tzlocal()):
fmod_since_deployment = fmod_since_deployment + 1
fmod_since_deployment.append(fpath)

if fmod_since_deployment > 0:
print("{0} output files modified since last deployment {1} ago.".format(str(fmod_since_deployment), self.human_time(last_deploy_offset)))
if len(fmod_since_deployment) > 0:
print("{0} output files modified since last deployment {1} ago.".format(str(len(fmod_since_deployment)), self.human_time(last_deploy_offset)))
if options['list_modified']:
for fpath in fmod_since_deployment:
print("Modified: '{0}'".format(fpath))
else:
print("Last deployment {0} ago.".format(self.human_time(last_deploy_offset)))

now = datetime.utcnow().replace(tzinfo=gettz("UTC"))

posts_count = len(self.site.all_posts)
posts_drafts = 0
posts_scheduled = 0
nearest_scheduled_offset = None

for post in self.site.all_posts:
if post.is_draft:
posts_drafts = posts_drafts + 1
if post.publish_later:
posts_scheduled = posts_scheduled + 1
post_due_offset = post.date - datetime.utcnow().replace(tzinfo=gettz("UTC"))
if (nearest_scheduled_offset is None) or (post_due_offset.seconds < nearest_scheduled_offset.seconds):
nearest_scheduled_offset = post_due_offset

if posts_scheduled > 0 and nearest_scheduled_offset is not None:
print("{0} to next scheduled post.".format(self.human_time(nearest_scheduled_offset)))
print("{0} posts in total, {1} scheduled, and {2} drafts.".format(posts_count, posts_scheduled, posts_drafts))

# find all drafts
posts_drafts = [post for post in self.site.all_posts if post.is_draft]
posts_drafts = sorted(posts_drafts, key=lambda post: post.source_path)

# find all scheduled posts with offset from now until publishing time
posts_scheduled = [(post.date - now, post) for post in self.site.all_posts if post.publish_later]
posts_scheduled = sorted(posts_scheduled, key=lambda offset_post: (offset_post[0], offset_post[1].source_path))

if len(posts_scheduled) > 0:
if options['list_scheduled']:
for offset, post in posts_scheduled:
print("Scheduled: '{1}' ({2}; source: {3}) in {0}".format(self.human_time(offset), post.meta('title'), post.permalink(), post.source_path))
else:
offset, post = posts_scheduled[0]
print("{0} to next scheduled post ('{1}'; {2}; source: {3}).".format(self.human_time(offset), post.meta('title'), post.permalink(), post.source_path))
if options['list_drafts']:
for post in posts_drafts:
print("Draft: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path))
print("{0} posts in total, {1} scheduled, and {2} drafts.".format(posts_count, len(posts_scheduled), len(posts_drafts)))

def human_time(self, dt):
days = dt.days
@@ -133,7 +133,7 @@ def render_listing(in_name, out_name, input_folder, output_folder, folders=[], f
os.path.join(
self.kw['output_folder'],
output_folder))))
if self.site.config['COPY_SOURCES']:
if self.site.config['COPY_SOURCES'] and in_name:
source_link = permalink[:-5] # remove '.html'
else:
source_link = None
@@ -58,17 +58,18 @@ def process_tree(self, src, dst):
continue
dst_file = os.path.join(dst_dir, src_name)
src_file = os.path.join(root, src_name)
thumb_file = '.thumbnail'.join(os.path.splitext(dst_file))
yield {
'name': dst_file,
'file_dep': [src_file],
'targets': [dst_file],
'actions': [(self.process_image, (src_file, dst_file))],
'targets': [dst_file, thumb_file],
'actions': [(self.process_image, (src_file, dst_file, thumb_file))],
'clean': True,
}

def process_image(self, src, dst):
def process_image(self, src, dst, thumb):
self.resize_image(src, dst, self.kw['max_image_size'], False)
self.resize_image(src, '.thumbnail'.join(os.path.splitext(dst)), self.kw['image_thumbnail_size'], False)
self.resize_image(src, thumb, self.kw['image_thumbnail_size'], False)

def gen_tasks(self):
"""Copy static files into the output folder."""

0 comments on commit d2867ae

Please sign in to comment.