improved static website generation

lib/python/moa/plugin/job/versioning.py

@@ -21,7 +21,7 @@
 from moa.sysConf import sysConf
 
 l = moa.logger.getLogger(__name__)
-l.setLevel(moa.logger.DEBUG)
+l.setLevel(moa.logger.WARNING)
 
 
 def _run_cl(cl):
@@ -104,7 +104,7 @@ def hook_pelican(job):
 
     jenv = sysConf.plugins.pelican.jenv
     jtemplate = jenv.select_template(['versioning.page.jinja2'])
-    with open('./doc/pages/version.md', 'w') as F:
+    with open('.moa/doc/pages/version.md', 'w') as F:
         F.write(jtemplate.render({
             'last_run_id': last_run_id,
             'vrange': vrange,

lib/python/moa/plugin/system/doc/__init__.py

@@ -57,15 +57,18 @@ def hook_defineOptions():
         dest='changeMessage', help='Change message for this operation')
 
 
-def _writeMessage(category, txt):
+def _writeMessage(category, txt, title=None):
     """
     Append a markdown formatted message to either CHANGELOG or BLOG
 
+    :param category: the category of message
+    :param title: an optional title for the message
+    :type title: string
     :param txt: message to save
     :type txt: array of strings
     """
 
-    dirname = os.path.join('doc', category)
+    dirname = os.path.join('.moa', 'doc', category)
     if not os.path.exists(dirname):
         os.makedirs(dirname)
 
@@ -75,8 +78,11 @@ def _writeMessage(category, txt):
         category, now.year, now.month, now.day,
         now.hour, now.minute, now.second))
 
-    title = txt[0]
-    txt = txt[1:]
+    if title is None:
+        title = txt[0]
+        txt = txt[1:]
+
+    txt = "\n".join(txt).rstrip() + "\n"
 
     #title = "%s" % category.capitalize()
 
@@ -221,7 +227,6 @@ def change(job, args):
 
 @moa.args.doNotLog
 @moa.args.needsJob
-@moa.args.addFlag('-f', '--force', help='force rewriting of configuration')
 @moa.args.command
 def pelican(job, args):
     """
@@ -233,12 +238,13 @@ def pelican(job, args):
     sysConf.plugins.pelican.jenv = jenv
 
     themedir = os.path.join(os.path.dirname(__file__), 'theme')
+
     sysConf.doc.server = socket.gethostname()
     peliconf = '.moa/pelican.conf.py'
-    renderdir = 'doc/pelican'
+    renderdir = '.moa/doc/pelican'
 
-    if not os.path.exists('doc'):
-        os.makedirs('doc')
+    if not os.path.exists('.moa/doc/pages'):
+        os.makedirs('.moa/doc/pages')
 
     #call a plugin hook to let other plugins generate pelican pages
     # (if they want to)
@@ -251,18 +257,18 @@ def pelican(job, args):
     pelican_util.generate_readme_page(job)
     pelican_util.generate_template_page(job)
 
-    if args.force or (not os.path.exists(peliconf)):
-        jtemplate = jenv.select_template(['pelican.conf.jinja2'])
+    jtemplate = jenv.select_template(['pelican.conf.jinja2'])
 
-        txt = jtemplate.render(sysConf)
-        with open(peliconf, 'w') as F:
-            F.write(txt)
+    txt = jtemplate.render(sysConf)
+    with open(peliconf, 'w') as F:
+        F.write(txt)
 
     if not os.path.exists(renderdir):
         os.makedirs(renderdir)
 
-    cl = 'pelican -q -t %s -s .moa/pelican.conf.py -o doc/pelican/ doc/' % (
-        themedir)
+    cl = ('pelican -q -t %s -m md -s .moa/pelican.conf.py ' +
+          '-o .moa/doc/pelican/ .moa/doc/') % (themedir)
+
     l.debug("Executing pelican:")
     l.debug("   %s" % cl)
     subprocess.Popen(cl, shell=True)

lib/python/moa/plugin/system/doc/pelican_util.py

@@ -2,7 +2,9 @@
 Functions to support pelican
 """
 import datetime
+import collections
 import os
+import glob
 
 import moa.logger
 from Yaco import Yaco
@@ -22,7 +24,7 @@ def _getpagename(name):
     :param name: Name of the page
     :type name: String
     """
-    pagedir = os.path.join('doc', 'pages')
+    pagedir = os.path.join('.moa', 'doc', 'pages')
     if not os.path.exists(pagedir):
         os.makedirs(pagedir)
 
@@ -37,15 +39,21 @@ def _getpostname(category):
     :type category: string
     """
 
-    pagedir = os.path.join('doc', category)
+    pagedir = os.path.join('.moa', 'doc', category)
     if not os.path.exists(pagedir):
         os.makedirs(pagedir)
 
+    flist = glob.glob(os.path.join(pagedir, '*.md'))
+    if len(flist) == 0:
+        latest = None
+    else:
+        latest = max(flist, key=lambda x: os.stat(x).st_mtime)
+
     now = datetime.datetime.now()
     filename = os.path.join(pagedir, '%s_%d%d%d_%d%d%d.md' % (
         category, now.year, now.month, now.day,
         now.hour, now.minute, now.second))
-    return filename
+    return filename, latest
 
 
 def generate_redirect(job):
@@ -56,7 +64,6 @@ def generate_redirect(job):
     :type job: moa.job.Job object
     """
     jtemplate = jenv.select_template(['redirect.jinja2'])
-    pagename = _getpagename('template.md')
     if os.path.exists('index.html'):
         os.unlink('index.html')
     with open('index.html', 'w') as F:
@@ -71,12 +78,14 @@ def generate_template_page(job):
     :type job: moa.job.Job object
     """
     jtemplate = jenv.select_template(['template.page.jinja2'])
-    pagename = _getpostname('template')
+    pagename = _getpagename('template.md')
+
+    l.debug("generate template page")
+    newpage = jtemplate.render({
+        't': job.template})
 
-    l.warning("generate template page")
     with open(pagename, 'w') as F:
-        F.write(jtemplate.render({
-            't': job.template}))
+        F.write(newpage)
 
 
 def generate_readme_page(job):
@@ -86,7 +95,7 @@ def generate_readme_page(job):
     if not os.path.exists('README.md'):
         return
 
-    targetdir = os.path.join('doc', 'pages')
+    targetdir = os.path.join('.moa', 'doc', 'pages')
     if not os.path.exists(targetdir):
         os.makedirs(targetdir)
     targetfile = os.path.join(targetdir, 'readme.md')
@@ -158,27 +167,30 @@ def generate_file_page(job):
         else:
             data.single['fsid'].files = files
 
-    if len(fsets + fmaps) == 0:
-        with open(pagename, 'w') as F:
-            F.write(jtemplate.render(data))
-        return
-
     #rearrange the files into logical sets
     nofiles = len(job.data.filesets[(fsets + fmaps)[0]].files)
 
     data.sets = []
+    all_categories = ['input', 'output']
     for i in range(nofiles):
-        thisSet = []
+        thisFiles = collections.defaultdict(list)
+        max_in_category = 0
         for j, fsid in enumerate((fsets + fmaps)):
             files = job.data.filesets[fsid].files
             templateInfo = job.template.filesets[fsid]
-            thisSet.append((templateInfo.category,
-                            templateInfo.type,
-                            fsid,
-                            files[i],
-                            os.path.dirname(files[i]),
-                            os.path.basename(files[i])))
-        data.sets.append(thisSet)
+            cat = templateInfo.category
+
+            if not cat in all_categories:
+                all_categories.append(cat)
+
+            thisFiles[cat].append(
+                [templateInfo.type,
+                 fsid,
+                 files[i],
+                 os.path.dirname(files[i]),
+                 os.path.basename(files[i])])
+        max_in_category = max([len(x) for x in thisFiles.values()])
+        data.sets.append((max_in_category, thisFiles))
 
     with open(pagename, 'w') as F:
         F.write(jtemplate.render(data))

lib/python/moa/plugin/system/doc/templates/file.page.jinja2

@@ -1,18 +1,25 @@
-Title: Filesets
-Category: Filesets
+Title: file sets
 
-
-Set  | Name         | Typ | Cat     | File
------|--------------|-----|---------|----------------------------------------------
-{% for fset in sets %}{%- set outerindex=loop.index -%}
-{%- for f in fset -%}
+Set  | Input                   | Output                 |
+-----|-------------------------|------------------------|
+{% for no_in_cat, fset in sets %}{%- set outerindex=loop.index -%}
+{%- for i in range(no_in_cat) -%}
 {%- if loop.index0 == 0 -%}
 {{ "%03d"|format(outerindex) }}. |
 {%- else -%}
 {{ "%4s"|format(" ") }} |
 {%- endif -%}
- {{ "%-12s"|format('**' + f[0][:8] + '**') }} | {{ "%3s"|format(f[1][:3]) }} | {% set dummy=1 -%}
-{{ "%6s"|format(f[2][:6]) }} | {% set dummy=2 -%}
-[{{ f[4] }}]({{ f[4] }}) **/** [{{ f[5] }}]({{ f[3] }})
-{% endfor -%}
-{% endfor %}
+{%- if fset.input|length <= i -%}
+{{ '                         |' }}
+{%- else -%}{% set f=fset['input'][i] -%}
+[{{f[3]}}]({{ f[3] }}) / [&psi;]({{ f[3] }}/index.html) / [{{ f[4] }}]({{ f[2] }}) |
+{%- endif -%}
+{%- if fset.output|length <= i -%}
+{{ '                         |' }}
+{%- else -%}{% set f=fset['output'][i] -%}
+[{{f[3]}}]({{ f[3] }}) / [&psi;]({{ f[3] }}/index.html) / [{{ f[4] }}]({{ f[2] }})
+{%- endif -%}
+{{ '\n' }}
+{%- endfor -%}
+{%- endfor %}
+

lib/python/moa/plugin/system/doc/templates/pelican.conf.jinja2

@@ -1,21 +1,23 @@
 # -*- coding: utf-8 -*-
 
-SITEURL = './doc/pelican/'
+SITEURL = './.moa/doc/pelican/'
 SITENAME = "Moa"
 DEFAULT_LANG = 'en'
 SERVER = u'{{ doc.server }}'
 WD= u'{{ job.wd }}'
+TITLE= u'{{ job.conf.title }}'
+MOATEMPLATE= u'{{ job.template.name }}'
 RELATIVE_URLS = True
 DISPLAY_PAGES_ON_MENU = True
-DEFAULT_PAGINATION = 15
-OUTPUT_PATH = './doc/pelican/'
-#PATH = 'doc'
+DEFAULT_PAGINATION = 25
+OUTPUT_PATH = './.moa/doc/pelican/'
 REVERSE_ARCHIVE_ORDER = True
-CONTENT_STATIC_LOC = '../../'
+CONTENT_STATIC_LOC = '../../../'
 MD_EXTENSIONS = ['tables', 'def_list']
 SUMMARY_MAX_LENGTH = 50
 
 LINKS = (('Moa docs', 'http://mfiers.github.com/Moa/'),
          ('Moa source', 'https://github.com/mfiers/Moa'),
          )
 
+

lib/python/moa/plugin/system/doc/templates/redirect.jinja2

@@ -2,8 +2,8 @@
 <html>
 <head>
 <title>Moa pelican page for {{ WD }}</title>
-<meta http-equiv="REFRESH" content="0;url=./doc/pelican/index.html"></HEAD>
+<meta http-equiv="REFRESH" content="0;url=./.moa/doc/pelican/index.html"></HEAD>
 <BODY>
 ..should have redirected...
 </BODY>
-</HTML>
+</HTML>

lib/python/moa/plugin/system/doc/theme/templates/base.html

@@ -11,17 +11,18 @@
 		<script src="{{ SITEURL }}/theme/html5.js"></script>
 	<![endif]-->
 
-	<!-- Le styles -->
-	<link href="{{ SITEURL }}/theme/bootstrap.min.css" rel="stylesheet">
-	<link href="{{ SITEURL }}/theme/local.css" rel="stylesheet">
-	<link href="{{ SITEURL }}/theme/pygments.css" rel="stylesheet">
+	<link rel="shortcut icon" href="{{ SITEURL }}/theme/images/icons/favicon.ico" />
+	<link href="{{ SITEURL }}/theme/bootstrap.min.css" rel="stylesheet" />
+	<link href="{{ SITEURL }}/theme/local.css" rel="stylesheet" />
+	<link href="{{ SITEURL }}/theme/pygments.css" rel="stylesheet" />
 </head>
 <body>
 	<div class="topbar">
 	  <div class="topbar-inner">
 	    <div class="container-fluid">
-	      <a class="brand" href="{{ SITEURL }}/index.html">{{ SITENAME }}</a>
-              x {{ WD }} x
+	      <a class="brand" href="{{ SITEURL }}/index.html">
+		<img src="{{SITEURL}}/theme/images/moa.logo.very.small.png">
+		{{ SITENAME }} | {{ MOATEMPLATE }} | <b> {{ TITLE }} </b> | </a>
 	      <ul class="nav">
                 <li><a href="{% block jobdir %}../../{% endblock %}">{{WD}}</a></li>
 	      </ul>
@@ -52,7 +53,7 @@ <h3>Job</h3>
                 </li>
 		{% endfor %}
 	      </ul>
-              
+
 	      <h3>Links</h3>
 	      <ul>
 		{% for name, link in LINKS %}
@@ -68,7 +69,7 @@ <h3>Links</h3>
 	      <p> Powered by <a href="http://alexis.notmyidea.org/pelican/">Pelican</a>, theme based on <a href="http://twitter.github.com/bootstrap/">Bootstrap from Twitter</a>&copy; {{ AUTHOR }} 2011</p>
 	    </footer>
 	  </div>
-   
+
 	</div>
 </body>
 </html>

lib/python/moa/plugin/system/doc/theme/templates/index.html

@@ -6,9 +6,8 @@
 <div class='article'>
   <h2>{{ article.category}}: {{ article.title }}</h2>
   <div class="well small">{% include "metadata.html" %}</div>
-  <div class="summary">{{ article.summary }} &nbsp;&nbsp;<a class="btn primary xsmall" href="{{ SITEURL }}/{{ article.url }}">more…</a>
-  </div>
-</div>	
+
+</div>
 {% endfor %}
 {%endif%}
 
@@ -22,7 +21,10 @@ <h2>{{ article.category}}: {{ article.title }}</h2>
     <li class="prev disabled"><a href="#">&larr; Previous</a></li>
     {% endif %}
     {% for num in range( 1, 1 + articles_paginator.num_pages ) %}
-    <li class="{{ 'active' if num == articles_page.number else '' }}"><a href="{{ SITEURL }}/{{ page_name }}{{ num if num > 1 else '' }}.html">{{ num }}</a></li>
+    <li class="{{ 'active' if num == articles_page.number else '' }}">
+      <a href="{{ SITEURL }}/{{ page_name }}{{ num if num > 1 else '' }}.html">
+	{{ num }}
+    </a></li>
     {% endfor %}
     {% if articles_page.has_next() %}
     <li class="next"><a href="{{ SITEURL }}/{{ page_name }}{{ articles_page.next_page_number() }}.html">Next &rarr;</a></li>

lib/python/moa/plugin/system/logger.py

@@ -162,7 +162,7 @@ def hook_finish():
 
 
 def hook_pelican():
-    l.critical("logger pelican hook")
+    l.debug("logger pelican hook")
     return
     l.debug("pelican versioning output")
     nov = 10