[doc] Make "front matter" optional.

split_doc.py --strict makes it required.

- add in_progress notification
- Make title detection smarter

build/doc.sh

@@ -157,8 +157,12 @@ cmark() {
 
 readonly DOCS=(
   # polished
-  osh-manual oil-manual known-differences eggex 
+  osh-manual known-differences eggex 
   # needs polish
+  # Note: docs about the Oil language are prefixed 'oil-'.
+  # data-model and command-vs-expression-mode span both OSH and Oil.
+
+  index
   oil-options
   oil-keywords-and-builtins
   oil-assignment
@@ -167,7 +171,7 @@ readonly DOCS=(
   oil-literal-syntax
   oil-word-language
   oil-special-vars
-  func-proc-blocks
+  oil-func-proc-block
   eggex
   data-model
 )
@@ -192,6 +196,7 @@ split-and-render() {
     -v oil_version="$OIL_VERSION" \
     -v css_files='../web/manual.css ../web/toc.css' \
     -v all_docs_url='.' \
+    -v repo_url="$src" \
     $src $prefix
 
   #ls -l _tmp/doc
@@ -254,7 +259,8 @@ all() {
   # TODO: We can set repo_url here!  Then we don't need it for most docs.
   # split_doc.py can return {} if the doc doesn't start with ---
 
-  for d in doc/index.md doc/known-differences.md doc/*-manual.md doc/eggex.md; do
+  for d in doc/index.md doc/known-differences.md doc/*-manual.md \
+    doc/eggex.md doc/oil-options.md; do
     #"${DOCS[@]}"; do
     split-and-render $d
   done

devtools/doc_html.py

@@ -30,10 +30,18 @@ def Header(meta, f):
       <a href="https://github.com/oilshell/oil/blob/master/%(repo_url)s" id="source-link">source</a> |
       <a href="%(all_docs_url)s">all docs</a>
         for <span id="version-in-header">version %(oil_version)s</span> |
+      <a href="/releases.html">all versions</a> |
       <a href="/">oilshell.org</a>
     </p>
 ''' % meta)
 
+  if 'in_progress' in meta:
+    f.write('''\
+    <p style="color: darkred; font-size: x-large;">
+      NOTE: This document is a work in progress!
+    </p>
+''')
+
 
 def Footer(meta, f):
   f.write('''\

devtools/split_doc.py

@@ -17,16 +17,19 @@
     r'(\S+): [ ]* (.*)', re.VERBOSE)
 
 
-def SplitDocument(default_vals, entry_f, meta_f, content_f):
+def SplitDocument(default_vals, entry_f, meta_f, content_f, strict=False):
   """Split a document into metadata JSON and content Markdown.
 
   Used for blog posts and index.md / cross-ref.md.
   """
-  first_line = entry_f.readline().strip()
+  first_line = entry_f.readline()
+  if strict and first_line.strip() != '---':
+    raise RuntimeError("Document should start with --- (got %r)" % first_line)
+
+  meta = {}
 
   # TODO: if first_line is ---, then read metadata in key: value format.
-  if first_line == '---':
-    meta = {}
+  if first_line.strip() == '---':
     while True:
       line = entry_f.readline().strip()
       if line == '---':
@@ -57,37 +60,43 @@ def SplitDocument(default_vals, entry_f, meta_f, content_f):
       else:
         meta[name] = value
 
+    #print('line = %r' % line, file=sys.stderr)
+    while True:
+      first_nonempty = entry_f.readline()
+      if first_nonempty.strip() != '':
+        break
+
   else:
-    raise AssertionError(
-        "Document should start with --- (got %r)" % first_line)
+    if first_line:
+      first_nonempty = first_line
+    else:
+      while True:
+        first_nonempty = entry_f.readline()
+        if first_nonempty.strip() != '':
+          break
+
+  # Invariant: we've read the first non-empty line here.  Now we need to see if
+  # it's the title.
 
+  #print('first_nonempty = %r' % first_nonempty, file=sys.stderr)
 
+  line_two = entry_f.readline()
+  if re.match('=+', line_two):
+    meta['title'] = first_nonempty.strip()
+
+  # Fill in defaults after parsing all values.
   for name, value in default_vals.iteritems():
     if name not in meta:
       meta[name] = value
 
-  # Parse the title like this:
-  # ---
-  # repo-url:
-  # ---
-  #
-  # Title
-  # =====
-
-  one = entry_f.readline()
-  two = entry_f.readline()
-  three = entry_f.readline()
-  if re.match('=+', three):
-    meta['title'] = two.strip()
-
   json.dump(meta, meta_f, indent=2)
 
   # Read the rest of the file and write it
   contents = entry_f.read()
 
-  content_f.write(one)
-  content_f.write(two)
-  content_f.write(three)
+  content_f.write(first_nonempty)
+  content_f.write(line_two)
+
   content_f.write(contents)
 
   comments_url = meta.get('comments_url', '')
@@ -106,6 +115,9 @@ def Options():
   p.add_option(
       '-v', dest='default_vals', action='append', default=[],
       help="If the doc's own metadata doesn't define 'name', set it to this value")
+  p.add_option(
+      '-s', '--strict', dest='strict', action='store_true', default=False,
+      help="Require metadata")
   return p
 
 
@@ -128,7 +140,7 @@ def main(argv):
       open(entry_path) as entry_f, \
       open(meta_path, 'w') as meta_f, \
       open(content_path, 'w') as content_f:
-    SplitDocument(default_vals, entry_f, meta_f, content_f)
+    SplitDocument(default_vals, entry_f, meta_f, content_f, strict=opts.strict)
 
 
 if __name__ == '__main__':

devtools/split_doc_test.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python2
+"""
+split_doc_test.py: Tests for split_doc.py
+"""
+from __future__ import print_function
+
+import unittest
+from cStringIO import StringIO
+
+import split_doc  # module under test
+
+
+class FooTest(unittest.TestCase):
+
+  def testStrict(self):
+    entry_f = StringIO('''\
+Title
+=====
+
+hello
+
+''')
+
+    meta_f = StringIO()
+    content_f = StringIO()
+
+    self.assertRaises(RuntimeError,
+        split_doc.SplitDocument, {}, entry_f, meta_f, content_f, strict=True)
+
+    print(meta_f.getvalue())
+    print(content_f.getvalue())
+
+  def testMetadataAndTitle(self):
+    print('_' * 40)
+    print()
+
+    entry_f = StringIO('''\
+---
+foo: bar
+---
+
+Title
+=====
+
+hello
+
+''')
+
+    meta_f = StringIO()
+    content_f = StringIO()
+
+    split_doc.SplitDocument({'default': 'd'}, entry_f, meta_f, content_f)
+
+    print(meta_f.getvalue())
+    print(content_f.getvalue())
+
+  def testMetadataAndTitleNoSpace(self):
+    print('_' * 40)
+    print()
+
+    entry_f = StringIO('''\
+---
+foo: bar
+---
+No Space Before Title
+=====================
+
+hello
+
+''')
+
+    meta_f = StringIO()
+    content_f = StringIO()
+
+    split_doc.SplitDocument({'default': 'd'}, entry_f, meta_f, content_f)
+
+    print(meta_f.getvalue())
+    print(content_f.getvalue())
+
+  def testTitleOnly(self):
+    print('_' * 40)
+    print()
+
+    entry_f = StringIO('''\
+No Space Before Title
+=====================
+
+hello
+
+''')
+
+    meta_f = StringIO()
+    content_f = StringIO()
+
+    split_doc.SplitDocument({'default': 'd'}, entry_f, meta_f, content_f)
+
+    print(meta_f.getvalue())
+    print(content_f.getvalue())
+
+
+if __name__ == '__main__':
+  unittest.main()

doc/index.md

@@ -1,10 +1,29 @@
----
-repo_url: doc/index.md
----
-
 Oil Documentation
 =================
 
+The Oil project aims to transform Unix shell into a better programming
+language.
+
+This manual covers the parts of Oil that are **new and unique**.  In contrast,
+the [OSH User Manual](osh-manual.html) describes parts of `osh` that overlap
+with other shells like `bash`.
+
+Everything described here is part of the `osh` binary.  In other words, the Oil
+language is implemented with a set of backward-compatible extensions, often
+using shell options that are toggled with the `shopt` builtin.
+
+(In the distant future, there may be a legacy-free `oil` binary.)
+
+<!-- cmark.py expands this -->
+<div id="toc">
+</div>
+
+
+
+## Other Topics
+
+
+
 TODO:
 
 - copy section from release-index.md
@@ -17,7 +36,23 @@ TODO:
 
 ### Oil
 
-- Lots of Stuff here
+- [Options](oil-options.html)
+- [Keywords and Builtins](oil-keywords-and-builtins.html)
+  - [Assignment](oil-assignment.html)
+- [Command vs. Expression Mode](command-vs-expression-mode.html)
+- [Oil Expressions](oil-expressions.html)
+  - [Literal Syntax](oil-literal-syntax.html)
+- [Word Language](oil-word-language.html) - Oil extends the "word language".
+- [Special Variables](oil-special-vars.html) - Oil extends the "word language".
+- [funcs, procs, and blocks](oil-func-proc-block.html)
+- [Eggex](eggex.html) -- regular expression language.
+- [Data Model](data-model.html) -- The interpreter
+
+Other:
+
+- scope
+- `evalblock()`
+
 
 ### Other
 

doc/oil-manual.md

@@ -4,44 +4,3 @@ repo_url: doc/oil-manual.md
 
 Oil User Manual
 ===============
-
-The Oil project aims to transform Unix shell into a better programming
-language.
-
-This manual covers the parts of Oil that are **new and unique**.  In contrast,
-the [OSH User Manual](osh-manual.html) describes parts of `osh` that overlap
-with other shells like `bash`.
-
-Everything described here is part of the `osh` binary.  In other words, the Oil
-language is implemented with a set of backward-compatible extensions, often
-using shell options that are toggled with the `shopt` builtin.
-
-(In the distant future, there may be a legacy-free `oil` binary.)
-
-<!-- cmark.py expands this -->
-<div id="toc">
-</div>
-
-
-## Sections
-
-
-- [Options](oil-options.html)
-- [Keywords and Builtins](oil-keywords-and-builtins.html)
-  - [Assignment](oil-assignment.html)
-- [Command vs. Expression Mode](command-vs-expression-mode.html)
-- [Oil Expressions](oil-expressions.html)
-  - [Literal Syntax](oil-literal-syntax.html)
-- [Word Language](oil-word-language.html) - Oil extends the "word language".
-- [Special Variables](oil-special-vars.html) - Oil extends the "word language".
-- [funcs, procs, and blocks](func-proc-blocks.html)
-- [Eggex](eggex.html) -- regular expression language.
-- [Data Model](data-model.html) -- The interpreter
-
-
-## Other Topics
-
-### Scope
-
-- `evalblock()`
-

doc/oil-options.md

@@ -1,3 +1,7 @@
+---
+in_progress: yes
+---
+
 Oil Options
 ===========