[doc] Start re-organizing docs.

I think we need a new doc index though.

build/doc.sh

@@ -152,7 +152,7 @@ osh-quick-ref() {
 
 cmark() {
   # h2 and h3 are shown in TOC.  The blog uses "legacy" h3 and h4.
-  devtools/cmark.py --toc-tag h2 --toc-tag h3 --pretty-href
+  devtools/cmark.py --toc-tag h2 --toc-tag h3 --toc-pretty-href
 }
 
 markdown2html() {
@@ -199,6 +199,23 @@ install() {
   markdown2html INSTALL.txt $root_dir/doc/INSTALL.html "$css_link"
 }
 
+readonly DOCS=(
+  # polished
+  osh-manual oil-manual known-differences eggex 
+  # needs polish
+  oil-options
+  oil-keywords-and-builtins
+  oil-assignment
+  command-vs-expression-mode
+  oil-expressions
+  oil-literal-syntax
+  oil-word-language
+  oil-special-vars
+  func-proc-blocks
+  eggex
+  data-model
+)
+
 manual() {
   local root_dir=${1:-_release/VERSION}
   local release_date=${2:-}
@@ -209,8 +226,7 @@ manual() {
   '
 
   # TODO: cmark.py could replace <span class="date"></span> with -v date=?
-  for d in \
-    osh-manual oil-manual known-differences eggex oil-expressions; do
+  for d in "${DOCS[@]}"; do
 
     markdown2html doc/$d.md $root_dir/doc/$d.html "$css_link" ''
   done

devtools/cmark.py

@@ -52,9 +52,12 @@ def demo():
 
 class TocExtractor(HTMLParser.HTMLParser):
   """
-  Look for.
+  When he hit h_tags (h2, h3, h4, etc.), append to self.headings, recording the
+  line number.
 
-  <div id="toc">
+  Later, we insert two things:
+  - <a name=""> before each heading
+  - The TOC after <div id="toc">
   """
   def __init__(self, h_tags):
     HTMLParser.HTMLParser.__init__(self)
@@ -96,7 +99,7 @@ def handle_starttag(self, tag, attrs):
           css_id = v
           break
       self.headings.append((line_num, tag, css_id, []))
-      self.capturing = True
+      self.capturing = True  # record the text inside <h2></h2> etc.
 
   def handle_endtag(self, tag):
     # Debug print
@@ -131,11 +134,17 @@ def handle_data(self, data):
       self._AppendHtml(data)
 
   def _AppendHtml(self, html):
+    """
+    Append HTML to the last heading.
+    """
     _, _, _, html_parts = self.headings[-1]
     html_parts.append(html)
 
 
-def _MakeTocAndAnchors(headings, toc_pos):
+TAG_TO_CSS = {'h2': 'toclevel1', 'h3': 'toclevel2', 'h4': 'toclevel3'}
+
+
+def _MakeTocAndAnchors(opts, headings, toc_pos):
   """
   Given a list of extract headings and TOC position, render HTML to insert.
   """
@@ -145,22 +154,38 @@ def _MakeTocAndAnchors(headings, toc_pos):
   # Yeah it's just a flat list, and then indentation is done with CSS.  Hm
   # that's easy.
 
-  toc_lines = ['<div id="toctitle">Table of Contents</div>']
+  toc_lines = ['<div id="toctitle">Table of Contents</div>\n']
   insertions = []
 
   i = 0
   for line_num, tag, css_id, html_parts in headings:
-    css_class = {'h2': 'toclevel1', 'h3': 'toclevel2', 'h4': 'toclevel3'}[tag]
+    css_class = TAG_TO_CSS[tag]
 
     # Add BOTH anchors, for stability.
     numeric_anchor = 'toc_%d' % i
+
+    # If there was an explicit CSS ID written by the user, use that as the href.
+    # I used this in the blog a few times.
+
     href = css_id or numeric_anchor
 
-    line = '<div class="%s"><a href="#%s">%s</a></div>\n' % (
+    if opts.toc_pretty_href:
+      # TODO: generate a target
+      pass
+
+    line = '  <div class="%s"><a href="#%s">%s</a></div>\n' % (
         css_class, href, ''.join(html_parts))
     toc_lines.append(line)
 
-    target = '<a name="%s"></a>\n' % numeric_anchor
+    auto_anchor = 'TODO'
+
+    if opts.toc_pretty_href:
+      anchor = auto_anchor
+    else:
+      anchor = numeric_anchor
+
+    # Add one or two targets.
+    target = '<a name="%s"></a>\n' % anchor
     if css_id:
       target += '<a name="%s"></a>\n' % css_id
     insertions.append((line_num, target))
@@ -212,7 +237,7 @@ def Render(opts, in_file, out_file):
     out_file.write(html)  # Pass through
     return
 
-  insertions = _MakeTocAndAnchors(parser.headings, parser.toc_begin_line)
+  insertions = _MakeTocAndAnchors(opts, parser.headings, parser.toc_begin_line)
 
   log('')
   log('*** Text Insertions:')
@@ -230,13 +255,29 @@ def Options():
   """Returns an option parser instance."""
   p = optparse.OptionParser('cmark.py [options]')
   p.add_option(
-      '--pretty-href', action='store_true', default=False,
+      '--toc-pretty-href', action='store_true', default=False,
       help='Generate textual hrefs #like-this rather than like #toc10')
   p.add_option(
       '--toc-tag', dest='toc_tags', action='append', default=[],
       help='h tags to include in the TOC, e.g. h2 h3')
   return p
 
+# --extract-metadata : title plus front matter
+#
+# Move shell markdown2html here then?
+# maybe add 
+#
+# What about update-src-versions?  That's for every doc I guess.
+
+
+"""
+- repo-url: doc/README.md
+
+Title is h1
+===========
+
+"""
+
 
 def main(argv):
   o = Options()

devtools/cmark_test.py

@@ -54,15 +54,56 @@ def f():
 </pre></div>
 """)
 
+NEW_DOC = cStringIO.StringIO("""
+Title
+=====
+
+<div id="toc">
+</div>
+
+## One
+
+hello one.
+
+### subheading
+
+## Two &amp; Three
+
+""")
+
+
+DOC_WITH_METADATA = cStringIO.StringIO("""
+- repo-url: doc/README.md
+
+Title
+=====
+
+## One
+""")
+
+
+
 class RenderTest(unittest.TestCase):
 
   def testRender(self):
+    opts, _ = cmark.Options().parse_args([])
+
     out_file = cStringIO.StringIO()
-    cmark.Render(SIMPLE_DOC, out_file)
+    cmark.Render(opts, SIMPLE_DOC, out_file)
     self.assertEqual('<p>hi</p>\n', out_file.getvalue())
 
     out_file = cStringIO.StringIO()
-    cmark.Render(TOC_DOC, out_file)
+    cmark.Render(opts, TOC_DOC, out_file)
+    print(out_file.getvalue())
+
+  def testNewRender(self):
+    # New style of doc
+
+    new_flags = ['--toc-tag', 'h2', '--toc-tag', 'h3']
+    opts, _ = cmark.Options().parse_args(new_flags)
+
+    out_file = cStringIO.StringIO()
+    cmark.Render(opts, NEW_DOC, out_file)
     print(out_file.getvalue())
 
 

doc/command-vs-expression-mode.md

@@ -4,7 +4,7 @@ Command vs. Expression Mode
 
 Here is a list of places we switch modes:
 
-# From Command Mode to Expression Mode
+## From Command Mode to Expression Mode
 
 Assignments:
 
@@ -47,14 +47,14 @@ for (x, y in pairs) { ... }
 ```
 
 
-# From Expression Mode to Command Mode
+## From Expression Mode to Command Mode
 
 ```
 var x = func(x) { echo hi; return x +1 }   # everything between {} is in command mode
 ```
 
 
-# Other
+## Other
 
 Braced Vars in Double Quotes:
 

doc/func-proc-blocks.md

@@ -1,6 +1,79 @@
 Procs, Funcs, and Blocks
 ========================
 
+## Builtins Can Accept Ruby-Style Blocks
+
+Example of syntax that works:
+
+```
+cd / {
+  echo $PWD
+}
+cd / { echo $PWD }
+cd / { echo $PWD }; cd / { echo $PWD }
+```
+
+Syntax errors:
+
+```
+a=1 { echo bad };        # assignments can't take blocks
+>out.txt { echo bad };   # bare redirects can't take blocks
+break { echo bad };      # control flow can't take blocks
+```
+
+Runtime errors
+
+```
+local a=1 { echo bad };  # assignment builtins can't take blocks
+```
+
+### Caveat: Blocks Are Space Sensitive
+
+```
+cd {a,b}  # brace substitution
+cd { a,b }  # tries to run command 'a,b', which probably doesn't exist
+```
+
+more:
+
+```
+echo these are literal braces not a block \{ \}
+echo these are literal braces not a block '{' '}'
+# etc.
+```
+
+
+### What's Allowed in Blocks?
+
+You can break out with `return`, and it accepts Oil**expressions** (not
+shell-like words) (note: not implemented yet).
+
+
+```
+cd {
+  # return is for FUNCTIONS.
+  return 1 + 2 * 3
+}
+```
+
+The block can set vars in enclosing scope:
+
+```
+setvar('name', 1+2, up=1)
+```
+
+They can also get the value:
+
+```
+var namespace = evalblock('name', 1+2, up=1)
+
+# _result is set if there was a return statement!
+
+# namespace has all vars except those prefixed with _
+var result = namespace->_result
+```
+
+
 * Procs Have Open or Closed Signatures
 * Functions Look Like Julia, JavaScript, and Go
 * Procs May Accept Block Arguments

doc/oil-expressions.md

@@ -2,6 +2,52 @@ The Expression Language Is Mostly Python
 ========================================
 
 
+
+## Expressions
+
+### Shell Array Literals with @()
+
+```
+var x = @(a b c)
+var x = @(
+  'single quoted'
+  "double quoted"
+  $'c string'
+  glob/*.py
+  brace-{a,b,c}-{1..3}
+)
+```
+
+### Shell Command Substitution with $()
+
+The `$(echo hi)` construct works in shell commands, and it also works in Oil
+expressions:
+
+```
+var x = $(echo hi)           # no quotes necessary
+var x = "$(echo hi) there"
+```
+
+### Splice Arrays with @array
+
+```
+var a1 = @(a b)
+var a2 = @(c d)
+echo / @a1 / @a2 /   # gives / a b / c d /
+```
+
+### Future
+
+- "Legacy-free" command substitution with `$[echo hi]`
+- "Legacy-free" and typed literals like
+  - `@[a 'b c' "hi $name"]`
+  - `@[1 2 3]` 
+  - `@[3.14 1.50 2.33]`
+- For details, see the wiki page [Implementing the Oil Expression
+  Language](https://github.com/oilshell/oil/wiki/Implementing-the-Oil-Expression-Language)
+
+
+
 Most of the operator language is now implemented (in the metacircular style).
 
 Oil's operators largely follow Python, except: