From 42eaea736b75f5f4afd6e23287ad95480f23c869 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Ooms?= <aurelien.ooms@gmail.com>
Date: Tue, 16 Oct 2018 21:42:14 +0200
Subject: [PATCH 1/7] first failing attempt

---
 src/grammar.js  |  9 +++++++--
 src/shaker.js   | 37 +++++++++++++++++++++++++++++--------
 src/tokens.js   | 34 ++++++++++++++++------------------
 test/src/all.js |  2 +-
 4 files changed, 53 insertions(+), 29 deletions(-)

diff --git a/src/grammar.js b/src/grammar.js
index 2428409..18fcd73 100644
--- a/src/grammar.js
+++ b/src/grammar.js
@@ -66,11 +66,16 @@ const productions = {
     "renewenvironment" : [ '=renewenvironment' , "&environment-definition" ] ,
     "\n" : [ '=\n' ] ,
     " " : [ '= ' ] ,
-    "arg" : [ '=arg' ] , // 1.12
+    "digit" : [ '=digit' ] ,
+    "argument" : [ '=#' , '&argument-subject' ] , // 1.12
     "$" : [ '=$' ] ,
     "math" : [ '=\\(' , '&anything' , '=\\)' ] ,
     "mathenv" : [ '=\\[' , '&anything' , '=\\]' ] ,
   } ,
+  "argument-subject" : {
+    "#" : [ "=#" ] ,
+    "digit" : [ "=digit" ] ,
+  } ,
   "endif" : { // endif : 2
     "elsefi" : [ '=else' , "&anything" , '=fi' ] , // 2.0
     "fi" : [ '=fi' ] , // 2.1
@@ -85,7 +90,7 @@ const productions = {
     "*{envname}[nargs][default]{begin}{end}" : [ '=*' , '={' , '=text' , '=}' , "&definition-parameters" , '={' , "&anything" , '=}' , '={' , "&anything" , '=}' ] ,
   } ,
   "definition-parameters" : {
-    "yes" : [ '=[' , '=text' , '=]' , '&default-argument-for-definition' , "&ignore" ] ,
+    "yes" : [ '=[' , '=digit' , '=]' , '&default-argument-for-definition' , "&ignore" ] ,
     "no" : [ ] ,
   } ,
   "default-argument-for-definition" : {
diff --git a/src/shaker.js b/src/shaker.js
index f1806db..be3b72a 100644
--- a/src/shaker.js
+++ b/src/shaker.js
@@ -37,8 +37,8 @@ async function parseDefinitionParameters ( parameters ) {
   if (parameters.production === 'yes') {
     const it2 = iter(parameters.children);
     await next(it2) ; // [
-    const text = await next(it2) ;
-    nargs = parseInt(text.buffer, 10);
+    const digit = await next(it2) ;
+    nargs = parseInt(digit.buffer, 10);
     await next(it2) ; // ]
 
     const dfltparam = await next(it2) ;
@@ -62,6 +62,12 @@ const empty = {
   'children' : [] ,
 } ;
 
+const hash = {
+  'type' : 'leaf' ,
+  'terminal' : '#' ,
+  'buffer' : '#' ,
+} ;
+
 const err = ( nonterminal , production ) => () => {
   throw new Error(`${nonterminal}.${production} should have been handled before`);
 } ;
@@ -383,13 +389,23 @@ export default {
 
     " " : tree => tree ,
 
-    "arg": async ( tree , match , { args , variables } ) => {
-      const arg = await next(iter(tree.children)) ;
+    "digit" : tree => tree ,
+
+    "argument": async ( tree , match , { args , variables } ) => {
+      const it = iter(tree.children) ;
+      await next(it); // #
+      const nonterminal = await next(it); // # or digit
+      const arg = await next(iter(nonterminal.children)) ;
       if ( args.length < 2 ) throw new Error(`Requesting ${arg.buffer} but got no arguments in context.`) ;
-      const i = parseInt(arg.buffer.substr(1), 10) - 1; // #arg
-      if ( i >= args[1].length ) throw new Error(`Requesting ${arg.buffer} but only got ${args[1].length} arguments.`) ;
-      const subtree = args[1][i] ; // arg
-      return t( subtree , match , { args: args[0] , variables } ) ;
+      if (nonterminal.production === 'digit') {
+	const i = parseInt(arg.buffer, 10) - 1; // #arg
+	if ( i >= args[1].length ) throw new Error(`Requesting ${arg.buffer} but only got ${args[1].length} arguments.`) ;
+	const subtree = args[1][i] ; // arg
+	return t( subtree , match , { args: args[0] , variables } ) ;
+      }
+      else {
+	return hash ;
+      } ;
     } ,
 
     "$" : tree => tree ,
@@ -399,6 +415,11 @@ export default {
 
   } ,
 
+  "argument-subject" : {
+    "#" : err("argument-subject", "#") ,
+    "digit" : err("argument-subject", "digit") ,
+  } ,
+
   "endif": {
     "elsefi" : recurse( 'endif', 'elsefi' ) ,
     "fi" : tree => tree ,
diff --git a/src/tokens.js b/src/tokens.js
index b6a081b..0c8e5ee 100644
--- a/src/tokens.js
+++ b/src/tokens.js
@@ -86,27 +86,25 @@ async function* _tokens ( tape ) {
       }
 
     }
-    else if ( c === '#' ) {
-      yield* flush();
-
-      // read arg number
-      let arg = '#' ;
-      while ( true ) {
-	const d = await tape.read();
-	if ( d === tape.eof ) break ;
-	else if ( d >= '0' && d <= '9' ) arg += d;
-	else {
-	  tape.unread(d);
-	  break;
-	}
-      }
-      if ( arg === '#' ) throw new Error('Incomplete #') ;
-      yield [ 'arg' , arg , new Position(line, position) ] ;
-      position += arg.length;
-    }
     else {
       switch ( c ) {
 
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	  yield* flush();
+	  yield [ 'digit' , c , new Position(line, position) ] ;
+	  ++position;
+	  break;
+
+	case '#':
 	case '{':
 	case '}':
 	case '[':
diff --git a/test/src/all.js b/test/src/all.js
index 28dc3db..2233d9b 100644
--- a/test/src/all.js
+++ b/test/src/all.js
@@ -237,7 +237,7 @@ for ( const filename of transformedInputFiles )
 test( transformFile , `${transformedInputFiledir}/${filename}` , `${transformedOutputFiledir}/${filename}` ) ;
 
 // argument escaping
-//test( transform , '\\newcommand\\x[1]{\\def\\#1[1]{##1}}\\x{test}' , '\\def\\test[1]{#1}' ) ;
+test( transform , '\\newcommand\\x[1]{\\newcommand\\y[1]{#1 ##1}}\\x{test}\\y{1212}' , 'test 1212' ) ;
 
 // default arguments with newcommand and renewcommand
 test( transform , '\\newcommand{\\price}[2][17.5]{\\pounds #2 excl VAT @ #1\\%}\\price{100}' , '\\pounds 100 excl VAT @ 17.5\\%') ;

From f3fe9676c87f52a88c471c991163d152c3447db2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Ooms?= <aurelien.ooms@gmail.com>
Date: Wed, 17 Oct 2018 18:05:20 +0200
Subject: [PATCH 2/7] fix old arg throw tests

---
 test/src/all.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/src/all.js b/test/src/all.js
index 2233d9b..4905812 100644
--- a/test/src/all.js
+++ b/test/src/all.js
@@ -155,8 +155,8 @@ test( 'Complex displaymath environment (matrix)' , immutable , '\\begin{displaym
 test( 'Escaped newline' , immutable , 'a\\\nb' ) ;
 
 // incomplete arg number
-test( throws , '#' , /Incomplete #/ ) ;
-test( throws , '#x' , /Incomplete #/ ) ;
+test( throws , '#' , /unexpected end of file/ ) ;
+test( throws , '#x' , /1:2/ ) ;
 
 // no arguments defined
 test( throws , '#1' , /no arguments in context/ ) ;

From f7f3c420d88587ae7cfd8152b40965966f6b6ddc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Ooms?= <aurelien.ooms@gmail.com>
Date: Wed, 17 Oct 2018 18:37:40 +0200
Subject: [PATCH 3/7] first attempt at create base identity transform

---
 src/index.js                  |  3 ++
 src/shaketape.js              |  2 +-
 src/transform/index.js        | 12 ++++++
 src/{ => transform}/shaker.js | 73 +++++++----------------------------
 src/transform/visitor.js      | 34 ++++++++++++++++
 5 files changed, 65 insertions(+), 59 deletions(-)
 create mode 100644 src/transform/index.js
 rename src/{ => transform}/shaker.js (83%)
 create mode 100644 src/transform/visitor.js

diff --git a/src/index.js b/src/index.js
index 00313e6..02e4b7b 100644
--- a/src/index.js
+++ b/src/index.js
@@ -5,6 +5,7 @@ import shakestream from './shakestream' ;
 import shakestring from './shakestring' ;
 import shaketape from './shaketape' ;
 import tokens from './tokens' ;
+import transform from './transform' ;
 
 export default {
 	Position ,
@@ -14,6 +15,7 @@ export default {
 	shakestring ,
 	shaketape ,
 	tokens ,
+	transform ,
 } ;
 
 export {
@@ -24,4 +26,5 @@ export {
 	shakestring ,
 	shaketape ,
 	tokens ,
+	transform ,
 } ;
diff --git a/src/shaketape.js b/src/shaketape.js
index a363e9e..0983ead 100644
--- a/src/shaketape.js
+++ b/src/shaketape.js
@@ -3,7 +3,7 @@ import tape from '@aureooms/js-tape' ;
 
 import tokens from './tokens' ;
 import grammar from './grammar' ;
-import shaker from './shaker' ;
+import shaker from './transform/shaker' ;
 
 export default async function shaketape ( inputTape , outputStream ) {
 
diff --git a/src/transform/index.js b/src/transform/index.js
new file mode 100644
index 0000000..7b6e604
--- /dev/null
+++ b/src/transform/index.js
@@ -0,0 +1,12 @@
+import shaker from './shaker' ;
+import visitor from './visitor' ;
+
+export default {
+	shaker ,
+	visitor ,
+} ;
+
+export {
+	shaker ,
+	visitor ,
+} ;
diff --git a/src/shaker.js b/src/transform/shaker.js
similarity index 83%
rename from src/shaker.js
rename to src/transform/shaker.js
index be3b72a..26100dc 100644
--- a/src/shaker.js
+++ b/src/transform/shaker.js
@@ -1,6 +1,8 @@
 import { StopIteration } from '@aureooms/js-itertools' ;
 import { ast } from '@aureooms/js-grammar' ;
 
+import visitor from './visitor' ;
+
 // TODO create library with those
 function iter ( object ) {
   // maybe we do not even need the second case
@@ -73,56 +75,27 @@ const err = ( nonterminal , production ) => () => {
 } ;
 
 const t = ast.transform ;
-//const t = ( tree , match , ctx ) => {
-  //console.log(tree);
-  //return ast.transform( tree , match , ctx ) ;
-//} ;
-const m = ( children , match , ctx ) => ast.cmap( async child => await t( child , match , ctx ) , children ) ;
-
-const recurse = ( nonterminal , production ) => ( tree , match , ctx ) => ({
-  "type" : "node" ,
-  nonterminal ,
-  production ,
-  "children" : ast.cmap( async x => x.type === 'leaf' ? x : await t( x , match , ctx ) , tree.children ) ,
-}) ;
-
-export default {
-
-  "document" : {
-    "contents" : recurse( 'document' , 'contents' ) ,
-  } ,
+const cmap = ast.cmap ;
+const m = ( children , match , ctx ) => cmap( async child => await t( child , match , ctx ) , children ) ;
+
+function extend ( transform, extension ) {
+  const result = { } ;
+  for ( const key in transform ) {
+    result[key] = Object.assign(extension[key] || {}, Object.assign(transform[key], {})) ;
+  }
+  return result ;
+}
+
+export default extend( visitor , {
 
   "anything" : {
-    "starts-with-othercmd" : recurse( 'anything' , 'starts-with-othercmd' ) ,
-    "starts-with-begin-environment" : recurse( 'anything' , 'starts-with-begin-environment' ) ,
-    "starts-with-end-environment" : recurse( 'anything' , 'starts-with-end-environment' ) ,
-    "starts-with-*" : recurse( 'anything' , 'starts-with-*' ) ,
-    "starts-with-[" : recurse( 'anything' , 'starts-with-[' ) ,
-    "starts-with-]" : recurse( 'anything' , 'starts-with-]' ) ,
-    "starts-with-a-group" : recurse( 'anything' , 'starts-with-a-group' ) ,
-    "starts-with-something-else" : recurse( 'anything' , 'starts-with-something-else' ) ,
     "end" : () => empty ,
   } ,
 
   "anything-but-]" : {
-    "starts-with-othercmd" : recurse( 'anything-but-]' , 'starts-with-othercmd' ) ,
-    "starts-with-begin-environment" : recurse( 'anything-but-]' , 'starts-with-begin-environment' ) ,
-    "starts-with-end-environment" : recurse( 'anything-but-]' , 'starts-with-end-environment' ) ,
-    "starts-with-*" : recurse( 'anything-but-]' , 'starts-with-*' ) ,
-    "starts-with-[" : recurse( 'anything-but-]' , 'starts-with-[' ) ,
-    "starts-with-a-group" : recurse( 'anything-but-]' , 'starts-with-a-group' ) ,
-    "starts-with-something-else" : recurse( 'anything-but-]' , 'starts-with-something-else' ) ,
     "end" : () => empty ,
   } ,
 
-  "group" : {
-    "group" : recurse('group', 'group') ,
-  } ,
-
-  "optgroup" : {
-    "group" : recurse('optgroup', 'group') ,
-  } ,
-
   "othercmd" : {
 
     "othercmd": async ( tree , match , ctx ) => {
@@ -410,9 +383,6 @@ export default {
 
     "$" : tree => tree ,
 
-    "math" : recurse('something-else', 'math') ,
-    "mathenv" : recurse('something-else', 'mathenv') ,
-
   } ,
 
   "argument-subject" : {
@@ -421,7 +391,6 @@ export default {
   } ,
 
   "endif": {
-    "elsefi" : recurse( 'endif', 'elsefi' ) ,
     "fi" : tree => tree ,
   } ,
 
@@ -501,7 +470,6 @@ export default {
       variables.get('env').set(env, [ nargs , dflt , begin , end ]);
       return empty;
     } ,
-    "*{envname}[nargs][default]{begin}{end}" : recurse( 'environment-definition' , '*{envname}[nargs][default]{begin}{end}' ) ,
   } ,
   "definition-parameters" : {
     "yes" : err('definition-parameters' , 'yes' ) ,
@@ -518,25 +486,14 @@ export default {
   } ,
 
   "cmdargs": {
-    "normal" : recurse('cmdargs', 'normal') ,
-    "optional" : recurse('cmdargs', 'optional') ,
     "end" : () => empty ,
   } ,
 
   "cmdafter": {
-    "othercmd" : recurse('cmdafter', 'othercmd' ) ,
-    "begin-environment" : recurse('cmdafter', 'begin-environment' ) ,
-    "end-environment" : recurse('cmdafter', 'end-environment' ) ,
-    "something-else-then-anything" : recurse('cmdafter', 'something-else-then-anything' ) ,
-    "]-then-anything" : recurse('cmdafter', ']-then-anything' ) ,
     "nothing" : () => empty ,
   } ,
 
   "cmdafter-but-not-]": {
-    "othercmd" : recurse('cmdafter-but-not-]', 'othercmd' ) ,
-    "begin-environment" : recurse('cmdafter-but-not-]', 'begin-environment' ) ,
-    "end-environment" : recurse('cmdafter-but-not-]', 'end-environment' ) ,
-    "something-else-then-anything" : recurse('cmdafter-but-not-]', 'something-else-then-anything' ) ,
     "nothing" : () => empty ,
   } ,
 
@@ -547,4 +504,4 @@ export default {
     "nothing" : err('ignore', 'nothing') ,
   } ,
 
-} ;
+} ) ;
diff --git a/src/transform/visitor.js b/src/transform/visitor.js
new file mode 100644
index 0000000..e0194fb
--- /dev/null
+++ b/src/transform/visitor.js
@@ -0,0 +1,34 @@
+import { ast } from '@aureooms/js-grammar' ;
+
+import grammar from '../grammar' ;
+
+const t = ast.transform ;
+const cmap = ast.cmap ;
+const recurse = ( nonterminal , production ) => ( tree , match , ctx ) => ({
+  "type" : "node" ,
+  nonterminal ,
+  production ,
+  "children" : cmap( async x => x.type === 'leaf' ? x : await t( x , match , ctx ) , tree.children ) ,
+}) ;
+
+
+// move to @js-grammar/ast.visitor
+function generateVisitor ( grammar ) {
+
+  const transform = { } ;
+
+  for ( const [ nonterminal , productions ] of grammar.productions.entries() ) {
+    const nonterminalTransform = { } ;
+
+    for ( const production of productions.keys() ) {
+      nonterminalTransform[production] = recurse( nonterminal , production ) ;
+    }
+
+    transform[nonterminal] = nonterminalTransform ;
+  }
+
+  return transform ;
+
+}
+
+export default generateVisitor( grammar ) ;

From b14998a65d9f8918108e437c89a1ecd40b0d8344 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Ooms?= <aurelien.ooms@gmail.com>
Date: Wed, 17 Oct 2018 20:56:37 +0200
Subject: [PATCH 4/7] fix index and extend in shaker

---
 src/index.js            | 3 ---
 src/transform/shaker.js | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/index.js b/src/index.js
index 02e4b7b..3a17077 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,6 +1,5 @@
 import Position from './Position' ;
 import grammar from './grammar' ;
-import shaker from './shaker' ;
 import shakestream from './shakestream' ;
 import shakestring from './shakestring' ;
 import shaketape from './shaketape' ;
@@ -10,7 +9,6 @@ import transform from './transform' ;
 export default {
 	Position ,
 	grammar ,
-	shaker ,
 	shakestream ,
 	shakestring ,
 	shaketape ,
@@ -21,7 +19,6 @@ export default {
 export {
 	Position ,
 	grammar ,
-	shaker ,
 	shakestream ,
 	shakestring ,
 	shaketape ,
diff --git a/src/transform/shaker.js b/src/transform/shaker.js
index 26100dc..65ec915 100644
--- a/src/transform/shaker.js
+++ b/src/transform/shaker.js
@@ -81,7 +81,7 @@ const m = ( children , match , ctx ) => cmap( async child => await t( child , ma
 function extend ( transform, extension ) {
   const result = { } ;
   for ( const key in transform ) {
-    result[key] = Object.assign(extension[key] || {}, Object.assign(transform[key], {})) ;
+    result[key] = Object.assign({}, transform[key], extension[key]) ;
   }
   return result ;
 }

From d0f089d7e9d7c81aa0be926d4a9a869d33a047d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Ooms?= <aurelien.ooms@gmail.com>
Date: Wed, 17 Oct 2018 21:11:18 +0200
Subject: [PATCH 5/7] add escaped argument test that should fail ...

but does not, and simplify throws testing macro
---
 test/src/all.js | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/test/src/all.js b/test/src/all.js
index 4905812..6601d2f 100644
--- a/test/src/all.js
+++ b/test/src/all.js
@@ -22,12 +22,12 @@ async function transform ( t , string , expected ) {
 
 function throws ( t , string , expected ) {
 	const out = { 'write' : buffer => undefined } ;
-	//await t.throws(async () => await shakestring(string, out), expected);
-	return shakestring(string, out)
-		.then( () => t.fail() )
-		.catch( error => {
-			t.true(expected.test(error.message));
-		} ) ;
+	return t.throwsAsync(shakestring(string, out), expected);
+	//return shakestring(string, out)
+		//.then( () => t.fail() )
+		//.catch( error => {
+			//t.true(expected.test(error.message));
+		//} ) ;
 }
 
 const immutable = async ( t , string ) => await transform( t , string , string ) ;
@@ -238,6 +238,7 @@ test( transformFile , `${transformedInputFiledir}/${filename}` , `${transformedO
 
 // argument escaping
 test( transform , '\\newcommand\\x[1]{\\newcommand\\y[1]{#1 ##1}}\\x{test}\\y{1212}' , 'test 1212' ) ;
+test( throws , '\\def\\y{##1}\\newcommand\\x[1]{\\y}\\x{test}' , /no arguments in context/ ) ;
 
 // default arguments with newcommand and renewcommand
 test( transform , '\\newcommand{\\price}[2][17.5]{\\pounds #2 excl VAT @ #1\\%}\\price{100}' , '\\pounds 100 excl VAT @ 17.5\\%') ;

From 0327d620c2559ee343aefc406e458425a4266bcc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Ooms?= <aurelien.ooms@gmail.com>
Date: Wed, 17 Oct 2018 22:51:26 +0200
Subject: [PATCH 6/7] fix last issue with escaped parameters

---
 src/shaketape.js        |   2 +-
 src/transform/shaker.js | 231 ++++++++++++++++++++++------------------
 test/src/all.js         |  12 ++-
 3 files changed, 140 insertions(+), 105 deletions(-)

diff --git a/src/shaketape.js b/src/shaketape.js
index 0983ead..563d604 100644
--- a/src/shaketape.js
+++ b/src/shaketape.js
@@ -25,8 +25,8 @@ export default async function shaketape ( inputTape , outputStream ) {
 
   const ctx = {
     env : [ ] ,
-    args : [ ] ,
     variables ,
+    parser ,
   } ;
 
   const transformed = await ast.transform( tree , shaker , ctx ) ;
diff --git a/src/transform/shaker.js b/src/transform/shaker.js
index 65ec915..6b4eb2c 100644
--- a/src/transform/shaker.js
+++ b/src/transform/shaker.js
@@ -1,5 +1,6 @@
 import { StopIteration } from '@aureooms/js-itertools' ;
 import { ast } from '@aureooms/js-grammar' ;
+import tape from '@aureooms/js-tape' ;
 
 import visitor from './visitor' ;
 
@@ -86,7 +87,7 @@ function extend ( transform, extension ) {
   return result ;
 }
 
-export default extend( visitor , {
+const optimizedVisitor = extend( visitor , {
 
   "anything" : {
     "end" : () => empty ,
@@ -96,6 +97,98 @@ export default extend( visitor , {
     "end" : () => empty ,
   } ,
 
+  "*" : {
+    "*" : tree => tree ,
+  } ,
+
+  "[" : {
+    "[" : tree => tree ,
+  } ,
+
+  "]" : {
+    "]" : tree => tree ,
+  } ,
+
+  "something-else" : {
+
+    "text" : tree => tree ,
+
+    "\n" : tree => tree ,
+
+    " " : tree => tree ,
+
+    "digit" : tree => tree ,
+
+    "$" : tree => tree ,
+
+  } ,
+
+  "cmdargs": {
+    "end" : () => empty ,
+  } ,
+
+  "cmdafter": {
+    "nothing" : () => empty ,
+  } ,
+
+  "cmdafter-but-not-]": {
+    "nothing" : () => empty ,
+  } ,
+
+  "endif": {
+    "fi" : tree => tree ,
+  } ,
+
+} ) ;
+
+const expandArguments = extend( optimizedVisitor , {
+  "something-else": {
+    "argument": async ( tree , match , { args } ) => {
+      const it = iter(tree.children) ;
+      await next(it); // #
+      const nonterminal = await next(it); // # or digit
+      const arg = await next(iter(nonterminal.children)) ;
+      if (nonterminal.production === '#') return hash ; // this and next line could be moved one line up
+      const i = parseInt(arg.buffer, 10) - 1; // #arg
+      if ( i >= args.length ) throw new Error(`Requesting ${arg.buffer} but only got ${args.length} arguments.`) ;
+      const subtree = args[i] ;
+      return subtree ;
+    } ,
+  } ,
+
+  "argument-subject" : {
+    "#" : err("argument-subject", "#") ,
+    "digit" : err("argument-subject", "digit") ,
+  } ,
+
+} ) ;
+
+function parseArguments ( args , dfltarg , type , name ) {
+
+  const cmdargs = [];
+  let arg_i = args
+  if ( arg_i.production === 'optional' ) {
+    if (dfltarg === null) throw new Error(`${type} ${name} is not defined with a default argument.`) ;
+    const [ optgroup , tail ] = arg_i.children ;
+    const [ _open , arg , _close ] = optgroup.children ;
+    cmdargs.push(arg);
+    arg_i = tail ;
+  }
+  else if (dfltarg !== null) cmdargs.push(dfltarg) ;
+  while ( arg_i.production === 'normal' ) {
+    const [ group , tail ] = arg_i.children ;
+    const [ _open , arg , _close ] = group.children ;
+    cmdargs.push(arg) ;
+    arg_i = tail ;
+  }
+  const complex = arg_i.production === 'optional' ;
+
+  return [ complex , cmdargs ] ;
+
+}
+
+export default extend( optimizedVisitor , {
+
   "othercmd" : {
 
     "othercmd": async ( tree , match , ctx ) => {
@@ -113,27 +206,18 @@ export default extend( visitor , {
       if ( ctx.variables.get('cmd').has(cmd) ) {
 
 	const [ nargs , dfltarg , expandsto ] = ctx.variables.get('cmd').get(cmd) ;
-	const cmdargs = [];
-	let arg_i = args
-	if ( arg_i.production === 'optional' ) {
-	  if (dfltarg === null) throw new Error(`Command ${cmd} is not defined with a default argument.`) ;
-	  const [ optgroup , tail ] = arg_i.children ;
-	  const [ _open , arg , _close ] = optgroup.children ;
-	  cmdargs.push(arg);
-	  arg_i = tail ;
-	}
-	else if (dfltarg !== null) cmdargs.push(dfltarg) ;
-	while ( arg_i.production === 'normal' ) {
-	  const [ group , tail ] = arg_i.children ;
-	  const [ _open , arg , _close ] = group.children ;
-	  cmdargs.push(arg) ;
-	  arg_i = tail ;
-	}
-	const complex = arg_i.production === 'optional' ;
+
+	const [ complex , cmdargs ] = parseArguments( args , dfltarg , 'Command' , cmd ) ;
+
 	if (!complex) {
 	  // do not parse complex syntax
 	  if (cmdargs.length !== nargs) throw new Error(`Command ${cmd} is defined with ${nargs} arguments but ${cmdargs.length} were given.`) ;
-	  return t( expandsto , match , { env: ctx.env, variables: ctx.variables , args: [ ctx.args , cmdargs ] } ) ;
+
+	  const withArguments = await t( expandsto , expandArguments , { args: cmdargs } )
+	  const flat = ast.flatten(withArguments) ;
+	  const tokensTape = tape.fromAsyncIterable(flat);
+	  const subtree = (await ast.materialize(ctx.parser.parse(tokensTape))).children[0] ;
+	  return t( subtree , match , ctx ) ;
 	}
 
       }
@@ -170,31 +254,20 @@ export default extend( visitor , {
 
 	const [ nargs , dfltarg , begin , end ] = ctx.variables.get('env').get(env) ;
 
-	const cmdargs = [];
-	let arg_i = args
-	if ( arg_i.production === 'optional' ) {
-	  if (dfltarg === null) throw new Error(`Environment ${env} is not defined with a default argument.`) ;
-	  const [ optgroup , tail ] = arg_i.children ;
-	  const [ _open , arg , _close ] = optgroup.children ;
-	  cmdargs.push(arg);
-	  arg_i = tail ;
-	}
-	else if (dfltarg !== null) cmdargs.push(dfltarg) ;
-	while ( arg_i.production === 'normal' ) {
-	  const [ group , tail ] = arg_i.children ;
-	  const [ _open , arg , _close ] = group.children ;
-	  cmdargs.push(arg) ;
-	  arg_i = tail ;
-	}
+	const [ complex , cmdargs ] = parseArguments( args , dfltarg , 'Environment' , env ) ;
 
-	const complex = arg_i.production === 'optional' ;
 	if (!complex) {
 	  envStackEntry.expand = true ;
 	  envStackEntry.args = cmdargs ;
 	  // do not parse complex syntax
 	  if (cmdargs.length !== nargs)
 	    throw new Error(`Environment ${env} is defined with ${nargs} arguments but ${cmdargs.length} were given.`) ;
-	  return t( begin , match , { env: envStackEntry.children , variables: ctx.variables , args: [ ctx.args , cmdargs ] } ) ;
+
+	  const withArguments = await t( begin , expandArguments , { args: cmdargs } )
+	  const flat = ast.flatten(withArguments) ;
+	  const tokensTape = tape.fromAsyncIterable(flat);
+	  const subtree = (await ast.materialize(ctx.parser.parse(tokensTape))).children[0] ;
+	  return t( subtree , match , { env: envStackEntry.children , variables: ctx.variables , parser: ctx.parser } ) ;
 	}
 
       }
@@ -226,14 +299,18 @@ export default extend( visitor , {
 	throw new Error(`Trying to end environment on an empty stack with \\end{${env}} (matching \\begin{${env}} is missing).`);
       }
 
-      const { expand , env: currentEnv , children , args } = ctx.env.pop();
+      const { expand , env: currentEnv , children , args: cmdargs } = ctx.env.pop();
 
       if ( currentEnv !== env ) {
 	throw new Error(`Trying to match \\begin{${currentEnv}} with \\end{${env}}.`);
       }
       else if (expand) {
 	const [ nargs , defaultarg , begin , end ] = ctx.variables.get('env').get(env) ;
-	return t( end , match , { env: children , variables: ctx.variables , args: [ ctx.args , args ] } ) ;
+	const withArguments = await t( end , expandArguments , { args: cmdargs } )
+	const flat = ast.flatten(withArguments) ;
+	const tokensTape = tape.fromAsyncIterable(flat);
+	const subtree = (await ast.materialize(ctx.parser.parse(tokensTape))).children[0] ;
+	return t( subtree , match , { env: children , variables: ctx.variables , parser: ctx.parser } ) ;
       }
       else {
 	return {
@@ -247,24 +324,16 @@ export default extend( visitor , {
 
   } ,
 
-  "*" : {
-    "*" : tree => tree ,
-  } ,
-
-  "[" : {
-    "[" : tree => tree ,
-  } ,
-
-  "]" : {
-    "]" : tree => tree ,
-  } ,
-
   "something-else" : {
 
-    "text" : tree => tree ,
-
     "newif": () => empty ,
 
+    "comment": ( ) => ({
+      'type' : 'leaf' ,
+      'terminal' : 'comment' ,
+      'buffer' : '%' ,
+    }) ,
+
     "ifcmd": async ( tree , match , ctx ) => {
 
       const it = iter(tree.children) ;
@@ -311,12 +380,6 @@ export default extend( visitor , {
       return empty;
     } ,
 
-    "comment": ( ) => ({
-      'type' : 'leaf' ,
-      'terminal' : 'comment' ,
-      'buffer' : '%' ,
-    }) ,
-
     "def": async ( tree , match , { variables } ) => {
       const it = iter(tree.children) ;
       await next(it) ; // \def
@@ -358,40 +421,16 @@ export default extend( visitor , {
       return t( envdef , match , ctx ) ;
     } ,
 
-    "\n" : tree => tree ,
-
-    " " : tree => tree ,
-
-    "digit" : tree => tree ,
-
-    "argument": async ( tree , match , { args , variables } ) => {
-      const it = iter(tree.children) ;
-      await next(it); // #
-      const nonterminal = await next(it); // # or digit
-      const arg = await next(iter(nonterminal.children)) ;
-      if ( args.length < 2 ) throw new Error(`Requesting ${arg.buffer} but got no arguments in context.`) ;
-      if (nonterminal.production === 'digit') {
-	const i = parseInt(arg.buffer, 10) - 1; // #arg
-	if ( i >= args[1].length ) throw new Error(`Requesting ${arg.buffer} but only got ${args[1].length} arguments.`) ;
-	const subtree = args[1][i] ; // arg
-	return t( subtree , match , { args: args[0] , variables } ) ;
-      }
-      else {
-	return hash ;
-      } ;
-    } ,
-
-    "$" : tree => tree ,
-
   } ,
 
   "argument-subject" : {
-    "#" : err("argument-subject", "#") ,
-    "digit" : err("argument-subject", "digit") ,
-  } ,
-
-  "endif": {
-    "fi" : tree => tree ,
+    "#" : () => {
+      throw new Error('Escaped hash (##) without argument context.') ;
+    } ,
+    "digit" : async tree => {
+      const digit = await next(iter(tree.children)) ;
+      throw new Error(`Requesting #${digit.buffer} without argument context.`) ;
+    } ,
   } ,
 
   "command-definition" : {
@@ -485,18 +524,6 @@ export default extend( visitor , {
     "no" : err( "cmd*" , "no" ) ,
   } ,
 
-  "cmdargs": {
-    "end" : () => empty ,
-  } ,
-
-  "cmdafter": {
-    "nothing" : () => empty ,
-  } ,
-
-  "cmdafter-but-not-]": {
-    "nothing" : () => empty ,
-  } ,
-
   "ignore" : {
     "starts-with-a-space" : err('ignore', 'starts-with-a-space') ,
     "starts-with-a-newline" : err('ignore', 'starts-with-a-newline') ,
diff --git a/test/src/all.js b/test/src/all.js
index 6601d2f..9de940c 100644
--- a/test/src/all.js
+++ b/test/src/all.js
@@ -159,7 +159,15 @@ test( throws , '#' , /unexpected end of file/ ) ;
 test( throws , '#x' , /1:2/ ) ;
 
 // no arguments defined
-test( throws , '#1' , /no arguments in context/ ) ;
+test( throws , '#1' , /#1 without argument context/ ) ;
+test( throws , '#2' , /#2 without argument context/ ) ;
+test( throws , '#3' , /#3 without argument context/ ) ;
+test( throws , '#4' , /#4 without argument context/ ) ;
+test( throws , '#5' , /#5 without argument context/ ) ;
+test( throws , '#6' , /#6 without argument context/ ) ;
+test( throws , '#7' , /#7 without argument context/ ) ;
+test( throws , '#8' , /#8 without argument context/ ) ;
+test( throws , '#9' , /#9 without argument context/ ) ;
 
 // escaped #
 test( immutable , '\\#' ) ;
@@ -238,7 +246,7 @@ test( transformFile , `${transformedInputFiledir}/${filename}` , `${transformedO
 
 // argument escaping
 test( transform , '\\newcommand\\x[1]{\\newcommand\\y[1]{#1 ##1}}\\x{test}\\y{1212}' , 'test 1212' ) ;
-test( throws , '\\def\\y{##1}\\newcommand\\x[1]{\\y}\\x{test}' , /no arguments in context/ ) ;
+test( throws , '\\def\\y{##1}\\newcommand\\x[1]{\\y}\\x{test}' , /#1 without argument context/ ) ;
 
 // default arguments with newcommand and renewcommand
 test( transform , '\\newcommand{\\price}[2][17.5]{\\pounds #2 excl VAT @ #1\\%}\\price{100}' , '\\pounds 100 excl VAT @ 17.5\\%') ;

From 04979a02172c069bef9b3bdb8d47965730b5d634 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Ooms?= <aurelien.ooms@gmail.com>
Date: Wed, 17 Oct 2018 23:15:51 +0200
Subject: [PATCH 7/7] increase coverage

---
 test/src/all.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/src/all.js b/test/src/all.js
index 9de940c..df1a8a3 100644
--- a/test/src/all.js
+++ b/test/src/all.js
@@ -159,6 +159,7 @@ test( throws , '#' , /unexpected end of file/ ) ;
 test( throws , '#x' , /1:2/ ) ;
 
 // no arguments defined
+test( throws , '##' , /Escaped hash \(##\) without argument context/ ) ;
 test( throws , '#1' , /#1 without argument context/ ) ;
 test( throws , '#2' , /#2 without argument context/ ) ;
 test( throws , '#3' , /#3 without argument context/ ) ;