diff --git a/src/grammar.js b/src/grammar.js index 2428409..18fcd73 100644 --- a/src/grammar.js +++ b/src/grammar.js @@ -66,11 +66,16 @@ const productions = { "renewenvironment" : [ '=renewenvironment' , "&environment-definition" ] , "\n" : [ '=\n' ] , " " : [ '= ' ] , - "arg" : [ '=arg' ] , // 1.12 + "digit" : [ '=digit' ] , + "argument" : [ '=#' , '&argument-subject' ] , // 1.12 "$" : [ '=$' ] , "math" : [ '=\\(' , '&anything' , '=\\)' ] , "mathenv" : [ '=\\[' , '&anything' , '=\\]' ] , } , + "argument-subject" : { + "#" : [ "=#" ] , + "digit" : [ "=digit" ] , + } , "endif" : { // endif : 2 "elsefi" : [ '=else' , "&anything" , '=fi' ] , // 2.0 "fi" : [ '=fi' ] , // 2.1 @@ -85,7 +90,7 @@ const productions = { "*{envname}[nargs][default]{begin}{end}" : [ '=*' , '={' , '=text' , '=}' , "&definition-parameters" , '={' , "&anything" , '=}' , '={' , "&anything" , '=}' ] , } , "definition-parameters" : { - "yes" : [ '=[' , '=text' , '=]' , '&default-argument-for-definition' , "&ignore" ] , + "yes" : [ '=[' , '=digit' , '=]' , '&default-argument-for-definition' , "&ignore" ] , "no" : [ ] , } , "default-argument-for-definition" : { diff --git a/src/index.js b/src/index.js index 00313e6..3a17077 100644 --- a/src/index.js +++ b/src/index.js @@ -1,27 +1,27 @@ import Position from './Position' ; import grammar from './grammar' ; -import shaker from './shaker' ; import shakestream from './shakestream' ; import shakestring from './shakestring' ; import shaketape from './shaketape' ; import tokens from './tokens' ; +import transform from './transform' ; export default { Position , grammar , - shaker , shakestream , shakestring , shaketape , tokens , + transform , } ; export { Position , grammar , - shaker , shakestream , shakestring , shaketape , tokens , + transform , } ; diff --git a/src/shaketape.js b/src/shaketape.js index a363e9e..563d604 100644 --- a/src/shaketape.js +++ b/src/shaketape.js @@ -3,7 +3,7 @@ import tape from '@aureooms/js-tape' ; import tokens from './tokens' ; import grammar from './grammar' ; -import shaker from './shaker' ; +import shaker from './transform/shaker' ; export default async function shaketape ( inputTape , outputStream ) { @@ -25,8 +25,8 @@ export default async function shaketape ( inputTape , outputStream ) { const ctx = { env : [ ] , - args : [ ] , variables , + parser , } ; const transformed = await ast.transform( tree , shaker , ctx ) ; diff --git a/src/tokens.js b/src/tokens.js index b6a081b..0c8e5ee 100644 --- a/src/tokens.js +++ b/src/tokens.js @@ -86,27 +86,25 @@ async function* _tokens ( tape ) { } } - else if ( c === '#' ) { - yield* flush(); - - // read arg number - let arg = '#' ; - while ( true ) { - const d = await tape.read(); - if ( d === tape.eof ) break ; - else if ( d >= '0' && d <= '9' ) arg += d; - else { - tape.unread(d); - break; - } - } - if ( arg === '#' ) throw new Error('Incomplete #') ; - yield [ 'arg' , arg , new Position(line, position) ] ; - position += arg.length; - } else { switch ( c ) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + yield* flush(); + yield [ 'digit' , c , new Position(line, position) ] ; + ++position; + break; + + case '#': case '{': case '}': case '[': diff --git a/src/transform/index.js b/src/transform/index.js new file mode 100644 index 0000000..7b6e604 --- /dev/null +++ b/src/transform/index.js @@ -0,0 +1,12 @@ +import shaker from './shaker' ; +import visitor from './visitor' ; + +export default { + shaker , + visitor , +} ; + +export { + shaker , + visitor , +} ; diff --git a/src/shaker.js b/src/transform/shaker.js similarity index 69% rename from src/shaker.js rename to src/transform/shaker.js index f1806db..6b4eb2c 100644 --- a/src/shaker.js +++ b/src/transform/shaker.js @@ -1,5 +1,8 @@ import { StopIteration } from '@aureooms/js-itertools' ; import { ast } from '@aureooms/js-grammar' ; +import tape from '@aureooms/js-tape' ; + +import visitor from './visitor' ; // TODO create library with those function iter ( object ) { @@ -37,8 +40,8 @@ async function parseDefinitionParameters ( parameters ) { if (parameters.production === 'yes') { const it2 = iter(parameters.children); await next(it2) ; // [ - const text = await next(it2) ; - nargs = parseInt(text.buffer, 10); + const digit = await next(it2) ; + nargs = parseInt(digit.buffer, 10); await next(it2) ; // ] const dfltparam = await next(it2) ; @@ -62,61 +65,130 @@ const empty = { 'children' : [] , } ; +const hash = { + 'type' : 'leaf' , + 'terminal' : '#' , + 'buffer' : '#' , +} ; + const err = ( nonterminal , production ) => () => { throw new Error(`${nonterminal}.${production} should have been handled before`); } ; const t = ast.transform ; -//const t = ( tree , match , ctx ) => { - //console.log(tree); - //return ast.transform( tree , match , ctx ) ; -//} ; -const m = ( children , match , ctx ) => ast.cmap( async child => await t( child , match , ctx ) , children ) ; - -const recurse = ( nonterminal , production ) => ( tree , match , ctx ) => ({ - "type" : "node" , - nonterminal , - production , - "children" : ast.cmap( async x => x.type === 'leaf' ? x : await t( x , match , ctx ) , tree.children ) , -}) ; - -export default { - - "document" : { - "contents" : recurse( 'document' , 'contents' ) , - } , +const cmap = ast.cmap ; +const m = ( children , match , ctx ) => cmap( async child => await t( child , match , ctx ) , children ) ; + +function extend ( transform, extension ) { + const result = { } ; + for ( const key in transform ) { + result[key] = Object.assign({}, transform[key], extension[key]) ; + } + return result ; +} + +const optimizedVisitor = extend( visitor , { "anything" : { - "starts-with-othercmd" : recurse( 'anything' , 'starts-with-othercmd' ) , - "starts-with-begin-environment" : recurse( 'anything' , 'starts-with-begin-environment' ) , - "starts-with-end-environment" : recurse( 'anything' , 'starts-with-end-environment' ) , - "starts-with-*" : recurse( 'anything' , 'starts-with-*' ) , - "starts-with-[" : recurse( 'anything' , 'starts-with-[' ) , - "starts-with-]" : recurse( 'anything' , 'starts-with-]' ) , - "starts-with-a-group" : recurse( 'anything' , 'starts-with-a-group' ) , - "starts-with-something-else" : recurse( 'anything' , 'starts-with-something-else' ) , "end" : () => empty , } , "anything-but-]" : { - "starts-with-othercmd" : recurse( 'anything-but-]' , 'starts-with-othercmd' ) , - "starts-with-begin-environment" : recurse( 'anything-but-]' , 'starts-with-begin-environment' ) , - "starts-with-end-environment" : recurse( 'anything-but-]' , 'starts-with-end-environment' ) , - "starts-with-*" : recurse( 'anything-but-]' , 'starts-with-*' ) , - "starts-with-[" : recurse( 'anything-but-]' , 'starts-with-[' ) , - "starts-with-a-group" : recurse( 'anything-but-]' , 'starts-with-a-group' ) , - "starts-with-something-else" : recurse( 'anything-but-]' , 'starts-with-something-else' ) , "end" : () => empty , } , - "group" : { - "group" : recurse('group', 'group') , + "*" : { + "*" : tree => tree , + } , + + "[" : { + "[" : tree => tree , } , - "optgroup" : { - "group" : recurse('optgroup', 'group') , + "]" : { + "]" : tree => tree , } , + "something-else" : { + + "text" : tree => tree , + + "\n" : tree => tree , + + " " : tree => tree , + + "digit" : tree => tree , + + "$" : tree => tree , + + } , + + "cmdargs": { + "end" : () => empty , + } , + + "cmdafter": { + "nothing" : () => empty , + } , + + "cmdafter-but-not-]": { + "nothing" : () => empty , + } , + + "endif": { + "fi" : tree => tree , + } , + +} ) ; + +const expandArguments = extend( optimizedVisitor , { + "something-else": { + "argument": async ( tree , match , { args } ) => { + const it = iter(tree.children) ; + await next(it); // # + const nonterminal = await next(it); // # or digit + const arg = await next(iter(nonterminal.children)) ; + if (nonterminal.production === '#') return hash ; // this and next line could be moved one line up + const i = parseInt(arg.buffer, 10) - 1; // #arg + if ( i >= args.length ) throw new Error(`Requesting ${arg.buffer} but only got ${args.length} arguments.`) ; + const subtree = args[i] ; + return subtree ; + } , + } , + + "argument-subject" : { + "#" : err("argument-subject", "#") , + "digit" : err("argument-subject", "digit") , + } , + +} ) ; + +function parseArguments ( args , dfltarg , type , name ) { + + const cmdargs = []; + let arg_i = args + if ( arg_i.production === 'optional' ) { + if (dfltarg === null) throw new Error(`${type} ${name} is not defined with a default argument.`) ; + const [ optgroup , tail ] = arg_i.children ; + const [ _open , arg , _close ] = optgroup.children ; + cmdargs.push(arg); + arg_i = tail ; + } + else if (dfltarg !== null) cmdargs.push(dfltarg) ; + while ( arg_i.production === 'normal' ) { + const [ group , tail ] = arg_i.children ; + const [ _open , arg , _close ] = group.children ; + cmdargs.push(arg) ; + arg_i = tail ; + } + const complex = arg_i.production === 'optional' ; + + return [ complex , cmdargs ] ; + +} + +export default extend( optimizedVisitor , { + "othercmd" : { "othercmd": async ( tree , match , ctx ) => { @@ -134,27 +206,18 @@ export default { if ( ctx.variables.get('cmd').has(cmd) ) { const [ nargs , dfltarg , expandsto ] = ctx.variables.get('cmd').get(cmd) ; - const cmdargs = []; - let arg_i = args - if ( arg_i.production === 'optional' ) { - if (dfltarg === null) throw new Error(`Command ${cmd} is not defined with a default argument.`) ; - const [ optgroup , tail ] = arg_i.children ; - const [ _open , arg , _close ] = optgroup.children ; - cmdargs.push(arg); - arg_i = tail ; - } - else if (dfltarg !== null) cmdargs.push(dfltarg) ; - while ( arg_i.production === 'normal' ) { - const [ group , tail ] = arg_i.children ; - const [ _open , arg , _close ] = group.children ; - cmdargs.push(arg) ; - arg_i = tail ; - } - const complex = arg_i.production === 'optional' ; + + const [ complex , cmdargs ] = parseArguments( args , dfltarg , 'Command' , cmd ) ; + if (!complex) { // do not parse complex syntax if (cmdargs.length !== nargs) throw new Error(`Command ${cmd} is defined with ${nargs} arguments but ${cmdargs.length} were given.`) ; - return t( expandsto , match , { env: ctx.env, variables: ctx.variables , args: [ ctx.args , cmdargs ] } ) ; + + const withArguments = await t( expandsto , expandArguments , { args: cmdargs } ) + const flat = ast.flatten(withArguments) ; + const tokensTape = tape.fromAsyncIterable(flat); + const subtree = (await ast.materialize(ctx.parser.parse(tokensTape))).children[0] ; + return t( subtree , match , ctx ) ; } } @@ -191,31 +254,20 @@ export default { const [ nargs , dfltarg , begin , end ] = ctx.variables.get('env').get(env) ; - const cmdargs = []; - let arg_i = args - if ( arg_i.production === 'optional' ) { - if (dfltarg === null) throw new Error(`Environment ${env} is not defined with a default argument.`) ; - const [ optgroup , tail ] = arg_i.children ; - const [ _open , arg , _close ] = optgroup.children ; - cmdargs.push(arg); - arg_i = tail ; - } - else if (dfltarg !== null) cmdargs.push(dfltarg) ; - while ( arg_i.production === 'normal' ) { - const [ group , tail ] = arg_i.children ; - const [ _open , arg , _close ] = group.children ; - cmdargs.push(arg) ; - arg_i = tail ; - } + const [ complex , cmdargs ] = parseArguments( args , dfltarg , 'Environment' , env ) ; - const complex = arg_i.production === 'optional' ; if (!complex) { envStackEntry.expand = true ; envStackEntry.args = cmdargs ; // do not parse complex syntax if (cmdargs.length !== nargs) throw new Error(`Environment ${env} is defined with ${nargs} arguments but ${cmdargs.length} were given.`) ; - return t( begin , match , { env: envStackEntry.children , variables: ctx.variables , args: [ ctx.args , cmdargs ] } ) ; + + const withArguments = await t( begin , expandArguments , { args: cmdargs } ) + const flat = ast.flatten(withArguments) ; + const tokensTape = tape.fromAsyncIterable(flat); + const subtree = (await ast.materialize(ctx.parser.parse(tokensTape))).children[0] ; + return t( subtree , match , { env: envStackEntry.children , variables: ctx.variables , parser: ctx.parser } ) ; } } @@ -247,14 +299,18 @@ export default { throw new Error(`Trying to end environment on an empty stack with \\end{${env}} (matching \\begin{${env}} is missing).`); } - const { expand , env: currentEnv , children , args } = ctx.env.pop(); + const { expand , env: currentEnv , children , args: cmdargs } = ctx.env.pop(); if ( currentEnv !== env ) { throw new Error(`Trying to match \\begin{${currentEnv}} with \\end{${env}}.`); } else if (expand) { const [ nargs , defaultarg , begin , end ] = ctx.variables.get('env').get(env) ; - return t( end , match , { env: children , variables: ctx.variables , args: [ ctx.args , args ] } ) ; + const withArguments = await t( end , expandArguments , { args: cmdargs } ) + const flat = ast.flatten(withArguments) ; + const tokensTape = tape.fromAsyncIterable(flat); + const subtree = (await ast.materialize(ctx.parser.parse(tokensTape))).children[0] ; + return t( subtree , match , { env: children , variables: ctx.variables , parser: ctx.parser } ) ; } else { return { @@ -268,24 +324,16 @@ export default { } , - "*" : { - "*" : tree => tree , - } , - - "[" : { - "[" : tree => tree , - } , - - "]" : { - "]" : tree => tree , - } , - "something-else" : { - "text" : tree => tree , - "newif": () => empty , + "comment": ( ) => ({ + 'type' : 'leaf' , + 'terminal' : 'comment' , + 'buffer' : '%' , + }) , + "ifcmd": async ( tree , match , ctx ) => { const it = iter(tree.children) ; @@ -332,12 +380,6 @@ export default { return empty; } , - "comment": ( ) => ({ - 'type' : 'leaf' , - 'terminal' : 'comment' , - 'buffer' : '%' , - }) , - "def": async ( tree , match , { variables } ) => { const it = iter(tree.children) ; await next(it) ; // \def @@ -379,29 +421,16 @@ export default { return t( envdef , match , ctx ) ; } , - "\n" : tree => tree , - - " " : tree => tree , - - "arg": async ( tree , match , { args , variables } ) => { - const arg = await next(iter(tree.children)) ; - if ( args.length < 2 ) throw new Error(`Requesting ${arg.buffer} but got no arguments in context.`) ; - const i = parseInt(arg.buffer.substr(1), 10) - 1; // #arg - if ( i >= args[1].length ) throw new Error(`Requesting ${arg.buffer} but only got ${args[1].length} arguments.`) ; - const subtree = args[1][i] ; // arg - return t( subtree , match , { args: args[0] , variables } ) ; - } , - - "$" : tree => tree , - - "math" : recurse('something-else', 'math') , - "mathenv" : recurse('something-else', 'mathenv') , - } , - "endif": { - "elsefi" : recurse( 'endif', 'elsefi' ) , - "fi" : tree => tree , + "argument-subject" : { + "#" : () => { + throw new Error('Escaped hash (##) without argument context.') ; + } , + "digit" : async tree => { + const digit = await next(iter(tree.children)) ; + throw new Error(`Requesting #${digit.buffer} without argument context.`) ; + } , } , "command-definition" : { @@ -480,7 +509,6 @@ export default { variables.get('env').set(env, [ nargs , dflt , begin , end ]); return empty; } , - "*{envname}[nargs][default]{begin}{end}" : recurse( 'environment-definition' , '*{envname}[nargs][default]{begin}{end}' ) , } , "definition-parameters" : { "yes" : err('definition-parameters' , 'yes' ) , @@ -496,29 +524,6 @@ export default { "no" : err( "cmd*" , "no" ) , } , - "cmdargs": { - "normal" : recurse('cmdargs', 'normal') , - "optional" : recurse('cmdargs', 'optional') , - "end" : () => empty , - } , - - "cmdafter": { - "othercmd" : recurse('cmdafter', 'othercmd' ) , - "begin-environment" : recurse('cmdafter', 'begin-environment' ) , - "end-environment" : recurse('cmdafter', 'end-environment' ) , - "something-else-then-anything" : recurse('cmdafter', 'something-else-then-anything' ) , - "]-then-anything" : recurse('cmdafter', ']-then-anything' ) , - "nothing" : () => empty , - } , - - "cmdafter-but-not-]": { - "othercmd" : recurse('cmdafter-but-not-]', 'othercmd' ) , - "begin-environment" : recurse('cmdafter-but-not-]', 'begin-environment' ) , - "end-environment" : recurse('cmdafter-but-not-]', 'end-environment' ) , - "something-else-then-anything" : recurse('cmdafter-but-not-]', 'something-else-then-anything' ) , - "nothing" : () => empty , - } , - "ignore" : { "starts-with-a-space" : err('ignore', 'starts-with-a-space') , "starts-with-a-newline" : err('ignore', 'starts-with-a-newline') , @@ -526,4 +531,4 @@ export default { "nothing" : err('ignore', 'nothing') , } , -} ; +} ) ; diff --git a/src/transform/visitor.js b/src/transform/visitor.js new file mode 100644 index 0000000..e0194fb --- /dev/null +++ b/src/transform/visitor.js @@ -0,0 +1,34 @@ +import { ast } from '@aureooms/js-grammar' ; + +import grammar from '../grammar' ; + +const t = ast.transform ; +const cmap = ast.cmap ; +const recurse = ( nonterminal , production ) => ( tree , match , ctx ) => ({ + "type" : "node" , + nonterminal , + production , + "children" : cmap( async x => x.type === 'leaf' ? x : await t( x , match , ctx ) , tree.children ) , +}) ; + + +// move to @js-grammar/ast.visitor +function generateVisitor ( grammar ) { + + const transform = { } ; + + for ( const [ nonterminal , productions ] of grammar.productions.entries() ) { + const nonterminalTransform = { } ; + + for ( const production of productions.keys() ) { + nonterminalTransform[production] = recurse( nonterminal , production ) ; + } + + transform[nonterminal] = nonterminalTransform ; + } + + return transform ; + +} + +export default generateVisitor( grammar ) ; diff --git a/test/src/all.js b/test/src/all.js index 28dc3db..df1a8a3 100644 --- a/test/src/all.js +++ b/test/src/all.js @@ -22,12 +22,12 @@ async function transform ( t , string , expected ) { function throws ( t , string , expected ) { const out = { 'write' : buffer => undefined } ; - //await t.throws(async () => await shakestring(string, out), expected); - return shakestring(string, out) - .then( () => t.fail() ) - .catch( error => { - t.true(expected.test(error.message)); - } ) ; + return t.throwsAsync(shakestring(string, out), expected); + //return shakestring(string, out) + //.then( () => t.fail() ) + //.catch( error => { + //t.true(expected.test(error.message)); + //} ) ; } const immutable = async ( t , string ) => await transform( t , string , string ) ; @@ -155,11 +155,20 @@ test( 'Complex displaymath environment (matrix)' , immutable , '\\begin{displaym test( 'Escaped newline' , immutable , 'a\\\nb' ) ; // incomplete arg number -test( throws , '#' , /Incomplete #/ ) ; -test( throws , '#x' , /Incomplete #/ ) ; +test( throws , '#' , /unexpected end of file/ ) ; +test( throws , '#x' , /1:2/ ) ; // no arguments defined -test( throws , '#1' , /no arguments in context/ ) ; +test( throws , '##' , /Escaped hash \(##\) without argument context/ ) ; +test( throws , '#1' , /#1 without argument context/ ) ; +test( throws , '#2' , /#2 without argument context/ ) ; +test( throws , '#3' , /#3 without argument context/ ) ; +test( throws , '#4' , /#4 without argument context/ ) ; +test( throws , '#5' , /#5 without argument context/ ) ; +test( throws , '#6' , /#6 without argument context/ ) ; +test( throws , '#7' , /#7 without argument context/ ) ; +test( throws , '#8' , /#8 without argument context/ ) ; +test( throws , '#9' , /#9 without argument context/ ) ; // escaped # test( immutable , '\\#' ) ; @@ -237,7 +246,8 @@ for ( const filename of transformedInputFiles ) test( transformFile , `${transformedInputFiledir}/${filename}` , `${transformedOutputFiledir}/${filename}` ) ; // argument escaping -//test( transform , '\\newcommand\\x[1]{\\def\\#1[1]{##1}}\\x{test}' , '\\def\\test[1]{#1}' ) ; +test( transform , '\\newcommand\\x[1]{\\newcommand\\y[1]{#1 ##1}}\\x{test}\\y{1212}' , 'test 1212' ) ; +test( throws , '\\def\\y{##1}\\newcommand\\x[1]{\\y}\\x{test}' , /#1 without argument context/ ) ; // default arguments with newcommand and renewcommand test( transform , '\\newcommand{\\price}[2][17.5]{\\pounds #2 excl VAT @ #1\\%}\\price{100}' , '\\pounds 100 excl VAT @ 17.5\\%') ;