From bd5004c23257250391d318ec566f1b0d0a8440a9 Mon Sep 17 00:00:00 2001 From: Andrew Ray Date: Sun, 2 Apr 2023 11:28:21 -0700 Subject: [PATCH] Test updates and minor refactor of location information - Fixes a typo in the readme - Removes the weird object type from grammarSource - Refactors scope location information into some functions - Minor cleanup in parser file (use node() for entrypoint) - Adds tests for scoping - Updates tests to print peggy formatted errors, including for parser generation - Bumps library version --- README.md | 4 +- package-lock.json | 36 ++++++---- package.json | 2 +- src/parser/glsl-grammar.pegjs | 126 ++++++++++++++++++---------------- src/parser/parse.test.ts | 119 +++++++++++++++++++++----------- src/parser/parser.d.ts | 2 +- 6 files changed, 171 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index 6006f1b..b09b04d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ The [Shaderfrog](https://shaderfrog.com/app) GLSL compiler is an open source GLSL 1.00 and 3.00 parser and preprocessor that compiles [back to -GLSL](parser/generator.ts). Both the parser and preprocessor can preserve +GLSL](src/parser/generator.ts). Both the parser and preprocessor can preserve comments and whitespace. The parser uses PEG grammar via the Peggy Javascript library. The PEG grammars @@ -49,7 +49,7 @@ Where `options` is: // The origin of the GLSL, for debugging. For example, "main.js", If the // parser raises an error (specifically a GrammarError), and you call // error.format([]) on it, the error shows { source: 'main.js', ... } - grammarSource: string | object, + grammarSource: string, // If true, sets location information on each AST node, in the form of // { column: number, line: number, offset: number } includeLocation: boolean diff --git a/package-lock.json b/package-lock.json index 29c2ac0..8086bdd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@shaderfrog/glsl-parser", - "version": "0.4.3", + "version": "1.3.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@shaderfrog/glsl-parser", - "version": "0.4.3", + "version": "1.3.0", "license": "ISC", "devDependencies": { "@babel/core": "^7.15.5", @@ -2496,14 +2496,24 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001261", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001261.tgz", - "integrity": "sha512-vM8D9Uvp7bHIN0fZ2KQ4wnmYFpJo/Etb4Vwsuc+ka0tfGDHvOPrFm6S/7CCNLSOkAUjenT2HnUPESdOIL91FaA==", - "dev": true, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - } + "version": "1.0.30001473", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001473.tgz", + "integrity": "sha512-ewDad7+D2vlyy+E4UJuVfiBsU69IL+8oVmTuZnH5Q6CIUbxNfI50uVpRHbUPDD6SUaN2o0Lh4DhTrvLG/Tn1yg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ] }, "node_modules/chalk": { "version": "4.1.1", @@ -7215,9 +7225,9 @@ "dev": true }, "caniuse-lite": { - "version": "1.0.30001261", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001261.tgz", - "integrity": "sha512-vM8D9Uvp7bHIN0fZ2KQ4wnmYFpJo/Etb4Vwsuc+ka0tfGDHvOPrFm6S/7CCNLSOkAUjenT2HnUPESdOIL91FaA==", + "version": "1.0.30001473", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001473.tgz", + "integrity": "sha512-ewDad7+D2vlyy+E4UJuVfiBsU69IL+8oVmTuZnH5Q6CIUbxNfI50uVpRHbUPDD6SUaN2o0Lh4DhTrvLG/Tn1yg==", "dev": true }, "chalk": { diff --git a/package.json b/package.json index f0593da..f1aedb6 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "engines": { "node": ">=16" }, - "version": "1.3.0", + "version": "1.4.0", "description": "A GLSL ES 1.0 and 3.0 parser and preprocessor that can preserve whitespace and comments", "scripts": { "prepare": "npm run build && ./prepublish.sh", diff --git a/src/parser/glsl-grammar.pegjs b/src/parser/glsl-grammar.pegjs index ef95942..0c014a8 100644 --- a/src/parser/glsl-grammar.pegjs +++ b/src/parser/glsl-grammar.pegjs @@ -6,14 +6,6 @@ // https://github.com/pegjs/pegjs/issues/187 const OPEN_CURLY = String.fromCharCode(123); - const makeScope = (name, parent) => ({ - name, - parent, - bindings: {}, - types: {}, - functions: {}, - }); - // Types (aka struct) scope const addTypes = (scope, ...types) => { types.forEach(([identifier, type]) => { @@ -293,7 +285,17 @@ // Per-parse initializations { - // location() (and etc. functions) are not available in global scope, + const getLocation = (loc) => { + // Try to avoid calling getLocation() more than neccessary + if(!options.includeLocation) { + return; + } + // Intentionally drop the "source" and "offset" keys from the location object + const { start, end } = loc || location(); + return { start, end }; + } + + // getLocation() (and etc. functions) are not available in global scope, // so node() is moved to per-parse scope const node = (type, attrs) => { const n = { @@ -301,12 +303,50 @@ ...attrs, } if(options.includeLocation) { - const { start, end } = location(); - n.location = { start, end } + n.location = getLocation(); } return n; }; + const makeScope = (name, parent, startLocation) => { + let newLocation = getLocation(startLocation); + + return { + name, + parent, + ...(newLocation ? { location: newLocation } : false), + bindings: {}, + types: {}, + functions: {}, + }; + }; + + const warn = (...args) => !options.quiet && console.warn(...args); + + let scope = makeScope('global'); + let scopes = [scope]; + + const pushScope = scope => { + // console.log('pushing scope at ',text()); + scopes.push(scope); + return scope; + }; + const popScope = scope => { + // console.log('popping scope at ',text()); + if(!scope.parent) { + throw new Error('popped bad scope', scope, 'at', text()); + } + return scope.parent; + }; + const setScopeEnd = (scope, end) => { + if(options.includeLocation) { + if(!scope.location) { + console.error('no end location at', text()); + } + scope.location.end = end; + } + }; + // Group the statements in a switch statement into cases / default arrays const groupCases = (statements) => statements.reduce((cases, stmt) => { const partial = stmt.partial || {}; @@ -349,37 +389,15 @@ }]; } }, []); - - const warn = (...args) => !options.quiet && console.warn(...args); - - let scope = makeScope('global'); - let scopes = [scope]; - - const pushScope = scope => { - // console.log('pushing scope at ',text()); - - if(options.includeLocation){ - let { start } = location(); - scope.location = { start }; - } - - scopes.push(scope); - return scope; - }; - const popScope = scope => { - // console.log('popping scope at ',text()); - if(!scope.parent) { - throw new Error('popped bad scope', scope, 'at', text()); - } - return scope.parent; - }; } -// Extra whitespace here at start is to help with screenshots by adding -// extra linebreaks +// Entrypoint to parsing! start = wsStart:_ program:translation_unit { - return { type: 'program', wsStart, program, scopes }; + // Set the global scope end to the end of the program + setScopeEnd(scope, getLocation()?.end); + return node('program', { wsStart, program, scopes }); } + // "compatibility profile only and vertex language only; same as in when in a // vertex shader" ATTRIBUTE = token:"attribute" t:terminal { return node('keyword', { token, whitespace: t }); } @@ -1080,7 +1098,7 @@ function_header_new_scope "function header" 'function_header', { returnType, name, lp } ); - scope = pushScope(makeScope(name.identifier, scope)); + scope = pushScope(makeScope(name.identifier, scope, lp.location)); return n; } @@ -1407,12 +1425,9 @@ compound_statement = }) statements:statement_list? rb:RIGHT_BRACE { - - if(options.includeLocation){ - // using start of right bracket, so trailing whitespace is not counted towards scope range - let end = rb.location.start; - scope.location.end = end; - } + // Use start of right bracket, so trailing whitespace is not counted towards + // scope range + setScopeEnd(scope, rb.location?.start); scope = popScope(scope); @@ -1505,12 +1520,9 @@ iteration_statement "iteration statement" condition:condition rp:RIGHT_PAREN body:statement_no_new_scope { - - if(options.includeLocation){ - // use right bracket or fallback to location.end - let end = body.rb ? body.rb.location.start : body.location.end; - scope.location.end = end; - } + // use right bracket or fallback to location.end + const end = body.rb ? body.rb.location?.start : body.location?.end; + setScopeEnd(scope, end); scope = popScope(scope); @@ -1561,11 +1573,8 @@ iteration_statement "iteration statement" operation:expression? rp:RIGHT_PAREN body:statement_no_new_scope { - - if(options.includeLocation){ - let end = body.rb ? body.rb.location.start : body.location.end; - scope.location.end = end; - } + const end = body.rb ? body.rb.location?.start : body.location?.end; + setScopeEnd(scope, end); scope = popScope(scope); @@ -1669,10 +1678,7 @@ function_definition = const n = node('function', { prototype, body }); - if(options.includeLocation){ - let end = body.rb.location.start; - scope.location.end = end; - } + setScopeEnd(scope, body.rb.location?.start); scope = popScope(scope); diff --git a/src/parser/parse.test.ts b/src/parser/parse.test.ts index e2b1e82..f06038a 100644 --- a/src/parser/parse.test.ts +++ b/src/parser/parse.test.ts @@ -10,12 +10,31 @@ import { preprocessAst } from '../preprocessor/preprocessor'; import generatePreprocess from '../preprocessor/generator'; const fileContents = (filePath: string) => fs.readFileSync(filePath).toString(); +const inspect = (arg: any) => console.log(util.inspect(arg, false, null, true)); + +// Most of this ceremony around building a parser is dealing with Peggy's error +// format() function, where the grammarSource has to line up in generate() and +// format() to get nicely formatted errors if there's a syntax error in the +// grammar +const buildParser = (file: string) => { + const grammar = fileContents(file); + try { + return peggy.generate(grammar, { + grammarSource: file, + cache: true, + }); + } catch (e) { + const err = e as SyntaxError; + if ('format' in err && typeof err.format === 'function') { + console.error(err.format([{ source: file, text: grammar }])); + } + throw e; + } +}; -// Preprocessor setup -const preprocessorGrammar = fileContents( +const preprocessParser = buildParser( './src/preprocessor/preprocessor-grammar.pegjs' ); -const preprocessParser = peggy.generate(preprocessorGrammar, { cache: true }); const preprocess = (program: string) => { const ast = preprocessParser.parse(program, { grammarSource: program }); @@ -39,10 +58,9 @@ const debugScopes = (scopes: Scope[]) => functions: debugEntry(s.functions), })); -const grammar = fileContents('./src/parser/glsl-grammar.pegjs'); const testFile = fileContents('./src/parser/glsltest.glsl'); -const parser = peggy.generate(grammar, { cache: true }) as Parser; +const parser = buildParser('./src/parser/glsl-grammar.pegjs'); const middle = /\/\* start \*\/((.|[\r\n])+)(\/\* end \*\/)?/m; @@ -63,25 +81,14 @@ const parseSrc = (src: string, options: ParserOptions = {}) => { } }; -const debugProgram = (program: string) => { - debugAst(parseSrc(program).program); -}; - -const debugAst = (ast: AstNode | AstNode[]) => { - console.log(util.inspect(ast, false, null, true)); +const debugSrc = (src: string) => { + inspect(parseSrc(src).program); }; const debugStatement = (stmt: AstNode) => { const program = `void main() {/* start */${stmt}/* end */}`; - const ast = parser.parse(program); - console.log( - util.inspect( - (ast.program[0] as FunctionNode).body.statements[0], - false, - null, - true - ) - ); + const ast = parseSrc(program); + inspect((ast.program[0] as FunctionNode).body.statements[0]); }; const expectParsedStatement = (src: string, options = {}) => { @@ -89,7 +96,7 @@ const expectParsedStatement = (src: string, options = {}) => { const ast = parseSrc(program, options); const glsl = generate(ast); if (glsl !== program) { - console.log(util.inspect(ast.program[0], false, null, true)); + inspect(ast.program[0]); // @ts-ignore expect(glsl.match(middle)[1]).toBe(src); } @@ -97,20 +104,20 @@ const expectParsedStatement = (src: string, options = {}) => { const parseStatement = (src: string, options: ParserOptions = {}) => { const program = `void main() {${src}}`; - return parser.parse(program, options); + return parseSrc(program, options); }; const expectParsedProgram = (src: string, options: ParserOptions = {}) => { const ast = parseSrc(src, options); const glsl = generate(ast); if (glsl !== src) { - console.log(util.inspect(ast, false, null, true)); + inspect(ast); expect(glsl).toBe(src); } }; test('scope bindings and type names', () => { - const ast = parser.parse(` + const ast = parseSrc(` float a, b = 1.0, c = a; vec2 texcoord1, texcoord2; vec3 position; @@ -150,7 +157,7 @@ coherent buffer Block { }); test('scope references', () => { - const ast = parser.parse(` + const ast = parseSrc(` float a, b = 1.0, c = a; mat2x2 myMat = mat2( vec2( 1.0, 0.0 ), vec2( 0.0, 1.0 ) ); struct { @@ -302,12 +309,12 @@ test('for loops', () => { test('switch error', () => { // Test the semantic analysis case expect(() => - parseStatement( - ` + parser.parse( + `void main() { switch (easingId) { result = cubicIn(); } - `, +}`, { quiet: true } ) ).toThrow(/must start with a case or default label/); @@ -446,7 +453,6 @@ test('postfix, unary, binary expressions', () => { }); test('parses a test file', () => { - // console.log(debugProgram(preprocess(testFile))); expectParsedProgram(preprocess(testFile)); }); @@ -539,7 +545,7 @@ test('subroutines', () => { }); test('struct constructor', () => { - const ast = parser.parse(` + const ast = parseSrc(` struct light { float intensity; vec3 position; @@ -550,7 +556,7 @@ light lightVar = light(3.0, vec3(1.0, 2.0, 3.0)); }); test('overloaded scope test', () => { - const ast = parser.parse(` + const ast = parseSrc(` vec4 overloaded(vec4 x) { return x; } @@ -562,7 +568,7 @@ float overloaded(float x) { test('overriding glsl builtin function', () => { // "noise" is a built-in GLSL function that should be identified and renamed - const ast = parser.parse(` + const ast = parseSrc(` float noise() {} float fn() { uv += noise(); @@ -579,7 +585,7 @@ float fn_FUNCTION() { }); test('rename bindings and functions', () => { - const ast = parser.parse( + const ast = parseSrc( ` float a, b = 1.0, c = a; mat2x2 myMat = mat2( vec2( 1.0, 0.0 ), vec2( 0.0, 1.0 ) ); @@ -674,7 +680,7 @@ vec4 linearToOutputTexel_FUNCTION( vec4 value ) { return LinearToLinear_FUNCTION }); test('detecting struct scope and usage', () => { - const ast = parser.parse(` + const ast = parseSrc(` struct StructName { vec3 color; }; @@ -701,7 +707,7 @@ void main() { }); test('fn args shadowing global scope identified as separate bindings', () => { - const ast = parser.parse(` + const ast = parseSrc(` attribute vec3 position; vec3 func(vec3 position) { return position; @@ -718,7 +724,7 @@ vec3 func(vec3 position) { }); test('I do not yet know what to do with layout()', () => { - const ast = parser.parse(` + const ast = parseSrc(` layout(std140,column_major) uniform; float a; uniform Material @@ -737,14 +743,45 @@ uniform vec2 vProp; };`); }); -test('Parser locations', () => { +test('Locations with location disabled', () => { const src = `void main() {}`; - let ast = parseSrc(src); + const ast = parseSrc(src); // default argument is no location information expect(ast.program[0].location).toBe(undefined); + expect(ast.scopes[0].location).toBe(undefined); +}); + +test('Parser locations', () => { + const src = `// Some comment +void main() { + float x = 1.0; - ast = parseSrc(src, { includeLocation: true }); + { + float x = 1.0; + } +}`; + const ast = parseSrc(src, { includeLocation: true }); + // The main fn location should start at "void" expect(ast.program[0].location).toStrictEqual({ - start: { column: 1, line: 1, offset: 0 }, - end: { column: 15, line: 1, offset: 14 }, + start: { line: 2, column: 1, offset: 16 }, + end: { line: 8, column: 2, offset: 76 }, + }); + + // The global scope is the entire program + expect(ast.scopes[0].location).toStrictEqual({ + start: { line: 1, column: 1, offset: 0 }, + end: { line: 8, column: 2, offset: 76 }, + }); + + // The scope created by the main fn should start at the open paren of the fn + // header, because fn scopes include fn arguments + expect(ast.scopes[1].location).toStrictEqual({ + start: { line: 2, column: 10, offset: 25 }, + end: { line: 8, column: 1, offset: 75 }, + }); + + // The inner compound statement { scope } + expect(ast.scopes[2].location).toStrictEqual({ + start: { line: 5, column: 3, offset: 50 }, + end: { line: 7, column: 3, offset: 73 }, }); }); diff --git a/src/parser/parser.d.ts b/src/parser/parser.d.ts index 6bc1425..123933d 100644 --- a/src/parser/parser.d.ts +++ b/src/parser/parser.d.ts @@ -2,7 +2,7 @@ import type { AstNode, Program } from '../ast'; export type ParserOptions = Partial<{ quiet: boolean; - grammarSource: string | object; + grammarSource: string; includeLocation: boolean; }>;