Skip to content

Commit

Permalink
Multiple changes: (Lots of test coverage with barely any test code, b…
Browse files Browse the repository at this point in the history 
…ugfix for escaped newlines after invocations, backwards compatibility for <= >= != operators, semantic compression on lexer)
  • Loading branch information
@byxor
byxor committed Sep 3, 2020
1 parent b314d8c commit e35acbb
Show file tree
Hide file tree
Showing 7 changed files with 559 additions and 309 deletions.
6 changes: 3 additions & 3 deletions cmd/ns/ns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ const (
The QB programming language.
----------------------------------------------------
`
version = "0.2"
version = "0.3"
)

func main() {
Expand All @@ -42,9 +42,9 @@ func RunNeverscript(arguments CommandLineArguments) {

fmt.Printf("\nCompiling '%s' (may freeze)...\n", *arguments.FileToCompile)
var lexer compiler.Lexer
var newParser compiler.NewParser
var parser compiler.Parser
var bytecodeCompiler compiler.BytecodeCompiler
compiler.Compile(*arguments.FileToCompile, outputFilename, &lexer, &newParser, &bytecodeCompiler)
compiler.Compile(*arguments.FileToCompile, outputFilename, &lexer, &parser, &bytecodeCompiler)
fmt.Printf(" Created '%s'.\n\n", outputFilename)

if *arguments.ShowHexDump {
Expand Down
2 changes: 2 additions & 0 deletions compiler/ast.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ const (
AstKind_LessThanExpression
AstKind_LessThanEqualsExpression
AstKind_EqualsExpression
AstKind_NotEqualExpression
AstKind_DotExpression
AstKind_ColonExpression
AstKind_UnaryExpression
Expand Down Expand Up @@ -77,6 +78,7 @@ func (astKind AstKind) String() string {
"AstKind_LessThanExpression",
"AstKind_LessThanEqualsExpression",
"AstKind_EqualsExpression",
"AstKind_NotEqualExpression",
"AstKind_DotExpression",
"AstKind_ColonExpression",
"AstKind_UnaryExpression",
Expand Down
12 changes: 6 additions & 6 deletions compiler/compiler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"strings"
)

func Compile(nsFilePath, qbFilePath string, lexer *Lexer, newParser *NewParser, bytecodeCompiler *BytecodeCompiler) {
func Compile(nsFilePath, qbFilePath string, lexer *Lexer, parser *Parser, bytecodeCompiler *BytecodeCompiler) {
{ // read source code into memory & store it in lexer
bytes, err := ioutil.ReadFile(nsFilePath)
if err != nil {
Expand All @@ -22,13 +22,13 @@ func Compile(nsFilePath, qbFilePath string, lexer *Lexer, newParser *NewParser,

LexSourceCode(lexer)

newParser.Tokens = lexer.Tokens
BuildAbstractSyntaxTree(newParser)
if !newParser.Result.WasSuccessful {
log.Fatal(newParser.Result.Reason)
parser.Tokens = lexer.Tokens
BuildAbstractSyntaxTree(parser)
if !parser.Result.WasSuccessful {
log.Fatal(parser.Result.Reason)
}

bytecodeCompiler.RootAstNode = newParser.Result.Node
bytecodeCompiler.RootAstNode = parser.Result.Node
GenerateBytecode(bytecodeCompiler)

ioutil.WriteFile(qbFilePath, bytecodeCompiler.Bytes, 0644)
Expand Down
220 changes: 105 additions & 115 deletions compiler/lexer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,11 @@ type Lexer struct {
Tokens []Token
NumTokens int

ScanMode ScanMode
StartOfIdentifier int
StartOfInteger int
StartOfString int
}

type ScanMode int

const (
ScanMode_Default ScanMode = iota
)

func LexSourceCode(lexer *Lexer) { // do lexical analysis (build an array of Tokens)

CanFindKeywordAtIndex := func(keyword string, index int) bool {
Expand Down Expand Up @@ -259,7 +252,6 @@ func LexSourceCode(lexer *Lexer) { // do lexical analysis (build an array of Tok

lexer.Tokens = make([]Token, 6500)
lexer.LineNumber = 1
lexer.ScanMode = ScanMode_Default
for {
if lexer.Index >= lexer.SourceCodeSize {
break
Expand All @@ -284,114 +276,112 @@ func LexSourceCode(lexer *Lexer) { // do lexical analysis (build an array of Tok
SaveToken(lexer, TokenKind_RawChecksum, data)
lexer.Index += len(data)
} else {
switch lexer.ScanMode {
case ScanMode_Default:
// Check for single-character tokens
switch lexer.SourceCode[lexer.Index] {
case '\t':
fallthrough
case ' ':
lexer.Index++
case '\r':
fallthrough
case '\n':
lexer.LineNumber++
SaveToken(lexer, TokenKind_NewLine, "\\n")
lexer.Index++
case '=':
SaveToken(lexer, TokenKind_Equals, "=")
lexer.Index++
case '[':
SaveToken(lexer, TokenKind_LeftSquareBracket, "[")
lexer.Index++
case ']':
SaveToken(lexer, TokenKind_RightSquareBracket, "]")
lexer.Index++
case '{':
SaveToken(lexer, TokenKind_LeftCurlyBrace, "{")
lexer.Index++
case '}':
SaveToken(lexer, TokenKind_RightCurlyBrace, "}")
lexer.Index++
case '(':
SaveToken(lexer, TokenKind_LeftParenthesis, "(")
lexer.Index++
case ')':
SaveToken(lexer, TokenKind_RightParenthesis, ")")
lexer.Index++
case '<':
SaveToken(lexer, TokenKind_LeftAngleBracket, "<")
lexer.Index++
case '>':
SaveToken(lexer, TokenKind_RightAngleBracket, ">")
lexer.Index++
case '+':
SaveToken(lexer, TokenKind_Plus, "+")
lexer.Index++
case '-':
SaveToken(lexer, TokenKind_Minus, "-")
lexer.Index++
case '*':
SaveToken(lexer, TokenKind_Asterisk, "*")
lexer.Index++
case '/':
SaveToken(lexer, TokenKind_ForwardSlash, "/")
lexer.Index++
case '\\':
SaveToken(lexer, TokenKind_BackwardSlash, "\\")
lexer.Index++
case ',':
SaveToken(lexer, TokenKind_Comma, ",")
lexer.Index++
case '.':
SaveToken(lexer, TokenKind_Dot, ".")
lexer.Index++
case '!':
SaveToken(lexer, TokenKind_Bang, "!")
lexer.Index++
case ':':
SaveToken(lexer, TokenKind_Colon, ":")
lexer.Index++
default:
// Check for multi-character tokens
if CanFindKeyword("or") {
SaveToken(lexer, TokenKind_Or, "or")
lexer.Index += 2
} else if CanFindKeyword("if") {
SaveToken(lexer, TokenKind_If, "if")
lexer.Index += 2
} else if CanFindKeyword("and") {
SaveToken(lexer, TokenKind_And, "and")
lexer.Index += 3
} else if CanFindKeyword("else") {
SaveToken(lexer, TokenKind_Else, "else")
lexer.Index += 4
} else if CanFindKeyword("while") {
SaveToken(lexer, TokenKind_While, "while")
lexer.Index += 5
} else if CanFindKeyword("break") {
SaveToken(lexer, TokenKind_Break, "break")
} else if CanFindKeyword("script") {
SaveToken(lexer, TokenKind_Script, "script")
lexer.Index += 6
} else if CanFindKeyword("random") {
SaveToken(lexer, TokenKind_Random, "random")
lexer.Index += 6
} else if CanFindKeyword("return") {
SaveToken(lexer, TokenKind_Return, "return")
lexer.Index += 6
} else if identifier, found := CanFindIdentifier(); found {
SaveToken(lexer, TokenKind_Identifier, identifier)
lexer.Index += len(identifier)
} else {
fmt.Println("Lexer failed")
fmt.Println("'" + string(lexer.SourceCode[lexer.Index:lexer.Index+10]) + "'...")
for i, token := range lexer.Tokens {
if i >= lexer.NumTokens {
break
}
fmt.Printf("%+v\n", token)
// Check for single-character tokens
switch lexer.SourceCode[lexer.Index] {
case '\t':
fallthrough
case ' ':
lexer.Index++
case '\r':
fallthrough
case '\n':
lexer.LineNumber++
SaveToken(lexer, TokenKind_NewLine, "\\n")
lexer.Index++
case '=':
SaveToken(lexer, TokenKind_Equals, "=")
lexer.Index++
case '[':
SaveToken(lexer, TokenKind_LeftSquareBracket, "[")
lexer.Index++
case ']':
SaveToken(lexer, TokenKind_RightSquareBracket, "]")
lexer.Index++
case '{':
SaveToken(lexer, TokenKind_LeftCurlyBrace, "{")
lexer.Index++
case '}':
SaveToken(lexer, TokenKind_RightCurlyBrace, "}")
lexer.Index++
case '(':
SaveToken(lexer, TokenKind_LeftParenthesis, "(")
lexer.Index++
case ')':
SaveToken(lexer, TokenKind_RightParenthesis, ")")
lexer.Index++
case '<':
SaveToken(lexer, TokenKind_LeftAngleBracket, "<")
lexer.Index++
case '>':
SaveToken(lexer, TokenKind_RightAngleBracket, ">")
lexer.Index++
case '+':
SaveToken(lexer, TokenKind_Plus, "+")
lexer.Index++
case '-':
SaveToken(lexer, TokenKind_Minus, "-")
lexer.Index++
case '*':
SaveToken(lexer, TokenKind_Asterisk, "*")
lexer.Index++
case '/':
SaveToken(lexer, TokenKind_ForwardSlash, "/")
lexer.Index++
case '\\':
SaveToken(lexer, TokenKind_BackwardSlash, "\\")
lexer.Index++
case ',':
SaveToken(lexer, TokenKind_Comma, ",")
lexer.Index++
case '.':
SaveToken(lexer, TokenKind_Dot, ".")
lexer.Index++
case '!':
SaveToken(lexer, TokenKind_Bang, "!")
lexer.Index++
case ':':
SaveToken(lexer, TokenKind_Colon, ":")
lexer.Index++
default:
// Check for multi-character tokens
if CanFindKeyword("or") {
SaveToken(lexer, TokenKind_Or, "or")
lexer.Index += 2
} else if CanFindKeyword("if") {
SaveToken(lexer, TokenKind_If, "if")
lexer.Index += 2
} else if CanFindKeyword("and") {
SaveToken(lexer, TokenKind_And, "and")
lexer.Index += 3
} else if CanFindKeyword("else") {
SaveToken(lexer, TokenKind_Else, "else")
lexer.Index += 4
} else if CanFindKeyword("while") {
SaveToken(lexer, TokenKind_While, "while")
lexer.Index += 5
} else if CanFindKeyword("break") {
SaveToken(lexer, TokenKind_Break, "break")
lexer.Index += 5
} else if CanFindKeyword("script") {
SaveToken(lexer, TokenKind_Script, "script")
lexer.Index += 6
} else if CanFindKeyword("random") {
SaveToken(lexer, TokenKind_Random, "random")
lexer.Index += 6
} else if CanFindKeyword("return") {
SaveToken(lexer, TokenKind_Return, "return")
lexer.Index += 6
} else if identifier, found := CanFindIdentifier(); found {
SaveToken(lexer, TokenKind_Identifier, identifier)
lexer.Index += len(identifier)
} else {
fmt.Println("Lexer failed")
fmt.Println("'" + string(lexer.SourceCode[lexer.Index:lexer.Index+10]) + "'...")
for i, token := range lexer.Tokens {
if i >= lexer.NumTokens {
break
}
fmt.Printf("%+v\n", token)
}
}
}
Expand Down

0 comments on commit e35acbb

Please sign in to comment.