fix windows line ending (\r\n) parse errors

irmen · Apr 4, 2021 · b4700af · b4700af
1 parent 374e2b3
commit b4700af
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 6 deletions.
diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleParsing.kt
@@ -41,8 +41,8 @@ class ModuleImporter {
         if(!Files.isReadable(filePath))
             throw ParsingFailedError("No such file: $filePath")
 
-        val input = CharStreams.fromPath(filePath)
-        return importModule(program, input, filePath, false, encoder, compilationTargetName)
+        val content = Files.readAllBytes(filePath).toString(Charsets.UTF_8).replace("\r\n", "\n")
+        return importModule(program, CharStreams.fromString(content), filePath, false, encoder, compilationTargetName)
     }
 
     fun importLibraryModule(program: Program, name: String,
@@ -122,7 +122,8 @@ class ModuleImporter {
                     val (resource, resourcePath) = rsc
                     resource.use {
                         println("importing '$moduleName' (library)")
-                        importModule(program, CharStreams.fromStream(it), Paths.get("@embedded@/$resourcePath"),
+                        val content = it.readAllBytes().toString(Charsets.UTF_8).replace("\r\n", "\n")
+                        importModule(program, CharStreams.fromString(content), Paths.get("@embedded@/$resourcePath"),
                             true, encoder, compilationTargetName)
                     }
                 } else {

diff --git a/parser/antlr/prog8.g4 b/parser/antlr/prog8.g4
@@ -5,6 +5,7 @@ NOTES:
 
 - whitespace is ignored. (tabs/spaces)
 - every position can be empty, be a comment, or contain ONE statement.
+- input is assumed to be a text file with UNIX line endings (\n).
 
 */
 
@@ -14,10 +15,10 @@ grammar prog8;
 package prog8.parser;
 }
 
-LINECOMMENT : [\r\n][ \t]* COMMENT -> channel(HIDDEN);
-COMMENT :  ';' ~[\r\n]* -> channel(HIDDEN) ;
+LINECOMMENT : [\n][ \t]* COMMENT -> channel(HIDDEN);
+COMMENT :  ';' ~[\n]* -> channel(HIDDEN) ;
 WS :  [ \t] -> skip ;
-EOL :  [\r\n]+ ;
+EOL :  [\n]+ ;
 // WS2 : '\\' EOL -> skip;
 VOID: 'void';
 NAME :  [a-zA-Z][a-zA-Z0-9_]* ;