use sedlex for utf-8 aware lexing (closes HaxeFoundation#5163)

nadako · Apr 11, 2017 · 4941ec4 · 4941ec4
1 parent b218ae1
commit 4941ec4
Show file tree

Hide file tree

Showing 9 changed files with 462 additions and 447 deletions.
diff --git a/.merlin b/.merlin
@@ -2,3 +2,4 @@ S src/**
 B _build/src/**
 S libs/**
 B libs/**
+PKG sedlex
diff --git a/.travis.yml b/.travis.yml
@@ -56,7 +56,7 @@ install_osx: &install_osx
   - travis_retry brew install opam;
   - export OPAMYES=1
   - opam init
-  - opam install camlp4 ocamlfind
+  - opam install camlp4 sedlex ocamlfind
   - eval `opam config env`
   # Install neko
   - travis_retry brew install neko --HEAD;

diff --git a/Makefile b/Makefile
@@ -8,7 +8,7 @@
 #  - use 'make -f Makefile.win' to build for Windows
 #  - use 'make MSVC=1 -f Makefile.win' to build for Windows with OCaml/MSVC
 #
-.SUFFIXES : .ml .mli .cmo .cmi .cmx .mll .mly
+.SUFFIXES : .ml .mli .cmo .cmi .cmx .mly
 
 INSTALL_DIR=$(DESTDIR)/usr
 INSTALL_BIN_DIR=$(INSTALL_DIR)/bin
@@ -29,7 +29,7 @@ STATICLINK?=0
 
 HAXE_DIRECTORIES=compiler context generators generators/gencommon macro filters optimization syntax typing display
 EXTLIB_LIBS=extlib extc neko javalib ziplib swflib xml-light ttflib ilib objsize pcre
-FINDLIB_LIBS=unix str threads
+FINDLIB_LIBS=unix str threads sedlex
 
 # Includes, packages and compiler
 

diff --git a/appveyor.yml b/appveyor.yml
@@ -34,7 +34,7 @@ install:
     - 7z x "opam32.tar.xz" -so | 7z x -aoa -si -ttar
     - '%CYG_ROOT%/bin/bash -lc "cd \"$OLDPWD\" && bash opam32/install.sh"'
     - '%CYG_ROOT%/bin/bash -lc "opam init mingw \"https://github.com/fdopen/opam-repository-mingw.git\" --comp 4.02.3+mingw32c --switch 4.02.3+mingw32c --auto-setup --yes"'
-    - '%CYG_ROOT%/bin/bash -lc "opam install camlp4 ocamlfind --yes"'
+    - '%CYG_ROOT%/bin/bash -lc "opam install camlp4 sedlex ocamlfind --yes"'
     # Install neko
     - choco install neko --prerelease --ignore-dependencies -s 'https://ci.appveyor.com/nuget/neko' -y
     - choco install chocolatey-core.extension php --ignore-dependencies -y

diff --git a/compile.sh b/compile.sh
@@ -3,10 +3,6 @@ if [ ! -f "_build/src/syntax/parser.ml" -o "src/syntax/parser.mly" -nt "_build/s
 	camlp4o -impl src/syntax/parser.mly -o _build/src/syntax/parser.ml
 fi
 
-if [ ! -f "_build/src/syntax/lexer.ml" -o "src/syntax/lexer.mll" -nt "_build/src/syntax/lexer.ml" ]; then
-	ocamllex -o _build/src/syntax/lexer.ml src/syntax/lexer.mll
-fi
-
 if [ "$1" -eq "0" -a ! -f "_build/src/compiler/version.ml" ]; then
 	echo let version_extra = None > _build/src/compiler/version.ml
 fi