Switch to a maintained fork of universalchardet

TypesettingTools · Feb 10, 2016 · 4531547 · 4531547
1 parent a315ce6
commit 4531547
Show file tree

Hide file tree

Showing 107 changed files with 154 additions and 18,516 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -41,3 +41,7 @@
 	path = vendor/fftw
 	url = https://github.com/Aegisub/fftw3.git
 	ignore = dirty
+[submodule "uchardet"]
+	path = vendor/uchardet
+	url = https://github.com/BYVoid/uchardet
+	ignore = dirty
diff --git a/.travis.yml b/.travis.yml
@@ -9,7 +9,7 @@ before_install:
   - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 60 --slave /usr/bin/g++ g++ /usr/bin/g++-4.8 --slave /usr/bin/gcov gcov /usr/bin/gcov-4.8
   - sudo pip install cpp-coveralls
 install:
-  - sudo apt-get install -y -qq libasound2-dev libfftw3-dev libhunspell-dev yasm libfribidi-dev libass-dev libffms2-dev libwxgtk3.0-dev libicu-dev luarocks
+  - sudo apt-get install -y -qq libasound2-dev libfftw3-dev libhunspell-dev yasm libfribidi-dev libass-dev libffms2-dev libwxgtk3.0-dev libicu-dev luarocks libuchardet-dev
   - sudo luarocks install busted > /dev/null
   - sudo luarocks install moonscript > /dev/null
   - sudo luarocks install uuid > /dev/null

diff --git a/Makefile.inc.in b/Makefile.inc.in
@@ -2,12 +2,13 @@
 # AEGISUB FEATURE SUPPORT
 #########################
 HAVE_ALSA       = @with_alsa@
+HAVE_FFMS2      = @with_ffms2@
 HAVE_HUNSPELL   = @with_hunspell@
+HAVE_LIBPULSE   = @with_libpulse@
 HAVE_OPENAL     = @with_openal@
 HAVE_OSS        = @with_oss@
 HAVE_PORTAUDIO  = @with_portaudio@
-HAVE_FFMS2      = @with_ffms2@
-HAVE_LIBPULSE   = @with_libpulse@
+HAVE_UCHARDET   = @with_uchardet@
 
 ##############
 # BUILD OUTPUT
@@ -83,7 +84,7 @@ CFLAGS_OPENAL      = @OPENAL_CFLAGS@
 CFLAGS_OSS         = @OSS_CFLAGS@
 CFLAGS_PORTAUDIO   = @PORTAUDIO_CFLAGS@
 CFLAGS_PTHREAD     = @PTHREAD_CFLAGS@
-CFLAGS_UCHARDET    = -I$(TOP)vendor/universalchardet
+CFLAGS_UCHARDET    = @UCHARDET_CFLAGS@
 
 LIBS_ALSA          = @ALSA_LIBS@
 LIBS_FFMS2         = @FFMS2_LIBS@
@@ -100,6 +101,7 @@ LIBS_LUA           = $(TOP)vendor/luajit/src/libluajit.a
 LIBS_OPENAL        = @OPENAL_LIBS@
 LIBS_PORTAUDIO     = @PORTAUDIO_LIBS@
 LIBS_PTHREAD       = @PTHREAD_LIBS@
+LIBS_UCHARDET      = @UCHARDET_LIBS@
 
 ifeq (yes, $(BUILD_DARWIN))
 LIBS_LUA           = $(TOP)vendor/luajit/src/libluajit-aegisub.so

diff --git a/acinclude.m4 b/acinclude.m4
@@ -1,5 +1,4 @@
 m4_include([m4macros/ac_agi.m4])
-m4_include([m4macros/ac_agi_mdcpucfg.m4])
 m4_include([m4macros/ac_flag.m4])
 m4_include([m4macros/agi_find_libheader.m4])
 m4_include([m4macros/ax_boost_base.m4])

diff --git a/build/libaegisub/libaegisub.vcxproj b/build/libaegisub/libaegisub.vcxproj
@@ -25,6 +25,7 @@
         NOMINMAX;
         _WIN32_WINNT=0x0501;
         _CRT_NONSTDC_NO_DEPRECATE;
+        WITH_UCHARDET;
         %(PreprocessorDefinitions)
       </PreprocessorDefinitions>
       <PrecompiledHeader>Use</PrecompiledHeader>

diff --git a/build/universalchardet/universalchardet.vcxproj b/build/universalchardet/universalchardet.vcxproj
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup Label="Globals">
     <ProjectGuid>{7B56955D-5162-4698-AA5B-47484EDC8783}</ProjectGuid>
@@ -8,7 +8,7 @@
   <!-- Aegisub project configuration -->
   <PropertyGroup Label="AegisubConfiguration">
     <AegisubProjectType>lib</AegisubProjectType>
-    <SrcDir>..\..\vendor\universalchardet\</SrcDir>
+    <SrcDir>..\..\vendor\uchardet\src\</SrcDir>
   </PropertyGroup>
 
   <ImportGroup Label="PropertySheets">
@@ -18,26 +18,37 @@
   <ItemDefinitionGroup>
     <ClCompile>
       <RuntimeTypeInfo>false</RuntimeTypeInfo>
+      <ExceptionHandling>false</ExceptionHandling>
+      <PreprocessorDefinitions>
+        _CRT_NONSTDC_NO_DEPRECATE;
+        %(PreprocessorDefinitions)
+      </PreprocessorDefinitions>
     </ClCompile>
+    <InstallHeader>
+      <HeaderRoot>$(SrcDir)</HeaderRoot>
+      <Destination>uchardet\</Destination>
+    </InstallHeader>
   </ItemDefinitionGroup>
 
   <!-- Source files -->
   <ItemGroup>
-    <None Include="$(SrcDir)Big5Freq.tab" />
-    <None Include="$(SrcDir)EUCKRFreq.tab" />
-    <None Include="$(SrcDir)EUCTWFreq.tab" />
-    <None Include="$(SrcDir)GB2312Freq.tab" />
-    <None Include="$(SrcDir)JISFreq.tab" />
+    <InstallHeader Include="$(SrcDir)uchardet.h" />
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="$(SrcDir)CharDistribution.cpp" />
     <ClCompile Include="$(SrcDir)JpCntx.cpp" />
-    <ClCompile Include="$(SrcDir)LangBulgarianModel.cpp" />
-    <ClCompile Include="$(SrcDir)LangCyrillicModel.cpp" />
-    <ClCompile Include="$(SrcDir)LangGreekModel.cpp" />
-    <ClCompile Include="$(SrcDir)LangHebrewModel.cpp" />
-    <ClCompile Include="$(SrcDir)LangHungarianModel.cpp" />
-    <ClCompile Include="$(SrcDir)LangThaiModel.cpp" />
+    <ClCompile Include="$(SrcDir)LangModels/LangArabicModel.cpp" />
+    <ClCompile Include="$(SrcDir)LangModels/LangBulgarianModel.cpp" />
+    <ClCompile Include="$(SrcDir)LangModels/LangEsperantoModel.cpp" />
+    <ClCompile Include="$(SrcDir)LangModels/LangFrenchModel.cpp" />
+    <ClCompile Include="$(SrcDir)LangModels/LangGermanModel.cpp" />
+    <ClCompile Include="$(SrcDir)LangModels/LangGreekModel.cpp" />
+    <ClCompile Include="$(SrcDir)LangModels/LangHebrewModel.cpp" />
+    <ClCompile Include="$(SrcDir)LangModels/LangHungarianModel.cpp" />
+    <ClCompile Include="$(SrcDir)LangModels/LangRussianModel.cpp" />
+    <ClCompile Include="$(SrcDir)LangModels/LangSpanishModel.cpp" />
+    <ClCompile Include="$(SrcDir)LangModels/LangThaiModel.cpp" />
+    <ClCompile Include="$(SrcDir)LangModels/LangTurkishModel.cpp" />
     <ClCompile Include="$(SrcDir)nsBig5Prober.cpp" />
     <ClCompile Include="$(SrcDir)nsCharSetProber.cpp" />
     <ClCompile Include="$(SrcDir)nsEscCharsetProber.cpp" />
@@ -55,15 +66,20 @@
     <ClCompile Include="$(SrcDir)nsSJISProber.cpp" />
     <ClCompile Include="$(SrcDir)nsUniversalDetector.cpp" />
     <ClCompile Include="$(SrcDir)nsUTF8Prober.cpp" />
+    <ClCompile Include="$(SrcDir)uchardet.cpp" />
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="$(SrcDir)Big5Freq.tab" />
+    <ClInclude Include="$(SrcDir)EUCKRFreq.tab" />
+    <ClInclude Include="$(SrcDir)EUCTWFreq.tab" />
+    <ClInclude Include="$(SrcDir)GB2312Freq.tab" />
+    <ClInclude Include="$(SrcDir)JISFreq.tab" />
     <ClInclude Include="$(SrcDir)CharDistribution.h" />
     <ClInclude Include="$(SrcDir)JpCntx.h" />
     <ClInclude Include="$(SrcDir)nsBig5Prober.h" />
     <ClInclude Include="$(SrcDir)nsCharSetProber.h" />
     <ClInclude Include="$(SrcDir)nsCodingStateMachine.h" />
     <ClInclude Include="$(SrcDir)nscore.h" />
-    <ClInclude Include="$(SrcDir)nsError.h" />
     <ClInclude Include="$(SrcDir)nsEscCharsetProber.h" />
     <ClInclude Include="$(SrcDir)nsEUCJPProber.h" />
     <ClInclude Include="$(SrcDir)nsEUCKRProber.h" />
@@ -78,10 +94,7 @@
     <ClInclude Include="$(SrcDir)nsSJISProber.h" />
     <ClInclude Include="$(SrcDir)nsUniversalDetector.h" />
     <ClInclude Include="$(SrcDir)nsUTF8Prober.h" />
-    <ClInclude Include="$(SrcDir)prcpucfg.h" />
     <ClInclude Include="$(SrcDir)prmem.h" />
-    <ClInclude Include="$(SrcDir)protypes.h" />
-    <ClInclude Include="$(SrcDir)prtypes.h" />
-    <ClInclude Include="$(SrcDir)xpcom-config.h" />
+    <ClInclude Include="$(SrcDir)uchardet.h" />
   </ItemGroup>
 </Project>
diff --git a/build/universalchardet/universalchardet.vcxproj.filters b/build/universalchardet/universalchardet.vcxproj.filters
@@ -14,48 +14,13 @@
       <Extensions>*.tab</Extensions>
     </Filter>
   </ItemGroup>
-  <ItemGroup>
-    <None Include="$(SrcDir)Big5Freq.tab">
-      <Filter>Frequency tables</Filter>
-    </None>
-    <None Include="$(SrcDir)JISFreq.tab">
-      <Filter>Frequency tables</Filter>
-    </None>
-    <None Include="$(SrcDir)EUCKRFreq.tab">
-      <Filter>Frequency tables</Filter>
-    </None>
-    <None Include="$(SrcDir)EUCTWFreq.tab">
-      <Filter>Frequency tables</Filter>
-    </None>
-    <None Include="$(SrcDir)GB2312Freq.tab">
-      <Filter>Frequency tables</Filter>
-    </None>
-  </ItemGroup>
   <ItemGroup>
     <ClCompile Include="$(SrcDir)CharDistribution.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
     <ClCompile Include="$(SrcDir)JpCntx.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="$(SrcDir)LangBulgarianModel.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="$(SrcDir)LangCyrillicModel.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="$(SrcDir)LangGreekModel.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="$(SrcDir)LangHebrewModel.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="$(SrcDir)LangHungarianModel.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="$(SrcDir)LangThaiModel.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="$(SrcDir)nsBig5Prober.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -107,6 +72,45 @@
     <ClCompile Include="$(SrcDir)nsUTF8Prober.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="$(SrcDir)LangModels/LangArabicModel.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SrcDir)LangModels/LangBulgarianModel.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SrcDir)LangModels/LangEsperantoModel.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SrcDir)LangModels/LangFrenchModel.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SrcDir)LangModels/LangGermanModel.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SrcDir)LangModels/LangGreekModel.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SrcDir)LangModels/LangHebrewModel.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SrcDir)LangModels/LangHungarianModel.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SrcDir)LangModels/LangRussianModel.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SrcDir)LangModels/LangSpanishModel.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SrcDir)LangModels/LangThaiModel.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SrcDir)LangModels/LangTurkishModel.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SrcDir)uchardet.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="$(SrcDir)CharDistribution.h">
@@ -127,9 +131,6 @@
     <ClInclude Include="$(SrcDir)nscore.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="$(SrcDir)nsError.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
     <ClInclude Include="$(SrcDir)nsEscCharsetProber.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -172,20 +173,26 @@
     <ClInclude Include="$(SrcDir)nsUTF8Prober.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="$(SrcDir)prcpucfg.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
     <ClInclude Include="$(SrcDir)prmem.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="$(SrcDir)protypes.h">
+    <ClInclude Include="$(SrcDir)uchardet.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="$(SrcDir)prtypes.h">
-      <Filter>Header Files</Filter>
+    <ClInclude Include="$(SrcDir)Big5Freq.tab">
+      <Filter>Frequency tables</Filter>
     </ClInclude>
-    <ClInclude Include="$(SrcDir)xpcom-config.h">
-      <Filter>Header Files</Filter>
+    <ClInclude Include="$(SrcDir)EUCKRFreq.tab">
+      <Filter>Frequency tables</Filter>
+    </ClInclude>
+    <ClInclude Include="$(SrcDir)EUCTWFreq.tab">
+      <Filter>Frequency tables</Filter>
+    </ClInclude>
+    <ClInclude Include="$(SrcDir)GB2312Freq.tab">
+      <Filter>Frequency tables</Filter>
+    </ClInclude>
+    <ClInclude Include="$(SrcDir)JISFreq.tab">
+      <Filter>Frequency tables</Filter>
     </ClInclude>
   </ItemGroup>
 </Project>
diff --git a/configure.ac b/configure.ac
@@ -38,9 +38,6 @@ AS_CASE([$host],
         [*-*-darwin*], [build_darwin="yes"],
         [*-*-linux*],  [build_linux="yes"])
 
-# Used for universalchardet.
-AC_AGI_MDCPUCFG($host)
-
 AC_SUBST(build_darwin)
 
 ########################
@@ -384,6 +381,7 @@ AGI_OPT_PKG(ffms2, [build without ffms2 A/V provider [auto]], [Enable FFMS2 supp
 
 AGI_OPT_PKG(fftw3, [build without fftw support [auto]], [Enable fftw support])
 AGI_OPT_PKG(hunspell, [build without hunspell spell-checking [auto]], [Enable Hunspell support])
+AGI_OPT_PKG(uchardet, [build without automatic character set detection [auto]], [Enable uchardet support])
 
 ######################################################
 # Debugging support
@@ -571,6 +569,7 @@ Audio Players
   PulseAudio:            $with_libpulse $libpulse_disabled
 
 Misc Packages
+  uchardet:              $with_uchardet $uchardet_disabled
   Hunspell:              $with_hunspell $hunspell_disabled
   FFTW3:                 $with_fftw3 $fftw3_disabled
 ])
diff --git a/header.mk b/header.mk
@@ -13,8 +13,7 @@ subdirs := \
 	tests \
 	tools \
 	vendor/luabins \
-	vendor/luajit \
-	vendor/universalchardet
+	vendor/luajit
 
 subdirs := $(addprefix $(TOP),$(addsuffix /Makefile,$(subdirs)))
 

diff --git a/libaegisub/Makefile b/libaegisub/Makefile
@@ -46,6 +46,7 @@ endif
 aegisub_PCH := $(d)lagi_pre.h
 aegisub_CPPFLAGS := -I$(d)include -I$(TOP) $(CPPFLAGS_BOOST) $(CFLAGS_LUA)
 
+$(d)common/charset.o_FLAGS := $(CFLAGS_UCHARDET)
 $(d)common/charset_conv.o_FLAGS := $(CFLAGS_ICONV)
 $(d)common/parser.o_FLAGS := -ftemplate-depth=256
 $(d)unix/path.o_FLAGS := -DP_DATA=\"$(P_DATA)\"