Nelson uses simdutf library (#718)

nelson-lang · Aug 29, 2022 · a1c2017 · a1c2017
1 parent 01a5762
commit a1c2017
Show file tree

Hide file tree

Showing 14 changed files with 27,821 additions and 78 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## 0.6.9 (UNRELEASED)
+
+### Changed
+
+- Nelson uses [`simdutf`](https://github.com/simdutf/simdutf) library to convert characters encoding faster.
+
 ## 0.6.8 (2022-08-27)
 
 ### Added

diff --git a/THIRDPARTY.md b/THIRDPARTY.md
@@ -328,3 +328,11 @@ A C++17 Thread Pool for High-Performance Scientific Computing
 > <https://github.com/bshoshany/thread-pool>
 
 > <https://github.com/bshoshany/thread-pool/blob/master/LICENSE.txt> MIT license
+
+- simdutf
+
+Unicode routines (UTF8, UTF16, UTF32): billions of characters per second using SSE2, AVX2, NEON, AVX-512.
+
+> <https://github.com/simdutf/simdutf>
+
+> <https://github.com/simdutf/simdutf/blob/master/LICENSE-MIT> MIT license
diff --git a/modules/characters_encoding/CMakeLists.txt b/modules/characters_encoding/CMakeLists.txt
@@ -11,10 +11,13 @@ set(module_name characters_encoding)
 set(module_library_name nlsCharacters_encoding)
 set(module_library_builtin_name ${module_library_name}_builtin)
 # ==============================================================================
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src/include ${ICU_INCLUDE_DIRS}
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src/include
+                    ${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/simdutf
+                    ${ICU_INCLUDE_DIRS}
                     ${Boost_INCLUDE_DIRS})
 # ==============================================================================
-file(GLOB CHARACTERS_ENCODING_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/*.cpp")
+file(GLOB CHARACTERS_ENCODING_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/*.cpp"
+"${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/simdutf/*.cpp")
 # ==============================================================================
 add_library(${module_library_name} SHARED ${CHARACTERS_ENCODING_SRC})
 # ==============================================================================

diff --git a/modules/characters_encoding/src/c/dllMain.c b/modules/characters_encoding/src/c/dllMain.c
@@ -0,0 +1,30 @@
+//=============================================================================
+// Copyright (c) 2016-present Allan CORNET (Nelson)
+//=============================================================================
+// This file is part of the Nelson.
+//=============================================================================
+// LICENCE_BLOCK_BEGIN
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// LICENCE_BLOCK_END
+//=============================================================================
+#include <Windows.h>
+//=============================================================================
+#pragma comment(lib, "icuuc.lib")
+#pragma comment(lib, "icuin.lib")
+//=============================================================================
+int WINAPI
+DllMain(HINSTANCE hInstance, DWORD reason, PVOID pvReserved)
+{
+    switch (reason) {
+    case DLL_PROCESS_ATTACH:
+        break;
+    case DLL_PROCESS_DETACH:
+        break;
+    case DLL_THREAD_ATTACH:
+        break;
+    case DLL_THREAD_DETACH:
+        break;
+    }
+    return 1;
+}
+//=============================================================================
diff --git a/modules/characters_encoding/src/c/nlsCharacters_encoding.vcxproj b/modules/characters_encoding/src/c/nlsCharacters_encoding.vcxproj
@@ -95,7 +95,7 @@
       <Optimization>Disabled</Optimization>
       <PreprocessorDefinitions>WIN32;_DEBUG;_WINDOWS;_USRDLL;NLSCHARACTERS_ENCODING_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <SDLCheck>true</SDLCheck>
-      <AdditionalIncludeDirectories>$(SolutionDir)modules/characters_encoding/src/include;$(SolutionDir)modules/i18n/src/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/icu/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/Boost</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)modules/characters_encoding/src/include;$(SolutionDir)modules/characters_encoding/src/cpp/simdutf;$(SolutionDir)modules/types/src/include;$(SolutionDir)modules/i18n/src/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/icu/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/Boost</AdditionalIncludeDirectories>
       <OpenMPSupport>true</OpenMPSupport>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <LanguageStandard>stdcpp17</LanguageStandard>
@@ -115,7 +115,7 @@
       <Optimization>Disabled</Optimization>
       <PreprocessorDefinitions>WIN32;_DEBUG;_WINDOWS;_USRDLL;NLSCHARACTERS_ENCODING_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <SDLCheck>true</SDLCheck>
-      <AdditionalIncludeDirectories>$(SolutionDir)modules/characters_encoding/src/include;$(SolutionDir)modules/i18n/src/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/icu/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/Boost</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)modules/characters_encoding/src/include;$(SolutionDir)modules/characters_encoding/src/cpp/simdutf;$(SolutionDir)modules/types/src/include;$(SolutionDir)modules/i18n/src/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/icu/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/Boost</AdditionalIncludeDirectories>
       <OpenMPSupport>true</OpenMPSupport>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <LanguageStandard>stdcpp17</LanguageStandard>
@@ -137,7 +137,7 @@
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;_USRDLL;NLSCHARACTERS_ENCODING_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <SDLCheck>true</SDLCheck>
-      <AdditionalIncludeDirectories>$(SolutionDir)modules/characters_encoding/src/include;$(SolutionDir)modules/i18n/src/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/icu/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/Boost</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)modules/characters_encoding/src/include;$(SolutionDir)modules/characters_encoding/src/cpp/simdutf;$(SolutionDir)modules/types/src/include;$(SolutionDir)modules/i18n/src/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/icu/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/Boost</AdditionalIncludeDirectories>
       <OpenMPSupport>true</OpenMPSupport>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <LanguageStandard>stdcpp17</LanguageStandard>
@@ -161,7 +161,7 @@
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;_USRDLL;NLSCHARACTERS_ENCODING_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <SDLCheck>true</SDLCheck>
-      <AdditionalIncludeDirectories>$(SolutionDir)modules/characters_encoding/src/include;$(SolutionDir)modules/i18n/src/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/icu/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/Boost</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)modules/characters_encoding/src/include;$(SolutionDir)modules/characters_encoding/src/cpp/simdutf;$(SolutionDir)modules/types/src/include;$(SolutionDir)modules/i18n/src/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/icu/include;$(SolutionDir)../NelSon-thirdparty-$(PlatformName)/Boost</AdditionalIncludeDirectories>
       <OpenMPSupport>true</OpenMPSupport>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <LanguageStandard>stdcpp17</LanguageStandard>
@@ -178,9 +178,12 @@
   <ItemGroup>
     <ClInclude Include="..\..\src\include\characters_encoding.hpp" />
     <ClInclude Include="..\..\src\include\nlsCharacters_encoding_exports.h" />
+    <ClInclude Include="..\cpp\simdutf\simdutf.h" />
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\cpp\characters_encoding.cpp" />
+    <ClCompile Include="..\cpp\simdutf\simdutf.cpp" />
+    <ClCompile Include="dllMain.c" />
   </ItemGroup>
   <ItemGroup>
     <Text Include="..\..\CMakeLists.txt" />

diff --git a/modules/characters_encoding/src/c/nlsCharacters_encoding.vcxproj.filters b/modules/characters_encoding/src/c/nlsCharacters_encoding.vcxproj.filters
@@ -21,11 +21,20 @@
     <ClInclude Include="..\..\src\include\nlsCharacters_encoding_exports.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\cpp\simdutf\simdutf.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\cpp\characters_encoding.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\cpp\simdutf\simdutf.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="dllMain.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <Text Include="..\..\CMakeLists.txt" />