Merge pull request #433 from evo-lua/196-lua-utf8-integration

Add a new utf8 library (powered by lua-utf8)

.cppcheck

@@ -1,6 +1,9 @@
 // The performance impact is irrelevant where deliberately used, but it affects readability
 passedByValue
 
+// Copy/pasted library code (temporary... hopefully)
+*:Runtime/Bindings/lutf8.c
+
 // Debug functions left in on purpose
 unusedFunction:Runtime/LuaVirtualMachine.cpp
 // Callback-based interface (hooks that will be called by RML)

.gitmodules

@@ -82,3 +82,7 @@
 	path = deps/roberto-ieru/LPeg
 	url = https://github.com/roberto-ieru/LPeg
 	ignore = dirty
+[submodule "deps/starwing/luautf8"]
+	path = deps/starwing/luautf8
+	url = https://github.com/starwing/luautf8
+	ignore = dirty

BuildTools/BuildTarget.lua

@@ -32,28 +32,42 @@ end
 function BuildTarget:SetCompilerToolchain(toolchainInfo)
 	local ninjaFile = self.ninjaFile
 
+	ninjaFile:AddVariable("C_COMPILER", toolchainInfo.C_COMPILER)
 	ninjaFile:AddVariable("CPP_COMPILER", toolchainInfo.CPP_COMPILER)
-	ninjaFile:AddVariable("COMPILER_FLAGS", toolchainInfo.COMPILER_FLAGS)
+	ninjaFile:AddVariable("COMPILER_FLAGS_C", toolchainInfo.COMPILER_FLAGS_C)
+	ninjaFile:AddVariable("COMPILER_FLAGS_CPP", toolchainInfo.COMPILER_FLAGS_CPP)
+	ninjaFile:AddVariable("C_LINKER", toolchainInfo.CPP_LINKER)
 	ninjaFile:AddVariable("CPP_LINKER", toolchainInfo.CPP_LINKER)
 	ninjaFile:AddVariable("LINKER_FLAGS", toolchainInfo.LINKER_FLAGS)
+	ninjaFile:AddVariable("C_ARCHIVER", toolchainInfo.C_ARCHIVER)
 	ninjaFile:AddVariable("CPP_ARCHIVER", toolchainInfo.CPP_ARCHIVER)
 	ninjaFile:AddVariable("ARCHIVER_FLAGS", toolchainInfo.ARCHIVER_FLAGS)
 
 	-- Technically, this is still specific to GCC due to the emitted deps file, but that could easily be changed later (if needed)
 	ninjaFile:AddRule(
 		"compile",
-		"$CPP_COMPILER -c $in -o $out -MT $out -MMD -MF $out.d $COMPILER_FLAGS $includes $defines",
+		"$CPP_COMPILER -c $in -o $out -MT $out -MMD -MF $out.d $COMPILER_FLAGS_CPP $includes $defines",
 		{
 			description = "Compiling $in ...",
 			deps = "$C_COMPILER", --  g++ uses the same format as gcc
 			depfile = "$out.d",
 		}
 	)
+	ninjaFile:AddRule(
+		"ccompile",
+		"$C_COMPILER -c $in -o $out -MT $out -MMD -MF $out.d $COMPILER_FLAGS_C $includes $defines",
+		{
+			description = "Compiling $in ...",
+			deps = "$C_COMPILER",
+			depfile = "$out.d",
+		}
+	)
 	ninjaFile:AddRule(
 		"link",
 		"$CPP_LINKER $in -o $out $libs $LINKER_FLAGS",
 		{ description = "Linking target $out ..." }
 	)
+	ninjaFile:AddRule("clink", "$C_LINKER $in -o $out $libs $LINKER_FLAGS", { description = "Linking target $out ..." })
 
 	self.toolchain = toolchainInfo
 end
@@ -114,6 +128,17 @@ function BuildTarget:ProcessNativeSources()
 
 		table.insert(objectFiles, outputFile)
 	end
+
+	for index, cSourceFilePath in ipairs(self.cSources) do
+		local outputFile =
+			string.format("%s/%s.%s", self.BUILD_DIR, cSourceFilePath, NinjaBuildTools.OBJECT_FILE_EXTENSION)
+
+		-- Some dependencies demand special treatment because of how they use defines (questionably?)
+		local defines = self:GetDefines()
+		ninjaFile:AddBuildEdge(outputFile, "ccompile " .. cSourceFilePath, { includes = includes, defines = defines })
+
+		table.insert(objectFiles, outputFile)
+	end
 end
 
 function BuildTarget:GetDefines()

BuildTools/NinjaBuildTools.lua

@@ -1,8 +1,16 @@
 local ffi = require("ffi")
 
+local format = string.format
+
 local isWindows = (ffi.os == "Windows")
 local isMacOS = (ffi.os == "OSX")
 
+local GCC_RELEASE_FLAGS = "-O3 -DNDEBUG"
+local GCC_DEBUG_FLAGS = "-g" -- For better stack traces
+local GCC_DIAGNOSTICS_FLAGS =
+	"-Wall -Wextra -Wno-missing-field-initializers -Wno-unused-parameter -fvisibility=hidden -fno-strict-aliasing -fdiagnostics-color -Wfatal-errors"
+local DEFAULT_COMPILER_FLAGS = format("%s %s %s", GCC_RELEASE_FLAGS, GCC_DEBUG_FLAGS, GCC_DIAGNOSTICS_FLAGS)
+
 local C_BuildTools = {
 	OBJECT_FILE_EXTENSION = (isWindows and "obj" or "o"),
 	STATIC_LIBRARY_EXTENSION = (isWindows and ".lib" or ".a"),
@@ -11,12 +19,16 @@ local C_BuildTools = {
 	DEFAULT_BUILD_DIRECTORY_NAME = "ninjabuild-" .. (isWindows and "windows" or "unix"),
 	GCC_COMPILATION_SETTINGS = {
 		displayName = "GNU Compiler Collection",
+		C_COMPILER = "gcc",
 		CPP_COMPILER = "g++",
-		COMPILER_FLAGS = "-O3 -DNDEBUG -g -std=c++20 -Wall -Wextra -Wno-missing-field-initializers -Wno-unused-parameter -fvisibility=hidden -fno-strict-aliasing -fdiagnostics-color -Wfatal-errors"
-			.. (isMacOS and " -ObjC++" or ""), -- Forced Objc++ should be removed once glfw3webgpu is merged into the GLFW core library... until then, it's a necessary evil
+		COMPILER_FLAGS_CPP = DEFAULT_COMPILER_FLAGS .. " -std=c++20",
+		-- Forced ObjC compilation should be removed once glfw3webgpu is merged into the GLFW core library
+		COMPILER_FLAGS_C = DEFAULT_COMPILER_FLAGS .. " -std=c11" .. (isMacOS and " -ObjC" or ""),
+		C_LINKER = "gcc",
 		CPP_LINKER = "g++",
 		-- Must export the entry point of bytecode objects so that LuaJIT can load them via require()
 		LINKER_FLAGS = isWindows and "-Wl,--export-all-symbols" or "-rdynamic",
+		C_ARCHIVER = "ar",
 		CPP_ARCHIVER = "ar",
 		ARCHIVER_FLAGS = "-rcs",
 	},
@@ -181,8 +193,6 @@ function C_BuildTools.FetchNotableChanges(versionTag)
 	return notableChanges[versionTag] or {}
 end
 
-local format = _G.format or string.format -- Can be removed after the format alias has been re-introduced
-
 -- This is screaming to get out and become its own class, but for the time being it shall remain imprisoned...
 local function markdownFile_AddCategory(markdownFile, category)
 	if #category.entries == 0 then

BuildTools/Targets/EvoBuildTarget.lua

@@ -52,6 +52,10 @@ local EvoBuildTarget = {
 		"Runtime/Libraries/vfs.lua",
 		"Runtime/Libraries/v8.lua",
 	},
+	cSources = {
+		"Runtime/Bindings/glfw_webgpu.c",
+		"Runtime/Bindings/lutf8.c",
+	},
 	cppSources = {
 		"Runtime/main.cpp",
 		"Runtime/Bindings/crypto_argon2.cpp",
@@ -87,6 +91,7 @@ local EvoBuildTarget = {
 		"deps/luvit/luv/deps/libuv/include",
 		"deps/mariusbancila/stduuid/include",
 		"deps/nothings/stb",
+		"deps/starwing/luautf8", -- unidata.h
 		"deps/webview/webview",
 		"deps/openssl/openssl/include",
 		"deps/zhaog/lua-openssl/deps/auxiliar",

Runtime/Bindings/glfw_ffi.cpp

@@ -2,7 +2,6 @@
 #include "interop_ffi.hpp"
 
 #include <glfw3webgpu.h>
-#include <glfw3webgpu.c>
 
 #include <string>
 

Runtime/Bindings/glfw_webgpu.c

@@ -0,0 +1,5 @@
+// Remove this once the upstream PR is merged: https://github.com/glfw/glfw/pull/2333
+
+// This glue library uses ObjectiveC, so it can't be included with the .cpp sources
+#include <glfw3webgpu.h>
+#include <glfw3webgpu.c>