[lua] Fix bugs handling errors inside app.transaction()

This errors was reported in aseprite#4431: The Tx wasn't rolled back correctly
in case of a Lua error inside the transaction because Lua needs to be
compiled as C++ to avoid longjmps and support stack
unwinding (i.e. calling destructors).

src/app/script/engine.cpp

@@ -42,6 +42,14 @@
 #include <stack>
 #include <string>
 
+// We use our own fopen() that supports Unicode filename on Windows
+// extern "C"
+FILE* lua_user_fopen(const char* fname,
+                     const char* mode)
+{
+  return base::open_file_raw(fname, mode);
+}
+
 namespace app {
 namespace script {
 
@@ -210,13 +218,6 @@ void register_websocket_class(lua_State* L);
 
 void set_app_params(lua_State* L, const Params& params);
 
-// We use our own fopen() that supports Unicode filename on Windows
-extern "C" FILE* lua_user_fopen(const char* fname,
-                                const char* mode)
-{
-  return base::open_file_raw(fname, mode);
-}
-
 Engine::Engine()
   : L(luaL_newstate())
   , m_delegate(nullptr)

src/app/script/luacpp.h

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2018-2023  Igara Studio S.A.
+// Copyright (C) 2018-2024  Igara Studio S.A.
 // Copyright (C) 2018  David Capello
 //
 // This program is distributed under the terms of
@@ -9,11 +9,12 @@
 #define APP_SCRIPT_LUACPP_H_INCLUDED
 #pragma once
 
-extern "C" {
-  #include "lua.h"
-  #include "lualib.h"
-  #include "lauxlib.h"
-}
+// We're compiling Lua with C++ support to handle error with
+// exceptions, so there is no need of extern "C" { ... } these
+// includes.
+#include "lua.h"
+#include "lualib.h"
+#include "lauxlib.h"
 
 #include "base/debug.h"
 

third_party/CMakeLists.txt

@@ -205,7 +205,19 @@ if(ENABLE_SCRIPTING)
     target_compile_definitions(lua PUBLIC LUA_USE_LINUX=1)
   endif()
 
-  target_compile_definitions(lua PUBLIC LUA_FLOORN2I=1)
+  # Compile Lua as C++ to control errors with exceptions and have
+  # stack unwinding (i.e. calling destructors correctly).
+  if(MSVC)
+    target_compile_options(lua PRIVATE -TP)
+    target_compile_options(lualib PRIVATE -TP)
+    target_compile_options(lauxlib PRIVATE -TP)
+  else()
+    target_compile_options(lua PRIVATE -xc++)
+    target_compile_options(lualib PRIVATE -xc++)
+    target_compile_options(lauxlib PRIVATE -xc++)
+  endif()
+
+  target_compile_definitions(lua PUBLIC LUA_FLOORN2I=F2Ifloor)
   target_compile_definitions(lualib PRIVATE HAVE_SYSTEM)
   target_include_directories(lua PUBLIC lua)
   target_include_directories(lauxlib PUBLIC lua)