Skip to content

Commit 94cf928

Browse files
blattersturmblattersturm
committed
natives: codegen changes, lazy natives for Lua to reduce RAM usage
1 parent c71d624 commit 94cf928

File tree

9 files changed

+358
-13
lines changed

9 files changed

+358
-13
lines changed

‎code/components/citizen-scripting-lua/src/LuaNativesLoader.cpp

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#include "StdInc.h"
2+
3+
#ifndef IS_FXSERVER
4+
#include <VFSManager.h>
5+
#include <VFSZipFile.h>
6+
#include <ResourceManager.h>
7+
8+
static void MountNatives(const std::string& name)
9+
{
10+
fwRefContainer<vfs::ZipFile> device = new vfs::ZipFile();
11+
12+
if (device->OpenArchive(fmt::sprintf("citizen:/scripting/lua/%s.zip", name)))
13+
{
14+
vfs::Mount(device, fmt::sprintf("nativesLua:/%s/", name));
15+
}
16+
}
17+
18+
static InitFunction initFunction([]()
19+
{
20+
fx::ResourceManager::OnInitializeInstance.Connect([](fx::ResourceManager*)
21+
{
22+
static bool mountedFiles = false;
23+
24+
if (!mountedFiles)
25+
{
26+
MountNatives("natives_universal");
27+
MountNatives("natives_21e43a33");
28+
MountNatives("natives_0193d0af");
29+
30+
mountedFiles = true;
31+
}
32+
});
33+
});
34+
#endif

‎code/components/citizen-scripting-lua/src/LuaScriptRuntime.cpp

+90-1
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ class LuaScriptRuntime : public OMClass<LuaScriptRuntime, IScriptRuntime, IScrip
117117

118118
int m_instanceId;
119119

120+
std::string m_nativesDir;
121+
120122
public:
121123
inline LuaScriptRuntime()
122124
{
@@ -169,6 +171,11 @@ class LuaScriptRuntime : public OMClass<LuaScriptRuntime, IScriptRuntime, IScrip
169171
return resourceName;
170172
}
171173

174+
inline std::string GetNativesDir()
175+
{
176+
return m_nativesDir;
177+
}
178+
172179
private:
173180
result_t LoadFileInternal(OMPtr<fxIStream> stream, char* scriptFile);
174181

@@ -180,6 +187,8 @@ class LuaScriptRuntime : public OMClass<LuaScriptRuntime, IScriptRuntime, IScrip
180187

181188
result_t LoadSystemFile(char* scriptFile);
182189

190+
result_t LoadNativesBuild(const std::string& nativeBuild);
191+
183192
public:
184193
NS_DECL_ISCRIPTRUNTIME;
185194

@@ -940,6 +949,45 @@ int Lua_InvokeNative(lua_State* L)
940949
return numResults;
941950
}
942951

952+
int Lua_LoadNative(lua_State* L)
953+
{
954+
const char* fn = luaL_checkstring(L, 1);
955+
956+
auto runtime = LuaScriptRuntime::GetCurrent();
957+
958+
OMPtr<fxIStream> stream;
959+
960+
result_t hr = runtime->GetScriptHost()->OpenSystemFile(const_cast<char*>(va("%s0x%08x.lua", runtime->GetNativesDir(), HashRageString(fn))), stream.GetAddressOf());
961+
962+
if (!FX_SUCCEEDED(hr))
963+
{
964+
lua_pushnil(L);
965+
return 1;
966+
}
967+
968+
// read file data
969+
uint64_t length;
970+
971+
if (FX_FAILED(hr = stream->GetLength(&length)))
972+
{
973+
lua_pushnil(L);
974+
return 1;
975+
}
976+
977+
std::vector<char> fileData(length + 1);
978+
if (FX_FAILED(hr = stream->Read(&fileData[0], length, nullptr)))
979+
{
980+
lua_pushnil(L);
981+
return 1;
982+
}
983+
984+
fileData[length] = '\0';
985+
986+
lua_pushlstring(L, fileData.data(), length);
987+
988+
return 1;
989+
}
990+
943991
template<LuaMetaFields metaField>
944992
int Lua_GetMetaField(lua_State* L)
945993
{
@@ -998,6 +1046,7 @@ static const struct luaL_Reg g_citizenLib[] =
9981046
{ "SetEventRoutine", Lua_SetEventRoutine },
9991047
{ "Trace", Lua_Trace },
10001048
{ "InvokeNative", Lua_InvokeNative },
1049+
{ "LoadNative", Lua_LoadNative },
10011050
// ref things
10021051
{ "SetCallRefRoutine", Lua_SetCallRefRoutine },
10031052
{ "SetDeleteRefRoutine", Lua_SetDeleteRefRoutine },
@@ -1093,7 +1142,7 @@ result_t LuaScriptRuntime::Create(IScriptHost *scriptHost)
10931142
return hr;
10941143
}
10951144

1096-
if (FX_FAILED(hr = LoadSystemFile(const_cast<char*>(va("citizen:/scripting/lua/%s", nativesBuild)))))
1145+
if (FX_FAILED(hr = LoadNativesBuild(nativesBuild)))
10971146
{
10981147
return hr;
10991148
}
@@ -1120,6 +1169,46 @@ result_t LuaScriptRuntime::Create(IScriptHost *scriptHost)
11201169
return FX_S_OK;
11211170
}
11221171

1172+
result_t LuaScriptRuntime::LoadNativesBuild(const std::string& nativesBuild)
1173+
{
1174+
result_t hr = FX_S_OK;
1175+
1176+
bool useLazyNatives = false;
1177+
1178+
#ifndef IS_FXSERVER
1179+
useLazyNatives = true;
1180+
1181+
int32_t numFields = 0;
1182+
1183+
if (FX_SUCCEEDED(hr = m_resourceHost->GetNumResourceMetaData("disable_lazy_natives", &numFields)))
1184+
{
1185+
if (numFields > 0)
1186+
{
1187+
useLazyNatives = false;
1188+
}
1189+
}
1190+
#endif
1191+
1192+
if (!useLazyNatives)
1193+
{
1194+
if (FX_FAILED(hr = LoadSystemFile(const_cast<char*>(va("citizen:/scripting/lua/%s", nativesBuild)))))
1195+
{
1196+
return hr;
1197+
}
1198+
}
1199+
else
1200+
{
1201+
m_nativesDir = "nativesLua:/" + nativesBuild.substr(0, nativesBuild.length() - 4) + "/";
1202+
1203+
if (FX_FAILED(hr = LoadSystemFile(const_cast<char*>(va("citizen:/scripting/lua/%s", "natives_loader.lua")))))
1204+
{
1205+
return hr;
1206+
}
1207+
}
1208+
1209+
return FX_S_OK;
1210+
}
1211+
11231212
result_t LuaScriptRuntime::Destroy()
11241213
{
11251214
// destroy any routines that may be referencing the Lua state

‎data/shared/citizen/scripting/lua/.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
11
natives_*.lua
2+
natives_*.zip
3+
natives_gta/

‎data/shared/citizen/scripting/lua/natives_loader.lua

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
-- global loader bits
2+
local _i, _f, _v, _r, _ri, _rf, _rl, _s, _rv, _ro, _in, _ii, _fi =
3+
Citizen.PointerValueInt(), Citizen.PointerValueFloat(), Citizen.PointerValueVector(),
4+
Citizen.ReturnResultAnyway(), Citizen.ResultAsInteger(), Citizen.ResultAsFloat(), Citizen.ResultAsLong(), Citizen.ResultAsString(), Citizen.ResultAsVector(), Citizen.ResultAsObject(),
5+
Citizen.InvokeNative, Citizen.PointerValueIntInitialized, Citizen.PointerValueFloatInitialized
6+
7+
local g = _G
8+
local rg = rawget
9+
local rs = rawset
10+
local _ln = Citizen.LoadNative
11+
local load = load
12+
local msgpack = msgpack
13+
local _tostring = tostring
14+
local function _ts(num)
15+
if num == 0 or not num then -- workaround for users calling string parameters with '0', also nil being translated
16+
return nil
17+
end
18+
return _tostring(num)
19+
end
20+
local function _ch(hash)
21+
if g.type(hash) == 'string' then
22+
return g.GetHashKey(hash)
23+
end
24+
25+
return hash
26+
end
27+
28+
local function _mfr(fn)
29+
return g.Citizen.GetFunctionReference(fn)
30+
end
31+
32+
local Global = setmetatable({}, { __newindex = function(_, n, v)
33+
g[n] = v
34+
35+
rs(_, n, v)
36+
end})
37+
38+
local nativeEnv = {
39+
Global = Global,
40+
_mfr = _mfr,
41+
_ch = _ch,
42+
_ts = _ts,
43+
msgpack = msgpack,
44+
rs = rs,
45+
_i = _i,
46+
_f = _f,
47+
_v = _v,
48+
_r = _r,
49+
_ri = _ri,
50+
_rf = _rf,
51+
_rl = _rl,
52+
_s = _s,
53+
_rv = _rv,
54+
_ro = _ro,
55+
_in = _in,
56+
_ii = _ii,
57+
_fi = _fi
58+
}
59+
60+
local nilCache = {}
61+
62+
setmetatable(g, {
63+
__index = function(t, n)
64+
local v = rg(t, n)
65+
66+
if not v and not nilCache[n] then
67+
local nativeString = _ln(n)
68+
69+
if nativeString then
70+
local chunk = load(nativeString, '@' .. n .. '.lua', 't', nativeEnv)
71+
72+
if chunk then
73+
chunk()
74+
75+
v = rg(t, n)
76+
end
77+
else
78+
nilCache[n] = true
79+
end
80+
end
81+
82+
return v
83+
end
84+
})

‎ext/natives/Makefile

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
GTA_INPUTS=$(wildcard natives_stash/gta_*.lua)
2+
3+
GTA_OUTPUTS_LUA=$(GTA_INPUTS:natives_stash/gta_%=out/natives_%)
4+
GTA_OUTPUTS_JS=$(GTA_OUTPUTS_LUA:.lua=.js)
5+
GTA_OUTPUTS_ZIP=$(GTA_OUTPUTS_LUA:.lua=.zip)
6+
GTA_OUTPUTS_DTS=$(GTA_OUTPUTS_LUA:.lua=.d.ts)
7+
8+
CODEGEN_SCRIPTS=codegen.lua codegen_cfx_natives.lua codegen_types.lua markdown.lua
9+
10+
define NATIVES_FIVE_HEADER
11+
#if GTA_FIVE
12+
namespace CitizenFX.Core.Native
13+
{
14+
endef
15+
16+
define NATIVES_FIVE_FOOTER
17+
}
18+
#endif
19+
endef
20+
21+
export NATIVES_FIVE_HEADER
22+
export NATIVES_FIVE_FOOTER
23+
24+
define NATIVES_SERVER_HEADER
25+
#if IS_FXSERVER
26+
namespace CitizenFX.Core.Native
27+
{
28+
endef
29+
30+
define NATIVES_SERVER_FOOTER
31+
}
32+
#endif
33+
endef
34+
35+
export NATIVES_SERVER_HEADER
36+
export NATIVES_SERVER_FOOTER
37+
38+
all: out \
39+
out/NativesFive.cs out/NativesServer.cs \
40+
out/rpc_natives.json \
41+
out/natives_server.lua out/natives_server.d.ts out/natives_server.js \
42+
$(GTA_OUTPUTS_LUA) $(GTA_OUTPUTS_JS) $(GTA_OUTPUTS_DTS) $(GTA_OUTPUTS_ZIP)
43+
44+
.PHONY: clean
45+
clean:
46+
rm -r out
47+
48+
out:
49+
mkdir -p out
50+
51+
out/NativesFive.cs: natives_stash/gta_universal.lua $(CODEGEN_SCRIPTS) codegen_out_cs.lua codegen_out_enum.lua
52+
echo "$$NATIVES_FIVE_HEADER" > $@
53+
./lua53 codegen.lua natives_stash/gta_universal.lua enum >> $@
54+
./lua53 codegen.lua natives_stash/gta_universal.lua cs >> $@
55+
echo "$$NATIVES_FIVE_FOOTER" >> $@
56+
57+
out/NativesServer.cs: $(CODEGEN_SCRIPTS) codegen_out_cs.lua
58+
echo "$$NATIVES_SERVER_HEADER" > $@
59+
./lua53 codegen.lua natives_stash/gta_universal.lua enum server >> $@
60+
./lua53 codegen.lua natives_stash/gta_universal.lua cs server >> $@
61+
echo "$$NATIVES_SERVER_FOOTER" >> $@
62+
63+
out/rpc_natives.json: $(CODEGEN_SCRIPTS) natives_stash/gta_universal.lua codegen_out_rpc.lua rpc_spec_natives.lua
64+
./lua53 codegen.lua natives_stash/gta_universal.lua rpc server > $@
65+
66+
out/natives_server.lua: $(CODEGEN_SCRIPTS) natives_stash/gta_universal.lua codegen_out_lua.lua
67+
./lua53 codegen.lua natives_stash/gta_universal.lua lua server > $@
68+
69+
out/natives_server.d.ts: $(CODEGEN_SCRIPTS) natives_stash/gta_universal.lua codegen_out_dts.lua
70+
./lua53 codegen.lua natives_stash/gta_universal.lua dts server > $@
71+
72+
out/natives_server.js: $(CODEGEN_SCRIPTS) natives_stash/gta_universal.lua codegen_out_js.lua
73+
./lua53 codegen.lua natives_stash/gta_universal.lua js server > $@
74+
75+
out/natives_%.lua: natives_stash/gta_%.lua $(CODEGEN_SCRIPTS) codegen_out_lua.lua
76+
./lua53 codegen.lua $< lua > $@
77+
78+
out/natives_%.d.ts: natives_stash/gta_%.lua $(CODEGEN_SCRIPTS) codegen_out_dts.lua
79+
./lua53 codegen.lua $< dts > $@
80+
81+
out/natives_%.js: natives_stash/gta_%.lua $(CODEGEN_SCRIPTS) codegen_out_js.lua
82+
./lua53 codegen.lua $< js > $@
83+
84+
out/natives_%.zip: natives_stash/gta_%.lua $(CODEGEN_SCRIPTS) codegen_out_lua.lua codegen_out_slua.lua
85+
$(eval DIR := $(shell mktemp -d -p out/))
86+
NATIVES_DIR="$(DIR)" ./lua53 codegen.lua $< slua > /dev/null
87+
sh -c 'pushd $(DIR) && ../../../../code/tools/ci/7z.exe a ../../$@ \*.lua && popd'
88+
rm -r $(DIR) || true

0 commit comments

Comments
 (0)