Fix ContentDB packages timing out by using download_file instead (#7891)

Author: rubenwardy

builtin/mainmenu/dlg_contentstore.lua

@@ -268,9 +268,37 @@ function package_dialog.create(package)
 end
 
 function store.load()
-	store.packages_full = core.get_package_list()
-	store.packages = store.packages_full
-	store.loaded = true
+	local tmpdir = os.tempfolder()
+	local target = tmpdir .. DIR_DELIM .. "packages.json"
+
+	assert(core.create_dir(tmpdir))
+
+	local base_url     = core.settings:get("contentdb_url")
+	local show_nonfree = core.settings:get_bool("show_nonfree_packages")
+	local url = base_url ..
+		"/api/packages/?type=mod&type=game&type=txp&protocol_version=" ..
+		core.get_max_supp_proto() ..
+		"&nonfree=" ..
+		(show_nonfree and "true" or "false")
+
+	core.download_file(url, target)
+
+	local file = io.open(target, "r")
+	if file then
+		store.packages_full = core.parse_json(file:read("*all"))
+		file:close()
+
+		for _, package in pairs(store.packages_full) do
+			package.url = base_url .. "/packages/" ..
+				package.author .. "/" .. package.name ..
+				"/releases/" .. package.release .. "/download/"
+		end
+
+		store.packages = store.packages_full
+		store.loaded = true
+	end
+
+	core.delete_dir(tmpdir)
 end
 
 function store.update_paths()

doc/menu_lua_api.txt

@@ -146,17 +146,6 @@ Package - content which is downloadable from the content db, may or may not be i
 			optional_depends = {"mod", "names"}, -- mods only
 		}
 
-* core.get_package_list() -> downloads package list from content db
-	* returns a list of:
-
-		{
-			name = "basename",
-			title = "human readable title",
-			author = "username",
-			type = "", -- mod, game, txp
-			short_description = "description",
-			url = "",
-		}
 
 Favorites:
 core.get_favorites(location) -> list of favorites (possible in async calls)

src/script/lua_api/l_mainmenu.cpp

@@ -991,68 +991,6 @@ int ModApiMainMenu::l_get_screen_info(lua_State *L)
 	return 1;
 }
 
-int ModApiMainMenu::l_get_package_list(lua_State *L)
-{
-	std::string url   = g_settings->get("contentdb_url");
-	bool show_nonfree = g_settings->getBool("show_nonfree_packages");
-	std::vector<Package> packages = getPackagesFromURL(url +
-			"/api/packages/?type=mod&type=game&type=txp&protocol_version="
-			LATEST_PROTOCOL_VERSION_STRING "&nonfree=" +
-			(show_nonfree ? "true" : "false"));
-
-	// Make table
-	lua_newtable(L);
-	int top = lua_gettop(L);
-	unsigned int index = 1;
-
-	// Fill table
-	for (const auto &package : packages) {
-		lua_pushnumber(L, index);
-		lua_newtable(L);
-
-		int top_lvl2 = lua_gettop(L);
-
-		lua_pushstring(L, "author");
-		lua_pushstring(L, package.author.c_str());
-		lua_settable  (L, top_lvl2);
-
-		lua_pushstring(L, "name");
-		lua_pushstring(L, package.name.c_str());
-		lua_settable  (L, top_lvl2);
-
-		lua_pushstring(L, "title");
-		lua_pushstring(L, package.title.c_str());
-		lua_settable  (L, top_lvl2);
-
-		lua_pushstring(L, "type");
-		lua_pushstring(L, package.type.c_str());
-		lua_settable  (L, top_lvl2);
-
-		lua_pushstring(L, "short_description");
-		lua_pushstring(L, package.shortDesc.c_str());
-		lua_settable  (L, top_lvl2);
-
-		lua_pushstring (L, "release");
-		lua_pushinteger(L, package.release);
-		lua_settable   (L, top_lvl2);
-
-		if (package.thumbnail != "") {
-			lua_pushstring(L, "thumbnail");
-			lua_pushstring(L, package.thumbnail.c_str());
-			lua_settable  (L, top_lvl2);
-		}
-
-		lua_pushstring(L, "url");
-		lua_pushstring(L, package.getDownloadURL(url).c_str());
-		lua_settable  (L, top_lvl2);
-
-		lua_settable(L, top);
-		index++;
-	}
-
-	return 1;
-}
-
 /******************************************************************************/
 int ModApiMainMenu::l_get_min_supp_proto(lua_State *L)
 {
@@ -1123,7 +1061,6 @@ void ModApiMainMenu::Initialize(lua_State *L, int top)
 	API_FCT(get_video_drivers);
 	API_FCT(get_video_modes);
 	API_FCT(get_screen_info);
-	API_FCT(get_package_list);
 	API_FCT(get_min_supp_proto);
 	API_FCT(get_max_supp_proto);
 	API_FCT(do_async_callback);
@@ -1147,5 +1084,4 @@ void ModApiMainMenu::InitializeAsync(lua_State *L, int top)
 	//API_FCT(extract_zip); //TODO remove dependency to GuiEngine
 	API_FCT(download_file);
 	//API_FCT(gettext); (gettext lib isn't threadsafe)
-	API_FCT(get_package_list);
 }

src/script/lua_api/l_mainmenu.h

@@ -133,9 +133,6 @@ class ModApiMainMenu: public ModApiBase
 
 	static int l_get_video_modes(lua_State *L);
 
-	//content store
-	static int l_get_package_list(lua_State *L);
-
 	//version compatibility
 	static int l_get_min_supp_proto(lua_State *L);