Commit mini-db

.gitignore

@@ -52,6 +52,9 @@
 /bin/GuiConfigs/*.dat
 /bin/GuiConfigs/*.dat.*
 
+# Some data from git
+!/bin/git/
+
 # Visual Studio Files
 .vs/*
 .vscode/*

bin/git/README.md

@@ -0,0 +1 @@
+Cached data from GitHub API.

rpcs3/main.cpp

@@ -9,9 +9,14 @@
 #include <QTimer>
 #include <QObject>
 #include <QStyleFactory>
+#include <QByteArray>
+#include <QJsonArray>
+#include <QJsonObject>
+#include <QJsonDocument>
 
 #include "rpcs3qt/gui_application.h"
 #include "rpcs3qt/fatal_error_dialog.h"
+#include "rpcs3qt/curl_handle.h"
 
 #include "headless_application.h"
 #include "Utilities/sema.h"
@@ -40,6 +45,7 @@ DYNAMIC_IMPORT("ntdll.dll", NtSetTimerResolution, NTSTATUS(ULONG DesiredResoluti
 #include "Utilities/Config.h"
 #include "Utilities/Thread.h"
 #include "Utilities/File.h"
+#include "Utilities/StrUtil.h"
 #include "rpcs3_version.h"
 #include "Emu/System.h"
 #include <thread>
@@ -172,17 +178,18 @@ struct pause_on_fatal final : logs::listener
 	}
 };
 
-const char* arg_headless   = "headless";
-const char* arg_no_gui     = "no-gui";
-const char* arg_high_dpi   = "hidpi";
-const char* arg_rounding   = "dpi-rounding";
-const char* arg_styles     = "styles";
-const char* arg_style      = "style";
-const char* arg_stylesheet = "stylesheet";
-const char* arg_config     = "config";
-const char* arg_q_debug    = "qDebug";
-const char* arg_error      = "error";
-const char* arg_updating   = "updating";
+constexpr auto arg_headless   = "headless";
+constexpr auto arg_no_gui     = "no-gui";
+constexpr auto arg_high_dpi   = "hidpi";
+constexpr auto arg_rounding   = "dpi-rounding";
+constexpr auto arg_styles     = "styles";
+constexpr auto arg_style      = "style";
+constexpr auto arg_stylesheet = "stylesheet";
+constexpr auto arg_config     = "config";
+constexpr auto arg_q_debug    = "qDebug";
+constexpr auto arg_error      = "error";
+constexpr auto arg_updating   = "updating";
+constexpr auto arg_commit_db  = "get-commit-db";
 
 int find_arg(std::string arg, int& argc, char* argv[])
 {
@@ -474,12 +481,181 @@ int main(int argc, char** argv)
 	parser.addOption(QCommandLineOption(arg_q_debug, "Log qDebug to RPCS3.log."));
 	parser.addOption(QCommandLineOption(arg_error, "For internal usage."));
 	parser.addOption(QCommandLineOption(arg_updating, "For internal usage."));
+	parser.addOption(QCommandLineOption(arg_commit_db, "Update commits.lst cache."));
 	parser.process(app->arguments());
 
 	// Don't start up the full rpcs3 gui if we just want the version or help.
 	if (parser.isSet(version_option) || parser.isSet(help_option))
 		return 0;
 
+	if (parser.isSet(arg_commit_db))
+	{
+		fs::file file(argc > 2 ? argv[2] : "bin/git/commits.lst", fs::read + fs::write + fs::append + fs::create);
+
+		if (file)
+		{
+			// Get existing list
+			std::string data = file.to_string();
+			std::vector<std::string> list = fmt::split(data, {"\n"});
+
+			const bool was_empty = data.empty();
+
+			// SHA to start
+			std::string from, last;
+
+			if (argc > 3)
+			{
+				from = argv[3];
+			}
+
+			if (!list.empty())
+			{
+				// Decode last entry to check last written commit
+				QByteArray buf(list.back().c_str(), list.back().size());
+				QJsonDocument doc = QJsonDocument::fromJson(buf);
+
+				if (doc.isObject())
+				{
+					last = doc["sha"].toString().toStdString();
+				}
+			}
+
+			list.clear();
+
+			// JSON buffer
+			QByteArray buf;
+
+			// CURL handle to work with GitHub API
+			curl_handle curl;
+
+			struct curl_slist* hhdr{};
+			hhdr = curl_slist_append(hhdr, "Accept: application/vnd.github.v3+json");
+			hhdr = curl_slist_append(hhdr, "User-Agent: curl/7.37.0");
+
+			curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hhdr);
+			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, +[](const char* ptr, usz, usz size, void* json) -> usz
+			{
+				reinterpret_cast<QByteArray*>(json)->append(ptr, size);
+				return size;
+			});
+			curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buf);
+
+			u32 page = 1;
+
+			constexpr u32 per_page = 100;
+
+			while (page <= 55)
+			{
+				std::string url = "https://api.github.com/repos/RPCS3/rpcs3/commits?per_page=";
+				fmt::append(url, "%u&page=%u", per_page, page++);
+				if (!from.empty())
+					fmt::append(url, "&sha=%s", from);
+
+				curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+				curl_easy_perform(curl);
+
+				QJsonDocument info = QJsonDocument::fromJson(buf);
+
+				if (!info.isArray()) [[unlikely]]
+				{
+					fprintf(stderr, "Bad response:\n%s", buf.data());
+					break;
+				}
+
+				u32 count = 0;
+
+				for (auto&& ref : info.array())
+				{
+					if (ref.isObject())
+					{
+						count++;
+
+						QJsonObject result, author, committer;
+						QJsonObject commit = ref.toObject();
+
+						auto commit_ = commit["commit"].toObject();
+						auto author_ = commit_["author"].toObject();
+						auto committer_ = commit_["committer"].toObject();
+						auto _author = commit["author"].toObject();
+						auto _committer = commit["committer"].toObject();
+
+						result["sha"] = commit["sha"];
+						result["msg"] = commit_["message"];
+
+						author["name"] = author_["name"];
+						author["date"] = author_["date"];
+						author["email"] = author_["email"];
+						author["login"] = _author["login"];
+						author["avatar"] = _author["avatar_url"];
+
+						committer["name"] = committer_["name"];
+						committer["date"] = committer_["date"];
+						committer["email"] = committer_["email"];
+						committer["login"] = _committer["login"];
+						committer["avatar"] = _committer["avatar_url"];
+
+						result["author"] = author;
+						result["committer"] = committer;
+
+						QJsonDocument out(result);
+						buf = out.toJson(QJsonDocument::JsonFormat::Compact);
+						buf += "\n";
+
+						if (was_empty || !from.empty())
+						{
+							data = buf.toStdString() + std::move(data);
+						}
+						else if (commit["sha"].toString().toStdString() == last)
+						{
+							page = -1;
+							break;
+						}
+						else
+						{
+							// Append to the list
+							list.emplace_back(buf.data(), buf.size());
+						}
+					}
+					else
+					{
+						page = -1;
+						break;
+					}
+				}
+
+				buf.clear();
+
+				if (count < per_page)
+				{
+					break;
+				}
+			}
+
+			if (was_empty || !from.empty())
+			{
+				file.trunc(0);
+				file.write(data);
+			}
+			else
+			{
+				// Append list in reverse order
+				for (usz i = list.size() - 1; ~i; --i)
+				{
+					file.write(list[i]);
+				}
+			}
+
+			curl_slist_free_all(hhdr);
+		}
+		else
+		{
+			fprintf(stderr, "Failed to open file: %s.\n", argv[2]);
+			return 1;
+		}
+
+		return 0;
+	}
+
 	if (parser.isSet(arg_q_debug))
 	{
 		qInstallMessageHandler(log_q_debug);

rpcs3/rpcs3qt/curl_handle.h

@@ -17,4 +17,9 @@ class curl_handle : public QObject
 	~curl_handle();
 
 	CURL* get_curl();
+
+	operator CURL*()
+	{
+		return get_curl();
+	}
 };

rpcs3/rpcs3qt/downloader.cpp

@@ -48,14 +48,14 @@ void downloader::start(const std::string& url, bool follow_location, bool show_p
 	m_curl_buf.clear();
 	m_curl_abort = false;
 
-	curl_easy_setopt(m_curl->get_curl(), CURLOPT_URL, url.c_str());
-	curl_easy_setopt(m_curl->get_curl(), CURLOPT_WRITEFUNCTION, curl_write_cb_compat);
-	curl_easy_setopt(m_curl->get_curl(), CURLOPT_WRITEDATA, this);
-	curl_easy_setopt(m_curl->get_curl(), CURLOPT_FOLLOWLOCATION, follow_location ? 1 : 0);
+	curl_easy_setopt(m_curl, CURLOPT_URL, url.c_str());
+	curl_easy_setopt(m_curl, CURLOPT_WRITEFUNCTION, curl_write_cb_compat);
+	curl_easy_setopt(m_curl, CURLOPT_WRITEDATA, this);
+	curl_easy_setopt(m_curl, CURLOPT_FOLLOWLOCATION, follow_location ? 1 : 0);
 
 	m_thread = QThread::create([this]
 	{
-		const auto result = curl_easy_perform(m_curl->get_curl());
+		const auto result = curl_easy_perform(m_curl);
 		m_curl_success = result == CURLE_OK;
 
 		if (!m_curl_success && !m_curl_abort)
@@ -179,7 +179,7 @@ usz downloader::update_buffer(char* data, usz size)
 
 	if (m_actual_download_size < 0)
 	{
-		if (curl_easy_getinfo(m_curl->get_curl(), CURLINFO_CONTENT_LENGTH_DOWNLOAD, &m_actual_download_size) == CURLE_OK && m_actual_download_size > 0)
+		if (curl_easy_getinfo(m_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &m_actual_download_size) == CURLE_OK && m_actual_download_size > 0)
 		{
 			max = static_cast<int>(m_actual_download_size);
 		}

rpcs3/rpcs3qt/update_manager.cpp

@@ -209,6 +209,12 @@ bool update_manager::handle_json(bool automatic, bool check_only, const QByteArr
 	m_expected_hash = latest[os]["checksum"].toString().toStdString();
 	m_expected_size = latest[os]["size"].toInt();
 
+	if (!m_request_url.starts_with("https://github.com/RPCS3/rpcs3"))
+	{
+		update_log.fatal("Bad url: %s", m_request_url);
+		return false;
+	}
+
 	if (check_only)
 	{
 		m_downloader->close_progress_dialog();