diff --git a/core/io/remote_filesystem_client.cpp b/core/io/remote_filesystem_client.cpp new file mode 100644 index 00000000000000..fa76a02ca10668 --- /dev/null +++ b/core/io/remote_filesystem_client.cpp @@ -0,0 +1,301 @@ +/**************************************************************************/ +/* remote_filesystem_client.cpp */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#include "remote_filesystem_client.h" +#include "core/io/dir_access.h" +#include "core/io/file_access.h" +#include "core/io/stream_peer_tcp.h" +#include "core/string/string_builder.h" + +#define FILESYSTEM_CACHE_VERSION 1 +#define FILESYSTEM_PROTOCOL_VERSION 1 +#define PASSWORD_LENGTH 32 + +#define FILES_SUBFOLDER "remote_filesystem_files" +#define FILES_CACHE_FILE "remote_filesystem.cache" + +Vector RemoteFilesystemClient::_load_cache_file() { + Ref fa = FileAccess::open(cache_path.path_join(FILES_CACHE_FILE), FileAccess::READ); + if (!fa.is_valid()) { + return Vector(); // No cache, return empty + } + + int version = fa->get_line().to_int(); + if (version != FILESYSTEM_CACHE_VERSION) { + return Vector(); // Version mismatch, ignore everything. + } + + String file_path = cache_path.path_join(FILES_SUBFOLDER); + + Vector file_cache; + + while (!fa->eof_reached()) { + String l = fa->get_line(); + Vector fields = l.split("::"); + if (fields.size() != 3) { + break; + } + FileCache fc; + fc.path = fields[0]; + fc.server_modified_time = fields[1].to_int(); + fc.modified_time = fields[2].to_int(); + + String full_path = file_path.path_join(fc.path); + if (!FileAccess::exists(full_path)) { + continue; // File is gone. + } + + if (FileAccess::get_modified_time(full_path) != fc.modified_time) { + DirAccess::remove_absolute(full_path); // Take the chance to remove this file and assume we no longer have it. + continue; + } + + file_cache.push_back(fc); + } + + return file_cache; +} + +Error RemoteFilesystemClient::_store_file(const String &p_path, const LocalVector &p_file, uint64_t &modified_time) { + modified_time = 0; + String full_path = cache_path.path_join(FILES_SUBFOLDER).path_join(p_path); + String base_file_dir = full_path.get_base_dir(); + + if (!validated_directories.has(base_file_dir)) { + // Verify that path exists before writing file, but only verify once for performance. + DirAccess::make_dir_recursive_absolute(base_file_dir); + validated_directories.insert(base_file_dir); + } + + Ref f = FileAccess::open(full_path, FileAccess::WRITE); + ERR_FAIL_COND_V_MSG(f.is_null(), ERR_FILE_CANT_OPEN, "Unable to open file for writing to remote filesystem cache: " + p_path); + f->store_buffer(p_file.ptr(), p_file.size()); + Error err = f->get_error(); + if (err) { + return err; + } + f.unref(); // Unref to ensure file is not locked and modified time can be obtained. + + modified_time = FileAccess::get_modified_time(full_path); + return OK; +} + +Error RemoteFilesystemClient::_remove_file(const String &p_path) { + return DirAccess::remove_absolute(cache_path.path_join(FILES_SUBFOLDER).path_join(p_path)); +} +Error RemoteFilesystemClient::_store_cache_file(const Vector &p_cache) { + String full_path = cache_path.path_join(FILES_CACHE_FILE); + String base_file_dir = full_path.get_base_dir(); + Error err = DirAccess::make_dir_recursive_absolute(base_file_dir); + ERR_FAIL_COND_V_MSG(err != OK, err, "Unable to create base directory to store cache file: " + base_file_dir); + + Ref f = FileAccess::open(full_path, FileAccess::WRITE); + ERR_FAIL_COND_V_MSG(f.is_null(), ERR_FILE_CANT_OPEN, "Unable to open the remote cache file for writing: " + full_path); + f->store_line(itos(FILESYSTEM_CACHE_VERSION)); + for (int i = 0; i < p_cache.size(); i++) { + String l = p_cache[i].path + "::" + itos(p_cache[i].server_modified_time) + "::" + itos(p_cache[i].modified_time); + f->store_line(l); + } + return OK; +} + +Error RemoteFilesystemClient::synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path) { + Error err = _synchronize_with_server(p_host, p_port, p_password, r_cache_path); + // Ensure no memory is kept + validated_directories.reset(); + cache_path = String(); + return err; +} + +void RemoteFilesystemClient::_update_cache_path(String &r_cache_path) { + r_cache_path = cache_path.path_join(FILES_SUBFOLDER); +} + +Error RemoteFilesystemClient::_synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path) { + cache_path = r_cache_path; + { + Ref dir = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); + dir->change_dir(cache_path); + cache_path = dir->get_current_dir(); + } + + Ref tcp_client; + tcp_client.instantiate(); + + IPAddress ip = p_host.is_valid_ip_address() ? ip : IP::get_singleton()->resolve_hostname(p_host); + ERR_FAIL_COND_V_MSG(!ip.is_valid(), ERR_INVALID_PARAMETER, "Unable to resolve remote filesystem server hostname: " + p_host); + print_verbose("Remote Filesystem: Connecting to host " + String(ip) + ", port: " + itos(p_port)); + Error err = tcp_client->connect_to_host(ip, p_port); + ERR_FAIL_COND_V_MSG(err != OK, err, "Unable to open connection to remote file server (" + String(p_host) + ", port " + itos(p_port) + ") failed."); + + while (tcp_client->get_status() == StreamPeerTCP::STATUS_CONNECTING) { + tcp_client->poll(); + OS::get_singleton()->delay_usec(100); + } + + if (tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED) { + ERR_FAIL_V_MSG(ERR_CANT_CONNECT, "Connection to remote file server (" + String(p_host) + ", port " + itos(p_port) + ") failed."); + } + + // Connection OK, now send the current file state. + print_verbose("Remote Filesystem: Connection OK."); + + // Header (GRFS) - Godot Remote File System + print_verbose("Remote Filesystem: Sending Header"); + tcp_client->put_u8('G'); + tcp_client->put_u8('R'); + tcp_client->put_u8('F'); + tcp_client->put_u8('S'); + // Protocol version + tcp_client->put_32(FILESYSTEM_PROTOCOL_VERSION); + print_verbose("Remote Filesystem: Send password"); + uint8_t password[PASSWORD_LENGTH]; // Send fixed size password, since its easier and safe to validate. + for (int i = 0; i < PASSWORD_LENGTH; i++) { + if (i < p_password.length()) { + password[i] = p_password[i]; + } else { + password[i] = 0; + } + } + tcp_client->put_data(password, PASSWORD_LENGTH); + print_verbose("Remote Filesystem: Tags"); + Vector tags; + { + String os_name = OS::get_singleton()->get_name().to_lower(); + if (os_name == "Linux" || os_name == "FreeBSD" || os_name == "NetBSD" || os_name == "OpenBSD" || os_name == "BSD") { + os_name = "linuxbsd"; + } + tags.push_back(os_name); + switch (OS::get_singleton()->get_preferred_texture_format()) { + case OS::PREFERRED_TEXTURE_FORMAT_S3TC_BPTC: { + tags.push_back("bptc"); + tags.push_back("s3tc"); + } break; + case OS::PREFERRED_TEXTURE_FORMAT_ETC2_ASTC: { + tags.push_back("etc2"); + tags.push_back("astc"); + } break; + } + } + + tcp_client->put_32(tags.size()); + for (int i = 0; i < tags.size(); i++) { + tcp_client->put_utf8_string(tags[i]); + } + // Size of compressed list of files + print_verbose("Remote Filesystem: Sending File List"); + + Vector file_cache = _load_cache_file(); + + // Encode file cache to send it via network. + Vector file_cache_buffer; + if (file_cache.size()) { + StringBuilder sbuild; + for (int i = 0; i < file_cache.size(); i++) { + sbuild.append(file_cache[i].path); + sbuild.append("::"); + sbuild.append(itos(file_cache[i].server_modified_time)); + sbuild.append("\n"); + } + String s = sbuild.as_string(); + CharString cs = s.utf8(); + file_cache_buffer.resize(Compression::get_max_compressed_buffer_size(cs.length(), Compression::MODE_ZSTD)); + int res_len = Compression::compress(file_cache_buffer.ptrw(), (const uint8_t *)cs.ptr(), cs.length(), Compression::MODE_ZSTD); + file_cache_buffer.resize(res_len); + + tcp_client->put_32(cs.length()); // Size of buffer uncompressed + tcp_client->put_32(file_cache_buffer.size()); // Size of buffer compressed + tcp_client->put_data(file_cache_buffer.ptr(), file_cache_buffer.size()); // Buffer + } else { + tcp_client->put_32(0); // No file cache buffer + } + + tcp_client->poll(); + ERR_FAIL_COND_V_MSG(tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED, ERR_CONNECTION_ERROR, "Remote filesystem server disconnected after sending header."); + + uint32_t file_count = tcp_client->get_32(); + + ERR_FAIL_COND_V_MSG(tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED, ERR_CONNECTION_ERROR, "Remote filesystem server disconnected while waiting for file list"); + + LocalVector file_buffer; + + Vector new_file_cache; + HashSet files_processed; + + for (uint32_t i = 0; i < file_count; i++) { + String file = tcp_client->get_utf8_string(); + ERR_FAIL_COND_V_MSG(file == String(), ERR_CONNECTION_ERROR, "Invalid file name received from remote filesystem."); + uint64_t server_modified_time = tcp_client->get_u64(); + ERR_FAIL_COND_V_MSG(tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED, ERR_CONNECTION_ERROR, "Remote filesystem server disconnected while waiting for file info."); + + if (server_modified_time == 0) { + // File was removed, remove locally + _remove_file(file); + files_processed.insert(file); + continue; + } + + uint64_t file_size = tcp_client->get_u64(); + ERR_FAIL_COND_V_MSG(file_size == 0, ERR_CONNECTION_ERROR, "Invalid file size received from remote filesystem."); + file_buffer.resize(file_size); + ERR_FAIL_COND_V_MSG(file_buffer.size() != file_size, ERR_CONNECTION_ERROR, "Remote filesystem sent a file (" + file + ") too big (Unable to allocate memory): " + String::humanize_size(file_size)); + err = tcp_client->get_data(file_buffer.ptr(), file_size); + ERR_FAIL_COND_V_MSG(err != OK, ERR_CONNECTION_ERROR, "Error retrieving file from remote filesystem: " + file); + + uint64_t modified_time; + err = _store_file(file, file_buffer, modified_time); + ERR_FAIL_COND_V_MSG(err != OK, ERR_FILE_CANT_OPEN, "Error writing file retrieved from remote filesystem: " + file); + print_verbose("Receiving file " + itos(i + 1) + "/" + itos(file_count) + ": " + file); + FileCache fc; + fc.path = file; + fc.server_modified_time = server_modified_time; + fc.modified_time = modified_time; + new_file_cache.push_back(fc); + files_processed.insert(file); + } + + print_verbose("Updating the cache file."); + + // Merge file cache. + for (int i = 0; i < file_cache.size(); i++) { + if (files_processed.has(file_cache[i].path)) { + continue; // This was either added or removed, so skip. + } + new_file_cache.push_back(file_cache[i]); + } + + err = _store_cache_file(new_file_cache); + ERR_FAIL_COND_V_MSG(err != OK, ERR_FILE_CANT_OPEN, "Error writing the remote filesystem file cache. "); + + print_verbose("Remote Filesystem: Update success."); + + _update_cache_path(r_cache_path); + return OK; +} diff --git a/core/io/remote_filesystem_client.h b/core/io/remote_filesystem_client.h new file mode 100644 index 00000000000000..820f370a0abb6e --- /dev/null +++ b/core/io/remote_filesystem_client.h @@ -0,0 +1,65 @@ +/**************************************************************************/ +/* remote_filesystem_client.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#ifndef REMOTE_FILESYSTEM_CLIENT_H +#define REMOTE_FILESYSTEM_CLIENT_H + +#include "core/io/ip_address.h" +#include "core/string/ustring.h" +#include "core/templates/hash_set.h" +#include "core/templates/local_vector.h" + +class RemoteFilesystemClient { + String cache_path; + HashSet validated_directories; + +protected: + String _get_cache_path() { return cache_path; } + struct FileCache { + String path; // Local path (as in "folder/to/file.png") + uint64_t server_modified_time; // MD5 checksum. + uint64_t modified_time; + }; + virtual bool _is_configured() { return !cache_path.is_empty(); } + // Can be re-implemented per platform. If so, feel free to ignore get_cache_path() + virtual Vector _load_cache_file(); + virtual Error _store_file(const String &p_path, const LocalVector &p_file, uint64_t &modified_time); + virtual Error _remove_file(const String &p_path); + virtual Error _store_cache_file(const Vector &p_cache); + virtual Error _synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path); + + virtual void _update_cache_path(String &r_cache_path); + +public: + Error synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path); + ~RemoteFilesystemClient() {} +}; + +#endif // REMOTE_FILESYSTEM_CLIENT_H diff --git a/core/os/os.cpp b/core/os/os.cpp index c82f87c731d55b..e8ccf44d5f8dff 100644 --- a/core/os/os.cpp +++ b/core/os/os.cpp @@ -569,6 +569,10 @@ void OS::add_frame_delay(bool p_can_draw) { } } +Error OS::setup_remote_filesystem(const String &p_server_host, int p_port, const String &p_password, String &r_project_path) { + return default_rfs.synchronize_with_server(p_server_host, p_port, p_password, r_project_path); +} + OS::PreferredTextureFormat OS::get_preferred_texture_format() const { #if defined(__arm__) || defined(__aarch64__) || defined(_M_ARM) || defined(_M_ARM64) return PREFERRED_TEXTURE_FORMAT_ETC2_ASTC; // By rule, ARM hardware uses ETC texture compression. diff --git a/core/os/os.h b/core/os/os.h index 3248330c74c0f2..2656532e093fc5 100644 --- a/core/os/os.h +++ b/core/os/os.h @@ -34,6 +34,7 @@ #include "core/config/engine.h" #include "core/io/image.h" #include "core/io/logger.h" +#include "core/io/remote_filesystem_client.h" #include "core/os/time_enums.h" #include "core/string/ustring.h" #include "core/templates/list.h" @@ -72,6 +73,8 @@ class OS { String _current_rendering_driver_name; String _current_rendering_method; + RemoteFilesystemClient default_rfs; + protected: void _set_logger(CompositeLogger *p_logger); @@ -292,6 +295,8 @@ class OS { virtual void process_and_drop_events() {} + virtual Error setup_remote_filesystem(const String &p_server_host, int p_port, const String &p_password, String &r_project_path); + enum PreferredTextureFormat { PREFERRED_TEXTURE_FORMAT_S3TC_BPTC, PREFERRED_TEXTURE_FORMAT_ETC2_ASTC diff --git a/core/string/print_string.cpp b/core/string/print_string.cpp index 7b894d83bf0ea4..7b90710308cd05 100644 --- a/core/string/print_string.cpp +++ b/core/string/print_string.cpp @@ -193,10 +193,8 @@ void print_error(String p_string) { _global_unlock(); } -void print_verbose(String p_string) { - if (OS::get_singleton()->is_stdout_verbose()) { - print_line(p_string); - } +bool is_print_verbose_enabled() { + return OS::get_singleton()->is_stdout_verbose(); } String stringify_variants(Variant p_var) { diff --git a/core/string/print_string.h b/core/string/print_string.h index 6496384b3f6a90..d1eeb15815322c 100644 --- a/core/string/print_string.h +++ b/core/string/print_string.h @@ -59,7 +59,14 @@ void remove_print_handler(const PrintHandlerList *p_handler); extern void __print_line(String p_string); extern void __print_line_rich(String p_string); extern void print_error(String p_string); -extern void print_verbose(String p_string); +extern bool is_print_verbose_enabled(); + +// This version avoids processing the text to be printed until it actually has to be printed, saving some CPU usage. +#define print_verbose(m_text) \ + { \ + if (is_print_verbose_enabled()) \ + print_line(m_text); \ + } inline void print_line(Variant v) { __print_line(stringify_variants(v)); diff --git a/core/variant/variant_utility.cpp b/core/variant/variant_utility.cpp index fd079dbeeaaa63..bfade19c956d7d 100644 --- a/core/variant/variant_utility.cpp +++ b/core/variant/variant_utility.cpp @@ -803,7 +803,7 @@ struct VariantUtilityFunctions { print_line_rich(s); r_error.error = Callable::CallError::CALL_OK; } - +#undef print_verbose static inline void print_verbose(const Variant **p_args, int p_arg_count, Callable::CallError &r_error) { if (OS::get_singleton()->is_stdout_verbose()) { String s; diff --git a/editor/debugger/editor_file_server.cpp b/editor/debugger/editor_file_server.cpp index ba5dd9e6e2c3e3..a0a383f9efaacf 100644 --- a/editor/debugger/editor_file_server.cpp +++ b/editor/debugger/editor_file_server.cpp @@ -32,273 +32,229 @@ #include "../editor_settings.h" #include "core/io/marshalls.h" +#include "editor/editor_node.h" +#include "editor/export/editor_export_platform.h" + +#define FILESYSTEM_PROTOCOL_VERSION 1 +#define PASSWORD_LENGTH 32 +#define MAX_FILE_BUFFER_SIZE 100 * 1024 * 1024 // 100mb max file buffer size (description of files to update, compressed). + +static void _add_file(String f, const uint64_t &p_modified_time, HashMap &files_to_send, HashMap &cached_files) { + f = f.replace_first("res://", ""); // remove res:// + const uint64_t *cached_mt = cached_files.getptr(f); + if (cached_mt && *cached_mt == p_modified_time) { + // File is good, skip it. + cached_files.erase(f); // Erase to mark this file as existing. Remaning files not added to files_to_send will be considered erased here, so they need to be erased in the client too. + return; + } + files_to_send.insert(f, p_modified_time); +} -//#define DEBUG_PRINT(m_p) print_line(m_p) -//#define DEBUG_TIME(m_what) printf("MS: %s - %lu\n", m_what, OS::get_singleton()->get_ticks_usec()); +void EditorFileServer::_scan_files_changed(EditorFileSystemDirectory *efd, const Vector &p_tags, HashMap &files_to_send, HashMap &cached_files) { + for (int i = 0; i < efd->get_file_count(); i++) { + String f = efd->get_file_path(i); + if (FileAccess::exists(f + ".import")) { + // is imported, determine what to do + // Todo the modified times of remapped files should most likely be kept in EditorFileSystem to speed this up in the future. + Ref cf; + cf.instantiate(); + Error err = cf->load(f + ".import"); + + ERR_CONTINUE(err != OK); + { + uint64_t mt = FileAccess::get_modified_time(f + ".import"); + _add_file(f + ".import", mt, files_to_send, cached_files); + } -#define DEBUG_PRINT(m_what) -#define DEBUG_TIME(m_what) + if (!cf->has_section("remap")) { + continue; + } -void EditorFileServer::_close_client(ClientData *cd) { - cd->connection->disconnect_from_host(); - { - MutexLock lock(cd->efs->wait_mutex); - cd->efs->to_wait.insert(cd->thread); + List remaps; + cf->get_section_keys("remap", &remaps); + + for (const String &F : remaps) { + String remap = F; + if (remap == "path") { + String remapped_path = cf->get_value("remap", remap); + uint64_t mt = FileAccess::get_modified_time(remapped_path); + _add_file(remapped_path, mt, files_to_send, cached_files); + } else if (remap.begins_with("path.")) { + String feature = remap.get_slice(".", 1); + if (p_tags.find(feature) != -1) { + String remapped_path = cf->get_value("remap", remap); + uint64_t mt = FileAccess::get_modified_time(remapped_path); + _add_file(remapped_path, mt, files_to_send, cached_files); + } + } + } + } else { + uint64_t mt = efd->get_file_modified_time(i); + _add_file(f, mt, files_to_send, cached_files); + } } - while (cd->files.size()) { - cd->files.remove(cd->files.begin()); + + for (int i = 0; i < efd->get_subdir_count(); i++) { + _scan_files_changed(efd->get_subdir(i), p_tags, files_to_send, cached_files); } - memdelete(cd); } -void EditorFileServer::_subthread_start(void *s) { - ClientData *cd = static_cast(s); - - cd->connection->set_no_delay(true); - uint8_t buf4[8]; - Error err = cd->connection->get_data(buf4, 4); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); +static void _add_custom_file(const String f, HashMap &files_to_send, HashMap &cached_files) { + if (!FileAccess::exists(f)) { + return; } + _add_file(f, FileAccess::get_modified_time(f), files_to_send, cached_files); +} - int passlen = decode_uint32(buf4); - - if (passlen > 512) { - _close_client(cd); - ERR_FAIL_COND(passlen > 512); - } else if (passlen > 0) { - Vector passutf8; - passutf8.resize(passlen + 1); - err = cd->connection->get_data((uint8_t *)passutf8.ptr(), passlen); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - passutf8.write[passlen] = 0; - String s2; - s2.parse_utf8(passutf8.ptr()); - if (s2 != cd->efs->password) { - encode_uint32(ERR_INVALID_DATA, buf4); - cd->connection->put_data(buf4, 4); - OS::get_singleton()->delay_usec(1000000); - _close_client(cd); - ERR_PRINT("CLIENT PASSWORD MISMATCH"); - ERR_FAIL(); - } - } else { - if (!cd->efs->password.is_empty()) { - encode_uint32(ERR_INVALID_DATA, buf4); - cd->connection->put_data(buf4, 4); - OS::get_singleton()->delay_usec(1000000); - _close_client(cd); - ERR_PRINT("CLIENT PASSWORD MISMATCH (should be empty!)"); - ERR_FAIL(); - } +void EditorFileServer::poll() { + if (!active) { + return; } - encode_uint32(OK, buf4); - cd->connection->put_data(buf4, 4); + if (!server->is_connection_available()) { + return; + } - while (!cd->quit) { - //wait for ID - err = cd->connection->get_data(buf4, 4); - DEBUG_TIME("get_data") + Ref tcp_peer = server->take_connection(); + ERR_FAIL_COND(tcp_peer.is_null()); + + // Got a connection! + EditorProgress pr("updating_remote_file_system", TTR("Updating assets on target device:"), 104); + + pr.step(TTR("Syncinc headers"), 0, true); + print_verbose("EFS: Connecting taken!"); + char header[4]; + Error err = tcp_peer->get_data((uint8_t *)&header, 4); + ERR_FAIL_COND(err != OK); + ERR_FAIL_COND(header[0] != 'G'); + ERR_FAIL_COND(header[1] != 'R'); + ERR_FAIL_COND(header[2] != 'F'); + ERR_FAIL_COND(header[3] != 'S'); + + uint32_t protocol_version = tcp_peer->get_u32(); + ERR_FAIL_COND(protocol_version != FILESYSTEM_PROTOCOL_VERSION); + + char password[PASSWORD_LENGTH + 1]; + err = tcp_peer->get_data((uint8_t *)password, PASSWORD_LENGTH); + password[PASSWORD_LENGTH] = 0; + ERR_FAIL_COND(err != OK); + String pw = password; + print_verbose("EFS: Got password: " + pw); + ERR_FAIL_COND_MSG(pw != password, "Client disconnected because password mismatch."); + + uint32_t tag_count = tcp_peer->get_u32(); + print_verbose("EFS: Getting tags: " + itos(tag_count)); + + ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED); + Vector tags; + for (uint32_t i = 0; i < tag_count; i++) { + String tag = tcp_peer->get_utf8_string(); + print_verbose("EFS: tag #" + itos(i) + ": " + tag); + ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED); + tags.push_back(tag); + } - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - int id = decode_uint32(buf4); + uint32_t file_buffer_decompressed_size = tcp_peer->get_32(); + HashMap cached_files; - //wait for command - err = cd->connection->get_data(buf4, 4); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - int cmd = decode_uint32(buf4); - - switch (cmd) { - case FileAccessNetwork::COMMAND_FILE_EXISTS: - case FileAccessNetwork::COMMAND_GET_MODTIME: - case FileAccessNetwork::COMMAND_OPEN_FILE: { - DEBUG_TIME("open_file") - err = cd->connection->get_data(buf4, 4); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } + if (file_buffer_decompressed_size > 0) { + pr.step(TTR("Getting remote file system"), 1, true); - int namelen = decode_uint32(buf4); - Vector fileutf8; - fileutf8.resize(namelen + 1); - err = cd->connection->get_data((uint8_t *)fileutf8.ptr(), namelen); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - fileutf8.write[namelen] = 0; - String s2; - s2.parse_utf8(fileutf8.ptr()); + // Got files cached by client. + uint32_t file_buffer_size = tcp_peer->get_32(); + print_verbose("EFS: Getting file buffer: compressed - " + String::humanize_size(file_buffer_size) + " decompressed: " + String::humanize_size(file_buffer_decompressed_size)); - if (cmd == FileAccessNetwork::COMMAND_FILE_EXISTS) { - print_verbose("FILE EXISTS: " + s2); - } - if (cmd == FileAccessNetwork::COMMAND_GET_MODTIME) { - print_verbose("MOD TIME: " + s2); - } - if (cmd == FileAccessNetwork::COMMAND_OPEN_FILE) { - print_verbose("OPEN: " + s2); - } + ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED); + ERR_FAIL_COND(file_buffer_size > MAX_FILE_BUFFER_SIZE); + LocalVector file_buffer; + file_buffer.resize(file_buffer_size); + LocalVector file_buffer_decompressed; + file_buffer_decompressed.resize(file_buffer_decompressed_size); - if (!s2.begins_with("res://")) { - _close_client(cd); - ERR_FAIL_COND(!s2.begins_with("res://")); - } - ERR_CONTINUE(cd->files.has(id)); - - if (cmd == FileAccessNetwork::COMMAND_FILE_EXISTS) { - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_FILE_EXISTS, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccess::exists(s2), buf4); - cd->connection->put_data(buf4, 4); - DEBUG_TIME("open_file_end") - break; - } + err = tcp_peer->get_data(file_buffer.ptr(), file_buffer_size); - if (cmd == FileAccessNetwork::COMMAND_GET_MODTIME) { - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_GET_MODTIME, buf4); - cd->connection->put_data(buf4, 4); - encode_uint64(FileAccess::get_modified_time(s2), buf4); - cd->connection->put_data(buf4, 8); - DEBUG_TIME("open_file_end") - break; - } + pr.step(TTR("Decompressing remote file system"), 2, true); - Ref fa = FileAccess::open(s2, FileAccess::READ); - if (fa.is_null()) { - //not found, continue - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_OPEN, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(ERR_FILE_NOT_FOUND, buf4); - cd->connection->put_data(buf4, 4); - DEBUG_TIME("open_file_end") - break; - } + ERR_FAIL_COND(err != OK); + // Decompress the text with all the files + Compression::decompress(file_buffer_decompressed.ptr(), file_buffer_decompressed.size(), file_buffer.ptr(), file_buffer.size(), Compression::MODE_ZSTD); + String files_text = String::utf8((const char *)file_buffer_decompressed.ptr(), file_buffer_decompressed.size()); + Vector files = files_text.split("\n"); + + print_verbose("EFS: Total cached files received: " + itos(files.size())); + for (int i = 0; i < files.size(); i++) { + if (files[i].get_slice_count("::") != 2) { + continue; + } + String file = files[i].get_slice("::", 0); + uint64_t modified_time = files[i].get_slice("::", 1).to_int(); - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_OPEN, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(OK, buf4); - cd->connection->put_data(buf4, 4); - encode_uint64(fa->get_length(), buf4); - cd->connection->put_data(buf4, 8); - - cd->files[id] = fa; - DEBUG_TIME("open_file_end") - - } break; - case FileAccessNetwork::COMMAND_READ_BLOCK: { - err = cd->connection->get_data(buf4, 8); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } + cached_files.insert(file, modified_time); + } + } else { + // Client does not have any files stored. + } - ERR_CONTINUE(!cd->files.has(id)); + pr.step(TTR("Scanning for local changes"), 3, true); - uint64_t offset = decode_uint64(buf4); + print_verbose("EFS: Scanning changes:"); - err = cd->connection->get_data(buf4, 4); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } + HashMap files_to_send; + // Scan files to send. + _scan_files_changed(EditorFileSystem::get_singleton()->get_filesystem(), tags, files_to_send, cached_files); + // Add forced export files + Vector forced_export = EditorExportPlatform::get_forced_export_files(); + for (int i = 0; i < forced_export.size(); i++) { + _add_custom_file(forced_export[i], files_to_send, cached_files); + } - int blocklen = decode_uint32(buf4); - ERR_CONTINUE(blocklen > (16 * 1024 * 1024)); - - cd->files[id]->seek(offset); - Vector buf; - buf.resize(blocklen); - uint32_t read = cd->files[id]->get_buffer(buf.ptrw(), blocklen); - - print_verbose("GET BLOCK - offset: " + itos(offset) + ", blocklen: " + itos(blocklen)); - - //not found, continue - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_DATA, buf4); - cd->connection->put_data(buf4, 4); - encode_uint64(offset, buf4); - cd->connection->put_data(buf4, 8); - encode_uint32(read, buf4); - cd->connection->put_data(buf4, 4); - cd->connection->put_data(buf.ptr(), read); - - } break; - case FileAccessNetwork::COMMAND_CLOSE: { - print_verbose("CLOSED"); - ERR_CONTINUE(!cd->files.has(id)); - cd->files.erase(id); - } break; + _add_custom_file("res://project.godot", files_to_send, cached_files); + // Check which files were removed and also add them + for (KeyValue K : cached_files) { + if (!files_to_send.has(K.key)) { + files_to_send.insert(K.key, 0); //0 means removed } } - _close_client(cd); -} + print_verbose("EFS: Sending " + itos(files_to_send.size()) + " files."); -void EditorFileServer::_thread_start(void *s) { - EditorFileServer *self = static_cast(s); - while (!self->quit) { - if (self->cmd == CMD_ACTIVATE) { - self->server->listen(self->port); - self->active = true; - self->cmd = CMD_NONE; - } else if (self->cmd == CMD_STOP) { - self->server->stop(); - self->active = false; - self->cmd = CMD_NONE; - } + tcp_peer->put_32(files_to_send.size()); - if (self->active) { - if (self->server->is_connection_available()) { - ClientData *cd = memnew(ClientData); - cd->connection = self->server->take_connection(); - cd->efs = self; - cd->quit = false; - cd->thread = memnew(Thread); - cd->thread->start(_subthread_start, cd); - } - } + int idx = 0; + for (KeyValue K : files_to_send) { + pr.step(TTR("Sending file: ") + K.key.get_file(), 4 + idx * 100 / files_to_send.size(), false); + idx++; - self->wait_mutex.lock(); - while (self->to_wait.size()) { - Thread *w = *self->to_wait.begin(); - self->to_wait.erase(w); - self->wait_mutex.unlock(); - w->wait_to_finish(); - self->wait_mutex.lock(); - } - self->wait_mutex.unlock(); + tcp_peer->put_utf8_string(K.key); - OS::get_singleton()->delay_usec(100000); + Vector array = FileAccess::_get_file_as_bytes("res://" + K.key); + if (array.size() == 0 && !FileAccess::exists("res://" + K.key)) { + WARN_PRINT_ED("File " + K.key + " was not found."); + tcp_peer->put_64(0); // Send file removal + continue; + } + tcp_peer->put_64(K.value); + if (K.value == 0) { + // File removal. + continue; + } + tcp_peer->put_64(array.size()); + tcp_peer->put_data(array.ptr(), array.size()); + ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED); } + + print_verbose("EFS: Done."); } void EditorFileServer::start() { - stop(); + if (active) { + stop(); + } port = EDITOR_GET("filesystem/file_server/port"); password = EDITOR_GET("filesystem/file_server/password"); - cmd = CMD_ACTIVATE; + Error err = server->listen(port); + ERR_FAIL_COND_MSG(err != OK, "EditorFileServer: Unable to listen on port " + itos(port)); + active = true; } bool EditorFileServer::is_active() const { @@ -306,18 +262,19 @@ bool EditorFileServer::is_active() const { } void EditorFileServer::stop() { - cmd = CMD_STOP; + if (active) { + server->stop(); + active = false; + } } EditorFileServer::EditorFileServer() { server.instantiate(); - thread.start(_thread_start, this); EDITOR_DEF("filesystem/file_server/port", 6010); EDITOR_DEF("filesystem/file_server/password", ""); } EditorFileServer::~EditorFileServer() { - quit = true; - thread.wait_to_finish(); + stop(); } diff --git a/editor/debugger/editor_file_server.h b/editor/debugger/editor_file_server.h index ff2742e73fea4c..f77c65668ab82e 100644 --- a/editor/debugger/editor_file_server.h +++ b/editor/debugger/editor_file_server.h @@ -36,41 +36,20 @@ #include "core/io/tcp_server.h" #include "core/object/class_db.h" #include "core/os/thread.h" +#include "editor/editor_file_system.h" class EditorFileServer : public Object { GDCLASS(EditorFileServer, Object); - enum Command { - CMD_NONE, - CMD_ACTIVATE, - CMD_STOP, - }; - - struct ClientData { - Thread *thread = nullptr; - Ref connection; - HashMap> files; - EditorFileServer *efs = nullptr; - bool quit = false; - }; - Ref server; - HashSet to_wait; - - static void _close_client(ClientData *cd); - static void _subthread_start(void *s); - - Mutex wait_mutex; - Thread thread; - static void _thread_start(void *); - bool quit = false; - Command cmd = CMD_NONE; - String password; int port = 0; bool active = false; + void _scan_files_changed(EditorFileSystemDirectory *efd, const Vector &p_tags, HashMap &files_to_send, HashMap &cached_files); public: + void poll(); + void start(); void stop(); diff --git a/editor/export/editor_export_platform.cpp b/editor/export/editor_export_platform.cpp index 121088f27a51d7..c74ebff974f42c 100644 --- a/editor/export/editor_export_platform.cpp +++ b/editor/export/editor_export_platform.cpp @@ -818,12 +818,56 @@ String EditorExportPlatform::_export_customize(const String &p_path, LocalVector return save_path.is_empty() ? p_path : save_path; } +Vector EditorExportPlatform::get_forced_export_files() { + Vector files; + + files.push_back(ProjectSettings::get_singleton()->get_global_class_list_path()); + + String icon = GLOBAL_GET("application/config/icon"); + String splash = GLOBAL_GET("application/boot_splash/image"); + if (!icon.is_empty() && FileAccess::exists(icon)) { + files.push_back(icon); + } + if (!splash.is_empty() && FileAccess::exists(splash) && icon != splash) { + files.push_back(splash); + } + String resource_cache_file = ResourceUID::get_cache_file(); + if (FileAccess::exists(resource_cache_file)) { + files.push_back(resource_cache_file); + } + + String extension_list_config_file = GDExtension::get_extension_list_config_file(); + if (FileAccess::exists(extension_list_config_file)) { + files.push_back(extension_list_config_file); + } + + // Store text server data if it is supported. + if (TS->has_feature(TextServer::FEATURE_USE_SUPPORT_DATA)) { + bool use_data = GLOBAL_GET("internationalization/locale/include_text_server_data"); + if (use_data) { + // Try using user provided data file. + String ts_data = "res://" + TS->get_support_data_filename(); + if (FileAccess::exists(ts_data)) { + files.push_back(ts_data); + } else { + // Use default text server data. + String icu_data_file = EditorPaths::get_singleton()->get_cache_dir().path_join("tmp_icu_data"); + ERR_FAIL_COND_V(!TS->save_support_data(icu_data_file), files); + files.push_back(icu_data_file); + //Remove later + MessageQueue::get_singleton()->push_callable(callable_mp_static(DirAccess::remove_absolute), icu_data_file); + } + } + } + + return files; +} + Error EditorExportPlatform::export_project_files(const Ref &p_preset, bool p_debug, EditorExportSaveFunction p_func, void *p_udata, EditorExportSaveSharedObject p_so_func) { //figure out paths of files that will be exported HashSet paths; Vector path_remaps; - paths.insert(ProjectSettings::get_singleton()->get_global_class_list_path()); if (p_preset->get_export_filter() == EditorExportPreset::EXPORT_ALL_RESOURCES) { //find stuff _export_find_resources(EditorFileSystem::get_singleton()->get_filesystem(), paths); @@ -1295,69 +1339,15 @@ Error EditorExportPlatform::export_project_files(const Ref & } } - // Store icon and splash images directly, they need to bypass the import system and be loaded as images - String icon = GLOBAL_GET("application/config/icon"); - String splash = GLOBAL_GET("application/boot_splash/image"); - if (!icon.is_empty() && FileAccess::exists(icon)) { - Vector array = FileAccess::get_file_as_bytes(icon); - err = p_func(p_udata, icon, array, idx, total, enc_in_filters, enc_ex_filters, key); - if (err != OK) { - return err; - } - } - if (!splash.is_empty() && FileAccess::exists(splash) && icon != splash) { - Vector array = FileAccess::get_file_as_bytes(splash); - err = p_func(p_udata, splash, array, idx, total, enc_in_filters, enc_ex_filters, key); - if (err != OK) { - return err; - } - } - String resource_cache_file = ResourceUID::get_cache_file(); - if (FileAccess::exists(resource_cache_file)) { - Vector array = FileAccess::get_file_as_bytes(resource_cache_file); - err = p_func(p_udata, resource_cache_file, array, idx, total, enc_in_filters, enc_ex_filters, key); - if (err != OK) { - return err; - } - } - - String extension_list_config_file = GDExtension::get_extension_list_config_file(); - if (FileAccess::exists(extension_list_config_file)) { - Vector array = FileAccess::get_file_as_bytes(extension_list_config_file); - err = p_func(p_udata, extension_list_config_file, array, idx, total, enc_in_filters, enc_ex_filters, key); + Vector forced_export = get_forced_export_files(); + for (int i = 0; i < forced_export.size(); i++) { + Vector array = FileAccess::get_file_as_bytes(forced_export[i]); + err = p_func(p_udata, forced_export[i], array, idx, total, enc_in_filters, enc_ex_filters, key); if (err != OK) { return err; } } - // Store text server data if it is supported. - if (TS->has_feature(TextServer::FEATURE_USE_SUPPORT_DATA)) { - bool use_data = GLOBAL_GET("internationalization/locale/include_text_server_data"); - if (use_data) { - // Try using user provided data file. - String ts_data = "res://" + TS->get_support_data_filename(); - if (FileAccess::exists(ts_data)) { - Vector array = FileAccess::get_file_as_bytes(ts_data); - err = p_func(p_udata, ts_data, array, idx, total, enc_in_filters, enc_ex_filters, key); - if (err != OK) { - return err; - } - } else { - // Use default text server data. - String icu_data_file = EditorPaths::get_singleton()->get_cache_dir().path_join("tmp_icu_data"); - if (!TS->save_support_data(icu_data_file)) { - return ERR_INVALID_DATA; - } - Vector array = FileAccess::get_file_as_bytes(icu_data_file); - err = p_func(p_udata, ts_data, array, idx, total, enc_in_filters, enc_ex_filters, key); - DirAccess::remove_file_or_error(icu_data_file); - if (err != OK) { - return err; - } - } - } - } - String config_file = "project.binary"; String engine_cfb = EditorPaths::get_singleton()->get_cache_dir().path_join("tmp" + config_file); ProjectSettings::get_singleton()->save_custom(engine_cfb, custom_map, custom_list); diff --git a/editor/export/editor_export_platform.h b/editor/export/editor_export_platform.h index 21d871990224a1..df5a66f099a738 100644 --- a/editor/export/editor_export_platform.h +++ b/editor/export/editor_export_platform.h @@ -196,6 +196,8 @@ class EditorExportPlatform : public RefCounted { return worst_type; } + static Vector get_forced_export_files(); + virtual bool fill_log_messages(RichTextLabel *p_log, Error p_err); virtual void get_export_options(List *r_options) const = 0; diff --git a/editor/plugins/debugger_editor_plugin.cpp b/editor/plugins/debugger_editor_plugin.cpp index 7863e6d19e4f8d..f43f0397ad2929 100644 --- a/editor/plugins/debugger_editor_plugin.cpp +++ b/editor/plugins/debugger_editor_plugin.cpp @@ -126,8 +126,10 @@ void DebuggerEditorPlugin::_menu_option(int p_option) { if (ischecked) { file_server->stop(); + set_process(false); } else { file_server->start(); + set_process(true); } debug_menu->set_item_checked(debug_menu->get_item_index(RUN_FILE_SERVER), !ischecked); @@ -190,6 +192,9 @@ void DebuggerEditorPlugin::_notification(int p_what) { case NOTIFICATION_READY: { _update_debug_options(); } break; + case NOTIFICATION_PROCESS: { + file_server->poll(); + } break; } } diff --git a/main/main.cpp b/main/main.cpp index 5e0187cc7fd982..97f807c528e558 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -129,7 +129,6 @@ static Time *time_singleton = nullptr; #ifdef MINIZIP_ENABLED static ZipArchive *zip_packed_data = nullptr; #endif -static FileAccessNetworkClient *file_access_network_client = nullptr; static MessageQueue *message_queue = nullptr; // Initialized in setup2() @@ -1384,9 +1383,7 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph // Network file system needs to be configured before globals, since globals are based on the // 'project.godot' file which will only be available through the network if this is enabled - FileAccessNetwork::configure(); if (!remotefs.is_empty()) { - file_access_network_client = memnew(FileAccessNetworkClient); int port; if (remotefs.contains(":")) { port = remotefs.get_slicec(':', 1).to_int(); @@ -1394,14 +1391,14 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph } else { port = 6010; } + Error err = OS::get_singleton()->setup_remote_filesystem(remotefs, port, remotefs_pass, project_path); - Error err = file_access_network_client->connect(remotefs, port, remotefs_pass); if (err) { OS::get_singleton()->printerr("Could not connect to remotefs: %s:%i.\n", remotefs.utf8().get_data(), port); goto error; } - FileAccess::make_default(FileAccess::ACCESS_RESOURCES); + print_line("the path is now: " + project_path); } if (globals->setup(project_path, main_pack, upwards, editor) == OK) { @@ -1937,9 +1934,6 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph if (packed_data) { memdelete(packed_data); } - if (file_access_network_client) { - memdelete(file_access_network_client); - } unregister_core_driver_types(); unregister_core_extensions(); @@ -3448,9 +3442,6 @@ void Main::cleanup(bool p_force) { if (packed_data) { memdelete(packed_data); } - if (file_access_network_client) { - memdelete(file_access_network_client); - } if (performance) { memdelete(performance); }