From 7d35423a404a72508666d44f8592dba0b3acd327 Mon Sep 17 00:00:00 2001 From: Juan Linietsky Date: Fri, 28 Apr 2023 13:15:36 +0200 Subject: [PATCH] Redo how the remote filesystem works Instead of reading files over the network, the new version uses a local file cache and only updates files when it changes. The original remote filesystem was created 14 years ago, when ethernet was faster than hard drives or even flash. Also, mobile devices have a very small amount of storage. Nowadays, this is no longer the case so the approach is changed to using a persistent cache in the target device. --- core/config/project_settings.cpp | 12 - core/io/file_access_network.cpp | 498 ---------------------- core/io/file_access_network.h | 167 -------- core/io/remote_filesystem_client.cpp | 333 +++++++++++++++ core/io/remote_filesystem_client.h | 65 +++ core/os/os.cpp | 4 + core/os/os.h | 5 + core/string/print_string.cpp | 6 +- core/string/print_string.h | 9 +- core/variant/variant_utility.cpp | 2 +- editor/debugger/editor_file_server.cpp | 425 +++++++++--------- editor/debugger/editor_file_server.h | 30 +- editor/export/editor_export_platform.cpp | 108 +++-- editor/export/editor_export_platform.h | 2 + editor/plugins/debugger_editor_plugin.cpp | 5 + main/main.cpp | 14 +- 16 files changed, 672 insertions(+), 1013 deletions(-) delete mode 100644 core/io/file_access_network.cpp delete mode 100644 core/io/file_access_network.h create mode 100644 core/io/remote_filesystem_client.cpp create mode 100644 core/io/remote_filesystem_client.h diff --git a/core/config/project_settings.cpp b/core/config/project_settings.cpp index 9fd7ef998876d8..264259eb2f32c8 100644 --- a/core/config/project_settings.cpp +++ b/core/config/project_settings.cpp @@ -36,7 +36,6 @@ #include "core/io/config_file.h" #include "core/io/dir_access.h" #include "core/io/file_access.h" -#include "core/io/file_access_network.h" #include "core/io/file_access_pack.h" #include "core/io/marshalls.h" #include "core/os/keyboard.h" @@ -502,17 +501,6 @@ Error ProjectSettings::_setup(const String &p_path, const String &p_main_pack, b } } - // If looking for files in a network client, use it directly - - if (FileAccessNetworkClient::get_singleton()) { - Error err = _load_settings_text_or_binary("res://project.godot", "res://project.binary"); - if (err == OK && !p_ignore_override) { - // Optional, we don't mind if it fails - _load_settings_text("res://override.cfg"); - } - return err; - } - // Attempt with a user-defined main pack first if (!p_main_pack.is_empty()) { diff --git a/core/io/file_access_network.cpp b/core/io/file_access_network.cpp deleted file mode 100644 index 7fabff26ac55a5..00000000000000 --- a/core/io/file_access_network.cpp +++ /dev/null @@ -1,498 +0,0 @@ -/**************************************************************************/ -/* file_access_network.cpp */ -/**************************************************************************/ -/* This file is part of: */ -/* GODOT ENGINE */ -/* https://godotengine.org */ -/**************************************************************************/ -/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ -/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ -/* */ -/* Permission is hereby granted, free of charge, to any person obtaining */ -/* a copy of this software and associated documentation files (the */ -/* "Software"), to deal in the Software without restriction, including */ -/* without limitation the rights to use, copy, modify, merge, publish, */ -/* distribute, sublicense, and/or sell copies of the Software, and to */ -/* permit persons to whom the Software is furnished to do so, subject to */ -/* the following conditions: */ -/* */ -/* The above copyright notice and this permission notice shall be */ -/* included in all copies or substantial portions of the Software. */ -/* */ -/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ -/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ -/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ -/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ -/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ -/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ -/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -/**************************************************************************/ - -#include "file_access_network.h" - -#include "core/config/project_settings.h" -#include "core/io/ip.h" -#include "core/io/marshalls.h" -#include "core/os/os.h" - -//#define DEBUG_PRINT(m_p) print_line(m_p) -//#define DEBUG_TIME(m_what) printf("MS: %s - %lli\n",m_what,OS::get_singleton()->get_ticks_usec()); -#define DEBUG_PRINT(m_p) -#define DEBUG_TIME(m_what) - -void FileAccessNetworkClient::lock_mutex() { - mutex.lock(); - lockcount++; -} - -void FileAccessNetworkClient::unlock_mutex() { - lockcount--; - mutex.unlock(); -} - -void FileAccessNetworkClient::put_32(int p_32) { - uint8_t buf[4]; - encode_uint32(p_32, buf); - client->put_data(buf, 4); - DEBUG_PRINT("put32: " + itos(p_32)); -} - -void FileAccessNetworkClient::put_64(int64_t p_64) { - uint8_t buf[8]; - encode_uint64(p_64, buf); - client->put_data(buf, 8); - DEBUG_PRINT("put64: " + itos(p_64)); -} - -int FileAccessNetworkClient::get_32() { - uint8_t buf[4]; - client->get_data(buf, 4); - return decode_uint32(buf); -} - -int64_t FileAccessNetworkClient::get_64() { - uint8_t buf[8]; - client->get_data(buf, 8); - return decode_uint64(buf); -} - -void FileAccessNetworkClient::_thread_func() { - client->set_no_delay(true); - while (!quit) { - DEBUG_PRINT("SEM WAIT - " + itos(sem->get())); - sem.wait(); - DEBUG_TIME("sem_unlock"); - //DEBUG_PRINT("semwait returned "+itos(werr)); - DEBUG_PRINT("MUTEX LOCK " + itos(lockcount)); - lock_mutex(); - DEBUG_PRINT("MUTEX PASS"); - - { - MutexLock lock(blockrequest_mutex); - while (block_requests.size()) { - put_32(block_requests.front()->get().id); - put_32(FileAccessNetwork::COMMAND_READ_BLOCK); - put_64(block_requests.front()->get().offset); - put_32(block_requests.front()->get().size); - block_requests.pop_front(); - } - } - - DEBUG_PRINT("THREAD ITER"); - - DEBUG_TIME("sem_read"); - int id = get_32(); - - int response = get_32(); - DEBUG_PRINT("GET RESPONSE: " + itos(response)); - - FileAccessNetwork *fa = nullptr; - - if (response != FileAccessNetwork::RESPONSE_DATA) { - if (!accesses.has(id)) { - unlock_mutex(); - ERR_FAIL_COND(!accesses.has(id)); - } - } - - if (accesses.has(id)) { - fa = accesses[id]; - } - - switch (response) { - case FileAccessNetwork::RESPONSE_OPEN: { - DEBUG_TIME("sem_open"); - int status = get_32(); - if (status != OK) { - fa->_respond(0, Error(status)); - } else { - int64_t len = get_64(); - fa->_respond(len, Error(status)); - } - - fa->sem.post(); - - } break; - case FileAccessNetwork::RESPONSE_DATA: { - int64_t offset = get_64(); - int32_t len = get_32(); - - Vector resp_block; - resp_block.resize(len); - client->get_data(resp_block.ptrw(), len); - - if (fa) { //may have been queued - fa->_set_block(offset, resp_block); - } - - } break; - case FileAccessNetwork::RESPONSE_FILE_EXISTS: { - int status = get_32(); - fa->exists_modtime = status != 0; - fa->sem.post(); - - } break; - case FileAccessNetwork::RESPONSE_GET_MODTIME: { - uint64_t status = get_64(); - fa->exists_modtime = status; - fa->sem.post(); - - } break; - } - - unlock_mutex(); - } -} - -void FileAccessNetworkClient::_thread_func(void *s) { - FileAccessNetworkClient *self = static_cast(s); - - self->_thread_func(); -} - -Error FileAccessNetworkClient::connect(const String &p_host, int p_port, const String &p_password) { - IPAddress ip; - - if (p_host.is_valid_ip_address()) { - ip = p_host; - } else { - ip = IP::get_singleton()->resolve_hostname(p_host); - } - - DEBUG_PRINT("IP: " + String(ip) + " port " + itos(p_port)); - Error err = client->connect_to_host(ip, p_port); - ERR_FAIL_COND_V_MSG(err != OK, err, "Cannot connect to host with IP: " + String(ip) + " and port: " + itos(p_port)); - while (client->get_status() == StreamPeerTCP::STATUS_CONNECTING) { - //DEBUG_PRINT("trying to connect...."); - OS::get_singleton()->delay_usec(1000); - } - - if (client->get_status() != StreamPeerTCP::STATUS_CONNECTED) { - return ERR_CANT_CONNECT; - } - - CharString cs = p_password.utf8(); - put_32(cs.length()); - client->put_data((const uint8_t *)cs.ptr(), cs.length()); - - int e = get_32(); - - if (e != OK) { - return ERR_INVALID_PARAMETER; - } - - thread.start(_thread_func, this); - - return OK; -} - -FileAccessNetworkClient *FileAccessNetworkClient::singleton = nullptr; - -FileAccessNetworkClient::FileAccessNetworkClient() { - singleton = this; - client.instantiate(); -} - -FileAccessNetworkClient::~FileAccessNetworkClient() { - quit = true; - sem.post(); - thread.wait_to_finish(); -} - -void FileAccessNetwork::_set_block(uint64_t p_offset, const Vector &p_block) { - int32_t page = p_offset / page_size; - ERR_FAIL_INDEX(page, pages.size()); - if (page < pages.size() - 1) { - ERR_FAIL_COND(p_block.size() != page_size); - } else { - ERR_FAIL_COND((uint64_t)p_block.size() != total_size % page_size); - } - - { - MutexLock lock(buffer_mutex); - pages.write[page].buffer = p_block; - pages.write[page].queued = false; - } - - if (waiting_on_page == page) { - waiting_on_page = -1; - page_sem.post(); - } -} - -void FileAccessNetwork::_respond(uint64_t p_len, Error p_status) { - DEBUG_PRINT("GOT RESPONSE - len: " + itos(p_len) + " status: " + itos(p_status)); - response = p_status; - if (response != OK) { - return; - } - opened = true; - total_size = p_len; - int32_t pc = ((total_size - 1) / page_size) + 1; - pages.resize(pc); -} - -Error FileAccessNetwork::open_internal(const String &p_path, int p_mode_flags) { - ERR_FAIL_COND_V(p_mode_flags != READ, ERR_UNAVAILABLE); - _close(); - - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - DEBUG_PRINT("open: " + p_path); - - DEBUG_TIME("open_begin"); - - nc->lock_mutex(); - nc->put_32(id); - nc->accesses[id] = this; - nc->put_32(COMMAND_OPEN_FILE); - CharString cs = p_path.utf8(); - nc->put_32(cs.length()); - nc->client->put_data((const uint8_t *)cs.ptr(), cs.length()); - pos = 0; - eof_flag = false; - last_page = -1; - last_page_buff = nullptr; - - //buffers.clear(); - nc->unlock_mutex(); - DEBUG_PRINT("OPEN POST"); - DEBUG_TIME("open_post"); - nc->sem.post(); //awaiting answer - DEBUG_PRINT("WAIT..."); - sem.wait(); - DEBUG_TIME("open_end"); - DEBUG_PRINT("WAIT ENDED..."); - - return response; -} - -void FileAccessNetwork::_close() { - if (!opened) { - return; - } - - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - - DEBUG_PRINT("CLOSE"); - nc->lock_mutex(); - nc->put_32(id); - nc->put_32(COMMAND_CLOSE); - pages.clear(); - opened = false; - nc->unlock_mutex(); -} - -bool FileAccessNetwork::is_open() const { - return opened; -} - -void FileAccessNetwork::seek(uint64_t p_position) { - ERR_FAIL_COND_MSG(!opened, "File must be opened before use."); - - eof_flag = p_position > total_size; - - if (p_position >= total_size) { - p_position = total_size; - } - - pos = p_position; -} - -void FileAccessNetwork::seek_end(int64_t p_position) { - seek(total_size + p_position); -} - -uint64_t FileAccessNetwork::get_position() const { - ERR_FAIL_COND_V_MSG(!opened, 0, "File must be opened before use."); - return pos; -} - -uint64_t FileAccessNetwork::get_length() const { - ERR_FAIL_COND_V_MSG(!opened, 0, "File must be opened before use."); - return total_size; -} - -bool FileAccessNetwork::eof_reached() const { - ERR_FAIL_COND_V_MSG(!opened, false, "File must be opened before use."); - return eof_flag; -} - -uint8_t FileAccessNetwork::get_8() const { - uint8_t v; - get_buffer(&v, 1); - return v; -} - -void FileAccessNetwork::_queue_page(int32_t p_page) const { - if (p_page >= pages.size()) { - return; - } - if (pages[p_page].buffer.is_empty() && !pages[p_page].queued) { - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - { - MutexLock lock(nc->blockrequest_mutex); - - FileAccessNetworkClient::BlockRequest br; - br.id = id; - br.offset = (uint64_t)p_page * page_size; - br.size = page_size; - nc->block_requests.push_back(br); - pages.write[p_page].queued = true; - } - DEBUG_PRINT("QUEUE PAGE POST"); - nc->sem.post(); - DEBUG_PRINT("queued " + itos(p_page)); - } -} - -uint64_t FileAccessNetwork::get_buffer(uint8_t *p_dst, uint64_t p_length) const { - ERR_FAIL_COND_V(!p_dst && p_length > 0, -1); - - if (pos + p_length > total_size) { - eof_flag = true; - } - if (pos + p_length >= total_size) { - p_length = total_size - pos; - } - - uint8_t *buff = last_page_buff; - - for (uint64_t i = 0; i < p_length; i++) { - int32_t page = pos / page_size; - - if (page != last_page) { - buffer_mutex.lock(); - if (pages[page].buffer.is_empty()) { - waiting_on_page = page; - for (int32_t j = 0; j < read_ahead; j++) { - _queue_page(page + j); - } - buffer_mutex.unlock(); - DEBUG_PRINT("wait"); - page_sem.wait(); - DEBUG_PRINT("done"); - } else { - for (int32_t j = 0; j < read_ahead; j++) { - _queue_page(page + j); - } - buffer_mutex.unlock(); - } - - buff = pages.write[page].buffer.ptrw(); - last_page_buff = buff; - last_page = page; - } - - p_dst[i] = buff[pos - uint64_t(page) * page_size]; - pos++; - } - - return p_length; -} - -Error FileAccessNetwork::get_error() const { - return pos == total_size ? ERR_FILE_EOF : OK; -} - -void FileAccessNetwork::flush() { - ERR_FAIL(); -} - -void FileAccessNetwork::store_8(uint8_t p_dest) { - ERR_FAIL(); -} - -bool FileAccessNetwork::file_exists(const String &p_path) { - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - nc->lock_mutex(); - nc->put_32(id); - nc->put_32(COMMAND_FILE_EXISTS); - CharString cs = p_path.utf8(); - nc->put_32(cs.length()); - nc->client->put_data((const uint8_t *)cs.ptr(), cs.length()); - nc->unlock_mutex(); - DEBUG_PRINT("FILE EXISTS POST"); - nc->sem.post(); - sem.wait(); - - return exists_modtime != 0; -} - -uint64_t FileAccessNetwork::_get_modified_time(const String &p_file) { - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - nc->lock_mutex(); - nc->put_32(id); - nc->put_32(COMMAND_GET_MODTIME); - CharString cs = p_file.utf8(); - nc->put_32(cs.length()); - nc->client->put_data((const uint8_t *)cs.ptr(), cs.length()); - nc->unlock_mutex(); - DEBUG_PRINT("MODTIME POST"); - nc->sem.post(); - sem.wait(); - - return exists_modtime; -} - -uint32_t FileAccessNetwork::_get_unix_permissions(const String &p_file) { - ERR_PRINT("Getting UNIX permissions from network drives is not implemented yet"); - return 0; -} - -Error FileAccessNetwork::_set_unix_permissions(const String &p_file, uint32_t p_permissions) { - ERR_PRINT("Setting UNIX permissions on network drives is not implemented yet"); - return ERR_UNAVAILABLE; -} - -void FileAccessNetwork::configure() { - GLOBAL_DEF(PropertyInfo(Variant::INT, "network/remote_fs/page_size", PROPERTY_HINT_RANGE, "1,65536,1,or_greater"), 65536); // Is used as denominator and can't be zero - GLOBAL_DEF(PropertyInfo(Variant::INT, "network/remote_fs/page_read_ahead", PROPERTY_HINT_RANGE, "0,8,1,or_greater"), 4); -} - -void FileAccessNetwork::close() { - _close(); - - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - nc->lock_mutex(); - nc->accesses.erase(id); - nc->unlock_mutex(); -} - -FileAccessNetwork::FileAccessNetwork() { - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - nc->lock_mutex(); - id = nc->last_id++; - nc->accesses[id] = this; - nc->unlock_mutex(); - page_size = GLOBAL_GET("network/remote_fs/page_size"); - read_ahead = GLOBAL_GET("network/remote_fs/page_read_ahead"); -} - -FileAccessNetwork::~FileAccessNetwork() { - _close(); - - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - nc->lock_mutex(); - nc->accesses.erase(id); - nc->unlock_mutex(); -} diff --git a/core/io/file_access_network.h b/core/io/file_access_network.h deleted file mode 100644 index 78c19347ce3b9e..00000000000000 --- a/core/io/file_access_network.h +++ /dev/null @@ -1,167 +0,0 @@ -/**************************************************************************/ -/* file_access_network.h */ -/**************************************************************************/ -/* This file is part of: */ -/* GODOT ENGINE */ -/* https://godotengine.org */ -/**************************************************************************/ -/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ -/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ -/* */ -/* Permission is hereby granted, free of charge, to any person obtaining */ -/* a copy of this software and associated documentation files (the */ -/* "Software"), to deal in the Software without restriction, including */ -/* without limitation the rights to use, copy, modify, merge, publish, */ -/* distribute, sublicense, and/or sell copies of the Software, and to */ -/* permit persons to whom the Software is furnished to do so, subject to */ -/* the following conditions: */ -/* */ -/* The above copyright notice and this permission notice shall be */ -/* included in all copies or substantial portions of the Software. */ -/* */ -/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ -/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ -/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ -/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ -/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ -/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ -/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -/**************************************************************************/ - -#ifndef FILE_ACCESS_NETWORK_H -#define FILE_ACCESS_NETWORK_H - -#include "core/io/file_access.h" -#include "core/io/stream_peer_tcp.h" -#include "core/os/semaphore.h" -#include "core/os/thread.h" - -class FileAccessNetwork; - -class FileAccessNetworkClient { - struct BlockRequest { - int32_t id; - uint64_t offset; - int32_t size; - }; - - List block_requests; - - Semaphore sem; - Thread thread; - bool quit = false; - Mutex mutex; - Mutex blockrequest_mutex; - HashMap accesses; - Ref client; - int32_t last_id = 0; - int32_t lockcount = 0; - - Vector block; - - void _thread_func(); - static void _thread_func(void *s); - - void put_32(int32_t p_32); - void put_64(int64_t p_64); - int32_t get_32(); - int64_t get_64(); - void lock_mutex(); - void unlock_mutex(); - - friend class FileAccessNetwork; - static FileAccessNetworkClient *singleton; - -public: - static FileAccessNetworkClient *get_singleton() { return singleton; } - - Error connect(const String &p_host, int p_port, const String &p_password = ""); - - FileAccessNetworkClient(); - ~FileAccessNetworkClient(); -}; - -class FileAccessNetwork : public FileAccess { - Semaphore sem; - Semaphore page_sem; - Mutex buffer_mutex; - bool opened = false; - uint64_t total_size = 0; - mutable uint64_t pos = 0; - int32_t id = -1; - mutable bool eof_flag = false; - mutable int32_t last_page = -1; - mutable uint8_t *last_page_buff = nullptr; - - int32_t page_size = 0; - int32_t read_ahead = 0; - - mutable int waiting_on_page = -1; - - struct Page { - int activity = 0; - bool queued = false; - Vector buffer; - }; - - mutable Vector pages; - - mutable Error response; - - uint64_t exists_modtime = 0; - - friend class FileAccessNetworkClient; - void _queue_page(int32_t p_page) const; - void _respond(uint64_t p_len, Error p_status); - void _set_block(uint64_t p_offset, const Vector &p_block); - void _close(); - -public: - enum Command { - COMMAND_OPEN_FILE, - COMMAND_READ_BLOCK, - COMMAND_CLOSE, - COMMAND_FILE_EXISTS, - COMMAND_GET_MODTIME, - }; - - enum Response { - RESPONSE_OPEN, - RESPONSE_DATA, - RESPONSE_FILE_EXISTS, - RESPONSE_GET_MODTIME, - }; - - virtual Error open_internal(const String &p_path, int p_mode_flags) override; ///< open a file - virtual bool is_open() const override; ///< true when file is open - - virtual void seek(uint64_t p_position) override; ///< seek to a given position - virtual void seek_end(int64_t p_position = 0) override; ///< seek from the end of file - virtual uint64_t get_position() const override; ///< get position in the file - virtual uint64_t get_length() const override; ///< get size of the file - - virtual bool eof_reached() const override; ///< reading passed EOF - - virtual uint8_t get_8() const override; ///< get a byte - virtual uint64_t get_buffer(uint8_t *p_dst, uint64_t p_length) const override; - - virtual Error get_error() const override; ///< get last error - - virtual void flush() override; - virtual void store_8(uint8_t p_dest) override; ///< store a byte - - virtual bool file_exists(const String &p_path) override; ///< return true if a file exists - - virtual uint64_t _get_modified_time(const String &p_file) override; - virtual uint32_t _get_unix_permissions(const String &p_file) override; - virtual Error _set_unix_permissions(const String &p_file, uint32_t p_permissions) override; - - virtual void close() override; - - static void configure(); - - FileAccessNetwork(); - ~FileAccessNetwork(); -}; - -#endif // FILE_ACCESS_NETWORK_H diff --git a/core/io/remote_filesystem_client.cpp b/core/io/remote_filesystem_client.cpp new file mode 100644 index 00000000000000..5cd300d171f568 --- /dev/null +++ b/core/io/remote_filesystem_client.cpp @@ -0,0 +1,333 @@ +/**************************************************************************/ +/* remote_filesystem_client.cpp */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#include "remote_filesystem_client.h" +#include "core/io/dir_access.h" +#include "core/io/file_access.h" +#include "core/io/stream_peer_tcp.h" +#include "core/string/string_builder.h" + +#define FILESYSTEM_CACHE_VERSION 1 +#define FILESYSTEM_PROTOCOL_VERSION 1 +#define PASSWORD_LENGTH 32 + +#define FILES_SUBFOLDER "remote_filesystem_files" +#define FILES_CACHE_FILE "remote_filesystem.cache" + +Vector RemoteFilesystemClient::_load_cache_file() { + Ref fa = FileAccess::open(cache_path.path_join(FILES_CACHE_FILE), FileAccess::READ); + if (!fa.is_valid()) { + return Vector(); // No cache, return empty + } + + int version = fa->get_line().to_int(); + if (version != FILESYSTEM_CACHE_VERSION) { + return Vector(); // Version mismatch, ignore everything. + } + + String file_path = cache_path.path_join(FILES_SUBFOLDER); + + Vector file_cache; + + while (!fa->eof_reached()) { + String l = fa->get_line(); + Vector fields = l.split("::"); + if (fields.size() != 3) { + break; + } + FileCache fc; + fc.path = fields[0]; + fc.server_modified_time = fields[1].to_int(); + fc.modified_time = fields[2].to_int(); + + String full_path = file_path.path_join(fc.path); + if (!FileAccess::exists(full_path)) { + continue; // File is gone. + } + + if (FileAccess::get_modified_time(full_path) != fc.modified_time) { + DirAccess::remove_absolute(full_path); // Take the chance to remove this file and assume we no longer have it. + continue; + } + + file_cache.push_back(fc); + } + + return file_cache; +} + +Error RemoteFilesystemClient::_store_file(const String &p_path, const LocalVector &p_file, uint64_t &modified_time) { + modified_time = 0; + String full_path = cache_path.path_join(FILES_SUBFOLDER).path_join(p_path); + String base_file_dir = full_path.get_base_dir(); + + if (!validated_directories.has(base_file_dir)) { + // Verify that path exists before writing file, but only verify once for performance. + DirAccess::make_dir_recursive_absolute(base_file_dir); + validated_directories.insert(base_file_dir); + } + + Ref f = FileAccess::open(full_path, FileAccess::WRITE); + ERR_FAIL_COND_V_MSG(f.is_null(), ERR_FILE_CANT_OPEN, "Unable to open file for writing to remote filesystem cache: " + p_path); + f->store_buffer(p_file.ptr(), p_file.size()); + Error err = f->get_error(); + if (err) { + return err; + } + f.unref(); // Unref to ensure file is not locked and modified time can be obtained. + + modified_time = FileAccess::get_modified_time(full_path); + return OK; +} + +Error RemoteFilesystemClient::_remove_file(const String &p_path) { + return DirAccess::remove_absolute(cache_path.path_join(FILES_SUBFOLDER).path_join(p_path)); +} +Error RemoteFilesystemClient::_store_cache_file(const Vector &p_cache) { + String full_path = cache_path.path_join(FILES_CACHE_FILE); + String base_file_dir = full_path.get_base_dir(); + Error err = DirAccess::make_dir_recursive_absolute(base_file_dir); + ERR_FAIL_COND_V_MSG(err != OK, err, "Unable to create base directory to store cache file: " + base_file_dir); + + Ref f = FileAccess::open(full_path, FileAccess::WRITE); + ERR_FAIL_COND_V_MSG(f.is_null(), ERR_FILE_CANT_OPEN, "Unable to open the remote cache file for writing: " + full_path); + f->store_line(itos(FILESYSTEM_CACHE_VERSION)); + for (int i = 0; i < p_cache.size(); i++) { + String l = p_cache[i].path + "::" + itos(p_cache[i].server_modified_time) + "::" + itos(p_cache[i].modified_time); + f->store_line(l); + } + return OK; +} + +Error RemoteFilesystemClient::synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path) { + Error err = _synchronize_with_server(p_host, p_port, p_password, r_cache_path); + // Ensure no memory is kept + validated_directories.reset(); + cache_path = String(); + return err; +} + +void RemoteFilesystemClient::_update_cache_path(String &r_cache_path) { + r_cache_path = cache_path.path_join(FILES_SUBFOLDER); +} + +Error RemoteFilesystemClient::_synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path) { + cache_path = r_cache_path; + { + Ref dir = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); + dir->change_dir(cache_path); + cache_path = dir->get_current_dir(); + } + + Ref tcp_client; + tcp_client.instantiate(); + + IPAddress ip = p_host.is_valid_ip_address() ? IPAddress(p_host) : IP::get_singleton()->resolve_hostname(p_host); + ERR_FAIL_COND_V_MSG(!ip.is_valid(), ERR_INVALID_PARAMETER, "Unable to resolve remote filesystem server hostname: " + p_host); + print_verbose("Remote Filesystem: Connecting to host " + String(ip) + ", port: " + itos(p_port)); + Error err = tcp_client->connect_to_host(ip, p_port); + ERR_FAIL_COND_V_MSG(err != OK, err, "Unable to open connection to remote file server (" + String(p_host) + ", port " + itos(p_port) + ") failed."); + + while (tcp_client->get_status() == StreamPeerTCP::STATUS_CONNECTING) { + tcp_client->poll(); + OS::get_singleton()->delay_usec(100); + } + + if (tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED) { + ERR_FAIL_V_MSG(ERR_CANT_CONNECT, "Connection to remote file server (" + String(p_host) + ", port " + itos(p_port) + ") failed."); + } + + // Connection OK, now send the current file state. + print_verbose("Remote Filesystem: Connection OK."); + + // Header (GRFS) - Godot Remote File System + print_verbose("Remote Filesystem: Sending Header"); + tcp_client->put_u8('G'); + tcp_client->put_u8('R'); + tcp_client->put_u8('F'); + tcp_client->put_u8('S'); + // Protocol version + tcp_client->put_32(FILESYSTEM_PROTOCOL_VERSION); + print_verbose("Remote Filesystem: Send password"); + uint8_t password[PASSWORD_LENGTH]; // Send fixed size password, since its easier and safe to validate. + for (int i = 0; i < PASSWORD_LENGTH; i++) { + if (i < p_password.length()) { + password[i] = p_password[i]; + } else { + password[i] = 0; + } + } + tcp_client->put_data(password, PASSWORD_LENGTH); + print_verbose("Remote Filesystem: Tags"); + Vector tags; + { + String os_name = OS::get_singleton()->get_name().to_lower(); + if (os_name == "Linux" || os_name == "FreeBSD" || os_name == "NetBSD" || os_name == "OpenBSD" || os_name == "BSD") { + os_name = "linuxbsd"; + } + tags.push_back(os_name); + switch (OS::get_singleton()->get_preferred_texture_format()) { + case OS::PREFERRED_TEXTURE_FORMAT_S3TC_BPTC: { + tags.push_back("bptc"); + tags.push_back("s3tc"); + } break; + case OS::PREFERRED_TEXTURE_FORMAT_ETC2_ASTC: { + tags.push_back("etc2"); + tags.push_back("astc"); + } break; + } + } + + tcp_client->put_32(tags.size()); + for (int i = 0; i < tags.size(); i++) { + tcp_client->put_utf8_string(tags[i]); + } + // Size of compressed list of files + print_verbose("Remote Filesystem: Sending File List"); + + Vector file_cache = _load_cache_file(); + + // Encode file cache to send it via network. + Vector file_cache_buffer; + if (file_cache.size()) { + StringBuilder sbuild; + for (int i = 0; i < file_cache.size(); i++) { + sbuild.append(file_cache[i].path); + sbuild.append("::"); + sbuild.append(itos(file_cache[i].server_modified_time)); + sbuild.append("\n"); + } + String s = sbuild.as_string(); + CharString cs = s.utf8(); + file_cache_buffer.resize(Compression::get_max_compressed_buffer_size(cs.length(), Compression::MODE_ZSTD)); + int res_len = Compression::compress(file_cache_buffer.ptrw(), (const uint8_t *)cs.ptr(), cs.length(), Compression::MODE_ZSTD); + file_cache_buffer.resize(res_len); + + tcp_client->put_32(cs.length()); // Size of buffer uncompressed + tcp_client->put_32(file_cache_buffer.size()); // Size of buffer compressed + tcp_client->put_data(file_cache_buffer.ptr(), file_cache_buffer.size()); // Buffer + } else { + tcp_client->put_32(0); // No file cache buffer + } + + tcp_client->poll(); + ERR_FAIL_COND_V_MSG(tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED, ERR_CONNECTION_ERROR, "Remote filesystem server disconnected after sending header."); + + uint32_t file_count = tcp_client->get_32(); + + ERR_FAIL_COND_V_MSG(tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED, ERR_CONNECTION_ERROR, "Remote filesystem server disconnected while waiting for file list"); + + LocalVector file_buffer; + + Vector temp_file_cache; + + HashSet files_processed; + for (uint32_t i = 0; i < file_count; i++) { + String file = tcp_client->get_utf8_string(); + ERR_FAIL_COND_V_MSG(file == String(), ERR_CONNECTION_ERROR, "Invalid file name received from remote filesystem."); + uint64_t server_modified_time = tcp_client->get_u64(); + ERR_FAIL_COND_V_MSG(tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED, ERR_CONNECTION_ERROR, "Remote filesystem server disconnected while waiting for file info."); + + FileCache fc; + fc.path = file; + fc.server_modified_time = server_modified_time; + temp_file_cache.push_back(fc); + + files_processed.insert(file); + } + + Vector new_file_cache; + + // Get the actual files. As a robustness measure, if the connection is interrupted here, any file not yet received will be considered removed. + // Since the file changed anyway, this makes it the easiest way to keep robustness. + + bool server_disconnected = false; + for (uint32_t i = 0; i < file_count; i++) { + String file = temp_file_cache[i].path; + + if (temp_file_cache[i].server_modified_time == 0 || server_disconnected) { + // File was removed, or server disconnected before tranfering it. Since its no longer valid, remove anyway. + _remove_file(file); + continue; + } + + uint64_t file_size = tcp_client->get_u64(); + file_buffer.resize(file_size); + if (file_buffer.size() != file_size) { + server_disconnected = true; + ERR_PRINT("Remote filesystem sent a file (" + file + ") too big (Unable to allocate memory): " + String::humanize_size(file_size)); + } else { + err = tcp_client->get_data(file_buffer.ptr(), file_size); + if (err != OK) { + ERR_PRINT("Error retrieving file from remote filesystem: " + file); + server_disconnected = true; + } + } + + if (tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED) { + // Early disconnect, stop accepting files. + server_disconnected = true; + } + + if (server_disconnected) { + // No more server, transfer is invalid, remove this file. + _remove_file(file); + continue; + } + + uint64_t modified_time; + err = _store_file(file, file_buffer, modified_time); + if (err != OK) { + server_disconnected = true; + continue; + } + FileCache fc = temp_file_cache[i]; + fc.modified_time = modified_time; + new_file_cache.push_back(fc); + } + + print_verbose("Updating the cache file."); + + // Merge file cache. + for (int i = 0; i < file_cache.size(); i++) { + if (files_processed.has(file_cache[i].path)) { + continue; // This was either added or removed, so skip. + } + new_file_cache.push_back(file_cache[i]); + } + + err = _store_cache_file(new_file_cache); + ERR_FAIL_COND_V_MSG(err != OK, ERR_FILE_CANT_OPEN, "Error writing the remote filesystem file cache. "); + + print_verbose("Remote Filesystem: Update success."); + + _update_cache_path(r_cache_path); + return OK; +} diff --git a/core/io/remote_filesystem_client.h b/core/io/remote_filesystem_client.h new file mode 100644 index 00000000000000..42eba98eb100a7 --- /dev/null +++ b/core/io/remote_filesystem_client.h @@ -0,0 +1,65 @@ +/**************************************************************************/ +/* remote_filesystem_client.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#ifndef REMOTE_FILESYSTEM_CLIENT_H +#define REMOTE_FILESYSTEM_CLIENT_H + +#include "core/io/ip_address.h" +#include "core/string/ustring.h" +#include "core/templates/hash_set.h" +#include "core/templates/local_vector.h" + +class RemoteFilesystemClient { + String cache_path; + HashSet validated_directories; + +protected: + String _get_cache_path() { return cache_path; } + struct FileCache { + String path; // Local path (as in "folder/to/file.png") + uint64_t server_modified_time; // MD5 checksum. + uint64_t modified_time; + }; + virtual bool _is_configured() { return !cache_path.is_empty(); } + // Can be re-implemented per platform. If so, feel free to ignore get_cache_path() + virtual Vector _load_cache_file(); + virtual Error _store_file(const String &p_path, const LocalVector &p_file, uint64_t &modified_time); + virtual Error _remove_file(const String &p_path); + virtual Error _store_cache_file(const Vector &p_cache); + virtual Error _synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path); + + virtual void _update_cache_path(String &r_cache_path); + +public: + Error synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path); + virtual ~RemoteFilesystemClient() {} +}; + +#endif // REMOTE_FILESYSTEM_CLIENT_H diff --git a/core/os/os.cpp b/core/os/os.cpp index c82f87c731d55b..e8ccf44d5f8dff 100644 --- a/core/os/os.cpp +++ b/core/os/os.cpp @@ -569,6 +569,10 @@ void OS::add_frame_delay(bool p_can_draw) { } } +Error OS::setup_remote_filesystem(const String &p_server_host, int p_port, const String &p_password, String &r_project_path) { + return default_rfs.synchronize_with_server(p_server_host, p_port, p_password, r_project_path); +} + OS::PreferredTextureFormat OS::get_preferred_texture_format() const { #if defined(__arm__) || defined(__aarch64__) || defined(_M_ARM) || defined(_M_ARM64) return PREFERRED_TEXTURE_FORMAT_ETC2_ASTC; // By rule, ARM hardware uses ETC texture compression. diff --git a/core/os/os.h b/core/os/os.h index 3248330c74c0f2..2656532e093fc5 100644 --- a/core/os/os.h +++ b/core/os/os.h @@ -34,6 +34,7 @@ #include "core/config/engine.h" #include "core/io/image.h" #include "core/io/logger.h" +#include "core/io/remote_filesystem_client.h" #include "core/os/time_enums.h" #include "core/string/ustring.h" #include "core/templates/list.h" @@ -72,6 +73,8 @@ class OS { String _current_rendering_driver_name; String _current_rendering_method; + RemoteFilesystemClient default_rfs; + protected: void _set_logger(CompositeLogger *p_logger); @@ -292,6 +295,8 @@ class OS { virtual void process_and_drop_events() {} + virtual Error setup_remote_filesystem(const String &p_server_host, int p_port, const String &p_password, String &r_project_path); + enum PreferredTextureFormat { PREFERRED_TEXTURE_FORMAT_S3TC_BPTC, PREFERRED_TEXTURE_FORMAT_ETC2_ASTC diff --git a/core/string/print_string.cpp b/core/string/print_string.cpp index 7b894d83bf0ea4..7b90710308cd05 100644 --- a/core/string/print_string.cpp +++ b/core/string/print_string.cpp @@ -193,10 +193,8 @@ void print_error(String p_string) { _global_unlock(); } -void print_verbose(String p_string) { - if (OS::get_singleton()->is_stdout_verbose()) { - print_line(p_string); - } +bool is_print_verbose_enabled() { + return OS::get_singleton()->is_stdout_verbose(); } String stringify_variants(Variant p_var) { diff --git a/core/string/print_string.h b/core/string/print_string.h index 6496384b3f6a90..d1eeb15815322c 100644 --- a/core/string/print_string.h +++ b/core/string/print_string.h @@ -59,7 +59,14 @@ void remove_print_handler(const PrintHandlerList *p_handler); extern void __print_line(String p_string); extern void __print_line_rich(String p_string); extern void print_error(String p_string); -extern void print_verbose(String p_string); +extern bool is_print_verbose_enabled(); + +// This version avoids processing the text to be printed until it actually has to be printed, saving some CPU usage. +#define print_verbose(m_text) \ + { \ + if (is_print_verbose_enabled()) \ + print_line(m_text); \ + } inline void print_line(Variant v) { __print_line(stringify_variants(v)); diff --git a/core/variant/variant_utility.cpp b/core/variant/variant_utility.cpp index fd079dbeeaaa63..bfade19c956d7d 100644 --- a/core/variant/variant_utility.cpp +++ b/core/variant/variant_utility.cpp @@ -803,7 +803,7 @@ struct VariantUtilityFunctions { print_line_rich(s); r_error.error = Callable::CallError::CALL_OK; } - +#undef print_verbose static inline void print_verbose(const Variant **p_args, int p_arg_count, Callable::CallError &r_error) { if (OS::get_singleton()->is_stdout_verbose()) { String s; diff --git a/editor/debugger/editor_file_server.cpp b/editor/debugger/editor_file_server.cpp index ba5dd9e6e2c3e3..cd0dcdaa0305c4 100644 --- a/editor/debugger/editor_file_server.cpp +++ b/editor/debugger/editor_file_server.cpp @@ -32,273 +32,231 @@ #include "../editor_settings.h" #include "core/io/marshalls.h" +#include "editor/editor_node.h" +#include "editor/export/editor_export_platform.h" + +#define FILESYSTEM_PROTOCOL_VERSION 1 +#define PASSWORD_LENGTH 32 +#define MAX_FILE_BUFFER_SIZE 100 * 1024 * 1024 // 100mb max file buffer size (description of files to update, compressed). + +static void _add_file(String f, const uint64_t &p_modified_time, HashMap &files_to_send, HashMap &cached_files) { + f = f.replace_first("res://", ""); // remove res:// + const uint64_t *cached_mt = cached_files.getptr(f); + if (cached_mt && *cached_mt == p_modified_time) { + // File is good, skip it. + cached_files.erase(f); // Erase to mark this file as existing. Remaning files not added to files_to_send will be considered erased here, so they need to be erased in the client too. + return; + } + files_to_send.insert(f, p_modified_time); +} -//#define DEBUG_PRINT(m_p) print_line(m_p) -//#define DEBUG_TIME(m_what) printf("MS: %s - %lu\n", m_what, OS::get_singleton()->get_ticks_usec()); +void EditorFileServer::_scan_files_changed(EditorFileSystemDirectory *efd, const Vector &p_tags, HashMap &files_to_send, HashMap &cached_files) { + for (int i = 0; i < efd->get_file_count(); i++) { + String f = efd->get_file_path(i); + if (FileAccess::exists(f + ".import")) { + // is imported, determine what to do + // Todo the modified times of remapped files should most likely be kept in EditorFileSystem to speed this up in the future. + Ref cf; + cf.instantiate(); + Error err = cf->load(f + ".import"); + + ERR_CONTINUE(err != OK); + { + uint64_t mt = FileAccess::get_modified_time(f + ".import"); + _add_file(f + ".import", mt, files_to_send, cached_files); + } -#define DEBUG_PRINT(m_what) -#define DEBUG_TIME(m_what) + if (!cf->has_section("remap")) { + continue; + } -void EditorFileServer::_close_client(ClientData *cd) { - cd->connection->disconnect_from_host(); - { - MutexLock lock(cd->efs->wait_mutex); - cd->efs->to_wait.insert(cd->thread); + List remaps; + cf->get_section_keys("remap", &remaps); + + for (const String &F : remaps) { + String remap = F; + if (remap == "path") { + String remapped_path = cf->get_value("remap", remap); + uint64_t mt = FileAccess::get_modified_time(remapped_path); + _add_file(remapped_path, mt, files_to_send, cached_files); + } else if (remap.begins_with("path.")) { + String feature = remap.get_slice(".", 1); + if (p_tags.find(feature) != -1) { + String remapped_path = cf->get_value("remap", remap); + uint64_t mt = FileAccess::get_modified_time(remapped_path); + _add_file(remapped_path, mt, files_to_send, cached_files); + } + } + } + } else { + uint64_t mt = efd->get_file_modified_time(i); + _add_file(f, mt, files_to_send, cached_files); + } } - while (cd->files.size()) { - cd->files.remove(cd->files.begin()); + + for (int i = 0; i < efd->get_subdir_count(); i++) { + _scan_files_changed(efd->get_subdir(i), p_tags, files_to_send, cached_files); } - memdelete(cd); } -void EditorFileServer::_subthread_start(void *s) { - ClientData *cd = static_cast(s); +static void _add_custom_file(const String f, HashMap &files_to_send, HashMap &cached_files) { + if (!FileAccess::exists(f)) { + return; + } + _add_file(f, FileAccess::get_modified_time(f), files_to_send, cached_files); +} - cd->connection->set_no_delay(true); - uint8_t buf4[8]; - Error err = cd->connection->get_data(buf4, 4); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); +void EditorFileServer::poll() { + if (!active) { + return; } - int passlen = decode_uint32(buf4); - - if (passlen > 512) { - _close_client(cd); - ERR_FAIL_COND(passlen > 512); - } else if (passlen > 0) { - Vector passutf8; - passutf8.resize(passlen + 1); - err = cd->connection->get_data((uint8_t *)passutf8.ptr(), passlen); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - passutf8.write[passlen] = 0; - String s2; - s2.parse_utf8(passutf8.ptr()); - if (s2 != cd->efs->password) { - encode_uint32(ERR_INVALID_DATA, buf4); - cd->connection->put_data(buf4, 4); - OS::get_singleton()->delay_usec(1000000); - _close_client(cd); - ERR_PRINT("CLIENT PASSWORD MISMATCH"); - ERR_FAIL(); - } - } else { - if (!cd->efs->password.is_empty()) { - encode_uint32(ERR_INVALID_DATA, buf4); - cd->connection->put_data(buf4, 4); - OS::get_singleton()->delay_usec(1000000); - _close_client(cd); - ERR_PRINT("CLIENT PASSWORD MISMATCH (should be empty!)"); - ERR_FAIL(); - } + if (!server->is_connection_available()) { + return; } - encode_uint32(OK, buf4); - cd->connection->put_data(buf4, 4); + Ref tcp_peer = server->take_connection(); + ERR_FAIL_COND(tcp_peer.is_null()); + + // Got a connection! + EditorProgress pr("updating_remote_file_system", TTR("Updating assets on target device:"), 105); + + pr.step(TTR("Syncinc headers"), 0, true); + print_verbose("EFS: Connecting taken!"); + char header[4]; + Error err = tcp_peer->get_data((uint8_t *)&header, 4); + ERR_FAIL_COND(err != OK); + ERR_FAIL_COND(header[0] != 'G'); + ERR_FAIL_COND(header[1] != 'R'); + ERR_FAIL_COND(header[2] != 'F'); + ERR_FAIL_COND(header[3] != 'S'); + + uint32_t protocol_version = tcp_peer->get_u32(); + ERR_FAIL_COND(protocol_version != FILESYSTEM_PROTOCOL_VERSION); + + char cpassword[PASSWORD_LENGTH + 1]; + err = tcp_peer->get_data((uint8_t *)cpassword, PASSWORD_LENGTH); + cpassword[PASSWORD_LENGTH] = 0; + ERR_FAIL_COND(err != OK); + print_verbose("EFS: Got password: " + String(cpassword)); + ERR_FAIL_COND_MSG(password != cpassword, "Client disconnected because password mismatch."); + + uint32_t tag_count = tcp_peer->get_u32(); + print_verbose("EFS: Getting tags: " + itos(tag_count)); + + ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED); + Vector tags; + for (uint32_t i = 0; i < tag_count; i++) { + String tag = tcp_peer->get_utf8_string(); + print_verbose("EFS: tag #" + itos(i) + ": " + tag); + ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED); + tags.push_back(tag); + } - while (!cd->quit) { - //wait for ID - err = cd->connection->get_data(buf4, 4); - DEBUG_TIME("get_data") + uint32_t file_buffer_decompressed_size = tcp_peer->get_32(); + HashMap cached_files; - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - int id = decode_uint32(buf4); + if (file_buffer_decompressed_size > 0) { + pr.step(TTR("Getting remote file system"), 1, true); - //wait for command - err = cd->connection->get_data(buf4, 4); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - int cmd = decode_uint32(buf4); - - switch (cmd) { - case FileAccessNetwork::COMMAND_FILE_EXISTS: - case FileAccessNetwork::COMMAND_GET_MODTIME: - case FileAccessNetwork::COMMAND_OPEN_FILE: { - DEBUG_TIME("open_file") - err = cd->connection->get_data(buf4, 4); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } + // Got files cached by client. + uint32_t file_buffer_size = tcp_peer->get_32(); + print_verbose("EFS: Getting file buffer: compressed - " + String::humanize_size(file_buffer_size) + " decompressed: " + String::humanize_size(file_buffer_decompressed_size)); - int namelen = decode_uint32(buf4); - Vector fileutf8; - fileutf8.resize(namelen + 1); - err = cd->connection->get_data((uint8_t *)fileutf8.ptr(), namelen); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - fileutf8.write[namelen] = 0; - String s2; - s2.parse_utf8(fileutf8.ptr()); + ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED); + ERR_FAIL_COND(file_buffer_size > MAX_FILE_BUFFER_SIZE); + LocalVector file_buffer; + file_buffer.resize(file_buffer_size); + LocalVector file_buffer_decompressed; + file_buffer_decompressed.resize(file_buffer_decompressed_size); - if (cmd == FileAccessNetwork::COMMAND_FILE_EXISTS) { - print_verbose("FILE EXISTS: " + s2); - } - if (cmd == FileAccessNetwork::COMMAND_GET_MODTIME) { - print_verbose("MOD TIME: " + s2); - } - if (cmd == FileAccessNetwork::COMMAND_OPEN_FILE) { - print_verbose("OPEN: " + s2); - } + err = tcp_peer->get_data(file_buffer.ptr(), file_buffer_size); - if (!s2.begins_with("res://")) { - _close_client(cd); - ERR_FAIL_COND(!s2.begins_with("res://")); - } - ERR_CONTINUE(cd->files.has(id)); - - if (cmd == FileAccessNetwork::COMMAND_FILE_EXISTS) { - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_FILE_EXISTS, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccess::exists(s2), buf4); - cd->connection->put_data(buf4, 4); - DEBUG_TIME("open_file_end") - break; - } + pr.step(TTR("Decompressing remote file system"), 2, true); - if (cmd == FileAccessNetwork::COMMAND_GET_MODTIME) { - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_GET_MODTIME, buf4); - cd->connection->put_data(buf4, 4); - encode_uint64(FileAccess::get_modified_time(s2), buf4); - cd->connection->put_data(buf4, 8); - DEBUG_TIME("open_file_end") - break; - } - - Ref fa = FileAccess::open(s2, FileAccess::READ); - if (fa.is_null()) { - //not found, continue - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_OPEN, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(ERR_FILE_NOT_FOUND, buf4); - cd->connection->put_data(buf4, 4); - DEBUG_TIME("open_file_end") - break; - } + ERR_FAIL_COND(err != OK); + // Decompress the text with all the files + Compression::decompress(file_buffer_decompressed.ptr(), file_buffer_decompressed.size(), file_buffer.ptr(), file_buffer.size(), Compression::MODE_ZSTD); + String files_text = String::utf8((const char *)file_buffer_decompressed.ptr(), file_buffer_decompressed.size()); + Vector files = files_text.split("\n"); + + print_verbose("EFS: Total cached files received: " + itos(files.size())); + for (int i = 0; i < files.size(); i++) { + if (files[i].get_slice_count("::") != 2) { + continue; + } + String file = files[i].get_slice("::", 0); + uint64_t modified_time = files[i].get_slice("::", 1).to_int(); - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_OPEN, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(OK, buf4); - cd->connection->put_data(buf4, 4); - encode_uint64(fa->get_length(), buf4); - cd->connection->put_data(buf4, 8); - - cd->files[id] = fa; - DEBUG_TIME("open_file_end") - - } break; - case FileAccessNetwork::COMMAND_READ_BLOCK: { - err = cd->connection->get_data(buf4, 8); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } + cached_files.insert(file, modified_time); + } + } else { + // Client does not have any files stored. + } - ERR_CONTINUE(!cd->files.has(id)); + pr.step(TTR("Scanning for local changes"), 3, true); - uint64_t offset = decode_uint64(buf4); + print_verbose("EFS: Scanning changes:"); - err = cd->connection->get_data(buf4, 4); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } + HashMap files_to_send; + // Scan files to send. + _scan_files_changed(EditorFileSystem::get_singleton()->get_filesystem(), tags, files_to_send, cached_files); + // Add forced export files + Vector forced_export = EditorExportPlatform::get_forced_export_files(); + for (int i = 0; i < forced_export.size(); i++) { + _add_custom_file(forced_export[i], files_to_send, cached_files); + } - int blocklen = decode_uint32(buf4); - ERR_CONTINUE(blocklen > (16 * 1024 * 1024)); - - cd->files[id]->seek(offset); - Vector buf; - buf.resize(blocklen); - uint32_t read = cd->files[id]->get_buffer(buf.ptrw(), blocklen); - - print_verbose("GET BLOCK - offset: " + itos(offset) + ", blocklen: " + itos(blocklen)); - - //not found, continue - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_DATA, buf4); - cd->connection->put_data(buf4, 4); - encode_uint64(offset, buf4); - cd->connection->put_data(buf4, 8); - encode_uint32(read, buf4); - cd->connection->put_data(buf4, 4); - cd->connection->put_data(buf.ptr(), read); - - } break; - case FileAccessNetwork::COMMAND_CLOSE: { - print_verbose("CLOSED"); - ERR_CONTINUE(!cd->files.has(id)); - cd->files.erase(id); - } break; + _add_custom_file("res://project.godot", files_to_send, cached_files); + // Check which files were removed and also add them + for (KeyValue K : cached_files) { + if (!files_to_send.has(K.key)) { + files_to_send.insert(K.key, 0); //0 means removed } } - _close_client(cd); -} + tcp_peer->put_32(files_to_send.size()); -void EditorFileServer::_thread_start(void *s) { - EditorFileServer *self = static_cast(s); - while (!self->quit) { - if (self->cmd == CMD_ACTIVATE) { - self->server->listen(self->port); - self->active = true; - self->cmd = CMD_NONE; - } else if (self->cmd == CMD_STOP) { - self->server->stop(); - self->active = false; - self->cmd = CMD_NONE; - } + print_verbose("EFS: Sending list of changed files."); + pr.step(TTR("Sending list of changed files:"), 4, true); - if (self->active) { - if (self->server->is_connection_available()) { - ClientData *cd = memnew(ClientData); - cd->connection = self->server->take_connection(); - cd->efs = self; - cd->quit = false; - cd->thread = memnew(Thread); - cd->thread->start(_subthread_start, cd); - } - } + // Send list of changed files first, to ensure that if connecting breaks, the client is not found in a broken state. + for (KeyValue K : files_to_send) { + tcp_peer->put_utf8_string(K.key); + tcp_peer->put_64(K.value); + } + + print_verbose("EFS: Sending " + itos(files_to_send.size()) + " files."); + + int idx = 0; + for (KeyValue K : files_to_send) { + pr.step(TTR("Sending file: ") + K.key.get_file(), 5 + idx * 100 / files_to_send.size(), false); + idx++; - self->wait_mutex.lock(); - while (self->to_wait.size()) { - Thread *w = *self->to_wait.begin(); - self->to_wait.erase(w); - self->wait_mutex.unlock(); - w->wait_to_finish(); - self->wait_mutex.lock(); + if (K.value == 0 || !FileAccess::exists("res://" + K.key)) { // File was removed + continue; } - self->wait_mutex.unlock(); - OS::get_singleton()->delay_usec(100000); + Vector array = FileAccess::_get_file_as_bytes("res://" + K.key); + tcp_peer->put_64(array.size()); + tcp_peer->put_data(array.ptr(), array.size()); + ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED); } + + tcp_peer->put_data((const uint8_t *)"GEND", 4); // End marker. + + print_verbose("EFS: Done."); } void EditorFileServer::start() { - stop(); + if (active) { + stop(); + } port = EDITOR_GET("filesystem/file_server/port"); password = EDITOR_GET("filesystem/file_server/password"); - cmd = CMD_ACTIVATE; + Error err = server->listen(port); + ERR_FAIL_COND_MSG(err != OK, "EditorFileServer: Unable to listen on port " + itos(port)); + active = true; } bool EditorFileServer::is_active() const { @@ -306,18 +264,19 @@ bool EditorFileServer::is_active() const { } void EditorFileServer::stop() { - cmd = CMD_STOP; + if (active) { + server->stop(); + active = false; + } } EditorFileServer::EditorFileServer() { server.instantiate(); - thread.start(_thread_start, this); EDITOR_DEF("filesystem/file_server/port", 6010); EDITOR_DEF("filesystem/file_server/password", ""); } EditorFileServer::~EditorFileServer() { - quit = true; - thread.wait_to_finish(); + stop(); } diff --git a/editor/debugger/editor_file_server.h b/editor/debugger/editor_file_server.h index ff2742e73fea4c..4374f508ee36e9 100644 --- a/editor/debugger/editor_file_server.h +++ b/editor/debugger/editor_file_server.h @@ -31,46 +31,24 @@ #ifndef EDITOR_FILE_SERVER_H #define EDITOR_FILE_SERVER_H -#include "core/io/file_access_network.h" #include "core/io/packet_peer.h" #include "core/io/tcp_server.h" #include "core/object/class_db.h" #include "core/os/thread.h" +#include "editor/editor_file_system.h" class EditorFileServer : public Object { GDCLASS(EditorFileServer, Object); - enum Command { - CMD_NONE, - CMD_ACTIVATE, - CMD_STOP, - }; - - struct ClientData { - Thread *thread = nullptr; - Ref connection; - HashMap> files; - EditorFileServer *efs = nullptr; - bool quit = false; - }; - Ref server; - HashSet to_wait; - - static void _close_client(ClientData *cd); - static void _subthread_start(void *s); - - Mutex wait_mutex; - Thread thread; - static void _thread_start(void *); - bool quit = false; - Command cmd = CMD_NONE; - String password; int port = 0; bool active = false; + void _scan_files_changed(EditorFileSystemDirectory *efd, const Vector &p_tags, HashMap &files_to_send, HashMap &cached_files); public: + void poll(); + void start(); void stop(); diff --git a/editor/export/editor_export_platform.cpp b/editor/export/editor_export_platform.cpp index 121088f27a51d7..c74ebff974f42c 100644 --- a/editor/export/editor_export_platform.cpp +++ b/editor/export/editor_export_platform.cpp @@ -818,12 +818,56 @@ String EditorExportPlatform::_export_customize(const String &p_path, LocalVector return save_path.is_empty() ? p_path : save_path; } +Vector EditorExportPlatform::get_forced_export_files() { + Vector files; + + files.push_back(ProjectSettings::get_singleton()->get_global_class_list_path()); + + String icon = GLOBAL_GET("application/config/icon"); + String splash = GLOBAL_GET("application/boot_splash/image"); + if (!icon.is_empty() && FileAccess::exists(icon)) { + files.push_back(icon); + } + if (!splash.is_empty() && FileAccess::exists(splash) && icon != splash) { + files.push_back(splash); + } + String resource_cache_file = ResourceUID::get_cache_file(); + if (FileAccess::exists(resource_cache_file)) { + files.push_back(resource_cache_file); + } + + String extension_list_config_file = GDExtension::get_extension_list_config_file(); + if (FileAccess::exists(extension_list_config_file)) { + files.push_back(extension_list_config_file); + } + + // Store text server data if it is supported. + if (TS->has_feature(TextServer::FEATURE_USE_SUPPORT_DATA)) { + bool use_data = GLOBAL_GET("internationalization/locale/include_text_server_data"); + if (use_data) { + // Try using user provided data file. + String ts_data = "res://" + TS->get_support_data_filename(); + if (FileAccess::exists(ts_data)) { + files.push_back(ts_data); + } else { + // Use default text server data. + String icu_data_file = EditorPaths::get_singleton()->get_cache_dir().path_join("tmp_icu_data"); + ERR_FAIL_COND_V(!TS->save_support_data(icu_data_file), files); + files.push_back(icu_data_file); + //Remove later + MessageQueue::get_singleton()->push_callable(callable_mp_static(DirAccess::remove_absolute), icu_data_file); + } + } + } + + return files; +} + Error EditorExportPlatform::export_project_files(const Ref &p_preset, bool p_debug, EditorExportSaveFunction p_func, void *p_udata, EditorExportSaveSharedObject p_so_func) { //figure out paths of files that will be exported HashSet paths; Vector path_remaps; - paths.insert(ProjectSettings::get_singleton()->get_global_class_list_path()); if (p_preset->get_export_filter() == EditorExportPreset::EXPORT_ALL_RESOURCES) { //find stuff _export_find_resources(EditorFileSystem::get_singleton()->get_filesystem(), paths); @@ -1295,69 +1339,15 @@ Error EditorExportPlatform::export_project_files(const Ref & } } - // Store icon and splash images directly, they need to bypass the import system and be loaded as images - String icon = GLOBAL_GET("application/config/icon"); - String splash = GLOBAL_GET("application/boot_splash/image"); - if (!icon.is_empty() && FileAccess::exists(icon)) { - Vector array = FileAccess::get_file_as_bytes(icon); - err = p_func(p_udata, icon, array, idx, total, enc_in_filters, enc_ex_filters, key); - if (err != OK) { - return err; - } - } - if (!splash.is_empty() && FileAccess::exists(splash) && icon != splash) { - Vector array = FileAccess::get_file_as_bytes(splash); - err = p_func(p_udata, splash, array, idx, total, enc_in_filters, enc_ex_filters, key); - if (err != OK) { - return err; - } - } - String resource_cache_file = ResourceUID::get_cache_file(); - if (FileAccess::exists(resource_cache_file)) { - Vector array = FileAccess::get_file_as_bytes(resource_cache_file); - err = p_func(p_udata, resource_cache_file, array, idx, total, enc_in_filters, enc_ex_filters, key); - if (err != OK) { - return err; - } - } - - String extension_list_config_file = GDExtension::get_extension_list_config_file(); - if (FileAccess::exists(extension_list_config_file)) { - Vector array = FileAccess::get_file_as_bytes(extension_list_config_file); - err = p_func(p_udata, extension_list_config_file, array, idx, total, enc_in_filters, enc_ex_filters, key); + Vector forced_export = get_forced_export_files(); + for (int i = 0; i < forced_export.size(); i++) { + Vector array = FileAccess::get_file_as_bytes(forced_export[i]); + err = p_func(p_udata, forced_export[i], array, idx, total, enc_in_filters, enc_ex_filters, key); if (err != OK) { return err; } } - // Store text server data if it is supported. - if (TS->has_feature(TextServer::FEATURE_USE_SUPPORT_DATA)) { - bool use_data = GLOBAL_GET("internationalization/locale/include_text_server_data"); - if (use_data) { - // Try using user provided data file. - String ts_data = "res://" + TS->get_support_data_filename(); - if (FileAccess::exists(ts_data)) { - Vector array = FileAccess::get_file_as_bytes(ts_data); - err = p_func(p_udata, ts_data, array, idx, total, enc_in_filters, enc_ex_filters, key); - if (err != OK) { - return err; - } - } else { - // Use default text server data. - String icu_data_file = EditorPaths::get_singleton()->get_cache_dir().path_join("tmp_icu_data"); - if (!TS->save_support_data(icu_data_file)) { - return ERR_INVALID_DATA; - } - Vector array = FileAccess::get_file_as_bytes(icu_data_file); - err = p_func(p_udata, ts_data, array, idx, total, enc_in_filters, enc_ex_filters, key); - DirAccess::remove_file_or_error(icu_data_file); - if (err != OK) { - return err; - } - } - } - } - String config_file = "project.binary"; String engine_cfb = EditorPaths::get_singleton()->get_cache_dir().path_join("tmp" + config_file); ProjectSettings::get_singleton()->save_custom(engine_cfb, custom_map, custom_list); diff --git a/editor/export/editor_export_platform.h b/editor/export/editor_export_platform.h index 21d871990224a1..df5a66f099a738 100644 --- a/editor/export/editor_export_platform.h +++ b/editor/export/editor_export_platform.h @@ -196,6 +196,8 @@ class EditorExportPlatform : public RefCounted { return worst_type; } + static Vector get_forced_export_files(); + virtual bool fill_log_messages(RichTextLabel *p_log, Error p_err); virtual void get_export_options(List *r_options) const = 0; diff --git a/editor/plugins/debugger_editor_plugin.cpp b/editor/plugins/debugger_editor_plugin.cpp index 7863e6d19e4f8d..f43f0397ad2929 100644 --- a/editor/plugins/debugger_editor_plugin.cpp +++ b/editor/plugins/debugger_editor_plugin.cpp @@ -126,8 +126,10 @@ void DebuggerEditorPlugin::_menu_option(int p_option) { if (ischecked) { file_server->stop(); + set_process(false); } else { file_server->start(); + set_process(true); } debug_menu->set_item_checked(debug_menu->get_item_index(RUN_FILE_SERVER), !ischecked); @@ -190,6 +192,9 @@ void DebuggerEditorPlugin::_notification(int p_what) { case NOTIFICATION_READY: { _update_debug_options(); } break; + case NOTIFICATION_PROCESS: { + file_server->poll(); + } break; } } diff --git a/main/main.cpp b/main/main.cpp index 5e0187cc7fd982..15e61bf75e6265 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -41,7 +41,6 @@ #include "core/input/input.h" #include "core/input/input_map.h" #include "core/io/dir_access.h" -#include "core/io/file_access_network.h" #include "core/io/file_access_pack.h" #include "core/io/file_access_zip.h" #include "core/io/image_loader.h" @@ -129,7 +128,6 @@ static Time *time_singleton = nullptr; #ifdef MINIZIP_ENABLED static ZipArchive *zip_packed_data = nullptr; #endif -static FileAccessNetworkClient *file_access_network_client = nullptr; static MessageQueue *message_queue = nullptr; // Initialized in setup2() @@ -1384,9 +1382,7 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph // Network file system needs to be configured before globals, since globals are based on the // 'project.godot' file which will only be available through the network if this is enabled - FileAccessNetwork::configure(); if (!remotefs.is_empty()) { - file_access_network_client = memnew(FileAccessNetworkClient); int port; if (remotefs.contains(":")) { port = remotefs.get_slicec(':', 1).to_int(); @@ -1394,14 +1390,14 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph } else { port = 6010; } + Error err = OS::get_singleton()->setup_remote_filesystem(remotefs, port, remotefs_pass, project_path); - Error err = file_access_network_client->connect(remotefs, port, remotefs_pass); if (err) { OS::get_singleton()->printerr("Could not connect to remotefs: %s:%i.\n", remotefs.utf8().get_data(), port); goto error; } - FileAccess::make_default(FileAccess::ACCESS_RESOURCES); + print_line("the path is now: " + project_path); } if (globals->setup(project_path, main_pack, upwards, editor) == OK) { @@ -1937,9 +1933,6 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph if (packed_data) { memdelete(packed_data); } - if (file_access_network_client) { - memdelete(file_access_network_client); - } unregister_core_driver_types(); unregister_core_extensions(); @@ -3448,9 +3441,6 @@ void Main::cleanup(bool p_force) { if (packed_data) { memdelete(packed_data); } - if (file_access_network_client) { - memdelete(file_access_network_client); - } if (performance) { memdelete(performance); }