-
Notifications
You must be signed in to change notification settings - Fork 151
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
RCORE-1969 Fix bundled encrypted Realm regression #7535
Changes from 7 commits
31df8bd
0fc35e8
6aeb01f
aa1d26f
594ee26
b1da5c9
38c340c
12cf9d4
326f3ba
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -74,7 +74,7 @@ size_t SlabAlloc::get_total_slab_size() noexcept | |
|
||
SlabAlloc::SlabAlloc() | ||
{ | ||
m_initial_section_size = 1UL << section_shift; // page_size(); | ||
m_initial_section_size = section_size(); | ||
m_free_space_state = free_space_Clean; | ||
m_baseline = 0; | ||
} | ||
|
@@ -718,16 +718,9 @@ bool SlabAlloc::align_filesize_for_mmap(ref_type top_ref, Config& cfg) | |
// check if online compaction allows us to shrink the file: | ||
if (top_ref) { | ||
// Get the expected file size by looking up logical file size stored in top array | ||
constexpr size_t max_top_size = (Group::s_file_size_ndx + 1) * 8 + sizeof(Header); | ||
size_t top_page_base = top_ref & ~(page_size() - 1); | ||
size_t top_offset = top_ref - top_page_base; | ||
size_t map_size = std::min(max_top_size + top_offset, size - top_page_base); | ||
File::Map<char> map_top(m_file, top_page_base, File::access_ReadOnly, map_size, 0, m_write_observer); | ||
realm::util::encryption_read_barrier(map_top, top_offset, max_top_size); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Here we were requesting the All of this is rather tricky to calculate accurately because we'd first have to map/decrypt the top_ref header, and then based on the array width, calculate the number of bytes to the |
||
auto top_header = map_top.get_addr() + top_offset; | ||
auto top_data = NodeHeader::get_data_from_header(top_header); | ||
auto w = NodeHeader::get_width_from_header(top_header); | ||
auto logical_size = size_t(get_direct(top_data, w, Group::s_file_size_ndx)) >> 1; | ||
Array top(*this); | ||
top.init_from_ref(top_ref); | ||
size_t logical_size = Group::get_logical_file_size(top); | ||
// make sure we're page aligned, so the code below doesn't first | ||
// truncate the file, then expand it again | ||
expected_size = round_up_to_page_size(logical_size); | ||
|
@@ -738,7 +731,6 @@ bool SlabAlloc::align_filesize_for_mmap(ref_type top_ref, Config& cfg) | |
detach(true); // keep m_file open | ||
m_file.resize(expected_size); | ||
m_file.close(); | ||
size = expected_size; | ||
return true; | ||
} | ||
|
||
|
@@ -849,7 +841,7 @@ ref_type SlabAlloc::attach_file(const std::string& path, Config& cfg, util::Writ | |
if (REALM_UNLIKELY(cfg.read_only)) | ||
throw InvalidDatabase("Read-only access to empty Realm file", path); | ||
|
||
size_t initial_size = page_size(); // m_initial_section_size; | ||
size_t initial_size = page_size(); | ||
// exFAT does not allocate a unique id for the file until it is non-empty. It must be | ||
// valid at this point because File::get_unique_id() is used to distinguish | ||
// mappings_for_file in the encryption layer. So the prealloc() is required before | ||
|
@@ -882,6 +874,12 @@ ref_type SlabAlloc::attach_file(const std::string& path, Config& cfg, util::Writ | |
DetachGuard dg(*this); | ||
|
||
reset_free_space_tracking(); | ||
|
||
// the file could have been produced on a device with a different | ||
// page size than our own so don't expect the size to be aligned | ||
if (cfg.encryption_key && size != 0 && size != round_up_to_page_size(size)) { | ||
size = round_up_to_page_size(size); | ||
} | ||
update_reader_view(size); | ||
REALM_ASSERT(m_mappings.size()); | ||
m_data = m_mappings[0].primary_mapping.get_addr(); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -23,6 +23,7 @@ | |
#include <realm/util/file_mapper.hpp> | ||
|
||
#include <algorithm> | ||
#include <atomic> | ||
#include <cerrno> | ||
#include <climits> | ||
#include <cstddef> | ||
|
@@ -49,6 +50,7 @@ | |
using namespace realm::util; | ||
|
||
namespace { | ||
constexpr size_t c_min_supported_page_size = 4096; | ||
size_t get_page_size() | ||
{ | ||
#ifdef _WIN32 | ||
|
@@ -60,13 +62,13 @@ size_t get_page_size() | |
#else | ||
long size = sysconf(_SC_PAGESIZE); | ||
#endif | ||
REALM_ASSERT(size > 0 && size % 4096 == 0); | ||
REALM_ASSERT(size > 0 && size % c_min_supported_page_size == 0); | ||
return static_cast<size_t>(size); | ||
} | ||
|
||
// This variable exists such that page_size() can return the page size without having to make any system calls. | ||
// It could also have been a static local variable, but Valgrind/Helgrind gives a false error on that. | ||
size_t cached_page_size = get_page_size(); | ||
std::atomic<size_t> cached_page_size = get_page_size(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Probably doesn't matter, but page_size() should do relaxed reads of this. |
||
|
||
bool for_each_helper(const std::string& path, const std::string& dir, realm::util::File::ForEachHandler& handler) | ||
{ | ||
|
@@ -410,7 +412,18 @@ std::string make_temp_file(const char* prefix) | |
|
||
size_t page_size() | ||
{ | ||
return cached_page_size; | ||
return cached_page_size.load(std::memory_order::memory_order_relaxed); | ||
} | ||
|
||
OnlyForTestingPageSizeChange::OnlyForTestingPageSizeChange(size_t new_page_size) | ||
{ | ||
REALM_ASSERT(new_page_size % c_min_supported_page_size == 0); | ||
cached_page_size = new_page_size; | ||
} | ||
|
||
OnlyForTestingPageSizeChange::~OnlyForTestingPageSizeChange() | ||
{ | ||
cached_page_size = get_page_size(); | ||
} | ||
|
||
void File::open_internal(const std::string& path, AccessMode a, CreateMode c, int flags, bool* success) | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -48,6 +48,7 @@ | |
#include <sstream> | ||
#include <regex> | ||
#include <thread> | ||
#include <iostream> | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Doesn't appear to be used. |
||
|
||
#include <realm/util/file.hpp> | ||
#include <realm/util/errno.hpp> | ||
|
@@ -128,7 +129,6 @@ int64_t fetch_value_in_file(const std::string& fname, const char* scan_pattern) | |
return PageReclaimGovernor::no_match; | ||
} | ||
|
||
|
||
/* Default reclaim governor | ||
* | ||
*/ | ||
|
@@ -485,13 +485,14 @@ SharedFileInfo* get_file_info_for_file(File& file) | |
return it->info.get(); | ||
} | ||
|
||
|
||
namespace { | ||
EncryptedFileMapping* add_mapping(void* addr, size_t size, const FileAttributes& file, size_t file_offset) | ||
{ | ||
size_t fs = to_size_t(File::get_size_static(file.fd)); | ||
if (fs > 0 && fs < page_size()) | ||
throw DecryptionFailed(); | ||
if (fs > 0 && fs < c_min_encrypted_file_size) | ||
throw DecryptionFailed( | ||
util::format("file size %1 is less than the minimum encrypted file size of %2 for '%3'", fs, | ||
c_min_encrypted_file_size, file.path)); | ||
|
||
LockGuard lock(mapping_mutex); | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Not sure why the Header is part of this calculation? Plus, multiplication by 8 only gives the true size if the top array width is maxed out. This is fine for a "maximum" but causes problems later on because we actually use this as the size to read from the file.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I assume the use of
sizeof(Header)
(24) should have actually beenNodeHeader::header_size
(8)