Skip to content

Commit 1e0c4e8

Browse files
committed
This PR introduces several performance optimizations for the binary write/build process
- it changes the way how the number of symbols is reduced and how offsets to names are calculated. Instead of n^2 algorithms, it introduces nlog(n) algorithm. When the symbols optimization step is performed map<symbol_name, offset_to_name> is created so it saves another n^2 pass - a couple of performance improvements to filter_iterator - from pre to post incrementation to save an unneeded copy, memoization of end iterator as the compiler cannot do it on its own and recreates it every loop iteration, simplified construction of the end iterator on the test binary that has a couple of hundreds of MB, it reduces the execution time of elf_add_section example form 20 minutes to a couple of seconds - adds an ability to modify single value inplace in the sectiton to save coying it just to do that - the reason behind this patch is to use LIEF as a patchelf replacement so the performance is one of the factors Signed-off-by: Janusz Lisiecki <jlisiecki@nvidia.com>
1 parent 84ffe2d commit 1e0c4e8

File tree

6 files changed

+168
-67
lines changed

6 files changed

+168
-67
lines changed

include/LIEF/ELF/Builder.hpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/* Copyright 2017 R. Thomas
22
* Copyright 2017 Quarkslab
3+
* Copyright 2020, NVIDIA CORPORATION. All rights reserved
34
*
45
* Licensed under the Apache License, Version 2.0 (the "License");
56
* you may not use this file except in compliance with the License.
@@ -20,6 +21,7 @@
2021
#include <memory>
2122
#include <algorithm>
2223
#include <string>
24+
#include <unordered_map>
2325

2426
#include "LIEF/visibility.h"
2527
#include "LIEF/iostream.hpp"
@@ -96,7 +98,8 @@ class LIEF_API Builder {
9698
void build_symbol_definition(void);
9799

98100
template<typename T, typename HANDLER>
99-
std::vector<std::string> optimize(const HANDLER& e);
101+
std::vector<std::string> optimize(const HANDLER& e,
102+
std::unordered_map<std::string, size_t> *of_map_p=nullptr);
100103

101104
template<typename ELF_T>
102105
void build_symbol_version(void);

include/LIEF/ELF/Segment.hpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,9 @@ class LIEF_API Segment : public Object {
8181
void alignment(uint64_t alignment);
8282
void content(const std::vector<uint8_t>& content);
8383
void content(std::vector<uint8_t>&& content);
84+
template<typename T> T get_content_value(size_t offset) const;
85+
template<typename T> void set_content_value(size_t offset, T value);
86+
size_t get_content_size() const;
8487

8588
it_sections sections(void);
8689
it_const_sections sections(void) const;

include/LIEF/iterators.hpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/* Copyright 2017 R. Thomas
22
* Copyright 2017 Quarkslab
3+
* Copyright 2020, NVIDIA CORPORATION. All rights reserved.
34
*
45
* Licensed under the Apache License, Version 2.0 (the "License");
56
* you may not use this file except in compliance with the License.
@@ -367,7 +368,8 @@ class filter_iterator : public std::iterator<
367368
}
368369

369370
filter_iterator end(void) const {
370-
filter_iterator it_end{this->container_, this->filters_};
371+
// we don't need filter for the end iterator
372+
filter_iterator it_end{this->container_};
371373

372374
it_end.it_ = it_end.container_.end();
373375
it_end.distance_ = it_end.container_.size();
@@ -432,7 +434,8 @@ class filter_iterator : public std::iterator<
432434
filter_iterator it = this->begin();
433435
size_t size = 0;
434436

435-
while (it++ != std::end(it)) size++;
437+
auto end_iter = std::end(it);
438+
for (; it != end_iter; ++it) ++size;
436439
this->size_c_ = size;
437440
return this->size_c_;
438441
}

src/ELF/Binary.tcc

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -234,8 +234,8 @@ void Binary::patch_addend(Relocation& relocation, uint64_t from, uint64_t shift)
234234
VLOG(VDEBUG) << "Patch addend relocation at address: 0x" << std::hex << address;
235235
Segment& segment = segment_from_virtual_address(address);
236236
const uint64_t relative_offset = this->virtual_address_to_offset(address) - segment.file_offset();
237-
std::vector<uint8_t> segment_content = segment.content();
238-
const size_t segment_size = segment_content.size();
237+
238+
const size_t segment_size = segment.get_content_size();
239239

240240
if (segment_size == 0) {
241241
LOG(WARNING) << "Segment is empty nothing to do";
@@ -247,13 +247,13 @@ void Binary::patch_addend(Relocation& relocation, uint64_t from, uint64_t shift)
247247
return;
248248
}
249249

250-
T* value = reinterpret_cast<T*>(segment_content.data() + relative_offset);
250+
T value = segment.get_content_value<T>(relative_offset);
251251

252-
if (value != nullptr and *value >= from) {
253-
*value += shift;
252+
if (value >= from) {
253+
value += shift;
254254
}
255255

256-
segment.content(segment_content);
256+
segment.set_content_value(relative_offset, value);
257257
}
258258

259259

src/ELF/Builder.tcc

Lines changed: 94 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/* Copyright 2017 R. Thomas
22
* Copyright 2017 Quarkslab
3+
* Copyright 2020, NVIDIA CORPORATION. All rights reserved.
34
*
45
* Licensed under the Apache License, Version 2.0 (the "License");
56
* you may not use this file except in compliance with the License.
@@ -14,6 +15,7 @@
1415
* limitations under the License.
1516
*/
1617
#include <numeric>
18+
#include <unordered_map>
1719

1820
#include "LIEF/logging++.hpp"
1921

@@ -144,15 +146,16 @@ void Builder::build(void) {
144146

145147
}
146148

147-
148149
template<typename T, typename HANDLER>
149-
std::vector<std::string> Builder::optimize(const HANDLER& container) {
150+
std::vector<std::string> Builder::optimize(const HANDLER& container,
151+
std::unordered_map<std::string, size_t> *of_map_p) {
150152

151153
std::set<std::string> string_table;
152-
153154
std::vector<std::string> string_table_optimized;
155+
string_table_optimized.reserve(container.size());
154156

155-
// Insert all strings in a std::set<> ordered by size
157+
// reverse all symbol names and sort them so we can merge then in the linear time:
158+
// aaa, aadd, aaaa, cca, ca -> aaaa, aaa, acc, ac ddaa
156159
std::transform(
157160
std::begin(container),
158161
std::end(container),
@@ -161,32 +164,73 @@ std::vector<std::string> Builder::optimize(const HANDLER& container) {
161164
std::end(string_table)),
162165
std::mem_fn(static_cast<const std::string& (T::*)(void) const>(&T::name)));
163166

164-
std::vector<std::string> sorted_table;
165-
sorted_table.reserve(container.size());
166-
std::move(std::begin(string_table), std::end(string_table),
167-
std::back_inserter(sorted_table));
168-
169-
std::stable_sort(std::begin(sorted_table), std::end(sorted_table),
170-
[] (const std::string& lhs, const std::string& rhs) {
171-
return lhs.size() >= rhs.size();
172-
});
167+
for (auto &val: string_table) {
168+
string_table_optimized.emplace_back(std::move(val));
169+
std::reverse(std::begin(string_table_optimized.back()), std::end(string_table_optimized.back()));
170+
}
173171

174-
// Optimize the string table
175-
std::copy_if(
176-
std::begin(sorted_table),
177-
std::end(sorted_table),
178-
std::back_inserter(string_table_optimized),
179-
[&string_table_optimized] (const std::string& name) {
180-
// Check if the given string **IS** the suffix of another string
181-
auto it = std::find_if(
182-
std::begin(string_table_optimized),
183-
std::end(string_table_optimized),
184-
[&name] (const std::string& nameOpti) {
185-
return nameOpti.substr(nameOpti.size() - name.size()) == name ;
186-
});
187-
return (it == std::end(string_table_optimized));
172+
std::sort(std::begin(string_table_optimized), std::end(string_table_optimized),
173+
[] (const std::string& lhs, const std::string& rhs) {
174+
bool ret = false;
175+
if (lhs.size() > rhs.size()) {
176+
auto res = lhs.compare(0, rhs.size(), rhs);
177+
ret = (res <= 0);
178+
} else {
179+
auto res = rhs.compare(0, lhs.size(), lhs);
180+
ret = (res > 0);
181+
}
182+
return ret;
188183
});
189184

185+
// as all elements that can be merged are adjacent we can just go through the list once
186+
// and memorize one we merged to calculate the offsets later
187+
std::unordered_map<std::string, std::string> merged_map;
188+
size_t to_set_idx = 0, cur_elm_idx = 1;
189+
for (; cur_elm_idx < string_table_optimized.size(); ++cur_elm_idx) {
190+
auto &cur_elm = string_table_optimized[cur_elm_idx];
191+
auto &to_set_elm = string_table_optimized[to_set_idx];
192+
if (to_set_elm.size() >= cur_elm.size()) {
193+
auto ret = to_set_elm.compare(0, cur_elm.size(), cur_elm);
194+
if (ret == 0) {
195+
// when memorizing reverse back symbol names
196+
std::string rev_cur_elm = cur_elm;
197+
std::string rev_to_set_elm = to_set_elm;
198+
std::reverse(std::begin(rev_cur_elm), std::end(rev_cur_elm));
199+
std::reverse(std::begin(rev_to_set_elm), std::end(rev_to_set_elm));
200+
merged_map[rev_cur_elm] = rev_to_set_elm;
201+
continue;
202+
}
203+
}
204+
++to_set_idx;
205+
std::swap(string_table_optimized[to_set_idx], cur_elm);
206+
}
207+
// if the first one is empty
208+
if (string_table_optimized[0].size() == 0) {
209+
std::swap(string_table_optimized[0], string_table_optimized[to_set_idx]);
210+
--to_set_idx;
211+
}
212+
string_table_optimized.resize(to_set_idx + 1);
213+
214+
//reverse symbols back and sort them again
215+
for (auto &val: string_table_optimized) {
216+
std::reverse(std::begin(val), std::end(val));
217+
}
218+
std::sort(std::begin(string_table_optimized), std::end(string_table_optimized));
219+
220+
if (of_map_p) {
221+
std::unordered_map<std::string, size_t> offset_map;
222+
offset_map[""] = 0;
223+
size_t offset_counter = 1;
224+
for (const auto &v : string_table_optimized) {
225+
offset_map[v] = offset_counter;
226+
offset_counter += v.size() + 1;
227+
}
228+
for (const auto &kv : merged_map) {
229+
offset_map[kv.first] = offset_map[kv.second] + (kv.second.size() - kv.first.size());
230+
}
231+
*of_map_p = std::move(offset_map);
232+
}
233+
190234
return string_table_optimized;
191235
}
192236

@@ -442,10 +486,12 @@ void Builder::build_static_symbols(void) {
442486
vector_iostream content(this->should_swap());
443487
content.reserve(this->binary_->static_symbols_.size() * sizeof(Elf_Sym));
444488
std::vector<uint8_t> string_table_raw;
489+
std::unordered_map<std::string, size_t> offset_name_map;
445490

446491
// Container which will hold symbols name (optimized)
447492
std::vector<std::string> string_table_optimize =
448-
this->optimize<Symbol, decltype(this->binary_->static_symbols_)>(this->binary_->static_symbols_);
493+
this->optimize<Symbol, decltype(this->binary_->static_symbols_)>(this->binary_->static_symbols_,
494+
&offset_name_map);
449495

450496
// We can't start with a symbol name
451497
string_table_raw.push_back(0);
@@ -459,19 +505,13 @@ void Builder::build_static_symbols(void) {
459505
VLOG(VDEBUG) << "Dealing with symbol: " << symbol->name();
460506
const std::string& name = symbol->name();
461507

462-
// Check if name is already pressent
463-
auto&& it_name = std::search(
464-
std::begin(string_table_raw),
465-
std::end(string_table_raw),
466-
name.c_str(),
467-
name.c_str() + name.size() + 1);
468-
469-
470-
if (it_name == std::end(string_table_raw)) {
471-
throw LIEF::not_found("Unable to find symbol '" + name + "' in the string table");
508+
auto offset_it = offset_name_map.find(name);
509+
if (offset_it == std::end(offset_name_map)) {
510+
throw LIEF::not_found("Unable to find symbol '" + name + "' in the string table");
472511
}
473512

474-
const Elf_Off name_offset = static_cast<Elf_Off>(std::distance(std::begin(string_table_raw), it_name));
513+
const Elf_Off name_offset = static_cast<Elf_Off>(offset_it->second);
514+
475515

476516
Elf_Sym sym_hdr;
477517
memset(&sym_hdr, 0, sizeof(sym_hdr));
@@ -1052,33 +1092,29 @@ void Builder::build_dynamic_symbols(void) {
10521092

10531093
// Build symbols string table
10541094
std::vector<uint8_t> string_table_raw = string_table_section.content();
1095+
std::unordered_map<std::string, size_t> offset_name_map;
1096+
size_t additional_offset = string_table_raw.size() - 1;
10551097

10561098
std::vector<std::string> string_table_optimized =
1057-
this->optimize<Symbol, decltype(this->binary_->dynamic_symbols_)>(this->binary_->dynamic_symbols_);
1099+
this->optimize<Symbol, decltype(this->binary_->dynamic_symbols_)>(this->binary_->dynamic_symbols_,
1100+
&offset_name_map);
10581101

10591102
for (const std::string& name : string_table_optimized) {
10601103
string_table_raw.insert(std::end(string_table_raw), std::begin(name), std::end(name));
10611104
string_table_raw.push_back(0);
10621105
}
10631106

1064-
10651107
//
10661108
// Build symbols
10671109
//
10681110
vector_iostream symbol_table_raw(this->should_swap());
10691111
for (const Symbol* symbol : this->binary_->dynamic_symbols_) {
10701112
const std::string& name = symbol->name();
1071-
// Check if name is already pressent
1072-
auto&& it_name = std::search(
1073-
std::begin(string_table_raw),
1074-
std::end(string_table_raw),
1075-
name.c_str(),
1076-
name.c_str() + name.size() + 1);
1077-
1078-
if (it_name == std::end(string_table_raw)) {
1113+
auto offset_it = offset_name_map.find(name);
1114+
if (offset_it == std::end(offset_name_map)) {
10791115
throw LIEF::not_found("Unable to find the symbol in the string table");
10801116
}
1081-
const Elf_Off name_offset = static_cast<Elf_Off>(std::distance(std::begin(string_table_raw), it_name));
1117+
const Elf_Off name_offset = static_cast<Elf_Off>(offset_name_map[name] + additional_offset);
10821118

10831119
Elf_Sym sym_header;
10841120

@@ -1285,15 +1321,15 @@ void Builder::build_dynamic_relocations(void) {
12851321
VLOG(VDEBUG) << "[+] Building dynamic relocations";
12861322

12871323
it_dynamic_relocations dynamic_relocations = this->binary_->dynamic_relocations();
1288-
1324+
auto end_iter = std::end(dynamic_relocations);
12891325
bool isRela = dynamic_relocations[0].is_rela();
1290-
if (not std::all_of(
1291-
std::begin(dynamic_relocations),
1292-
std::end(dynamic_relocations),
1293-
[isRela] (const Relocation& relocation) {
1294-
return relocation.is_rela() == isRela;
1295-
})) {
1296-
throw LIEF::type_error("Relocation are not of the same type");
1326+
for (; dynamic_relocations != end_iter; ++dynamic_relocations) {
1327+
if (dynamic_relocations->is_rela() != isRela) {
1328+
break;
1329+
}
1330+
}
1331+
if (dynamic_relocations != end_iter) {
1332+
throw LIEF::type_error("Relocation are not of the same type");
12971333
}
12981334

12991335
dynamic_entries_t::iterator it_dyn_relocation;

src/ELF/Segment.cpp

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,62 @@ std::vector<uint8_t> Segment::content(void) const {
182182
return {binary_content.data() + node.offset(), binary_content.data() + node.offset() + node.size()};
183183
}
184184

185+
size_t Segment::get_content_size() const {
186+
DataHandler::Node& node = this->datahandler_->get(
187+
this->file_offset(),
188+
this->physical_size(),
189+
DataHandler::Node::SEGMENT);
190+
return node.size();
191+
}
192+
template<typename T> T Segment::get_content_value(size_t offset) const {
193+
T ret;
194+
if (this->datahandler_ == nullptr) {
195+
VLOG(VDEBUG) << "Content from cache";
196+
memcpy(&ret, this->content_c_.data() + offset, sizeof(T));
197+
} else {
198+
DataHandler::Node& node = this->datahandler_->get(
199+
this->file_offset(),
200+
this->physical_size(),
201+
DataHandler::Node::SEGMENT);
202+
const std::vector<uint8_t>& binary_content = this->datahandler_->content();
203+
memcpy(&ret, binary_content.data() + node.offset() + offset, sizeof(T));
204+
}
205+
return ret;
206+
}
207+
208+
template unsigned short Segment::get_content_value<unsigned short>(size_t offset) const;
209+
template unsigned int Segment::get_content_value<unsigned int>(size_t offset) const;
210+
template unsigned long Segment::get_content_value<unsigned long>(size_t offset) const;
211+
template unsigned long long Segment::get_content_value<unsigned long long>(size_t offset) const;
212+
213+
template<typename T> void Segment::set_content_value(size_t offset, T value) {
214+
if (this->datahandler_ == nullptr) {
215+
VLOG(VDEBUG) << "Content from cache";
216+
if (offset + sizeof(T) > this->content_c_.size()) {
217+
this->content_c_.resize(offset + sizeof(T));
218+
this->physical_size(offset + sizeof(T));
219+
}
220+
memcpy(this->content_c_.data() + offset, &value, sizeof(T));
221+
} else {
222+
DataHandler::Node& node = this->datahandler_->get(
223+
this->file_offset(),
224+
this->physical_size(),
225+
DataHandler::Node::SEGMENT);
226+
std::vector<uint8_t>& binary_content = this->datahandler_->content();
227+
228+
if (offset + sizeof(T) > binary_content.size()) {
229+
this->datahandler_->reserve(node.offset(), offset + sizeof(T));
230+
LOG(WARNING) << "You inserted data in segment '"
231+
<< to_string(this->type()) << "' It may lead to overaly!" << std::endl;
232+
}
233+
this->physical_size(node.size());
234+
memcpy(binary_content.data() + node.offset() + offset, &value, sizeof(T));
235+
}
236+
}
237+
template void Segment::set_content_value<unsigned short>(size_t offset, unsigned short value);
238+
template void Segment::set_content_value<unsigned int>(size_t offset, unsigned int value);
239+
template void Segment::set_content_value<unsigned long>(size_t offset, unsigned long value);
240+
template void Segment::set_content_value<unsigned long long>(size_t offset, unsigned long long value);
185241

186242
it_const_sections Segment::sections(void) const {
187243
return {this->sections_};

0 commit comments

Comments
 (0)