diff --git a/CMakeLists.txt b/CMakeLists.txt index d110ef58..a537ef34 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,7 @@ add_subdirectory("vm") STEP_TARGETS build) if(UBPF_ENABLE_TESTS) + add_subdirectory("custom_tests") add_subdirectory("ubpf_plugin") if (NOT UBPF_SKIP_EXTERNAL) endif() diff --git a/cmake/settings.cmake b/cmake/settings.cmake index 3a06504f..0f2f0afb 100644 --- a/cmake/settings.cmake +++ b/cmake/settings.cmake @@ -17,7 +17,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) -Wall -Werror -Iinc - -O2 + -O0 + -g -Wunused-parameter -fPIC ) diff --git a/custom_tests/CMakeLists.txt b/custom_tests/CMakeLists.txt new file mode 100644 index 00000000..9cd82c1d --- /dev/null +++ b/custom_tests/CMakeLists.txt @@ -0,0 +1,62 @@ +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: Apache-2.0 + +set(CMAKE_CXX_STANDARD 20) + +file(GLOB test_descr_files ${CMAKE_SOURCE_DIR}/custom_tests/descrs/*.md) + +add_library(ubpf_custom_test_support srcs/ubpf_custom_test_support.cc) + +target_link_libraries( + ubpf_custom_test_support + ubpf + ubpf_settings +) + +target_include_directories(ubpf_custom_test_support PUBLIC ".srcs/") +target_include_directories(ubpf_custom_test_support PRIVATE + "${CMAKE_SOURCE_DIR}/vm" + "${CMAKE_BINARY_DIR}/vm" + "${CMAKE_SOURCE_DIR}/vm/inc" + "${CMAKE_BINARY_DIR}/vm/inc" +) + +set(QEMU_RUNNER "") +if(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64 AND (NOT CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL aarch64)) + set(QEMU_RUNNER qemu-aarch64 -L /usr/aarch64-linux-gnu) +endif() + +foreach(test_file ${test_descr_files}) + get_filename_component(test_name ${test_file} NAME_WE) + set(test_source_path "${CMAKE_SOURCE_DIR}/custom_tests/srcs/${test_name}.cc") + + add_executable( + ${test_name} + ${test_source_path} + ) + target_include_directories(${test_name} PRIVATE + "${CMAKE_SOURCE_DIR}/vm" + "${CMAKE_BINARY_DIR}/vm" + "${CMAKE_SOURCE_DIR}/vm/inc" + "${CMAKE_BINARY_DIR}/vm/inc" + ) + target_link_libraries( + ${test_name} + ubpf + ubpf_custom_test_support + ubpf_settings + ) + set(potential_input_file ${CMAKE_SOURCE_DIR}/custom_tests/data/${test_name}.input) + if (EXISTS ${potential_input_file}) + list(JOIN QEMU_RUNNER " " QEMU_RUNNER_STR) + add_test( + NAME ${test_name}-Custom + COMMAND sh -c "cat ${potential_input_file} | ${QEMU_RUNNER_STR} $" + ) + else() + add_test( + NAME ${test_name}-Custom + COMMAND ${QEMU_RUNNER} $ + ) + endif() +endforeach() diff --git a/custom_tests/README.md b/custom_tests/README.md new file mode 100644 index 00000000..5ce054c4 --- /dev/null +++ b/custom_tests/README.md @@ -0,0 +1,35 @@ +## Writing a uBPF Custom Tests + +Custom tests are enabled by creating two (2) or three (3) different files in the `custom_tests` directory. + +### Files Of a uBPF Custom Test + +#### Description Files + +The first file to create is the Description File. The Description File is a file with a `.md` extension that resides in the `descrs` directory. The purpose of this file is to identify the name of the test (everything before the `.md` extension) and provide a place to document the purpose of the test. + +#### Source Files + +The second file to create is the Source File. The Source file should reside in the `srcs` directory and have a name that matches its Description File (with the `.cc` extension rather than the `.md` extension). + +#### Input Files + +The final file is optional. The Input File resides in the `data` directory and should have the same name as the other two (2) files but with an `.input` extension rather than `.cc` or `.md` for the Source and Description File respectively. If present, the contents of this file will be given to the executed custom test over standard input. + +### Building + +The Source Files for a custom test are compiled using C++20 and are saved as an executable named according to the name of the test in the CMake build directory. + +### Return Values + +All successful tests should return `0`. All failing tests should return something other than `0`. + +### Supporting Libraries + +To reduce the boilerplate needed to write custom tests, there is a custom test library with several helpful functions. These functions are documented in the library's header file (`custom_tests/srcs/ubpf_custom_test_support.h`). + +### Putting It Together + +After describing the test's purpose in a Markdown syntax in a file named, say, `test_example.md` and stored in the `descrs` directory, you can write the test's Source Code (in C++20) and give it the name `test_example.cc` in the `srcs` directory. If the test needs input, you can save that input in the tests Input File (`test_input.input`) in the `data` directory. + +Because all the files are present, this test will be run when the CTest target is invoked. Because there the optional `test_input.input` file is present, the contents of that file will be given to the executable via standard input. \ No newline at end of file diff --git a/custom_tests/data/ubpf_test_external_dispatcher_context_overwrite.input b/custom_tests/data/ubpf_test_external_dispatcher_context_overwrite.input new file mode 100644 index 00000000..45c5a332 --- /dev/null +++ b/custom_tests/data/ubpf_test_external_dispatcher_context_overwrite.input @@ -0,0 +1 @@ +b7 01 00 00 01 02 03 04 85 00 00 00 01 00 00 00 95 00 00 00 00 00 00 00 diff --git a/custom_tests/data/ubpf_test_external_dispatcher_simple_context.input b/custom_tests/data/ubpf_test_external_dispatcher_simple_context.input new file mode 100644 index 00000000..12873c00 --- /dev/null +++ b/custom_tests/data/ubpf_test_external_dispatcher_simple_context.input @@ -0,0 +1 @@ +85 00 00 00 01 00 00 00 95 00 00 00 00 00 00 00 diff --git a/custom_tests/data/ubpf_test_jit_unexpected_instruction.input b/custom_tests/data/ubpf_test_jit_unexpected_instruction.input new file mode 100644 index 00000000..d5be5a56 --- /dev/null +++ b/custom_tests/data/ubpf_test_jit_unexpected_instruction.input @@ -0,0 +1 @@ +8f 00 00 00 01 00 00 00 \ No newline at end of file diff --git a/custom_tests/data/ubpf_test_update_dispatcher.input b/custom_tests/data/ubpf_test_update_dispatcher.input new file mode 100644 index 00000000..12873c00 --- /dev/null +++ b/custom_tests/data/ubpf_test_update_dispatcher.input @@ -0,0 +1 @@ +85 00 00 00 01 00 00 00 95 00 00 00 00 00 00 00 diff --git a/custom_tests/data/ubpf_test_update_helpers.input b/custom_tests/data/ubpf_test_update_helpers.input new file mode 100644 index 00000000..12873c00 --- /dev/null +++ b/custom_tests/data/ubpf_test_update_helpers.input @@ -0,0 +1 @@ +85 00 00 00 01 00 00 00 95 00 00 00 00 00 00 00 diff --git a/custom_tests/descrs/ubpf_test_external_dispatcher_context_overwrite.md b/custom_tests/descrs/ubpf_test_external_dispatcher_context_overwrite.md new file mode 100644 index 00000000..a743b7d9 --- /dev/null +++ b/custom_tests/descrs/ubpf_test_external_dispatcher_context_overwrite.md @@ -0,0 +1,7 @@ +## Test Description + +This custom test program tests whether JIT'd eBPF programs properly pass the original context +to external helper dispatcher even when (eBPF) register r0 has been modified. The original +context to the eBPF program is passed in (eBPF) register r0. Subsequent changes to that +register by the eBPF program should *not* affect that context (which is given to the +helper function external dispatcher). \ No newline at end of file diff --git a/custom_tests/descrs/ubpf_test_external_dispatcher_simple_context.md b/custom_tests/descrs/ubpf_test_external_dispatcher_simple_context.md new file mode 100644 index 00000000..4c484094 --- /dev/null +++ b/custom_tests/descrs/ubpf_test_external_dispatcher_simple_context.md @@ -0,0 +1,4 @@ +## Test Description + +This custom test program tests whether JIT'd eBPF programs properly pass the context +to external helper dispatcher. diff --git a/custom_tests/descrs/ubpf_test_jit_buffer_too_small.md b/custom_tests/descrs/ubpf_test_jit_buffer_too_small.md new file mode 100644 index 00000000..f7213f7f --- /dev/null +++ b/custom_tests/descrs/ubpf_test_jit_buffer_too_small.md @@ -0,0 +1,4 @@ +## Test Description + +This custom test program tests whether compilation fails (with the proper error message) when +the user gives a buffer that is too small to accommodate the size of the JIT'd code. diff --git a/custom_tests/descrs/ubpf_test_jit_unexpected_instruction.md b/custom_tests/descrs/ubpf_test_jit_unexpected_instruction.md new file mode 100644 index 00000000..f8f4c28f --- /dev/null +++ b/custom_tests/descrs/ubpf_test_jit_unexpected_instruction.md @@ -0,0 +1,4 @@ +## Test Description + +This custom test program tests that an eBPF program fails to load (with the proper error) in +the presence of a program with an invalid instruction opcode. diff --git a/custom_tests/descrs/ubpf_test_update_dispatcher.md b/custom_tests/descrs/ubpf_test_update_dispatcher.md new file mode 100644 index 00000000..9fe6d1e3 --- /dev/null +++ b/custom_tests/descrs/ubpf_test_update_dispatcher.md @@ -0,0 +1,4 @@ +## Test Description + +This custom test program tests whether it is possible to update the external helper dispatcher +after an eBPF program has been compiled. diff --git a/custom_tests/descrs/ubpf_test_update_helpers.md b/custom_tests/descrs/ubpf_test_update_helpers.md new file mode 100644 index 00000000..e0a872ad --- /dev/null +++ b/custom_tests/descrs/ubpf_test_update_helpers.md @@ -0,0 +1,4 @@ +## Test Description + +This custom test program tests whether it is possible to update the external helper +functions for an eBPF program that has already been JIT'd. diff --git a/custom_tests/srcs/test_helpers.h b/custom_tests/srcs/test_helpers.h new file mode 100644 index 00000000..1d190865 --- /dev/null +++ b/custom_tests/srcs/test_helpers.h @@ -0,0 +1,225 @@ +// Copyright (c) Microsoft Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once +#include "ubpf.h" +#include +#include +#include +#include + +#if !defined(UNREFERENCED_PARAMETER) +#define UNREFERENCED_PARAMETER(P) (void)(P) +#endif + +static uint64_t +gather_bytes(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + return ((uint64_t)(a & 0xff) << 32) | ((uint64_t)(b & 0xff) << 24) | ((uint64_t)(c & 0xff) << 16) | + ((uint64_t)(d & 0xff) << 8) | (e & 0xff); +}; + +static uint64_t +memfrob(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + + uint8_t* p = reinterpret_cast(a); + for (uint64_t i = 0; i < b; i++) { + p[i] ^= 42; + } + return 0; +}; + +; + +static uint64_t +no_op(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(a); + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + + return 0; +} + +static uint64_t +sqrti(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + + return static_cast(std::sqrt(a)); +} + +static uint64_t +strcmp_ext(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return strcmp(reinterpret_cast(a), reinterpret_cast(b)); +} + +static uint64_t +unwind(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return a; +} + +static std::map helper_functions = { + {0, gather_bytes}, + {1, memfrob}, + {2, no_op}, + {3, sqrti}, + {4, strcmp_ext}, + {5, unwind}, +}; + +static uint64_t +dispatcher_test_memfrob(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(a); + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return 42; + +} + +static uint64_t +updated_dispatcher_test_memfrob(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(a); + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return 43; +} + +static uint64_t +dispatcher_gather_bytes(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(a); + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return 44; + +} + +static uint64_t +updated_dispatcher_gather_bytes(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(a); + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return 45; +} + +static uint64_t +dispatcher_no_op(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(a); + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return 46; + +} + +static uint64_t +updated_dispatcher_no_op(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(a); + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return 47; +} + +static uint64_t +dispatcher_sqrti(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(a); + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return 48; + +} + +static uint64_t +updated_dispatcher_sqrti(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(a); + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return 49; +} + +static uint64_t +dispatcher_strcmp_ext(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(a); + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return 50; + +} + +static uint64_t +updated_dispatcher_strcmp_ext(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(a); + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return 51; +} + +static uint64_t +dispatcher_unwind(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(a); + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return 52; + +} + +static uint64_t +updated_dispatcher_unwind(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) +{ + UNREFERENCED_PARAMETER(a); + UNREFERENCED_PARAMETER(b); + UNREFERENCED_PARAMETER(c); + UNREFERENCED_PARAMETER(d); + UNREFERENCED_PARAMETER(e); + return 53; +} diff --git a/custom_tests/srcs/ubpf_custom_test_support.cc b/custom_tests/srcs/ubpf_custom_test_support.cc new file mode 100644 index 00000000..cdc2368e --- /dev/null +++ b/custom_tests/srcs/ubpf_custom_test_support.cc @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include + +extern "C" +{ +#include "ebpf.h" +#include "ubpf.h" +} + + +#include "ubpf_custom_test_support.h" + +std::vector +base16_decode(const std::string &input) +{ + std::vector output; + std::stringstream ss(input); + std::string value; + output.reserve(input.size() / 3); + while (std::getline(ss, value, ' ')) + { + try + { + output.push_back(static_cast(std::stoi(value, nullptr, 16))); + } + catch (...) + { + // Ignore invalid values. + } + } + return output; +} + +std::vector +bytes_to_ebpf_inst(std::vector bytes) +{ + std::vector instructions(bytes.size() / sizeof(ebpf_inst)); + memcpy(instructions.data(), bytes.data(), bytes.size()); + return instructions; +} + + +bool ubpf_setup_custom_test(ubpf_vm_up &vm, + const std::string program_string, + std::optional fixup_f, + ubpf_jit_fn &jit_fn, + std::string &error) +{ + jit_fn = nullptr; + std::vector program = bytes_to_ebpf_inst(base16_decode(program_string)); + char *error_s{nullptr}; + + if (vm == nullptr) + { + error = "VM not provided"; + return false; + } + + if (ubpf_set_unwind_function_index(vm.get(), 5) != 0) + { + error = "Failed to set unwind function index"; + return false; + } + + if (fixup_f.has_value()) + { + if (!(fixup_f.value())(vm, error)) { + return false; + } + } + + if (ubpf_load(vm.get(), program.data(), static_cast(program.size() * sizeof(ebpf_inst)), &error_s) != 0) + { + error = "Failed to load program: " + std::string{error_s}; + free(error_s); + return false; + } + + jit_fn = ubpf_compile(vm.get(), &error_s); + if (jit_fn == nullptr) + { + error = "Failed to compile: " + std::string{error_s}; + free(error_s); + return false; + } + + assert(error_s == nullptr); + free(error_s); + return true; +} diff --git a/custom_tests/srcs/ubpf_custom_test_support.h b/custom_tests/srcs/ubpf_custom_test_support.h new file mode 100644 index 00000000..8957c5c6 --- /dev/null +++ b/custom_tests/srcs/ubpf_custom_test_support.h @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include + +extern "C" +{ +#include "ebpf.h" +#include "ubpf.h" +} + +#define UNREFERENCED_PARAMETER (void) + +/** + * @brief Read in a string of hex bytes and return a vector of bytes. + * + * @param[in] input String containing hex bytes. + * @return Vector of bytes. + */ +std::vector +base16_decode(const std::string &input); + + +/** + * @brief Convert a vector of bytes to a vector of ebpf_inst. + * + * @param[in] bytes Vector of bytes. + * @return Vector of ebpf_inst. + */ +std::vector +bytes_to_ebpf_inst(std::vector bytes); + + +using ubpf_vm_up = std::unique_ptr; +using custom_test_fixup_cb = std::function; + + +/** + * @brief Do the common necessary work to setup a custom test. + * + * @param[in] vm The VM for which to prepare the test. + * @param[in] program_string A string of raw bytes that make up the eBPF program to execute under this test. + * @param[in] fixup_f A function that will be invoked after the program is loaded and before it is compiled. + * @param[out] jit_fn A function that can be invoked to run the jit'd program. + * @param[out] error A string containing the error message (if any) generated during custom test configuration. + * @return True or false depending on whether setting up the custom test succeeded. + */ +bool ubpf_setup_custom_test(ubpf_vm_up &vm, + const std::string program_string, + std::optional fixup_f, + ubpf_jit_fn &jit_fn, + std::string &error); + diff --git a/custom_tests/srcs/ubpf_test_external_dispatcher_context_overwrite.cc b/custom_tests/srcs/ubpf_test_external_dispatcher_context_overwrite.cc new file mode 100644 index 00000000..b99d0448 --- /dev/null +++ b/custom_tests/srcs/ubpf_test_external_dispatcher_context_overwrite.cc @@ -0,0 +1,76 @@ +// Copyright (c) Will Hawkins +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include + +extern "C" +{ +#include "ubpf.h" +} + +#include "ubpf_custom_test_support.h" + +uint64_t *cookie_pointer_value{nullptr}; +uint64_t +external_dispatcher(uint64_t p0, uint64_t p1, uint64_t p2, uint64_t p3, uint64_t p4, unsigned int idx, void* cookie) +{ + UNREFERENCED_PARAMETER(p0); + UNREFERENCED_PARAMETER(p1); + UNREFERENCED_PARAMETER(p2); + UNREFERENCED_PARAMETER(p3); + UNREFERENCED_PARAMETER(p4); + UNREFERENCED_PARAMETER(idx); + uint64_t* ccookie = (uint64_t*)cookie; + cookie_pointer_value = ccookie; + return 1; +} + +bool +external_dispatcher_validater(unsigned int idx, const struct ubpf_vm* cookie) +{ + UNREFERENCED_PARAMETER(idx); + UNREFERENCED_PARAMETER(cookie); + return true; +} + +int main(int argc, char **argv) +{ + std::vector args(argv, argv + argc); + std::string program_string{}; + ubpf_jit_fn jit_fn; + uint64_t memory{0x123456789}; + + // The program modifies (eBPF) r0 (see test description) and then invokes + // a helper function that will be invoked through the external + // dispatcher. + std::getline(std::cin, program_string); + + std::unique_ptr vm(ubpf_create(), ubpf_destroy); + std::string error{}; + if (!ubpf_setup_custom_test( + vm, + program_string, + [](ubpf_vm_up& vm, std::string &error) { + if (ubpf_register_external_dispatcher(vm.get(), external_dispatcher, external_dispatcher_validater) < 0) { + error = "Failed to register external dispatcher."; + return false; + } + return true; + }, + jit_fn, + error)) { + std::cerr << "Problem setting up custom test: " << error << std::endl; + return 1; + } + + [[maybe_unused]] auto result = jit_fn(&memory, sizeof(uint64_t)); + + // Ultimately, the cookie pointer that we got as context to the external helper dispatcher + // should match what we passed as argument 0 to jit_fn (even though (eBPF) r0 has been + // modified by the eBPF program). + return !(cookie_pointer_value == &memory); +} diff --git a/custom_tests/srcs/ubpf_test_external_dispatcher_simple_context.cc b/custom_tests/srcs/ubpf_test_external_dispatcher_simple_context.cc new file mode 100644 index 00000000..e3fa6e65 --- /dev/null +++ b/custom_tests/srcs/ubpf_test_external_dispatcher_simple_context.cc @@ -0,0 +1,73 @@ +// Copyright (c) Will Hawkins +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include + +extern "C" +{ +#include "ubpf.h" +} + +#include "ubpf_custom_test_support.h" + +uint64_t +external_dispatcher(uint64_t p0, uint64_t p1, uint64_t p2, uint64_t p3, uint64_t p4, unsigned int idx, void* cookie) +{ + UNREFERENCED_PARAMETER(p0); + UNREFERENCED_PARAMETER(p1); + UNREFERENCED_PARAMETER(p2); + UNREFERENCED_PARAMETER(p3); + UNREFERENCED_PARAMETER(p4); + UNREFERENCED_PARAMETER(idx); + uint64_t* ccookie = (uint64_t*)cookie; + return *ccookie; +} + +bool +external_dispatcher_validater(unsigned int idx, const struct ubpf_vm* cookie) +{ + UNREFERENCED_PARAMETER(idx); + UNREFERENCED_PARAMETER(cookie); + return true; +} + +int main(int argc, char **argv) +{ + std::vector args(argv, argv + argc); + std::string program_string{}; + ubpf_jit_fn jit_fn; + uint64_t memory{0x123456789}; + + // The test program invokes an external function (which is invoked via the registered + // external helper dispatcher). The result of that external function is given as the + // result of the eBPF's program execution. Therefore, ... + std::getline(std::cin, program_string); + + std::unique_ptr vm(ubpf_create(), ubpf_destroy); + std::string error{}; + if (!ubpf_setup_custom_test( + vm, + program_string, + [](ubpf_vm_up& vm, std::string &error) { + if (ubpf_register_external_dispatcher(vm.get(), external_dispatcher, external_dispatcher_validater) < 0) { + error = "Failed to register external dispatcher."; + return false; + } + return true; + }, + jit_fn, + error)) { + std::cerr << "Problem setting up custom test: " << error << std::endl; + return 1; + } + + [[maybe_unused]] auto result = jit_fn(&memory, sizeof(uint64_t)); + + // ... because of the semantics of external_dispatcher, the result of the eBPF + // program execution should point to the same place to which &memory points. + return !(result == memory); +} diff --git a/custom_tests/srcs/ubpf_test_jit_buffer_too_small.cc b/custom_tests/srcs/ubpf_test_jit_buffer_too_small.cc new file mode 100644 index 00000000..4c2f8a4a --- /dev/null +++ b/custom_tests/srcs/ubpf_test_jit_buffer_too_small.cc @@ -0,0 +1,53 @@ +// Copyright (c) Will Hawkins +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +extern "C" +{ +#include "ebpf.h" +#include "ubpf.h" +} + +#include "ubpf_custom_test_support.h" + +/** + * @brief This program reads BPF instructions from stdin and memory contents from + * the first agument. It then executes the BPF program and prints the + * value of %r0 at the end of execution. + */ +int main() +{ + std::string expected_error{"Failed to compile: Target buffer too small"}; + std::string program_string{"95 00 00 00 00 00 00 00"}; + ubpf_jit_fn jit_fn; + + std::vector program = bytes_to_ebpf_inst(base16_decode(program_string)); + + ubpf_vm_up vm(ubpf_create(), ubpf_destroy); + std::string error{}; + char *error_s{nullptr}; + + if (!ubpf_setup_custom_test( + vm, + program_string, + custom_test_fixup_cb{[](ubpf_vm_up& vm, std::string& error) { + if (ubpf_set_jit_code_size(vm.get(), 1) < 0) { + error = "Could not set the jit code size."; + return false; + } + return true; + }}, + jit_fn, + error)) { + free(error_s); + + // Only if the error is that the buffer was too small does this test pass. + if (jit_fn == nullptr && expected_error == error) + return 0; + } + + free(error_s); + return 1; +} diff --git a/custom_tests/srcs/ubpf_test_jit_unexpected_instruction.cc b/custom_tests/srcs/ubpf_test_jit_unexpected_instruction.cc new file mode 100644 index 00000000..5b819cc3 --- /dev/null +++ b/custom_tests/srcs/ubpf_test_jit_unexpected_instruction.cc @@ -0,0 +1,46 @@ +// Copyright (c) Will Hawkins +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +extern "C" +{ +#include "ebpf.h" +#include "ubpf.h" +} + +#include "ubpf_custom_test_support.h" + +int main() +{ + std::string expected_error{"Failed to load program: unknown opcode 0x8f at PC 0" }; + ubpf_jit_fn jit_fn; + std::string program_string; + + // The program's first instruction contains an invalid opcode. Attempting to load this + // program should elicit an error alerting the user to an unknown opcode (see above). + std::getline(std::cin, program_string); + + std::vector program = bytes_to_ebpf_inst(base16_decode(program_string)); + + ubpf_vm_up vm(ubpf_create(), ubpf_destroy); + std::string error{}; + + if (!ubpf_setup_custom_test( + vm, + program_string, + custom_test_fixup_cb{[](ubpf_vm_up&, std::string& ) { + return true; + }}, + jit_fn, + error)) { + + // Only if the error matches exactly what we expect should this test pass. + if (jit_fn == nullptr && expected_error == error) + return 0; + } + + return 1; +} diff --git a/custom_tests/srcs/ubpf_test_update_dispatcher.cc b/custom_tests/srcs/ubpf_test_update_dispatcher.cc new file mode 100644 index 00000000..5124122f --- /dev/null +++ b/custom_tests/srcs/ubpf_test_update_dispatcher.cc @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include + +extern "C" +{ +#include "ubpf.h" +} + +#include "ubpf_custom_test_support.h" + +const uint64_t dispatcher_test_dispatcher_failure{40}; +const uint64_t dispatcher_test_dispatcher_success{42}; +const uint64_t updated_dispatcher_test_dispatcher_failure{41}; +const uint64_t updated_dispatcher_test_dispatcher_success{43}; + +uint64_t +dispatcher_test_dispatcher( + uint64_t p0, uint64_t p1, uint64_t p2, uint64_t p3, uint64_t p4, unsigned int idx, void* cookie) +{ + UNREFERENCED_PARAMETER(p0); + UNREFERENCED_PARAMETER(p1); + UNREFERENCED_PARAMETER(p2); + UNREFERENCED_PARAMETER(p3); + UNREFERENCED_PARAMETER(p4); + UNREFERENCED_PARAMETER(cookie); + if (idx != 1) { + return dispatcher_test_dispatcher_failure; + } + return dispatcher_test_dispatcher_success; +} + +uint64_t +updated_dispatcher_test_dispatcher( + uint64_t p0, uint64_t p1, uint64_t p2, uint64_t p3, uint64_t p4, unsigned int idx, void* cookie) +{ + UNREFERENCED_PARAMETER(p0); + UNREFERENCED_PARAMETER(p1); + UNREFERENCED_PARAMETER(p2); + UNREFERENCED_PARAMETER(p3); + UNREFERENCED_PARAMETER(p4); + UNREFERENCED_PARAMETER(cookie); + if (idx != 1) { + return updated_dispatcher_test_dispatcher_failure; + } + return updated_dispatcher_test_dispatcher_success; +} + +bool +test_helpers_validater(unsigned int idx, const struct ubpf_vm* vm) +{ + UNREFERENCED_PARAMETER(idx); + UNREFERENCED_PARAMETER(vm); + return true; +} + +int +main(int argc, char** argv) +{ + std::vector args(argv, argv + argc); + std::string program_string; + std::string memory_string; + + std::getline(std::cin, program_string); + + ubpf_jit_fn jit_fn; + uint64_t memory{0x123456789}; + std::unique_ptr vm(ubpf_create(), ubpf_destroy); + std::string error{}; + if (!ubpf_setup_custom_test( + vm, + program_string, + [](ubpf_vm_up& vm, std::string& error) { + if (ubpf_register_external_dispatcher(vm.get(), dispatcher_test_dispatcher, test_helpers_validater)) { + error = "Failed to register the external dispatcher function"; + return false; + } + return true; + }, + jit_fn, + error)) { + std::cerr << "Problem setting up custom test: " << error << std::endl; + return 1; + } + + auto first_result = jit_fn(&memory, sizeof(uint64_t)); + + if (ubpf_register_external_dispatcher(vm.get(), updated_dispatcher_test_dispatcher, test_helpers_validater)) { + std::cout << "Failed to register updated dispatcher function\n"; + return 1; + } + + auto second_result = jit_fn(&memory, sizeof(uint64_t)); + + auto current_success{ + (first_result == dispatcher_test_dispatcher_success && + second_result == updated_dispatcher_test_dispatcher_success)}; + return current_success ? 0 : 1; +} diff --git a/custom_tests/srcs/ubpf_test_update_helpers.cc b/custom_tests/srcs/ubpf_test_update_helpers.cc new file mode 100644 index 00000000..8566e42d --- /dev/null +++ b/custom_tests/srcs/ubpf_test_update_helpers.cc @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation +// SPDX-License-Identifier: Apache-2.0 + +// This program reads BPF instructions from stdin and memory contents from +// the first agument. It then executes the BPF program and prints the +// value of %r0 at the end of execution. +// The program is intended to be used with the bpf conformance test suite. + +#include "ubpf_int.h" +#include +#include +#include +#include +#include + +extern "C" +{ +#include "ubpf.h" +} + +#include "ubpf_custom_test_support.h" +#include "test_helpers.h" + +struct HelperTestCase { + const char *testcase_name; + external_function_t helper_function1; + external_function_t helper_function2; + int index; + uint64_t result1; + uint64_t result2; +}; + +/** + * @brief This program reads BPF instructions from stdin and memory contents from + * the first agument. It then executes the BPF program and prints the + * value of %r0 at the end of execution. + */ +int main(int argc, char **argv) +{ + std::vector args(argv, argv + argc); + std::string program_string{}; + bool success{true}; + + const char memfrob_testcase_name[] = "memfrob"; + const char gather_bytes_testcase_name[] = "gather bytes"; + const char sqrti_testcase_name[] = "sqrti"; + const char no_op_testcase_name[] = "no op"; + const char strcmp_testcase_name[] = "strcmp"; + const char unwind_testcase_name[] = "unwind"; + + std::vector test_cases{ + { + .testcase_name = memfrob_testcase_name, + .helper_function1 = dispatcher_test_memfrob, + .helper_function2 = updated_dispatcher_test_memfrob, + .index = 1, + .result1 = 42, + .result2 = 43 + }, + { + .testcase_name = gather_bytes_testcase_name, + .helper_function1 = dispatcher_gather_bytes, + .helper_function2 = updated_dispatcher_gather_bytes, + .index = 1, + .result1 = 44, + .result2 = 45 + }, + { + .testcase_name = no_op_testcase_name, + .helper_function1 = dispatcher_no_op, + .helper_function2 = updated_dispatcher_no_op, + .index = 1, + .result1 = 46, + .result2 = 47 + }, + { + .testcase_name = sqrti_testcase_name, + .helper_function1 = dispatcher_sqrti, + .helper_function2 = updated_dispatcher_sqrti, + .index = 1, + .result1 = 48, + .result2 = 49 + }, + { + .testcase_name = strcmp_testcase_name, + .helper_function1 = dispatcher_strcmp_ext, + .helper_function2 = updated_dispatcher_strcmp_ext, + .index = 1, + .result1 = 50, + .result2 = 51 + }, + { + .testcase_name = unwind_testcase_name, + .helper_function1 = dispatcher_unwind, + .helper_function2 = updated_dispatcher_unwind, + .index = 1, + .result1 = 52, + .result2 = 53 + } + }; + + std::getline(std::cin, program_string); + + for (auto testcase : test_cases) { + ubpf_jit_fn jit_fn; + uint64_t memory{0x123456789}; + std::unique_ptr vm(ubpf_create(), ubpf_destroy); + std::string error{}; + if (!ubpf_setup_custom_test( + vm, + program_string, + [&testcase](ubpf_vm_up& vm, std::string& error) { + if (ubpf_register(vm.get(), testcase.index, "unnamed", testcase.helper_function1) != 0) { + error = "Failed to register helper function"; + return false; + } + return true; + }, + jit_fn, + error)) { + std::cerr << "Problem setting up custom test: " << error << std::endl; + return 1; + } + + [[maybe_unused]] auto first_result = jit_fn(&memory, sizeof(uint64_t)); + + if (ubpf_register(vm.get(), testcase.index, "unnamed", testcase.helper_function2) != 0) { + std::cout << "Failed to register helper function\n"; + return 1; + } + + [[maybe_unused]] auto second_result = jit_fn(&memory, sizeof(uint64_t)); + + auto current_success{(first_result == testcase.result1 && second_result == testcase.result2)}; + if (!current_success) { + std::cout << "There was a failure with test " << testcase.testcase_name << ": " << + testcase.result1 << " != " << first_result << " or " << + testcase.result2 << " != " << second_result << "!\n"; + } + success &= current_success; + } + return success ? 0 : 1; +} diff --git a/external/bpf_conformance b/external/bpf_conformance index 41ec3192..6356e3ee 160000 --- a/external/bpf_conformance +++ b/external/bpf_conformance @@ -1 +1 @@ -Subproject commit 41ec319242a4e598850650158dbaa91397df45a4 +Subproject commit 6356e3ee03b2c1a70fd6839a85713974bab999b3 diff --git a/ubpf_plugin/CMakeLists.txt b/ubpf_plugin/CMakeLists.txt index 5a9de1ef..6c64f411 100644 --- a/ubpf_plugin/CMakeLists.txt +++ b/ubpf_plugin/CMakeLists.txt @@ -30,6 +30,14 @@ target_link_libraries( ubpf_settings ) + +if(TARGET "ubpf_compat") + target_link_libraries("ubpf_plugin" + $ + ) +endif() + + file(GLOB external_files ${CMAKE_SOURCE_DIR}/external/bpf_conformance/tests/*.data) file(GLOB local_files ${CMAKE_SOURCE_DIR}/tests/*.data) diff --git a/ubpf_plugin/ubpf_plugin.cc b/ubpf_plugin/ubpf_plugin.cc index 3accc64e..18082db9 100644 --- a/ubpf_plugin/ubpf_plugin.cc +++ b/ubpf_plugin/ubpf_plugin.cc @@ -6,13 +6,13 @@ // value of %r0 at the end of execution. // The program is intended to be used with the bpf conformance test suite. -#include -#include #include #include #include #include #include +#include +#include "ubpf_int.h" extern "C" { @@ -27,8 +27,8 @@ uint64_t test_helpers_dispatcher(uint64_t p0, uint64_t p1,uint64_t p2,uint64_t p return helper_functions[idx](p0, p1, p2, p3, p4); } -bool test_helpers_validater(unsigned int idx, void *cookie) { - UNREFERENCED_PARAMETER(cookie); +bool test_helpers_validater(unsigned int idx, const struct ubpf_vm *vm) { + UNREFERENCED_PARAMETER(vm); return helper_functions.contains(idx); } @@ -141,18 +141,7 @@ int main(int argc, char **argv) return 1; } -/* - for (auto &[key, value] : helper_functions) - { - if (ubpf_register(vm.get(), key, "unnamed", value) != 0) - { - std::cerr << "Failed to register helper function" << std::endl; - return 1; - } - } -*/ - - ubpf_register_external_dispatcher(vm.get(), test_helpers_dispatcher, test_helpers_validater, NULL); + ubpf_register_external_dispatcher(vm.get(), test_helpers_dispatcher, test_helpers_validater); if (ubpf_set_unwind_function_index(vm.get(), 5) != 0) { @@ -168,27 +157,84 @@ int main(int argc, char **argv) return 1; } - uint64_t actual_result; + uint64_t external_dispatcher_result; if (jit) { + // Compile the program ... ubpf_jit_fn fn = ubpf_compile(vm.get(), &error); if (fn == nullptr) { std::cerr << "Failed to compile program: " << error << std::endl; - std::cout << "Failed to load code: " << error << std::endl; free(error); return 1; } - actual_result = fn(memory_ptr, memory.size()); + + // ... keep the original program memory safe from being trashed by test program so that + // it can be run again ... + std::vector usable_program_memory{memory}; + uint8_t *usable_program_memory_pointer{nullptr}; + if (usable_program_memory.size() != 0) { + usable_program_memory_pointer = usable_program_memory.data(); + } + + // ... execute the original copy of the JIT'd code ... + external_dispatcher_result = fn(usable_program_memory_pointer, usable_program_memory.size()); + + + // ... execute original code but with indexed dispatcher to helper functions ... + ubpf_register_external_dispatcher(vm.get(), nullptr, test_helpers_validater); + for (auto& [key, value] : helper_functions) { + if (ubpf_register(vm.get(), key, "unnamed", value) != 0) { + std::cerr << "Failed to register helper function" << std::endl; + return 1; + } + } + + uint64_t index_helper_result; + usable_program_memory = memory; + usable_program_memory_pointer = nullptr; + if (usable_program_memory.size() != 0) { + usable_program_memory_pointer = usable_program_memory.data(); + } + index_helper_result = fn(usable_program_memory_pointer, usable_program_memory.size()); + + // ... copy the JIT'd program ... + auto fn_copy_size = vm->jitted_size * sizeof(char); + void *fn_copy = mmap(0, fn_copy_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + fn = ubpf_copy_jit(vm.get(), fn_copy, fn_copy_size, &error); + if (fn == nullptr) { + std::cerr << "Failed to copy JIT'd program: " << error << std::endl; + free(error); + return 1; + } + mprotect(fn_copy, fn_copy_size, PROT_READ | PROT_EXEC); + + // ... execute the copy of the JIT'd code ... + uint64_t copy_result; + usable_program_memory = memory; + usable_program_memory_pointer = nullptr; + if (usable_program_memory.size() != 0) { + usable_program_memory_pointer = usable_program_memory.data(); + } + copy_result = fn(usable_program_memory_pointer, usable_program_memory.size()); + + // ... and make sure the results are the same. + if (external_dispatcher_result != index_helper_result || index_helper_result != copy_result) { + std::cerr << "Execution of the JIT'd code (with external and indexed helpers) and a copy of " + "the JIT'd code gave different results: 0x" << std::hex << external_dispatcher_result + << " vs 0x" << std::hex << index_helper_result + << " vs 0x" << std::hex << copy_result << "." << std::endl; + return 1; + } } else { - if (ubpf_exec(vm.get(), memory_ptr, memory.size(), &actual_result) != 0) + if (ubpf_exec(vm.get(), memory_ptr, memory.size(), &external_dispatcher_result) != 0) { std::cerr << "Failed to execute program" << std::endl; return 1; } } - std::cout << std::hex << actual_result << std::endl; + std::cout << std::hex << external_dispatcher_result << std::endl; return 0; } diff --git a/vm/CMakeLists.txt b/vm/CMakeLists.txt index 0c330fa8..f173aa5b 100644 --- a/vm/CMakeLists.txt +++ b/vm/CMakeLists.txt @@ -41,6 +41,8 @@ add_library("ubpf" ubpf_int.h ubpf_jit_arm64.c ubpf_jit.c + ubpf_jit_support.c + ubpf_jit_support.h ubpf_jit_x86_64.c ubpf_jit_x86_64.h ubpf_loader.c diff --git a/vm/compat/windows/sys/mman.h b/vm/compat/windows/sys/mman.h index cc85afba..5be898c2 100644 --- a/vm/compat/windows/sys/mman.h +++ b/vm/compat/windows/sys/mman.h @@ -12,6 +12,7 @@ #pragma once #include +#include #define PROT_READ 0x1 #define PROT_WRITE 0x2 @@ -20,7 +21,10 @@ #define MAP_FAILED NULL #define PROT_EXEC 0x4 -typedef int64_t off_t; +#ifdef __cplusplus +extern "C" +{ +#endif void* mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset); @@ -28,3 +32,7 @@ int munmap(void* addr, size_t length); int mprotect(void* addr, size_t len, int prot); + +#ifdef __cplusplus +} +#endif diff --git a/vm/compat/windows/unistd.h b/vm/compat/windows/unistd.h index 09de2a69..6749127e 100644 --- a/vm/compat/windows/unistd.h +++ b/vm/compat/windows/unistd.h @@ -15,7 +15,17 @@ #define STDIN_FILENO 0 +#ifdef __cplusplus +extern "C" +{ +#endif + int rand_r(unsigned int* seedp); int vasprintf(char** strp, const char* fmt, va_list ap); + + +#ifdef __cplusplus +} +#endif diff --git a/vm/inc/ubpf.h b/vm/inc/ubpf.h index 81c92a1e..e6417597 100644 --- a/vm/inc/ubpf.h +++ b/vm/inc/ubpf.h @@ -140,7 +140,7 @@ extern "C" /** * @brief The type of an external helper validation function. */ - typedef bool (*external_function_validate_t)(unsigned int index, void* cookie); + typedef bool (*external_function_validate_t)(unsigned int index, const struct ubpf_vm* vm); /** * @brief Register a function that dispatches to external helpers @@ -155,8 +155,6 @@ extern "C" * helper. * @param[in] validater The callback that will validate that a given index * is valid for an external helper. - * @param[in] cookie A pointer to some user-defined cookie that will be - * passed to the callbacks. * @retval 0 Success. * @retval -1 Failure. */ @@ -164,8 +162,7 @@ extern "C" ubpf_register_external_dispatcher( struct ubpf_vm* vm, external_function_dispatcher_t dispatcher, - external_function_validate_t validater, - void* cookie); + external_function_validate_t validater); /** * @brief Load code into a VM. @@ -274,8 +271,9 @@ extern "C" /** * @brief Compile a BPF program in the VM to native code. * - * A program must be loaded into the VM and all external functions must be - * registered before calling this function. + * A program must be loaded into the VM and all external functions (or + * the external helper dispatcher) must be registered before calling this + * function. * * @param[in] vm The VM to compile the program in. * @param[out] errmsg The error message, if any. This should be freed by the caller. @@ -284,18 +282,22 @@ extern "C" ubpf_jit_fn ubpf_compile(struct ubpf_vm* vm, char** errmsg); - /* - * Translate the eBPF byte code to x64 machine code, store in buffer, and - * write the resulting count of bytes to size. + /** + * @brief Copy the JIT'd program code to the given buffer. * - * This must be called after registering all functions. + * A program must have been loaded into the VM and already JIT'd before + * calling this function. * - * Returns 0 on success, -1 on error. In case of error a pointer to the error - * message will be stored in 'errmsg' and should be freed by the caller. + * @param[in] vm The VM of the already JIT'd program. + * @param[out] errmsg The error message, if any. This should be freed by the caller. + * @return ubpf_jit_fn A pointer to the compiled program (the same as buffer), or + * NULL on failure. */ + ubpf_jit_fn + ubpf_copy_jit(struct ubpf_vm* vm, void *buffer, size_t size, char** errmsg); /** - * @brief Translate the eBPF byte code to x64 machine code. + * @brief Translate the eBPF byte code to machine code. * * A program must be loaded into the VM and all external functions must be * registered before calling this function. @@ -397,6 +399,21 @@ extern "C" int ubpf_register_data_bounds_check(struct ubpf_vm* vm, void* user_context, ubpf_bounds_check bounds_check); + /** + * @brief Set a size for the buffer allocated to machine code generated during JIT compilation. + * The JIT compiler allocates a buffer to store the code while it is being generated. The default + * may be too big for some embedded platforms. Use this to customize the size of that buffer. + * Note: The buffer being sized here is *not* the final location of the machine code returned by + * ubpf_compile -- that buffer is perfectly sized to match the size of the generated machine code. + * + * @param[in] vm The VM to set the buffer size for. + * @param[in] code_size The size of the buffer to use. + * @retval 0 Success. + * @retval -1 Failure. + */ + int + ubpf_set_jit_code_size(struct ubpf_vm* vm, size_t code_size); + #ifdef __cplusplus } #endif diff --git a/vm/ubpf_int.h b/vm/ubpf_int.h index 861e6a80..9c5311ba 100644 --- a/vm/ubpf_int.h +++ b/vm/ubpf_int.h @@ -24,15 +24,33 @@ #include #include "ebpf.h" +#define UNUSED_PARAMETER(x) ((void)x) + struct ebpf_inst; typedef uint64_t (*ext_func)(uint64_t arg0, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4); +typedef enum { + UBPF_JIT_COMPILE_SUCCESS, + UBPF_JIT_COMPILE_FAILURE, +} upbf_jit_result_t; + +struct ubpf_jit_result { + uint32_t external_dispatcher_offset; + uint32_t external_helper_offset; + upbf_jit_result_t compile_result; + char *errmsg; +}; + +#define MAX_EXT_FUNCS 64 + struct ubpf_vm { struct ebpf_inst* insts; uint16_t num_insts; ubpf_jit_fn jitted; size_t jitted_size; + size_t jitter_buffer_size; + struct ubpf_jit_result jitted_result; ext_func* ext_funcs; bool* int_funcs; @@ -40,11 +58,12 @@ struct ubpf_vm external_function_dispatcher_t dispatcher; external_function_validate_t dispatcher_validate; - void* dispatcher_cookie; bool bounds_check_enabled; int (*error_printf)(FILE* stream, const char* format, ...); - int (*translate)(struct ubpf_vm* vm, uint8_t* buffer, size_t* size, char** errmsg); + struct ubpf_jit_result (*jit_translate)(struct ubpf_vm* vm, uint8_t* buffer, size_t* size); + bool (*jit_update_dispatcher)(struct ubpf_vm* vm, external_function_dispatcher_t new_dispatcher, uint8_t* buffer, size_t size, uint32_t offset); + bool (*jit_update_helper)(struct ubpf_vm* vm, ext_func new_helper, unsigned int idx, uint8_t* buffer, size_t size, uint32_t offset); int unwind_stack_extension_index; uint64_t pointer_secret; ubpf_data_relocation data_relocation_function; @@ -63,12 +82,24 @@ struct ubpf_stack_frame }; /* The various JIT targets. */ -int -ubpf_translate_arm64(struct ubpf_vm* vm, uint8_t* buffer, size_t* size, char** errmsg); -int -ubpf_translate_x86_64(struct ubpf_vm* vm, uint8_t* buffer, size_t* size, char** errmsg); -int -ubpf_translate_null(struct ubpf_vm* vm, uint8_t* buffer, size_t* size, char** errmsg); + +// arm64 +struct ubpf_jit_result +ubpf_translate_arm64(struct ubpf_vm* vm, uint8_t* buffer, size_t* size); +bool ubpf_jit_update_dispatcher_arm64(struct ubpf_vm* vm, external_function_dispatcher_t new_dispatcher, uint8_t* buffer, size_t size, uint32_t offset); +bool ubpf_jit_update_helper_arm64(struct ubpf_vm* vm, ext_func new_helper, unsigned int idx, uint8_t* buffer, size_t size, uint32_t offset); + +//x86_64 +struct ubpf_jit_result +ubpf_translate_x86_64(struct ubpf_vm* vm, uint8_t* buffer, size_t* size); +bool ubpf_jit_update_dispatcher_x86_64(struct ubpf_vm* vm, external_function_dispatcher_t new_dispatcher, uint8_t* buffer, size_t size, uint32_t offset); +bool ubpf_jit_update_helper_x86_64(struct ubpf_vm* vm, ext_func new_helper, unsigned int idx, uint8_t* buffer, size_t size, uint32_t offset); + +//uhm, hello? +struct ubpf_jit_result +ubpf_translate_null(struct ubpf_vm* vm, uint8_t* buffer, size_t* size); +bool ubpf_jit_update_dispatcher_null(struct ubpf_vm* vm, external_function_dispatcher_t new_dispatcher, uint8_t* buffer, size_t size, uint32_t offset); +bool ubpf_jit_update_helper_null(struct ubpf_vm* vm, ext_func new_helper, unsigned int idx, uint8_t* buffer, size_t size, uint32_t offset); char* ubpf_error(const char* fmt, ...); diff --git a/vm/ubpf_jit.c b/vm/ubpf_jit.c index 0e7b10fe..cc534ade 100644 --- a/vm/ubpf_jit.c +++ b/vm/ubpf_jit.c @@ -23,31 +23,65 @@ #include #include #include +#include #include -#include #include #include -#include #include "ubpf_int.h" -#include "ubpf_jit_x86_64.h" -#define UNUSED(x) ((void)x) int ubpf_translate(struct ubpf_vm* vm, uint8_t* buffer, size_t* size, char** errmsg) { - return vm->translate(vm, buffer, size, errmsg); + struct ubpf_jit_result jit_result = vm->jit_translate(vm, buffer, size); + vm->jitted_result = jit_result; + if (jit_result.errmsg) { + *errmsg = jit_result.errmsg; + } + return jit_result.compile_result == UBPF_JIT_COMPILE_SUCCESS ? 0 : -1; } -int -ubpf_translate_null(struct ubpf_vm* vm, uint8_t* buffer, size_t* size, char** errmsg) +struct ubpf_jit_result +ubpf_translate_null(struct ubpf_vm* vm, uint8_t* buffer, size_t* size) { + struct ubpf_jit_result compile_result; + compile_result.compile_result = UBPF_JIT_COMPILE_FAILURE; + compile_result.external_dispatcher_offset = 0; + /* NULL JIT target - just returns an error. */ - UNUSED(vm); - UNUSED(buffer); - UNUSED(size); - *errmsg = ubpf_error("Code can not be JITed on this target."); - return -1; + UNUSED_PARAMETER(vm); + UNUSED_PARAMETER(buffer); + UNUSED_PARAMETER(size); + compile_result.errmsg = ubpf_error("Code can not be JITed on this target."); + return compile_result; +} + +bool ubpf_jit_update_dispatcher_null(struct ubpf_vm* vm, external_function_dispatcher_t new_dispatcher, uint8_t* buffer, size_t size, uint32_t offset) +{ + UNUSED_PARAMETER(vm); + UNUSED_PARAMETER(new_dispatcher); + UNUSED_PARAMETER(buffer); + UNUSED_PARAMETER(size); + UNUSED_PARAMETER(offset); + return false; +} + +bool ubpf_jit_update_helper_null(struct ubpf_vm* vm, ext_func new_helper, unsigned int idx, uint8_t* buffer, size_t size, uint32_t offset) +{ + UNUSED_PARAMETER(vm); + UNUSED_PARAMETER(new_helper); + UNUSED_PARAMETER(idx); + UNUSED_PARAMETER(buffer); + UNUSED_PARAMETER(size); + UNUSED_PARAMETER(offset); + return false; +} + +int +ubpf_set_jit_code_size(struct ubpf_vm* vm, size_t code_size) +{ + vm->jitter_buffer_size = code_size; + return 0; } ubpf_jit_fn @@ -68,7 +102,7 @@ ubpf_compile(struct ubpf_vm* vm, char** errmsg) return NULL; } - jitted_size = 65536; + jitted_size = vm->jitter_buffer_size; buffer = calloc(jitted_size, 1); if (buffer == NULL) { *errmsg = ubpf_error("internal uBPF error: calloc failed: %s\n", strerror(errno)); @@ -102,3 +136,26 @@ ubpf_compile(struct ubpf_vm* vm, char** errmsg) } return vm->jitted; } + +ubpf_jit_fn +ubpf_copy_jit(struct ubpf_vm *vm, void *buffer, size_t size, char **errmsg) +{ + // If compilation was not successfull or it has not even been attempted, + // we cannot copy. + if (vm->jitted_result.compile_result != UBPF_JIT_COMPILE_SUCCESS || !vm->jitted) { + *errmsg = ubpf_error("Cannot copy JIT'd code before compilation"); + return (ubpf_jit_fn)NULL; + } + + // If the given buffer is not big enough to contain the JIT'd code, + // we cannot copy. + if (vm->jitted_size > size) { + *errmsg = ubpf_error("Buffer not big enough for copy"); + return (ubpf_jit_fn)NULL; + } + + // All good. Do the copy! + memcpy(buffer, vm->jitted, vm->jitted_size); + *errmsg = NULL; + return (ubpf_jit_fn)buffer; +} diff --git a/vm/ubpf_jit_arm64.c b/vm/ubpf_jit_arm64.c index e3749364..c6710406 100644 --- a/vm/ubpf_jit_arm64.c +++ b/vm/ubpf_jit_arm64.c @@ -28,47 +28,17 @@ #include #include #include -#include #include -#include #include #include "ubpf_int.h" +#include "ubpf_jit_support.h" #if !defined(_countof) #define _countof(array) (sizeof(array) / sizeof(array[0])) #endif -/* Special values for target_pc in struct jump */ -#define TARGET_PC_EXIT ~UINT32_C(0) -#define TARGET_PC_ENTER (~UINT32_C(0) & 0x0101) -#define TARGET_PC_EXTERNAL_DISPATCHER (~UINT32_C(0) & 0x1010) - // This is guaranteed to be an illegal A64 instruction. #define BAD_OPCODE ~UINT32_C(0) - -struct patchable_relative -{ - uint32_t offset_loc; - uint32_t target_pc; -}; - -struct jit_state -{ - uint8_t* buf; - uint32_t offset; - uint32_t size; - uint32_t* pc_locs; - uint32_t exit_loc; - uint32_t entry_loc; - uint32_t dispatcher_loc; - uint32_t unwind_loc; - struct patchable_relative* jumps; - struct patchable_relative* loads; - int num_jumps; - int num_loads; - uint32_t stack_size; -}; - // All A64 registers (note SP & RZ get encoded the same way). enum Registers { @@ -117,6 +87,8 @@ static enum Registers temp_register = R24; static enum Registers temp_div_register = R25; // Temp register for load/store offsets static enum Registers offset_register = R26; +// Special register for external dispatcher context. +static enum Registers VOLATILE_CTXT = R26; // Number of eBPF registers #define REGISTER_MAP_SIZE 11 @@ -169,8 +141,11 @@ static uint32_t inline align_to(uint32_t amount, uint64_t boundary) static void emit_bytes(struct jit_state* state, void* data, uint32_t len) { - assert(len <= state->size); - assert(state->offset <= state->size - len); + if (!(len <= state->size && state->offset <= state->size - len)) { + state->jit_status = NotEnoughSpace; + return; + } + if ((state->offset + len) > state->size) { state->offset = state->size; return; @@ -271,12 +246,21 @@ emit_loadstore_register( static void emit_loadstore_literal( - struct jit_state* state, enum LoadStoreOpcode op, enum Registers rt) + struct jit_state* state, enum LoadStoreOpcode op, enum Registers rt, uint32_t target) { + note_load(state, target); const uint32_t reg_op_base = 0x08000000U; emit_instruction(state, op | reg_op_base | rt); } +static void +emit_adr(struct jit_state *state, uint32_t offset, enum Registers rd) +{ + note_lea(state, offset); + uint32_t instr = 0x10000000 | rd; + emit_instruction(state, instr); +} + enum LoadStorePairOpcode { // op V L @@ -351,35 +335,12 @@ enum UnconditionalBranchImmediateOpcode UBR_BL = 0x94000000U, // 1001_0100_0000_0000_0000_0000_0000_0000 }; -static void -note_jump(struct jit_state* state, uint32_t target_pc) -{ - if (state->num_jumps == UBPF_MAX_INSTS) { - return; - } - struct patchable_relative* jump = &state->jumps[state->num_jumps++]; - jump->offset_loc = state->offset; - jump->target_pc = target_pc; -} - -static void -note_load(struct jit_state* state, uint32_t target_pc) -{ - if (state->num_loads == UBPF_MAX_INSTS) { - return; - } - struct patchable_relative* load = &state->loads[state->num_loads++]; - load->offset_loc = state->offset; - load->target_pc = target_pc; -} - - /* [ArmARM-A H.a]: C4.1.65: Unconditional branch (immediate). */ static void emit_unconditionalbranch_immediate( struct jit_state* state, enum UnconditionalBranchImmediateOpcode op, int32_t target_pc) { - note_jump(state, target_pc); + emit_patchable_relative(state->offset, target_pc, 0, state->jumps, state->num_jumps++); emit_instruction(state, op); } @@ -411,11 +372,13 @@ enum ConditionalBranchImmediateOpcode }; /* [ArmARM-A H.a]: C4.1.65: Conditional branch (immediate). */ -static void +static uint32_t emit_conditionalbranch_immediate(struct jit_state* state, enum Condition cond, uint32_t target_pc) { - note_jump(state, target_pc); + uint32_t source_offset = state->offset; + emit_patchable_relative(state->offset, target_pc, 0, state->jumps, state->num_jumps++); emit_instruction(state, BR_Bcond | (0 << 5) | cond); + return source_offset; } enum CompareBranchOpcode @@ -425,15 +388,6 @@ enum CompareBranchOpcode CBR_CBNZ = 0x35000000U, // 0011_0101_0000_0000_0000_0000_0000_0000 }; -#if 0 -static void -emit_comparebranch_immediate(struct jit_state *state, bool sixty_four, enum CompareBranchOpcode op, enum Registers rt, uint32_t target_pc) -{ - note_jump(state, target_pc); - emit_instruction(state, (sixty_four << 31) | op | rt); -} -#endif - enum DP1Opcode { // S op2--|op-----| @@ -545,38 +499,6 @@ emit_movewide_immediate(struct jit_state* state, bool sixty_four, enum Registers } } -static void -update_branch_immediate(struct jit_state* state, uint32_t offset, int32_t imm) -{ - assert((imm & 3) == 0); - uint32_t instr; - imm >>= 2; - memcpy(&instr, state->buf + offset, sizeof(uint32_t)); - if ((instr & 0xfe000000U) == 0x54000000U /* Conditional branch immediate. */ - || (instr & 0x7e000000U) == 0x34000000U) { /* Compare and branch immediate. */ - assert((imm >> 19) == INT64_C(-1) || (imm >> 19) == 0); - instr |= (imm & 0x7ffff) << 5; - } else if ((instr & 0x7c000000U) == 0x14000000U) { - /* Unconditional branch immediate. */ - assert((imm >> 26) == INT64_C(-1) || (imm >> 26) == 0); - instr |= (imm & 0x03ffffffU) << 0; - } else { - assert(false); - instr = BAD_OPCODE; - } - memcpy(state->buf + offset, &instr, sizeof(uint32_t)); -} - -static void -update_load_literal(struct jit_state* state, uint32_t instr_offset, int32_t target_offset) -{ - uint32_t instr; - target_offset = (0x7FFFF & target_offset) << 5; - memcpy(&instr, state->buf + instr_offset, sizeof(uint32_t)); - instr |= target_offset; - memcpy(state->buf + instr_offset, &instr, sizeof(uint32_t)); -} - /* Generate the function prologue. * * We set the stack to look like: @@ -609,6 +531,9 @@ emit_jit_prologue(struct jit_state* state, size_t ubpf_stack_size) /* Setup UBPF frame pointer. */ emit_addsub_immediate(state, true, AS_ADD, map_register(10), SP, state->stack_size); + /* Copy R0 to the volatile context for safe keeping. */ + emit_logical_register(state, true, LOG_ORR, VOLATILE_CTXT, RZ, R0); + emit_unconditionalbranch_immediate(state, UBR_BL, TARGET_PC_ENTER); emit_unconditionalbranch_immediate(state, UBR_B, TARGET_PC_EXIT); state->entry_loc = state->offset; @@ -617,20 +542,51 @@ emit_jit_prologue(struct jit_state* state, size_t ubpf_stack_size) static void emit_dispatched_external_helper_call(struct jit_state* state, struct ubpf_vm* vm, unsigned int idx) { + UNUSED_PARAMETER(vm); + + /* + * There are two paths through the function: + * 1. There is an external dispatcher registered. If so, we prioritize that. + * 2. We fall back to the regular registered helper. + * See translate and emit_dispatched_external_helper_call in ubpf_jit_x86_64.c for additional + * details. + */ + uint32_t stack_movement = align_to(8, 16); emit_addsub_immediate(state, true, AS_SUB, SP, SP, stack_movement); emit_loadstore_immediate(state, LS_STRX, R30, SP, 0); - // All parameters to the helper function are in the right spot - // for the dispatcher. All we need to do now is ... + // Determine whether to call it through a dispatcher or by index and then load up the address + // of that function. + emit_loadstore_literal(state, LS_LDRL, temp_register, TARGET_PC_EXTERNAL_DISPATCHER); + + // Check whether temp_register is empty. + emit_addsub_immediate(state, true, AS_SUBS, temp_register, temp_register, 0); + + // Jump if we are ready to roll (because we are using an external dispatcher). + uint32_t jump_source = emit_conditionalbranch_immediate(state, COND_NE, 0); + + // We are not ready to roll. So, load the helper function address by index. + emit_movewide_immediate(state, true, R5, idx); + emit_movewide_immediate(state, true, R6, 3); + emit_dataprocessing_twosource(state, true, DP2_LSLV, R5, R5, R6); + + emit_movewide_immediate(state, true, temp_register, 0); + emit_adr(state, TARGET_LOAD_HELPER_TABLE, temp_register); + emit_addsub_register(state, true, AS_ADD, temp_register, temp_register, R5); + emit_loadstore_immediate(state, LS_LDRX, temp_register, temp_register, 0); + + // And now we, too, are ready to roll. + + // Both paths meet here where we ... + emit_jump_target(state, jump_source); // ... set up the final two parameters. emit_movewide_immediate(state, true, R5, idx); - emit_movewide_immediate(state, true, R6, (uint64_t)vm->dispatcher_cookie); + // Use a sneaky way to copy the context register into the R6 register (as the final parameter). + emit_logical_register(state, true, LOG_ORR, R6, RZ, VOLATILE_CTXT); - // Call! - note_load(state, TARGET_PC_EXTERNAL_DISPATCHER); - emit_loadstore_literal(state, LS_LDRL, temp_register); + // Now, all that's left is to call! emit_unconditionalbranch_register(state, BR_BLR, temp_register); /* On exit need to move result from r0 to whichever register we've mapped EBPF r0 to. */ @@ -651,7 +607,6 @@ emit_local_call(struct jit_state* state, uint32_t target_pc) emit_loadstore_immediate(state, LS_STRX, R30, SP, 0); emit_loadstorepair_immediate(state, LSP_STPX, map_register(6), map_register(7), SP, 8); emit_loadstorepair_immediate(state, LSP_STPX, map_register(8), map_register(9), SP, 24); - note_jump(state, target_pc); emit_unconditionalbranch_immediate(state, UBR_BL, target_pc); emit_loadstore_immediate(state, LS_LDRX, R30, SP, 0); emit_loadstorepair_immediate(state, LSP_LDPX, map_register(6), map_register(7), SP, 8); @@ -702,6 +657,16 @@ emit_dispatched_external_helper_address(struct jit_state *state, uint64_t dispat return helper_address; } +static uint32_t +emit_helper_table(struct jit_state* state, struct ubpf_vm* vm) { + + uint32_t helper_table_address_target = state->offset; + for (int i = 0; iext_funcs[i], sizeof(uint64_t)); + } + return helper_table_address_target; +} + static bool is_imm_op(struct ebpf_inst const* inst) { @@ -976,6 +941,13 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) emit_jit_prologue(state, UBPF_STACK_SIZE); for (i = 0; i < vm->num_insts; i++) { + + if (state->jit_status != NoError) { + break; + } + + // All checks for errors during the encoding of _this_ instruction + // occur at the end of the loop. struct ebpf_inst inst = ubpf_fetch_instruction(vm, i); state->pc_locs[i] = state->offset; @@ -986,6 +958,9 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) int sixty_four = is_alu64_op(&inst); + // If this is an operation with an immediate operand (and that immediate + // operand is _not_ simple), then we convert the operation to the equivalent + // register version after moving the immediate into a temporary register. if (is_imm_op(&inst) && !is_simple_imm(&inst)) { emit_movewide_immediate(state, sixty_four, temp_register, (int64_t)inst.imm); src = temp_register; @@ -1193,16 +1168,53 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) case EBPF_OP_RSH64_IMM: case EBPF_OP_ARSH64_IMM: *errmsg = ubpf_error("Unexpected instruction at PC %d: opcode %02x, immediate %08x", i, opcode, inst.imm); - return -1; + state->jit_status = UnexpectedInstruction; default: *errmsg = ubpf_error("Unknown instruction at PC %d: opcode %02x", i, opcode); - return -1; + state->jit_status = UnknownInstruction; + } + } + + if (state->jit_status != NoError) { + switch (state->jit_status) { + case TooManyJumps: { + *errmsg = ubpf_error("Too many jump instructions."); + break; + } + case TooManyLoads: { + *errmsg = ubpf_error("Too many load instructions."); + break; + } + case TooManyLeas: { + *errmsg = ubpf_error("Too many LEA calculations."); + break; + } + case UnexpectedInstruction: { + // errmsg set at time the error was detected because the message requires + // information about the unexpected instruction. + break; + } + case UnknownInstruction: { + // errmsg set at time the error was detected because the message requires + // information about the unknown instruction. + break; + } + case NotEnoughSpace: { + *errmsg = ubpf_error("Target buffer too small"); + break; + } + case NoError: { + assert(false); + } } + return -1; } + emit_jit_epilogue(state); state->dispatcher_loc = emit_dispatched_external_helper_address(state, (uint64_t)vm->dispatcher); + state->helper_table_loc = emit_helper_table(state, vm); return 0; } @@ -1223,6 +1235,49 @@ divmod(struct jit_state* state, uint8_t opcode, int rd, int rn, int rm) } } +static void +resolve_branch_immediate(struct jit_state* state, uint32_t offset, int32_t imm) +{ + assert((imm & 3) == 0); + uint32_t instr; + imm >>= 2; + memcpy(&instr, state->buf + offset, sizeof(uint32_t)); + if ((instr & 0xfe000000U) == 0x54000000U /* Conditional branch immediate. */ + || (instr & 0x7e000000U) == 0x34000000U) { /* Compare and branch immediate. */ + assert((imm >> 19) == INT64_C(-1) || (imm >> 19) == 0); + instr |= (imm & 0x7ffff) << 5; + } else if ((instr & 0x7c000000U) == 0x14000000U) { + /* Unconditional branch immediate. */ + assert((imm >> 26) == INT64_C(-1) || (imm >> 26) == 0); + instr |= (imm & 0x03ffffffU) << 0; + } else { + assert(false); + instr = BAD_OPCODE; + } + memcpy(state->buf + offset, &instr, sizeof(uint32_t)); +} + +static void +resolve_load_literal(struct jit_state* state, uint32_t instr_offset, int32_t target_offset) +{ + uint32_t instr; + target_offset = (0x7FFFF & target_offset) << 5; + memcpy(&instr, state->buf + instr_offset, sizeof(uint32_t)); + instr |= target_offset; + memcpy(state->buf + instr_offset, &instr, sizeof(uint32_t)); +} + +static void +resolve_adr(struct jit_state* state, uint32_t instr_offset, int32_t immediate) +{ + uint32_t instr; + uint32_t immhi = (immediate & 0x00ffffff) << 5; + memcpy(&instr, state->buf + instr_offset, sizeof(uint32_t)); + instr |= immhi; + memcpy(state->buf + instr_offset, &instr, sizeof(uint32_t)); +} + + static bool resolve_jumps(struct jit_state* state) { @@ -1230,7 +1285,9 @@ resolve_jumps(struct jit_state* state) struct patchable_relative jump = state->jumps[i]; int32_t target_loc; - if (jump.target_pc == TARGET_PC_EXIT) { + if (jump.target_offset != 0) { + target_loc = jump.target_offset; + } else if (jump.target_pc == TARGET_PC_EXIT) { target_loc = state->exit_loc; } else if (jump.target_pc == TARGET_PC_ENTER) { target_loc = state->entry_loc; @@ -1239,7 +1296,7 @@ resolve_jumps(struct jit_state* state) } int32_t rel = target_loc - jump.offset_loc; - update_branch_immediate(state, jump.offset_loc, rel); + resolve_branch_immediate(state, jump.offset_loc, rel); } return true; } @@ -1251,6 +1308,7 @@ resolve_loads(struct jit_state* state) struct patchable_relative jump = state->loads[i]; int32_t target_loc; + // Right now it is only possible to load from the external dispatcher. if (jump.target_pc == TARGET_PC_EXTERNAL_DISPATCHER) { target_loc = state->dispatcher_loc; } else { @@ -1260,57 +1318,85 @@ resolve_loads(struct jit_state* state) int32_t rel = target_loc - jump.offset_loc; assert(rel % 4 == 0); rel >>= 2; - update_load_literal(state, jump.offset_loc, rel); + resolve_load_literal(state, jump.offset_loc, rel); + } + return true; +} + +static bool +resolve_leas(struct jit_state* state) +{ + for (unsigned i = 0; i < state->num_leas; ++i) { + struct patchable_relative jump = state->leas[i]; + + int32_t target_loc; + // Right now it is only possible to have leas to the helper table. + if (jump.target_pc == TARGET_LOAD_HELPER_TABLE) { + target_loc = state->helper_table_loc; + } else { + return false; + } + + int32_t rel = target_loc - jump.offset_loc; + assert(rel % 4 == 0); + rel >>= 2; + resolve_adr(state, jump.offset_loc, rel); } return true; } -int -ubpf_translate_arm64(struct ubpf_vm* vm, uint8_t* buffer, size_t* size, char** errmsg) +bool ubpf_jit_update_dispatcher_arm64(struct ubpf_vm* vm, external_function_dispatcher_t new_dispatcher, uint8_t* buffer, size_t size, uint32_t offset) { - struct jit_state state; - int result = -1; - - state.offset = 0; - state.size = *size; - state.buf = buffer; - state.pc_locs = calloc(UBPF_MAX_INSTS + 1, sizeof(state.pc_locs[0])); - state.jumps = calloc(UBPF_MAX_INSTS, sizeof(state.jumps[0])); - state.loads = calloc(UBPF_MAX_INSTS, sizeof(state.loads[0])); - state.num_jumps = 0; - state.num_loads = 0; - - if (!state.pc_locs || !state.jumps) { - *errmsg = ubpf_error("Out of memory"); - goto out; + UNUSED_PARAMETER(vm); + uint64_t jit_upper_bound = (uint64_t)buffer + size; + void *dispatcher_address = (void*)((uint64_t)buffer + offset); + if ((uint64_t)dispatcher_address + sizeof(void*) < jit_upper_bound) { + memcpy(dispatcher_address, &new_dispatcher, sizeof(void*)); + return true; } - if (translate(vm, &state, errmsg) < 0) { - goto out; + return false; +} + +bool ubpf_jit_update_helper_arm64(struct ubpf_vm* vm, ext_func new_helper, unsigned int idx, uint8_t* buffer, size_t size, uint32_t offset) +{ + UNUSED_PARAMETER(vm); + uint64_t jit_upper_bound = (uint64_t)buffer + size; + + void* dispatcher_address = (void*)((uint64_t)buffer + offset + (8 * idx)); + if ((uint64_t)dispatcher_address + sizeof(void*) < jit_upper_bound) { + memcpy(dispatcher_address, &new_helper, sizeof(void*)); + return true; } + return false; +} + +struct ubpf_jit_result +ubpf_translate_arm64(struct ubpf_vm* vm, uint8_t* buffer, size_t* size) +{ + struct jit_state state; + struct ubpf_jit_result compile_result; - if (state.num_jumps == UBPF_MAX_INSTS) { - *errmsg = ubpf_error("Excessive number of jump targets"); + if (initialize_jit_state_result(&state, &compile_result, buffer, *size, &compile_result.errmsg) < 0) { goto out; } - if (state.offset == state.size) { - *errmsg = ubpf_error("Target buffer too small"); + if (translate(vm, &state, &compile_result.errmsg) < 0) { goto out; } - if (!resolve_jumps(&state) || !resolve_loads(&state)) { - *errmsg = ubpf_error("Could not patch the relative addresses in the JIT'd code."); + if (!resolve_jumps(&state) || !resolve_loads(&state) || !resolve_leas(&state)) { + compile_result.errmsg = ubpf_error("Could not patch the relative addresses in the JIT'd code."); goto out; } - result = 0; + compile_result.compile_result = UBPF_JIT_COMPILE_SUCCESS; *size = state.offset; + compile_result.external_dispatcher_offset = state.dispatcher_loc; + compile_result.external_helper_offset = state.helper_table_loc; out: - free(state.pc_locs); - free(state.jumps); - free(state.loads); - return result; + release_jit_state_result(&state, &compile_result); + return compile_result; } diff --git a/vm/ubpf_jit_support.c b/vm/ubpf_jit_support.c new file mode 100644 index 00000000..ba96b9be --- /dev/null +++ b/vm/ubpf_jit_support.c @@ -0,0 +1,100 @@ +// Copyright (c) Will Hawkins +// SPDX-License-Identifier: Apache-2.0 + +/* + * Copyright Will Hawkins + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ubpf_jit_support.h" +#include +#include "ubpf_int.h" + +int +initialize_jit_state_result(struct jit_state *state, struct ubpf_jit_result *compile_result, uint8_t *buffer, uint32_t size, char **errmsg) { + compile_result->compile_result = UBPF_JIT_COMPILE_FAILURE; + compile_result->errmsg = NULL; + compile_result->external_dispatcher_offset = 0; + + state->offset = 0; + state->size = size; + state->buf = buffer; + state->pc_locs = calloc(UBPF_MAX_INSTS + 1, sizeof(state->pc_locs[0])); + state->jumps = calloc(UBPF_MAX_INSTS, sizeof(state->jumps[0])); + state->loads = calloc(UBPF_MAX_INSTS, sizeof(state->loads[0])); + state->leas = calloc(UBPF_MAX_INSTS, sizeof(state->leas[0])); + state->num_jumps = 0; + state->num_loads = 0; + state->num_leas = 0; + state->jit_status = NoError; + + if (!state->pc_locs || !state->jumps || !state->loads || !state->leas) { + *errmsg = ubpf_error("Could not allocate space needed to JIT compile eBPF program"); + return -1; + } + + return 0; +} + +void +release_jit_state_result(struct jit_state *state, struct ubpf_jit_result *compile_result) +{ + UNUSED_PARAMETER(compile_result); + free(state->pc_locs); + state->pc_locs = NULL; + free(state->jumps); + state->jumps = NULL; + free(state->loads); + state->loads = NULL; + free(state->leas); + state->leas = NULL; +} + +void +emit_patchable_relative(uint32_t offset, uint32_t target_pc, uint32_t manual_target_offset, struct patchable_relative *table, size_t index) +{ + struct patchable_relative* jump = &table[index]; + jump->offset_loc = offset; + jump->target_pc = target_pc; + jump->target_offset = manual_target_offset; +} + +void +note_load(struct jit_state* state, uint32_t target_pc) +{ + emit_patchable_relative(state->offset, target_pc, 0, state->loads, state->num_loads++); +} + +void +note_lea(struct jit_state* state, uint32_t offset) +{ + emit_patchable_relative(state->offset, offset, 0, state->leas, state->num_leas++); +} + + +void +fixup_jump_target(struct patchable_relative *table, size_t table_size, uint32_t src_offset, uint32_t dest_offset) +{ + for (size_t index = 0; index < table_size; index++) { + if (table[index].offset_loc == src_offset) { + table[index].target_offset = dest_offset; + } + } +} + +void +emit_jump_target(struct jit_state* state, uint32_t jump_src) +{ + fixup_jump_target(state->jumps, state->num_jumps, jump_src, state->offset); +} \ No newline at end of file diff --git a/vm/ubpf_jit_support.h b/vm/ubpf_jit_support.h new file mode 100644 index 00000000..19738975 --- /dev/null +++ b/vm/ubpf_jit_support.h @@ -0,0 +1,117 @@ +// Copyright (c) Will Hawkins +// SPDX-License-Identifier: Apache-2.0 + +/* + * Copyright 2015 Big Switch Networks, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Generic x86-64 code generation functions + */ + +#ifndef UBPF_JIT_SUPPORT_H +#define UBPF_JIT_SUPPORT_H + +#include +#include +#include "ubpf_int.h" + +enum JitProgress { + NoError, + TooManyJumps, + TooManyLoads, + TooManyLeas, + NotEnoughSpace, + UnexpectedInstruction, + UnknownInstruction +}; + +struct patchable_relative +{ + /* Where in the instruction stream should this relative address be patched. */ + uint32_t offset_loc; + /* Which PC should this target. The ultimate offset will be determined + * automatically unless ... */ + uint32_t target_pc; + /* ... the target_offset is set which overrides the automatic lookup. */ + uint32_t target_offset; +}; + +/* Special values for target_pc in struct jump */ +#define TARGET_PC_EXIT ~UINT32_C(0) +#define TARGET_PC_ENTER (~UINT32_C(0) & 0x01) +#define TARGET_PC_RETPOLINE (~UINT32_C(0) & 0x0101) +#define TARGET_PC_EXTERNAL_DISPATCHER (~UINT32_C(0) & 0x010101) +#define TARGET_LOAD_HELPER_TABLE (~UINT32_C(0) & 0x01010101) + +struct jit_state +{ + uint8_t* buf; + uint32_t offset; + uint32_t size; + uint32_t* pc_locs; + uint32_t exit_loc; + uint32_t entry_loc; + uint32_t unwind_loc; + /* The offset (from the start of the JIT'd code) to the location + * of the retpoline (if retpoline support is enabled). + */ + uint32_t retpoline_loc; + /* The offset (from the start of the JIT'd code) to the location + * of the address of the external helper dispatcher. The address + * at that location during execution may be null if no external + * helper dispatcher is registered. See commentary in ubpf_jit_x86_64.c. + */ + uint32_t dispatcher_loc; + /* The offset (from the start of the JIT'd code) to the location + * of a consecutive series of XXXX addresses that contain pointers + * to external helper functions. The address' position in the sequence + * corresponds to the index of the helper function. Addresses may + * be null but validation guarantees that (at the time the eBPF program + * is loaded), if a helper function is called, there is an appropriately + * registered handler. See commentary in ubpf_jit_x86_64.c. + */ + uint32_t helper_table_loc; + enum JitProgress jit_status; + struct patchable_relative* jumps; + struct patchable_relative* loads; + struct patchable_relative* leas; + int num_jumps; + int num_loads; + int num_leas; + uint32_t stack_size; +}; + +int +initialize_jit_state_result(struct jit_state *state, struct ubpf_jit_result *compile_result, uint8_t *buffer, uint32_t size, char **errmsg); + +void +release_jit_state_result(struct jit_state *state, struct ubpf_jit_result *compile_result); + +void +emit_patchable_relative(uint32_t offset, uint32_t target_pc, uint32_t manual_target_offset, struct patchable_relative *table, size_t index); + +void +note_load(struct jit_state* state, uint32_t target_pc); + +void +note_lea(struct jit_state* state, uint32_t offset); + +void +emit_jump_target(struct jit_state* state, uint32_t jump_src); + +void +fixup_jump_target(struct patchable_relative *table, size_t table_size, uint32_t src_offset, uint32_t dest_offset); +#endif diff --git a/vm/ubpf_jit_x86_64.c b/vm/ubpf_jit_x86_64.c index 45e7e67e..5fe7b98c 100644 --- a/vm/ubpf_jit_x86_64.c +++ b/vm/ubpf_jit_x86_64.c @@ -18,6 +18,7 @@ * limitations under the License. */ +#include "ubpf_jit_support.h" #define _GNU_SOURCE #include "ebpf.h" @@ -115,7 +116,7 @@ emit_local_call(struct jit_state* state, uint32_t target_pc) emit_alu64_imm32(state, 0x81, 5, RSP, 4 * sizeof(uint64_t)); #endif emit1(state, 0xe8); // e8 is the opcode for a CALL - emit_jump_target_address(state, target_pc); + emit_jump_address_reloc(state, target_pc); #if defined(_WIN32) /* Deallocate home register space - 4 registers */ emit_alu64_imm32(state, 0x81, 0, RSP, 4 * sizeof(uint64_t)); @@ -134,6 +135,16 @@ emit_dispatched_external_helper_address(struct jit_state* state, struct ubpf_vm* return external_helper_address_target; } +static uint32_t +emit_helper_table(struct jit_state* state, struct ubpf_vm* vm) { + + uint32_t helper_table_address_target = state->offset; + for (int i = 0; iext_funcs[i]); + } + return helper_table_address_target; +} + static uint32_t emit_retpoline(struct jit_state* state) { @@ -147,32 +158,27 @@ emit_retpoline(struct jit_state* state) /* label0: */ /* call label1 */ uint32_t retpoline_target = state->offset; - emit1(state, 0xe8); - uint32_t label1_call_offset = state->offset; - emit4(state, 0x00); + uint32_t label1_call_offset = emit_call(state, 0); /* capture_ret_spec: */ /* pause */ uint32_t capture_ret_spec = state->offset; - emit1(state, 0xf3); - emit1(state, 0x90); + emit_pause(state); /* jmp capture_ret_spec */ - emit1(state, 0xe9); - emit_jump_target_offset(state, state->offset, capture_ret_spec); - emit4(state, 0x00); + emit_jmp(state, capture_ret_spec); /* label1: */ /* mov rax, (rsp) */ uint32_t label1 = state->offset; emit1(state, 0x48); emit1(state, 0x89); - emit1(state, 0x04); - emit1(state, 0x24); + emit1(state, 0x04); // Mod: 00b Reg: 000b RM: 100b + emit1(state, 0x24); // Scale: 00b Index: 100b Base: 100b /* ret */ - emit1(state, 0xc3); + emit_ret(state); - emit_jump_target_offset(state, label1_call_offset, label1); + fixup_jump_target(state->jumps, state->num_jumps, label1_call_offset, label1); return retpoline_target; } @@ -200,6 +206,25 @@ ubpf_set_register_offset(int x) } } +/* + * In order to make it so that the generated code is completely standalone, all the necessary + * function pointers for external helpers are embedded in the jitted code. The layout looks like: + * + * state->buffer: CODE + * CODE + * CODE + * ... + * CODE + * External Helper External Dispatcher Function Pointer (8 bytes, maybe NULL) + * External Helper Function Pointer Idx 0 (8 bytes, maybe NULL) + * External Helper Function Pointer Idx 1 (8 bytes, maybe NULL) + * ... + * External Helper Function Pointer Idx MAX_EXT_FUNCS-1 (8 bytes, maybe NULL) + * state->buffer + state->offset: + * + * The layout and operation of this mechanism is identical for code JIT compiled for Arm. + */ + static int translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) { @@ -215,6 +240,11 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) emit_mov(state, platform_parameter_registers[0], map_register(BPF_REG_1)); } + /* Move the platform parameter register to the (volatile) register + * that holds the pointer to the context. + */ + emit_mov(state, platform_parameter_registers[0], VOLATILE_CTXT); + /* * Assuming that the stack is 16-byte aligned right before * the call insn that brought us to this code, when @@ -259,6 +289,10 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) emit_jmp(state, TARGET_PC_EXIT); for (i = 0; i < vm->num_insts; i++) { + if (state->jit_status != NoError) { + break; + } + struct ebpf_inst inst = ubpf_fetch_instruction(vm, i); state->pc_locs[i] = state->offset; @@ -614,7 +648,7 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) /* We reserve RCX for shifts */ if (inst.src == 0) { emit_mov(state, RCX_ALT, RCX); - emit_dispatched_external_helper_call(state, vm, inst.imm); + emit_dispatched_external_helper_call(state, inst.imm); if (inst.imm == vm->unwind_stack_extension_index) { emit_cmp_imm32(state, map_register(BPF_REG_0), 0); emit_jcc(state, 0x84, TARGET_PC_EXIT); @@ -679,11 +713,46 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) } default: + state->jit_status = UnknownInstruction; *errmsg = ubpf_error("Unknown instruction at PC %d: opcode %02x", i, inst.opcode); - return -1; } } + if (state->jit_status != NoError) { + switch (state->jit_status) { + case TooManyJumps: { + *errmsg = ubpf_error("Too many jump instructions"); + break; + } + case TooManyLoads: { + *errmsg = ubpf_error("Too many load instructions"); + break; + } + case TooManyLeas: { + *errmsg = ubpf_error("Too many LEA calculations"); + break; + } + case UnexpectedInstruction: { + // errmsg set at time the error was detected because the message requires + // information about the unexpected instruction. + break; + } + case UnknownInstruction: { + // errmsg set at time the error was detected because the message requires + // information about the unknown instruction. + break; + } + case NotEnoughSpace: { + *errmsg = ubpf_error("Target buffer too small"); + break; + } + case NoError: { + assert(false); + } + } + return -1; + } + /* Epilogue */ state->exit_loc = state->offset; @@ -708,6 +777,7 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) state->retpoline_loc = emit_retpoline(state); state->dispatcher_loc = emit_dispatched_external_helper_address(state, vm); + state->helper_table_loc = emit_helper_table(state, vm); return 0; } @@ -860,71 +930,96 @@ resolve_patchable_relatives(struct jit_state* state) uint8_t* offset_ptr = &state->buf[jump.offset_loc]; memcpy(offset_ptr, &rel, sizeof(uint32_t)); } + for (i = 0; i < state->num_loads; i++) { struct patchable_relative load = state->loads[i]; int target_loc; + // It is only possible to load from the external dispatcher's position. if (load.target_pc == TARGET_PC_EXTERNAL_DISPATCHER) { target_loc = state->dispatcher_loc; } else { target_loc = -1; return false; } - - /* Assumes jump offset is at end of instruction */ + /* Assumes load target is calculated relative to the end of instruction */ uint32_t rel = target_loc - (load.offset_loc + sizeof(uint32_t)); uint8_t* offset_ptr = &state->buf[load.offset_loc]; memcpy(offset_ptr, &rel, sizeof(uint32_t)); } + + for (i = 0; i < state->num_leas; i++) { + struct patchable_relative lea = state->leas[i]; + + int target_loc; + // It is only possible to LEA from the helper table. + if (lea.target_pc == TARGET_LOAD_HELPER_TABLE) { + target_loc = state->helper_table_loc; + } else { + target_loc = -1; + return false; + } + /* Assumes lea target is calculated relative to the end of instruction */ + uint32_t rel = target_loc - (lea.offset_loc + sizeof(uint32_t)); + + uint8_t* offset_ptr = &state->buf[lea.offset_loc]; + memcpy(offset_ptr, &rel, sizeof(uint32_t)); + } return true; } -int -ubpf_translate_x86_64(struct ubpf_vm* vm, uint8_t* buffer, size_t* size, char** errmsg) +struct ubpf_jit_result +ubpf_translate_x86_64(struct ubpf_vm* vm, uint8_t* buffer, size_t* size) { struct jit_state state; - int result = -1; - - state.offset = 0; - state.size = *size; - state.buf = buffer; - state.pc_locs = calloc(UBPF_MAX_INSTS + 1, sizeof(state.pc_locs[0])); - state.jumps = calloc(UBPF_MAX_INSTS, sizeof(state.jumps[0])); - state.loads = calloc(UBPF_MAX_INSTS, sizeof(state.loads[0])); - state.num_jumps = 0; - state.num_loads = 0; - - if (!state.pc_locs || !state.jumps) { - *errmsg = ubpf_error("Out of memory"); - goto out; - } + struct ubpf_jit_result compile_result; - if (translate(vm, &state, errmsg) < 0) { + if (initialize_jit_state_result(&state, &compile_result, buffer, *size, &compile_result.errmsg) < 0) { goto out; } - if (state.num_jumps == UBPF_MAX_INSTS) { - *errmsg = ubpf_error("Excessive number of jump targets"); + if (translate(vm, &state, &compile_result.errmsg) < 0) { goto out; } - if (state.offset == state.size) { - *errmsg = ubpf_error("Target buffer too small"); + if (!resolve_patchable_relatives(&state)) { + compile_result.errmsg = ubpf_error("Could not patch the relative addresses in the JIT'd code"); goto out; } - if (!resolve_patchable_relatives(&state)) { - *errmsg = ubpf_error("Could not patch the relative addresses in the JIT'd code."); - goto out; - } - - result = 0; + compile_result.compile_result = UBPF_JIT_COMPILE_SUCCESS; + compile_result.external_dispatcher_offset = state.dispatcher_loc; + compile_result.external_helper_offset = state.helper_table_loc; *size = state.offset; out: - free(state.pc_locs); - free(state.jumps); - free(state.loads); - return result; + release_jit_state_result(&state, &compile_result); + return compile_result; +} + +bool ubpf_jit_update_dispatcher_x86_64(struct ubpf_vm* vm, external_function_dispatcher_t new_dispatcher, uint8_t* buffer, size_t size, uint32_t offset) +{ + UNUSED_PARAMETER(vm); + uint64_t jit_upper_bound = (uint64_t)buffer + size; + void *dispatcher_address = (void*)((uint64_t)buffer + offset); + if ((uint64_t)dispatcher_address + sizeof(void*) < jit_upper_bound) { + memcpy(dispatcher_address, &new_dispatcher, sizeof(void*)); + return true; + } + + return false; +} + +bool ubpf_jit_update_helper_x86_64(struct ubpf_vm* vm, ext_func new_helper, unsigned int idx, uint8_t* buffer, size_t size, uint32_t offset) +{ + UNUSED_PARAMETER(vm); + uint64_t jit_upper_bound = (uint64_t)buffer + size; + + void* dispatcher_address = (void*)((uint64_t)buffer + offset + (8 * idx)); + if ((uint64_t)dispatcher_address + sizeof(void*) < jit_upper_bound) { + memcpy(dispatcher_address, &new_helper, sizeof(void*)); + return true; + } + return false; } diff --git a/vm/ubpf_jit_x86_64.h b/vm/ubpf_jit_x86_64.h index 7c093d70..c10a7f71 100644 --- a/vm/ubpf_jit_x86_64.h +++ b/vm/ubpf_jit_x86_64.h @@ -29,7 +29,7 @@ #include #include "ubpf.h" -#include "ubpf_int.h" +#include "ubpf_jit_support.h" #define RAX 0 #define RCX 1 @@ -49,6 +49,8 @@ #define R14 14 #define R15 15 +#define VOLATILE_CTXT 11 + enum operand_size { S8, @@ -57,42 +59,21 @@ enum operand_size S64, }; -struct patchable_relative -{ - uint32_t offset_loc; - uint32_t target_pc; - uint32_t target_offset; -}; - -/* Special values for target_pc in struct jump */ -#define TARGET_PC_EXIT -1 -#define TARGET_PC_RETPOLINE -3 -#define TARGET_PC_EXTERNAL_DISPATCHER -4 - -struct jit_state -{ - uint8_t* buf; - uint32_t offset; - uint32_t size; - uint32_t* pc_locs; - uint32_t exit_loc; - uint32_t unwind_loc; - uint32_t retpoline_loc; - uint32_t dispatcher_loc; - struct patchable_relative* jumps; - struct patchable_relative* loads; - int num_jumps; - int num_loads; -}; - static inline void emit_bytes(struct jit_state* state, void* data, uint32_t len) { - assert(state->offset <= state->size - len); + // Never emit any bytes if there is an error! + if (state->jit_status != NoError) { + return; + } + + // If we are trying to emit bytes to a spot outside the buffer, + // then there is not enough space! if ((state->offset + len) > state->size) { - state->offset = state->size; + state->jit_status = NotEnoughSpace; return; } + memcpy(state->buf + state->offset, data, len); state->offset += len; } @@ -121,32 +102,28 @@ emit8(struct jit_state* state, uint64_t x) emit_bytes(state, &x, sizeof(x)); } -static inline void -emit_jump_target_address(struct jit_state* state, int32_t target_pc) -{ - if (state->num_jumps == UBPF_MAX_INSTS) { - return; - } - struct patchable_relative* jump = &state->jumps[state->num_jumps++]; - jump->offset_loc = state->offset; - jump->target_pc = target_pc; +static void +emit_4byte_offset_placeholder(struct jit_state *state) { emit4(state, 0); } -static inline void -emit_jump_target_offset(struct jit_state* state, uint32_t jump_loc, uint32_t jump_state_offset) +static uint32_t +emit_jump_address_reloc(struct jit_state* state, int32_t target_pc) { if (state->num_jumps == UBPF_MAX_INSTS) { - return; + state->jit_status = TooManyJumps; + return 0; } - struct patchable_relative* jump = &state->jumps[state->num_jumps++]; - jump->offset_loc = jump_loc; - jump->target_offset = jump_state_offset; + uint32_t target_address_offset = state->offset; + emit_patchable_relative(state->offset, target_pc, 0, state->jumps, state->num_jumps++); + emit_4byte_offset_placeholder(state); + return target_address_offset; } static inline void emit_modrm(struct jit_state* state, int mod, int r, int m) { + // Only the top 2 bits of the mod should be used. assert(!(mod & ~0xc0)); emit1(state, (mod & 0xc0) | ((r & 7) << 3) | (m & 7)); } @@ -292,12 +269,12 @@ emit_cmp32(struct jit_state* state, int src, int dst) emit_alu32(state, 0x39, src, dst); } -static inline void +static inline uint32_t emit_jcc(struct jit_state* state, int code, int32_t target_pc) { emit1(state, 0x0f); emit1(state, code); - emit_jump_target_address(state, target_pc); + return emit_jump_address_reloc(state, target_pc); } /* Load [src + offset] into dst */ @@ -332,20 +309,37 @@ emit_load_imm(struct jit_state* state, int dst, int64_t imm) } } -/* Load sign-extended immediate into register */ -static inline void -emit_load_relative(struct jit_state* state, int target_pc) +static uint32_t +emit_rip_relative_load(struct jit_state* state, int dst, int relative_load_tgt) { if (state->num_loads == UBPF_MAX_INSTS) { - return; + state->jit_status = TooManyLoads; + return 0; } - emit1(state, 0x48); + + emit_rex(state, 1, 0, 0, 0); emit1(state, 0x8b); - emit1(state, 0x05); - struct patchable_relative* load = &state->loads[state->num_loads++]; - load->offset_loc = state->offset; - load->target_pc = target_pc; - emit4(state, 0); + emit_modrm(state, 0, dst, 0x05); + uint32_t load_target_offset = state->offset; + note_load(state, relative_load_tgt); + emit_4byte_offset_placeholder(state); + return load_target_offset; +} + +static void +emit_rip_relative_lea(struct jit_state* state, int dst, int lea_tgt) +{ + if (state->num_leas == UBPF_MAX_INSTS) { + state->jit_status = TooManyLeas; + return; + } + + // lea dst, [rip + HELPER TABLE ADDRESS] + emit_rex(state, 1, 1, 0, 0); + emit1(state, 0x8d); + emit_modrm(state, 0, dst, 0x05); + note_lea(state, lea_tgt); + emit_4byte_offset_placeholder(state); } /* Store register src to [dst + offset] */ @@ -392,52 +386,178 @@ static inline void emit_jmp(struct jit_state* state, uint32_t target_pc) { emit1(state, 0xe9); - emit_jump_target_address(state, target_pc); + emit_jump_address_reloc(state, target_pc); +} + +static inline uint32_t +emit_call(struct jit_state* state, uint32_t target_pc) +{ + emit1(state, 0xe8); + uint32_t call_src = state->offset; + emit_jump_address_reloc(state, target_pc); + return call_src; } static inline void -emit_dispatched_external_helper_call(struct jit_state* state, const struct ubpf_vm* vm, unsigned int idx) +emit_pause(struct jit_state *state) +{ + emit1(state, 0xf3); + emit1(state, 0x90); +} + +static inline void +emit_dispatched_external_helper_call(struct jit_state* state, unsigned int idx) { /* + * Note: We do *not* have to preserve any x86-64 registers here ... + * ... according to the SystemV ABI: rbx (eBPF6), + * r13 (eBPF7), + * r14 (eBPF8), + * r15 (eBPF9), and + * rbp (eBPF10) are all preserved. + * ... according to the Windows ABI: r15 (eBPF6) + * rdi (eBPF7), + * rsi (eBPF8), + * rbx (eBPF9), and + * rbp (eBPF10) are all preserved. + * * When we enter here, our stack is 16-byte aligned. Keep * it that way! */ + /* + * There are two things that could happen: + * 1. The user has registered an external dispatcher and we need to + * send control there to invoke an external helper. + * 2. The user is relying on the default dispatcher to pass control + * to the registered external helper. + * To determine which action to take, we will first consider the 8 + * bytes at TARGET_PC_EXTERNAL_DISPATCHER. If those 8 bytes have an + * address, that represents the address of the user-registered external + * dispatcher and we pass control there. That function signature looks like + * uint64_t, uint64_t, uint64_t, uint64_t, uint64_t, unsigned int index, void* cookie + * so we make sure that the arguments are done properly depending on the abi. + * + * If there is no external dispatcher registered, the user is expected + * to have registered a handler with us for the helper with index idx. + * There is a table of MAX_ function pointers starting at TARGET_LOAD_HELPER_TABLE. + * Each of those functions has a signature that looks like + * uint64_t, uint64_t, uint64_t, uint64_t, uint64_t, void* cookie + * We load the appropriate function pointer by using idx to index it and then + * make sure that the arguments are set properly depending on the abi. + */ + + // Save r9 -- I need it for a parameter! + emit_push(state, R9); + // Save register where volatile context is stored. + emit_push(state, VOLATILE_CTXT); + // ^^ Stack is aligned here. + #if defined(_WIN32) - /* We have to create a little space to keep our 16-byte - * alignment happy + /* Because we may need 24 bytes on the stack but at least 16, we have to take 32 + * to keep alignment happy. We may ultimately need it all, but we certainly + * need 16! Later, though, there is a push that always happens (MARKER2), so + * we only allocate 24 here. */ - emit_alu64_imm32(state, 0x81, 5, RSP, sizeof(uint64_t)); + emit_alu64_imm32(state, 0x81, 5, RSP, 3*sizeof(uint64_t)); +#endif - emit_load_imm(state, RAX, (uint64_t)vm->dispatcher_cookie); - emit_push(state, RAX); + emit_rip_relative_load(state, RAX, TARGET_PC_EXTERNAL_DISPATCHER); + // cmp rax, 0 + emit_cmp_imm32(state, RAX, 0); + // jne skip_default_dispatcher_label + uint32_t skip_default_dispatcher_source = emit_jcc(state, 0x85, 0); - emit_load_imm(state, RAX, idx); - emit_push(state, RAX); + // Default dispatcher: - /* Windows x64 ABI spills 5th parameter to stack */ - emit_push(state, map_register(5)); + // Load the address of the helper function from the table. + // mov rax, idx + emit_alu32(state, 0xc7, 0, RAX); + emit4(state, idx); + // shl rax, 3 (i.e., multiply the index by 8 because addresses are that size on x86-64) + emit_alu64_imm8(state, 0xc1, 4, RAX, 3); - /* Windows x64 ABI requires home register space. - * Allocate home register space - 4 registers. - */ - emit_alu64_imm32(state, 0x81, 5, RSP, 4 * sizeof(uint64_t)); + // lea r9, [rip + HELPER TABLE ADDRESS] + emit_rip_relative_lea(state, R9, TARGET_LOAD_HELPER_TABLE); + + // add rax, r9 + emit_alu64(state, 0x01, R9, RAX); + // load rax, [rax] + emit_load(state, S64, RAX, RAX, 0); + + // There is no index for the registered helper function. They just get + // 5 arguments and a context, which becomes the 6th argument to the function ... +#if defined(_WIN32) + // and spills to the stack on Windows. + // mov qword [rsp], VOLATILE_CTXT + emit1(state, 0x4c); + emit1(state, 0x89); + emit1(state, 0x5c); + emit1(state, 0x24); + emit1(state, 0x00); #else - // Save r9 -- I need it for a parameter! - emit_push(state, R9); + // and goes in R9 on SystemV. + emit_mov(state, VOLATILE_CTXT, R9); +#endif - // Before it's a parameter, use it for a push. - emit_load_imm(state, R9, (uint64_t)vm->dispatcher_cookie); - emit_push(state, R9); + // jmp call_label + emit1(state, 0xe9); + uint32_t skip_external_dispatcher_source = state->offset; + emit_4byte_offset_placeholder(state); + + // External dispatcher: + // skip_default_dispatcher_label: + emit_jump_target(state, skip_default_dispatcher_source); + + // Using an external dispatcher. They get a total of 7 arguments. The + // 6th argument is the index of the function to call which ... + + +#if defined(_WIN32) + // and spills to the stack on Windows. + + // mov qword [rsp + 8], VOLATILE_CTXT + emit1(state, 0x4c); + emit1(state, 0x89); + emit1(state, 0x5c); + emit1(state, 0x24); + emit1(state, 0x08); + + emit_load_imm(state, VOLATILE_CTXT, (uint64_t)idx); + + // mov qword [rsp + 8], VOLATILE_CTXT + emit1(state, 0x4c); + emit1(state, 0x89); + emit1(state, 0x5c); + emit1(state, 0x24); + emit1(state, 0x00); +#else + // and goes in R9 on SystemV. emit_load_imm(state, R9, (uint64_t)idx); + // And the 7th is already spilled to the stack in the right spot because + // we wanted to save it -- cool (see MARKER1, above). + + // Intentional no-op for 7th argument. #endif - emit_load_relative(state, TARGET_PC_EXTERNAL_DISPATCHER); + // Control flow converges for call: + + // call_label: + emit_jump_target(state, skip_external_dispatcher_source); + +#if defined(_WIN32) + /* Windows x64 ABI spills 5th parameter to stack (MARKER2) */ + emit_push(state, map_register(5)); + + /* Windows x64 ABI requires home register space. + * Allocate home register space - 4 registers. + */ + emit_alu64_imm32(state, 0x81, 5, RSP, 4 * sizeof(uint64_t)); +#endif #ifndef UBPF_DISABLE_RETPOLINES - emit1(state, 0xe8); // e8 is the opcode for a CALL - emit_jump_target_address(state, TARGET_PC_RETPOLINE); + emit_call(state, TARGET_PC_RETPOLINE); #else /* TODO use direct call when possible */ /* callq *%rax */ @@ -456,12 +576,12 @@ emit_dispatched_external_helper_call(struct jit_state* state, const struct ubpf_ // Just rationalize the stack! #if defined(_WIN32) - /* Deallocate home register space + 3 spilled parameters + alignment space */ + /* Deallocate home register space + (up to ) 3 spilled parameters + alignment space */ emit_alu64_imm32(state, 0x81, 0, RSP, (4 + 3 + 1) * sizeof(uint64_t)); -#else - emit_pop(state, R9); // First one is a throw away (it's where our parameter was!) - emit_pop(state, R9); // This one is real! #endif + + emit_pop(state, VOLATILE_CTXT); // Restore register where volatile context is stored. + emit_pop(state, R9); // Restore r9. } #endif diff --git a/vm/ubpf_vm.c b/vm/ubpf_vm.c index 8e9341d0..a7db73a4 100644 --- a/vm/ubpf_vm.c +++ b/vm/ubpf_vm.c @@ -30,9 +30,9 @@ #include "ubpf_int.h" #include -#define MAX_EXT_FUNCS 64 #define SHIFT_MASK_32_BIT(X) ((X) & 0x1f) #define SHIFT_MASK_64_BIT(X) ((X) & 0x3f) +#define DEFAULT_JITTER_BUFFER_SIZE 65536 static bool validate(const struct ubpf_vm* vm, const struct ebpf_inst* insts, uint32_t num_insts, char** errmsg); @@ -65,20 +65,9 @@ ubpf_set_error_print(struct ubpf_vm* vm, int (*error_printf)(FILE* stream, const } static uint64_t -ubpf_default_external_dispatcher(uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5, unsigned int index, void* cookie) +ubpf_default_external_dispatcher(uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5, unsigned int index, external_function_t *external_fns) { - struct ubpf_vm *vm = (struct ubpf_vm*)cookie; - return vm->ext_funcs[index](arg1, arg2, arg3, arg4, arg5); -} - -static bool -ubpf_default_external_validator(unsigned int index, void* cookie) -{ - struct ubpf_vm *vm = (struct ubpf_vm*)cookie; - if (index < MAX_EXT_FUNCS) { - return vm->ext_funcs[index] != NULL; - } - return false; + return external_fns[index](arg1, arg2, arg3, arg4, arg5); } struct ubpf_vm* @@ -105,19 +94,20 @@ ubpf_create(void) vm->error_printf = fprintf; #if defined(__x86_64__) || defined(_M_X64) - vm->translate = ubpf_translate_x86_64; + vm->jit_translate = ubpf_translate_x86_64; + vm->jit_update_dispatcher = ubpf_jit_update_dispatcher_x86_64; + vm->jit_update_helper = ubpf_jit_update_helper_x86_64; #elif defined(__aarch64__) || defined(_M_ARM64) - vm->translate = ubpf_translate_arm64; + vm->jit_translate = ubpf_translate_arm64; + vm->jit_update_dispatcher = ubpf_jit_update_dispatcher_arm64; + vm->jit_update_helper = ubpf_jit_update_helper_arm64; #else vm->translate = ubpf_translate_null; #endif vm->unwind_stack_extension_index = -1; - // By default, we will set an internal function to be the dispatcher. - // If the user wants to override it, that's great (see ubpf_register_external_dispatcher). - vm->dispatcher = ubpf_default_external_dispatcher; - vm->dispatcher_validate = ubpf_default_external_validator; - vm->dispatcher_cookie = vm; + vm->jitted_result.compile_result = UBPF_JIT_COMPILE_FAILURE; + vm->jitter_buffer_size = DEFAULT_JITTER_BUFFER_SIZE; return vm; } @@ -148,19 +138,51 @@ ubpf_register(struct ubpf_vm* vm, unsigned int idx, const char* name, external_f vm->ext_funcs[idx] = (ext_func)fn; vm->ext_func_names[idx] = name; - return 0; + int success = 0; + + if (vm->jitted_result.compile_result == UBPF_JIT_COMPILE_SUCCESS) { + if (mprotect(vm->jitted, vm->jitted_size, PROT_READ | PROT_WRITE) < 0) { + return -1; + } + + // Now, update! + if (!vm->jit_update_helper(vm, fn, idx, (uint8_t*)vm->jitted, vm->jitted_size, vm->jitted_result.external_helper_offset)) { + // Can't immediately stop here because we have unprotected memory! + success = -1; + } + + if (mprotect(vm->jitted, vm->jitted_size, PROT_READ | PROT_EXEC) < 0) { + return -1; + } + } + return success; } int ubpf_register_external_dispatcher( - struct ubpf_vm* vm, external_function_dispatcher_t dispatcher, external_function_validate_t validater, void* cookie) + struct ubpf_vm* vm, external_function_dispatcher_t dispatcher, external_function_validate_t validater) { vm->dispatcher = dispatcher; vm->dispatcher_validate = validater; - vm->dispatcher_cookie = cookie; - /* TODO: If the code is already JIT'd, update the dispatcher's address. */ - return 0; + int success = 0; + + if (vm->jitted_result.compile_result == UBPF_JIT_COMPILE_SUCCESS) { + if (mprotect(vm->jitted, vm->jitted_size, PROT_READ | PROT_WRITE) < 0) { + return -1; + } + + // Now, update! + if (!vm->jit_update_dispatcher(vm, dispatcher, (uint8_t*)vm->jitted, vm->jitted_size, vm->jitted_result.external_dispatcher_offset)) { + // Can't immediately stop here because we have unprotected memory! + success = -1; + } + + if (mprotect(vm->jitted, vm->jitted_size, PROT_READ | PROT_EXEC) < 0) { + return -1; + } + } + return success; } int @@ -187,16 +209,6 @@ ubpf_lookup_registered_function(struct ubpf_vm* vm, const char* name) return -1; } -bool -ubpf_validate_external_helper(const struct ubpf_vm* vm, unsigned int idx) -{ - if (vm->dispatcher_validate) { - return vm->dispatcher_validate(idx, vm->dispatcher_cookie); - } - - return vm->ext_funcs[idx] != NULL; -} - int ubpf_load(struct ubpf_vm* vm, const void* code, uint32_t code_len, char** errmsg) { @@ -342,6 +354,7 @@ ubpf_exec(const struct ubpf_vm* vm, void* mem, size_t mem_len, uint64_t* bpf_ret uint64_t _reg[16]; uint64_t ras_index = 0; int return_value = -1; + void *external_dispatcher_cookie = mem; // Windows Kernel mode limits stack usage to 12K, so we need to allocate it dynamically. #if defined(NTDDI_VERSION) && defined(WINNT) @@ -899,9 +912,11 @@ ubpf_exec(const struct ubpf_vm* vm, void* mem, size_t mem_len, uint64_t* bpf_ret // program was assembled with the same endianess as the host machine. if (inst.src == 0) { // Handle call by address to external function. - // reg[0] = vm->ext_funcs[inst.imm](reg[1], reg[2], reg[3], reg[4], reg[5]); - reg[0] = vm->dispatcher(reg[1], reg[2], reg[3], reg[4], reg[5], inst.imm, (void*)vm->dispatcher_cookie); - // Unwind the stack if unwind extension returns success. + if (vm->dispatcher != NULL) { + reg[0] = vm->dispatcher(reg[1], reg[2], reg[3], reg[4], reg[5], inst.imm, external_dispatcher_cookie); + } else { + reg[0] = ubpf_default_external_dispatcher(reg[1], reg[2], reg[3], reg[4], reg[5], inst.imm, vm->ext_funcs); + } if (inst.imm == vm->unwind_stack_extension_index && reg[0] == 0) { *bpf_return_value = reg[0]; return_value = 0; @@ -1111,7 +1126,7 @@ validate(const struct ubpf_vm* vm, const struct ebpf_inst* insts, uint32_t num_i *errmsg = ubpf_error("invalid call immediate at PC %d", i); return false; } - if ((vm->dispatcher != NULL && !vm->dispatcher_validate(inst.imm, vm->dispatcher_cookie)) || + if ((vm->dispatcher != NULL && !vm->dispatcher_validate(inst.imm, vm)) || (vm->dispatcher == NULL && !vm->ext_funcs[inst.imm])) { *errmsg = ubpf_error("call to nonexistent function %u at PC %d", inst.imm, i); return false;