Skip to content

Commit

Permalink
Add new conformance tests
Browse files Browse the repository at this point in the history 
The bpf conformance repo added more tests, so use them
Also distinguish between reasons for failure, so "safe" failures
(verification failed, and verification succeeded but r0 is top)
can be allowed.

Signed-off-by: Dave Thaler <dthaler@microsoft.com>
  • Loading branch information
@dthaler @elazarg
dthaler authored and elazarg committed Nov 16, 2022
1 parent a993f06 commit 67cb4d1
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 42 deletions.
2 changes: 1 addition & 1 deletion external/bpf_conformance
Submodule bpf_conformance updated from db5934 to b428b2
13 changes: 7 additions & 6 deletions src/ebpf_yaml.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ static InstructionSeq raw_cfg_to_instruction_seq(const vector<std::tuple<string,
for (const auto& [label_name, raw_block] : raw_blocks) {
label_name_to_label.emplace(label_name, label_index);
// don't count large instructions as 2
label_index += raw_block.size();
label_index += (int)raw_block.size();
}

InstructionSeq res;
Expand Down Expand Up @@ -262,7 +262,7 @@ string_invariant stack_contents_invariant(const std::vector<uint8_t>& memory_byt
"s[" + std::to_string(EBPF_STACK_SIZE - memory_bytes.size()) + "..." +
std::to_string(EBPF_STACK_SIZE - 1) + "].type=number"};

int offset = EBPF_STACK_SIZE - memory_bytes.size();
int offset = EBPF_STACK_SIZE - (int)memory_bytes.size();
if (offset % 2 != 0) {
add_stack_variable<uint8_t>(more, offset, memory_bytes);
}
Expand All @@ -279,7 +279,7 @@ string_invariant stack_contents_invariant(const std::vector<uint8_t>& memory_byt
return string_invariant(more);
}

std::optional<uint64_t> run_conformance_test_case(const std::vector<uint8_t>& memory_bytes,
std::tuple<bool, std::optional<uint64_t>> run_conformance_test_case(const std::vector<uint8_t>& memory_bytes,
const std::vector<uint8_t>& program_bytes, bool debug) {
ebpf_context_descriptor_t context_descriptor{64, -1, -1, -1};
EbpfProgramType program_type = make_program_type("conformance_check", &context_descriptor);
Expand All @@ -292,11 +292,12 @@ std::optional<uint64_t> run_conformance_test_case(const std::vector<uint8_t>& me
if (!memory_bytes.empty()) {
if (memory_bytes.size() > EBPF_STACK_SIZE) {
std::cerr << "memory size overflow\n";
return false;
return {};
}
pre_invariant = pre_invariant + stack_contents_invariant(memory_bytes);
}
raw_program raw_prog{.prog = insts};
raw_prog.info.platform = &g_ebpf_platform_linux;

// Convert the raw program section to a set of instructions.
std::variant<InstructionSeq, std::string> prog_or_error = unmarshal(raw_prog);
Expand All @@ -321,14 +322,14 @@ std::optional<uint64_t> run_conformance_test_case(const std::vector<uint8_t>& me

for (const std::string& invariant : actual_last_invariant.value()) {
if (invariant.rfind("r0.value=", 0) == 0) {
return std::stoull(invariant.substr(9));
return {result, std::stoull(invariant.substr(9))};
}
}
return {result, {}};
} catch (std::exception) {
// Catch exceptions thrown in ebpf_domain.cpp.
return {};
}
return {};
}

void print_failure(const Failure& failure, std::ostream& out) {
Expand Down
2 changes: 1 addition & 1 deletion src/ebpf_yaml.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,6 @@ struct Failure {
void print_failure(const Failure& failure, std::ostream& out);

std::optional<Failure> run_yaml_test_case(const TestCase& test_case);
std::optional<uint64_t> run_conformance_test_case(const std::vector<uint8_t>& memory_bytes, const std::vector<uint8_t>& program_bytes, bool debug);
std::tuple<bool, std::optional<uint64_t>> run_conformance_test_case(const std::vector<uint8_t>& memory_bytes, const std::vector<uint8_t>& program_bytes, bool debug);

bool run_yaml(const std::string& path);
10 changes: 7 additions & 3 deletions src/test/conformance_check.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,14 @@ int main(int argc, char** argv) {
std::getline(std::cin, program_string);
}

std::optional<uint64_t> r0_value = run_conformance_test_case(base16_decode(memory_string), base16_decode(program_string), debug);
const auto& [result, r0_value] =
run_conformance_test_case(base16_decode(memory_string), base16_decode(program_string), debug);
if (!r0_value) {
if (debug) {
std::cerr << "Verification failed\n";
// Write failure reason to stdout which is all that the bpf conformance library looks at.
if (!result) {
std::cout << "Verification failed\n";
} else {
std::cout << "Couldn't determine r0 value\n";
}
return 1;
}
Expand Down
108 changes: 77 additions & 31 deletions src/test/test_conformance.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,121 +6,167 @@

#define CONFORMANCE_TEST_PATH "external/bpf_conformance/tests/"

void test_conformance(std::string filename, bpf_conformance_test_result_t expected_result) {
void test_conformance(std::string filename, bpf_conformance_test_result_t expected_result, std::string expected_reason) {
std::vector<std::filesystem::path> test_files = {CONFORMANCE_TEST_PATH + filename};
bool list_opcodes_tested = false;
std::string plugin_options;
boost::filesystem::path test_path = boost::dll::program_location();
boost::filesystem::path extension = test_path.extension();
std::filesystem::path plugin_path =
test_path.remove_filename().append("conformance_check" + extension.string()).string();
std::map<std::filesystem::path, std::tuple<bpf_conformance_test_result_t, std::string>> result =
bpf_conformance(test_files, plugin_path, plugin_options, list_opcodes_tested);
bpf_conformance(test_files, plugin_path, {});
for (auto file : test_files) {
auto& [file_result, _] = result[file];
auto& [file_result, reason] = result[file];
REQUIRE(file_result == expected_result);
if (file_result != bpf_conformance_test_result_t::TEST_RESULT_PASS) {
REQUIRE(reason == "Plugin returned error code 1 and output " + expected_reason + "\r");
}
}
}

#define TEST_CONFORMANCE(filename) \
TEST_CASE("conformance_check " filename, "[conformance]") { \
test_conformance(filename, bpf_conformance_test_result_t::TEST_RESULT_PASS); \
test_conformance(filename, bpf_conformance_test_result_t::TEST_RESULT_PASS, {}); \
}

// Some tests don't pass yet, but ought to in the future.
#define TEST_CONFORMANCE_FAIL(filename) \
TEST_CASE("conformance_check " filename, "[conformance]") { \
test_conformance(filename, bpf_conformance_test_result_t::TEST_RESULT_FAIL); \
// Any tests that fail verification are safe, but might prevent
// legitimate programs from being usable.
#define TEST_CONFORMANCE_VERIFICATION_FAILED(filename) \
TEST_CASE("conformance_check " filename, "[conformance]") { \
test_conformance(filename, bpf_conformance_test_result_t::TEST_RESULT_FAIL, "Verification failed"); \
}

// Any tests that return top are safe, but are not as precise as they
// could be and so may prevent legitimate programs from being usable.
#define TEST_CONFORMANCE_TOP(filename) \
TEST_CASE("conformance_check " filename, "[conformance]") { \
test_conformance(filename, bpf_conformance_test_result_t::TEST_RESULT_FAIL, "Couldn't determine r0 value"); \
}

TEST_CONFORMANCE("add.data")
TEST_CONFORMANCE("add64.data")
TEST_CONFORMANCE("alu-arith.data")
TEST_CONFORMANCE("alu-bit.data")
TEST_CONFORMANCE("alu64-arith.data")
TEST_CONFORMANCE_FAIL("alu64-bit.data")
TEST_CONFORMANCE_TOP("alu64-bit.data")
TEST_CONFORMANCE("arsh-reg.data")
TEST_CONFORMANCE("arsh.data")
TEST_CONFORMANCE("arsh32-high-shift.data")
TEST_CONFORMANCE_FAIL("arsh64.data")
TEST_CONFORMANCE_TOP("arsh64.data")
TEST_CONFORMANCE("be16-high.data")
TEST_CONFORMANCE("be16.data")
TEST_CONFORMANCE("be32-high.data")
TEST_CONFORMANCE("be32.data")
TEST_CONFORMANCE("be64.data")
TEST_CONFORMANCE_FAIL("call_unwind_fail.data")
TEST_CONFORMANCE("call_unwind_fail.data")
TEST_CONFORMANCE("div-by-zero-reg.data")
TEST_CONFORMANCE("div32-high-divisor.data")
TEST_CONFORMANCE("div32-imm.data")
TEST_CONFORMANCE("div32-reg.data")
TEST_CONFORMANCE("div64-by-zero-reg.data")
TEST_CONFORMANCE("div64-imm.data")
TEST_CONFORMANCE("div64-negative-imm.data")
TEST_CONFORMANCE("div64-negative-reg.data")
TEST_CONFORMANCE("div64-reg.data")
TEST_CONFORMANCE("exit-not-last.data")
TEST_CONFORMANCE("exit.data")
TEST_CONFORMANCE("jeq-imm.data")
TEST_CONFORMANCE("jeq-reg.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jeq32-imm.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jeq32-reg.data")
TEST_CONFORMANCE("jge-imm.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jge32-imm.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jge32-reg.data")
TEST_CONFORMANCE("jgt-imm.data")
TEST_CONFORMANCE("jgt-reg.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jgt32-imm.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jgt32-reg.data")
TEST_CONFORMANCE("jit-bounce.data")
TEST_CONFORMANCE("jle-imm.data")
TEST_CONFORMANCE("jle-reg.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jle32-imm.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jle32-reg.data")
TEST_CONFORMANCE("jlt-imm.data")
TEST_CONFORMANCE("jlt-reg.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jlt32-imm.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jlt32-reg.data")
TEST_CONFORMANCE("jne-reg.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jne32-imm.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jne32-reg.data")
TEST_CONFORMANCE("jset-imm.data")
TEST_CONFORMANCE("jset-reg.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jset32-imm.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jset32-reg.data")
TEST_CONFORMANCE("jsge-imm.data")
TEST_CONFORMANCE("jsge-reg.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jsge32-imm.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jsge32-reg.data")
TEST_CONFORMANCE("jsgt-imm.data")
TEST_CONFORMANCE("jsgt-reg.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jsgt32-imm.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jsgt32-reg.data")
TEST_CONFORMANCE("jsle-imm.data")
TEST_CONFORMANCE("jsle-reg.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jsle32-imm.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jsle32-reg.data")
TEST_CONFORMANCE("jslt-imm.data")
TEST_CONFORMANCE("jslt-reg.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jslt32-imm.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("jslt32-reg.data")
TEST_CONFORMANCE("lddw.data")
TEST_CONFORMANCE("lddw2.data")
TEST_CONFORMANCE_FAIL("ldxb-all.data")
TEST_CONFORMANCE_FAIL("ldxb.data")
TEST_CONFORMANCE_FAIL("ldxdw.data")
TEST_CONFORMANCE_FAIL("ldxh-all.data")
TEST_CONFORMANCE_FAIL("ldxh-all2.data")
TEST_CONFORMANCE_FAIL("ldxh-same-reg.data")
TEST_CONFORMANCE_FAIL("ldxh.data")
TEST_CONFORMANCE_FAIL("ldxw-all.data")
TEST_CONFORMANCE_FAIL("ldxw.data")
TEST_CONFORMANCE_TOP("ldxb-all.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("ldxb.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("ldxdw.data")
TEST_CONFORMANCE_TOP("ldxh-all.data")
TEST_CONFORMANCE_TOP("ldxh-all2.data")
TEST_CONFORMANCE_TOP("ldxh-same-reg.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("ldxh.data")
TEST_CONFORMANCE_TOP("ldxw-all.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("ldxw.data")
TEST_CONFORMANCE("le16.data")
TEST_CONFORMANCE("le32.data")
TEST_CONFORMANCE("le64.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("lock_add.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("lock_add32.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("lock_and.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("lock_and32.data")
TEST_CONFORMANCE("lock_cmpxchg.data") // FAILS!!
TEST_CONFORMANCE("lock_cmpxchg32.data") // FAILS!!
TEST_CONFORMANCE_VERIFICATION_FAILED("lock_or.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("lock_or32.data")
TEST_CONFORMANCE("lock_xchg.data")
TEST_CONFORMANCE("lock_xchg32.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("lock_xor.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("lock_xor32.data")
TEST_CONFORMANCE("lsh-reg.data")
TEST_CONFORMANCE_FAIL("mem-len.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("mem-len.data")
TEST_CONFORMANCE("mod-by-zero-reg.data")
TEST_CONFORMANCE("mod.data")
TEST_CONFORMANCE("mod32.data")
TEST_CONFORMANCE("mod64-by-zero-reg.data")
TEST_CONFORMANCE_FAIL("mod64.data")
TEST_CONFORMANCE_TOP("mod64.data")
TEST_CONFORMANCE("mov.data")
TEST_CONFORMANCE("mov64-sign-extend.data")
TEST_CONFORMANCE("mul32-imm.data")
TEST_CONFORMANCE("mul32-reg-overflow.data")
TEST_CONFORMANCE("mul32-reg.data")
TEST_CONFORMANCE("mul64-imm.data")
TEST_CONFORMANCE("mul64-reg.data")
TEST_CONFORMANCE("neg.data")
TEST_CONFORMANCE("neg64.data")
TEST_CONFORMANCE_FAIL("prime.data")
TEST_CONFORMANCE("prime.data") // FAILS!!
TEST_CONFORMANCE("rsh-reg.data")
TEST_CONFORMANCE("rsh32.data")
TEST_CONFORMANCE_FAIL("stack.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("stack.data")
TEST_CONFORMANCE("stb.data")
TEST_CONFORMANCE("stdw.data")
TEST_CONFORMANCE("sth.data")
TEST_CONFORMANCE("stw.data")
TEST_CONFORMANCE_FAIL("stxb-all.data")
TEST_CONFORMANCE_FAIL("stxb-all2.data")
TEST_CONFORMANCE_FAIL("stxb-chain.data")
TEST_CONFORMANCE_TOP("stxb-all.data")
TEST_CONFORMANCE_TOP("stxb-all2.data")
TEST_CONFORMANCE_TOP("stxb-chain.data")
TEST_CONFORMANCE("stxb.data")
TEST_CONFORMANCE_FAIL("stxdw.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("stxdw.data")
TEST_CONFORMANCE("stxh.data")
TEST_CONFORMANCE("stxw.data")
TEST_CONFORMANCE_FAIL("subnet.data")
TEST_CONFORMANCE_VERIFICATION_FAILED("subnet.data")

0 comments on commit 67cb4d1

Please sign in to comment.