forked from commaai/panda
-
Notifications
You must be signed in to change notification settings - Fork 1
/
test_mutation.py
executable file
·92 lines (79 loc) · 2.61 KB
/
test_mutation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
#!/usr/bin/env python3
import os
import glob
import pytest
import shutil
import subprocess
import tempfile
import hashlib
import random
HERE = os.path.abspath(os.path.dirname(__file__))
ROOT = os.path.join(HERE, "../../")
IGNORED_PATHS = (
'board/obj',
'board/jungle',
'board/stm32h7/inc',
'board/stm32fx/inc',
'board/fake_stm.h',
# bootstub only files
'board/flasher.h',
'board/bootstub.c',
'board/bootstub_declarations.h',
'board/stm32fx/llflash.h'
)
mutations = [
# default
(None, None, False),
# F4 only
("board/stm32fx/llbxcan.h", "s/1U/1/g", True),
# H7 only
("board/stm32h7/llfdcan.h", "s/return ret;/if (true) { return ret; } else { return false; }/g", True),
# general safety
("board/safety/safety_toyota.h", "s/is_lkas_msg =.*;/is_lkas_msg = addr == 1 || addr == 2;/g", True),
]
patterns = [
# misra-c2012-13.3
"$a void test(int tmp) { int tmp2 = tmp++ + 2; if (tmp2) {;}}",
# misra-c2012-13.4
"$a int test(int x, int y) { return (x=2) && (y=2); }",
# misra-c2012-13.5
"$a void test(int tmp) { if (true && tmp++) {;} }",
# misra-c2012-13.6
"$a void test(int tmp) { if (sizeof(tmp++)) {;} }",
# misra-c2012-14.1
"$a void test(float len) { for (float j = 0; j < len; j++) {;} }",
# misra-c2012-14.4
"$a void test(int len) { if (len - 8) {;} }",
# misra-c2012-16.4
r"$a void test(int temp) {switch (temp) { case 1: ; }}\n",
# misra-c2012-17.8
"$a void test(int cnt) { for (cnt=0;;cnt++) {;} }",
# misra-c2012-20.4
r"$a #define auto 1\n",
# misra-c2012-20.5
r"$a #define TEST 1\n#undef TEST\n",
]
all_files = glob.glob('board/**', root_dir=ROOT, recursive=True)
files = [f for f in all_files if f.endswith(('.c', '.h')) and not f.startswith(IGNORED_PATHS)]
assert len(files) > 70, all(d in files for d in ('board/main.c', 'board/stm32fx/llbxcan.h', 'board/stm32h7/llfdcan.h', 'board/safety/safety_toyota.h'))
for p in patterns:
mutations.append((random.choice(files), p, True))
@pytest.mark.parametrize("fn, patch, should_fail", mutations)
def test_misra_mutation(fn, patch, should_fail):
key = hashlib.md5((str(fn) + str(patch)).encode()).hexdigest()
tmp = os.path.join(tempfile.gettempdir(), key)
if os.path.exists(tmp):
shutil.rmtree(tmp)
shutil.copytree(ROOT, tmp)
# apply patch
if fn is not None:
r = os.system(f"cd {tmp} && sed -i '{patch}' {fn}")
assert r == 0
# run test
env = {'SKIP_BUILD': '1'} | os.environ.copy()
r = subprocess.run("tests/misra/test_misra.sh", cwd=tmp, shell=True, env=env)
failed = r.returncode != 0
assert failed == should_fail
shutil.rmtree(tmp)
if __name__ == "__main__":
pytest.main([__file__, "-n 8"])