Skip to content

Commit a0483bd

Browse files
Buristanigormunkin
Buristan
authored and
igormunkin
committed
test: introduce module for C tests
We need an instrument to write tests in plain C for LuaJIT, to be able: * easily test LuaC API * test patches without usage of plain Lua * write unit tests * startup LuaJIT with custom memory allocator, to test some GC issues * maybe, in future, use custom hashing function to test a behavior of LuaJIT tables and so on. The <test.c> module serves to achieve these goals without too fancy features. It's functionality inspired by CMocka API [1], but only TAP14 [2] protocol is supported (Version of TAP set to 13 to be compatible with old TAP13 harnesses). The group of unit tests is declared like the following: | void *t_state = NULL; | const struct test_unit tgroup[] = { | test_unit_def(test_base), | test_unit_def(test_subtest), | }; | return test_run_group(tgroup, t_state); `test_run_group()` runs the whole group of tests, returns `TEST_EXIT_SUCCESS` or `TEST_EXIT_FAILURE`. If a similar group is declared inside unit test, this group will be considered as a subtest. This library provides an API similar to glibc (3) `assert()` to use inside unit tests. `assert_[true,false]()` are useful for condition checks and `assert_{type}_[not_,]_equal()` are useful for value comparisons. If some assertion fails diagnostic is set, all test considered as failing and finished via `longjmp()`, so these assertions can be used inside custom subroutines. Also, this module provides ability to skip one test or all tests, mark test as todo, bail out all tests. Just use `return skip("reason")`, `skip_all("reason")` or `todo("reason")` for early return. They should be used only in the test body to make skipping clear. `skip_all("reason")` may be used both for the parent test and for a subtest. `bail_out("reason")` prints an error message and exits the process. As a part of this commit, tarantool-c-tests directory is created with the corresponding CMakeLists.txt file to build this test library. Tests to be rewritten in C with this library in the next commit and placed as unit tests are: * lj-49-bad-lightuserdata.test.lua * misclib-getmetrics-capi.test.lua * misclib-sysprof-capi.test.lua For now the tarantool-c-tests target just build the test library without new tests to run. The library itself is tested via some primitive tests for `ok` case, `skip` and `todo` directives. The TAP13 format is tested via prove, that we are using for running our tests. TAP14 format is compatible with TAP13, so there are no other tests required. Also, .c_test suffix is added to the <.gitignore>. [1]: https://github.com/clibs/cmocka [2]: https://testanything.org/tap-version-14-specification.html Part of tarantool/tarantool#7900 Reviewed-by: Maxim Kokryashkin <m.kokryashkin@tarantool.org> Reviewed-by: Sergey Bronnikov <sergeyb@tarantool.org> Signed-off-by: Igor Munkin <imun@tarantool.org>
1 parent 451d4ec commit a0483bd

File tree

7 files changed

+614
-0
lines changed

7 files changed

+614
-0
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,4 @@ install_manifest.txt
2424
luajit-parse-memprof
2525
luajit-parse-sysprof
2626
luajit.pc
27+
*.c_test

‎test/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,14 @@ separate_arguments(LUAJIT_TEST_COMMAND)
4848
add_subdirectory(LuaJIT-tests)
4949
add_subdirectory(PUC-Rio-Lua-5.1-tests)
5050
add_subdirectory(lua-Harness-tests)
51+
add_subdirectory(tarantool-c-tests)
5152
add_subdirectory(tarantool-tests)
5253

5354
add_custom_target(${PROJECT_NAME}-test DEPENDS
5455
LuaJIT-tests
5556
PUC-Rio-Lua-5.1-tests
5657
lua-Harness-tests
58+
tarantool-c-tests
5759
tarantool-tests
5860
)
5961

‎test/tarantool-c-tests/CMakeLists.txt

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
find_program(PROVE prove)
2+
if(NOT PROVE)
3+
message(WARNING "`prove' is not found, so tarantool-c-tests target is not generated")
4+
return()
5+
endif()
6+
7+
set(C_TEST_SUFFIX .c_test)
8+
set(C_TEST_FLAGS --failures --shuffle)
9+
10+
if(CMAKE_VERBOSE_MAKEFILE)
11+
list(APPEND C_TEST_FLAGS --verbose)
12+
endif()
13+
14+
# Build libtest.
15+
16+
set(TEST_LIB_NAME "test")
17+
add_library(libtest STATIC EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/test.c)
18+
target_include_directories(libtest PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
19+
set_target_properties(libtest PROPERTIES
20+
COMPILE_FLAGS "-Wall -Wextra"
21+
OUTPUT_NAME "${TEST_LIB_NAME}"
22+
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
23+
)
24+
25+
# TARGET_C_FLAGS is required here to be sure that headers like
26+
# lj_arch.h in compiled test are consistent with the LuaJIT library
27+
# to link.
28+
AppendFlags(TESTS_C_FLAGS ${TARGET_C_FLAGS})
29+
30+
set(CTEST_SRC_SUFFIX ".test.c")
31+
file(GLOB tests "${CMAKE_CURRENT_SOURCE_DIR}/*${CTEST_SRC_SUFFIX}")
32+
foreach(test_source ${tests})
33+
# Get test name without suffix. Needed to set OUTPUT_NAME.
34+
get_filename_component(exe ${test_source} NAME_WE)
35+
add_executable(${exe} EXCLUDE_FROM_ALL ${test_source})
36+
target_include_directories(${exe} PRIVATE
37+
${CMAKE_CURRENT_SOURCE_DIR}
38+
${LUAJIT_SOURCE_DIR}
39+
)
40+
set_target_properties(${exe} PROPERTIES
41+
COMPILE_FLAGS "${TESTS_C_FLAGS}"
42+
OUTPUT_NAME "${exe}${C_TEST_SUFFIX}"
43+
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
44+
)
45+
target_link_libraries(${exe} libtest ${LUAJIT_LIBRARY})
46+
LIST(APPEND TESTS_COMPILED ${exe})
47+
endforeach()
48+
49+
add_custom_target(tarantool-c-tests
50+
DEPENDS libluajit libtest ${TESTS_COMPILED}
51+
)
52+
53+
add_custom_command(TARGET tarantool-c-tests
54+
COMMENT "Running Tarantool C tests"
55+
COMMAND
56+
${PROVE}
57+
${CMAKE_CURRENT_BINARY_DIR}
58+
--ext ${C_TEST_SUFFIX}
59+
--jobs ${CMAKE_BUILD_PARALLEL_LEVEL}
60+
# Report any TAP parse errors, if any, since test module is
61+
# maintained by us.
62+
--parse
63+
${C_TEST_FLAGS}
64+
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
65+
)
66+

‎test/tarantool-c-tests/README.md

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Tarantool C tests
2+
This directory contains C tests for Tarantool's fork of LuaJIT.
3+
4+
They should test C API, some unit functionality, etc..
5+
6+
## How to start
7+
8+
The group of unit tests is declared like the following:
9+
```c
10+
void *t_state = NULL;
11+
const struct test_unit tgroup[] = {
12+
test_unit_def(test_base),
13+
test_unit_def(test_simple),
14+
};
15+
return test_run_group(tgroup, t_state);
16+
```
17+
18+
Each of the unit tests has the following definition:
19+
20+
```c
21+
static int test_base(void *test_state)
22+
```
23+
24+
## Assertions
25+
26+
The following assertions are defined in <test.h> to be used instead default
27+
glibc `assert()`:
28+
* `assert_true(cond)` -- check that condition `cond` is true.
29+
* `assert_false(cond)` -- check that condition `cond` is false.
30+
* `assert_ptr{_not}_equal(a, b)` -- check that pointer `a` is {not} equal to
31+
the `b`.
32+
* `assert_int{_not}_equal(a, b)` -- check that `int` variable `a` is {not}
33+
equal to the `b`.
34+
* `assert_sizet{_not}_equal(a, b)` -- check that `size_t` variable `a` is {not}
35+
equal to the `b`.
36+
* `assert_double{_not}_equal(a, b)` -- check that two doubles are {not}
37+
**exactly** equal.
38+
39+
## Directives
40+
41+
The following directives are supported to stop unit test or group of tests
42+
earlier:
43+
* `skip("reason")` -- skip the current test.
44+
* `skip_all("reason")` -- skip the current group of tests.
45+
* `todo("reason")` -- skip the current test marking as TODO.
46+
* `bail_out("reason")` -- exit the entire process due to some emergency.

‎test/tarantool-c-tests/test.c

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
#include "test.h"
2+
3+
/*
4+
* Test module, based on TAP 14 specification [1].
5+
* [1]: https://testanything.org/tap-version-14-specification.html
6+
*/
7+
8+
/* Need for `PATH_MAX` in diagnostic definition. */
9+
#include <limits.h>
10+
#include <setjmp.h>
11+
#include <stdarg.h>
12+
/* Need for `strchr()` in diagnostic parsing. */
13+
#include <string.h>
14+
15+
/*
16+
* Test level: 0 for the parent test, >0 for any subtests.
17+
*/
18+
static int level = -1;
19+
20+
/*
21+
* The last diagnostic data to be used in the YAML Diagnostic
22+
* block.
23+
*
24+
* Contains filename, line number and failed expression or assert
25+
* name and "got" and "expected" fields. All entries are separated
26+
* by \n.
27+
* The longest field is filename here, so PATH_MAX * 3 as
28+
* the diagnostic string length should be enough.
29+
*
30+
* The first \0 means the end of diagnostic data.
31+
*
32+
* As far as `strchr()` searches until \0, all previous entries
33+
* are suppressed by the last one. If the first byte is \0 --
34+
* diagnostic is empty.
35+
*/
36+
#define TEST_DIAG_DATA_MAX (PATH_MAX * 3)
37+
char test_diag_buf[TEST_DIAG_DATA_MAX] = {0};
38+
39+
const char *skip_reason = NULL;
40+
const char *todo_reason = NULL;
41+
42+
/* Indent for the TAP. 4 spaces is default for subtest. */
43+
static void indent(void)
44+
{
45+
int i;
46+
for (i = 0; i < level; i++)
47+
printf(" ");
48+
}
49+
50+
void test_message(const char *fmt, ...)
51+
{
52+
va_list ap;
53+
indent();
54+
va_start(ap, fmt);
55+
vprintf(fmt, ap);
56+
printf("\n");
57+
va_end(ap);
58+
}
59+
60+
static void test_print_tap_version(void)
61+
{
62+
/*
63+
* Since several TAP13 parsers in popular usage treat
64+
* a repeated Version declaration as an error, even if the
65+
* Version is indented, Subtests _should not_ include a
66+
* Version, if TAP13 Harness compatibility is
67+
* desirable [1].
68+
*/
69+
if (level == 0)
70+
test_message("TAP version %d", TAP_VERSION);
71+
}
72+
73+
static void test_start_comment(const char *t_name)
74+
{
75+
if (level > -1)
76+
/*
77+
* Inform about starting subtest, easier for
78+
* humans to read.
79+
* Subtest with a name must be terminated by a
80+
* Test Point with a matching Description [1].
81+
*/
82+
test_comment("Subtest: %s", t_name);
83+
}
84+
85+
void _test_print_skip_all(const char *group_name, const char *reason)
86+
{
87+
test_start_comment(group_name);
88+
/*
89+
* XXX: This test isn't started yet, so set indent level
90+
* manually.
91+
*/
92+
level++;
93+
test_print_tap_version();
94+
/*
95+
* XXX: `SKIP_DIRECTIVE` is not necessary here according
96+
* to the TAP14 specification [1], but some harnesses may
97+
* fail to parse the output without it.
98+
*/
99+
test_message("1..0" SKIP_DIRECTIVE "%s", reason);
100+
level--;
101+
}
102+
103+
/* Just inform TAP parser how many tests we want to run. */
104+
static void test_plan(size_t planned)
105+
{
106+
test_message("1..%lu", planned);
107+
}
108+
109+
/* Human-readable output how many tests/subtests are failed. */
110+
static void test_finish(size_t planned, size_t failed)
111+
{
112+
const char *t_type = level == 0 ? "tests" : "subtests";
113+
if (failed > 0)
114+
test_comment("Failed %lu %s out of %lu",
115+
failed, t_type, planned);
116+
fflush(stdout);
117+
}
118+
119+
void test_set_skip_reason(const char *reason)
120+
{
121+
skip_reason = reason;
122+
}
123+
124+
void test_set_todo_reason(const char *reason)
125+
{
126+
todo_reason = reason;
127+
}
128+
129+
void test_save_diag_data(const char *fmt, ...)
130+
{
131+
va_list ap;
132+
va_start(ap, fmt);
133+
vsnprintf(test_diag_buf, TEST_DIAG_DATA_MAX, fmt, ap);
134+
va_end(ap);
135+
}
136+
137+
static void test_clear_diag_data(void)
138+
{
139+
/*
140+
* Limit buffer with zero byte to show that there is no
141+
* any entry.
142+
*/
143+
test_diag_buf[0] = '\0';
144+
}
145+
146+
static int test_diagnostic_is_set(void)
147+
{
148+
return test_diag_buf[0] != '\0';
149+
}
150+
151+
/*
152+
* Parse the last diagnostic data entry and print it in YAML
153+
* format with the corresponding additional half-indent in TAP
154+
* (2 spaces).
155+
* Clear diagnostic message to be sure that it's printed once.
156+
* XXX: \n separators are changed to \0 during parsing and
157+
* printing output for convenience in usage.
158+
*/
159+
static void test_diagnostic(void)
160+
{
161+
test_message(" ---");
162+
char *ent = test_diag_buf;
163+
char *ent_end = NULL;
164+
while ((ent_end = strchr(ent, '\n')) != NULL) {
165+
char *next_ent = ent_end + 1;
166+
/*
167+
* Limit string with with the zero byte for
168+
* formatted output. Anyway, don't need this \n
169+
* anymore.
170+
*/
171+
*ent_end = '\0';
172+
test_message(" %s", ent);
173+
ent = next_ent;
174+
}
175+
test_message(" ...");
176+
test_clear_diag_data();
177+
}
178+
179+
static jmp_buf test_run_env;
180+
181+
TEST_NORET void _test_exit(int status)
182+
{
183+
longjmp(test_run_env, status);
184+
}
185+
186+
static int test_run(const struct test_unit *test, size_t test_number,
187+
void *test_state)
188+
{
189+
int status = TEST_EXIT_SUCCESS;
190+
/*
191+
* Run unit test. Diagnostic in case of failure setup by
192+
* helpers assert macros defined in the header.
193+
*/
194+
int jmp_status;
195+
if ((jmp_status = setjmp(test_run_env)) == 0) {
196+
if (test->f(test_state) != TEST_EXIT_SUCCESS)
197+
status = TEST_EXIT_FAILURE;
198+
} else {
199+
status = jmp_status - TEST_JMP_STATUS_SHIFT;
200+
}
201+
const char *result = status == TEST_EXIT_SUCCESS ? "ok" : "not ok";
202+
203+
/*
204+
* Format suffix of the test message for SKIP or TODO
205+
* directives.
206+
*/
207+
#define SUFFIX_SZ 1024
208+
char suffix[SUFFIX_SZ] = {0};
209+
if (skip_reason) {
210+
snprintf(suffix, SUFFIX_SZ, SKIP_DIRECTIVE "%s", skip_reason);
211+
skip_reason = NULL;
212+
} else if (todo_reason) {
213+
/* Prevent count this test as failed. */
214+
status = TEST_EXIT_SUCCESS;
215+
snprintf(suffix, SUFFIX_SZ, TODO_DIRECTIVE "%s", todo_reason);
216+
todo_reason = NULL;
217+
}
218+
#undef SUFFIX_SZ
219+
220+
test_message("%s %lu - %s%s", result, test_number, test->name,
221+
suffix);
222+
223+
if (status && test_diagnostic_is_set())
224+
test_diagnostic();
225+
return status;
226+
}
227+
228+
int _test_run_group(const char *group_name, const struct test_unit tests[],
229+
size_t n_tests, void *test_state)
230+
{
231+
test_start_comment(group_name);
232+
233+
level++;
234+
test_print_tap_version();
235+
236+
test_plan(n_tests);
237+
238+
size_t n_failed = 0;
239+
240+
size_t i;
241+
for (i = 0; i < n_tests; i++) {
242+
size_t test_number = i + 1;
243+
/* Return 1 on failure, 0 on success. */
244+
n_failed += test_run(&tests[i], test_number, test_state);
245+
}
246+
247+
test_finish(n_tests, n_failed);
248+
249+
level--;
250+
return n_failed > 0 ? TEST_EXIT_FAILURE : TEST_EXIT_SUCCESS;
251+
}

0 commit comments

Comments
 (0)