Skip to content

Commit

Permalink
Created new test_runner.c and test_.py
Browse files Browse the repository at this point in the history
This is to try a different design for testing, the goals are to make the
test infrastructure a bit simpler, with clear stages for building and
running, and faster, by avoiding rebuilding lfs.c n-times.
  • Loading branch information
geky committed Apr 16, 2022
1 parent 40dba4a commit 56a9903
Show file tree
Hide file tree
Showing 4 changed files with 505 additions and 1 deletion.
33 changes: 32 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ override BUILDDIR := $(BUILDDIR)/
$(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \
$(BUILDDIR) \
$(BUILDDIR)bd \
$(BUILDDIR)runners \
$(BUILDDIR)tests))
endif

Expand All @@ -25,12 +26,19 @@ NM ?= nm
OBJDUMP ?= objdump
LCOV ?= lcov

SRC ?= $(wildcard *.c)
SRC ?= $(filter-out $(wildcard *.*.c),$(wildcard *.c))
OBJ := $(SRC:%.c=$(BUILDDIR)%.o)
DEP := $(SRC:%.c=$(BUILDDIR)%.d)
ASM := $(SRC:%.c=$(BUILDDIR)%.s)
CGI := $(SRC:%.c=$(BUILDDIR)%.ci)

TESTS ?= $(wildcard tests_/*.toml)
TEST_TSRC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) \
$(SRC:%.c=$(BUILDDIR)%.t.c) \
$(BUILDDIR)runners/test_runner.t.c
TEST_TASRC := $(TEST_TSRC:%.t.c=%.t.a.c)
TEST_TAOBJ := $(TEST_TASRC:%.t.a.c=%.t.a.o)

ifdef DEBUG
override CFLAGS += -O0
else
Expand Down Expand Up @@ -103,6 +111,9 @@ test:
test%: tests/test$$(firstword $$(subst \#, ,%)).toml
./scripts/test.py $@ $(TESTFLAGS)

.PHONY: test_
test_: $(BUILDDIR)runners/test_runner

.PHONY: code
code: $(OBJ)
./scripts/code.py $^ -S $(CODEFLAGS)
Expand Down Expand Up @@ -131,6 +142,7 @@ summary: $(BUILDDIR)lfs.csv
# rules
-include $(DEP)
.SUFFIXES:
.SECONDARY:

$(BUILDDIR)lfs: $(OBJ)
$(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
Expand All @@ -147,6 +159,9 @@ $(BUILDDIR)lfs.csv: $(OBJ) $(CGI)
./scripts/coverage.py $(BUILDDIR)tests/*.toml.info \
-q -m $@ $(COVERAGEFLAGS) -o $@)

$(BUILDDIR)runners/test_runner: $(TEST_TAOBJ)
$(CC) $(CFLAGS) $^ $(LFLAGS) -o $@

$(BUILDDIR)%.o: %.c
$(CC) -c -MMD $(CFLAGS) $< -o $@

Expand All @@ -160,14 +175,30 @@ $(BUILDDIR)%.s: %.c
$(BUILDDIR)%.ci: %.c | $(BUILDDIR)%.o
$(CC) -c -MMD -fcallgraph-info=su $(CFLAGS) $< -o $|

$(BUILDDIR)%.a.c: %.c
./scripts/explode_asserts.py $< -o $@

$(BUILDDIR)%.a.c: $(BUILDDIR)%.c
./scripts/explode_asserts.py $< -o $@

$(BUILDDIR)%.t.c: %.toml
./scripts/test_.py -c $< -o $@

$(BUILDDIR)%.t.c: %.c $(TESTS)
./scripts/test_.py -c $(TESTS) -s $< -o $@

# clean everything
.PHONY: clean
clean:
rm -f $(BUILDDIR)lfs
rm -f $(BUILDDIR)lfs.a
rm -f $(BUILDDIR)lfs.csv
rm -f $(BUILDDIR)runners/test_runner
rm -f $(OBJ)
rm -f $(CGI)
rm -f $(DEP)
rm -f $(ASM)
rm -f $(BUILDDIR)tests/*.toml.*
rm -f $(TEST_TSRC)
rm -f $(TEST_TASRC)
rm -f $(TEST_TAOBJ)
178 changes: 178 additions & 0 deletions runners/test_runner.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@

#include "runners/test_runner.h"
#include <getopt.h>


// disk geometries
struct test_geometry {
const char *name;
lfs_size_t read_size;
lfs_size_t prog_size;
lfs_size_t erase_size;
lfs_size_t erase_count;
};

const struct test_geometry test_geometries[] = {
// Made up geometries that works well for testing
{"small", 16, 16, 512, (1024*1024)/512},
{"medium", 16, 16, 4096, (1024*1024)/4096},
{"big", 16, 16, 32*1024, (1024*1024)/(32*1024)},
// EEPROM/NVRAM
{"eeprom", 1, 1, 512, (1024*1024)/512},
// SD/eMMC
{"emmc", 512, 512, 512, (1024*1024)/512},
// NOR flash
{"nor", 1, 1, 4096, (1024*1024)/4096},
// NAND flash
{"nand", 4096, 4096, 32*1024, (1024*1024)/(32*1024)},
};
const size_t test_geometry_count = (
sizeof(test_geometries) / sizeof(test_geometries[0]));


// option handling
enum opt_flags {
OPT_HELP = 'h',
OPT_LIST = 'l',
OPT_LIST_PATHS = 1,
OPT_LIST_DEFINES = 2,
OPT_LIST_GEOMETRIES = 3,
};

const struct option long_opts[] = {
{"help", no_argument, NULL, OPT_HELP},
{"list", no_argument, NULL, OPT_LIST},
{"list-paths", no_argument, NULL, OPT_LIST_PATHS},
{"list-defines", no_argument, NULL, OPT_LIST_DEFINES},
{"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES},
{NULL, 0, NULL, 0},
};

const char *const help_text[] = {
"Show this help message.",
"List test cases.",
"List the path for each test case.",
"List the defines for each test permutation.",
"List the disk geometries used for testing.",
};

int main(int argc, char **argv) {
bool list = false;
bool list_paths = false;
bool list_defines = false;
bool list_geometries = false;

// parse options
while (true) {
int index = 0;
int c = getopt_long(argc, argv, "hl", long_opts, &index);
switch (c) {
// generate help message
case OPT_HELP: {
printf("usage: %s [options] [test_case]\n", argv[0]);
printf("\n");

printf("options:\n");
size_t i = 0;
while (long_opts[i].name) {
size_t indent;
if (long_opts[i].val >= '0' && long_opts[i].val < 'z') {
printf(" -%c, --%-16s",
long_opts[i].val,
long_opts[i].name);
indent = 8+strlen(long_opts[i].name);
} else {
printf(" --%-20s", long_opts[i].name);
indent = 4+strlen(long_opts[i].name);
}

// a quick, hacky, byte-level method for text wrapping
size_t len = strlen(help_text[i]);
size_t j = 0;
if (indent < 24) {
printf("%.80s\n", &help_text[i][j]);
j += 80;
}

while (j < len) {
printf("%24s%.80s\n", "", &help_text[i][j]);
j += 80;
}

i += 1;
}

printf("\n");
exit(0);
}
// list flags
case OPT_LIST:
list = true;
break;
case OPT_LIST_PATHS:
list_paths = true;
break;
case OPT_LIST_DEFINES:
list_defines = true;
break;
case OPT_LIST_GEOMETRIES:
list_geometries = true;
break;
// done parsing
case -1:
goto getopt_done;
// unknown arg, getopt prints a message for us
default:
exit(-1);
}
}
getopt_done:

// what do we need to do?
if (list) {
printf("%-36s %-12s %-12s %7s %7s\n",
"id", "suite", "case", "type", "perms");
for (size_t i = 0; i < test_suite_count; i++) {
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
printf("%-36s %-12s %-12s %7s %7d\n",
test_suites[i]->cases[j]->id,
test_suites[i]->name,
test_suites[i]->cases[j]->name,
"n", // TODO
test_suites[i]->cases[j]->permutations);
}
}

} else if (list_paths) {
printf("%-36s %-36s\n", "id", "path");
for (size_t i = 0; i < test_suite_count; i++) {
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
printf("%-36s %-36s\n",
test_suites[i]->cases[j]->id,
test_suites[i]->cases[j]->path);
}
}
} else if (list_defines) {
// TODO
} else if (list_geometries) {
printf("%-12s %7s %7s %7s %7s %7s\n",
"name", "read", "prog", "erase", "count", "size");
for (size_t i = 0; i < test_geometry_count; i++) {
printf("%-12s %7d %7d %7d %7d %7d\n",
test_geometries[i].name,
test_geometries[i].read_size,
test_geometries[i].prog_size,
test_geometries[i].erase_size,
test_geometries[i].erase_count,
test_geometries[i].erase_size
* test_geometries[i].erase_count);
}
} else {
printf("remaining: ");
for (int i = optind; i < argc; i++) {
printf("%s ", argv[i]);
}
printf("\n");
}
}

26 changes: 26 additions & 0 deletions runners/test_runner.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#ifndef TEST_RUNNER_H
#define TEST_RUNNER_H

#include "lfs.h"


struct test_case {
const char *id;
const char *name;
const char *path;
uint32_t permutations;
void (*run)(struct lfs_config *cfg, uint32_t perm);
};

struct test_suite {
const char *id;
const char *name;
const char *path;
const struct test_case *const *cases;
size_t case_count;
};

extern const struct test_suite *test_suites[];
extern const size_t test_suite_count;

#endif

0 comments on commit 56a9903

Please sign in to comment.