Tweaked/fixed a number of small runner things after a bit of use

- Added support for negative numbers in the leb16 encoding with an optional 'w' prefix. - Changed prettyasserts.py rule to .a.c => .c, allowing other .a.c files in the future. - Updated .gitignore with missing generated files (tags, .csv). - Removed suite-namespacing of test symbols, these are no longer needed. - Changed test define overrides to have higher priority than explicit defines encoded in test ids. So: ./runners/bench_runner bench_dir_open:0f1g12gg2b8c8dgg4e0 -DREAD_SIZE=16 Behaves as expected. Otherwise it's not easy to experiment with known failing test cases. - Fixed issue where the -b flag ignored explicit test/bench ids.
littlefs-project · Dec 17, 2022 · 801cf27 · 801cf27
1 parent 1f37eb5
commit 801cf27
Show file tree

Hide file tree

Showing 6 changed files with 170 additions and 113 deletions.
diff --git a/.gitignore b/.gitignore
@@ -4,19 +4,31 @@
 *.a
 *.ci
 *.csv
-*.t.c
-*.b.c
-*.a.c
+*.t.*
+*.b.*
 *.gcno
 *.gcda
 *.perf
+lfs
+liblfs.a
 
 # Testing things
-blocks/
-lfs
-test.c
-tests/*.toml.*
-scripts/__pycache__
-.gdb_history
 runners/test_runner
 runners/bench_runner
+lfs.code.csv
+lfs.data.csv
+lfs.stack.csv
+lfs.structs.csv
+lfs.cov.csv
+lfs.perf.csv
+lfs.perfbd.csv
+lfs.test.csv
+lfs.bench.csv
+
+# Misc
+tags
+.gdb_history
+scripts/__pycache__
+
+# Historical, probably should remove at some point
+tests/*.toml.*
diff --git a/Makefile b/Makefile
@@ -28,43 +28,43 @@ VALGRIND ?= valgrind
 GDB		 ?= gdb
 PERF	 ?= perf
 
-SRC  ?= $(filter-out $(wildcard *.*.c),$(wildcard *.c))
+SRC  ?= $(filter-out $(wildcard *.t.* *.b.*),$(wildcard *.c))
 OBJ  := $(SRC:%.c=$(BUILDDIR)/%.o)
 DEP  := $(SRC:%.c=$(BUILDDIR)/%.d)
 ASM  := $(SRC:%.c=$(BUILDDIR)/%.s)
 CI   := $(SRC:%.c=$(BUILDDIR)/%.ci)
-GCDA := $(SRC:%.c=$(BUILDDIR)/%.t.a.gcda)
+GCDA := $(SRC:%.c=$(BUILDDIR)/%.t.gcda)
 
 TESTS ?= $(wildcard tests/*.toml)
 TEST_SRC ?= $(SRC) \
-		$(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \
+		$(filter-out $(wildcard bd/*.t.* bd/*.b.*),$(wildcard bd/*.c)) \
 		runners/test_runner.c
 TEST_RUNNER ?= $(BUILDDIR)/runners/test_runner
-TEST_TC    := $(TESTS:%.toml=$(BUILDDIR)/%.t.c) \
-		$(TEST_SRC:%.c=$(BUILDDIR)/%.t.c)
-TEST_TAC   := $(TEST_TC:%.t.c=%.t.a.c)
-TEST_OBJ   := $(TEST_TAC:%.t.a.c=%.t.a.o)
-TEST_DEP   := $(TEST_TAC:%.t.a.c=%.t.a.d)
-TEST_CI	   := $(TEST_TAC:%.t.a.c=%.t.a.ci)
-TEST_GCNO  := $(TEST_TAC:%.t.a.c=%.t.a.gcno)
-TEST_GCDA  := $(TEST_TAC:%.t.a.c=%.t.a.gcda)
+TEST_A     := $(TESTS:%.toml=$(BUILDDIR)/%.t.a.c) \
+		$(TEST_SRC:%.c=$(BUILDDIR)/%.t.a.c)
+TEST_C     := $(TEST_A:%.t.a.c=%.t.c)
+TEST_OBJ   := $(TEST_C:%.t.c=%.t.o)
+TEST_DEP   := $(TEST_C:%.t.c=%.t.d)
+TEST_CI	   := $(TEST_C:%.t.c=%.t.ci)
+TEST_GCNO  := $(TEST_C:%.t.c=%.t.gcno)
+TEST_GCDA  := $(TEST_C:%.t.c=%.t.gcda)
 TEST_PERF  := $(TEST_RUNNER:%=%.perf)
 TEST_TRACE := $(TEST_RUNNER:%=%.trace)
 TEST_CSV   := $(TEST_RUNNER:%=%.csv)
 
 BENCHES ?= $(wildcard benches/*.toml)
 BENCH_SRC ?= $(SRC) \
-		$(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \
+		$(filter-out $(wildcard bd/*.t.* bd/*.b.*),$(wildcard bd/*.c)) \
 		runners/bench_runner.c
 BENCH_RUNNER ?= $(BUILDDIR)/runners/bench_runner
-BENCH_BC    := $(BENCHES:%.toml=$(BUILDDIR)/%.b.c) \
-		$(BENCH_SRC:%.c=$(BUILDDIR)/%.b.c)
-BENCH_BAC   := $(BENCH_BC:%.b.c=%.b.a.c)
-BENCH_OBJ   := $(BENCH_BAC:%.b.a.c=%.b.a.o)
-BENCH_DEP   := $(BENCH_BAC:%.b.a.c=%.b.a.d)
-BENCH_CI    := $(BENCH_BAC:%.b.a.c=%.b.a.ci)
-BENCH_GCNO  := $(BENCH_BAC:%.b.a.c=%.b.a.gcno)
-BENCH_GCDA  := $(BENCH_BAC:%.b.a.c=%.b.a.gcda)
+BENCH_A     := $(BENCHES:%.toml=$(BUILDDIR)/%.b.a.c) \
+		$(BENCH_SRC:%.c=$(BUILDDIR)/%.b.a.c)
+BENCH_C     := $(BENCH_A:%.b.a.c=%.b.c)
+BENCH_OBJ   := $(BENCH_C:%.b.c=%.b.o)
+BENCH_DEP   := $(BENCH_C:%.b.c=%.b.d)
+BENCH_CI    := $(BENCH_C:%.b.c=%.b.ci)
+BENCH_GCNO  := $(BENCH_C:%.b.c=%.b.gcno)
+BENCH_GCDA  := $(BENCH_C:%.b.c=%.b.gcda)
 BENCH_PERF  := $(BENCH_RUNNER:%=%.perf)
 BENCH_TRACE := $(BENCH_RUNNER:%=%.trace)
 BENCH_CSV   := $(BENCH_RUNNER:%=%.csv)
@@ -517,22 +517,22 @@ $(BUILDDIR)/%.o $(BUILDDIR)/%.ci: %.c
 $(BUILDDIR)/%.s: %.c
 	$(CC) -S $(CFLAGS) $< -o $@
 
-$(BUILDDIR)/%.a.c: %.c
+$(BUILDDIR)/%.c: %.a.c
 	./scripts/prettyasserts.py -p LFS_ASSERT $< -o $@
 
-$(BUILDDIR)/%.a.c: $(BUILDDIR)/%.c
+$(BUILDDIR)/%.c: $(BUILDDIR)/%.a.c
 	./scripts/prettyasserts.py -p LFS_ASSERT $< -o $@
 
-$(BUILDDIR)/%.t.c: %.toml
+$(BUILDDIR)/%.t.a.c: %.toml
 	./scripts/test.py -c $< $(TESTCFLAGS) -o $@
 
-$(BUILDDIR)/%.t.c: %.c $(TESTS)
+$(BUILDDIR)/%.t.a.c: %.c $(TESTS)
 	./scripts/test.py -c $(TESTS) -s $< $(TESTCFLAGS) -o $@
 
-$(BUILDDIR)/%.b.c: %.toml
+$(BUILDDIR)/%.b.a.c: %.toml
 	./scripts/bench.py -c $< $(BENCHCFLAGS) -o $@
 
-$(BUILDDIR)/%.b.c: %.c $(BENCHES)
+$(BUILDDIR)/%.b.a.c: %.c $(BENCHES)
 	./scripts/bench.py -c $(BENCHES) -s $< $(BENCHCFLAGS) -o $@
 
 ## Clean everything
@@ -554,8 +554,8 @@ clean:
 	rm -f $(ASM)
 	rm -f $(CI)
 	rm -f $(TEST_RUNNER)
-	rm -f $(TEST_TC)
-	rm -f $(TEST_TAC)
+	rm -f $(TEST_A)
+	rm -f $(TEST_C)
 	rm -f $(TEST_OBJ)
 	rm -f $(TEST_DEP)
 	rm -f $(TEST_CI)
@@ -565,8 +565,8 @@ clean:
 	rm -f $(TEST_TRACE)
 	rm -f $(TEST_CSV)
 	rm -f $(BENCH_RUNNER)
-	rm -f $(BENCH_BC)
-	rm -f $(BENCH_BAC)
+	rm -f $(BENCH_A)
+	rm -f $(BENCH_C)
 	rm -f $(BENCH_OBJ)
 	rm -f $(BENCH_DEP)
 	rm -f $(BENCH_CI)

diff --git a/runners/bench_runner.c b/runners/bench_runner.c
@@ -52,6 +52,12 @@ void *mappend(void **p,
 
 // a quick self-terminating text-safe varint scheme
 static void leb16_print(uintmax_t x) {
+    // allow 'w' to indicate negative numbers
+    if ((intmax_t)x < 0) {
+        printf("w");
+        x = -x;
+    }
+
     while (true) {
         char nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0);
         printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10);
@@ -63,7 +69,17 @@ static void leb16_print(uintmax_t x) {
 }
 
 static uintmax_t leb16_parse(const char *s, char **tail) {
+    bool neg = false;
     uintmax_t x = 0;
+    if (tail) {
+        *tail = (char*)s;
+    }
+
+    if (s[0] == 'w') {
+        neg = true;
+        s = s+1;
+    }
+
     size_t i = 0;
     while (true) {
         uintmax_t nibble = s[i];
@@ -73,23 +89,21 @@ static uintmax_t leb16_parse(const char *s, char **tail) {
             nibble = nibble - 'a' + 10;
         } else {
             // invalid?
-            if (tail) {
-                *tail = (char*)s;
-            }
             return 0;
         }
 
         x |= (nibble & 0xf) << (4*i);
         i += 1;
         if (!(nibble & 0x10)) {
+            s = s + i;
             break;
         }
     }
 
     if (tail) {
-        *tail = (char*)s + i;
+        *tail = (char*)s;
     }
-    return x;
+    return neg ? -x : x;
 }
 
 
@@ -145,8 +159,8 @@ intmax_t bench_define_lit(void *data) {
     BENCH_IMPLICIT_DEFINES
 #undef BENCH_DEF
 
-#define BENCH_DEFINE_MAP_EXPLICIT    0
-#define BENCH_DEFINE_MAP_OVERRIDE    1
+#define BENCH_DEFINE_MAP_OVERRIDE    0
+#define BENCH_DEFINE_MAP_EXPLICIT    1
 #define BENCH_DEFINE_MAP_PERMUTATION 2
 #define BENCH_DEFINE_MAP_GEOMETRY    3
 #define BENCH_DEFINE_MAP_IMPLICIT    4
@@ -681,11 +695,18 @@ static void case_forperm(
             const struct bench_suite *suite,
             const struct bench_case *case_),
         void *data) {
+    // explicit permutation?
     if (defines) {
         bench_define_explicit(defines, define_count);
-        bench_define_flush();
 
-        cb(data, suite, case_);
+        for (size_t v = 0; v < bench_override_define_permutations; v++) {
+            // define override permutation
+            bench_define_override(v);
+            bench_define_flush();
+
+            cb(data, suite, case_);
+        }
+
         return;
     }
 

diff --git a/runners/test_runner.c b/runners/test_runner.c
@@ -52,6 +52,12 @@ void *mappend(void **p,
 
 // a quick self-terminating text-safe varint scheme
 static void leb16_print(uintmax_t x) {
+    // allow 'w' to indicate negative numbers
+    if ((intmax_t)x < 0) {
+        printf("w");
+        x = -x;
+    }
+
     while (true) {
         char nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0);
         printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10);
@@ -63,7 +69,17 @@ static void leb16_print(uintmax_t x) {
 }
 
 static uintmax_t leb16_parse(const char *s, char **tail) {
+    bool neg = false;
     uintmax_t x = 0;
+    if (tail) {
+        *tail = (char*)s;
+    }
+
+    if (s[0] == 'w') {
+        neg = true;
+        s = s+1;
+    }
+
     size_t i = 0;
     while (true) {
         uintmax_t nibble = s[i];
@@ -73,23 +89,21 @@ static uintmax_t leb16_parse(const char *s, char **tail) {
             nibble = nibble - 'a' + 10;
         } else {
             // invalid?
-            if (tail) {
-                *tail = (char*)s;
-            }
             return 0;
         }
 
         x |= (nibble & 0xf) << (4*i);
         i += 1;
         if (!(nibble & 0x10)) {
+            s = s + i;
             break;
         }
     }
 
     if (tail) {
-        *tail = (char*)s + i;
+        *tail = (char*)s;
     }
-    return x;
+    return neg ? -x : x;
 }
 
 
@@ -158,8 +172,8 @@ intmax_t test_define_lit(void *data) {
     TEST_IMPLICIT_DEFINES
 #undef TEST_DEF
 
-#define TEST_DEFINE_MAP_EXPLICIT    0
-#define TEST_DEFINE_MAP_OVERRIDE    1
+#define TEST_DEFINE_MAP_OVERRIDE    0
+#define TEST_DEFINE_MAP_EXPLICIT    1
 #define TEST_DEFINE_MAP_PERMUTATION 2
 #define TEST_DEFINE_MAP_GEOMETRY    3
 #define TEST_DEFINE_MAP_IMPLICIT    4
@@ -675,26 +689,34 @@ static void case_forperm(
             const struct test_case *case_,
             const test_powerloss_t *powerloss),
         void *data) {
+    // explicit permutation?
     if (defines) {
         test_define_explicit(defines, define_count);
-        test_define_flush();
 
-        if (cycles) {
-            cb(data, suite, case_, &(test_powerloss_t){
-                    .run=run_powerloss_cycles,
-                    .cycles=cycles,
-                    .cycle_count=cycle_count});
-        } else {
-            for (size_t p = 0; p < test_powerloss_count; p++) {
-                // skip non-reentrant tests when powerloss testing
-                if (test_powerlosses[p].run != run_powerloss_none
-                        && !(case_->flags & TEST_REENTRANT)) {
-                    continue;
-                }
+        for (size_t v = 0; v < test_override_define_permutations; v++) {
+            // define override permutation
+            test_define_override(v);
+            test_define_flush();
+
+            // explicit powerloss cycles?
+            if (cycles) {
+                cb(data, suite, case_, &(test_powerloss_t){
+                        .run=run_powerloss_cycles,
+                        .cycles=cycles,
+                        .cycle_count=cycle_count});
+            } else {
+                for (size_t p = 0; p < test_powerloss_count; p++) {
+                    // skip non-reentrant tests when powerloss testing
+                    if (test_powerlosses[p].run != run_powerloss_none
+                            && !(case_->flags & TEST_REENTRANT)) {
+                        continue;
+                    }
 
-                cb(data, suite, case_, &test_powerlosses[p]);
+                    cb(data, suite, case_, &test_powerlosses[p]);
+                }
             }
         }
+
         return;
     }