feat(kcov): add code coverage support for tests + fuzzers

.github/workflows/check.yml

@@ -52,6 +52,44 @@ jobs:
       - name: test
         run: zig build test
 
+  kcov_test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{matrix.os}}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: setup-zig
+        uses: goto-bus-stop/setup-zig@v1
+        with:
+          version: 0.13.0
+
+      - name: Set up dependencies
+        run: sudo apt-get update
+
+      - name: Install kcov
+        run: |
+          sudo apt-get install -y binutils-dev libssl-dev libcurl4-openssl-dev zlib1g-dev libdw-dev libiberty-dev
+          git clone https://github.com/SimonKagstrom/kcov.git
+          cd kcov
+          mkdir build
+          cd build
+          cmake ..
+          make
+          sudo make install
+
+      - name: Run kcov
+        run: |
+          bash scripts/kcov_test.sh
+
+      - name: Print coverage report
+        run: |
+          python parse_kcov.py kcov-output/test/coverage.json
+
   benchmarks:
     strategy:
       matrix:

.gitignore

@@ -4,6 +4,7 @@ zig-out/
 data/
 
 index_storage/
+kcov-output
 
 /gossip-dumps
 

.vscode/launch.json

@@ -0,0 +1,14 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "type": "lldb",
+        "request": "launch",
+        "name": "Debug Tests",
+        "program": "${workspaceFolder}/zig-out/bin/test",
+        "args": [],
+        "cwd": "${workspaceFolder}",
+        "preLaunchTask": "zig build"
+      }
+    ]
+  }

.vscode/tasks.json

@@ -0,0 +1,13 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "zig build",
+            "type": "shell",
+            "command": "zig",
+            "args": [
+                "build"
+            ],
+        }
+    ]
+}

docs/fuzzing.md

@@ -0,0 +1,79 @@
+# fuzzing documentation
+
+supported components: 
+- gossip 
+- accountsdb 
+
+main code paths:
+- `fuzz.zig` <- main entrypoint
+- `gossip/fuzz_table.zig`
+- `gossip/fuzz_service.zig`
+- `accountsdb/fuzz.zig`
+- `scripts/` <- kcov + fuzz bash scripts
+
+kcov will give you coverage information on what was and was not fuzzed
+- [https://github.com/SimonKagstrom/kcov](https://github.com/SimonKagstrom/kcov)
+
+*note:* view the full script for helpful install instructions
+
+![](imgs/2024-07-10-09-39-25.png)
+
+![](imgs/2024-07-10-09-39-57.png)
+
+## gossip 
+
+### service
+
+gossip supports two types of fuzzing 
+- *option1*: sending fuzz packets to an arbitrary endpoint
+- *option2*: sending fuzz packets directly to a sig client
+
+![](imgs/2024-07-10-09-33-37.png)
+
+example commands: 
+```bash
+#                                 <seed>  <n_messages>  <to_endpoint>    (option1)
+./zig-out/bin/fuzz gossip_service 10      4_000         127.0.0.1:8001
+
+#                                 <seed>  <n_messages> (option2)
+./zig-out/bin/fuzz gossip_service 10      4_000
+
+#                                 (run with random seed for inf)
+./zig-out/bin/fuzz gossip_service 
+```
+
+### table
+
+you can also fuzz the `GossipTable` (see `src/gossip/table.zig`) for reads/writes/trims/remove-old-labels using the following:
+
+example commands: 
+```bash
+#                               <seed>  <n_messages>
+./zig-out/bin/fuzz gossip_table 10      4_000
+
+#                               (run with random seed for inf)
+./zig-out/bin/fuzz gossip_table 
+```
+
+## accounts-db
+
+the fuzzer does a few things
+- runs the manager loop which flushes/cleans/shrinks/deletes account files in one thread
+- another thread reads/writes accounts and verifies the data read is correct
+
+the goal was to makes sure accounts-db was thread-safe for parallel access while also flushing/cleaning/purging
+
+example command: 
+```bash
+#                             <seed>  <n_messages>
+./zig-out/bin/fuzz accountsdb 10      4_000
+#                         (run with random seed for inf)
+./zig-out/bin/fuzz accountsdb
+```
+
+## running with kcov 
+
+commands to run:
+- `bash scripts/kcov_fuzz_gossip_service.sh`
+- `bash scripts/kcov_fuzz_gossip_table.sh`
+- `bash scripts/kcov_fuzz_accountsdb.sh`

parse_kcov.py

@@ -0,0 +1,36 @@
+import json 
+
+# read path from cli
+import sys
+if len(sys.argv) != 2:
+    print("Usage: python parse_kcov.py <path_to_coverage.json>")
+    sys.exit(1)
+coverage_path = sys.argv[1]
+
+with open(coverage_path, "r") as f:
+    coverage = json.load(f)
+
+max_path_length = max(len(file_info["file"]) for file_info in coverage["files"])
+
+# order by coverage percentage
+coverage["files"].sort(key=lambda x: float(x["percent_covered"]), reverse=False)
+
+output = ""
+for file_info in coverage["files"]:
+    path = file_info["file"]
+    path = path.split("sig/")[2]
+    file_coverage = float(file_info["percent_covered"])
+
+    # Determine the color based on the coverage percentage
+    if file_coverage < 50:
+        color = "\033[91m"  # Red
+    elif file_coverage < 75:
+        color = "\033[93m"  # Yellow
+    else:
+        color = "\033[92m"  # Green
+
+    # Reset color
+    reset = "\033[0m"
+    output += f"{color}{path:<{max_path_length}} --- {file_coverage:>10}%{reset}\n"
+
+print(output)

scripts/kcov_fuzz_accountsdb.sh

@@ -0,0 +1,31 @@
+# to install kcov follow the instructions at:
+#   https://github.com/SimonKagstrom/kcov/blob/master/INSTALL.md
+# to build on mac the following should work:
+#   ```
+#   cd /path/to/kcov/
+#   mkdir build
+#   cd build
+#   cmake ..
+#   make
+#   make install
+#   export PATH=$PATH:/path/to/kcov/build/src
+#   ```
+
+echo "=> Clearing kcov-output directory" 
+rm -rf kcov-output 
+mkdir kcov-output 
+
+echo "=> Building Sig" 
+zig build 
+
+echo "=> Running kcov on accountsdb" 
+kcov \
+    --include-pattern=src/accountsdb/ \
+    # not sure why this is necessary with --include-pattern but it is
+    --exclude-pattern=$HOME/.cache \
+    kcov-output/ \
+    ./zig-out/bin/fuzz accountsdb
+
+# open report
+echo "=> Opening kcov-output/index.html" 
+open kcov-output/index.html

scripts/kcov_fuzz_gossip.sh

@@ -0,0 +1,30 @@
+# to install kcov follow the instructions at:
+#   https://github.com/SimonKagstrom/kcov/blob/master/INSTALL.md
+# to build on mac the following should work:
+#   ```
+#   cd /path/to/kcov/
+#   mkdir build
+#   cd build
+#   cmake ..
+#   make
+#   make install
+#   export PATH=$PATH:/path/to/kcov/build/src
+#   ```
+
+echo "=> Clearing kcov-output directory" 
+rm -rf kcov-output 
+mkdir kcov-output 
+
+echo "=> Building Sig" 
+zig build 
+
+echo "=> Running kcov on gossip spy" 
+kcov \
+    --include-pattern=src/gossip/ \
+    --exclude-pattern=$HOME/.cache \
+    kcov-output/ \
+    ./zig-out/bin/fuzz gossip_service 19 50_000
+
+# open report
+echo "=> Opening kcov-output/index.html" 
+open kcov-output/index.html

scripts/kcov_fuzz_gossip_table.sh

@@ -0,0 +1,30 @@
+# to install kcov follow the instructions at:
+#   https://github.com/SimonKagstrom/kcov/blob/master/INSTALL.md
+# to build on mac the following should work:
+#   ```
+#   cd /path/to/kcov/
+#   mkdir build
+#   cd build
+#   cmake ..
+#   make
+#   make install
+#   export PATH=$PATH:/path/to/kcov/build/src
+#   ```
+
+echo "=> Clearing kcov-output directory" 
+rm -rf kcov-output 
+mkdir kcov-output 
+
+echo "=> Building Sig" 
+zig build 
+
+echo "=> Running kcov on gossip spy" 
+kcov \
+    --include-pattern=src/gossip/ \
+    --exclude-pattern=$HOME/.cache \
+    kcov-output/ \
+    ./zig-out/bin/fuzz gossip_table 19 50_000
+
+# open report
+echo "=> Opening kcov-output/index.html" 
+open kcov-output/index.html

scripts/kcov_test.sh

@@ -0,0 +1,29 @@
+# to install kcov follow the instructions at:
+#   https://github.com/SimonKagstrom/kcov/blob/master/INSTALL.md
+# to build on mac the following should work:
+#   ```
+#   cd /path/to/kcov/
+#   mkdir build
+#   cd build
+#   cmake ..
+#   make
+#   make install
+#   export PATH=$PATH:/path/to/kcov/build/src
+#   ```
+
+echo "=> Cleaning up" 
+rm -rf kcov-output 
+mkdir kcov-output 
+
+echo "=> Building Sig" 
+zig build 
+
+echo "=> Running kcov on tests" 
+kcov \
+    --include-pattern=src/ \
+    --exclude-pattern=$HOME/.cache \
+    kcov-output \
+    ./zig-out/bin/test
+
+echo "=> Opening kcov-output/index.html" 
+open kcov-output/index.html || echo "=> Failed to open kcov-output/index.html"