Tests framework (#375)

* Add prototype of unit tests for pwndbg

* Add test for pwndbg [filter]

* Fix isort, e2e tests, add pytest requirement

* Add comment about not handling exceptions for unittests

* Fixes after rebase

* Fix test_loads_without_crashing

* e2e tests: no colors & loading pwndbg tests

* Fix isort

* Add example of no file loaded test

* Move tests to unit_tests, add binary, add memory tests

* Isort fixes

* Move from e2e/unit tests to tests

* Add info about tests to DEVELOPING.md

* Fix tests

* review fixes

* commands filtering test: check for contents, not for equality

* Add tests launcher bash script

* Change tests launcher name from unittests to pytests

* Cleanup; better test file paths

* Add theme param to disable colors

* Better test_loads

* Skip some tests locally that can run on travis

* Fix test_loads according to travis

* Fix travis tests

.travis.yml

@@ -14,9 +14,9 @@ install:
   - pip install -r requirements.txt
   - sudo ./setup.sh
 script:
-  - futurize --all-imports --stage1 --print-function --write --unicode-literals pwndbg
-  - git diff-index --quiet HEAD -- pwndbg
-  - isort --check-only --diff --recursive pwndbg
-  - nosetests ./tests/
+  - futurize --all-imports --stage1 --print-function --write --unicode-literals pwndbg tests
+  - git diff-index --quiet HEAD -- pwndbg tests
+  - isort --check-only --diff --recursive pwndbg tests
+  - PWNDBG_TRAVIS_TEST_RUN=1 ./tests.sh
   - python2.7 -m py_compile ida_script.py $(git ls-files 'pwndbg/*.py')
   - python3   -m py_compile               $(git ls-files 'pwndbg/*.py')

DEVELOPING.md

@@ -30,3 +30,9 @@ Feel free to update the list below!
 
 * We would like to add proper tests for pwndbg - see tests framework PR if you want to help on that.
 
+# Testing
+
+Our tests are written using [pytest](https://docs.pytest.org/en/latest/). It uses some magic so that Python's `assert` can be used for asserting things in tests and it injects dependencies which are called fixtures, into test functions.
+
+The fixtures should be defined in [tests/conftest.py](tests/conftest.py). If you need help with writing tests, feel free to reach out on gitub issues/pr or on our irc channel on freenode.
+

pwndbg/color/__init__.py

@@ -5,10 +5,13 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import os
 import re
 
 import pwndbg.memoize
 
+from . import theme as theme
+
 NORMAL         = "\x1b[0m"
 BLACK          = "\x1b[30m"
 RED            = "\x1b[31m"
@@ -53,14 +56,23 @@ def bold(x): return colorize(x, BOLD)
 def underline(x): return colorize(x, UNDERLINE)
 def colorize(x, color): return color + terminateWith(str(x), color) + NORMAL
 
+
+disable_colors = theme.Parameter('disable-colors', bool(os.environ.get('PWNDBG_DISABLE_COLORS')), 'whether to color the output or not')
+
+
 @pwndbg.memoize.reset_on_stop
 def generateColorFunctionInner(old, new):
     def wrapper(text):
         return new(old(text))
+
     return wrapper
 
 def generateColorFunction(config):
     function = lambda x: x
+
+    if disable_colors:
+        return function
+
     for color in config.split(','):
         function = generateColorFunctionInner(function, globals()[color.lower().replace('-', '_')])
     return function

pwndbg/color/syntax_highlight.py

@@ -8,6 +8,7 @@
 import re
 
 import pwndbg.config
+from pwndbg.color import disable_colors
 from pwndbg.color import message
 from pwndbg.color import theme
 
@@ -40,7 +41,7 @@ def check_style():
 
 def syntax_highlight(code, filename='.asm'):
     # No syntax highlight if pygment is not installed
-    if not pygments:
+    if not pygments or disable_colors:
         return code
 
     filename = os.path.basename(filename)

pwndbg/commands/misc.py

@@ -50,11 +50,29 @@ def errno(err):
 
 @_pwndbg.commands.ArgparsedCommand(parser)
 def pwndbg(filter_pattern):
+    for name, docs in list_and_filter_commands(filter_pattern):
+        print("%-20s %s" % (name, docs))
+
+
+@_pwndbg.commands.ParsedCommand
+def distance(a, b):
+    '''Print the distance between the two arguments'''
+    a = int(a) & _arch.ptrmask
+    b = int(b) & _arch.ptrmask
+
+    distance = (b-a)
+
+    print("%#x->%#x is %#x bytes (%#x words)" % (a, b, distance, distance // _arch.ptrsize))
+
+
+def list_and_filter_commands(filter_str):
     sorted_commands = list(_pwndbg.commands.commands)
     sorted_commands.sort(key=lambda x: x.__name__)
 
-    if filter_pattern:
-        filter_pattern = filter_pattern.lower()
+    if filter_str:
+        filter_str = filter_str.lower()
+
+    results = []
 
     for c in sorted_commands:
         name = c.__name__
@@ -63,16 +81,7 @@ def pwndbg(filter_pattern):
         if docs: docs = docs.strip()
         if docs: docs = docs.splitlines()[0]
 
-        if not filter_pattern or filter_pattern in name.lower() or (docs and filter_pattern in docs.lower()):
-            print("%-20s %s" % (name, docs))
-
-
-@_pwndbg.commands.ParsedCommand
-def distance(a, b):
-    '''Print the distance between the two arguments'''
-    a = int(a) & _arch.ptrmask
-    b = int(b) & _arch.ptrmask
-
-    distance = (b-a)
+        if not filter_str or filter_str in name.lower() or (docs and filter_str in docs.lower()):
+            results.append((name, docs))
 
-    print("%#x->%#x is %#x bytes (%#x words)" % (a, b, distance, distance // _arch.ptrsize))
+    return results

pwndbg/commands/telescope.py

@@ -123,7 +123,7 @@ def telescope(address=None, count=telescope_lines, to_string=False):
     return result
 
 
-parser = argparse.ArgumentParser(description='dereferences on stack data with specified count and offset')
+parser = argparse.ArgumentParser(description='dereferences on stack data with specified count and offset.')
 parser.add_argument('count', nargs='?', default=8, type=int,
                     help='number of element to dump')
 parser.add_argument('offset', nargs='?', default=0, type=int,

pwndbg/commands/windbg.py

@@ -8,12 +8,11 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from builtins import str
-
 import argparse
 import codecs
 import math
 import sys
+from builtins import str
 
 import gdb
 

pwndbg/exception.py

@@ -47,6 +47,14 @@ def handle(name='Error'):
         - ``set exception-verbose on`` enables stack traces.
         - ``set exception-debugger on`` enables the post-mortem debugger.
     """
+
+    # This is for unit tests so they fail on exceptions instead of displaying them.
+    if getattr(sys, '_pwndbg_unittest_run', False) is True:
+        E, V, T = sys.exc_info()
+        e = E(V)
+        e.__traceback__ = T
+        raise e
+
     # Display the error
     if debug or verbose:
         exception_msg = traceback.format_exc()

pwndbg/prompt.py

@@ -10,6 +10,7 @@
 import pwndbg.events
 import pwndbg.gdbutils
 import pwndbg.memoize
+from pwndbg.color import disable_colors
 from pwndbg.color import message
 
 funcs_list_str = ', '.join(message.notice('$' + f.name) for f in pwndbg.gdbutils.functions.functions)
@@ -42,12 +43,15 @@ def prompt_hook_on_stop(*a):
     pwndbg.commands.context.context()
 
 
-@pwndbg.config.Trigger([message.config_prompt_color])
+@pwndbg.config.Trigger([message.config_prompt_color, disable_colors])
 def set_prompt():
     prompt = "pwndbg> "
-    prompt = "\x02" + prompt + "\x01"  # STX + prompt + SOH
-    prompt = message.prompt(prompt)
-    prompt = "\x01" + prompt + "\x02"  # SOH + prompt + STX
+
+    if not disable_colors:
+        prompt = "\x02" + prompt + "\x01"  # STX + prompt + SOH
+        prompt = message.prompt(prompt)
+        prompt = "\x01" + prompt + "\x02"  # SOH + prompt + STX
+
     gdb.execute('set prompt %s' % prompt)
 
 

pytests_launcher.py

@@ -0,0 +1,30 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import os
+import pytest
+import sys
+print(sys.argv)
+
+sys._pwndbg_unittest_run = True
+
+TESTS_PATH = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'tests'
+)
+
+# If you want to debug tests locally, add '--pdb' here
+args = [TESTS_PATH, '-vvv', '-s', '--showlocals', '--color=yes']
+
+print('Launching pytest with args: %s' % args)
+
+return_code = pytest.main(args)
+
+if return_code != 0:
+    print('-' * 80)
+    print('If you want to debug tests locally, modify tests_launcher.py and add --pdb to its args')
+    print('-' * 80)
+
+sys.exit(return_code)

requirements.txt

@@ -11,3 +11,4 @@ unicorn>=1.0.0
 pygments
 https://github.com/aquynh/capstone/archive/next.zip#subdirectory=bindings/python
 enum34
+pytest

tests.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+PWNDBG_DISABLE_COLORS=1 gdb --silent --nx --nh --command gdbinit.py --command pytests_launcher.py

tests/__init__.py

@@ -1,2 +1,6 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from . import binaries

tests/binaries/__init__.py

@@ -0,0 +1,13 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+
+from . import old_bash
+
+path = os.path.dirname(__file__)
+
+def get(x):
+    return os.path.join(path, x)

tests/binaries/makefile

@@ -0,0 +1,36 @@
+CC              =       gcc
+DEBUG           =       1
+CFLAGS         +=       -Wall
+SOURCES         =       $(wildcard *.c)
+COMPILED        =       $(SOURCES:.c=.o)
+LINKED          =       $(SOURCES:.c=.out)
+LDFLAGS         =
+EXTRA_FLAGS	    =
+
+ifeq ($(TARGET), x86)
+CFLAGS          +=      -m32
+endif
+
+ifeq ($(DEBUG), 1)
+CFLAGS          +=      -DDEBUG=1 -ggdb -O0
+else
+CFLAGS          +=      -O1
+endif
+
+
+.PHONY : all clean
+
+all: $(LINKED)
+
+
+%.out : %.c
+	@echo "[+] Building '$@'"
+	@$(CC) $(CFLAGS) $(EXTRA_FLAGS) -o $@ $? $(LDFLAGS)
+
+clean :
+	@echo "[+] Cleaning stuff"
+	@rm -f $(COMPILED) $(LINKED)
+
+
+reference-binary.out: EXTRA_FLAGS := -Dexample=1
+

tests/binaries/old_bash/__init__.py

@@ -0,0 +1,11 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+
+path = os.path.dirname(__file__)
+
+def get(x):
+    return os.path.join(path, x)

tests/binaries/reference-binary.c

@@ -0,0 +1,8 @@
+#include <stdio.h>
+
+void foo() {}
+
+int main() {
+    puts("Hello world");
+    foo();
+}

tests/conftest.py

@@ -0,0 +1,22 @@
+"""
+This file should consist of global test fixtures.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import gdb
+import pytest
+
+
+@pytest.fixture
+def entry_binary():
+    """
+    Returns function that launches given binary with 'entry' command
+    """
+    def _entry_binary(name):
+        gdb.execute('file ./tests/binaries/' + name)
+        gdb.execute('entry')
+
+    return _entry_binary