Merge branch 'dev' into asm_cache

CHANGELOG.md

@@ -71,8 +71,16 @@ The table below shows which release corresponds to each branch, and what date th
 
 ## 4.14.0 (`dev`)
 
+- [#2360][2360] Add offline parameter for `search_by_hash` series function
+- [#2356][2356] Add local libc database provider for libcdb
+- [#2374][2374] libcdb.unstrip_libc: debug symbols are fetched only if not present
+- [#2327][2327] Add basic support to debug processes on Windows
 - [#2358][2358] Cache output of `asm()`
 
+[2360]: https://github.com/Gallopsled/pwntools/pull/2360
+[2356]: https://github.com/Gallopsled/pwntools/pull/2356
+[2374]: https://github.com/Gallopsled/pwntools/pull/2374
+[2327]: https://github.com/Gallopsled/pwntools/pull/2327
 [2358]: https://github.com/Gallopsled/pwntools/pull/2358
 
 ## 4.13.0 (`beta`)

docs/requirements.txt

@@ -8,7 +8,8 @@ isort
 mako>=1.0.0
 paramiko>=1.15.2
 pip>=6.0.8
-pyelftools>=0.2.3
+pyelftools>=0.29, <0.30; python_version<'3'
+pyelftools>=0.29; python_version>='3'
 pygments>=2.0
 pypandoc
 pyserial>=2.7

docs/source/index.rst

@@ -77,6 +77,7 @@ Each of the ``pwntools`` modules is documented here.
    update
    useragents
    util/*
+   windbg
 
 .. toctree::
    :hidden:

docs/source/windbg.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+    from pwn import *
+
+:mod:`pwnlib.windbg` --- Working with WinDbg
+======================================
+
+.. automodule:: pwnlib.windbg
+   :members:

extra/docker/base/Dockerfile

@@ -40,10 +40,10 @@ RUN apt-get update \
         patchelf \
     && locale-gen en_US.UTF-8 \
     && update-locale LANG=en_US.UTF-8 \
-    && PYTHONPATH=`echo /usr/share/python-wheels/pip-*.whl` python2.7 -m pip install --upgrade pip setuptools wheel \
-    && python2.7 -m pip install --upgrade pwntools \
-    && python3 -m pip install --upgrade pip \
-    && python3 -m pip install --upgrade pwntools \
+    && PYTHONPATH=`echo /usr/share/python-wheels/pip-*.whl` python2.7 -m pip install --no-cache-dir --upgrade pip setuptools wheel \
+    && python2.7 -m pip install --no-cache-dir --upgrade pwntools \
+    && python3 -m pip install --no-cache-dir --upgrade pip \
+    && python3 -m pip install --no-cache-dir --upgrade pwntools \
     && PWNLIB_NOTERM=1 pwn update \
     && useradd -m pwntools \
     && passwd --delete --unlock pwntools \

extra/docker/beta/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:stable
 
 USER root
-RUN python2.7 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools@beta \
-    && python3 -m pip install --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@beta
+RUN python2.7 -m pip install --no-cache-dir --upgrade git+https://github.com/Gallopsled/pwntools@beta \
+    && python3 -m pip install --no-cache-dir --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@beta
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/buster/Dockerfile

@@ -5,4 +5,4 @@ RUN apt-get -y dist-upgrade
 RUN apt-get -y install python3 python3-pip
 RUN apt-get -y install git wget unzip
 
-RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@dev
+RUN pip3 install --no-cache-dir --upgrade git+https://github.com/Gallopsled/pwntools@dev

extra/docker/stable/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:base
 
 USER root
-RUN python2.7 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable \
-    && python3 -m pip install --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@stable
+RUN python2.7 -m pip install --no-cache-dir --upgrade git+https://github.com/Gallopsled/pwntools@stable \
+    && python3 -m pip install --no-cache-dir --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@stable
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

pwnlib/__init__.py

@@ -36,6 +36,7 @@
     'util',
     'update',
     'version',
+    'windbg',
 ]
 
 from . import args

pwnlib/args.py

@@ -159,6 +159,11 @@ def STDERR(v):
     """Sends logging to ``stderr`` by default, instead of ``stdout``"""
     context.log_console = sys.stderr
 
+def LOCAL_LIBCDB(v):
+    """Sets path to local libc-database via ``context.local_libcdb``, e.g. 
+    ``LOCAL_LIBCDB='/path/to/libc-databse'``"""
+    context.local_libcdb = v
+
 hooks = {
     'LOG_LEVEL': LOG_LEVEL,
     'LOG_FILE': LOG_FILE,
@@ -170,6 +175,7 @@ def STDERR(v):
     'NOASLR': NOASLR,
     'NOPTRACE': NOPTRACE,
     'STDERR': STDERR,
+    'LOCAL_LIBCDB': LOCAL_LIBCDB,
 }
 
 def initialize():

pwnlib/context/__init__.py

@@ -360,6 +360,7 @@ class ContextType(object):
         'endian': 'little',
         'gdbinit': "",
         'kernel': None,
+        'local_libcdb': "/var/lib/libc-database",
         'log_level': logging.INFO,
         'log_file': _devnull(),
         'log_console': sys.stdout,
@@ -1071,6 +1072,32 @@ def log_console(self, stream):
             stream = open(stream, 'wt')
         return stream
 
+    @_validator
+    def local_libcdb(self, path):
+        """ 
+        Sets path to local libc-database, get more information for libc-database:
+        https://github.com/niklasb/libc-database
+
+        Works in :attr:`pwnlib.libcdb` when searching by local database provider.
+
+        The default value is ``/var/lib/libc-database``.
+
+        Sets `context.local_libcdb` to empty string or `None` will turn off local libc-database integration.
+
+        Examples:
+
+            >>> context.local_libcdb = pwnlib.data.elf.path
+            >>> context.local_libcdb = 'foobar'
+            Traceback (most recent call last):
+            ...
+            AttributeError: 'foobar' does not exist, please download libc-database first
+        """
+
+        if not os.path.isdir(path):
+            raise AttributeError("'%s' does not exist, please download libc-database first" % path)
+
+        return path
+
     @property
     def mask(self):
         return (1 << self.bits) - 1

pwnlib/libcdb.py

@@ -11,6 +11,7 @@
 
 from pwnlib.context import context
 from pwnlib.elf import ELF
+from pwnlib.filesystem.path import Path
 from pwnlib.log import getLogger
 from pwnlib.tubes.process import process
 from pwnlib.util.fiddling import enhex
@@ -126,9 +127,28 @@ def provider_local_system(hex_encoded_id, hash_type):
         return local_libc.data
     return None
 
-PROVIDERS = [provider_local_system, provider_libcdb, provider_libc_rip]
+# Offline search https://github.com/niklasb/libc-database for hash type
+def provider_local_database(hex_encoded_id, hash_type):
+    if not context.local_libcdb:
+        return None
+
+    localdb = Path(context.local_libcdb)
+    if not localdb.is_dir():
+        return None
+
+    log.debug("Searching local libc database, %s: %s", hash_type, hex_encoded_id)
+    for libc_path in localdb.rglob("*.so"):
+        if hex_encoded_id == HASHES[hash_type](libc_path):
+            return read(libc_path)
+
+    return None
 
-def search_by_hash(hex_encoded_id, hash_type='build_id', unstrip=True):
+PROVIDERS = {
+    "offline": [provider_local_system, provider_local_database],
+    "online": [provider_libcdb, provider_libc_rip]
+}
+
+def search_by_hash(hex_encoded_id, hash_type='build_id', unstrip=True, offline_only=False):
     assert hash_type in HASHES, hash_type
 
     # Ensure that the libcdb cache directory exists
@@ -140,8 +160,12 @@ def search_by_hash(hex_encoded_id, hash_type='build_id', unstrip=True):
     if cache is None:
         return None
 
+    providers = PROVIDERS["offline"]
+    if not offline_only:
+        providers += PROVIDERS["online"]
+
     # Run through all available libc database providers to see if we have a match.
-    for provider in PROVIDERS:
+    for provider in providers:
         data = provider(hex_encoded_id, hash_type)
         if data and data.startswith(b'\x7FELF'):
             break
@@ -274,6 +298,10 @@ def unstrip_libc(filename):
         log.warn_once('Given libc does not have a buildid. Cannot look for debuginfo to unstrip.')
         return False
 
+    if libc.debuginfo:
+        log.debug('Given libc already contains debug information. Skipping unstrip.')
+        return True
+
     log.debug('Trying debuginfod servers: %r', DEBUGINFOD_SERVERS)
 
     for server_url in DEBUGINFOD_SERVERS:
@@ -586,7 +614,7 @@ def search_by_symbol_offsets(symbols, select_index=None, unstrip=True, return_as
     selected_libc = _handle_multiple_matching_libcs(matching_libcs)
     return search_by_build_id(selected_libc['buildid'], unstrip=unstrip)
 
-def search_by_build_id(hex_encoded_id, unstrip=True):
+def search_by_build_id(hex_encoded_id, unstrip=True, offline_only=False):
     """
     Given a hex-encoded Build ID, attempt to download a matching libc from libcdb.
 
@@ -595,6 +623,10 @@ def search_by_build_id(hex_encoded_id, unstrip=True):
             Hex-encoded Build ID (e.g. 'ABCDEF...') of the library
         unstrip(bool):
             Try to fetch debug info for the libc and apply it to the downloaded file.
+        offline_only(bool):
+            Both offline and online providers are used by default. When pass
+            `offline_only=True`, libcdb enable an exclusive offline search mode,
+            which will disable online providers.
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -610,9 +642,9 @@ def search_by_build_id(hex_encoded_id, unstrip=True):
         >>> hex(ELF(filename).symbols.read)
         '0xeef40'
     """
-    return search_by_hash(hex_encoded_id, 'build_id', unstrip)
+    return search_by_hash(hex_encoded_id, 'build_id', unstrip, offline_only)
 
-def search_by_md5(hex_encoded_id, unstrip=True):
+def search_by_md5(hex_encoded_id, unstrip=True, offline_only=False):
     """
     Given a hex-encoded md5sum, attempt to download a matching libc from libcdb.
 
@@ -621,6 +653,10 @@ def search_by_md5(hex_encoded_id, unstrip=True):
             Hex-encoded md5sum (e.g. 'ABCDEF...') of the library
         unstrip(bool):
             Try to fetch debug info for the libc and apply it to the downloaded file.
+        offline_only(bool):
+            Both offline and online providers are used by default. When pass
+            `offline_only=True`, libcdb enable an exclusive offline search mode,
+            which will disable online providers.
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -636,9 +672,9 @@ def search_by_md5(hex_encoded_id, unstrip=True):
         >>> hex(ELF(filename).symbols.read)
         '0xeef40'
     """
-    return search_by_hash(hex_encoded_id, 'md5', unstrip)
+    return search_by_hash(hex_encoded_id, 'md5', unstrip, offline_only)
 
-def search_by_sha1(hex_encoded_id, unstrip=True):
+def search_by_sha1(hex_encoded_id, unstrip=True, offline_only=False):
     """
     Given a hex-encoded sha1, attempt to download a matching libc from libcdb.
 
@@ -647,6 +683,10 @@ def search_by_sha1(hex_encoded_id, unstrip=True):
             Hex-encoded sha1sum (e.g. 'ABCDEF...') of the library
         unstrip(bool):
             Try to fetch debug info for the libc and apply it to the downloaded file.
+        offline_only(bool):
+            Both offline and online providers are used by default. When pass
+            `offline_only=True`, libcdb enable an exclusive offline search mode,
+            which will disable online providers.
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -662,10 +702,9 @@ def search_by_sha1(hex_encoded_id, unstrip=True):
         >>> hex(ELF(filename).symbols.read)
         '0xeef40'
     """
-    return search_by_hash(hex_encoded_id, 'sha1', unstrip)
-
+    return search_by_hash(hex_encoded_id, 'sha1', unstrip, offline_only)
 
-def search_by_sha256(hex_encoded_id, unstrip=True):
+def search_by_sha256(hex_encoded_id, unstrip=True, offline_only=False):
     """
     Given a hex-encoded sha256, attempt to download a matching libc from libcdb.
 
@@ -674,6 +713,10 @@ def search_by_sha256(hex_encoded_id, unstrip=True):
             Hex-encoded sha256sum (e.g. 'ABCDEF...') of the library
         unstrip(bool):
             Try to fetch debug info for the libc and apply it to the downloaded file.
+        offline_only(bool):
+            Both offline and online providers are used by default. When pass
+            `offline_only=True`, libcdb enable an exclusive offline search mode,
+            which will disable online providers.
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -689,7 +732,7 @@ def search_by_sha256(hex_encoded_id, unstrip=True):
         >>> hex(ELF(filename).symbols.read)
         '0xeef40'
     """
-    return search_by_hash(hex_encoded_id, 'sha256', unstrip)
+    return search_by_hash(hex_encoded_id, 'sha256', unstrip, offline_only)
 
 
 

pwnlib/tubes/process.py

@@ -802,15 +802,6 @@ def close(self):
         # First check if we are already dead
         self.poll()
 
-        # close file descriptors
-        for fd in [self.proc.stdin, self.proc.stdout, self.proc.stderr]:
-            if fd is not None:
-                try:
-                    fd.close()
-                except IOError as e:
-                    if e.errno != errno.EPIPE and e.errno != errno.EINVAL:
-                        raise
-
         if not self._stop_noticed:
             try:
                 self.proc.kill()
@@ -820,6 +811,15 @@ def close(self):
             except OSError:
                 pass
 
+        # close file descriptors
+        for fd in [self.proc.stdin, self.proc.stdout, self.proc.stderr]:
+            if fd is not None:
+                try:
+                    fd.close()
+                except IOError as e:
+                    if e.errno != errno.EPIPE and e.errno != errno.EINVAL:
+                        raise
+
 
     def fileno(self):
         if not self.connected():