Merge branch 'dev' into gdb-api

CHANGELOG.md

@@ -58,8 +58,10 @@ The table below shows which release corresponds to each branch, and what date th
 
 ## 4.5.0 (`dev`)
 
+- [#1261][1261] Misc `run_in_new_terminal` improvements (notably gdb terminated by default)
 - [#1695][1695] Allow using GDB Python API
 
+[1261]: https://github.com/Gallopsled/pwntools/pull/1261
 [1695]: https://github.com/Gallopsled/pwntools/pull/1695
 
 ## 4.4.0 (`beta`)
@@ -84,6 +86,7 @@ The table below shows which release corresponds to each branch, and what date th
 - [#1688][1688] Add `__setattr__` and `__call__` interfaces to `ROP` for setting registers
 - [#1692][1692] Remove python2 shebangs where appropriate
 - [#1703][1703] Update libcdb buildid offsets for amd64 and i386
+- [#1704][1704] Try https://libc.rip/ for libcdb lookup
 
 [1541]: https://github.com/Gallopsled/pwntools/pull/1541
 [1602]: https://github.com/Gallopsled/pwntools/pull/1602
@@ -103,6 +106,7 @@ The table below shows which release corresponds to each branch, and what date th
 [1688]: https://github.com/Gallopsled/pwntools/pull/1688
 [1692]: https://github.com/Gallopsled/pwntools/pull/1692
 [1703]: https://github.com/Gallopsled/pwntools/pull/1703
+[1704]: https://github.com/Gallopsled/pwntools/pull/1704
 
 ## 4.3.0 (`stable`)
 

docs/source/tubes/ssh.rst

@@ -14,6 +14,10 @@
       :members: kill, poll, interactive
       :show-inheritance:
 
+   .. autoclass:: pwnlib.tubes.ssh.ssh_process
+      :members:
+      :show-inheritance:
+
    .. autoclass:: pwnlib.tubes.ssh.ssh_connecter()
       :show-inheritance:
 

pwn/toplevel.py

@@ -4,6 +4,7 @@
 import math
 import operator
 import os
+import platform
 import re
 import requests
 import socks

pwnlib/data/syscalls/generate.py

@@ -75,8 +75,8 @@
 
         # The argument is not a register.  It is a string value, and we
         # are expecting a string value
-        elif name in can_pushstr and isinstance(arg, (bytes, six.text_type)):
-            if not isinstance(arg, bytes):
+        elif name in can_pushstr and isinstance(arg, (six.binary_type, six.text_type)):
+            if isinstance(arg, six.text_type):
                 arg = arg.encode('utf-8')
             string_arguments[name] = arg
 
@@ -150,6 +150,8 @@ def fix_bad_arg_names(func, arg):
         return 'length'
     if arg.name == 'repr':
         return 'repr_'
+    if arg.name == 'from':
+        return 'from_'
 
     if func.name == 'open' and arg.name == 'vararg':
         return 'mode'

pwnlib/libcdb.py

@@ -7,6 +7,7 @@
 import codecs
 import json
 import os
+import requests
 import tempfile
 
 from six.moves import urllib
@@ -25,6 +26,64 @@
 
 HASHES = ['build_id', 'sha1', 'sha256', 'md5']
 
+# https://gitlab.com/libcdb/libcdb wasn't updated after 2019,
+# but still is a massive database of older libc binaries.
+def provider_libcdb(hex_encoded_id, hash_type):
+    # Build the URL using the requested hash type
+    url_base = "https://gitlab.com/libcdb/libcdb/raw/master/hashes/%s/" % hash_type
+    url      = urllib.parse.urljoin(url_base, hex_encoded_id)
+
+    data     = b""
+    log.debug("Downloading data from LibcDB: %s", url)
+    try:
+        while not data.startswith(b'\x7fELF'):
+            data = wget(url, timeout=20)
+
+            if not data:
+                log.warn_once("Could not fetch libc for %s %s from libcdb", hash_type, hex_encoded_id)
+                break
+
+            # GitLab serves up symlinks with
+            if data.startswith(b'..'):
+                url = os.path.dirname(url) + '/'
+                url = urllib.parse.urljoin(url.encode('utf-8'), data)
+    except requests.RequestException as e:
+        log.warn_once("Failed to fetch libc for %s %s from libcdb: %s", hash_type, hex_encoded_id, e)
+    return data
+
+# https://libc.rip/
+def provider_libc_rip(hex_encoded_id, hash_type):
+    # Build the request for the hash type
+    # https://github.com/niklasb/libc-database/blob/master/searchengine/api.yml
+    if hash_type == 'build_id':
+        hash_type = 'buildid'
+    url    = "https://libc.rip/api/find"
+    params = {hash_type: hex_encoded_id}
+
+    data = b""
+    try:
+        result = requests.post(url, json=params, timeout=20)
+        if result.status_code != 200 or len(result.json()) == 0:
+            log.warn_once("Could not find libc for %s %s on libc.rip", hash_type, hex_encoded_id)
+            log.debug("Error: %s", result.text)
+            return None
+
+        libc_match = result.json()
+        assert len(libc_match) == 1, 'Invalid libc.rip response.'
+
+        url = libc_match[0]['download_url']
+        log.debug("Downloading data from libc.rip: %s", url)
+        data = wget(url, timeout=20)
+
+        if not data:
+            log.warn_once("Could not fetch libc for %s %s from libc.rip", hash_type, hex_encoded_id)
+            return None
+    except requests.RequestException as e:
+        log.warn_once("Failed to fetch libc for %s %s from libc.rip: %s", hash_type, hex_encoded_id, e)
+    return data
+
+PROVIDERS = [provider_libcdb, provider_libc_rip]
+
 def search_by_hash(hex_encoded_id, hash_type='build_id'):
     assert hash_type in HASHES, hash_type
 
@@ -49,23 +108,14 @@ def search_by_hash(hex_encoded_id, hash_type='build_id'):
             log.info_once("Skipping unavailable libc %s", hex_encoded_id)
             return None
 
-    # Build the URL using the requested hash type
-    url_base = "https://gitlab.com/libcdb/libcdb/raw/master/hashes/%s/" % hash_type
-    url      = urllib.parse.urljoin(url_base, hex_encoded_id)
-
-    data   = b""
-    while not data.startswith(b'\x7fELF'):
-        log.debug("Downloading data from LibcDB: %s", url)
-        data = wget(url, timeout=20)
-
-        if not data:
-            log.warn_once("Could not fetch libc for build_id %s", hex_encoded_id)
+    # Run through all available libc database providers to see if we have a match.
+    for provider in PROVIDERS:
+        data = provider(hex_encoded_id, hash_type)
+        if data and data.startswith(b'\x7FELF'):
             break
 
-        # GitLab serves up symlinks with
-        if data.startswith(b'..'):
-            url = os.path.dirname(url) + '/'
-            url = urllib.parse.urljoin(url.encode('utf-8'), data)
+    if not data:
+        log.warn_once("Could not find libc for %s %s anywhere", hash_type, hex_encoded_id)
 
     # Save whatever we got to the cache
     write(cache, data or b'')
@@ -94,6 +144,9 @@ def search_by_build_id(hex_encoded_id):
         '0xda260'
         >>> None == search_by_build_id('XX')
         True
+        >>> filename = search_by_build_id('a5a3c3f65fd94f4c7f323a175707c3a79cbbd614')
+        >>> hex(ELF(filename).symbols.read)
+        '0xeef40'
     """
     return search_by_hash(hex_encoded_id, 'build_id')
 
@@ -103,7 +156,7 @@ def search_by_md5(hex_encoded_id):
 
     Arguments:
         hex_encoded_id(str):
-            Hex-encoded Build ID (e.g. 'ABCDEF...') of the library
+            Hex-encoded md5sum (e.g. 'ABCDEF...') of the library
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -112,8 +165,11 @@ def search_by_md5(hex_encoded_id):
         >>> filename = search_by_md5('7a71dafb87606f360043dcd638e411bd')
         >>> hex(ELF(filename).symbols.read)
         '0xda260'
-        >>> None == search_by_build_id('XX')
+        >>> None == search_by_md5('XX')
         True
+        >>> filename = search_by_md5('74f2d3062180572fc8bcd964b587eeae')
+        >>> hex(ELF(filename).symbols.read)
+        '0xeef40'
     """
     return search_by_hash(hex_encoded_id, 'md5')
 
@@ -123,7 +179,7 @@ def search_by_sha1(hex_encoded_id):
 
     Arguments:
         hex_encoded_id(str):
-            Hex-encoded Build ID (e.g. 'ABCDEF...') of the library
+            Hex-encoded sha1sum (e.g. 'ABCDEF...') of the library
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -134,6 +190,9 @@ def search_by_sha1(hex_encoded_id):
         '0xda260'
         >>> None == search_by_sha1('XX')
         True
+        >>> filename = search_by_sha1('0041d2f397bc2498f62aeb4134d522c5b2635e87')
+        >>> hex(ELF(filename).symbols.read)
+        '0xeef40'
     """
     return search_by_hash(hex_encoded_id, 'sha1')
 
@@ -144,7 +203,7 @@ def search_by_sha256(hex_encoded_id):
 
     Arguments:
         hex_encoded_id(str):
-            Hex-encoded Build ID (e.g. 'ABCDEF...') of the library
+            Hex-encoded sha256sum (e.g. 'ABCDEF...') of the library
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -155,6 +214,9 @@ def search_by_sha256(hex_encoded_id):
         '0xda260'
         >>> None == search_by_sha256('XX')
         True
+        >>> filename = search_by_sha256('5d78fc60054df18df20480c71f3379218790751090f452baffb62ac6b2aff7ee')
+        >>> hex(ELF(filename).symbols.read)
+        '0xeef40'
     """
     return search_by_hash(hex_encoded_id, 'sha256')
 

pwnlib/shellcraft/templates/common/linux/syscalls/ioperm.asm

@@ -5,7 +5,7 @@ import pwnlib.constants
 import pwnlib.shellcraft
 import six
 %>
-<%docstring>ioperm(from, num, turn_on) -> str
+<%docstring>ioperm(from_, num, turn_on) -> str
 
 Invokes the syscall ioperm.
 
@@ -18,7 +18,7 @@ Arguments:
 Returns:
     int
 </%docstring>
-<%page args="from=0, num=0, turn_on=0"/>
+<%page args="from_=0, num=0, turn_on=0"/>
 <%
     abi = pwnlib.abi.ABI.syscall()
     stack = abi.stack
@@ -28,8 +28,8 @@ Returns:
     can_pushstr = []
     can_pushstr_array = []
 
-    argument_names = ['from', 'num', 'turn_on']
-    argument_values = [from, num, turn_on]
+    argument_names = ['from_', 'num', 'turn_on']
+    argument_values = [from_, num, turn_on]
 
     # Load all of the arguments into their destination registers / stack slots.
     register_arguments = dict()

pwnlib/shellcraft/templates/common/linux/syscalls/link.asm

@@ -5,7 +5,7 @@ import pwnlib.constants
 import pwnlib.shellcraft
 import six
 %>
-<%docstring>link(from, to) -> str
+<%docstring>link(from_, to) -> str
 
 Invokes the syscall link.
 
@@ -17,7 +17,7 @@ Arguments:
 Returns:
     int
 </%docstring>
-<%page args="from=0, to=0"/>
+<%page args="from_=0, to=0"/>
 <%
     abi = pwnlib.abi.ABI.syscall()
     stack = abi.stack
@@ -27,8 +27,8 @@ Returns:
     can_pushstr = ['from', 'to']
     can_pushstr_array = []
 
-    argument_names = ['from', 'to']
-    argument_values = [from, to]
+    argument_names = ['from_', 'to']
+    argument_values = [from_, to]
 
     # Load all of the arguments into their destination registers / stack slots.
     register_arguments = dict()

pwnlib/shellcraft/templates/common/linux/syscalls/linkat.asm

@@ -5,7 +5,7 @@ import pwnlib.constants
 import pwnlib.shellcraft
 import six
 %>
-<%docstring>linkat(fromfd, from, tofd, to, flags) -> str
+<%docstring>linkat(fromfd, from_, tofd, to, flags) -> str
 
 Invokes the syscall linkat.
 
@@ -20,7 +20,7 @@ Arguments:
 Returns:
     int
 </%docstring>
-<%page args="fromfd=0, from=0, tofd=0, to=0, flags=0"/>
+<%page args="fromfd=0, from_=0, tofd=0, to=0, flags=0"/>
 <%
     abi = pwnlib.abi.ABI.syscall()
     stack = abi.stack
@@ -30,8 +30,8 @@ Returns:
     can_pushstr = ['from', 'to']
     can_pushstr_array = []
 
-    argument_names = ['fromfd', 'from', 'tofd', 'to', 'flags']
-    argument_values = [fromfd, from, tofd, to, flags]
+    argument_names = ['fromfd', 'from_', 'tofd', 'to', 'flags']
+    argument_values = [fromfd, from_, tofd, to, flags]
 
     # Load all of the arguments into their destination registers / stack slots.
     register_arguments = dict()

pwnlib/shellcraft/templates/common/linux/syscalls/symlink.asm

@@ -5,7 +5,7 @@ import pwnlib.constants
 import pwnlib.shellcraft
 import six
 %>
-<%docstring>symlink(from, to) -> str
+<%docstring>symlink(from_, to) -> str
 
 Invokes the syscall symlink.
 
@@ -17,7 +17,7 @@ Arguments:
 Returns:
     int
 </%docstring>
-<%page args="from=0, to=0"/>
+<%page args="from_=0, to=0"/>
 <%
     abi = pwnlib.abi.ABI.syscall()
     stack = abi.stack
@@ -27,8 +27,8 @@ Returns:
     can_pushstr = ['from', 'to']
     can_pushstr_array = []
 
-    argument_names = ['from', 'to']
-    argument_values = [from, to]
+    argument_names = ['from_', 'to']
+    argument_values = [from_, to]
 
     # Load all of the arguments into their destination registers / stack slots.
     register_arguments = dict()

pwnlib/shellcraft/templates/common/linux/syscalls/symlinkat.asm

@@ -5,7 +5,7 @@ import pwnlib.constants
 import pwnlib.shellcraft
 import six
 %>
-<%docstring>symlinkat(from, tofd, to) -> str
+<%docstring>symlinkat(from_, tofd, to) -> str
 
 Invokes the syscall symlinkat.
 
@@ -18,7 +18,7 @@ Arguments:
 Returns:
     int
 </%docstring>
-<%page args="from=0, tofd=0, to=0"/>
+<%page args="from_=0, tofd=0, to=0"/>
 <%
     abi = pwnlib.abi.ABI.syscall()
     stack = abi.stack
@@ -28,8 +28,8 @@ Returns:
     can_pushstr = ['from', 'to']
     can_pushstr_array = []
 
-    argument_names = ['from', 'tofd', 'to']
-    argument_values = [from, tofd, to]
+    argument_names = ['from_', 'tofd', 'to']
+    argument_values = [from_, tofd, to]
 
     # Load all of the arguments into their destination registers / stack slots.
     register_arguments = dict()

pwnlib/shellcraft/templates/mips/linux/sh.asm

@@ -3,6 +3,8 @@
 
 Example:
 
+    >>> b'\0' in pwnlib.asm.asm(shellcraft.mips.linux.sh())
+    False
     >>> p = run_assembly(shellcraft.mips.linux.sh())
     >>> p.sendline(b'echo Hello')
     >>> p.recv()