From 545f8dddd42a3880a2f14d2f8f9f963fbd6a005f Mon Sep 17 00:00:00 2001 From: peace-maker Date: Sat, 24 Oct 2020 13:53:50 +0200 Subject: [PATCH 01/10] Try libc.rip for libc lookup (#1704) * Try libc.rip for libc lookup As of #1171 the libcdb repository isn't seeing any updates anymore. When looking up a libc by a hash or build_id try the libc database maintained by @niklasb as well. The database uses a REST api documented at https://github.com/niklasb/libc-database/blob/master/searchengine/api.yml * Catch exceptions properly * Update CHANGELOG * Add doctests Co-authored-by: Arusekk --- CHANGELOG.md | 2 + pwnlib/libcdb.py | 100 ++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 83 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a35cee62..d43bf2e99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,7 @@ The table below shows which release corresponds to each branch, and what date th - [#1688][1688] Add `__setattr__` and `__call__` interfaces to `ROP` for setting registers - [#1692][1692] Remove python2 shebangs where appropriate - [#1703][1703] Update libcdb buildid offsets for amd64 and i386 +- [#1704][1704] Try https://libc.rip/ for libcdb lookup [1541]: https://github.com/Gallopsled/pwntools/pull/1541 [1602]: https://github.com/Gallopsled/pwntools/pull/1602 @@ -99,6 +100,7 @@ The table below shows which release corresponds to each branch, and what date th [1688]: https://github.com/Gallopsled/pwntools/pull/1688 [1692]: https://github.com/Gallopsled/pwntools/pull/1692 [1703]: https://github.com/Gallopsled/pwntools/pull/1703 +[1704]: https://github.com/Gallopsled/pwntools/pull/1704 ## 4.3.0 (`stable`) diff --git a/pwnlib/libcdb.py b/pwnlib/libcdb.py index 51701f27a..e78e23a74 100644 --- a/pwnlib/libcdb.py +++ b/pwnlib/libcdb.py @@ -7,6 +7,7 @@ import codecs import json import os +import requests import tempfile from six.moves import urllib @@ -25,6 +26,64 @@ HASHES = ['build_id', 'sha1', 'sha256', 'md5'] +# https://gitlab.com/libcdb/libcdb wasn't updated after 2019, +# but still is a massive database of older libc binaries. +def provider_libcdb(hex_encoded_id, hash_type): + # Build the URL using the requested hash type + url_base = "https://gitlab.com/libcdb/libcdb/raw/master/hashes/%s/" % hash_type + url = urllib.parse.urljoin(url_base, hex_encoded_id) + + data = b"" + log.debug("Downloading data from LibcDB: %s", url) + try: + while not data.startswith(b'\x7fELF'): + data = wget(url, timeout=20) + + if not data: + log.warn_once("Could not fetch libc for %s %s from libcdb", hash_type, hex_encoded_id) + break + + # GitLab serves up symlinks with + if data.startswith(b'..'): + url = os.path.dirname(url) + '/' + url = urllib.parse.urljoin(url.encode('utf-8'), data) + except requests.RequestException as e: + log.warn_once("Failed to fetch libc for %s %s from libcdb: %s", hash_type, hex_encoded_id, e) + return data + +# https://libc.rip/ +def provider_libc_rip(hex_encoded_id, hash_type): + # Build the request for the hash type + # https://github.com/niklasb/libc-database/blob/master/searchengine/api.yml + if hash_type == 'build_id': + hash_type = 'buildid' + url = "https://libc.rip/api/find" + params = {hash_type: hex_encoded_id} + + data = b"" + try: + result = requests.post(url, json=params, timeout=20) + if result.status_code != 200 or len(result.json()) == 0: + log.warn_once("Could not find libc for %s %s on libc.rip", hash_type, hex_encoded_id) + log.debug("Error: %s", result.text) + return None + + libc_match = result.json() + assert len(libc_match) == 1, 'Invalid libc.rip response.' + + url = libc_match[0]['download_url'] + log.debug("Downloading data from libc.rip: %s", url) + data = wget(url, timeout=20) + + if not data: + log.warn_once("Could not fetch libc for %s %s from libc.rip", hash_type, hex_encoded_id) + return None + except requests.RequestException as e: + log.warn_once("Failed to fetch libc for %s %s from libc.rip: %s", hash_type, hex_encoded_id, e) + return data + +PROVIDERS = [provider_libcdb, provider_libc_rip] + def search_by_hash(hex_encoded_id, hash_type='build_id'): assert hash_type in HASHES, hash_type @@ -49,23 +108,14 @@ def search_by_hash(hex_encoded_id, hash_type='build_id'): log.info_once("Skipping unavailable libc %s", hex_encoded_id) return None - # Build the URL using the requested hash type - url_base = "https://gitlab.com/libcdb/libcdb/raw/master/hashes/%s/" % hash_type - url = urllib.parse.urljoin(url_base, hex_encoded_id) - - data = b"" - while not data.startswith(b'\x7fELF'): - log.debug("Downloading data from LibcDB: %s", url) - data = wget(url, timeout=20) - - if not data: - log.warn_once("Could not fetch libc for build_id %s", hex_encoded_id) + # Run through all available libc database providers to see if we have a match. + for provider in PROVIDERS: + data = provider(hex_encoded_id, hash_type) + if data and data.startswith(b'\x7FELF'): break - # GitLab serves up symlinks with - if data.startswith(b'..'): - url = os.path.dirname(url) + '/' - url = urllib.parse.urljoin(url.encode('utf-8'), data) + if not data: + log.warn_once("Could not find libc for %s %s anywhere", hash_type, hex_encoded_id) # Save whatever we got to the cache write(cache, data or b'') @@ -94,6 +144,9 @@ def search_by_build_id(hex_encoded_id): '0xda260' >>> None == search_by_build_id('XX') True + >>> filename = search_by_build_id('a5a3c3f65fd94f4c7f323a175707c3a79cbbd614') + >>> hex(ELF(filename).symbols.read) + '0xeef40' """ return search_by_hash(hex_encoded_id, 'build_id') @@ -103,7 +156,7 @@ def search_by_md5(hex_encoded_id): Arguments: hex_encoded_id(str): - Hex-encoded Build ID (e.g. 'ABCDEF...') of the library + Hex-encoded md5sum (e.g. 'ABCDEF...') of the library Returns: Path to the downloaded library on disk, or :const:`None`. @@ -112,8 +165,11 @@ def search_by_md5(hex_encoded_id): >>> filename = search_by_md5('7a71dafb87606f360043dcd638e411bd') >>> hex(ELF(filename).symbols.read) '0xda260' - >>> None == search_by_build_id('XX') + >>> None == search_by_md5('XX') True + >>> filename = search_by_md5('74f2d3062180572fc8bcd964b587eeae') + >>> hex(ELF(filename).symbols.read) + '0xeef40' """ return search_by_hash(hex_encoded_id, 'md5') @@ -123,7 +179,7 @@ def search_by_sha1(hex_encoded_id): Arguments: hex_encoded_id(str): - Hex-encoded Build ID (e.g. 'ABCDEF...') of the library + Hex-encoded sha1sum (e.g. 'ABCDEF...') of the library Returns: Path to the downloaded library on disk, or :const:`None`. @@ -134,6 +190,9 @@ def search_by_sha1(hex_encoded_id): '0xda260' >>> None == search_by_sha1('XX') True + >>> filename = search_by_sha1('0041d2f397bc2498f62aeb4134d522c5b2635e87') + >>> hex(ELF(filename).symbols.read) + '0xeef40' """ return search_by_hash(hex_encoded_id, 'sha1') @@ -144,7 +203,7 @@ def search_by_sha256(hex_encoded_id): Arguments: hex_encoded_id(str): - Hex-encoded Build ID (e.g. 'ABCDEF...') of the library + Hex-encoded sha256sum (e.g. 'ABCDEF...') of the library Returns: Path to the downloaded library on disk, or :const:`None`. @@ -155,6 +214,9 @@ def search_by_sha256(hex_encoded_id): '0xda260' >>> None == search_by_sha256('XX') True + >>> filename = search_by_sha256('5d78fc60054df18df20480c71f3379218790751090f452baffb62ac6b2aff7ee') + >>> hex(ELF(filename).symbols.read) + '0xeef40' """ return search_by_hash(hex_encoded_id, 'sha256') From 031b3cbadf9e4be55af7e84f6df2a14f820ec80b Mon Sep 17 00:00:00 2001 From: Arusekk Date: Mon, 26 Oct 2020 22:57:14 +0100 Subject: [PATCH 02/10] Fix races in server tube from #1067 and apply #1569 (#1707) --- pwnlib/tubes/listen.py | 2 -- pwnlib/tubes/server.py | 23 +++++++---------------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/pwnlib/tubes/listen.py b/pwnlib/tubes/listen.py index b27b5cf34..2221af470 100644 --- a/pwnlib/tubes/listen.py +++ b/pwnlib/tubes/listen.py @@ -79,8 +79,6 @@ def __init__(self, port=0, bindaddr='::', super(listen, self).__init__(*args, **kwargs) port = int(port) - fam = {socket.AF_INET: 'ipv4', - socket.AF_INET6: 'ipv6'}.get(fam, fam) fam = self._get_family(fam) typ = self._get_type(typ) diff --git a/pwnlib/tubes/server.py b/pwnlib/tubes/server.py index 9c61274e4..6f630f5bb 100644 --- a/pwnlib/tubes/server.py +++ b/pwnlib/tubes/server.py @@ -9,6 +9,7 @@ from pwnlib.log import getLogger from pwnlib.tubes.sock import sock from pwnlib.tubes.remote import remote +from six.moves.queue import Queue log = getLogger(__name__) @@ -67,19 +68,17 @@ class server(sock): _accepter = None - def __init__(self, port=0, bindaddr = "0.0.0.0", fam = "any", typ = "tcp", + def __init__(self, port=0, bindaddr = "::", fam = "any", typ = "tcp", callback = None, blocking = False, *args, **kwargs): super(server, self).__init__(*args, **kwargs) port = int(port) - fam = {socket.AF_INET: 'ipv4', - socket.AF_INET6: 'ipv6'}.get(fam, fam) fam = self._get_family(fam) typ = self._get_type(typ) - if fam == socket.AF_INET6 and bindaddr == '0.0.0.0': - bindaddr = '::' + if fam == socket.AF_INET and bindaddr == '::': + bindaddr = '0.0.0.0' h = self.waitfor('Trying to bind to %s on port %d' % (bindaddr, port)) @@ -104,8 +103,7 @@ def __init__(self, port=0, bindaddr = "0.0.0.0", fam = "any", typ = "tcp", h.success() self.sock = listen_sock - self.connections_waiting = threading.Event() - self.connections = [] + self.connections = Queue() def accepter(): while True: h = self.waitfor('Waiting for connections on %s:%s' % (self.lhost, self.lport)) @@ -139,21 +137,14 @@ def accepter(): else: callback(r) else: - self.connections.append(r) - if not self.connections_waiting.is_set(): - self.connections_waiting.set() + self.connections.put(r) self._accepter = context.Thread(target = accepter) self._accepter.daemon = True self._accepter.start() def next_connection(self): - if not self.connections_waiting.is_set(): - self.connections_waiting.wait() - conn = self.connections.pop(0) - if not self.connections: - self.connections_waiting.clear() - return conn + return self.connections.get() def close(self): # since `close` is scheduled to run on exit we must check that we got From 40b01582b83f725de6c106334986ee1e726d0829 Mon Sep 17 00:00:00 2001 From: Arusekk Date: Tue, 27 Oct 2020 09:18:07 +0100 Subject: [PATCH 03/10] Use caching to avoid being hit by rate limiting Closes #1708 --- pwnlib/update.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pwnlib/update.py b/pwnlib/update.py index 8f5d8c03a..23b327124 100644 --- a/pwnlib/update.py +++ b/pwnlib/update.py @@ -27,6 +27,7 @@ import os import time +from functools import lru_cache from six.moves.xmlrpc_client import ServerProxy import packaging.version @@ -66,6 +67,7 @@ def read_update_config(settings): register_config('update', read_update_config) +@lru_cache def available_on_pypi(prerelease=current_version.is_prerelease): """Return True if an update is available on PyPI. From 681be1284c79997569756af7db39538c9da61847 Mon Sep 17 00:00:00 2001 From: Arusekk Date: Tue, 27 Oct 2020 10:08:18 +0100 Subject: [PATCH 04/10] Apparently python 2 does not have lru_cache. --- pwnlib/update.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pwnlib/update.py b/pwnlib/update.py index 23b327124..0c04e8799 100644 --- a/pwnlib/update.py +++ b/pwnlib/update.py @@ -27,7 +27,6 @@ import os import time -from functools import lru_cache from six.moves.xmlrpc_client import ServerProxy import packaging.version @@ -67,7 +66,6 @@ def read_update_config(settings): register_config('update', read_update_config) -@lru_cache def available_on_pypi(prerelease=current_version.is_prerelease): """Return True if an update is available on PyPI. @@ -76,8 +74,12 @@ def available_on_pypi(prerelease=current_version.is_prerelease): >>> available_on_pypi(prerelease=False).is_prerelease False """ - client = ServerProxy('https://pypi.python.org/pypi') - versions = client.package_releases('pwntools', True) + versions = getattr(available_on_pypi, 'cached', None) + if versions is None: + client = ServerProxy('https://pypi.python.org/pypi') + versions = client.package_releases('pwntools', True) + available_on_pypi.cached = versions + versions = map(packaging.version.Version, versions) if not prerelease: From b6bb1ea17e96de5ef0d25f4725654fc52f4373e0 Mon Sep 17 00:00:00 2001 From: Arusekk Date: Wed, 28 Oct 2020 12:33:25 +0100 Subject: [PATCH 05/10] Don't introduce null bytes in MIPS shellcode. (#1710) * Don't introduce null bytes in MIPS shellcode. As a verification, you can run: ```sh pwn shellcraft mips.linux.sh |pwn phd ``` Closes #807 * Add a test * Fix doctest under py2 --- pwnlib/shellcraft/templates/mips/linux/sh.asm | 2 ++ pwnlib/shellcraft/templates/mips/pushstr_array.asm | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pwnlib/shellcraft/templates/mips/linux/sh.asm b/pwnlib/shellcraft/templates/mips/linux/sh.asm index 0e065d0e4..673a2dd13 100644 --- a/pwnlib/shellcraft/templates/mips/linux/sh.asm +++ b/pwnlib/shellcraft/templates/mips/linux/sh.asm @@ -3,6 +3,8 @@ Example: + >>> b'\0' in pwnlib.asm.asm(shellcraft.mips.linux.sh()) + False >>> p = run_assembly(shellcraft.mips.linux.sh()) >>> p.sendline(b'echo Hello') >>> p.recv() diff --git a/pwnlib/shellcraft/templates/mips/pushstr_array.asm b/pwnlib/shellcraft/templates/mips/pushstr_array.asm index c7a389c4a..9ca5f287f 100644 --- a/pwnlib/shellcraft/templates/mips/pushstr_array.asm +++ b/pwnlib/shellcraft/templates/mips/pushstr_array.asm @@ -31,7 +31,7 @@ offset = len(array_str) + word_size ${mips.push(reg)} /* null terminate */ % for i,arg in enumerate(reversed(array)): ${mips.mov(reg, offset + word_size*i - len(arg))} - add ${reg}, $sp + add ${reg}, $sp, ${reg} ${mips.push(reg)} /* ${repr(arg)} */ <% offset -= len(arg) %>\ % endfor From 4572f8711a31fea1ca4e117824ce046401eed0d3 Mon Sep 17 00:00:00 2001 From: Arusekk Date: Sat, 31 Oct 2020 21:51:38 +0100 Subject: [PATCH 06/10] Extend shellcraft forbidden names (#1715) Closes #1664 --- pwnlib/data/syscalls/generate.py | 6 ++++-- .../shellcraft/templates/common/linux/syscalls/ioperm.asm | 8 ++++---- .../shellcraft/templates/common/linux/syscalls/link.asm | 8 ++++---- .../shellcraft/templates/common/linux/syscalls/linkat.asm | 8 ++++---- .../templates/common/linux/syscalls/symlink.asm | 8 ++++---- .../templates/common/linux/syscalls/symlinkat.asm | 8 ++++---- 6 files changed, 24 insertions(+), 22 deletions(-) diff --git a/pwnlib/data/syscalls/generate.py b/pwnlib/data/syscalls/generate.py index fb31841a2..5ee93fd7e 100644 --- a/pwnlib/data/syscalls/generate.py +++ b/pwnlib/data/syscalls/generate.py @@ -75,8 +75,8 @@ # The argument is not a register. It is a string value, and we # are expecting a string value - elif name in can_pushstr and isinstance(arg, (bytes, six.text_type)): - if not isinstance(arg, bytes): + elif name in can_pushstr and isinstance(arg, (six.binary_type, six.text_type)): + if isinstance(arg, six.text_type): arg = arg.encode('utf-8') string_arguments[name] = arg @@ -150,6 +150,8 @@ def fix_bad_arg_names(func, arg): return 'length' if arg.name == 'repr': return 'repr_' + if arg.name == 'from': + return 'from_' if func.name == 'open' and arg.name == 'vararg': return 'mode' diff --git a/pwnlib/shellcraft/templates/common/linux/syscalls/ioperm.asm b/pwnlib/shellcraft/templates/common/linux/syscalls/ioperm.asm index c09a3e930..322fcf3ca 100644 --- a/pwnlib/shellcraft/templates/common/linux/syscalls/ioperm.asm +++ b/pwnlib/shellcraft/templates/common/linux/syscalls/ioperm.asm @@ -5,7 +5,7 @@ import pwnlib.constants import pwnlib.shellcraft import six %> -<%docstring>ioperm(from, num, turn_on) -> str +<%docstring>ioperm(from_, num, turn_on) -> str Invokes the syscall ioperm. @@ -18,7 +18,7 @@ Arguments: Returns: int -<%page args="from=0, num=0, turn_on=0"/> +<%page args="from_=0, num=0, turn_on=0"/> <% abi = pwnlib.abi.ABI.syscall() stack = abi.stack @@ -28,8 +28,8 @@ Returns: can_pushstr = [] can_pushstr_array = [] - argument_names = ['from', 'num', 'turn_on'] - argument_values = [from, num, turn_on] + argument_names = ['from_', 'num', 'turn_on'] + argument_values = [from_, num, turn_on] # Load all of the arguments into their destination registers / stack slots. register_arguments = dict() diff --git a/pwnlib/shellcraft/templates/common/linux/syscalls/link.asm b/pwnlib/shellcraft/templates/common/linux/syscalls/link.asm index 87bab61cb..63cbe54c2 100644 --- a/pwnlib/shellcraft/templates/common/linux/syscalls/link.asm +++ b/pwnlib/shellcraft/templates/common/linux/syscalls/link.asm @@ -5,7 +5,7 @@ import pwnlib.constants import pwnlib.shellcraft import six %> -<%docstring>link(from, to) -> str +<%docstring>link(from_, to) -> str Invokes the syscall link. @@ -17,7 +17,7 @@ Arguments: Returns: int -<%page args="from=0, to=0"/> +<%page args="from_=0, to=0"/> <% abi = pwnlib.abi.ABI.syscall() stack = abi.stack @@ -27,8 +27,8 @@ Returns: can_pushstr = ['from', 'to'] can_pushstr_array = [] - argument_names = ['from', 'to'] - argument_values = [from, to] + argument_names = ['from_', 'to'] + argument_values = [from_, to] # Load all of the arguments into their destination registers / stack slots. register_arguments = dict() diff --git a/pwnlib/shellcraft/templates/common/linux/syscalls/linkat.asm b/pwnlib/shellcraft/templates/common/linux/syscalls/linkat.asm index c63f6e775..03f15f9c3 100644 --- a/pwnlib/shellcraft/templates/common/linux/syscalls/linkat.asm +++ b/pwnlib/shellcraft/templates/common/linux/syscalls/linkat.asm @@ -5,7 +5,7 @@ import pwnlib.constants import pwnlib.shellcraft import six %> -<%docstring>linkat(fromfd, from, tofd, to, flags) -> str +<%docstring>linkat(fromfd, from_, tofd, to, flags) -> str Invokes the syscall linkat. @@ -20,7 +20,7 @@ Arguments: Returns: int -<%page args="fromfd=0, from=0, tofd=0, to=0, flags=0"/> +<%page args="fromfd=0, from_=0, tofd=0, to=0, flags=0"/> <% abi = pwnlib.abi.ABI.syscall() stack = abi.stack @@ -30,8 +30,8 @@ Returns: can_pushstr = ['from', 'to'] can_pushstr_array = [] - argument_names = ['fromfd', 'from', 'tofd', 'to', 'flags'] - argument_values = [fromfd, from, tofd, to, flags] + argument_names = ['fromfd', 'from_', 'tofd', 'to', 'flags'] + argument_values = [fromfd, from_, tofd, to, flags] # Load all of the arguments into their destination registers / stack slots. register_arguments = dict() diff --git a/pwnlib/shellcraft/templates/common/linux/syscalls/symlink.asm b/pwnlib/shellcraft/templates/common/linux/syscalls/symlink.asm index 894f30e87..eb785e9d9 100644 --- a/pwnlib/shellcraft/templates/common/linux/syscalls/symlink.asm +++ b/pwnlib/shellcraft/templates/common/linux/syscalls/symlink.asm @@ -5,7 +5,7 @@ import pwnlib.constants import pwnlib.shellcraft import six %> -<%docstring>symlink(from, to) -> str +<%docstring>symlink(from_, to) -> str Invokes the syscall symlink. @@ -17,7 +17,7 @@ Arguments: Returns: int -<%page args="from=0, to=0"/> +<%page args="from_=0, to=0"/> <% abi = pwnlib.abi.ABI.syscall() stack = abi.stack @@ -27,8 +27,8 @@ Returns: can_pushstr = ['from', 'to'] can_pushstr_array = [] - argument_names = ['from', 'to'] - argument_values = [from, to] + argument_names = ['from_', 'to'] + argument_values = [from_, to] # Load all of the arguments into their destination registers / stack slots. register_arguments = dict() diff --git a/pwnlib/shellcraft/templates/common/linux/syscalls/symlinkat.asm b/pwnlib/shellcraft/templates/common/linux/syscalls/symlinkat.asm index bc2396bb6..916c4faa3 100644 --- a/pwnlib/shellcraft/templates/common/linux/syscalls/symlinkat.asm +++ b/pwnlib/shellcraft/templates/common/linux/syscalls/symlinkat.asm @@ -5,7 +5,7 @@ import pwnlib.constants import pwnlib.shellcraft import six %> -<%docstring>symlinkat(from, tofd, to) -> str +<%docstring>symlinkat(from_, tofd, to) -> str Invokes the syscall symlinkat. @@ -18,7 +18,7 @@ Arguments: Returns: int -<%page args="from=0, tofd=0, to=0"/> +<%page args="from_=0, tofd=0, to=0"/> <% abi = pwnlib.abi.ABI.syscall() stack = abi.stack @@ -28,8 +28,8 @@ Returns: can_pushstr = ['from', 'to'] can_pushstr_array = [] - argument_names = ['from', 'tofd', 'to'] - argument_values = [from, tofd, to] + argument_names = ['from_', 'tofd', 'to'] + argument_values = [from_, tofd, to] # Load all of the arguments into their destination registers / stack slots. register_arguments = dict() From d98e60e4e5dd3498593a4fa52dab568427858a3f Mon Sep 17 00:00:00 2001 From: Arusekk Date: Sun, 1 Nov 2020 12:27:16 +0100 Subject: [PATCH 07/10] Revert "alternative to PR #563; this works by simply never emitting CR" (#1716) This reverts commit 08c8c2e8e42dbb707242331b412e49fff5d1fe55. Closes #1653 --- pwnlib/term/term.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pwnlib/term/term.py b/pwnlib/term/term.py index df35e68a5..2d0f41d6c 100644 --- a/pwnlib/term/term.py +++ b/pwnlib/term/term.py @@ -425,7 +425,7 @@ def render_cell(cell, clear_after = False): put('\x08') col -= 1 elif t == CR: -# put('\r') + put('\r') col = 0 elif t == SOH: put('\x01') From 42b060e48a0f2d29d1874cc41b1ba14f26be8575 Mon Sep 17 00:00:00 2001 From: Arusekk Date: Sun, 1 Nov 2020 12:46:00 +0100 Subject: [PATCH 08/10] Improve tubes.ssh test coverage, fix discovered issues (#1711) * Improve tubes.ssh test coverage, fix discovered issues * Fix test under py2 --- docs/source/tubes/ssh.rst | 4 ++ pwnlib/tubes/ssh.py | 100 ++++++++++++++++---------------------- 2 files changed, 47 insertions(+), 57 deletions(-) diff --git a/docs/source/tubes/ssh.rst b/docs/source/tubes/ssh.rst index d48b94162..ae351cc3b 100644 --- a/docs/source/tubes/ssh.rst +++ b/docs/source/tubes/ssh.rst @@ -14,6 +14,10 @@ :members: kill, poll, interactive :show-inheritance: + .. autoclass:: pwnlib.tubes.ssh.ssh_process + :members: + :show-inheritance: + .. autoclass:: pwnlib.tubes.ssh.ssh_connecter() :show-inheritance: diff --git a/pwnlib/tubes/ssh.py b/pwnlib/tubes/ssh.py index 4ea23ef27..bf48a0317 100644 --- a/pwnlib/tubes/ssh.py +++ b/pwnlib/tubes/ssh.py @@ -318,7 +318,7 @@ def libs(self): """ maps = self.parent.libs(self.executable) - maps_raw = self.parent.cat('/proc/%d/maps' % self.pid) + maps_raw = self.parent.cat('/proc/%d/maps' % self.pid).decode() for lib in maps: remote_path = lib.split(self.parent.host)[-1] @@ -337,6 +337,12 @@ def libc(self): Returns an ELF for the libc for the current process. If possible, it is adjusted to the correct address automatically. + + Examples: + >>> s = ssh(host='example.pwnme') + >>> p = s.process('true') + >>> p.libc # doctest: +ELLIPSIS + ELF(.../libc.so.6') """ from pwnlib.elf import ELF @@ -373,15 +379,27 @@ def corefile(self): return pwnlib.elf.corefile.Corefile(finder.core_path) def getenv(self, variable, **kwargs): - """Retrieve the address of an environment variable in the remote process. + r"""Retrieve the address of an environment variable in the remote process. + + Examples: + >>> s = ssh(host='example.pwnme') + >>> p = s.process(['python', '-c', 'print("Hello")']) + >>> hex(p.getenv('PATH')) # doctest: +ELLIPSIS + '0x...' + >>> p.recvall() + b'Hello\n' """ argv0 = self.argv[0] + variable = context._encode(variable) + script = ';'.join(('from ctypes import *', 'import os', 'libc = CDLL("libc.so.6")', + 'getenv = libc.getenv', + 'getenv.restype = c_void_p', 'print(os.path.realpath(%r))' % self.executable, - 'print(libc.getenv(%r))' % variable,)) + 'print(getenv(%r))' % variable,)) try: with context.local(log_level='error'): @@ -395,13 +413,13 @@ def getenv(self, variable, **kwargs): env=self.env, **kwargs) path = io.recvline() - address = int(io.recvline()) + address = int(io.recvall()) address -= len(python) address += len(path) return int(address) & context.mask - except: + except Exception: self.exception("Could not look up environment variable %r" % variable) def _close_msg(self): @@ -562,13 +580,9 @@ def __init__(self, user=None, host=None, port=22, password=None, key=None, .. doctest:: :skipif: github_actions - >>> s1 = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s1 = ssh(host='example.pwnme') >>> r1 = s1.remote('localhost', 22) >>> s2 = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass', ... proxy_sock=r1.sock) >>> r2 = s2.remote('localhost', 22) # and so on... >>> for x in r2, s2, r1, s1: x.close() @@ -615,6 +629,8 @@ def __init__(self, user=None, host=None, port=22, password=None, key=None, host_config = ssh_config.lookup(host) if 'hostname' in host_config: self.host = host = host_config['hostname'] + if not user and 'user' in host_config: + self.user = user = host_config['user'] if not keyfile and 'identityfile' in host_config: keyfile = host_config['identityfile'][0] if keyfile.lower() == 'none': @@ -703,9 +719,7 @@ def shell(self, shell = None, tty = True, timeout = Timeout.default): Return a :class:`pwnlib.tubes.ssh.ssh_channel` object. Examples: - >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s = ssh(host='example.pwnme') >>> sh = s.shell('/bin/sh') >>> sh.sendline(b'echo Hello; exit') >>> print(b'Hello' in sh.recvall()) @@ -782,9 +796,7 @@ def process(self, argv=None, executable=None, tty=True, cwd=None, env=None, time Requires Python on the remote server. Examples: - >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s = ssh(host='example.pwnme') >>> sh = s.process('/bin/sh', env={'PS1':''}) >>> sh.sendline(b'echo Hello; exit') >>> sh.recvall() @@ -845,13 +857,13 @@ def process(self, argv=None, executable=None, tty=True, cwd=None, env=None, time argv = argv or [] aslr = aslr if aslr is not None else context.aslr - if isinstance(argv, (six.text_type, six.binary_type)): + if isinstance(argv, (six.text_type, bytes, bytearray)): argv = [argv] if not isinstance(argv, (list, tuple)): self.error('argv must be a list or tuple') - if not all(isinstance(arg, (six.text_type, six.binary_type)) for arg in argv): + if not all(isinstance(arg, (six.text_type, bytes, bytearray)) for arg in argv): self.error("argv must be strings or bytes: %r" % argv) if shell: @@ -1144,9 +1156,7 @@ def system(self, process, tty = True, wd = None, env = None, timeout = None, raw Return a :class:`pwnlib.tubes.ssh.ssh_channel` object. Examples: - >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s = ssh(host='example.pwnme') >>> py = s.run('python -i') >>> _ = py.recvuntil(b'>>> ') >>> py.sendline(b'print(2+2)') @@ -1207,9 +1217,7 @@ def run_to_end(self, process, tty = False, wd = None, env = None): a TTY on the remote server. Examples: - >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s = ssh(host='example.pwnme') >>> print(s.run_to_end('echo Hello; exit 17')) (b'Hello\n', 17) """ @@ -1232,9 +1240,7 @@ def connect_remote(self, host, port, timeout = Timeout.default): Examples: >>> from pwn import * >>> l = listen() - >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s = ssh(host='example.pwnme') >>> a = s.connect_remote(s.host, l.lport) >>> a=a; b = l.wait_for_connection() # a=a; prevents hangs >>> a.sendline(b'Hello') @@ -1257,9 +1263,7 @@ def listen_remote(self, port = 0, bind_address = '', timeout = Timeout.default): Examples: >>> from pwn import * - >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s = ssh(host='example.pwnme') >>> l = s.listen_remote() >>> a = remote(s.host, l.port) >>> a=a; b = l.wait_for_connection() # a=a; prevents hangs @@ -1277,9 +1281,7 @@ def __getitem__(self, attr): Examples: - >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s = ssh(host='example.pwnme') >>> print(repr(s['echo hello'])) b'hello' """ @@ -1290,9 +1292,7 @@ def __call__(self, attr): Examples: - >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s = ssh(host='example.pwnme') >>> print(repr(s('echo hello'))) b'hello' """ @@ -1303,9 +1303,7 @@ def __getattr__(self, attr): Examples: - >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s = ssh(host='example.pwnme') >>> s.echo('hello') b'hello' >>> s.whoami() @@ -1336,9 +1334,7 @@ def connected(self): Example: - >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s = ssh(host='example.pwnme') >>> s.connected() True >>> s.close() @@ -1495,8 +1491,6 @@ def download_data(self, remote): >>> with open('/tmp/bar','w+') as f: ... _ = f.write('Hello, world') >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass', ... cache=False) >>> s.download_data('/tmp/bar') b'Hello, world' @@ -1585,9 +1579,7 @@ def upload_data(self, data, remote): remote(str): The filename to upload it to. Example: - >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s = ssh(host='example.pwnme') >>> s.upload_data(b'Hello, world', '/tmp/upload_foo') >>> print(open('/tmp/upload_foo').read()) Hello, world @@ -1816,18 +1808,14 @@ def set_working_directory(self, wd = None, symlink = False): that all files in the "old" working directory should be symlinked. Examples: - >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s = ssh(host='example.pwnme') >>> cwd = s.set_working_directory() >>> s.ls() b'' >>> s.pwd() == cwd True - >>> s = ssh(host='example.pwnme', - ... user='travis', - ... password='demopass') + >>> s = ssh(host='example.pwnme') >>> homedir = s.pwd() >>> _=s.touch('foo') @@ -1926,10 +1914,8 @@ def preexec(): return with self.process(['lsb_release', '-irs']) as io: - self._platform_info.update({ - 'distro': io.recvline().strip().decode(), - 'distro_ver': io.recvline().strip().decode() - }) + lsb_info = io.recvall().strip().decode() + self._platform_info['distro'], self._platform_info['distro_ver'] = lsb_info.split() except Exception: pass From 9e76e50c9a66c9f5b10511ca8419d2b79609698d Mon Sep 17 00:00:00 2001 From: Ricky Zhou Date: Sun, 1 Nov 2020 11:02:58 -0800 Subject: [PATCH 09/10] Misc run_in_new_terminal improvements. (#1261) * run_in_new_terminal: Prefer tmux/screen over TERM_PROGRAM and x-terminal-emulator. * run_in_new_terminal: Kill running commands when the main process exits. Currently, when a gdb window is opened via gdb.debug(), terminating the main pwntools process leaves the gdb window/process around. It is inconvenient to have to close these leftover gdb windows when iterating on an exploit. Add a kill_at_exit parameter to run_in_new_terminal() which attempts to terminate the command at main process exit. This only works for terminals which do not daemonize, or for tmux, which supports returning the command's PID. This change also removes the special case for closing stdin/stdout/stderr on Mac OS X, as the problem no longer reproduces under tmux 2.8. * Fix regression and add CHANGELOG Co-authored-by: Arusekk --- CHANGELOG.md | 4 ++++ pwn/toplevel.py | 1 + pwnlib/util/misc.py | 50 +++++++++++++++++++++++++++------------------ 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d43bf2e99..367aae89d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,10 @@ The table below shows which release corresponds to each branch, and what date th ## 4.5.0 (`dev`) +- [#1261][1261] Misc `run_in_new_terminal` improvements (notably gdb terminated by default) + +[1261]: https://github.com/Gallopsled/pwntools/pull/1261 + ## 4.4.0 (`beta`) - [#1541][1541] Use `context.newline` for tubes by default diff --git a/pwn/toplevel.py b/pwn/toplevel.py index 9b253e802..68c984bc1 100644 --- a/pwn/toplevel.py +++ b/pwn/toplevel.py @@ -4,6 +4,7 @@ import math import operator import os +import platform import re import requests import socks diff --git a/pwnlib/util/misc.py b/pwnlib/util/misc.py index 46b228771..23f2cd96f 100644 --- a/pwnlib/util/misc.py +++ b/pwnlib/util/misc.py @@ -3,15 +3,15 @@ import base64 import errno import os -import platform import re +import signal import six import socket import stat import string +import subprocess -import six - +from pwnlib import atexit from pwnlib.context import context from pwnlib.log import getLogger from pwnlib.util import fiddling @@ -181,8 +181,8 @@ def which(name, all = False): else: return None -def run_in_new_terminal(command, terminal = None, args = None): - """run_in_new_terminal(command, terminal = None) -> None +def run_in_new_terminal(command, terminal = None, args = None, kill_at_exit = True): + """run_in_new_terminal(command, terminal = None, args = None, kill_at_exit = True) -> int Run a command in a new terminal. @@ -190,21 +190,25 @@ def run_in_new_terminal(command, terminal = None, args = None): - If ``context.terminal`` is set it will be used. If it is an iterable then ``context.terminal[1:]`` are default arguments. - If a ``pwntools-terminal`` command exists in ``$PATH``, it is used - - If ``$TERM_PROGRAM`` is set, that is used. - - If X11 is detected (by the presence of the ``$DISPLAY`` environment - variable), ``x-terminal-emulator`` is used. - If tmux is detected (by the presence of the ``$TMUX`` environment variable), a new pane will be opened. - If GNU Screen is detected (by the presence of the ``$STY`` environment variable), a new screen will be opened. + - If ``$TERM_PROGRAM`` is set, that is used. + - If X11 is detected (by the presence of the ``$DISPLAY`` environment + variable), ``x-terminal-emulator`` is used. - If WSL (Windows Subsystem for Linux) is detected (by the presence of a ``wsl.exe`` binary in the ``$PATH`` and ``/proc/sys/kernel/osrelease`` containing ``Microsoft``), a new ``cmd.exe`` window will be opened. + If `kill_at_exit` is :const:`True`, try to close the command/terminal when the + current process exits. This may not work for all terminal types. + Arguments: command (str): The command to run. terminal (str): Which terminal to use. args (list): Arguments to pass to the terminal + kill_at_exit (bool): Whether to close the command/terminal on process exit. Note: The command is opened with ``/dev/null`` for stdin, stdout, stderr. @@ -221,7 +225,7 @@ def run_in_new_terminal(command, terminal = None, args = None): args = [] elif 'TMUX' in os.environ and which('tmux'): terminal = 'tmux' - args = ['splitw'] + args = ['splitw', '-F' '#{pane_pid}', '-P'] elif 'STY' in os.environ and which('screen'): terminal = 'screen' args = ['-t','pwntools-gdb','bash','-c'] @@ -261,17 +265,23 @@ def run_in_new_terminal(command, terminal = None, args = None): log.debug("Launching a new terminal: %r" % argv) - pid = os.fork() - - if pid == 0: - # Closing the file descriptors makes everything fail under tmux on OSX. - if platform.system() != 'Darwin': - devnull = open(os.devnull, 'r+b') - os.dup2(devnull.fileno(), 0) - os.dup2(devnull.fileno(), 1) - os.dup2(devnull.fileno(), 2) - os.execv(argv[0], argv) - os._exit(1) + stdin = stdout = stderr = open(os.devnull, 'rwb') + if terminal == 'tmux': + stdout = subprocess.PIPE + + p = subprocess.Popen(argv, stdin=stdin, stdout=stdout, stderr=stderr) + + if terminal == 'tmux': + out, _ = p.communicate() + pid = int(out) + else: + pid = p.pid + + if kill_at_exit: + if terminal == 'tmux': + atexit.register(lambda: os.kill(pid, signal.SIGTERM)) + else: + atexit.register(lambda: p.terminate()) return pid From 0b1f430935842d23aa6f4027eea7ac1bb292bcc1 Mon Sep 17 00:00:00 2001 From: Arusekk Date: Tue, 3 Nov 2020 23:48:38 +0100 Subject: [PATCH 10/10] Fix #1261 regression There is no such thing as rwb in py3 apparently --- pwnlib/util/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pwnlib/util/misc.py b/pwnlib/util/misc.py index 23f2cd96f..ff4e35f89 100644 --- a/pwnlib/util/misc.py +++ b/pwnlib/util/misc.py @@ -265,7 +265,7 @@ def run_in_new_terminal(command, terminal = None, args = None, kill_at_exit = Tr log.debug("Launching a new terminal: %r" % argv) - stdin = stdout = stderr = open(os.devnull, 'rwb') + stdin = stdout = stderr = open(os.devnull, 'r+b') if terminal == 'tmux': stdout = subprocess.PIPE