Merge branch 'dev' into libc_offset_lookup

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
         key: ${{ matrix.os }}-cache-pip
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -59,33 +59,9 @@ jobs:
           binutils-s390x-linux-gnu \
           binutils-sparc64-linux-gnu \
           gcc-multilib \
-          libc6-dbg
+          libc6-dbg \
+          elfutils
       
-    - name: Cache for elfutils
-      uses: actions/cache@v1
-      id: cache-elfutils
-      with:
-        path: ~/.cache/elfutils
-        key: ${{ matrix.os }}-cache-elfutils
-
-    # Install newer elfutils version due to regression in 0.176 available in focal.
-    - name: Install newer elfutils
-      env:
-        ELFUTILS_VERSION: 0.181
-      run: |
-        if [[ ! -d ~/.cache/elfutils/elfutils-${ELFUTILS_VERSION} ]]
-        then
-          wget -O /tmp/elfutils-${ELFUTILS_VERSION}.tar.bz2 https://sourceware.org/elfutils/ftp/${ELFUTILS_VERSION}/elfutils-${ELFUTILS_VERSION}.tar.bz2
-          mkdir -p ~/.cache/elfutils && cd ~/.cache/elfutils
-          tar -xf /tmp/elfutils-${ELFUTILS_VERSION}.tar.bz2
-          cd elfutils-${ELFUTILS_VERSION}
-          LDFLAGS=-Wl,-rpath=/usr/local/lib,--enable-new-dtags ./configure --disable-libdebuginfod --disable-debuginfod && make && sudo make install
-        else
-          cd ~/.cache/elfutils/elfutils-${ELFUTILS_VERSION}
-          sudo make install
-        fi
-        eu-unstrip --version
-
     - name: Install RPyC for GDB
       run: |
         sudo apt-get install -y python3-pip

.github/workflows/pylint.yml

@@ -28,10 +28,10 @@ jobs:
         set -x
         pip install pylint
         pip install --upgrade -r requirements.txt
-        pylint --exit-zero --errors-only  pwnlib > current.txt
+        pylint --exit-zero --errors-only pwnlib -f parseable | cut -d ' ' -f2- > current.txt
         git fetch origin
         git checkout origin/"$GITHUB_BASE_REF"
-        pylint --exit-zero --errors-only  pwnlib > base.txt
+        pylint --exit-zero --errors-only pwnlib -f parseable | cut -d ' ' -f2- > base.txt
         if diff base.txt current.txt | grep '>'; then
           false
         fi

CHANGELOG.md

@@ -70,11 +70,15 @@ The table below shows which release corresponds to each branch, and what date th
 - [#2092][2092] shellcraft: dup() is now called dupio() consistently across all supported arches
 - [#2093][2093] setresuid() in shellcraft uses current euid by default
 - [#2103][2103] Add search for libc binary by leaked function addresses `libcdb.search_by_symbol_offsets()`
+- [#2125][2125] Allow tube.recvregex to return capture groups
+- [#2144][2144] Removes `p2align 2` `asm()` headers from `x86-32`, `x86-64` and `mips` architectures to avoid inconsistent instruction length when patching binaries
 
 [2062]: https://github.com/Gallopsled/pwntools/pull/2062
 [2092]: https://github.com/Gallopsled/pwntools/pull/2092
 [2093]: https://github.com/Gallopsled/pwntools/pull/2093
 [2103]: https://github.com/Gallopsled/pwntools/pull/2103
+[2125]: https://github.com/Gallopsled/pwntools/pull/2125
+[2144]: https://github.com/Gallopsled/pwntools/pull/2144
 
 ## 4.9.0 (`beta`)
 
@@ -86,6 +90,8 @@ The table below shows which release corresponds to each branch, and what date th
 - [#2033][2033] Quote file and core path in generated GDB script
 - [#2035][2035] Change Buffer's parent class to object
 - [#2037][2037] Allow SSH tunnel to be treated like a TCP socket (with 'raw=True')
+- [#2123][2123] Fix ROP without a writeable cache directory
+- [#2124][2124] Fix `tube.recvpred()` timeout argument
 
 [1975]: https://github.com/Gallopsled/pwntools/pull/1975
 [1979]: https://github.com/Gallopsled/pwntools/pull/1979
@@ -95,6 +101,8 @@ The table below shows which release corresponds to each branch, and what date th
 [2033]: https://github.com/Gallopsled/pwntools/pull/2033
 [2035]: https://github.com/Gallopsled/pwntools/pull/2035
 [2037]: https://github.com/Gallopsled/pwntools/pull/2037
+[2123]: https://github.com/Gallopsled/pwntools/pull/2123
+[2124]: https://github.com/Gallopsled/pwntools/pull/2124
 
 ## 4.8.0 (`stable`)
 

README.md

@@ -4,7 +4,7 @@
 [![PyPI](https://img.shields.io/pypi/v/pwntools?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
 [![Travis](https://img.shields.io/travis/Gallopsled/pwntools/dev?logo=Travis)](https://travis-ci.org/Gallopsled/pwntools)
-[![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/Gallopsled/pwntools/Continuous%20Integration/dev?logo=GitHub)](https://github.com/Gallopsled/pwntools/actions?query=workflow%3A%22Continuous+Integration%22+branch%3Adev)
+[![GitHub Workflow Status (dev)](https://img.shields.io/github/actions/workflow/status/Gallopsled/pwntools/ci.yml?branch=dev&logo=GitHub)](https://github.com/Gallopsled/pwntools/actions/workflows/ci.yml?query=branch%3Adev)
 [![Coveralls](https://img.shields.io/coveralls/github/Gallopsled/pwntools/dev?logo=coveralls)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://choosealicense.com/licenses/mit/)
 [![Packaging status](https://img.shields.io/repology/repositories/python:pwntools)](https://repology.org/project/python:pwntools/versions)

SECURITY.md

@@ -0,0 +1,32 @@
+# Security Policy
+
+## Supported Versions
+
+| Version       | Supported          |
+| ------------- | ------------------ |
+| latest dev    | :white_check_mark: |
+| latest beta   | :white_check_mark: |
+| latest stable | :white_check_mark: |
+| anything else | :x: |
+
+## Reporting a Vulnerability
+
+The aim of pwntools is exploiting software vulnerabilities, which is an unusual position, but it nevertheless can have its own security issues.
+Especially that an attacker (=re-victim) is usually not prepared to be attacked back (by the re-attacker).
+
+The first question to ask yourself is: is this an actual vulnerability?
+- can it be triggered by a re-attacker (malicious honeypot pretending to be a vulnerable service)?
+- does it impact the attacker (=re-victim)?
+- is it serious?
+  * *availability: medium* means *at least* exhausting RAM or disk space of the attacker (=re-victim)
+  * *confidentiality: medium* means *at least* reading the filesystem of the attacker (=re-victim)
+  * *integrity: medium* means *at least* performing uncontrolled actions or data corruption on behalf of the attacker (=re-victim)
+  * if crucial for some sophisticated exploit chain, it is always serious
+  * `safe_eval` bypasses **are** serious.
+  * an example of what was **kind of** serious: [#1732](https://github.com/Gallopsled/pwntools/pull/1732)
+- can it be fixed without compromising on Pwntools' usability?
+
+If at least one of the answers is no, then this is NOT a vulnerability, so just file a bug report or feature request, without the weird confidential disclosure dance.
+
+Just e-mail the maintainers.  Arusekk is the one that is currently the most excited to fix vulnerabilities.
+Or create a CTF task!  Prove a point the good old hacker way!

extra/docker/Makefile

@@ -1,4 +1,4 @@
-subdirs=$(shell find . -type d -depth 1 | xargs basename)
+subdirs=$(shell find . -mindepth 1 -maxdepth 1 -type d | xargs -n1 basename)
 tags=base stable beta dev
 ROOT=$(shell git rev-parse --show-toplevel)
 CMD ?= zsh

extra/docker/base/Dockerfile

@@ -35,6 +35,7 @@ RUN apt-get update \
         binutils-powerpc64-linux-gnu \
         binutils-sparc64-linux-gnu \
         tmux \
+        patchelf \
     && pip install --upgrade pip \
     && python -m pip install --upgrade pwntools \
     && pip3 install --upgrade pip \

extra/docker/develop/Dockerfile

@@ -31,7 +31,7 @@ RUN python  -m pip install -U ipython ipdb \
  && python3 -m pip install -U ipython ipdb
 
 # Dependencies from .travis.yml addons -> apt -> packages
-RUN sudo apt-get install -y \
+RUN sudo apt-get update && sudo apt-get install -y \
 	ash \
 	bash \
 	bash-static \

pwnlib/asm.py

@@ -136,8 +136,23 @@ def print_binutils_instructions(util, context):
 %(instructions)s
 """.strip() % locals())
 
+
+def check_binutils_version(util):
+    if util_versions[util]:
+        return util_versions[util]
+    result = subprocess.check_output([util, '--version','/dev/null'],
+                                     stderr=subprocess.STDOUT, universal_newlines=True)
+    if 'clang' in result:
+        log.warn_once('Your binutils is clang-based and may not work!\n'
+            'Try installing with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+            'Reported version: %r', result.strip())
+    version = re.search(r' (\d+\.\d+)', result).group(1)
+    util_versions[util] = version = tuple(map(int, version.split('.')))
+    return version
+
+
 @LocalContext
-def which_binutils(util):
+def which_binutils(util, check_version=False):
     """
     Finds a binutils in the PATH somewhere.
     Expects that the utility is prefixed with the architecture name.
@@ -204,17 +219,23 @@ def which_binutils(util):
 
             for pattern in patterns:
                 for dir in environ['PATH'].split(':'):
-                    res = sorted(glob(path.join(dir, pattern)))
-                    if res:
-                        return res[0]
+                    for res in sorted(glob(path.join(dir, pattern))):
+                        if check_version:
+                            ver = check_binutils_version(res)
+                            return res, ver
+                        return res
 
     # No dice!
     print_binutils_instructions(util, context)
 
-checked_assembler_version = defaultdict(lambda: False)
+util_versions = defaultdict(tuple)
 
 def _assembler():
-    gas = which_binutils('as')
+    gas, version = which_binutils('as', check_version=True)
+    if version < (2, 19):
+        log.warn_once('Your binutils version is too old and may not work!\n'
+            'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+            'Reported version: %r', version)
 
     E = {
         'big':    '-EB',
@@ -246,25 +267,10 @@ def _assembler():
 
     assembler = assemblers.get(context.arch, [gas])
 
-    if not checked_assembler_version[gas]:
-        checked_assembler_version[gas] = True
-        result = subprocess.check_output([gas, '--version','/dev/null'],
-                                         stderr=subprocess.STDOUT, universal_newlines=True)
-        version = re.search(r' (\d+\.\d+)', result).group(1)
-        if 'clang' in result:
-            log.warn_once('Your binutils is clang version and may not work!\n'
-                'Try install with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
-                'Reported Version: %r', result.strip())
-        elif version < '2.19':
-            log.warn_once('Your binutils version is too old and may not work!\n'
-                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
-                'Reported Version: %r', result.strip())
-
-
     return assembler
 
 def _linker():
-    ld  = [which_binutils('ld')]
+    ld, _ = which_binutils('ld', check_version=True)
     bfd = ['--oformat=' + _bfdname()]
 
     E = {
@@ -276,7 +282,16 @@ def _linker():
         'i386': ['-m', 'elf_i386'],
     }.get(context.arch, [])
 
-    return ld + bfd + [E] + arguments
+    return [ld] + bfd + [E] + arguments
+
+
+def _execstack(linker):
+    ldflags = ['-z', 'execstack']
+    version = util_versions[linker[0]]
+    if version >= (2, 39):
+        return ldflags + ['--no-warn-execstack', '--no-warn-rwx-segments']
+    return ldflags
+
 
 def _objcopy():
     return [which_binutils('objcopy')]
@@ -305,20 +320,23 @@ def _arch_header():
     prefix  = ['.section .shellcode,"awx"',
                 '.global _start',
                 '.global __start',
-                '.p2align 2',
                 '_start:',
                 '__start:']
     headers = {
-        'i386'  :  ['.intel_syntax noprefix'],
-        'amd64' :  ['.intel_syntax noprefix'],
+        'i386'  :  ['.intel_syntax noprefix', '.p2align 0'],
+        'amd64' :  ['.intel_syntax noprefix', '.p2align 0'],
         'arm'   : ['.syntax unified',
                    '.arch armv7-a',
-                   '.arm'],
+                   '.arm',
+                   '.p2align 2'],
         'thumb' : ['.syntax unified',
                    '.arch armv7-a',
-                   '.thumb'],
+                   '.thumb',
+                   '.p2align 2'
+                   ],
         'mips'  : ['.set mips2',
                    '.set noreorder',
+                   '.p2align 2'
                    ],
     }
 
@@ -595,7 +613,7 @@ def make_elf(data,
 
         _run(assembler + ['-o', step2, step1])
 
-        linker_options = ['-z', 'execstack']
+        linker_options = _execstack(linker)
         if vma is not None:
             linker_options += ['--section-start=.shellcode=%#x' % vma,
                                '--entry=%#x' % vma]
@@ -689,7 +707,7 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
             shutil.copy(step2, step3)
 
         if vma or not extract:
-            ldflags = ['-z', 'execstack', '-o', step3, step2]
+            ldflags = _execstack(linker) + ['-o', step3, step2]
             if vma:
                 ldflags += ['--section-start=.shellcode=%#x' % vma,
                             '--entry=%#x' % vma]
@@ -771,7 +789,7 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
         >>> print(disasm(unhex('4ff00500'), arch = 'thumb', bits=32))
            0:   f04f 0005       mov.w   r0, #5
         >>> print(disasm(unhex('656664676665400F18A4000000000051'), byte=0, arch='amd64'))
-           0:   gs data16 fs data16 rex nop/reserved BYTE PTR gs:[eax+eax*1+0x0]
+           0:   gs data16 fs rex nop WORD PTR gs:[eax+eax*1+0x0]
            f:   push   rcx
         >>> print(disasm(unhex('01000000'), arch='sparc64'))
            0:   01 00 00 00     nop

pwnlib/commandline/libcdb.py

@@ -4,7 +4,6 @@
 from __future__ import print_function
 
 import re
-import requests
 import shutil
 import sys
 
@@ -138,6 +137,7 @@
 common_symbols = ['dup2', 'printf', 'puts', 'read', 'system', 'write']
 
 def find_libc(params):
+    import requests
     url    = "https://libc.rip/api/find"
     result = requests.post(url, json=params, timeout=20)
     log.debug('Request: %s', params)

pwnlib/elf/corefile.py

@@ -445,7 +445,7 @@ class Corefile(ELF):
         will be loaded.
 
         >>> process('bash').corefile.libc # doctest: +ELLIPSIS
-        Mapping('/.../libc-....so', start=0x..., stop=0x..., size=0x..., flags=..., page_offset=...)
+        Mapping('.../libc...so...', start=0x..., stop=0x..., size=0x..., flags=..., page_offset=...)
 
         The corefile also contains a :attr:`.stack` property, which gives
         us direct access to the stack contents.  On Linux, the very top of the stack
@@ -759,7 +759,7 @@ def vsyscall(self):
     @property
     def libc(self):
         """:class:`Mapping`: First mapping for ``libc.so``"""
-        expr = r'libc\b.*so$'
+        expr = r'^libc\b.*so(?:\.6)?$'
 
         for m in self.mappings:
             if not m.name:

pwnlib/elf/elf.py

@@ -1165,7 +1165,7 @@ def search(self, needle, writable = False, executable = False):
             won't work.
 
         Arguments:
-            needle(str): String to search for.
+            needle(bytes): String to search for.
             writable(bool): Search only writable sections.
             executable(bool): Search only executable sections.
 
@@ -1346,7 +1346,7 @@ def read(self, address, count):
             count(int): Number of bytes to read
 
         Returns:
-            A :class:`str` object, or :const:`None`.
+            A :class:`bytes` object, or :const:`None`.
 
         Examples:
             The simplest example is just to read the ELF header.
@@ -1507,7 +1507,7 @@ def get_data(self):
 
     @property
     def data(self):
-        """:class:`str`: Raw data of the ELF file.
+        """:class:`bytes`: Raw data of the ELF file.
 
         See:
             :meth:`get_data`
@@ -1535,7 +1535,7 @@ def asm(self, address, assembly):
         This modifies the ELF in-place.
         The resulting binary can be saved with :meth:`.ELF.save`
         """
-        binary = asm(assembly, vma=address)
+        binary = asm(assembly, vma=address, arch=self.arch, endian=self.endian, bits=self.bits)
         self.write(address, binary)
 
     def bss(self, offset=0):
@@ -1915,7 +1915,7 @@ def checksec(self, banner=True, color=True):
 
     @property
     def buildid(self):
-        """:class:`str`: GNU Build ID embedded into the binary"""
+        """:class:`bytes`: GNU Build ID embedded into the binary"""
         section = self.get_section_by_name('.note.gnu.build-id')
         if section:
             return section.data()[16:]