Merge branch 'dev' into gdb-api

.github/workflows/changelog.yml

@@ -17,4 +17,13 @@ jobs:
 
     - name: Check changelog for updates
       run: |
-        git log --stat ${GITHUB_BASE}..HEAD | grep CHANGELOG.md
+        if git log --stat ${GITHUB_BASE}..HEAD | grep CHANGELOG.md; then
+           echo 'Changelog updated :D'
+        else
+           if git log --stat ${GITHUB_BASE}..HEAD | grep '++\|--'; then
+              echo 'Major changes detected, changelog required!'
+              false
+           else
+              echo 'Minor changes detected, no changelog required!'
+           fi
+        fi

.github/workflows/ci.yml

@@ -2,7 +2,7 @@ name: Continuous Integration
 on: [push, pull_request]
 
 jobs:
-  build:
+  test:
     strategy:
       matrix:
         python-version: [2.7, 3.8]
@@ -202,7 +202,7 @@ jobs:
   staging-merge:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'Gallopsled' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/heads/') && endsWith(github.event.ref, '-staging')
-    needs: build
+    needs: test
     steps:
     - uses: actions/checkout@v2
       with:
@@ -220,7 +220,7 @@ jobs:
   pypi:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
-    needs: build
+    needs: test
     steps:
     - name: Download artifacts
       uses: actions/download-artifact@v2

.github/workflows/lint.yml

@@ -2,7 +2,7 @@ name: Lint
 on: [push, pull_request]
 
 jobs:
-  build:
+  lint:
     strategy:
       matrix:
         python-version: [3.8]

CHANGELOG.md

@@ -9,9 +9,10 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.4.0](#440)    | `dev`    | Sep 1, 2020 (planned)
-| [4.3.0](#430)    | `beta`   | Aug 1, 2020 (planned)
-| [4.2.0](#420)    | `stable` | Jul 3, 2020 (planned)
+| [4.5.0](#450)    | `dev`    | Dec 20, 2020 (planned)
+| [4.4.0](#440)    | `beta`   | Nov 20, 2020 (planned)
+| [4.3.0](#430)    | `stable` | Oct 20, 2020
+| [4.2.0](#420)    |          | Jul 3, 2020
 | [4.1.7](#417)    |          | Jun 30, 2020
 | [4.1.5](#415)    |          | Jun 27, 2020
 | [4.1.4](#414)    |          | Jun 26, 2020
@@ -55,8 +56,15 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.4.0 (`dev`)
+## 4.5.0 (`dev`)
 
+- [#1695][1695] Allow using GDB Python API
+
+[1695]: https://github.com/Gallopsled/pwntools/pull/1695
+
+## 4.4.0 (`beta`)
+
+- [#1541][1541] Use `context.newline` for tubes by default
 - [#1602][1602] Fix bytes handling in ssh tubes
 - [#1606][1606] Fix `asm()` and `disasm()` for MSP430, S390
 - [#1616][1616] Fix `cyclic` cli for 64 bit integers
@@ -75,8 +83,9 @@ The table below shows which release corresponds to each branch, and what date th
 - [#1687][1687] Actually import `requests` when doing `from pwn import *`
 - [#1688][1688] Add `__setattr__` and `__call__` interfaces to `ROP` for setting registers
 - [#1692][1692] Remove python2 shebangs where appropriate
-- [#1695][1695] Allow using GDB Python API
+- [#1703][1703] Update libcdb buildid offsets for amd64 and i386
 
+[1541]: https://github.com/Gallopsled/pwntools/pull/1541
 [1602]: https://github.com/Gallopsled/pwntools/pull/1602
 [1606]: https://github.com/Gallopsled/pwntools/pull/1606
 [1616]: https://github.com/Gallopsled/pwntools/pull/1616
@@ -93,8 +102,9 @@ The table below shows which release corresponds to each branch, and what date th
 [1687]: https://github.com/Gallopsled/pwntools/pull/1687
 [1688]: https://github.com/Gallopsled/pwntools/pull/1688
 [1692]: https://github.com/Gallopsled/pwntools/pull/1692
+[1703]: https://github.com/Gallopsled/pwntools/pull/1703
 
-## 4.3.0 (`beta`)
+## 4.3.0 (`stable`)
 
 - [#1576][1576] Add `executable=` argument to `ELF.search`
 - [#1584][1584] Add `jmp_esp`/`jmp_rsp` attribute to `ROP`
@@ -118,33 +128,14 @@ The table below shows which release corresponds to each branch, and what date th
 [1564]: https://github.com/Gallopsled/pwntools/pull/1564
 [1621]: https://github.com/Gallopsled/pwntools/pull/1621
 
-## 4.3.0 (`beta`)
-
-- [#1576][1576] Add `executable=` argument to `ELF.search`
-- [#1584][1584] Add `jmp_esp`/`jmp_rsp` attribute to `ROP`
-- [#1592][1592] Fix over-verbose logging of process() environment
-- [#1593][1593] Colorize output of `pwn template`
-- [#1601][1601] Add `pwn version` command line tool
-- [#1605][1605] Add to `fiddling.hexdump` a way to suppress the total at the end
-- [#1613][1613] Permit `--password` for `pwn template`
-- [#1564][1564] Fix `asm()` and `disasm()` for PowerPC64, MIPS64, Sparc64
-- [#1621][1621] Permit negative values in flat() and fit()
-
-[1576]: https://github.com/Gallopsled/pwntools/pull/1576
-[1584]: https://github.com/Gallopsled/pwntools/pull/1584
-[1592]: https://github.com/Gallopsled/pwntools/pull/1592
-[1593]: https://github.com/Gallopsled/pwntools/pull/1593
-[1601]: https://github.com/Gallopsled/pwntools/pull/1601
-[1605]: https://github.com/Gallopsled/pwntools/pull/1605
-[1613]: https://github.com/Gallopsled/pwntools/pull/1613
-[1564]: https://github.com/Gallopsled/pwntools/pull/1564
-[1621]: https://github.com/Gallopsled/pwntools/pull/1621
-
-## 4.2.1 (`stable`)
+## 4.2.1
 
 - [#1625][1625] GDB now properly loads executables with QEMU
+- [#1663][1663] Change lookup algorithm of `adb.which`
+- [#1699][1699] Fix broken linux shellcraft templates
 
 [1625]: https://github.com/Gallopsled/pwntools/pull/1625
+[1699]: https://github.com/Gallopsled/pwntools/pull/1699
 
 ## 4.2.0
 

pwnlib/adb/adb.py

@@ -827,10 +827,12 @@ def which(name, all = False, *a, **kw):
     """
     # Unfortunately, there is no native 'which' on many phones.
     which_cmd = '''
-echo $PATH | while read -d: directory; do
-    [ -x "$directory/{name}" ] || continue;
-    echo -n "$directory/{name}\\x00";
-done
+(IFS=:
+  for directory in $PATH; do
+      [ -x "$directory/{name}" ] || continue;
+      echo -n "$directory/{name}\\x00";
+  done
+)
 [ -x "{name}" ] && echo -n "$PWD/{name}\\x00"
 '''.format(name=name)
 

pwnlib/context/__init__.py

@@ -358,7 +358,7 @@ class ContextType(object):
         'log_console': sys.stdout,
         'randomize': False,
         'rename_corefiles': True,
-        'newline': '\n',
+        'newline': b'\n',
         'noptrace': False,
         'os': 'linux',
         'proxy': None,
@@ -1278,6 +1278,15 @@ def rename_corefiles(self, v):
         """
         return bool(v)
 
+    @_validator
+    def newline(self, v):
+        """Line ending used for Tubes by default.
+
+        This configures the newline emitted by e.g. ``sendline`` or that is used
+        as a delimiter for e.g. ``recvline``.
+        """
+        return six.ensure_binary(v)
+
 
     @_validator
     def gdbinit(self, value):

pwnlib/elf/config.py

@@ -21,8 +21,8 @@ def __init__(self, name, title, requires=[], excludes=[], minver=0, maxver=99):
         self.excludes = set(excludes)
 
         #: Kernel version that this check should be enforced on
-        self.minver = map(int, str(minver).split('.'))
-        self.maxver = map(int, str(maxver).split('.'))
+        self.minver = list(map(int, str(minver).split('.')))
+        self.maxver = list(map(int, str(maxver).split('.')))
 
     def relevant(self, config):
 

pwnlib/elf/elf.py

@@ -319,7 +319,7 @@ def __init__(self, path, checksec=True):
                 config = gz.read()
 
             if config:
-                self.config = parse_kconfig(config)
+                self.config = parse_kconfig(config.decode())
 
         #: ``True`` if the ELF is a statically linked executable
         self.statically_linked = bool(self.elftype == 'EXEC' and self.load_addr)
@@ -1055,7 +1055,10 @@ def _populate_kernel_version(self):
             return
 
         banner = self.string(self.symbols.linux_banner)
-
+
+        # convert banner into a utf-8 string since re.search does not accept bytes anymore
+        banner = banner.decode('utf-8')
+
         # 'Linux version 3.18.31-gd0846ecc
         regex = r'Linux version (\S+)'
         match = re.search(regex, banner)

pwnlib/fmtstr.py

@@ -851,7 +851,7 @@ def leak_stack(self, offset, prefix=b""):
         payload = b"START%%%d$pEND" % offset
         leak = self.execute_fmt(prefix + payload)
         try:
-            leak = re.findall(br"START(.*)END", leak, re.MULTILINE | re.DOTALL)[0]
+            leak = re.findall(br"START(.*?)END", leak, re.MULTILINE | re.DOTALL)[0]
             leak = int(leak, 16)
         except ValueError:
             leak = 0

pwnlib/libcdb.py

@@ -46,7 +46,7 @@ def search_by_hash(hex_encoded_id, hash_type='build_id'):
             log.info_once("Using cached data from %r", cache)
             return cache
         else:
-            log.info_once("Skipping unavialable libc %s", hex_encoded_id)
+            log.info_once("Skipping unavailable libc %s", hex_encoded_id)
             return None
 
     # Build the URL using the requested hash type
@@ -186,7 +186,7 @@ def get_build_id_offsets():
     return {
     # $ check_arch 80386
     #     181 Displaying notes found at file offset 0x00000174 with length 0x00000024:
-        'i386': [0x174],
+        'i386': [0x174, 0x1b4, 0x1d4],
     # $ check_arch "ARM, EABI5"
     #      69 Displaying notes found at file offset 0x00000174 with length 0x00000024:
         'arm':  [0x174],
@@ -197,7 +197,7 @@ def get_build_id_offsets():
     # $ check_arch "x86-64"
     #       6 Displaying notes found at file offset 0x00000174 with length 0x00000024:
     #      82 Displaying notes found at file offset 0x00000270 with length 0x00000024:
-        'amd64': [0x270, 0x174, 0x2e0],
+        'amd64': [0x270, 0x174, 0x2e0, 0x370],
     # $ check_arch "PowerPC or cisco"
     #      88 Displaying notes found at file offset 0x00000174 with length 0x00000024:
         'powerpc': [0x174],

pwnlib/rop/rop.py

@@ -1148,7 +1148,7 @@ def __bytes__(self):
 
     def __get_cachefile_name(self, files):
         """Given an ELF or list of ELF objects, return a cache file for the set of files"""
-        cachedir = os.path.join(tempfile.gettempdir(), 'pwntools-rop-cache-%d.%d' % sys.version_info[:2])
+        cachedir = os.path.join(context.cache_dir, 'rop-cache')
         if not os.path.exists(cachedir):
             os.mkdir(cachedir)
 
@@ -1164,7 +1164,7 @@ def __get_cachefile_name(self, files):
     @staticmethod
     def clear_cache():
         """Clears the ROP gadget cache"""
-        cachedir = os.path.join(tempfile.gettempdir(), 'pwntools-rop-cache')
+        cachedir = os.path.join(context.cache_dir, 'rop-cache')
         shutil.rmtree(cachedir)
 
     def __cache_load(self, elf):
@@ -1187,15 +1187,15 @@ def __load(self):
         #
         # - leave
         # - pop reg
-        # - add $sp, value
+        # - add $sp, <hexadecimal value>
         # - ret
         #
         # Currently, ROPgadget does not detect multi-byte "C2" ret.
         # https://github.com/JonathanSalwan/ROPgadget/issues/53
         #
 
         pop   = re.compile(r'^pop (.{3})')
-        add   = re.compile(r'^add [er]sp, (\S+)$')
+        add   = re.compile(r'^add [er]sp, ((?:0[xX])?[0-9a-fA-F]+)$')
         ret   = re.compile(r'^ret$')
         leave = re.compile(r'^leave$')
         int80 = re.compile(r'int +0x80')
@@ -1211,6 +1211,8 @@ def __load(self):
         # False
         # >>> valid('add esp, 0x24')
         # True
+        # >>> valid('add esp, esi')
+        # False
         #
         valid = lambda insn: any(map(lambda pattern: pattern.match(insn), [pop,add,ret,leave,int80,syscall,sysenter]))
 

pwnlib/shellcraft/templates/i386/linux/acceptloop_ipv4.asm

@@ -20,22 +20,22 @@ ${acceptloop}:
     /*  Socket file descriptor is placed in EBP */
 
     /*  sock = socket(AF_INET, SOCK_STREAM, 0) */
-    ${i386.linux.push(0)}
-    ${i386.linux.push('SOCK_STREAM')}
-    ${i386.linux.push('AF_INET')}
+    ${i386.push(0)}
+    ${i386.push('SOCK_STREAM')}
+    ${i386.push('AF_INET')}
     ${i386.linux.syscall('SYS_socketcall', 'SYS_socketcall_socket', 'esp')}
 
-    ${i386.linux.mov('esi', 'eax')}      /* keep socket fd */
+    ${i386.mov('esi', 'eax')}      /* keep socket fd */
 
     /*  bind(sock, &addr, sizeof addr); // sizeof addr == 0x10 */
-    ${i386.linux.push(0)}
+    ${i386.push(0)}
     /* ${htons(port)} == htons(${port}) */
-    ${i386.linux.push('AF_INET | (%d << 16)' % htons(port))}
-    ${i386.linux.mov('ecx', 'esp')}
+    ${i386.push('AF_INET | (%d << 16)' % htons(port))}
+    ${i386.mov('ecx', 'esp')}
 
-    ${i386.linux.push(0x10)}    /* sizeof addr */
-    ${i386.linux.push('ecx')}   /* &addr */
-    ${i386.linux.push('eax')}   /* sock */
+    ${i386.push(0x10)}    /* sizeof addr */
+    ${i386.push('ecx')}   /* &addr */
+    ${i386.push('eax')}   /* sock */
     ${i386.linux.syscall('SYS_socketcall', 'SYS_socketcall_bind', 'esp')}
 
     /*  listen(sock, whatever) */
@@ -44,17 +44,17 @@ ${acceptloop}:
 
 ${looplabel}:
     /*  accept(sock, NULL, NULL) */
-    ${i386.linux.push(0x0)}
-    ${i386.linux.push('esi')} /* sock */
+    ${i386.push(0x0)}
+    ${i386.push('esi')} /* sock */
     ${i386.linux.syscall('SYS_socketcall', 'SYS_socketcall_accept', 'esp')}
 
-    ${i386.linux.mov('ebp', 'eax')}      /* keep in-coming socket fd */
+    ${i386.mov('ebp', 'eax')}      /* keep in-coming socket fd */
 
     ${i386.linux.syscall('SYS_fork')}
     xchg eax, edi
 
     test edi, edi
-    ${i386.linux.mov('ebx', 'ebp')}
+    ${i386.mov('ebx', 'ebp')}
     cmovz ebx, esi /*  on child we close the server sock instead */
 
     /*  close(sock) */

pwnlib/shellcraft/templates/i386/linux/mprotect_all.asm

@@ -22,6 +22,6 @@
 %endif
 ${label}:
     ${i386.linux.syscall('SYS_mprotect', 'ebx', 'ecx', 'PROT_READ | PROT_WRITE | PROT_EXEC')}
-    ${i386.linux.mov('ecx', 0x1000)}
+    ${i386.mov('ecx', 0x1000)}
     add ebx, ecx
     jnz ${label}

pwnlib/tubes/server.py

@@ -117,7 +117,7 @@ def accepter():
                             data, rhost = listen_sock.recvfrom(4096)
                             listen_sock.connect(rhost)
                             sock = listen_sock
-                            sock.unrecv(data)
+                            self.unrecv(data)
                         sock.settimeout(self.timeout)
                         break
                     except socket.error as e: