From 18082f33c43c0aa5b958f332ba08d7890c2d62ec Mon Sep 17 00:00:00 2001 From: gpotter2 <10530980+gpotter2@users.noreply.github.com> Date: Wed, 22 May 2024 21:03:48 +0200 Subject: [PATCH] HTTP_Client/Server doc, other minor win tweaks (#4393) * Add HTTP_Client/HTTP_Server doc, fix tests * Sort arping() results by IP * Various minor Kerberos improvements (packets, client) --- doc/scapy/layers/http.rst | 97 ++++++++++++++++++++++++++++------- scapy/asn1/asn1.py | 2 + scapy/asn1fields.py | 29 ++++++++--- scapy/layers/kerberos.py | 102 +++++++++++++++++++++++++++++++++---- scapy/layers/l2.py | 28 +++++++--- test/scapy/layers/http.uts | 6 ++- test/scapy/layers/l2.uts | 2 +- 7 files changed, 222 insertions(+), 44 deletions(-) diff --git a/doc/scapy/layers/http.rst b/doc/scapy/layers/http.rst index aef497386cd..ec9879d1d29 100644 --- a/doc/scapy/layers/http.rst +++ b/doc/scapy/layers/http.rst @@ -84,19 +84,87 @@ All common header fields should be supported. Use Scapy to send/receive HTTP 1.X __________________________________ -To handle this decompression, Scapy uses `Sessions classes <../usage.html#advanced-sniffing-sessions>`_, more specifically the ``TCPSession`` class. -You have several ways of using it: +Scapy uses `Sessions classes <../usage.html#advanced-sniffing-sessions>`_ (more specifically the ``TCPSession`` class), in order to dissect and reconstruct HTTP packets. +This handles Content-Length, chunks and/or compression. -+--------------------------------------------+-------------------------------------------+ -| ``sniff(session=TCPSession, [...])`` | ``TCP_client.tcplink(HTTP, host, 80)`` | -+============================================+===========================================+ -| | Perform decompression / defragmentation | | Acts as a TCP client: handles SYN/ACK, | -| | on all TCP streams simultaneously, but | | and all TCP actions, but only creates | -| | only acts passively. | | one stream. | -+--------------------------------------------+-------------------------------------------+ +Here are the main ways of using HTTP 1.X with Scapy: + +- :class:`~scapy.layers.http.HTTP_Client`: Automata that send HTTP requests. It supports the :func:`~scapy.layers.gssapi.SSP` mechanism to support authorization with NTLM, Kerberos, etc. +- :class:`~scapy.layers.http.HTTP_Server`: Automata to handle incoming HTTP requests. Also supports :func:`~scapy.layers.gssapi.SSP`. +- ``sniff(session=TCPSession, [...])``: Perform decompression / defragmentation on all TCP streams simultaneously, but only acts passively. +- ``TCP_client.tcplink(HTTP, host, 80)``: Acts as a raw TCP client, handles SYN/ACK, and all TCP actions, but only creates one stream. It however supports some specific features, such as changing the source IP. **Examples:** +- :class:`~scapy.layers.http.HTTP_Client`: + +Let's perform a very simple GET request to an HTTP server: + +.. code:: python + + from scapy.layers.http import * # or load_layer("http") + client = HTTP_Client() + resp = client.request("http://127.0.0.1:8080") + client.close() + +You can use the following shorthand to do the same very basic feature: :func:`~scapy.layers.http.http_request`, usable as so: + +.. code:: python + + load_layer("http") + http_request("www.google.com", "/") # first argument is Host, second is Path + +Let's do the same request, but this time to a server that requires NTLM authentication: + +.. code:: python + + from scapy.layers.http import * # or load_layer("http") + client = HTTP_Client( + HTTP_AUTH_MECHS.NTLM, + ssp=NTLMSSP(UPN="user", PASSWORD="password"), + ) + resp = client.request("http://127.0.0.1:8080") + client.close() + +- :class:`~scapy.layers.http.HTTP_Server`: + +Start an unauthenticated HTTP server automaton: + +.. code:: python + + from scapy.layers.http import * + from scapy.layers.ntlm import * + + class Custom_HTTP_Server(HTTP_Server): + def answer(self, pkt): + if pkt.Path == b"/": + return HTTPResponse() / ( + "

OK

" + ) + else: + return HTTPResponse( + Status_Code=b"404", + Reason_Phrase=b"Not Found", + ) / ( + "

404 - Not Found

" + ) + + server = HTTP_Server.spawn( + port=8080, + iface="eth0", + ) + +We could also have started the same server, but requiring NTLM authorization using: + +.. code:: python + + server = HTTP_Server.spawn( + port=8080, + iface="eth0", + HTTP_AUTH_MECHS.NTLM, + ssp=NTLMSSP(IDENTITIES={"user": MD4le("password")}), + ) + - ``TCP_client.tcplink``: Send an HTTPRequest to ``www.secdev.org`` and write the result in a file: @@ -120,18 +188,9 @@ Send an HTTPRequest to ``www.secdev.org`` and write the result in a file: ``TCP_client.tcplink`` makes it feel like it only received one packet, but in reality it was recombined in ``TCPSession``. If you performed a plain ``sniff()``, you would have seen those packets. -**This code is implemented in a utility function:** ``http_request()``, usable as so: - -.. code:: python - - load_layer("http") - http_request("www.google.com", "/", display=True) - -This will open the webpage in your default browser thanks to ``display=True``. - - ``sniff()``: -Dissect a pcap which contains a JPEG image that was sent over HTTP using chunks. +Dissect a pcap which contains a JPEG image that was sent over HTTP using chunks. This is able to reconstruct all HTTP streams in parallel. .. note:: diff --git a/scapy/asn1/asn1.py b/scapy/asn1/asn1.py index 0a101d16c7f..5df2810efe4 100644 --- a/scapy/asn1/asn1.py +++ b/scapy/asn1/asn1.py @@ -501,6 +501,8 @@ def set(self, i, val): """ val = str(val) assert val in ['0', '1'] + if len(self.val) < i: + self.val += "0" * (i - len(self.val)) self.val = self.val[:i] + val + self.val[i + 1:] def __repr__(self): diff --git a/scapy/asn1fields.py b/scapy/asn1fields.py index 2f1da38b657..e4957b0920d 100644 --- a/scapy/asn1fields.py +++ b/scapy/asn1fields.py @@ -513,7 +513,12 @@ class ASN1F_SET(ASN1F_SEQUENCE): ASN1_tag = ASN1_Class_UNIVERSAL.SET -_SEQ_T = Union['ASN1_Packet', Type[ASN1F_field[Any, Any]], 'ASN1F_PACKET'] +_SEQ_T = Union[ + 'ASN1_Packet', + Type[ASN1F_field[Any, Any]], + 'ASN1F_PACKET', + ASN1F_field[Any, Any], +] class ASN1F_SEQUENCE_OF(ASN1F_field[List[_SEQ_T], @@ -533,10 +538,13 @@ def __init__(self, explicit_tag=None, # type: Optional[Any] ): # type: (...) -> None - if isinstance(cls, type) and issubclass(cls, ASN1F_field): - self.fld = cls - self._extract_packet = lambda s, pkt: self.fld( - self.name, b"").m2i(pkt, s) + if isinstance(cls, type) and issubclass(cls, ASN1F_field) or \ + isinstance(cls, ASN1F_field): + if isinstance(cls, type): + self.fld = cls(name, b"") + else: + self.fld = cls + self._extract_packet = lambda s, pkt: self.fld.m2i(pkt, s) self.holds_packets = 0 elif hasattr(cls, "ASN1_root") or callable(cls): self.cls = cast("Type[ASN1_Packet]", cls) @@ -594,12 +602,21 @@ def build(self, pkt): s = b"".join(raw(i) for i in val) return self.i2m(pkt, s) + def i2repr(self, pkt, x): + # type: (ASN1_Packet, _I) -> str + if self.holds_packets: + return super(ASN1F_SEQUENCE_OF, self).i2repr(pkt, x) # type: ignore + else: + return "[%s]" % ", ".join( + self.fld.i2repr(pkt, x) for x in x # type: ignore + ) + def randval(self): # type: () -> Any if self.holds_packets: return packet.fuzz(self.cls()) else: - return self.fld(self.name, b"").randval() + return self.fld.randval() def __repr__(self): # type: () -> str diff --git a/scapy/layers/kerberos.py b/scapy/layers/kerberos.py index a3e7ec4212b..7b4c06f7790 100644 --- a/scapy/layers/kerberos.py +++ b/scapy/layers/kerberos.py @@ -579,7 +579,7 @@ def m2i(self, pkt, s): 141: "KERB-AUTH-DATA-TOKEN-RESTRICTIONS", 142: "KERB-LOCAL", 143: "AD-AUTH-DATA-AP-OPTIONS", - 144: "AD-TARGET-PRINCIPAL", # not an official name + 144: "KERB-AUTH-DATA-CLIENT-TARGET", } @@ -665,11 +665,13 @@ class AD_AND_OR(ASN1_Packet): 144: "PA-OTP-PIN-CHANGE", 145: "PA-EPAK-AS-REQ", 146: "PA-EPAK-AS-REP", - 147: "PA_PKINIT_KX", - 148: "PA_PKU2U_NAME", + 147: "PA-PKINIT-KX", + 148: "PA-PKU2U-NAME", 149: "PA-REQ-ENC-PA-REP", - 150: "PA_AS_FRESHNESS", + 150: "PA-AS-FRESHNESS", 151: "PA-SPAKE", + 161: "KERB-KEY-LIST-REQ", + 162: "KERB-KEY-LIST-REP", 165: "PA-SUPPORTED-ENCTYPES", 166: "PA-EXTENDED-ERROR", 167: "PA-PAC-OPTIONS", @@ -885,6 +887,7 @@ class KERB_AUTH_DATA_AP_OPTIONS(Packet): 0x4000, { 0x4000: "KERB_AP_OPTIONS_CBT", + 0x8000: "KERB_AP_OPTIONS_UNVERIFIED_TARGET_NAME", }, ), ] @@ -893,18 +896,17 @@ class KERB_AUTH_DATA_AP_OPTIONS(Packet): _AUTHORIZATIONDATA_VALUES[143] = KERB_AUTH_DATA_AP_OPTIONS -# This has no doc..? not in [MS-KILE] at least. -# We use the name wireshark/samba gave it +# This has no doc..? [MS-KILE] only mentions its name. -class KERB_AD_TARGET_PRINCIPAL(Packet): +class KERB_AUTH_DATA_CLIENT_TARGET(Packet): name = "KERB-AD-TARGET-PRINCIPAL" fields_desc = [ StrFieldUtf16("spn", ""), ] -_AUTHORIZATIONDATA_VALUES[144] = KERB_AD_TARGET_PRINCIPAL +_AUTHORIZATIONDATA_VALUES[144] = KERB_AUTH_DATA_CLIENT_TARGET # RFC6806 sect 6 @@ -969,6 +971,69 @@ class PA_PAC_OPTIONS(ASN1_Packet): _PADATA_CLASSES[167] = PA_PAC_OPTIONS +# [MS-KILE] sect 2.2.11 + + +class KERB_KEY_LIST_REQ(ASN1_Packet): + ASN1_codec = ASN1_Codecs.BER + ASN1_root = ASN1F_SEQUENCE_OF( + "keytypes", + [], + ASN1F_enum_INTEGER("", 0, _KRB_E_TYPES), + ) + + +_PADATA_CLASSES[161] = KERB_KEY_LIST_REQ + +# [MS-KILE] sect 2.2.12 + + +class KERB_KEY_LIST_REP(ASN1_Packet): + ASN1_codec = ASN1_Codecs.BER + ASN1_root = ASN1F_SEQUENCE_OF( + "keys", + [], + ASN1F_PACKET("", None, EncryptionKey), + ) + + +_PADATA_CLASSES[162] = KERB_KEY_LIST_REP + +# [MS-KILE] sect 2.2.13 + + +class KERB_SUPERSEDED_BY_USER(ASN1_Packet): + ASN1_codec = ASN1_Codecs.BER + ASN1_root = ASN1F_SEQUENCE( + ASN1F_PACKET("name", None, PrincipalName, explicit_tag=0xA0), + Realm("realm", None, explicit_tag=0xA1), + ) + + +# [MS-KILE] sect 2.2.14 + + +class KERB_DMSA_KEY_PACKAGE(ASN1_Packet): + ASN1_codec = ASN1_Codecs.BER + ASN1_root = ASN1F_SEQUENCE( + ASN1F_SEQUENCE_OF( + "currentKeys", + [], + ASN1F_PACKET("", None, EncryptionKey), + explicit_tag=0xA0, + ), + ASN1F_optional( + ASN1F_SEQUENCE_OF( + "previousKeys", + [], + ASN1F_PACKET("", None, EncryptionKey), + explicit_tag=0xA0, + ), + ), + KerberosTime("expirationInterval", GeneralizedTime(), explicit_tag=0xA2), + KerberosTime("fetchInterval", GeneralizedTime(), explicit_tag=0xA4), + ) + # RFC6113 sect 5.4.1 @@ -1737,7 +1802,8 @@ def m2i(self, pkt, s): # 24: KDC_ERR_PREAUTH_FAILED # 25: KDC_ERR_PREAUTH_REQUIRED return MethodData(val[0].val, _underlayer=pkt), val[1] - elif pkt.errorCode.val in [18, 29, 41, 60]: + elif pkt.errorCode.val in [13, 18, 29, 41, 60]: + # 13: KDC_ERR_BADOPTION # 18: KDC_ERR_CLIENT_REVOKED # 29: KDC_ERR_SVC_UNAVAILABLE # 41: KRB_AP_ERR_MODIFIED @@ -2444,6 +2510,7 @@ def __init__( additional_tickets=[], u2u=False, for_user=None, + s4u2proxy=False, etypes=None, key=None, port=88, @@ -2519,6 +2586,7 @@ def __init__( self.additional_tickets = additional_tickets # U2U + S4U2Proxy self.u2u = u2u # U2U self.for_user = for_user # FOR-USER + self.s4u2proxy = s4u2proxy # S4U2Proxy self.key = key # See RFC4120 - sect 7.2.2 # This marks whether we should follow-up after an EOF @@ -2674,6 +2742,20 @@ def tgs_req(self): ) ) + # [MS-SFU] S4U2proxy - sect 3.1.5.2.1 + if self.s4u2proxy: + # "PA-PAC-OPTIONS with resource-based constrained-delegation bit set" + tgsreq.root.padata.append( + PADATA( + padataType=ASN1_INTEGER(167), # PA-PAC-OPTIONS + padataValue=PA_PAC_OPTIONS( + options="Resource-based-constrained-delegation", + ), + ) + ) + # "kdc-options field: MUST include the new cname-in-addl-tkt options flag" + kdc_req.kdcOptions.set(14, 1) + # Compute checksum if self.key.cksumtype: authenticator.cksum = Checksum() @@ -2901,6 +2983,7 @@ def krb_tgs_req( u2u=False, etypes=None, for_user=None, + s4u2proxy=False, **kwargs, ): r""" @@ -2950,6 +3033,7 @@ def krb_tgs_req( u2u=u2u, etypes=etypes, for_user=for_user, + s4u2proxy=s4u2proxy, **kwargs, ) cli.run() diff --git a/scapy/layers/l2.py b/scapy/layers/l2.py index 427b11d3592..120868dbb9e 100644 --- a/scapy/layers/l2.py +++ b/scapy/layers/l2.py @@ -59,8 +59,18 @@ _PacketList, ) from scapy.sendrecv import sendp, srp, srp1, srploop -from scapy.utils import checksum, hexdump, hexstr, inet_ntoa, inet_aton, \ - mac2str, valid_mac, valid_net, valid_net6 +from scapy.utils import ( + checksum, + hexdump, + hexstr, + inet_aton, + inet_ntoa, + mac2str, + pretty_list, + valid_mac, + valid_net, + valid_net6, +) # Typing imports from typing import ( @@ -987,18 +997,20 @@ def show(self, *args, **kwargs): """ Print the list of discovered MAC addresses. """ - - data = list() - padding = 0 + data = list() # type: List[Tuple[str | List[str], ...]] for s, r in self.res: manuf = conf.manufdb._get_short_manuf(r.src) manuf = "unknown" if manuf == r.src else manuf - padding = max(padding, len(manuf)) data.append((r[Ether].src, manuf, r[ARP].psrc)) - for src, manuf, psrc in data: - print(" %-17s %-*s %s" % (src, padding, manuf, psrc)) + print( + pretty_list( + data, + [("src", "manuf", "psrc")], + sortBy=2, + ) + ) @conf.commands.register diff --git a/test/scapy/layers/http.uts b/test/scapy/layers/http.uts index 2b2134e2f0c..5d5a4642530 100644 --- a/test/scapy/layers/http.uts +++ b/test/scapy/layers/http.uts @@ -297,7 +297,11 @@ class run_httpserver: print("@ Server started !") def __exit__(self, exc_type, exc_value, traceback): print("@ Stopping http server !") - self.server.shutdown(socket.SHUT_RDWR) + try: + self.server.shutdown(socket.SHUT_RDWR) + except OSError: + pass + self.server.close() if traceback: # failed print("\nTest failed.") diff --git a/test/scapy/layers/l2.uts b/test/scapy/layers/l2.uts index d4d5185670d..4f05574768c 100644 --- a/test/scapy/layers/l2.uts +++ b/test/scapy/layers/l2.uts @@ -25,7 +25,7 @@ with ContextManagerCaptureOutput() as cmco: ar.show() result_ar = cmco.get_output() -assert result_ar.startswith(" 70:ee:50:50:ee:70 Netatmo 192.168.0.1") +assert "70:ee:50:50:ee:70 Netatmo 192.168.0.1" in result_ar = arp_mitm - IP to IP ~ arp_mitm