diff --git a/.gitattributes b/.gitattributes index 7ddf3f340cb..0250658a1b5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ scapy/__init__.py export-subst * text=auto +*.sh text eol=lf *.bat text eol=crlf diff --git a/doc/scapy/_static/vethrelay.sh b/doc/scapy/_static/vethrelay.sh index 0e62f7b4903..d39767d62a8 100755 --- a/doc/scapy/_static/vethrelay.sh +++ b/doc/scapy/_static/vethrelay.sh @@ -1,7 +1,8 @@ #!/bin/bash -# Setup iptables for IP relay by creating an interface configured -# to be the destination of TPROXY rules. +# Setup nftables for IP relay by creating an interface configured +# to be the destination of TPROXY rules. All nft rules live in a +# dedicated 'scapy-tproxy' table. if [ "$EUID" -ne 0 ] then echo "Please run as root" @@ -14,19 +15,31 @@ if [ "$1" != "setup" ] && [ "$1" != "unsetup" ]; then fi IFACE="vethrelay" -IP="2.2.2.2" +NFT_TABLE="scapy-tproxy" + +if [ "$1" == "setup" ]; then + if ! modprobe nft_tproxy; then + echo -e "\x1b[31mError: nft_tproxy is required for transparent proxying.\x1b[0m" >&2 + exit 1 + fi + NFT_SOCKET_AVAILABLE=true + if ! modprobe nft_socket; then + echo -e "\x1b[33mWarning: nft_socket is unavailable; skipping the socket-match optimisation.\x1b[0m" >&2 + NFT_SOCKET_AVAILABLE=false + fi +fi # Linux doc about TPROXY and example regarding this: # https://www.kernel.org/doc/Documentation/networking/tproxy.txt # https://powerdns.org/tproxydoc/tproxy.md.html function checkSetup() { - iptables -t mangle -n --list "DIVERT" >/dev/null 2>&1 + nft list table ip "$NFT_TABLE" >/dev/null 2>&1 return $? } if [ "$1" == "setup" ]; then - # Add "DIVERT" chain if it doesn't exist + # Add the scapy-tproxy table if it doesn't exist checkSetup if [ $? -eq 0 ]; then echo "vethrelay already setup !" @@ -36,39 +49,57 @@ if [ "$1" == "setup" ]; then ip link add dev $IFACE type dummy sysctl net.ipv6.conf.$IFACE.disable_ipv6=1 >/dev/null ip link set dev $IFACE up - ip addr add dev $IFACE $IP/32 - # Create mangle "DIVERT" chain as an optimisation. -m socket matches - # packets from already established sockets. Those are marked as 1 then - # accepted directly. - iptables -t mangle -N DIVERT - iptables -t mangle -A PREROUTING -p tcp -m socket -j DIVERT - iptables -t mangle -A DIVERT -j MARK --set-mark 1 - iptables -t mangle -A DIVERT -j ACCEPT - # Packets marked with 1 are routed through table 100 instead of the - # default routing table + # All TPROXY rules live in a dedicated nftables table so that + # iptables-nft (which manages the ip filter/nat/mangle tables) + # is left untouched. + # The DIVERT chain is an optimisation. The socket match catches + # packets from already established sockets. Those are marked as 1 + # then accepted directly so that TPROXY does not run again. + nft add table ip "$NFT_TABLE" + nft add chain ip "$NFT_TABLE" DIVERT "{ type filter hook prerouting priority -150; }" + if [ "$NFT_SOCKET_AVAILABLE" == "true" ]; then + nft add rule ip "$NFT_TABLE" DIVERT meta l4proto tcp socket transparent 1 meta mark set 1 accept + fi + # Also add an OUTPUT rule so that we can test from loopback + nft add chain ip "$NFT_TABLE" OUTPUT "{ type route hook output priority mangle; policy accept; }" + # Packets marked with 1 -> use table 100 ip rule add fwmark 1 lookup 100 # In routing table 100, all IPs are local to 'vethrelay' ip route add local 0.0.0.0/0 dev $IFACE table 100 - echo -e "\x1b[32mInterface $IFACE is now setup with IPv4: $IP !\x1b[0m\n" - echo -e "Add listening rules as follow:\n" - echo "# TPROXY incoming TCP packets on port 80 to $IFACE on port 8080" - echo "iptables -t mangle -A PREROUTING -p tcp --dport 80 -j TPROXY --tproxy-mark 0x1/0x1 --on-port 8080 --on-ip $IP" + + echo -e "\n\x1b[32mInterface $IFACE is now setup !\x1b[0m" + echo -e "\n\x1b[1mIncoming (FORWARD) traffic\x1b[0m" + echo " Redirect TCP port 80 to the proxy on port 8080:" + echo + printf '\033[36m nft add rule ip %s DIVERT \\\n tcp dport 80 tproxy to :8080 meta mark set 0x1 accept\033[0m\n' "$NFT_TABLE" + echo + echo -e "\x1b[1mOutgoing traffic (optional)\x1b[0m" + echo " Outgoing traffic also requires the Incoming (FORWARD) rule above." + echo " To redirect outgoing TCP port 80, additionally add:" + echo " Priority 1 excludes the proxy's upstream connections." + echo + printf '\033[36m nft add rule ip %s OUTPUT \\\n meta priority != 0:1 tcp dport 80 meta mark set 0x1\033[0m\n' "$NFT_TABLE" + echo + echo -e "\x1b[1mAllow incoming traffic\x1b[0m" + echo " Allow INPUT on the original destination port and incoming interface." + echo " Example: TCP port 80 arriving on wlp4s0 (replace ):" + echo + printf '\033[36m nft add rule ip INPUT \\\n iifname wlp4s0 tcp dport 80 accept\033[0m\n' + echo + echo " Or, using iptables:" + echo + echo -e "\x1b[36m iptables -A INPUT -i wlp4s0 -p tcp --dport 80 -j ACCEPT\x1b[0m" echo - echo "# Listen on wlp4s0 for incoming packets on port 80 (on the interface where it really comes from)" - echo "iptables -A INPUT -i wlp4s0 -p tcp --dport 80 -j ACCEPT" elif [ "$1" == "unsetup" ]; then checkSetup if [ $? -ne 0 ]; then echo "vethrelay not setup !" exit 1 fi - # Remove all setup rules - sudo ip rule del fwmark 1 lookup 100 - sudo ip route del local 0.0.0.0/0 dev $IFACE table 100 - sudo iptables -t mangle -D DIVERT -j ACCEPT - sudo iptables -t mangle -D DIVERT -j MARK --set-mark 1 - sudo iptables -t mangle -D PREROUTING -p tcp -m socket -j DIVERT - sudo iptables -t mangle -X DIVERT - sudo ip link del dev $IFACE + # Remove all setup rules by deleting the whole nftables table + ip rule del fwmark 1 lookup 100 + ip route del local 0.0.0.0/0 dev $IFACE table 100 + nft delete table ip "$NFT_TABLE" + ip link del dev $IFACE echo -e "\x1b[32mInterface $IFACE unsetup !\x1b[0m" fi diff --git a/doc/scapy/advanced_usage/fwdmachine.rst b/doc/scapy/advanced_usage/fwdmachine.rst index 6f5b979a7f8..38ede35d143 100644 --- a/doc/scapy/advanced_usage/fwdmachine.rst +++ b/doc/scapy/advanced_usage/fwdmachine.rst @@ -38,7 +38,7 @@ initial destinations afterwards. More details on how to set up TPROXY are provid # Run it NOPFwdMachine( mode=ForwardMachine.MODE.TPROXY, - port=80, + port=8080, cls=HTTP, # we specify the class of the payload we are receiving ).run() diff --git a/scapy/base_classes.py b/scapy/base_classes.py index c91cc25fff1..e2d42b77218 100644 --- a/scapy/base_classes.py +++ b/scapy/base_classes.py @@ -310,6 +310,8 @@ def __contains__(self, other): if isinstance(other, int): return self.start <= other <= self.stop if isinstance(other, str): + if other == "%": + return isinstance(self, (Net, _ScopedIP)) and bool(self.scope) return self.__class__(other) in self if type(other) is not self.__class__: return False diff --git a/scapy/fwdmachine.py b/scapy/fwdmachine.py index 31e0a5eca0a..8284cc10e23 100644 --- a/scapy/fwdmachine.py +++ b/scapy/fwdmachine.py @@ -18,6 +18,7 @@ from scapy.asn1.asn1 import ASN1_OID from scapy.config import conf +from scapy.consts import LINUX from scapy.data import MTU from scapy.packet import Packet from scapy.supersocket import StreamSocket, StreamSocketPeekless @@ -27,6 +28,7 @@ from scapy.layers.tls.all import ( Cert, + CertTree, PrivKeyECDSA, ) from scapy.layers.x509 import ( @@ -77,8 +79,9 @@ class ForwardMachine: :param remote_port: the port to use in SERVER mode. (else use 'port') :param remote_af: (optional) if provided, use a different address family to connect to the remote host. - :param bind_address: the IP to bind locally. "0.0.0.0" by default in SERVER mode, - but "2.2.2.2" by default in TPROXY (if you are using the provided + :param bind_address: the IP to bind locally. "0.0.0.0" by default + :param iface: the interface to bind (defaults to ANY in SERVER, + but "vethrelay" by default in TPROXY (if you are using the provided 'vethrelay.sh' script). :param tls: enable TLS (in both the server and client) :param no_check_certificate: with TLS, do not check the certificate of the @@ -115,6 +118,7 @@ def __init__( remote_port: int = None, remote_af: Optional[socket.AddressFamily] = None, bind_address: str = None, + iface: str = None, tls: bool = False, no_check_certificate: bool = False, crtfile: Optional[str] = None, @@ -143,18 +147,14 @@ def __init__( self.sockcls = StreamSocketPeekless else: self.sockcls = StreamSocket - # Chose 'bind_address' depending on the mode - self.bind_address = bind_address - if self.bind_address is None: - if self.mode == ForwardMachine.MODE.SERVER: - self.bind_address = "0.0.0.0" - elif self.mode == ForwardMachine.MODE.TPROXY: - self.bind_address = "2.2.2.2" - else: - raise ValueError("Unknown mode :/") - red = lambda z: functools.reduce(lambda x, y: x + y, z) + self.bind_address = bind_address or "0.0.0.0" + # Chose 'iface' depending on the mode + if iface is None and self.mode == ForwardMachine.MODE.TPROXY: + iface = "vethrelay" + self.iface = iface # Utils self.ct = DefaultTheme() + red = lambda z: functools.reduce(lambda x, y: x + y, z) self.local_ips = red(red(list(x.ips.values())) for x in conf.ifaces.values()) self.cache = {} super(ForwardMachine, self).__init__(**kwargs) @@ -167,6 +167,12 @@ def run(self): self.ssock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if self.mode == ForwardMachine.MODE.TPROXY: self.ssock.setsockopt(socket.SOL_IP, socket.IP_TRANSPARENT, 1) # TPROXY ! + if self.iface is not None and False: + self.ssock.setsockopt( + socket.SOL_SOCKET, + socket.SO_BINDTODEVICE, + self.iface.encode(), + ) self.ssock.bind((self.bind_address, self.port)) self.ssock.listen(5) print(self.ct.green("Relay server waiting on port %s" % self.port)) @@ -237,10 +243,21 @@ class CONTEXT: CONTEXT object kept during a session """ - def __init__(self, fwdm, addr, dest): + def __init__(self, fwdm, addr, dest, remote_af, proto, tls, cls): self.addr = addr self.dest = dest + self.remote_af = remote_af + self.proto = proto + self.tls = tls self.tls_sni_name = None # Retrieved when receiving a connection + self.cls = cls + + @property + def sockcls(self): + if self.tls or self.remote_af == 40: # TLS or VSOCK + return StreamSocketPeekless + else: + return StreamSocket def vprint(self, evt, ctx, cs, req, rep): if evt == self.FORWARD: @@ -294,7 +311,16 @@ def _getpeersock(self, dest, ctx, server_hostname=None): """ Get peer socket """ - s = socket.socket(self.remote_af, self.proto) + s = socket.socket(ctx.remote_af, ctx.proto) + if ( + LINUX + and self.mode == ForwardMachine.MODE.TPROXY + and ctx.remote_af in (socket.AF_INET, socket.AF_INET6) + ): + # Here's a trick: we set SO_PRIORITY to 1 so that we can exclude those + # packets from the nftables rules that setup the TPROXY when an OUTPUT + # rule is configured. + s.setsockopt(socket.SOL_SOCKET, socket.SO_PRIORITY, 1) s.settimeout(self.timeout) ndest = self.destalias(dest) if ndest != dest: @@ -308,15 +334,21 @@ def gen_alike_chain(self, certs, privkey): Modify a real certificate chain to be served by our own privatekey """ c, certs = certs[0], certs[1:] + + # Set SubjectPublicKeyInfo to the one from our private key + c.setSubjectPublicKeyFromPrivateKey(privkey) + c.updateSubjectKeyIdentifier() + if certs: # Recursive: if there are certificates above this one in the chain, do them # first. certs = self.gen_alike_chain(certs, privkey) + c.updateAuthorityKeyIdentifier(certs[0]) else: # Last certificate of the chain. Make it self-signed c.tbsCertificate.issuer = c.tbsCertificate.subject - # Set SubjectPublicKeyInfo to the one from our private key - c.setSubjectPublicKeyFromPrivateKey(privkey) + c.updateAuthorityKeyIdentifier(c) + # Filter out extensions that would cause trouble c.tbsCertificate.serialNumber.val = int( RandInt() @@ -330,8 +362,6 @@ def gen_alike_chain(self, certs, privkey): "2.5.29.31", # cRLDistributionPoints "1.3.6.1.5.5.7.1.1", # authorityInfoAccess "1.3.6.1.4.1.11129.2.4.2", # SCT - "2.5.29.14", # subjectKeyIdentifier - "2.5.29.35", # authorityKeyIdentifier ] ] # For now, we only provide a RSA private key, so we can only sign with that :/ @@ -355,14 +385,14 @@ def get_key_and_alike_chain(self, cas, dest, server_name): return self.cache[ident] # Parse CAs certs = [Cert(c.public_bytes()) for c in cas] - # certs = certs[:1] + # XXX - Only get last cert. We shouldn't require this + # but we need to add support for multiple private keys... + chain = CertTree(certs, certs) + certs = [chain.getleaves()[0]] # Generate Private Key privkey = PrivKeyECDSA() # Iterate certs = self.gen_alike_chain(certs, privkey) - # Build a chain object. This checks that everything is properly signed, and - # re-order the certs. - # chain = Chain(certs, cert0=certs[-1]) self.cache[ident] = privkey, certs return privkey, certs @@ -370,12 +400,23 @@ def handler(self, sock, addr, dest): """ Handler of a client socket """ - ctx = self.CONTEXT(self, addr, dest) # we have a context object + # we have a context object + ctx = self.CONTEXT( + self, + addr=addr, + dest=dest, + remote_af=self.remote_af, + proto=self.proto, + tls=self.tls, + cls=self.cls, + ) self.newconn(ctx) + # Initialize peer socket ss = self._getpeersock(dest, ctx) + # Wrap both server and peer sockets in SSL - if self.tls: + if ctx.tls: # Build client SSL context if self.no_check_certificate: clisslcontext = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) @@ -453,8 +494,8 @@ def cb_sni(sock, server_name, _): return ss = _clisock[0] # Wrap the sockets - sock = self.sockcls(sock, self.cls) - ss = self.sockcls(ss, self.cls) + sock = ctx.sockcls(sock, ctx.cls) + ss = ctx.sockcls(ss, ctx.cls) sock.streamsession = ss.streamsession try: while True: @@ -471,8 +512,9 @@ def cb_sni(sock, server_name, _): # get data try: data = thissock.recv(self.MTU) - except EOFError: - raise RuntimeError + except EOFError as ex: + othersock.ins.shutdown(socket.SHUT_RDWR) + raise RuntimeError(str(ex)) if not data: # Session needs more data continue @@ -497,7 +539,7 @@ def cb_sni(sock, server_name, _): ctx, server_hostname=ex.server_hostname, ) - ss = self.sockcls(ss, self.cls) + ss = ctx.sockcls(ss, ctx.cls) self.vprint(self.REDIRECT_TO, ctx, 1, ctx.dest, ex.dest) ctx.dest = ex.dest # update context # Shut the old one. @@ -537,8 +579,8 @@ def cb_sni(sock, server_name, _): traceback.print_exception(ex) othersock.send(data) self.vprint(self.FORWARD, ctx, cs, data, None) - except RuntimeError: - print(self.ct.red("%s DISCONNECTED !" % repr(addr))) + except RuntimeError as ex: + print(self.ct.red("%s DISCONNECTED !: %s" % (repr(addr), str(ex)))) self.delconn(ctx) sock.close() ss.close() diff --git a/scapy/layers/http.py b/scapy/layers/http.py index c971ec60c08..59bb1105796 100644 --- a/scapy/layers/http.py +++ b/scapy/layers/http.py @@ -616,7 +616,10 @@ def do_dissect(self, s): return body def mysummary(self): - return self.sprintf("%HTTPRequest.Method% '%HTTPRequest.Path%' ") + if self.Host: + return self.sprintf("%HTTPRequest.Method% '%HTTPRequest.Path%' [%Host%]") + else: + return self.sprintf("%HTTPRequest.Method% '%HTTPRequest.Path%'") class HTTPResponse(_HTTPContent): @@ -741,8 +744,8 @@ def tcp_reassemble(cls, data, metadata, session): # use it. When the total size of the frags is high enough, # we have the packet - if session.pop("head_request", False): - # Answer to a HEAD request. + if session.pop("simple_request", False): + # Answer to a HEAD/CONNECT request. detect_end = lambda dat: dat.find(b"\r\n\r\n") # Subtract the length of the "HTTP*" layer @@ -774,12 +777,12 @@ def tcp_reassemble(cls, data, metadata, session): metadata["detect_unknown"] = True if ( isinstance(http_packet.payload, cls.clsreq) - and http_packet.Method == b"HEAD" + and http_packet.Method in [b"HEAD", b"CONNECT"] ): - session["head_request"] = True + session["simple_request"] = True elif is_response and ( http_packet.Status_Code == b"101" - or session.pop("head_request", False) + or session.pop("simple_request", False) ): # If it's an upgrade response, it may also hold a # different protocol data. make sure all headers are present diff --git a/scapy/layers/tls/cert.py b/scapy/layers/tls/cert.py index f717ead2a3b..04ad61d58d6 100644 --- a/scapy/layers/tls/cert.py +++ b/scapy/layers/tls/cert.py @@ -141,11 +141,14 @@ X509_CRL, X509_Cert, X509_DNSName, + X509_ExtAuthorityKeyIdentifier, + X509_ExtSubjectKeyIdentifier, + X509_Extension, X509_IPAddress, X509_OneAsymmetricKey, X509_SubjectPublicKeyInfo, ) -from scapy.layers.tls.crypto.hash import _get_hash +from scapy.layers.tls.crypto.hash import _get_hash, Hash_SHA from scapy.layers.tls.crypto.pkcs1 import ( _DecryptAndSignRSA, _EncryptAndVerifyRSA, @@ -1109,6 +1112,8 @@ def import_from_asn1pkt(self, cert): self.authorityKeyID = extn.extnValue.keyIdentifier.val elif extn.extnID.oidname == "subjectAltName": self.subjectAltName = extn.extnValue.subjectAltName + elif extn.extnID.oidname == "subjectKeyIdentifier": + self.skid = extn.extnValue.keyIdentifier.val self.signatureValue = bytes(cert.signatureValue) self.signatureLen = len(self.signatureValue) @@ -1165,6 +1170,54 @@ def setSubjectPublicKeyFromPrivateKey(self, key): else: raise ValueError("Unknown type 'key', should be PubKey or PrivKey") + def updateSubjectKeyIdentifier(self): + """ + Update the subjetKeyIdentifier + """ + try: + # Find existing + ski = next( + x.extnValue + for x in self.tbsCertificate.extensions + if x.extnID == "2.5.29.14" + ) + except StopIteration: + # Need to add one + ski = X509_ExtSubjectKeyIdentifier() + self.tbsCertificate.extensions.append(X509_Extension( + extnID=ASN1_OID("subjectKeyIdentifier"), + extnValue=ski + )) + + ski.keyIdentifier = ASN1_STRING( + Hash_SHA().digest(bytes( + self.tbsCertificate.subjectPublicKeyInfo.subjectPublicKey + )) + ) + self.skid = ski.keyIdentifier.val + + def updateAuthorityKeyIdentifier(self, parent: 'Cert'): + """ + Update the authorityKeyIdentifier + """ + try: + # Find existing + aki = next( + x.extnValue + for x in self.tbsCertificate.extensions + if x.extnID == "2.5.29.35" + ) + except StopIteration: + # Need to add one + aki = X509_ExtAuthorityKeyIdentifier() + self.tbsCertificate.extensions.append(X509_Extension( + extnID=ASN1_OID("authorityKeyIdentifier"), + extnValue=aki + )) + + aki.keyIdentifier = ASN1_STRING(parent.skid) + self.authorityKeyID = aki.keyIdentifier.val + def resignWith(self, key): """ Resign a certificate with a specific key @@ -1521,13 +1574,13 @@ def import_from_asn1pkt(self, csr): for x in certReqInfo.attributes if x.type.val == "1.2.840.113549.1.9.14" # extKeyUsage ) - self.sid = next( + self.skid = next( x.extnValue.keyIdentifier for x in extReq.extensions if x.extnID.val == "2.5.29.14" # subjectKeyIdentifier ) except StopIteration: - self.sid = None + self.skid = None @property def certReq(self): @@ -1561,7 +1614,7 @@ def __hash__(self): return hash(self.der) def isIssuer(self, other): - return other.sid == self.sid + return other.skid == self.skid def isSelfSigned(self): return True @@ -1687,7 +1740,7 @@ def findCertBySid(self, sid): if cert.issuer == sid.get_issuer(): return cert elif isinstance(cert, CSR) and isinstance(sid, CMS_SubjectKeyIdentifier): - if cert.sid == sid.sid: + if cert.skid == sid.sid: return cert raise KeyError("Certificate not found !") @@ -1921,6 +1974,19 @@ def verify( % (cert.subject_str, eku.oidname) ) + def getleaves(self): + """ + Return leaves certificates + """ + def _rec_getchain(curtree): + for c, subtree in curtree: + if subtree: + yield from _rec_getchain(subtree) + else: + yield c + + return list(_rec_getchain(self.tree)) + def show(self, ret: bool = False): """ Return the CertTree as a string certificate tree diff --git a/scapy/layers/tls/crypto/h_mac.py b/scapy/layers/tls/crypto/h_mac.py index db984dc22a2..3af543a337a 100644 --- a/scapy/layers/tls/crypto/h_mac.py +++ b/scapy/layers/tls/crypto/h_mac.py @@ -8,7 +8,7 @@ HMAC classes. """ -from scapy.config import conf +from scapy.config import conf, crypto_validator from scapy.layers.tls.crypto.hash import _tls_hash_algs if conf.crypto_valid: @@ -61,6 +61,7 @@ class _GenericHMAC(metaclass=_GenericHMACMetaclass): def __init__(self, key=None): self.key = key or b"" + @crypto_validator def digest(self, tbd): if self.key is None: raise HMACError diff --git a/scapy/supersocket.py b/scapy/supersocket.py index ec78c782dc4..7bcc42a9b6c 100644 --- a/scapy/supersocket.py +++ b/scapy/supersocket.py @@ -543,8 +543,8 @@ def recv(self, x=None, **kwargs): # Block try: data = self.ins.recv(x) - except OSError: - raise EOFError + except OSError as ex: + raise EOFError(str(ex)) try: pkt = self.sess.process(data, cls=self.basecls) # type: ignore except struct.error: diff --git a/test/regression.uts b/test/regression.uts index d0801d670fd..233a95f24bf 100644 --- a/test/regression.uts +++ b/test/regression.uts @@ -4257,6 +4257,8 @@ assert Net("1.2.3.0/24") == Net("1.2.3.0", "1.2.3.255") assert hash(Net("1.2.3.0/24")) == hash(Net("1.2.3.0", "1.2.3.255")) +assert "%" in Net("1.1.1.1", scope=conf.iface) + = Net with a scope identifier iface = conf.iface.network_name