Implementing Relay Descriptor verification

cf https://trac.torproject.org/projects/tor/ticket/5810

1) Implemented relay descriptor verification using the python-crypto lib.
   Code is only run if python-crypto can be imported. [cf stem.prereq.is_crypto_available()]
   NOTE: constructing a RelayDescriptor will now raise an exception if invalid descriptor content is used.
2) Refactored the digest() function in server_descriptor.py.
3) Added a function to the mocking lib to sign a descriptor with an auto-generated key
4) Add usage of new sign_descriptor_content() in unit tests where necessary.
5) Updated the non-ascii-descriptor file to be correctly signed.
6) Updated extra info descriptor test to use new fingerprint in non-ascii-descriptor file
7) Removed server descriptor tests that do not make sense if data is being generated dynamically.
   e.g. Removed test fingerprint valid test, since data now dynamically generated.

stem/descriptor/server_descriptor.py

@@ -39,6 +39,7 @@
 import stem.version
 import stem.util.connection
 import stem.util.tor_tools
+import stem.util.log as log
 
 # relay descriptors must have exactly one of the following
 REQUIRED_FIELDS = (
@@ -593,52 +594,123 @@ def __init__(self, raw_contents, validate = True, annotations = None):
 
     super(RelayDescriptor, self).__init__(raw_contents, validate, annotations)
 
-    # if we have a fingerprint then checks that our fingerprint is a hash of
-    # our signing key
+    # validate the descriptor if required
+    if validate:
+      # ensure the digest of the descriptor has been calculated
+      self.digest()
+      self._validate_content()
+
+  def digest(self):
+    # Digest is calculated from everything in the
+    # descriptor except the router-signature.
+    raw_descriptor = str(self)
+    start_token = "router "
+    sig_token = "\nrouter-signature\n"
+    start = raw_descriptor.find(start_token)
+    sig_start = raw_descriptor.find(sig_token)
+    end = sig_start + len(sig_token)
+    if start >= 0 and sig_start > 0 and end > start:
+      for_digest = raw_descriptor[start:end]
+      digest_hash = hashlib.sha1(for_digest)
+      self._digest = digest_hash.hexdigest()
+    else:
+      log.warn("unable to calculate digest for descriptor")
+      raise ValueError("unable to calculate digest for descriptor")
 
-    if validate and self.fingerprint and stem.prereq.is_rsa_available():
-      import rsa
-      pubkey = rsa.PublicKey.load_pkcs1(self.signing_key)
-      der_encoded = pubkey.save_pkcs1(format = "DER")
-      key_hash = hashlib.sha1(der_encoded).hexdigest()
-
-      if key_hash != self.fingerprint.lower():
-        raise ValueError("Hash of our signing key doesn't match our fingerprint. Signing key hash: %s, fingerprint: %s" % (key_hash, self.fingerprint.lower()))
+    return self._digest
 
-  def is_valid(self):
+  def _validate_content(self):
     """
     Validates that our content matches our signature.
     
-    **Method implementation is incomplete, and will raise a NotImplementedError**
-    
-    :returns: **True** if our signature matches our content, **False** otherwise
+    :raises a ValueError if signature does not match content,
     """
 
-    raise NotImplementedError # TODO: finish implementing
-
-    # without validation we may be missing our signature
-    if not self.signature: return False
-
-    # gets base64 encoded bytes of our signature without newlines nor the
-    # "-----[BEGIN|END] SIGNATURE-----" header/footer
+    if not self.signature:
+      log.warn("Signature missing")
+      raise ValueError("Signature missing")
 
-    sig_content = self.signature.replace("\n", "")[25:-23]
-    sig_bytes = base64.b64decode(sig_content)
+    # strips off the '-----BEGIN RSA PUBLIC KEY-----' header and corresponding footer
+    key_as_string = ''.join(self.signing_key.split('\n')[1:4])
 
-    # TODO: Decrypt the signature bytes with the signing key and remove
-    # the PKCS1 padding to get the original message, and encode the message
-    # in hex and compare it to the digest of the descriptor.
+    # calculate the signing key hash
+    key_as_der = base64.b64decode(key_as_string)
+    key_der_as_hash = hashlib.sha1(key_as_der).hexdigest()
 
-    return True
-
-  def digest(self):
-    if self._digest is None:
-      # our digest is calculated from everything except our signature
-      raw_content, ending = str(self), "\nrouter-signature\n"
-      raw_content = raw_content[:raw_content.find(ending) + len(ending)]
-      self._digest = hashlib.sha1(raw_content).hexdigest().upper()
-
-    return self._digest
+    # if we have a fingerprint then check that our fingerprint is a hash of
+    # our signing key
+    if self.fingerprint:
+      if key_der_as_hash != self.fingerprint.lower():
+        log.warn("Hash of our signing key doesn't match our fingerprint. Signing key hash: %s, fingerprint: %s" % (key_der_as_hash, self.fingerprint.lower()))
+        raise ValueError("Fingerprint does not match hash")
+    else:
+      log.notice("No fingerprint for this descriptor")
+
+    try:
+      self._verify_descriptor(key_as_der)
+      log.info("Descriptor verified.")
+    except ValueError, e:
+      log.warn("Failed to verify descriptor: %s" % e)
+      raise e
+
+  def _verify_descriptor(self, key_as_der):
+    if not stem.prereq.is_crypto_available():
+      return
+    else:
+      from Crypto.Util import asn1
+      from Crypto.Util.number import bytes_to_long, long_to_bytes
+
+      # get the ASN.1 sequence
+      seq = asn1.DerSequence()
+      seq.decode(key_as_der)
+      modulus = seq[0]
+      public_exponent = seq[1] #should always be 65537
+
+      # convert the descriptor signature to an int before decrypting it
+      sig_as_string = ''.join(self.signature.split('\n')[1:4])
+      sig_as_bytes = base64.b64decode(sig_as_string)
+      sig_as_long = bytes_to_long(sig_as_bytes)
+
+      # use the public exponent[e] & the modulus[n] to decrypt the int
+      decrypted_int = pow(sig_as_long, public_exponent ,modulus)
+      # block size will always be 128 for a 1024 bit key
+      blocksize = 128
+      # convert the int to a byte array.
+      decrypted_bytes = long_to_bytes(decrypted_int, blocksize)
+
+      ############################################################################
+      ## The decrypted bytes should have a structure exactly along these lines.
+      ## 1 byte  - [null '\x00']
+      ## 1 byte  - [block type identifier '\x01'] - Should always be 1
+      ## N bytes - [padding '\xFF' ]
+      ## 1 byte  - [separator '\x00' ]
+      ## M bytes - [message]
+      ## Total   - 128 bytes
+      ## More info here http://www.ietf.org/rfc/rfc2313.txt
+      ##                esp the Notes in section 8.1
+      ############################################################################
+      try:
+        if decrypted_bytes.index('\x00\x01') != 0:
+          log.warn("Verification failed, identifier missing")
+          raise ValueError("Verification failed, identifier missing")
+      except ValueError:
+        log.warn("Verification failed, Malformed data")
+        raise ValueError("Verification failed, Malformed data")
+
+      try:
+        identifier_offset = 2
+        # Find the separator
+        seperator_index = decrypted_bytes.index('\x00', identifier_offset)
+      except ValueError:
+        log.warn("Verification failed, seperator not found")
+        raise ValueError("Verification failed, seperator not found")
+
+      digest = decrypted_bytes[seperator_index+1:]
+      # The local digest is stored in hex so need to encode the decrypted digest
+      digest_hex = digest.encode('hex')
+      if digest_hex != self._digest:
+        log.warn("Decrypted digest does not match local digest")
+        raise ValueError("Decrypted digest does not match local digest")
 
   def _parse(self, entries, validate):
     entries = dict(entries) # shallow copy since we're destructive

stem/prereq.py

@@ -24,7 +24,7 @@
 
 import stem.util.log as log
 
-IS_RSA_AVAILABLE = None
+IS_CRYPTO_AVAILABLE = None
 
 def check_requirements():
   """
@@ -59,20 +59,23 @@ def is_python_27():
 
   return _check_version(7)
 
-def is_rsa_available():
-  global IS_RSA_AVAILABLE
+def is_crypto_available():
+  global IS_CRYPTO_AVAILABLE
 
-  if IS_RSA_AVAILABLE == None:
+  if IS_CRYPTO_AVAILABLE == None:
     try:
-      import rsa
-      IS_RSA_AVAILABLE = True
+      from Crypto.PublicKey import RSA
+      from Crypto.Util import asn1
+      from Crypto.Util.number import long_to_bytes
+      IS_CRYPTO_AVAILABLE = True
     except ImportError:
-      IS_RSA_AVAILABLE = False
+      IS_CRYPTO_AVAILABLE = False
 
-      msg = "Unable to import the rsa module. Because of this we'll be unable to verify descriptor signature integrity."
-      log.log_once("stem.prereq.is_rsa_available", log.INFO, msg)
+      # the code that verifies relay descriptor signatures uses the python-crypto library
+      msg = "Unable to import the crypto module. Because of this we'll be unable to verify descriptor signature integrity."
+      log.log_once("stem.prereq.is_crypto_available", log.INFO, msg)
 
-  return IS_RSA_AVAILABLE
+  return IS_CRYPTO_AVAILABLE
 
 def _check_version(minor_req):
   major_version, minor_version = sys.version_info[0:2]

test/integ/descriptor/data/non-ascii_descriptor

@@ -3,7 +3,7 @@ router torrelay389752132 130.243.230.116 9001 0 0
 platform Tor 0.2.2.35 (git-4f42b0a93422f70e) on Linux x86_64
 opt protocols Link 1 2 Circuit 1
 published 2012-03-21 16:28:14
-opt fingerprint FEBC 7F99 2AC4 18BB E42B C13F E94E FCFE 6549 197E
+opt fingerprint 5D47 E91A 1F74 21A4 E325 5F4D 04E5 34E9 A214 07BB
 uptime 3103848
 bandwidth 81920 102400 84275
 opt extra-info-digest 51E9FD0DA7C235D8C0250BAFB6E1ABB5F1EF9F04
@@ -15,16 +15,16 @@ k3Rx75up+wsuBzhfwSYr7W+T+WkDQvz49RFPpns6Ef0qFpQ1TlHxAgMBAAE=
 -----END RSA PUBLIC KEY-----
 signing-key
 -----BEGIN RSA PUBLIC KEY-----
-MIGJAoGBAMSmtutGlXVdvh/IC4TyhQpgSajxrZItC2lS5/70Vr4uLevryPlBgVrW
-35CHxKYaj0MAOfkJQ0/OvTaXe7hlaCLrDDXScaH/XEDurcWrynsdzomsCvn/6VJ+
-xZFszt2Dn5myXKMvYy3j1oevC4iDaZXwxgpwx/UMJsFn7GOUPFYbAgMBAAE=
+MIGJAoGBALRHQWXGjGLNROY8At3dMnrcSxw4PF/9oLYuqCsXNAq0Gju+EBA5qfM4
+AMpeOk+7ZsZ6AsjdBPAPaOf7hm+z6Kr3Am/gC43dci+iuNHf2wYLR8TnW/C5Q6ZQ
+iXpSAGrOHnIptyPHa0j9ayM4WmHWrPBKnC0QA91CGrxnnNc6DHehAgMBAAE=
 -----END RSA PUBLIC KEY-----
 opt hidden-service-dir
 contact 2048R/F171EC1F Johan Blåbäck こんにちは
 reject *:*
 router-signature
 -----BEGIN SIGNATURE-----
-q3Tw41+miuycnXowX/k6CCOcHMjw0BCDjW56Wh/eHoICmVb/hBJdtuzTaorWHLWp
-OoTa4Sy4OrGFL+ldzajGC8+oqMvrYudiIxbJWmH3NXFyd7ZeEdnHzHxNOU8p1+X+
-hFwdOCEvzvvTbOuS2DwDt+TU8rljZunZfcMWgXktAD0=
+WqBgiomhJ+XewpbOGg1r+6KXlAkdxHRhgCB/D980yJVzXWbOCrRhwyyAH9Lx+yrK
+1EFXAtfQBBx2hmsw8CSYuUT6ckjXyUBAKEdABC25yRdi+fN3NfSQd56U9MvArjo9
+Y8oz244gH4BSVp4CScL8dK0EUsUrAxjs+OU7bnV5saA=
 -----END SIGNATURE-----

test/integ/descriptor/server_descriptor.py

@@ -86,7 +86,7 @@ def test_metrics_descriptor(self):
     self.assertEquals(expected_signing_key, desc.signing_key)
     self.assertEquals(expected_signature, desc.signature)
     self.assertEquals([], desc.get_unrecognized_lines())
-    self.assertEquals("2C7B27BEAB04B4E2459D89CA6D5CD1CC5F95A689", desc.digest())
+    self.assertEquals("2C7B27BEAB04B4E2459D89CA6D5CD1CC5F95A689", desc.digest().upper())
 
   def test_old_descriptor(self):
     """
@@ -190,7 +190,7 @@ def test_non_ascii_descriptor(self):
 
     desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents)
     self.assertEquals("torrelay389752132", desc.nickname)
-    self.assertEquals("FEBC7F992AC418BBE42BC13FE94EFCFE6549197E", desc.fingerprint)
+    self.assertEquals("5D47E91A1F7421A4E3255F4D04E534E9A21407BB", desc.fingerprint)
     self.assertEquals("130.243.230.116", desc.address)
     self.assertEquals(9001, desc.or_port)
     self.assertEquals(None, desc.socks_port)

test/mocking.py

@@ -46,6 +46,9 @@
       get_router_status_entry_micro_v3 - RouterStatusEntryMicroV3
 """
 
+
+import base64
+import hashlib
 import inspect
 import itertools
 import StringIO
@@ -541,6 +544,7 @@ def get_relay_server_descriptor(attr = None, exclude = (), content = False):
   if content:
     return desc_content
   else:
+    desc_content = sign_descriptor_content(desc_content)
     return stem.descriptor.server_descriptor.RelayDescriptor(desc_content, validate = True)
 
 def get_bridge_server_descriptor(attr = None, exclude = (), content = False):
@@ -783,3 +787,84 @@ def get_network_status_document_v3(attr = None, exclude = (), authorities = None
   else:
     return stem.descriptor.networkstatus.NetworkStatusDocumentV3(desc_content, validate = True)
 
+def sign_descriptor_content(desc_content):
+
+  if not stem.prereq.is_crypto_available():
+    return desc_content
+  else:
+    from Crypto.PublicKey import RSA
+    from Crypto.Util import asn1
+    from Crypto.Util.number import long_to_bytes
+
+    # generate a key
+    private_key = RSA.generate(1024)
+
+    # get a string representation of the public key
+    seq = asn1.DerSequence()
+    seq.append(private_key.n)
+    seq.append(private_key.e)
+    seq_as_string = seq.encode()
+    public_key_string = base64.b64encode(seq_as_string)
+
+    # split public key into lines 64 characters long
+    public_key_string =  public_key_string [:64] + "\n" +public_key_string[64:128] +"\n" +public_key_string[128:]
+
+    # generate the new signing key string
+    signing_key_token = "\nsigning-key\n" #note the trailing '\n' is important here so as not to match the string elsewhere
+    signing_key_token_start = "-----BEGIN RSA PUBLIC KEY-----\n"
+    signing_key_token_end = "\n-----END RSA PUBLIC KEY-----\n"
+    new_sk = signing_key_token+ signing_key_token_start+public_key_string+signing_key_token_end
+
+    # update the descriptor string with the new signing key
+    skt_start = desc_content.find(signing_key_token)
+    skt_end = desc_content.find(signing_key_token_end, skt_start)
+    desc_content = desc_content[:skt_start]+new_sk+ desc_content[skt_end+len(signing_key_token_end):]
+
+    # generate the new fingerprint string
+    key_hash = hashlib.sha1(seq_as_string).hexdigest().upper()
+    grouped_fingerprint = ""
+    for x in range(0, len(key_hash), 4):
+      grouped_fingerprint += " " + key_hash[x:x+4]
+      fingerprint_token = "\nfingerprint"
+      new_fp = fingerprint_token + grouped_fingerprint
+
+    # update the descriptor string with the new fingerprint
+    ft_start = desc_content.find(fingerprint_token)
+    if ft_start < 0:
+      fingerprint_token = "\nopt fingerprint"
+      ft_start = desc_content.find(fingerprint_token)
+
+    # if the descriptor does not already contain a fingerprint do not add one
+    if ft_start >= 0:
+      ft_end = desc_content.find("\n", ft_start+1)
+      desc_content = desc_content[:ft_start]+new_fp+desc_content[ft_end:]
+
+    # calculate the new digest for the descriptor
+    tempDesc = stem.descriptor.server_descriptor.RelayDescriptor(desc_content, validate=False)
+    new_digest_hex = tempDesc.digest()
+    # remove the hex encoding
+    new_digest = new_digest_hex.decode('hex')
+
+    # Generate the digest buffer.
+    #  block is 128 bytes in size
+    #  2 bytes for the type info
+    #  1 byte for the separator
+    padding = ""
+    for x in range(125 - len(new_digest)):
+      padding += '\xFF'
+      digestBuffer = '\x00\x01' + padding + '\x00' + new_digest
+
+    # generate a new signature by signing the digest buffer with the private key
+    (signature, ) = private_key.sign(digestBuffer, None)
+    signature_as_bytes = long_to_bytes(signature, 128)
+    signature_base64 = base64.b64encode(signature_as_bytes)
+    signature_base64 =  signature_base64 [:64] + "\n" +signature_base64[64:128] +"\n" +signature_base64[128:]
+
+    # update the descriptor string with the new signature
+    router_signature_token = "\nrouter-signature\n"
+    router_signature_start = "-----BEGIN SIGNATURE-----\n"
+    router_signature_end = "\n-----END SIGNATURE-----\n"
+    rst_start = desc_content.find(router_signature_token)
+    desc_content = desc_content[:rst_start] + router_signature_token + router_signature_start + signature_base64 + router_signature_end
+
+    return desc_content