Bug 5170 - simplify i2d_PublicKey in pkey_eq

changes/bug5170

@@ -1,4 +1,5 @@
   o Code simplification and refactoring:
     - Remove contrib/id_to_fp.c since it wasn't used anywhere.
-    - Since OpenSSL 0.9.7 i2d_RSAPublicKey supports allocating output buffer.
-      Use this feature to avoid calling this function twice. Fixes #5170.
+    - Since OpenSSL 0.9.7 i2d_* functions support allocating output
+      buffer. Avoid calling twice: i2d_RSAPublicKey, i2d_DHparams,
+      i2d_X509, i2d_PublicKey. Fixes #5170.

src/common/tortls.c

@@ -986,21 +986,19 @@ pkey_eq(EVP_PKEY *a, EVP_PKEY *b)
   /* We'd like to do this, but openssl 0.9.7 doesn't have it:
      return EVP_PKEY_cmp(a,b) == 1;
   */
-  unsigned char *a_enc=NULL, *b_enc=NULL, *a_ptr, *b_ptr;
-  int a_len1, b_len1, a_len2, b_len2, result;
-  a_len1 = i2d_PublicKey(a, NULL);
-  b_len1 = i2d_PublicKey(b, NULL);
-  if (a_len1 != b_len1)
-    return 0;
-  a_ptr = a_enc = tor_malloc(a_len1);
-  b_ptr = b_enc = tor_malloc(b_len1);
-  a_len2 = i2d_PublicKey(a, &a_ptr);
-  b_len2 = i2d_PublicKey(b, &b_ptr);
-  tor_assert(a_len2 == a_len1);
-  tor_assert(b_len2 == b_len1);
-  result = tor_memeq(a_enc, b_enc, a_len1);
-  tor_free(a_enc);
-  tor_free(b_enc);
+  unsigned char *a_enc = NULL, *b_enc = NULL;
+  int a_len, b_len, result;
+  a_len = i2d_PublicKey(a, &a_enc);
+  b_len = i2d_PublicKey(b, &b_enc);
+  if (a_len != b_len || a_len < 0) {
+    result = 0;
+  } else {
+    result = tor_memeq(a_enc, b_enc, a_len);
+  }
+  if (a_enc)
+    OPENSSL_free(a_enc);
+  if (b_enc)
+    OPENSSL_free(b_enc);
   return result;
 }