Bug 1517574 - fix leak in NSC_GenerateKeyPair when rejecting public R…

…SA exponents that are too small r=jcj

Reviewers: jcj

Reviewed By: jcj

Bug #: 1517574

Differential Revision: https://phabricator.services.mozilla.com/D19019

Try: https://treeherder.mozilla.org/#/jobs?repo=nss-try&revision=9b518a646aacc092b81e94421c09aa9b87f2cab1

--HG--
extra : rebase_source : 2a8282f39440c7859f3c288790c22feeae2ab933

gtests/pk11_gtest/pk11_rsapss_unittest.cc

@@ -93,6 +93,20 @@ TEST_F(Pkcs11RsaPssTest, GenerateAndSignAndVerify) {
   EXPECT_EQ(rv, SECFailure);
 }
 
+TEST_F(Pkcs11RsaPssTest, NoLeakWithInvalidExponent) {
+  // Attempt to generate an RSA key with a public exponent of 1. This should
+  // fail, but it shouldn't leak memory.
+  PK11RSAGenParams rsaGenParams = {1024, 0x01};
+
+  // Generate RSA key pair.
+  ScopedPK11SlotInfo slot(PK11_GetInternalSlot());
+  SECKEYPublicKey* pubKey = nullptr;
+  SECKEYPrivateKey* privKey =
+      PK11_GenerateKeyPair(slot.get(), CKM_RSA_PKCS_KEY_PAIR_GEN, &rsaGenParams,
+                           &pubKey, false, false, nullptr);
+  EXPECT_FALSE(privKey);
+  EXPECT_FALSE(pubKey);
+}
 class Pkcs11RsaPssVectorTest
     : public Pkcs11RsaPssTest,
       public ::testing::WithParamInterface<Pkcs11SignatureTestParams> {};

lib/softoken/pkcs11c.c

@@ -4838,6 +4838,7 @@ NSC_GenerateKeyPair(CK_SESSION_HANDLE hSession,
             bitSize = sftk_GetLengthInBits(pubExp.data, pubExp.len);
             if (bitSize < 2) {
                 crv = CKR_ATTRIBUTE_VALUE_INVALID;
+                PORT_Free(pubExp.data);
                 break;
             }
             crv = sftk_AddAttributeType(privateKey, CKA_PUBLIC_EXPONENT,