Bug 1009785: Expose CKM_RSA_PKCS_OAEP from softoken's C_Encrypt/C_Dec…

…rypt (and thus also C_Wrap/C_Unwrap)

r=wtc

--HG--
extra : rebase_source : e1defd9999a5a8bb1a6b24ad4f32dd5cb0e7a9ad

lib/softoken/pkcs11.c

@@ -266,6 +266,8 @@ static const struct mechanismList mechanisms[] = {
 				 CKF_DUZ_IT_ALL},       PR_TRUE},
      {CKM_RSA_PKCS_PSS,         {RSA_MIN_MODULUS_BITS,CK_MAX,
 				 CKF_SN_VR},            PR_TRUE},
+     {CKM_RSA_PKCS_OAEP,        {RSA_MIN_MODULUS_BITS,CK_MAX,
+                                 CKF_EN_DE_WR_UN},       PR_TRUE},
 #ifdef SFTK_RSA9796_SUPPORTED
      {CKM_RSA_9796,		{RSA_MIN_MODULUS_BITS,CK_MAX,
 				 CKF_DUZ_IT_ALL},       PR_TRUE},

lib/softoken/pkcs11c.c

@@ -302,6 +302,46 @@ GetHashTypeFromMechanism(CK_MECHANISM_TYPE mech)
     }
 }
 
+/*
+ * Returns true if "params" contains a valid set of PSS parameters
+ */
+static PRBool
+sftk_ValidatePssParams(const CK_RSA_PKCS_PSS_PARAMS *params)
+{
+    if (!params) {
+        return PR_FALSE;
+    }
+    if (GetHashTypeFromMechanism(params->hashAlg) == HASH_AlgNULL ||
+        GetHashTypeFromMechanism(params->mgf) == HASH_AlgNULL) {
+        return PR_FALSE;
+    }
+    return PR_TRUE;
+}
+
+/*
+ * Returns true if "params" contains a valid set of OAEP parameters
+ */
+static PRBool
+sftk_ValidateOaepParams(const CK_RSA_PKCS_OAEP_PARAMS *params)
+{
+    if (!params) {
+        return PR_FALSE;
+    }
+    /* The requirements of ulSourceLen/pSourceData come from PKCS #11, which
+     * state:
+     *   If the parameter is empty, pSourceData must be NULL and
+     *   ulSourceDataLen must be zero.
+     */
+    if (params->source != CKZ_DATA_SPECIFIED ||
+        (GetHashTypeFromMechanism(params->hashAlg) == HASH_AlgNULL) ||
+        (GetHashTypeFromMechanism(params->mgf) == HASH_AlgNULL) ||
+        (params->ulSourceDataLen == 0 && params->pSourceData != NULL) ||
+        (params->ulSourceDataLen != 0 && params->pSourceData == NULL)) {
+        return PR_FALSE;
+    }
+    return PR_TRUE;
+}
+
 /* 
  * return a context based on the SFTKContext type.
  */
@@ -588,11 +628,6 @@ sftk_RSAEncryptOAEP(SFTKOAEPEncryptInfo *info, unsigned char *output,
     hashAlg = GetHashTypeFromMechanism(info->params->hashAlg);
     maskHashAlg = GetHashTypeFromMechanism(info->params->mgf);
 
-    if (info->params->source != CKZ_DATA_SPECIFIED) {
-        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
-        return SECFailure;
-    }
-
     return RSA_EncryptOAEP(&info->key->u.rsa, hashAlg, maskHashAlg,
                            (const unsigned char*)info->params->pSourceData,
                            info->params->ulSourceDataLen, NULL, 0,
@@ -617,11 +652,6 @@ sftk_RSADecryptOAEP(SFTKOAEPDecryptInfo *info, unsigned char *output,
     hashAlg = GetHashTypeFromMechanism(info->params->hashAlg);
     maskHashAlg = GetHashTypeFromMechanism(info->params->mgf);
 
-    if (info->params->source != CKZ_DATA_SPECIFIED) {
-        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
-        return SECFailure;
-    }
-
     rv = RSA_DecryptOAEP(&info->key->u.rsa, hashAlg, maskHashAlg,
                          (const unsigned char*)info->params->pSourceData,
                          info->params->ulSourceDataLen,
@@ -710,19 +740,18 @@ sftk_CryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
 	}
 	context->destroy = sftk_Null;
 	break;
-/* XXX: Disabled until unit tests land.
     case CKM_RSA_PKCS_OAEP:
 	if (key_type != CKK_RSA) {
 	    crv = CKR_KEY_TYPE_INCONSISTENT;
 	    break;
 	}
-	context->multi = PR_FALSE;
-	context->rsa = PR_TRUE;
-	if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_OAEP_PARAMS)) {
+	if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_OAEP_PARAMS) ||
+	    !sftk_ValidateOaepParams((CK_RSA_PKCS_OAEP_PARAMS*)pMechanism->pParameter)) {
 	    crv = CKR_MECHANISM_PARAM_INVALID;
 	    break;
 	}
-	/\* XXX: Need Parameter validation here *\/
+	context->multi = PR_FALSE;
+	context->rsa = PR_TRUE;
 	if (isEncrypt) {
 	    SFTKOAEPEncryptInfo *info = PORT_New(SFTKOAEPEncryptInfo);
 	    if (info == NULL) {
@@ -758,7 +787,6 @@ sftk_CryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
 	}
 	context->destroy = (SFTKDestroy) sftk_Space;
 	break;
-*/
     case CKM_RC2_CBC_PAD:
 	context->doPad = PR_TRUE;
 	/* fall thru */
@@ -2386,7 +2414,8 @@ CK_RV NSC_SignInit(CK_SESSION_HANDLE hSession,
 	    break;
 	} 
 	context->rsa = PR_TRUE;
-	if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS)) {
+	if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS) ||
+	    !sftk_ValidatePssParams((const CK_RSA_PKCS_PSS_PARAMS*)pMechanism->pParameter)) {
 	    crv = CKR_MECHANISM_PARAM_INVALID;
 	    break;
 	}
@@ -3023,7 +3052,8 @@ CK_RV NSC_VerifyInit(CK_SESSION_HANDLE hSession,
 	    break;
 	} 
 	context->rsa = PR_TRUE;
-	if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS)) {
+	if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS) ||
+	    !sftk_ValidatePssParams((const CK_RSA_PKCS_PSS_PARAMS*)pMechanism->pParameter)) {
 	    crv = CKR_MECHANISM_PARAM_INVALID;
 	    break;
 	}