Bug 1347499 - Don't try to derive again when we have a curve25519 (EC…

…Point_XOnly) key and fix ASAN issue, r=ttaubert

Differential Revision: https://nss-review.dev.mozaws.net/D251

--HG--
extra : rebase_source : 90f494fac2ac2a05b387da6240eed637de26923e

lib/freebl/ecl/curve25519_64.c

@@ -206,7 +206,7 @@ fexpand(felem *output, const u8 *in)
     output[1] = (*((const uint64_t *)(in + 6)) >> 3) & MASK51;
     output[2] = (*((const uint64_t *)(in + 12)) >> 6) & MASK51;
     output[3] = (*((const uint64_t *)(in + 19)) >> 1) & MASK51;
-    output[4] = (*((const uint64_t *)(in + 25)) >> 4) & MASK51;
+    output[4] = (*((const uint64_t *)(in + 24)) >> 12) & MASK51;
 }
 
 /* Take a fully reduced polynomial form number and contract it into a

lib/pk11wrap/pk11skey.c

@@ -18,6 +18,8 @@
 #include "secerr.h"
 #include "hasht.h"
 
+static ECPointEncoding pk11_ECGetPubkeyEncoding(const SECKEYPublicKey *pubKey);
+
 static void
 pk11_EnterKeyMonitor(PK11SymKey *symKey)
 {
@@ -2005,7 +2007,7 @@ PK11_PubDerive(SECKEYPrivateKey *privKey, SECKEYPublicKey *pubKey,
 
             /* old PKCS #11 spec was ambiguous on what needed to be passed,
              * try this again with and encoded public key */
-            if (crv != CKR_OK) {
+            if (crv != CKR_OK && pk11_ECGetPubkeyEncoding(pubKey) != ECPoint_XOnly) {
                 SECItem *pubValue = SEC_ASN1EncodeItem(NULL, NULL,
                                                        &pubKey->u.ec.publicValue,
                                                        SEC_ASN1_GET(SEC_OctetStringTemplate));
@@ -2211,6 +2213,11 @@ pk11_PubDeriveECKeyWithKDF(
     /* old PKCS #11 spec was ambiguous on what needed to be passed,
      * try this again with an encoded public key */
     if (crv != CKR_OK) {
+        /* For curves that only use X as public value and no encoding we don't
+         * have to try again. (Currently onlye Curve25519) */
+        if (pk11_ECGetPubkeyEncoding(pubKey) == ECPoint_XOnly) {
+            goto loser;
+        }
         SECItem *pubValue = SEC_ASN1EncodeItem(NULL, NULL,
                                                &pubKey->u.ec.publicValue,
                                                SEC_ASN1_GET(SEC_OctetStringTemplate));