Factor out PKCS#1 padding

Signed-off-by: David Woodhouse <dwmw2@infradead.org>

gnutls.h

@@ -40,6 +40,8 @@ int tpm2_rsa_sign_hash_fn(gnutls_privkey_t key, gnutls_sign_algorithm_t algo,
 int tpm2_ec_sign_hash_fn(gnutls_privkey_t key, gnutls_sign_algorithm_t algo,
 			 void *_vpninfo, unsigned int flags,
 			 const gnutls_datum_t *data, gnutls_datum_t *sig);
+int oc_pkcs1_pad(struct openconnect_info *vpninfo,
+		 unsigned char *buf, int size, const gnutls_datum_t *data);
 
 /* GnuTLS 3.6.0+ provides this. We have our own for older GnuTLS. There is
  * also _gnutls_encode_ber_rs_raw() in some older versions, but there were

gnutls_tpm2.c

@@ -342,4 +342,24 @@ int oc_gnutls_encode_rs_value(gnutls_datum_t *sig, const gnutls_datum_t *sig_r,
 }
 #endif /* GnuTLS < 3.6.0 */
 
+/* EMSA-PKCS1-v1_5 padding in accordance with RFC3447 §9.2 */
+#define PKCS1_PAD_OVERHEAD 11
+int oc_pkcs1_pad(struct openconnect_info *vpninfo,
+		 unsigned char *buf, int size, const gnutls_datum_t *data)
+{
+	if (data->size + PKCS1_PAD_OVERHEAD > size) {
+		vpn_progress(vpninfo, PRG_ERR,
+			     _("TPM2 digest too large: %d > %d\n"),
+			     data->size, size - PKCS1_PAD_OVERHEAD);
+		return GNUTLS_E_PK_SIGN_FAILED;
+	}
+
+	buf[0] = 0;
+	buf[1] = 1;
+	memset(buf + 2, 0xff, size - data->size - 3);
+	buf[size - data->size - 1] = 0;
+	memcpy(buf + size - data->size, data->data, data->size);
+
+	return 0;
+}
 #endif /* HAVE_TSS2 */

gnutls_tpm2_esys.c

@@ -323,8 +323,6 @@ static int auth_tpm2_key(struct openconnect_info *vpninfo, ESYS_CONTEXT *ctx, ES
 	return 0;
 }
 
-#define PKCS1_PAD_OVERHEAD 11
-
 int tpm2_rsa_sign_hash_fn(gnutls_privkey_t key, gnutls_sign_algorithm_t algo,
 			  void *_vpninfo, unsigned int flags,
 			  const gnutls_datum_t *data, gnutls_datum_t *sig)
@@ -344,19 +342,8 @@ int tpm2_rsa_sign_hash_fn(gnutls_privkey_t key, gnutls_sign_algorithm_t algo,
 
 	digest.size = vpninfo->tpm2->pub.publicArea.unique.rsa.size;
 
-	if (data->size + PKCS1_PAD_OVERHEAD > digest.size) {
-		vpn_progress(vpninfo, PRG_ERR,
-			     _("TPM2 digest too large: %d > %d\n"),
-			     data->size, digest.size - PKCS1_PAD_OVERHEAD);
+	if (oc_pkcs1_pad(vpninfo, digest.buffer, digest.size, data))
 		return GNUTLS_E_PK_SIGN_FAILED;
-	}
-
-	/* PKCS#1 padding */
-	digest.buffer[0] = 0;
-	digest.buffer[1] = 1;
-	memset(digest.buffer + 2, 0xff, digest.size - data->size - 3);
-	digest.buffer[digest.size - data->size - 1] = 0;
-	memcpy(digest.buffer + digest.size - data->size, data->data, data->size);
 
 	if (init_tpm2_key(&ectx, &key_handle, vpninfo))
 		goto out;

gnutls_tpm2_ibm.c

@@ -355,8 +355,6 @@ static void tpm2_unload_key(TSS_CONTEXT *tssContext, TPM_HANDLE key)
 	TSS_Delete(tssContext);
 }
 
-#define PKCS1_PAD_OVERHEAD 11
-
 int tpm2_rsa_sign_hash_fn(gnutls_privkey_t key, gnutls_sign_algorithm_t algo,
 			  void *_vpninfo, unsigned int flags,
 			  const gnutls_datum_t *data, gnutls_datum_t *sig)
@@ -376,19 +374,8 @@ int tpm2_rsa_sign_hash_fn(gnutls_privkey_t key, gnutls_sign_algorithm_t algo,
 
 	in.cipherText.t.size = vpninfo->tpm2->pub.publicArea.unique.rsa.t.size;
 
-	if (data->size + PKCS1_PAD_OVERHEAD > in.cipherText.t.size) {
-		vpn_progress(vpninfo, PRG_ERR,
-			     _("TPM2 digest too large: %d > %d\n"),
-			     data->size, in.cipherText.t.size - PKCS1_PAD_OVERHEAD);
+	if (oc_pkcs1_pad(vpninfo, in.cipherText.t.buffer, in.cipherText.t.size, data))
 		return GNUTLS_E_PK_SIGN_FAILED;
-	}
-
-	/* PKCS#1 padding */
-	in.cipherText.t.buffer[0] = 0;
-	in.cipherText.t.buffer[1] = 1;
-	memset(in.cipherText.t.buffer + 2, 0xff, in.cipherText.t.size - data->size - 3);
-	in.cipherText.t.buffer[in.cipherText.t.size - data->size - 1] = 0;
-	memcpy(in.cipherText.t.buffer + in.cipherText.t.size - data->size, data->data, data->size);
 
 	in.inScheme.scheme = TPM_ALG_NULL;
 	in.keyHandle = tpm2_load_key(vpninfo, &tssContext);