Bug 373108 - Implement AES Galois Counter Mode (GCM)

Patch by relyea, review modified by wtc/ryan,  reviewed by relyea

security/nss/lib/freebl/blapii.h

@@ -10,6 +10,15 @@
 
 #include "blapit.h"
 
+/* max block size of supported block ciphers */
+#define MAX_BLOCK_SIZE 16
+
+typedef SECStatus (*freeblCipherFunc)(void *cx, unsigned char *output,
+                          unsigned int *outputLen, unsigned int maxOutputLen,
+                          const unsigned char *input, unsigned int inputLen,
+			  unsigned int blocksize);
+typedef void (*freeblDestroyFunc)(void *cx, PRBool freeit);
+
 SEC_BEGIN_PROTOS
 
 #if defined(XP_UNIX) && !defined(NO_FORK_CHECK)

security/nss/lib/freebl/blapit.h

@@ -34,6 +34,9 @@
 /* AES operation modes */
 #define NSS_AES                 0
 #define NSS_AES_CBC             1
+#define NSS_AES_CTS             2
+#define NSS_AES_CTR             3
+#define NSS_AES_GCM             4
 
 /* Camellia operation modes */
 #define NSS_CAMELLIA                 0

security/nss/lib/freebl/ctr.c

@@ -0,0 +1,167 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifdef FREEBL_NO_DEPEND
+#include "stubs.h"
+#endif
+#include "prtypes.h"
+#include "blapit.h"
+#include "blapii.h"
+#include "ctr.h"
+#include "pkcs11t.h"
+#include "secerr.h"
+
+SECStatus
+CTR_InitContext(CTRContext *ctr, void *context, freeblCipherFunc cipher,
+		const unsigned char *param, unsigned int blocksize)
+{
+    const CK_AES_CTR_PARAMS *ctrParams = (const CK_AES_CTR_PARAMS *)param;
+
+    if (ctrParams->ulCounterBits == 0 ||
+	ctrParams->ulCounterBits > blocksize * PR_BITS_PER_BYTE) {
+	PORT_SetError(SEC_ERROR_INVALID_ARGS);
+	return SECFailure;
+    }
+
+    /* Invariant: 0 < ctr->bufPtr <= blocksize */
+    ctr->bufPtr = blocksize; /* no unused data in the buffer */
+    ctr->cipher = cipher;
+    ctr->context = context;
+    ctr->counterBits = ctrParams->ulCounterBits;
+    if (blocksize > sizeof(ctr->counter) ||
+	blocksize > sizeof(ctrParams->cb)) {
+	PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
+	return SECFailure;
+    }
+    PORT_Memcpy(ctr->counter, ctrParams->cb, blocksize);
+    return SECSuccess;
+}
+
+CTRContext *
+CTR_CreateContext(void *context, freeblCipherFunc cipher,
+		  const unsigned char *param, unsigned int blocksize)
+{
+    CTRContext *ctr;
+    SECStatus rv;
+
+    /* first fill in the Counter context */
+    ctr = PORT_ZNew(CTRContext);
+    if (ctr == NULL) {
+	return NULL;
+    }
+    rv = CTR_InitContext(ctr, context, cipher, param, blocksize);
+    if (rv != SECSuccess) {
+	CTR_DestroyContext(ctr, PR_TRUE);
+	ctr = NULL;
+    }
+    return ctr;
+}
+
+void
+CTR_DestroyContext(CTRContext *ctr, PRBool freeit)
+{
+    PORT_Memset(ctr, 0, sizeof(CTRContext));
+    if (freeit) {
+	PORT_Free(ctr);
+    }
+}
+
+/*
+ * Used by counter mode. Increment the counter block. Not all bits in the
+ * counter block are part of the counter, counterBits tells how many bits
+ * are part of the counter. The counter block is blocksize long. It's a
+ * big endian value.
+ *
+ * XXX Does not handle counter rollover.
+ */
+static void
+ctr_GetNextCtr(unsigned char *counter, unsigned int counterBits,
+		unsigned int blocksize)
+{
+    unsigned char *counterPtr = counter + blocksize - 1;
+    unsigned char mask, count;
+
+    PORT_Assert(counterBits <= blocksize*PR_BITS_PER_BYTE);
+    while (counterBits >= PR_BITS_PER_BYTE) {
+	if (++(*(counterPtr--))) {
+	    return;
+	}
+	counterBits -= PR_BITS_PER_BYTE;
+    }
+    if (counterBits == 0) {
+	return;
+    }
+    /* increment the final partial byte */
+    mask = (1 << counterBits)-1;
+    count = ++(*counterPtr) & mask;
+    *counterPtr = ((*counterPtr) & ~mask) | count;
+    return;
+}
+
+static void
+ctr_xor(unsigned char *target, const unsigned char *x,
+	 const unsigned char *y, unsigned int count)
+{
+    unsigned int i;
+    for (i=0; i < count; i++) {
+	*target++ = *x++ ^ *y++;
+    }
+}
+
+SECStatus
+CTR_Update(CTRContext *ctr, unsigned char *outbuf,
+		unsigned int *outlen, unsigned int maxout,
+		const unsigned char *inbuf, unsigned int inlen,
+		unsigned int blocksize)
+{
+    unsigned int tmp;
+    SECStatus rv;
+
+    if (maxout < inlen) {
+	*outlen = inlen;
+	PORT_SetError(SEC_ERROR_OUTPUT_LEN);
+	return SECFailure;
+    }
+    *outlen = 0;
+    if (ctr->bufPtr != blocksize) {
+	unsigned int needed = PR_MIN(blocksize-ctr->bufPtr, inlen);
+	ctr_xor(outbuf, inbuf, ctr->buffer+ctr->bufPtr, needed);
+	ctr->bufPtr += needed;
+	outbuf += needed;
+	inbuf += needed;
+	*outlen += needed;
+	inlen -= needed;
+	if (inlen == 0) {
+	    return SECSuccess;
+	}
+	PORT_Assert(ctr->bufPtr == blocksize);
+    }
+
+    while (inlen >= blocksize) {
+	rv = (*ctr->cipher)(ctr->context, ctr->buffer, &tmp, blocksize,
+			ctr->counter, blocksize, blocksize);
+	ctr_GetNextCtr(ctr->counter, ctr->counterBits, blocksize);
+	if (rv != SECSuccess) {
+	    return SECFailure;
+	}
+	ctr_xor(outbuf, inbuf, ctr->buffer, blocksize);
+	outbuf += blocksize;
+	inbuf += blocksize;
+	*outlen += blocksize;
+	inlen -= blocksize;
+    }
+    if (inlen == 0) {
+	return SECSuccess;
+    }
+    rv = (*ctr->cipher)(ctr->context, ctr->buffer, &tmp, blocksize,
+			ctr->counter, blocksize, blocksize);
+    ctr_GetNextCtr(ctr->counter, ctr->counterBits, blocksize);
+    if (rv != SECSuccess) {
+	return SECFailure;
+    }
+    ctr_xor(outbuf, inbuf, ctr->buffer, inlen);
+    ctr->bufPtr = inlen;
+    *outlen += inlen;
+    return SECSuccess;
+}

security/nss/lib/freebl/ctr.h

@@ -0,0 +1,44 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef CTR_H
+#define CTR_H 1
+
+#include "blapii.h"
+
+/* This structure is defined in this header because both ctr.c and gcm.c
+ * need it. */
+struct CTRContextStr {
+   freeblCipherFunc cipher;
+   void *context;
+   unsigned char counter[MAX_BLOCK_SIZE];
+   unsigned char buffer[MAX_BLOCK_SIZE];
+   unsigned long counterBits;
+   unsigned int bufPtr;
+};
+
+typedef struct CTRContextStr CTRContext;
+
+SECStatus CTR_InitContext(CTRContext *ctr, void *context,
+			freeblCipherFunc cipher, const unsigned char *param,
+			unsigned int blocksize);
+
+/*
+ * The context argument is the inner cipher context to use with cipher. The
+ * CTRContext does not own context. context needs to remain valid for as long
+ * as the CTRContext is valid.
+ *
+ * The cipher argument is a block cipher in the ECB encrypt mode.
+ */
+CTRContext * CTR_CreateContext(void *context, freeblCipherFunc cipher,
+			const unsigned char *param, unsigned int blocksize);
+
+void CTR_DestroyContext(CTRContext *ctr, PRBool freeit);
+
+SECStatus CTR_Update(CTRContext *ctr, unsigned char *outbuf,
+			unsigned int *outlen, unsigned int maxout,
+			const unsigned char *inbuf, unsigned int inlen,
+			unsigned int blocksize);
+
+#endif