andre@0: /* This Source Code Form is subject to the terms of the Mozilla Public
andre@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
andre@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
andre@0: 
andre@0: /*
andre@0:  * RSA PKCS#1 v2.1 (RFC 3447) operations
andre@0:  */
andre@0: 
andre@0: #ifdef FREEBL_NO_DEPEND
andre@0: #include "stubs.h"
andre@0: #endif
andre@0: 
andre@0: #include "secerr.h"
andre@0: 
andre@0: #include "blapi.h"
andre@0: #include "secitem.h"
andre@0: #include "blapii.h"
andre@0: 
andre@0: #define RSA_BLOCK_MIN_PAD_LEN            8
andre@0: #define RSA_BLOCK_FIRST_OCTET            0x00
andre@0: #define RSA_BLOCK_PRIVATE_PAD_OCTET      0xff
andre@0: #define RSA_BLOCK_AFTER_PAD_OCTET        0x00
andre@0: 
andre@0: /*
andre@0:  * RSA block types
andre@0:  *
andre@0:  * The values of RSA_BlockPrivate and RSA_BlockPublic are fixed.
andre@0:  * The value of RSA_BlockRaw isn't fixed by definition, but we are keeping
andre@0:  * the value that NSS has been using in the past.
andre@0:  */
andre@0: typedef enum {
andre@0:     RSA_BlockPrivate = 1,   /* pad for a private-key operation */
andre@0:     RSA_BlockPublic = 2,    /* pad for a public-key operation */
andre@0:     RSA_BlockRaw = 4        /* simply justify the block appropriately */
andre@0: } RSA_BlockType;
andre@0: 
andre@0: /* Needed for RSA-PSS functions */
andre@0: static const unsigned char eightZeros[] = { 0, 0, 0, 0, 0, 0, 0, 0 };
andre@0: 
andre@0: /* Constant time comparison of a single byte.
andre@0:  * Returns 1 iff a == b, otherwise returns 0.
andre@0:  * Note: For ranges of bytes, use constantTimeCompare.
andre@0:  */
andre@0: static unsigned char constantTimeEQ8(unsigned char a, unsigned char b) {
andre@0:     unsigned char c = ~((a - b) | (b - a));
andre@0:     c >>= 7;
andre@0:     return c;
andre@0: }
andre@0: 
andre@0: /* Constant time comparison of a range of bytes.
andre@0:  * Returns 1 iff len bytes of a are identical to len bytes of b, otherwise
andre@0:  * returns 0.
andre@0:  */
andre@0: static unsigned char constantTimeCompare(const unsigned char *a,
andre@0:                                          const unsigned char *b,
andre@0:                                          unsigned int len) {
andre@0:     unsigned char tmp = 0;
andre@0:     unsigned int i;
andre@0:     for (i = 0; i < len; ++i, ++a, ++b)
andre@0:         tmp |= *a ^ *b;
andre@0:     return constantTimeEQ8(0x00, tmp);
andre@0: }
andre@0: 
andre@0: /* Constant time conditional.
andre@0:  * Returns a if c is 1, or b if c is 0. The result is undefined if c is
andre@0:  * not 0 or 1.
andre@0:  */
andre@0: static unsigned int constantTimeCondition(unsigned int c,
andre@0:                                           unsigned int a,
andre@0:                                           unsigned int b)
andre@0: {
andre@0:     return (~(c - 1) & a) | ((c - 1) & b);
andre@0: }
andre@0: 
andre@0: static unsigned int
andre@0: rsa_modulusLen(SECItem * modulus)
andre@0: {
andre@0:     unsigned char byteZero = modulus->data[0];
andre@0:     unsigned int modLen = modulus->len - !byteZero;
andre@0:     return modLen;
andre@0: }
andre@0: 
andre@0: /*
andre@0:  * Format one block of data for public/private key encryption using
andre@0:  * the rules defined in PKCS #1.
andre@0:  */
andre@0: static unsigned char *
andre@0: rsa_FormatOneBlock(unsigned modulusLen,
andre@0:                    RSA_BlockType blockType,
andre@0:                    SECItem * data)
andre@0: {
andre@0:     unsigned char *block;
andre@0:     unsigned char *bp;
andre@0:     int padLen;
andre@0:     int i, j;
andre@0:     SECStatus rv;
andre@0: 
andre@0:     block = (unsigned char *) PORT_Alloc(modulusLen);
andre@0:     if (block == NULL)
andre@0:         return NULL;
andre@0: 
andre@0:     bp = block;
andre@0: 
andre@0:     /*
andre@0:      * All RSA blocks start with two octets:
andre@0:      *	0x00 || BlockType
andre@0:      */
andre@0:     *bp++ = RSA_BLOCK_FIRST_OCTET;
andre@0:     *bp++ = (unsigned char) blockType;
andre@0: 
andre@0:     switch (blockType) {
andre@0: 
andre@0:       /*
andre@0:        * Blocks intended for private-key operation.
andre@0:        */
andre@0:       case RSA_BlockPrivate:	 /* preferred method */
andre@0:         /*
andre@0:          * 0x00 || BT || Pad || 0x00 || ActualData
andre@0:          *   1      1   padLen    1      data->len
andre@0:          * Pad is either all 0x00 or all 0xff bytes, depending on blockType.
andre@0:          */
andre@0:         padLen = modulusLen - data->len - 3;
andre@0:         PORT_Assert(padLen >= RSA_BLOCK_MIN_PAD_LEN);
andre@0:         if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
andre@0:             PORT_Free(block);
andre@0:             return NULL;
andre@0:         }
andre@0:         PORT_Memset(bp, RSA_BLOCK_PRIVATE_PAD_OCTET, padLen);
andre@0:         bp += padLen;
andre@0:         *bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
andre@0:         PORT_Memcpy(bp, data->data, data->len);
andre@0:         break;
andre@0: 
andre@0:       /*
andre@0:        * Blocks intended for public-key operation.
andre@0:        */
andre@0:       case RSA_BlockPublic:
andre@0:         /*
andre@0:          * 0x00 || BT || Pad || 0x00 || ActualData
andre@0:          *   1      1   padLen    1      data->len
andre@0:          * Pad is all non-zero random bytes.
andre@0:          *
andre@0:          * Build the block left to right.
andre@0:          * Fill the entire block from Pad to the end with random bytes.
andre@0:          * Use the bytes after Pad as a supply of extra random bytes from
andre@0:          * which to find replacements for the zero bytes in Pad.
andre@0:          * If we need more than that, refill the bytes after Pad with
andre@0:          * new random bytes as necessary.
andre@0:          */
andre@0:         padLen = modulusLen - (data->len + 3);
andre@0:         PORT_Assert(padLen >= RSA_BLOCK_MIN_PAD_LEN);
andre@0:         if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
andre@0:             PORT_Free(block);
andre@0:             return NULL;
andre@0:         }
andre@0:         j = modulusLen - 2;
andre@0:         rv = RNG_GenerateGlobalRandomBytes(bp, j);
andre@0:         if (rv == SECSuccess) {
andre@0:             for (i = 0; i < padLen; ) {
andre@0:                 unsigned char repl;
andre@0:                 /* Pad with non-zero random data. */
andre@0:                 if (bp[i] != RSA_BLOCK_AFTER_PAD_OCTET) {
andre@0:                     ++i;
andre@0:                     continue;
andre@0:                 }
andre@0:                 if (j <= padLen) {
andre@0:                     rv = RNG_GenerateGlobalRandomBytes(bp + padLen,
andre@0:                                           modulusLen - (2 + padLen));
andre@0:                     if (rv != SECSuccess)
andre@0:                         break;
andre@0:                     j = modulusLen - 2;
andre@0:                 }
andre@0:                 do {
andre@0:                     repl = bp[--j];
andre@0:                 } while (repl == RSA_BLOCK_AFTER_PAD_OCTET && j > padLen);
andre@0:                 if (repl != RSA_BLOCK_AFTER_PAD_OCTET) {
andre@0:                     bp[i++] = repl;
andre@0:                 }
andre@0:             }
andre@0:         }
andre@0:         if (rv != SECSuccess) {
andre@0:             PORT_Free(block);
andre@0:             PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
andre@0:             return NULL;
andre@0:         }
andre@0:         bp += padLen;
andre@0:         *bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
andre@0:         PORT_Memcpy(bp, data->data, data->len);
andre@0:         break;
andre@0: 
andre@0:       default:
andre@0:         PORT_Assert(0);
andre@0:         PORT_Free(block);
andre@0:         return NULL;
andre@0:     }
andre@0: 
andre@0:     return block;
andre@0: }
andre@0: 
andre@0: static SECStatus
andre@0: rsa_FormatBlock(SECItem * result,
andre@0:                 unsigned modulusLen,
andre@0:                 RSA_BlockType blockType,
andre@0:                 SECItem * data)
andre@0: {
andre@0:     switch (blockType) {
andre@0:       case RSA_BlockPrivate:
andre@0:       case RSA_BlockPublic:
andre@0:         /*
andre@0:          * 0x00 || BT || Pad || 0x00 || ActualData
andre@0:          *
andre@0:          * The "3" below is the first octet + the second octet + the 0x00
andre@0:          * octet that always comes just before the ActualData.
andre@0:          */
andre@0:         PORT_Assert(data->len <= (modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN)));
andre@0: 
andre@0:         result->data = rsa_FormatOneBlock(modulusLen, blockType, data);
andre@0:         if (result->data == NULL) {
andre@0:             result->len = 0;
andre@0:             return SECFailure;
andre@0:         }
andre@0:         result->len = modulusLen;
andre@0: 
andre@0:         break;
andre@0: 
andre@0:       case RSA_BlockRaw:
andre@0:         /*
andre@0:          * Pad || ActualData
andre@0:          * Pad is zeros. The application is responsible for recovering
andre@0:          * the actual data.
andre@0:          */
andre@0:         if (data->len > modulusLen ) {
andre@0:             return SECFailure;
andre@0:         }
andre@0:         result->data = (unsigned char*)PORT_ZAlloc(modulusLen);
andre@0:         result->len = modulusLen;
andre@0:         PORT_Memcpy(result->data + (modulusLen - data->len),
andre@0:                     data->data, data->len);
andre@0:         break;
andre@0: 
andre@0:       default:
andre@0:         PORT_Assert(0);
andre@0:         result->data = NULL;
andre@0:         result->len = 0;
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     return SECSuccess;
andre@0: }
andre@0: 
andre@0: /*
andre@0:  * Mask generation function MGF1 as defined in PKCS #1 v2.1 / RFC 3447.
andre@0:  */
andre@0: static SECStatus
andre@0: MGF1(HASH_HashType hashAlg,
andre@0:      unsigned char * mask,
andre@0:      unsigned int maskLen,
andre@0:      const unsigned char * mgfSeed,
andre@0:      unsigned int mgfSeedLen)
andre@0: {
andre@0:     unsigned int digestLen;
andre@0:     PRUint32 counter;
andre@0:     PRUint32 rounds;
andre@0:     unsigned char * tempHash;
andre@0:     unsigned char * temp;
andre@0:     const SECHashObject * hash;
andre@0:     void * hashContext;
andre@0:     unsigned char C[4];
andre@0: 
andre@0:     hash = HASH_GetRawHashObject(hashAlg);
andre@0:     if (hash == NULL)
andre@0:         return SECFailure;
andre@0: 
andre@0:     hashContext = (*hash->create)();
andre@0:     rounds = (maskLen + hash->length - 1) / hash->length;
andre@0:     for (counter = 0; counter < rounds; counter++) {
andre@0:         C[0] = (unsigned char)((counter >> 24) & 0xff);
andre@0:         C[1] = (unsigned char)((counter >> 16) & 0xff);
andre@0:         C[2] = (unsigned char)((counter >> 8) & 0xff);
andre@0:         C[3] = (unsigned char)(counter & 0xff);
andre@0: 
andre@0:         /* This could be optimized when the clone functions in
andre@0:          * rawhash.c are implemented. */
andre@0:         (*hash->begin)(hashContext);
andre@0:         (*hash->update)(hashContext, mgfSeed, mgfSeedLen);
andre@0:         (*hash->update)(hashContext, C, sizeof C);
andre@0: 
andre@0:         tempHash = mask + counter * hash->length;
andre@0:         if (counter != (rounds - 1)) {
andre@0:             (*hash->end)(hashContext, tempHash, &digestLen, hash->length);
andre@0:         } else { /* we're in the last round and need to cut the hash */
andre@0:             temp = (unsigned char *)PORT_Alloc(hash->length);
andre@0:             (*hash->end)(hashContext, temp, &digestLen, hash->length);
andre@0:             PORT_Memcpy(tempHash, temp, maskLen - counter * hash->length);
andre@0:             PORT_Free(temp);
andre@0:         }
andre@0:     }
andre@0:     (*hash->destroy)(hashContext, PR_TRUE);
andre@0: 
andre@0:     return SECSuccess;
andre@0: }
andre@0: 
andre@0: /* XXX Doesn't set error code */
andre@0: SECStatus
andre@0: RSA_SignRaw(RSAPrivateKey * key,
andre@0:             unsigned char * output,
andre@0:             unsigned int * outputLen,
andre@0:             unsigned int maxOutputLen,
andre@0:             const unsigned char * data,
andre@0:             unsigned int dataLen)
andre@0: {
andre@0:     SECStatus rv = SECSuccess;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0:     SECItem formatted;
andre@0:     SECItem unformatted;
andre@0: 
andre@0:     if (maxOutputLen < modulusLen)
andre@0:         return SECFailure;
andre@0: 
andre@0:     unformatted.len  = dataLen;
andre@0:     unformatted.data = (unsigned char*)data;
andre@0:     formatted.data   = NULL;
andre@0:     rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockRaw, &unformatted);
andre@0:     if (rv != SECSuccess)
andre@0:         goto done;
andre@0: 
andre@0:     rv = RSA_PrivateKeyOpDoubleChecked(key, output, formatted.data);
andre@0:     *outputLen = modulusLen;
andre@0: 
andre@0: done:
andre@0:     if (formatted.data != NULL)
andre@0:         PORT_ZFree(formatted.data, modulusLen);
andre@0:     return rv;
andre@0: }
andre@0: 
andre@0: /* XXX Doesn't set error code */
andre@0: SECStatus
andre@0: RSA_CheckSignRaw(RSAPublicKey * key,
andre@0:                  const unsigned char * sig,
andre@0:                  unsigned int sigLen,
andre@0:                  const unsigned char * hash,
andre@0:                  unsigned int hashLen)
andre@0: {
andre@0:     SECStatus rv;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0:     unsigned char * buffer;
andre@0: 
andre@0:     if (sigLen != modulusLen)
andre@0:         goto failure;
andre@0:     if (hashLen > modulusLen)
andre@0:         goto failure;
andre@0: 
andre@0:     buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
andre@0:     if (!buffer)
andre@0:         goto failure;
andre@0: 
andre@0:     rv = RSA_PublicKeyOp(key, buffer, sig);
andre@0:     if (rv != SECSuccess)
andre@0:         goto loser;
andre@0: 
andre@0:     /*
andre@0:      * make sure we get the same results
andre@0:      */
andre@0:     /* XXX(rsleevi): Constant time */
andre@0:     /* NOTE: should we verify the leading zeros? */
andre@0:     if (PORT_Memcmp(buffer + (modulusLen - hashLen), hash, hashLen) != 0)
andre@0:         goto loser;
andre@0: 
andre@0:     PORT_Free(buffer);
andre@0:     return SECSuccess;
andre@0: 
andre@0: loser:
andre@0:     PORT_Free(buffer);
andre@0: failure:
andre@0:     return SECFailure;
andre@0: }
andre@0: 
andre@0: /* XXX Doesn't set error code */
andre@0: SECStatus
andre@0: RSA_CheckSignRecoverRaw(RSAPublicKey * key,
andre@0:                         unsigned char * data,
andre@0:                         unsigned int * dataLen,
andre@0:                         unsigned int maxDataLen,
andre@0:                         const unsigned char * sig,
andre@0:                         unsigned int sigLen)
andre@0: {
andre@0:     SECStatus rv;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0: 
andre@0:     if (sigLen != modulusLen)
andre@0:         goto failure;
andre@0:     if (maxDataLen < modulusLen)
andre@0:         goto failure;
andre@0: 
andre@0:     rv = RSA_PublicKeyOp(key, data, sig);
andre@0:     if (rv != SECSuccess)
andre@0:         goto failure;
andre@0: 
andre@0:     *dataLen = modulusLen;
andre@0:     return SECSuccess;
andre@0: 
andre@0: failure:
andre@0:     return SECFailure;
andre@0: }
andre@0: 
andre@0: /* XXX Doesn't set error code */
andre@0: SECStatus
andre@0: RSA_EncryptRaw(RSAPublicKey * key,
andre@0:                unsigned char * output,
andre@0:                unsigned int * outputLen,
andre@0:                unsigned int maxOutputLen,
andre@0:                const unsigned char * input,
andre@0:                unsigned int inputLen)
andre@0: {
andre@0:     SECStatus rv;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0:     SECItem formatted;
andre@0:     SECItem unformatted;
andre@0: 
andre@0:     formatted.data = NULL;
andre@0:     if (maxOutputLen < modulusLen)
andre@0:         goto failure;
andre@0: 
andre@0:     unformatted.len  = inputLen;
andre@0:     unformatted.data = (unsigned char*)input;
andre@0:     formatted.data   = NULL;
andre@0:     rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockRaw, &unformatted);
andre@0:     if (rv != SECSuccess)
andre@0:         goto failure;
andre@0: 
andre@0:     rv = RSA_PublicKeyOp(key, output, formatted.data);
andre@0:     if (rv != SECSuccess)
andre@0:         goto failure;
andre@0: 
andre@0:     PORT_ZFree(formatted.data, modulusLen);
andre@0:     *outputLen = modulusLen;
andre@0:     return SECSuccess;
andre@0: 
andre@0: failure:
andre@0:     if (formatted.data != NULL)
andre@0:         PORT_ZFree(formatted.data, modulusLen);
andre@0:     return SECFailure;
andre@0: }
andre@0: 
andre@0: /* XXX Doesn't set error code */
andre@0: SECStatus
andre@0: RSA_DecryptRaw(RSAPrivateKey * key,
andre@0:                unsigned char * output,
andre@0:                unsigned int * outputLen,
andre@0:                unsigned int maxOutputLen,
andre@0:                const unsigned char * input,
andre@0:                unsigned int inputLen)
andre@0: {
andre@0:     SECStatus rv;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0: 
andre@0:     if (modulusLen > maxOutputLen)
andre@0:         goto failure;
andre@0:     if (inputLen != modulusLen)
andre@0:         goto failure;
andre@0: 
andre@0:     rv = RSA_PrivateKeyOp(key, output, input);
andre@0:     if (rv != SECSuccess)
andre@0:         goto failure;
andre@0: 
andre@0:     *outputLen = modulusLen;
andre@0:     return SECSuccess;
andre@0: 
andre@0: failure:
andre@0:     return SECFailure;
andre@0: }
andre@0: 
andre@0: /*
andre@0:  * Decodes an EME-OAEP encoded block, validating the encoding in constant
andre@0:  * time.
andre@0:  * Described in RFC 3447, section 7.1.2.
andre@0:  * input contains the encoded block, after decryption.
andre@0:  * label is the optional value L that was associated with the message.
andre@0:  * On success, the original message and message length will be stored in
andre@0:  * output and outputLen.
andre@0:  */
andre@0: static SECStatus
andre@0: eme_oaep_decode(unsigned char * output,
andre@0:                 unsigned int * outputLen,
andre@0:                 unsigned int maxOutputLen,
andre@0:                 const unsigned char * input,
andre@0:                 unsigned int inputLen,
andre@0:                 HASH_HashType hashAlg,
andre@0:                 HASH_HashType maskHashAlg,
andre@0:                 const unsigned char * label,
andre@0:                 unsigned int labelLen)
andre@0: {
andre@0:     const SECHashObject * hash;
andre@0:     void * hashContext;
andre@0:     SECStatus rv = SECFailure;
andre@0:     unsigned char labelHash[HASH_LENGTH_MAX];
andre@0:     unsigned int i;
andre@0:     unsigned int maskLen;
andre@0:     unsigned int paddingOffset;
andre@0:     unsigned char * mask = NULL;
andre@0:     unsigned char * tmpOutput = NULL;
andre@0:     unsigned char isGood;
andre@0:     unsigned char foundPaddingEnd;
andre@0: 
andre@0:     hash = HASH_GetRawHashObject(hashAlg);
andre@0: 
andre@0:     /* 1.c */
andre@0:     if (inputLen < (hash->length * 2) + 2) {
andre@0:         PORT_SetError(SEC_ERROR_INPUT_LEN);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     /* Step 3.a - Generate lHash */
andre@0:     hashContext = (*hash->create)();
andre@0:     if (hashContext == NULL) {
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         return SECFailure;
andre@0:     }
andre@0:     (*hash->begin)(hashContext);
andre@0:     if (labelLen > 0)
andre@0:         (*hash->update)(hashContext, label, labelLen);
andre@0:     (*hash->end)(hashContext, labelHash, &i, sizeof(labelHash));
andre@0:     (*hash->destroy)(hashContext, PR_TRUE);
andre@0: 
andre@0:     tmpOutput = (unsigned char*)PORT_Alloc(inputLen);
andre@0:     if (tmpOutput == NULL) {
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         goto done;
andre@0:     }
andre@0: 
andre@0:     maskLen = inputLen - hash->length - 1;
andre@0:     mask = (unsigned char*)PORT_Alloc(maskLen);
andre@0:     if (mask == NULL) {
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         goto done;
andre@0:     }
andre@0: 
andre@0:     PORT_Memcpy(tmpOutput, input, inputLen);
andre@0: 
andre@0:     /* 3.c - Generate seedMask */
andre@0:     MGF1(maskHashAlg, mask, hash->length, &tmpOutput[1 + hash->length],
andre@0:          inputLen - hash->length - 1);
andre@0:     /* 3.d - Unmask seed */
andre@0:     for (i = 0; i < hash->length; ++i)
andre@0:         tmpOutput[1 + i] ^= mask[i];
andre@0: 
andre@0:     /* 3.e - Generate dbMask */
andre@0:     MGF1(maskHashAlg, mask, maskLen, &tmpOutput[1], hash->length);
andre@0:     /* 3.f - Unmask DB */
andre@0:     for (i = 0; i < maskLen; ++i)
andre@0:         tmpOutput[1 + hash->length + i] ^= mask[i];
andre@0: 
andre@0:     /* 3.g - Compare Y, lHash, and PS in constant time
andre@0:      * Warning: This code is timing dependent and must not disclose which of
andre@0:      * these were invalid.
andre@0:      */
andre@0:     paddingOffset = 0;
andre@0:     isGood = 1;
andre@0:     foundPaddingEnd = 0;
andre@0: 
andre@0:     /* Compare Y */
andre@0:     isGood &= constantTimeEQ8(0x00, tmpOutput[0]);
andre@0: 
andre@0:     /* Compare lHash and lHash' */
andre@0:     isGood &= constantTimeCompare(&labelHash[0],
andre@0:                                   &tmpOutput[1 + hash->length],
andre@0:                                   hash->length);
andre@0: 
andre@0:     /* Compare that the padding is zero or more zero octets, followed by a
andre@0:      * 0x01 octet */
andre@0:     for (i = 1 + (hash->length * 2); i < inputLen; ++i) {
andre@0:         unsigned char isZero = constantTimeEQ8(0x00, tmpOutput[i]);
andre@0:         unsigned char isOne = constantTimeEQ8(0x01, tmpOutput[i]);
andre@0:         /* non-constant time equivalent:
andre@0:          * if (tmpOutput[i] == 0x01 && !foundPaddingEnd)
andre@0:          *     paddingOffset = i;
andre@0:          */
andre@0:         paddingOffset = constantTimeCondition(isOne & ~foundPaddingEnd, i,
andre@0:                                               paddingOffset);
andre@0:         /* non-constant time equivalent:
andre@0:          * if (tmpOutput[i] == 0x01)
andre@0:          *    foundPaddingEnd = true;
andre@0:          *
andre@0:          * Note: This may yield false positives, as it will be set whenever
andre@0:          * a 0x01 byte is encountered. If there was bad padding (eg:
andre@0:          * 0x03 0x02 0x01), foundPaddingEnd will still be set to true, and
andre@0:          * paddingOffset will still be set to 2.
andre@0:          */
andre@0:         foundPaddingEnd = constantTimeCondition(isOne, 1, foundPaddingEnd);
andre@0:         /* non-constant time equivalent:
andre@0:          * if (tmpOutput[i] != 0x00 && tmpOutput[i] != 0x01 &&
andre@0:          *     !foundPaddingEnd) {
andre@0:          *    isGood = false;
andre@0:          * }
andre@0:          *
andre@0:          * Note: This may yield false positives, as a message (and padding)
andre@0:          * that is entirely zeros will result in isGood still being true. Thus
andre@0:          * it's necessary to check foundPaddingEnd is positive below.
andre@0:          */
andre@0:         isGood = constantTimeCondition(~foundPaddingEnd & ~isZero, 0, isGood);
andre@0:     }
andre@0: 
andre@0:     /* While both isGood and foundPaddingEnd may have false positives, they
andre@0:      * cannot BOTH have false positives. If both are not true, then an invalid
andre@0:      * message was received. Note, this comparison must still be done in constant
andre@0:      * time so as not to leak either condition.
andre@0:      */
andre@0:     if (!(isGood & foundPaddingEnd)) {
andre@0:         PORT_SetError(SEC_ERROR_BAD_DATA);
andre@0:         goto done;
andre@0:     }
andre@0: 
andre@0:     /* End timing dependent code */
andre@0: 
andre@0:     ++paddingOffset; /* Skip the 0x01 following the end of PS */
andre@0: 
andre@0:     *outputLen = inputLen - paddingOffset;
andre@0:     if (*outputLen > maxOutputLen) {
andre@0:         PORT_SetError(SEC_ERROR_OUTPUT_LEN);
andre@0:         goto done;
andre@0:     }
andre@0: 
andre@0:     if (*outputLen)
andre@0:         PORT_Memcpy(output, &tmpOutput[paddingOffset], *outputLen);
andre@0:     rv = SECSuccess;
andre@0: 
andre@0: done:
andre@0:     if (mask)
andre@0:         PORT_ZFree(mask, maskLen);
andre@0:     if (tmpOutput)
andre@0:         PORT_ZFree(tmpOutput, inputLen);
andre@0:     return rv;
andre@0: }
andre@0: 
andre@0: /*
andre@0:  * Generate an EME-OAEP encoded block for encryption
andre@0:  * Described in RFC 3447, section 7.1.1
andre@0:  * We use input instead of M for the message to be encrypted
andre@0:  * label is the optional value L to be associated with the message.
andre@0:  */
andre@0: static SECStatus
andre@0: eme_oaep_encode(unsigned char * em,
andre@0:                 unsigned int emLen,
andre@0:                 const unsigned char * input,
andre@0:                 unsigned int inputLen,
andre@0:                 HASH_HashType hashAlg,
andre@0:                 HASH_HashType maskHashAlg,
andre@0:                 const unsigned char * label,
andre@0:                 unsigned int labelLen,
andre@0:                 const unsigned char * seed,
andre@0:                 unsigned int seedLen)
andre@0: {
andre@0:     const SECHashObject * hash;
andre@0:     void * hashContext;
andre@0:     SECStatus rv;
andre@0:     unsigned char * mask;
andre@0:     unsigned int reservedLen;
andre@0:     unsigned int dbMaskLen;
andre@0:     unsigned int i;
andre@0: 
andre@0:     hash = HASH_GetRawHashObject(hashAlg);
andre@0:     PORT_Assert(seed == NULL || seedLen == hash->length);
andre@0: 
andre@0:     /* Step 1.b */
andre@0:     reservedLen = (2 * hash->length) + 2;
andre@0:     if (emLen < reservedLen || inputLen > (emLen - reservedLen)) {
andre@0:         PORT_SetError(SEC_ERROR_INPUT_LEN);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     /*
andre@0:      * From RFC 3447, Section 7.1
andre@0:      *                      +----------+---------+-------+
andre@0:      *                 DB = |  lHash   |    PS   |   M   |
andre@0:      *                      +----------+---------+-------+
andre@0:      *                                     |
andre@0:      *           +----------+              V
andre@0:      *           |   seed   |--> MGF ---> xor
andre@0:      *           +----------+              |
andre@0:      *                 |                   |
andre@0:      *        +--+     V                   |
andre@0:      *        |00|    xor <----- MGF <-----|
andre@0:      *        +--+     |                   |
andre@0:      *          |      |                   |
andre@0:      *          V      V                   V
andre@0:      *        +--+----------+----------------------------+
andre@0:      *  EM =  |00|maskedSeed|          maskedDB          |
andre@0:      *        +--+----------+----------------------------+
andre@0:      *
andre@0:      * We use mask to hold the result of the MGF functions, and all other
andre@0:      * values are generated in their final resting place.
andre@0:      */
andre@0:     *em = 0x00;
andre@0: 
andre@0:     /* Step 2.a - Generate lHash */
andre@0:     hashContext = (*hash->create)();
andre@0:     if (hashContext == NULL) {
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         return SECFailure;
andre@0:     }
andre@0:     (*hash->begin)(hashContext);
andre@0:     if (labelLen > 0)
andre@0:         (*hash->update)(hashContext, label, labelLen);
andre@0:     (*hash->end)(hashContext, &em[1 + hash->length], &i, hash->length);
andre@0:     (*hash->destroy)(hashContext, PR_TRUE);
andre@0: 
andre@0:     /* Step 2.b - Generate PS */
andre@0:     if (emLen - reservedLen - inputLen > 0) {
andre@0:         PORT_Memset(em + 1 + (hash->length * 2), 0x00,
andre@0:                     emLen - reservedLen - inputLen);
andre@0:     }
andre@0: 
andre@0:     /* Step 2.c. - Generate DB
andre@0:      * DB = lHash || PS || 0x01 || M
andre@0:      * Note that PS and lHash have already been placed into em at their
andre@0:      * appropriate offsets. This just copies M into place
andre@0:      */
andre@0:     em[emLen - inputLen - 1] = 0x01;
andre@0:     if (inputLen)
andre@0:         PORT_Memcpy(em + emLen - inputLen, input, inputLen);
andre@0: 
andre@0:     if (seed == NULL) {
andre@0:         /* Step 2.d - Generate seed */
andre@0:         rv = RNG_GenerateGlobalRandomBytes(em + 1, hash->length);
andre@0:         if (rv != SECSuccess) {
andre@0:             return rv;
andre@0:         }
andre@0:     } else {
andre@0:         /* For Known Answer Tests, copy the supplied seed. */
andre@0:         PORT_Memcpy(em + 1, seed, seedLen);
andre@0:     }
andre@0: 
andre@0:     /* Step 2.e - Generate dbMask*/
andre@0:     dbMaskLen = emLen - hash->length - 1;
andre@0:     mask = (unsigned char*)PORT_Alloc(dbMaskLen);
andre@0:     if (mask == NULL) {
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         return SECFailure;
andre@0:     }
andre@0:     MGF1(maskHashAlg, mask, dbMaskLen, em + 1, hash->length);
andre@0:     /* Step 2.f - Compute maskedDB*/
andre@0:     for (i = 0; i < dbMaskLen; ++i)
andre@0:         em[1 + hash->length + i] ^= mask[i];
andre@0: 
andre@0:     /* Step 2.g - Generate seedMask */
andre@0:     MGF1(maskHashAlg, mask, hash->length, &em[1 + hash->length], dbMaskLen);
andre@0:     /* Step 2.h - Compute maskedSeed */
andre@0:     for (i = 0; i < hash->length; ++i)
andre@0:         em[1 + i] ^= mask[i];
andre@0: 
andre@0:     PORT_ZFree(mask, dbMaskLen);
andre@0:     return SECSuccess;
andre@0: }
andre@0: 
andre@0: SECStatus
andre@0: RSA_EncryptOAEP(RSAPublicKey * key,
andre@0:                 HASH_HashType hashAlg,
andre@0:                 HASH_HashType maskHashAlg,
andre@0:                 const unsigned char * label,
andre@0:                 unsigned int labelLen,
andre@0:                 const unsigned char * seed,
andre@0:                 unsigned int seedLen,
andre@0:                 unsigned char * output,
andre@0:                 unsigned int * outputLen,
andre@0:                 unsigned int maxOutputLen,
andre@0:                 const unsigned char * input,
andre@0:                 unsigned int inputLen)
andre@0: {
andre@0:     SECStatus rv = SECFailure;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0:     unsigned char * oaepEncoded = NULL;
andre@0: 
andre@0:     if (maxOutputLen < modulusLen) {
andre@0:         PORT_SetError(SEC_ERROR_OUTPUT_LEN);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
andre@0:         PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     if ((labelLen == 0 && label != NULL) ||
andre@0:         (labelLen > 0 && label == NULL)) {
andre@0:         PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     oaepEncoded = (unsigned char *)PORT_Alloc(modulusLen);
andre@0:     if (oaepEncoded == NULL) {
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         return SECFailure;
andre@0:     }
andre@0:     rv = eme_oaep_encode(oaepEncoded, modulusLen, input, inputLen,
andre@0:                          hashAlg, maskHashAlg, label, labelLen, seed, seedLen);
andre@0:     if (rv != SECSuccess)
andre@0:         goto done;
andre@0: 
andre@0:     rv = RSA_PublicKeyOp(key, output, oaepEncoded);
andre@0:     if (rv != SECSuccess)
andre@0:         goto done;
andre@0:     *outputLen = modulusLen;
andre@0: 
andre@0: done:
andre@0:     PORT_Free(oaepEncoded);
andre@0:     return rv;
andre@0: }
andre@0: 
andre@0: SECStatus
andre@0: RSA_DecryptOAEP(RSAPrivateKey * key,
andre@0:                 HASH_HashType hashAlg,
andre@0:                 HASH_HashType maskHashAlg,
andre@0:                 const unsigned char * label,
andre@0:                 unsigned int labelLen,
andre@0:                 unsigned char * output,
andre@0:                 unsigned int * outputLen,
andre@0:                 unsigned int maxOutputLen,
andre@0:                 const unsigned char * input,
andre@0:                 unsigned int inputLen)
andre@0: {
andre@0:     SECStatus rv = SECFailure;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0:     unsigned char * oaepEncoded = NULL;
andre@0: 
andre@0:     if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
andre@0:         PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     if (inputLen != modulusLen) {
andre@0:         PORT_SetError(SEC_ERROR_INPUT_LEN);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     if ((labelLen == 0 && label != NULL) ||
andre@0:         (labelLen > 0 && label == NULL)) {
andre@0:         PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     oaepEncoded = (unsigned char *)PORT_Alloc(modulusLen);
andre@0:     if (oaepEncoded == NULL) {
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     rv = RSA_PrivateKeyOpDoubleChecked(key, oaepEncoded, input);
andre@0:     if (rv != SECSuccess) {
andre@0:         goto done;
andre@0:     }
andre@0:     rv = eme_oaep_decode(output, outputLen, maxOutputLen, oaepEncoded,
andre@0:                          modulusLen, hashAlg, maskHashAlg, label,
andre@0:                          labelLen);
andre@0: 
andre@0: done:
andre@0:     if (oaepEncoded)
andre@0:         PORT_ZFree(oaepEncoded, modulusLen);
andre@0:     return rv;
andre@0: }
andre@0: 
andre@0: /* XXX Doesn't set error code */
andre@0: SECStatus
andre@0: RSA_EncryptBlock(RSAPublicKey * key,
andre@0:                  unsigned char * output,
andre@0:                  unsigned int * outputLen,
andre@0:                  unsigned int maxOutputLen,
andre@0:                  const unsigned char * input,
andre@0:                  unsigned int inputLen)
andre@0: {
andre@0:     SECStatus rv;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0:     SECItem formatted;
andre@0:     SECItem unformatted;
andre@0: 
andre@0:     formatted.data = NULL;
andre@0:     if (maxOutputLen < modulusLen)
andre@0:         goto failure;
andre@0: 
andre@0:     unformatted.len  = inputLen;
andre@0:     unformatted.data = (unsigned char*)input;
andre@0:     formatted.data   = NULL;
andre@0:     rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockPublic,
andre@0:                          &unformatted);
andre@0:     if (rv != SECSuccess)
andre@0:         goto failure;
andre@0: 
andre@0:     rv = RSA_PublicKeyOp(key, output, formatted.data);
andre@0:     if (rv != SECSuccess)
andre@0:         goto failure;
andre@0: 
andre@0:     PORT_ZFree(formatted.data, modulusLen);
andre@0:     *outputLen = modulusLen;
andre@0:     return SECSuccess;
andre@0: 
andre@0: failure:
andre@0:     if (formatted.data != NULL)
andre@0:         PORT_ZFree(formatted.data, modulusLen);
andre@0:     return SECFailure;
andre@0: }
andre@0: 
andre@0: /* XXX Doesn't set error code */
andre@0: SECStatus
andre@0: RSA_DecryptBlock(RSAPrivateKey * key,
andre@0:                  unsigned char * output,
andre@0:                  unsigned int * outputLen,
andre@0:                  unsigned int maxOutputLen,
andre@0:                  const unsigned char * input,
andre@0:                  unsigned int inputLen)
andre@0: {
andre@0:     SECStatus rv;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0:     unsigned int i;
andre@0:     unsigned char * buffer;
andre@0: 
andre@0:     if (inputLen != modulusLen)
andre@0:         goto failure;
andre@0: 
andre@0:     buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
andre@0:     if (!buffer)
andre@0:         goto failure;
andre@0: 
andre@0:     rv = RSA_PrivateKeyOp(key, buffer, input);
andre@0:     if (rv != SECSuccess)
andre@0:         goto loser;
andre@0: 
andre@0:     /* XXX(rsleevi): Constant time */
andre@0:     if (buffer[0] != RSA_BLOCK_FIRST_OCTET ||
andre@0:         buffer[1] != (unsigned char)RSA_BlockPublic) {
andre@0:         goto loser;
andre@0:     }
andre@0:     *outputLen = 0;
andre@0:     for (i = 2; i < modulusLen; i++) {
andre@0:         if (buffer[i] == RSA_BLOCK_AFTER_PAD_OCTET) {
andre@0:             *outputLen = modulusLen - i - 1;
andre@0:             break;
andre@0:         }
andre@0:     }
andre@0:     if (*outputLen == 0)
andre@0:         goto loser;
andre@0:     if (*outputLen > maxOutputLen)
andre@0:         goto loser;
andre@0: 
andre@0:     PORT_Memcpy(output, buffer + modulusLen - *outputLen, *outputLen);
andre@0: 
andre@0:     PORT_Free(buffer);
andre@0:     return SECSuccess;
andre@0: 
andre@0: loser:
andre@0:     PORT_Free(buffer);
andre@0: failure:
andre@0:     return SECFailure;
andre@0: }
andre@0: 
andre@0: /*
andre@0:  * Encode a RSA-PSS signature.
andre@0:  * Described in RFC 3447, section 9.1.1.
andre@0:  * We use mHash instead of M as input.
andre@0:  * emBits from the RFC is just modBits - 1, see section 8.1.1.
andre@0:  * We only support MGF1 as the MGF.
andre@0:  *
andre@0:  * NOTE: this code assumes modBits is a multiple of 8.
andre@0:  */
andre@0: static SECStatus
andre@0: emsa_pss_encode(unsigned char * em,
andre@0:                 unsigned int emLen,
andre@0:                 const unsigned char * mHash,
andre@0:                 HASH_HashType hashAlg,
andre@0:                 HASH_HashType maskHashAlg,
andre@0:                 const unsigned char * salt,
andre@0:                 unsigned int saltLen)
andre@0: {
andre@0:     const SECHashObject * hash;
andre@0:     void * hash_context;
andre@0:     unsigned char * dbMask;
andre@0:     unsigned int dbMaskLen;
andre@0:     unsigned int i;
andre@0:     SECStatus rv;
andre@0: 
andre@0:     hash = HASH_GetRawHashObject(hashAlg);
andre@0:     dbMaskLen = emLen - hash->length - 1;
andre@0: 
andre@0:     /* Step 3 */
andre@0:     if (emLen < hash->length + saltLen + 2) {
andre@0:         PORT_SetError(SEC_ERROR_OUTPUT_LEN);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     /* Step 4 */
andre@0:     if (salt == NULL) {
andre@0:         rv = RNG_GenerateGlobalRandomBytes(&em[dbMaskLen - saltLen], saltLen);
andre@0:         if (rv != SECSuccess) {
andre@0:             return rv;
andre@0:         }
andre@0:     } else {
andre@0:         PORT_Memcpy(&em[dbMaskLen - saltLen], salt, saltLen);
andre@0:     }
andre@0: 
andre@0:     /* Step 5 + 6 */
andre@0:     /* Compute H and store it at its final location &em[dbMaskLen]. */
andre@0:     hash_context = (*hash->create)();
andre@0:     if (hash_context == NULL) {
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         return SECFailure;
andre@0:     }
andre@0:     (*hash->begin)(hash_context);
andre@0:     (*hash->update)(hash_context, eightZeros, 8);
andre@0:     (*hash->update)(hash_context, mHash, hash->length);
andre@0:     (*hash->update)(hash_context, &em[dbMaskLen - saltLen], saltLen);
andre@0:     (*hash->end)(hash_context, &em[dbMaskLen], &i, hash->length);
andre@0:     (*hash->destroy)(hash_context, PR_TRUE);
andre@0: 
andre@0:     /* Step 7 + 8 */
andre@0:     PORT_Memset(em, 0, dbMaskLen - saltLen - 1);
andre@0:     em[dbMaskLen - saltLen - 1] = 0x01;
andre@0: 
andre@0:     /* Step 9 */
andre@0:     dbMask = (unsigned char *)PORT_Alloc(dbMaskLen);
andre@0:     if (dbMask == NULL) {
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         return SECFailure;
andre@0:     }
andre@0:     MGF1(maskHashAlg, dbMask, dbMaskLen, &em[dbMaskLen], hash->length);
andre@0: 
andre@0:     /* Step 10 */
andre@0:     for (i = 0; i < dbMaskLen; i++)
andre@0:         em[i] ^= dbMask[i];
andre@0:     PORT_Free(dbMask);
andre@0: 
andre@0:     /* Step 11 */
andre@0:     em[0] &= 0x7f;
andre@0: 
andre@0:     /* Step 12 */
andre@0:     em[emLen - 1] = 0xbc;
andre@0: 
andre@0:     return SECSuccess;
andre@0: }
andre@0: 
andre@0: /*
andre@0:  * Verify a RSA-PSS signature.
andre@0:  * Described in RFC 3447, section 9.1.2.
andre@0:  * We use mHash instead of M as input.
andre@0:  * emBits from the RFC is just modBits - 1, see section 8.1.2.
andre@0:  * We only support MGF1 as the MGF.
andre@0:  *
andre@0:  * NOTE: this code assumes modBits is a multiple of 8.
andre@0:  */
andre@0: static SECStatus
andre@0: emsa_pss_verify(const unsigned char * mHash,
andre@0:                 const unsigned char * em,
andre@0:                 unsigned int emLen,
andre@0:                 HASH_HashType hashAlg,
andre@0:                 HASH_HashType maskHashAlg,
andre@0:                 unsigned int saltLen)
andre@0: {
andre@0:     const SECHashObject * hash;
andre@0:     void * hash_context;
andre@0:     unsigned char * db;
andre@0:     unsigned char * H_;  /* H' from the RFC */
andre@0:     unsigned int i;
andre@0:     unsigned int dbMaskLen;
andre@0:     SECStatus rv;
andre@0: 
andre@0:     hash = HASH_GetRawHashObject(hashAlg);
andre@0:     dbMaskLen = emLen - hash->length - 1;
andre@0: 
andre@0:     /* Step 3 + 4 + 6 */
andre@0:     if ((emLen < (hash->length + saltLen + 2)) ||
andre@0:         (em[emLen - 1] != 0xbc) ||
andre@0:         ((em[0] & 0x80) != 0)) {
andre@0:         PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     /* Step 7 */
andre@0:     db = (unsigned char *)PORT_Alloc(dbMaskLen);
andre@0:     if (db == NULL) {
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         return SECFailure;
andre@0:     }
andre@0:     /* &em[dbMaskLen] points to H, used as mgfSeed */
andre@0:     MGF1(maskHashAlg, db, dbMaskLen, &em[dbMaskLen], hash->length);
andre@0: 
andre@0:     /* Step 8 */
andre@0:     for (i = 0; i < dbMaskLen; i++) {
andre@0:         db[i] ^= em[i];
andre@0:     }
andre@0: 
andre@0:     /* Step 9 */
andre@0:     db[0] &= 0x7f;
andre@0: 
andre@0:     /* Step 10 */
andre@0:     for (i = 0; i < (dbMaskLen - saltLen - 1); i++) {
andre@0:         if (db[i] != 0) {
andre@0:             PORT_Free(db);
andre@0:             PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
andre@0:             return SECFailure;
andre@0:         }
andre@0:     }
andre@0:     if (db[dbMaskLen - saltLen - 1] != 0x01) {
andre@0:         PORT_Free(db);
andre@0:         PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     /* Step 12 + 13 */
andre@0:     H_ = (unsigned char *)PORT_Alloc(hash->length);
andre@0:     if (H_ == NULL) {
andre@0:         PORT_Free(db);
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         return SECFailure;
andre@0:     }
andre@0:     hash_context = (*hash->create)();
andre@0:     if (hash_context == NULL) {
andre@0:         PORT_Free(db);
andre@0:         PORT_Free(H_);
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         return SECFailure;
andre@0:     }
andre@0:     (*hash->begin)(hash_context);
andre@0:     (*hash->update)(hash_context, eightZeros, 8);
andre@0:     (*hash->update)(hash_context, mHash, hash->length);
andre@0:     (*hash->update)(hash_context, &db[dbMaskLen - saltLen], saltLen);
andre@0:     (*hash->end)(hash_context, H_, &i, hash->length);
andre@0:     (*hash->destroy)(hash_context, PR_TRUE);
andre@0: 
andre@0:     PORT_Free(db);
andre@0: 
andre@0:     /* Step 14 */
andre@0:     if (PORT_Memcmp(H_, &em[dbMaskLen], hash->length) != 0) {
andre@0:         PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
andre@0:         rv = SECFailure;
andre@0:     } else {
andre@0:         rv = SECSuccess;
andre@0:     }
andre@0: 
andre@0:     PORT_Free(H_);
andre@0:     return rv;
andre@0: }
andre@0: 
andre@0: SECStatus
andre@0: RSA_SignPSS(RSAPrivateKey * key,
andre@0:             HASH_HashType hashAlg,
andre@0:             HASH_HashType maskHashAlg,
andre@0:             const unsigned char * salt,
andre@0:             unsigned int saltLength,
andre@0:             unsigned char * output,
andre@0:             unsigned int * outputLen,
andre@0:             unsigned int maxOutputLen,
andre@0:             const unsigned char * input,
andre@0:             unsigned int inputLen)
andre@0: {
andre@0:     SECStatus rv = SECSuccess;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0:     unsigned char *pssEncoded = NULL;
andre@0: 
andre@0:     if (maxOutputLen < modulusLen) {
andre@0:         PORT_SetError(SEC_ERROR_OUTPUT_LEN);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
andre@0:         PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     pssEncoded = (unsigned char *)PORT_Alloc(modulusLen);
andre@0:     if (pssEncoded == NULL) {
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         return SECFailure;
andre@0:     }
andre@0:     rv = emsa_pss_encode(pssEncoded, modulusLen, input, hashAlg,
andre@0:                          maskHashAlg, salt, saltLength);
andre@0:     if (rv != SECSuccess)
andre@0:         goto done;
andre@0: 
andre@0:     rv = RSA_PrivateKeyOpDoubleChecked(key, output, pssEncoded);
andre@0:     *outputLen = modulusLen;
andre@0: 
andre@0: done:
andre@0:     PORT_Free(pssEncoded);
andre@0:     return rv;
andre@0: }
andre@0: 
andre@0: SECStatus
andre@0: RSA_CheckSignPSS(RSAPublicKey * key,
andre@0:                  HASH_HashType hashAlg,
andre@0:                  HASH_HashType maskHashAlg,
andre@0:                  unsigned int saltLength,
andre@0:                  const unsigned char * sig,
andre@0:                  unsigned int sigLen,
andre@0:                  const unsigned char * hash,
andre@0:                  unsigned int hashLen)
andre@0: {
andre@0:     SECStatus rv;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0:     unsigned char * buffer;
andre@0: 
andre@0:     if (sigLen != modulusLen) {
andre@0:         PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
andre@0:         PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     buffer = (unsigned char *)PORT_Alloc(modulusLen);
andre@0:     if (!buffer) {
andre@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     rv = RSA_PublicKeyOp(key, buffer, sig);
andre@0:     if (rv != SECSuccess) {
andre@0:         PORT_Free(buffer);
andre@0:         PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
andre@0:         return SECFailure;
andre@0:     }
andre@0: 
andre@0:     rv = emsa_pss_verify(hash, buffer, modulusLen, hashAlg,
andre@0:                          maskHashAlg, saltLength);
andre@0:     PORT_Free(buffer);
andre@0: 
andre@0:     return rv;
andre@0: }
andre@0: 
andre@0: /* XXX Doesn't set error code */
andre@0: SECStatus
andre@0: RSA_Sign(RSAPrivateKey * key,
andre@0:          unsigned char * output,
andre@0:          unsigned int * outputLen,
andre@0:          unsigned int maxOutputLen,
andre@0:          const unsigned char * input,
andre@0:          unsigned int inputLen)
andre@0: {
andre@0:     SECStatus rv = SECSuccess;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0:     SECItem formatted;
andre@0:     SECItem unformatted;
andre@0: 
andre@0:     if (maxOutputLen < modulusLen)
andre@0:         return SECFailure;
andre@0: 
andre@0:     unformatted.len  = inputLen;
andre@0:     unformatted.data = (unsigned char*)input;
andre@0:     formatted.data   = NULL;
andre@0:     rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockPrivate,
andre@0:                          &unformatted);
andre@0:     if (rv != SECSuccess)
andre@0:         goto done;
andre@0: 
andre@0:     rv = RSA_PrivateKeyOpDoubleChecked(key, output, formatted.data);
andre@0:     *outputLen = modulusLen;
andre@0: 
andre@0:     goto done;
andre@0: 
andre@0: done:
andre@0:     if (formatted.data != NULL)
andre@0:         PORT_ZFree(formatted.data, modulusLen);
andre@0:     return rv;
andre@0: }
andre@0: 
andre@0: /* XXX Doesn't set error code */
andre@0: SECStatus
andre@0: RSA_CheckSign(RSAPublicKey * key,
andre@0:               const unsigned char * sig,
andre@0:               unsigned int sigLen,
andre@0:               const unsigned char * data,
andre@0:               unsigned int dataLen)
andre@0: {
andre@0:     SECStatus rv;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0:     unsigned int i;
andre@0:     unsigned char * buffer;
andre@0: 
andre@0:     if (sigLen != modulusLen)
andre@0:         goto failure;
andre@0:     /*
andre@0:      * 0x00 || BT || Pad || 0x00 || ActualData
andre@0:      *
andre@0:      * The "3" below is the first octet + the second octet + the 0x00
andre@0:      * octet that always comes just before the ActualData.
andre@0:      */
andre@0:     if (dataLen > modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN))
andre@0:         goto failure;
andre@0: 
andre@0:     buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
andre@0:     if (!buffer)
andre@0:         goto failure;
andre@0: 
andre@0:     rv = RSA_PublicKeyOp(key, buffer, sig);
andre@0:     if (rv != SECSuccess)
andre@0:         goto loser;
andre@0: 
andre@0:     /*
andre@0:      * check the padding that was used
andre@0:      */
andre@0:     if (buffer[0] != RSA_BLOCK_FIRST_OCTET ||
andre@0:         buffer[1] != (unsigned char)RSA_BlockPrivate) {
andre@0:         goto loser;
andre@0:     }
andre@0:     for (i = 2; i < modulusLen - dataLen - 1; i++) {
andre@0:         if (buffer[i] != RSA_BLOCK_PRIVATE_PAD_OCTET)
andre@0:             goto loser;
andre@0:     }
andre@0:     if (buffer[i] != RSA_BLOCK_AFTER_PAD_OCTET)
andre@0:         goto loser;
andre@0: 
andre@0:     /*
andre@0:      * make sure we get the same results
andre@0:      */
andre@0:     if (PORT_Memcmp(buffer + modulusLen - dataLen, data, dataLen) != 0)
andre@0:         goto loser;
andre@0: 
andre@0:     PORT_Free(buffer);
andre@0:     return SECSuccess;
andre@0: 
andre@0: loser:
andre@0:     PORT_Free(buffer);
andre@0: failure:
andre@0:     return SECFailure;
andre@0: }
andre@0: 
andre@0: /* XXX Doesn't set error code */
andre@0: SECStatus
andre@0: RSA_CheckSignRecover(RSAPublicKey * key,
andre@0:                      unsigned char * output,
andre@0:                      unsigned int * outputLen,
andre@0:                      unsigned int maxOutputLen,
andre@0:                      const unsigned char * sig,
andre@0:                      unsigned int sigLen)
andre@0: {
andre@0:     SECStatus rv;
andre@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
andre@0:     unsigned int i;
andre@0:     unsigned char * buffer;
andre@0: 
andre@0:     if (sigLen != modulusLen)
andre@0:         goto failure;
andre@0: 
andre@0:     buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
andre@0:     if (!buffer)
andre@0:         goto failure;
andre@0: 
andre@0:     rv = RSA_PublicKeyOp(key, buffer, sig);
andre@0:     if (rv != SECSuccess)
andre@0:         goto loser;
andre@0:     *outputLen = 0;
andre@0: 
andre@0:     /*
andre@0:      * check the padding that was used
andre@0:      */
andre@0:     if (buffer[0] != RSA_BLOCK_FIRST_OCTET ||
andre@0:         buffer[1] != (unsigned char)RSA_BlockPrivate) {
andre@0:         goto loser;
andre@0:     }
andre@0:     for (i = 2; i < modulusLen; i++) {
andre@0:         if (buffer[i] == RSA_BLOCK_AFTER_PAD_OCTET) {
andre@0:             *outputLen = modulusLen - i - 1;
andre@0:             break;
andre@0:         }
andre@0:         if (buffer[i] != RSA_BLOCK_PRIVATE_PAD_OCTET)
andre@0:             goto loser;
andre@0:     }
andre@0:     if (*outputLen == 0)
andre@0:         goto loser;
andre@0:     if (*outputLen > maxOutputLen)
andre@0:         goto loser;
andre@0: 
andre@0:     PORT_Memcpy(output, buffer + modulusLen - *outputLen, *outputLen);
andre@0: 
andre@0:     PORT_Free(buffer);
andre@0:     return SECSuccess;
andre@0: 
andre@0: loser:
andre@0:     PORT_Free(buffer);
andre@0: failure:
andre@0:     return SECFailure;
andre@0: }