andre@0: /*
andre@0:  * alg2268.c - implementation of the algorithm in RFC 2268
andre@0:  *
andre@0:  * This Source Code Form is subject to the terms of the Mozilla Public
andre@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
andre@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
andre@0: 
andre@0: #ifdef FREEBL_NO_DEPEND
andre@0: #include "stubs.h"
andre@0: #endif
andre@0: 
andre@0: #include "blapi.h"
andre@0: #include "secerr.h"
andre@0: #ifdef XP_UNIX_XXX
andre@0: #include <stddef.h>	/* for ptrdiff_t */
andre@0: #endif
andre@0: 
andre@0: /*
andre@0: ** RC2 symmetric block cypher
andre@0: */
andre@0: 
andre@0: typedef SECStatus (rc2Func)(RC2Context *cx, unsigned char *output,
andre@0: 		           const unsigned char *input, unsigned int inputLen);
andre@0: 
andre@0: /* forward declarations */
andre@0: static rc2Func rc2_EncryptECB;
andre@0: static rc2Func rc2_DecryptECB;
andre@0: static rc2Func rc2_EncryptCBC;
andre@0: static rc2Func rc2_DecryptCBC;
andre@0: 
andre@0: typedef union {
andre@0:     PRUint32	l[2];
andre@0:     PRUint16	s[4];
andre@0:     PRUint8	b[8];
andre@0: } RC2Block;
andre@0: 
andre@0: struct RC2ContextStr {
andre@0:     union {
andre@0:     	PRUint8  Kb[128];
andre@0: 	PRUint16 Kw[64];
andre@0:     } u;
andre@0:     RC2Block     iv;
andre@0:     rc2Func      *enc;
andre@0:     rc2Func      *dec;
andre@0: };
andre@0: 
andre@0: #define B u.Kb
andre@0: #define K u.Kw
andre@0: #define BYTESWAP(x) ((x) << 8 | (x) >> 8)
andre@0: #define SWAPK(i)  cx->K[i] = (tmpS = cx->K[i], BYTESWAP(tmpS))
andre@0: #define RC2_BLOCK_SIZE 8
andre@0: 
andre@0: #define LOAD_HARD(R) \
andre@0:     R[0] = (PRUint16)input[1] << 8 | input[0]; \
andre@0:     R[1] = (PRUint16)input[3] << 8 | input[2]; \
andre@0:     R[2] = (PRUint16)input[5] << 8 | input[4]; \
andre@0:     R[3] = (PRUint16)input[7] << 8 | input[6];
andre@0: #define LOAD_EASY(R) \
andre@0:     R[0] = ((PRUint16 *)input)[0]; \
andre@0:     R[1] = ((PRUint16 *)input)[1]; \
andre@0:     R[2] = ((PRUint16 *)input)[2]; \
andre@0:     R[3] = ((PRUint16 *)input)[3];
andre@0: #define STORE_HARD(R) \
andre@0:     output[0] =  (PRUint8)(R[0]);   output[1] = (PRUint8)(R[0] >> 8); \
andre@0:     output[2] =  (PRUint8)(R[1]);   output[3] = (PRUint8)(R[1] >> 8); \
andre@0:     output[4] =  (PRUint8)(R[2]);   output[5] = (PRUint8)(R[2] >> 8); \
andre@0:     output[6] =  (PRUint8)(R[3]);   output[7] = (PRUint8)(R[3] >> 8);
andre@0: #define STORE_EASY(R) \
andre@0:     ((PRUint16 *)output)[0] =  R[0]; \
andre@0:     ((PRUint16 *)output)[1] =  R[1]; \
andre@0:     ((PRUint16 *)output)[2] =  R[2]; \
andre@0:     ((PRUint16 *)output)[3] =  R[3];   
andre@0: 
andre@0: #if defined (NSS_X86_OR_X64)
andre@0: #define LOAD(R)  LOAD_EASY(R)
andre@0: #define STORE(R) STORE_EASY(R)
andre@0: #elif !defined(IS_LITTLE_ENDIAN)
andre@0: #define LOAD(R)  LOAD_HARD(R)
andre@0: #define STORE(R) STORE_HARD(R)
andre@0: #else
andre@0: #define LOAD(R) if ((ptrdiff_t)input & 1) { LOAD_HARD(R) } else { LOAD_EASY(R) }
andre@0: #define STORE(R) if ((ptrdiff_t)input & 1) { STORE_HARD(R) } else { STORE_EASY(R) }
andre@0: #endif
andre@0: 
andre@0: static const PRUint8 S[256] = {
andre@0: 0331,0170,0371,0304,0031,0335,0265,0355,0050,0351,0375,0171,0112,0240,0330,0235,
andre@0: 0306,0176,0067,0203,0053,0166,0123,0216,0142,0114,0144,0210,0104,0213,0373,0242,
andre@0: 0027,0232,0131,0365,0207,0263,0117,0023,0141,0105,0155,0215,0011,0201,0175,0062,
andre@0: 0275,0217,0100,0353,0206,0267,0173,0013,0360,0225,0041,0042,0134,0153,0116,0202,
andre@0: 0124,0326,0145,0223,0316,0140,0262,0034,0163,0126,0300,0024,0247,0214,0361,0334,
andre@0: 0022,0165,0312,0037,0073,0276,0344,0321,0102,0075,0324,0060,0243,0074,0266,0046,
andre@0: 0157,0277,0016,0332,0106,0151,0007,0127,0047,0362,0035,0233,0274,0224,0103,0003,
andre@0: 0370,0021,0307,0366,0220,0357,0076,0347,0006,0303,0325,0057,0310,0146,0036,0327,
andre@0: 0010,0350,0352,0336,0200,0122,0356,0367,0204,0252,0162,0254,0065,0115,0152,0052,
andre@0: 0226,0032,0322,0161,0132,0025,0111,0164,0113,0237,0320,0136,0004,0030,0244,0354,
andre@0: 0302,0340,0101,0156,0017,0121,0313,0314,0044,0221,0257,0120,0241,0364,0160,0071,
andre@0: 0231,0174,0072,0205,0043,0270,0264,0172,0374,0002,0066,0133,0045,0125,0227,0061,
andre@0: 0055,0135,0372,0230,0343,0212,0222,0256,0005,0337,0051,0020,0147,0154,0272,0311,
andre@0: 0323,0000,0346,0317,0341,0236,0250,0054,0143,0026,0001,0077,0130,0342,0211,0251,
andre@0: 0015,0070,0064,0033,0253,0063,0377,0260,0273,0110,0014,0137,0271,0261,0315,0056,
andre@0: 0305,0363,0333,0107,0345,0245,0234,0167,0012,0246,0040,0150,0376,0177,0301,0255
andre@0: };
andre@0: 
andre@0: RC2Context * RC2_AllocateContext(void)
andre@0: {
andre@0:     return PORT_ZNew(RC2Context);
andre@0: }
andre@0: SECStatus   
andre@0: RC2_InitContext(RC2Context *cx, const unsigned char *key, unsigned int len,
andre@0: 	        const unsigned char *input, int mode, unsigned int efLen8, 
andre@0: 		unsigned int unused)
andre@0: {
andre@0:     PRUint8    *L,*L2;
andre@0:     int         i;
andre@0: #if !defined(IS_LITTLE_ENDIAN)
andre@0:     PRUint16    tmpS;
andre@0: #endif
andre@0:     PRUint8     tmpB;
andre@0: 
andre@0:     if (!key || !cx || !len || len > (sizeof cx->B) || 
andre@0: 	efLen8 > (sizeof cx->B)) {
andre@0: 	PORT_SetError(SEC_ERROR_INVALID_ARGS);
andre@0:     	return SECFailure;
andre@0:     }
andre@0:     if (mode == NSS_RC2) {
andre@0:     	/* groovy */
andre@0:     } else if (mode == NSS_RC2_CBC) {
andre@0:     	if (!input) {
andre@0: 	    PORT_SetError(SEC_ERROR_INVALID_ARGS);
andre@0: 	    return SECFailure;
andre@0: 	}
andre@0:     } else {
andre@0: 	PORT_SetError(SEC_ERROR_INVALID_ARGS);
andre@0: 	return SECFailure;
andre@0:     }
andre@0: 
andre@0:     if (mode == NSS_RC2_CBC) {
andre@0:     	cx->enc = & rc2_EncryptCBC;
andre@0: 	cx->dec = & rc2_DecryptCBC;
andre@0: 	LOAD(cx->iv.s);
andre@0:     } else {
andre@0:     	cx->enc = & rc2_EncryptECB;
andre@0: 	cx->dec = & rc2_DecryptECB;
andre@0:     }
andre@0: 
andre@0:     /* Step 0. Copy key into table. */
andre@0:     memcpy(cx->B, key, len);
andre@0: 
andre@0:     /* Step 1. Compute all values to the right of the key. */
andre@0:     L2 = cx->B;
andre@0:     L = L2 + len;
andre@0:     tmpB = L[-1];
andre@0:     for (i = (sizeof cx->B) - len; i > 0; --i) {
andre@0: 	*L++ = tmpB = S[ (PRUint8)(tmpB + *L2++) ];
andre@0:     }
andre@0: 
andre@0:     /* step 2. Adjust left most byte of effective key. */
andre@0:     i = (sizeof cx->B) - efLen8;
andre@0:     L = cx->B + i;
andre@0:     *L = tmpB = S[*L];				/* mask is always 0xff */
andre@0: 
andre@0:     /* step 3. Recompute all values to the left of effective key. */
andre@0:     L2 = --L + efLen8;
andre@0:     while(L >= cx->B) {
andre@0: 	*L-- = tmpB = S[ tmpB ^ *L2-- ];
andre@0:     }
andre@0: 
andre@0: #if !defined(IS_LITTLE_ENDIAN)
andre@0:     for (i = 63; i >= 0; --i) {
andre@0:         SWAPK(i);		/* candidate for unrolling */
andre@0:     }
andre@0: #endif
andre@0:     return SECSuccess;
andre@0: }
andre@0: 
andre@0: /*
andre@0: ** Create a new RC2 context suitable for RC2 encryption/decryption.
andre@0: ** 	"key" raw key data
andre@0: ** 	"len" the number of bytes of key data
andre@0: ** 	"iv" is the CBC initialization vector (if mode is NSS_RC2_CBC)
andre@0: ** 	"mode" one of NSS_RC2 or NSS_RC2_CBC
andre@0: **	"effectiveKeyLen" in bytes, not bits.
andre@0: **
andre@0: ** When mode is set to NSS_RC2_CBC the RC2 cipher is run in "cipher block
andre@0: ** chaining" mode.
andre@0: */
andre@0: RC2Context *
andre@0: RC2_CreateContext(const unsigned char *key, unsigned int len,
andre@0: 		  const unsigned char *iv, int mode, unsigned efLen8)
andre@0: {
andre@0:     RC2Context *cx = PORT_ZNew(RC2Context);
andre@0:     if (cx) {
andre@0: 	SECStatus rv = RC2_InitContext(cx, key, len, iv, mode, efLen8, 0);
andre@0: 	if (rv != SECSuccess) {
andre@0: 	    RC2_DestroyContext(cx, PR_TRUE);
andre@0: 	    cx = NULL;
andre@0: 	}
andre@0:     }
andre@0:     return cx;
andre@0: }
andre@0: 
andre@0: /*
andre@0: ** Destroy an RC2 encryption/decryption context.
andre@0: **	"cx" the context
andre@0: **	"freeit" if PR_TRUE then free the object as well as its sub-objects
andre@0: */
andre@0: void 
andre@0: RC2_DestroyContext(RC2Context *cx, PRBool freeit)
andre@0: {
andre@0:     if (cx) {
andre@0: 	memset(cx, 0, sizeof *cx);
andre@0: 	if (freeit) {
andre@0: 	    PORT_Free(cx);
andre@0: 	}
andre@0:     }
andre@0: }
andre@0: 
andre@0: #define ROL(x,k) (x << k | x >> (16-k))
andre@0: #define MIX(j) \
andre@0:     R0 = R0 + cx->K[ 4*j+0] + (R3 & R2) + (~R3 & R1);  R0 = ROL(R0,1);\
andre@0:     R1 = R1 + cx->K[ 4*j+1] + (R0 & R3) + (~R0 & R2);  R1 = ROL(R1,2);\
andre@0:     R2 = R2 + cx->K[ 4*j+2] + (R1 & R0) + (~R1 & R3);  R2 = ROL(R2,3);\
andre@0:     R3 = R3 + cx->K[ 4*j+3] + (R2 & R1) + (~R2 & R0);  R3 = ROL(R3,5)
andre@0: #define MASH \
andre@0:     R0 = R0 + cx->K[R3 & 63];\
andre@0:     R1 = R1 + cx->K[R0 & 63];\
andre@0:     R2 = R2 + cx->K[R1 & 63];\
andre@0:     R3 = R3 + cx->K[R2 & 63]
andre@0: 
andre@0: /* Encrypt one block */
andre@0: static void 
andre@0: rc2_Encrypt1Block(RC2Context *cx, RC2Block *output, RC2Block *input)
andre@0: {
andre@0:     register PRUint16 R0, R1, R2, R3;
andre@0: 
andre@0:     /* step 1. Initialize input. */
andre@0:     R0 = input->s[0];
andre@0:     R1 = input->s[1];
andre@0:     R2 = input->s[2];
andre@0:     R3 = input->s[3];
andre@0: 
andre@0:     /* step 2.  Expand Key (already done, in context) */
andre@0:     /* step 3.  j = 0 */
andre@0:     /* step 4.  Perform 5 mixing rounds. */
andre@0: 
andre@0:     MIX(0);
andre@0:     MIX(1);
andre@0:     MIX(2);
andre@0:     MIX(3);
andre@0:     MIX(4);
andre@0: 
andre@0:     /* step 5. Perform 1 mashing round. */
andre@0:     MASH;
andre@0: 
andre@0:     /* step 6. Perform 6 mixing rounds. */
andre@0: 
andre@0:     MIX(5);
andre@0:     MIX(6);
andre@0:     MIX(7);
andre@0:     MIX(8);
andre@0:     MIX(9);
andre@0:     MIX(10);
andre@0: 
andre@0:     /* step 7. Perform 1 mashing round. */
andre@0:     MASH;
andre@0: 
andre@0:     /* step 8. Perform 5 mixing rounds. */
andre@0: 
andre@0:     MIX(11);
andre@0:     MIX(12);
andre@0:     MIX(13);
andre@0:     MIX(14);
andre@0:     MIX(15);
andre@0: 
andre@0:     /* output results */
andre@0:     output->s[0] = R0;
andre@0:     output->s[1] = R1;
andre@0:     output->s[2] = R2;
andre@0:     output->s[3] = R3;
andre@0: }
andre@0: 
andre@0: #define ROR(x,k) (x >> k | x << (16-k))
andre@0: #define R_MIX(j) \
andre@0:     R3 = ROR(R3,5); R3 = R3 - cx->K[ 4*j+3] - (R2 & R1) - (~R2 & R0);  \
andre@0:     R2 = ROR(R2,3); R2 = R2 - cx->K[ 4*j+2] - (R1 & R0) - (~R1 & R3);  \
andre@0:     R1 = ROR(R1,2); R1 = R1 - cx->K[ 4*j+1] - (R0 & R3) - (~R0 & R2);  \
andre@0:     R0 = ROR(R0,1); R0 = R0 - cx->K[ 4*j+0] - (R3 & R2) - (~R3 & R1)
andre@0: #define R_MASH \
andre@0:     R3 = R3 - cx->K[R2 & 63];\
andre@0:     R2 = R2 - cx->K[R1 & 63];\
andre@0:     R1 = R1 - cx->K[R0 & 63];\
andre@0:     R0 = R0 - cx->K[R3 & 63]
andre@0: 
andre@0: /* Encrypt one block */
andre@0: static void 
andre@0: rc2_Decrypt1Block(RC2Context *cx, RC2Block *output, RC2Block *input)
andre@0: {
andre@0:     register PRUint16 R0, R1, R2, R3;
andre@0: 
andre@0:     /* step 1. Initialize input. */
andre@0:     R0 = input->s[0];
andre@0:     R1 = input->s[1];
andre@0:     R2 = input->s[2];
andre@0:     R3 = input->s[3];
andre@0: 
andre@0:     /* step 2.  Expand Key (already done, in context) */
andre@0:     /* step 3.  j = 63 */
andre@0:     /* step 4.  Perform 5 r_mixing rounds. */
andre@0:     R_MIX(15);
andre@0:     R_MIX(14);
andre@0:     R_MIX(13);
andre@0:     R_MIX(12);
andre@0:     R_MIX(11);
andre@0: 
andre@0:     /* step 5.  Perform 1 r_mashing round. */
andre@0:     R_MASH;
andre@0: 
andre@0:     /* step 6.  Perform 6 r_mixing rounds. */
andre@0:     R_MIX(10);
andre@0:     R_MIX(9);
andre@0:     R_MIX(8);
andre@0:     R_MIX(7);
andre@0:     R_MIX(6);
andre@0:     R_MIX(5);
andre@0: 
andre@0:     /* step 7.  Perform 1 r_mashing round. */
andre@0:     R_MASH;
andre@0: 
andre@0:     /* step 8.  Perform 5 r_mixing rounds. */
andre@0:     R_MIX(4);
andre@0:     R_MIX(3);
andre@0:     R_MIX(2);
andre@0:     R_MIX(1);
andre@0:     R_MIX(0);
andre@0: 
andre@0:     /* output results */
andre@0:     output->s[0] = R0;
andre@0:     output->s[1] = R1;
andre@0:     output->s[2] = R2;
andre@0:     output->s[3] = R3;
andre@0: }
andre@0: 
andre@0: static SECStatus
andre@0: rc2_EncryptECB(RC2Context *cx, unsigned char *output,
andre@0: 	       const unsigned char *input, unsigned int inputLen)
andre@0: {
andre@0:     RC2Block  iBlock;
andre@0: 
andre@0:     while (inputLen > 0) {
andre@0:     	LOAD(iBlock.s)
andre@0: 	rc2_Encrypt1Block(cx, &iBlock, &iBlock);
andre@0: 	STORE(iBlock.s)
andre@0: 	output   += RC2_BLOCK_SIZE;
andre@0: 	input    += RC2_BLOCK_SIZE;
andre@0: 	inputLen -= RC2_BLOCK_SIZE;
andre@0:     }
andre@0:     return SECSuccess;
andre@0: }
andre@0: 
andre@0: static SECStatus
andre@0: rc2_DecryptECB(RC2Context *cx, unsigned char *output,
andre@0: 	       const unsigned char *input, unsigned int inputLen)
andre@0: {
andre@0:     RC2Block  iBlock;
andre@0: 
andre@0:     while (inputLen > 0) {
andre@0:     	LOAD(iBlock.s)
andre@0: 	rc2_Decrypt1Block(cx, &iBlock, &iBlock);
andre@0: 	STORE(iBlock.s)
andre@0: 	output   += RC2_BLOCK_SIZE;
andre@0: 	input    += RC2_BLOCK_SIZE;
andre@0: 	inputLen -= RC2_BLOCK_SIZE;
andre@0:     }
andre@0:     return SECSuccess;
andre@0: }
andre@0: 
andre@0: static SECStatus
andre@0: rc2_EncryptCBC(RC2Context *cx, unsigned char *output,
andre@0: 	       const unsigned char *input, unsigned int inputLen)
andre@0: {
andre@0:     RC2Block  iBlock;
andre@0: 
andre@0:     while (inputLen > 0) {
andre@0: 
andre@0: 	LOAD(iBlock.s)
andre@0: 	iBlock.l[0] ^= cx->iv.l[0];
andre@0: 	iBlock.l[1] ^= cx->iv.l[1];
andre@0: 	rc2_Encrypt1Block(cx, &iBlock, &iBlock);
andre@0: 	cx->iv = iBlock;
andre@0: 	STORE(iBlock.s)
andre@0: 	output   += RC2_BLOCK_SIZE;
andre@0: 	input    += RC2_BLOCK_SIZE;
andre@0: 	inputLen -= RC2_BLOCK_SIZE;
andre@0:     }
andre@0:     return SECSuccess;
andre@0: }
andre@0: 
andre@0: static SECStatus
andre@0: rc2_DecryptCBC(RC2Context *cx, unsigned char *output,
andre@0: 	       const unsigned char *input, unsigned int inputLen)
andre@0: {
andre@0:     RC2Block  iBlock;
andre@0:     RC2Block  oBlock;
andre@0: 
andre@0:     while (inputLen > 0) {
andre@0: 	LOAD(iBlock.s)
andre@0: 	rc2_Decrypt1Block(cx, &oBlock, &iBlock);
andre@0: 	oBlock.l[0] ^= cx->iv.l[0];
andre@0: 	oBlock.l[1] ^= cx->iv.l[1];
andre@0: 	cx->iv = iBlock;
andre@0: 	STORE(oBlock.s)
andre@0: 	output   += RC2_BLOCK_SIZE;
andre@0: 	input    += RC2_BLOCK_SIZE;
andre@0: 	inputLen -= RC2_BLOCK_SIZE;
andre@0:     }
andre@0:     return SECSuccess;
andre@0: }
andre@0: 
andre@0: 
andre@0: /*
andre@0: ** Perform RC2 encryption.
andre@0: **	"cx" the context
andre@0: **	"output" the output buffer to store the encrypted data.
andre@0: **	"outputLen" how much data is stored in "output". Set by the routine
andre@0: **	   after some data is stored in output.
andre@0: **	"maxOutputLen" the maximum amount of data that can ever be
andre@0: **	   stored in "output"
andre@0: **	"input" the input data
andre@0: **	"inputLen" the amount of input data
andre@0: */
andre@0: SECStatus RC2_Encrypt(RC2Context *cx, unsigned char *output,
andre@0: 		      unsigned int *outputLen, unsigned int maxOutputLen,
andre@0: 		      const unsigned char *input, unsigned int inputLen)
andre@0: {
andre@0:     SECStatus rv = SECSuccess;
andre@0:     if (inputLen) {
andre@0: 	if (inputLen % RC2_BLOCK_SIZE) {
andre@0: 	    PORT_SetError(SEC_ERROR_INPUT_LEN);
andre@0: 	    return SECFailure;
andre@0: 	}
andre@0: 	if (maxOutputLen < inputLen) {
andre@0: 	    PORT_SetError(SEC_ERROR_OUTPUT_LEN);
andre@0: 	    return SECFailure;
andre@0: 	}
andre@0: 	rv = (*cx->enc)(cx, output, input, inputLen);
andre@0:     }
andre@0:     if (rv == SECSuccess) {
andre@0:     	*outputLen = inputLen;
andre@0:     }
andre@0:     return rv;
andre@0: }
andre@0: 
andre@0: /*
andre@0: ** Perform RC2 decryption.
andre@0: **	"cx" the context
andre@0: **	"output" the output buffer to store the decrypted data.
andre@0: **	"outputLen" how much data is stored in "output". Set by the routine
andre@0: **	   after some data is stored in output.
andre@0: **	"maxOutputLen" the maximum amount of data that can ever be
andre@0: **	   stored in "output"
andre@0: **	"input" the input data
andre@0: **	"inputLen" the amount of input data
andre@0: */
andre@0: SECStatus RC2_Decrypt(RC2Context *cx, unsigned char *output,
andre@0: 		      unsigned int *outputLen, unsigned int maxOutputLen,
andre@0: 		      const unsigned char *input, unsigned int inputLen)
andre@0: {
andre@0:     SECStatus rv = SECSuccess;
andre@0:     if (inputLen) {
andre@0: 	if (inputLen % RC2_BLOCK_SIZE) {
andre@0: 	    PORT_SetError(SEC_ERROR_INPUT_LEN);
andre@0: 	    return SECFailure;
andre@0: 	}
andre@0: 	if (maxOutputLen < inputLen) {
andre@0: 	    PORT_SetError(SEC_ERROR_OUTPUT_LEN);
andre@0: 	    return SECFailure;
andre@0: 	}
andre@0: 	rv = (*cx->dec)(cx, output, input, inputLen);
andre@0:     }
andre@0:     if (rv == SECSuccess) {
andre@0: 	*outputLen = inputLen;
andre@0:     }
andre@0:     return rv;
andre@0: }
andre@0: