andre@0: /* This Source Code Form is subject to the terms of the Mozilla Public
andre@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
andre@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
andre@0: 
andre@0: #include "seccomon.h"
andre@0: #include "secport.h"
andre@0: 
andre@0: #ifdef TEST_UTF8
andre@0: #include <assert.h>
andre@0: #undef PORT_Assert
andre@0: #define PORT_Assert assert
andre@0: #endif
andre@0: 
andre@0: /*
andre@0:  * From RFC 2044:
andre@0:  *
andre@0:  * UCS-4 range (hex.)           UTF-8 octet sequence (binary)
andre@0:  * 0000 0000-0000 007F   0xxxxxxx
andre@0:  * 0000 0080-0000 07FF   110xxxxx 10xxxxxx
andre@0:  * 0000 0800-0000 FFFF   1110xxxx 10xxxxxx 10xxxxxx
andre@0:  * 0001 0000-001F FFFF   11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
andre@0:  * 0020 0000-03FF FFFF   111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
andre@0:  * 0400 0000-7FFF FFFF   1111110x 10xxxxxx ... 10xxxxxx
andre@0:  */  
andre@0: 
andre@0: /*
andre@0:  * From http://www.imc.org/draft-hoffman-utf16
andre@0:  *
andre@0:  * For U on [0x00010000,0x0010FFFF]:  Let U' = U - 0x00010000
andre@0:  *
andre@0:  * U' = yyyyyyyyyyxxxxxxxxxx
andre@0:  * W1 = 110110yyyyyyyyyy
andre@0:  * W2 = 110111xxxxxxxxxx
andre@0:  */
andre@0: 
andre@0: /*
andre@0:  * This code is assuming NETWORK BYTE ORDER for the 16- and 32-bit
andre@0:  * character values.  If you wish to use this code for working with
andre@0:  * host byte order values, define the following:
andre@0:  *
andre@0:  * #if IS_BIG_ENDIAN
andre@0:  * #define L_0 0
andre@0:  * #define L_1 1
andre@0:  * #define L_2 2
andre@0:  * #define L_3 3
andre@0:  * #define H_0 0
andre@0:  * #define H_1 1
andre@0:  * #else / * not everyone has elif * /
andre@0:  * #if IS_LITTLE_ENDIAN
andre@0:  * #define L_0 3
andre@0:  * #define L_1 2
andre@0:  * #define L_2 1
andre@0:  * #define L_3 0
andre@0:  * #define H_0 1
andre@0:  * #define H_1 0
andre@0:  * #else
andre@0:  * #error "PDP and NUXI support deferred"
andre@0:  * #endif / * IS_LITTLE_ENDIAN * /
andre@0:  * #endif / * IS_BIG_ENDIAN * /
andre@0:  */
andre@0: 
andre@0: #define L_0 0
andre@0: #define L_1 1
andre@0: #define L_2 2
andre@0: #define L_3 3
andre@0: #define H_0 0
andre@0: #define H_1 1
andre@0: 
andre@0: #define BAD_UTF8 ((PRUint32)-1)
andre@0: 
andre@0: /*
andre@0:  * Parse a single UTF-8 character per the spec. in section 3.9 (D36)
andre@0:  * of Unicode 4.0.0.
andre@0:  *
andre@0:  * Parameters:
andre@0:  * index - Points to the byte offset in inBuf of character to read.  On success,
andre@0:  *         updated to the offset of the following character.
andre@0:  * inBuf - Input buffer, UTF-8 encoded
andre@0:  * inbufLen - Length of input buffer, in bytes.
andre@0:  *
andre@0:  * Returns:
andre@0:  * Success - The UCS4 encoded character
andre@0:  * Failure - BAD_UTF8
andre@0:  */
andre@0: static PRUint32
andre@0: sec_port_read_utf8(unsigned int *index, unsigned char *inBuf, unsigned int inBufLen)
andre@0: {
andre@0:   PRUint32 result;
andre@0:   unsigned int i = *index;
andre@0:   int bytes_left;
andre@0:   PRUint32 min_value;
andre@0: 
andre@0:   PORT_Assert(i < inBufLen);
andre@0: 
andre@0:   if ( (inBuf[i] & 0x80) == 0x00 ) {
andre@0:     result = inBuf[i++];
andre@0:     bytes_left = 0;
andre@0:     min_value = 0;
andre@0:   } else if ( (inBuf[i] & 0xE0) == 0xC0 ) {
andre@0:     result = inBuf[i++] & 0x1F;
andre@0:     bytes_left = 1;
andre@0:     min_value = 0x80;
andre@0:   } else if ( (inBuf[i] & 0xF0) == 0xE0) {
andre@0:     result = inBuf[i++] & 0x0F;
andre@0:     bytes_left = 2;
andre@0:     min_value = 0x800;
andre@0:   } else if ( (inBuf[i] & 0xF8) == 0xF0) {
andre@0:     result = inBuf[i++] & 0x07;
andre@0:     bytes_left = 3;
andre@0:     min_value = 0x10000;
andre@0:   } else {
andre@0:     return BAD_UTF8;
andre@0:   }
andre@0: 
andre@0:   while (bytes_left--) {
andre@0:     if (i >= inBufLen || (inBuf[i] & 0xC0) != 0x80) return BAD_UTF8;
andre@0:     result = (result << 6) | (inBuf[i++] & 0x3F);
andre@0:   }
andre@0: 
andre@0:   /* Check for overlong sequences, surrogates, and outside unicode range */
andre@0:   if (result < min_value || (result & 0xFFFFF800) == 0xD800 || result > 0x10FFFF) {
andre@0:     return BAD_UTF8;
andre@0:   }
andre@0: 
andre@0:   *index = i;
andre@0:   return result;
andre@0: }
andre@0: 
andre@0: PRBool
andre@0: sec_port_ucs4_utf8_conversion_function
andre@0: (
andre@0:   PRBool toUnicode,
andre@0:   unsigned char *inBuf,
andre@0:   unsigned int inBufLen,
andre@0:   unsigned char *outBuf,
andre@0:   unsigned int maxOutBufLen,
andre@0:   unsigned int *outBufLen
andre@0: )
andre@0: {
andre@0:   PORT_Assert((unsigned int *)NULL != outBufLen);
andre@0: 
andre@0:   if( toUnicode ) {
andre@0:     unsigned int i, len = 0;
andre@0: 
andre@0:     for( i = 0; i < inBufLen; ) {
andre@0:       if( (inBuf[i] & 0x80) == 0x00 ) i += 1;
andre@0:       else if( (inBuf[i] & 0xE0) == 0xC0 ) i += 2;
andre@0:       else if( (inBuf[i] & 0xF0) == 0xE0 ) i += 3;
andre@0:       else if( (inBuf[i] & 0xF8) == 0xF0 ) i += 4;
andre@0:       else return PR_FALSE;
andre@0: 
andre@0:       len += 4;
andre@0:     }
andre@0: 
andre@0:     if( len > maxOutBufLen ) {
andre@0:       *outBufLen = len;
andre@0:       return PR_FALSE;
andre@0:     }
andre@0: 
andre@0:     len = 0;
andre@0: 
andre@0:     for( i = 0; i < inBufLen; ) {
andre@0:       PRUint32 ucs4 = sec_port_read_utf8(&i, inBuf, inBufLen);
andre@0: 
andre@0:       if (ucs4 == BAD_UTF8) return PR_FALSE;
andre@0:            
andre@0:       outBuf[len+L_0] = 0x00;
andre@0:       outBuf[len+L_1] = (unsigned char)(ucs4 >> 16);
andre@0:       outBuf[len+L_2] = (unsigned char)(ucs4 >> 8);
andre@0:       outBuf[len+L_3] = (unsigned char)ucs4;
andre@0: 
andre@0:       len += 4;
andre@0:     }
andre@0: 
andre@0:     *outBufLen = len;
andre@0:     return PR_TRUE;
andre@0:   } else {
andre@0:     unsigned int i, len = 0;
andre@0:     PORT_Assert((inBufLen % 4) == 0);
andre@0:     if ((inBufLen % 4) != 0) {
andre@0:       *outBufLen = 0;
andre@0:       return PR_FALSE;
andre@0:     }
andre@0: 
andre@0:     for( i = 0; i < inBufLen; i += 4 ) {
andre@0:       if( (inBuf[i+L_0] > 0x00) || (inBuf[i+L_1] > 0x10) ) {
andre@0: 	*outBufLen = 0;
andre@0: 	return PR_FALSE;
andre@0:       } else if( inBuf[i+L_1] >= 0x01 ) len += 4;
andre@0:       else if( inBuf[i+L_2] >= 0x08 ) len += 3;
andre@0:       else if( (inBuf[i+L_2] > 0x00) || (inBuf[i+L_3] >= 0x80) ) len += 2;
andre@0:       else len += 1;
andre@0:     }
andre@0: 
andre@0:     if( len > maxOutBufLen ) {
andre@0:       *outBufLen = len;
andre@0:       return PR_FALSE;
andre@0:     }
andre@0: 
andre@0:     len = 0;
andre@0: 
andre@0:     for( i = 0; i < inBufLen; i += 4 ) {
andre@0:       if( inBuf[i+L_1] >= 0x01 ) {
andre@0:         /* 0001 0000-001F FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
andre@0:         /* 00000000 000abcde fghijklm nopqrstu ->
andre@0:            11110abc 10defghi 10jklmno 10pqrstu */
andre@0: 
andre@0:         outBuf[len+0] = 0xF0 | ((inBuf[i+L_1] & 0x1C) >> 2);
andre@0:         outBuf[len+1] = 0x80 | ((inBuf[i+L_1] & 0x03) << 4)
andre@0:                              | ((inBuf[i+L_2] & 0xF0) >> 4);
andre@0:         outBuf[len+2] = 0x80 | ((inBuf[i+L_2] & 0x0F) << 2)
andre@0:                              | ((inBuf[i+L_3] & 0xC0) >> 6);
andre@0:         outBuf[len+3] = 0x80 | ((inBuf[i+L_3] & 0x3F) >> 0);
andre@0: 
andre@0:         len += 4;
andre@0:       } else if( inBuf[i+L_2] >= 0x08 ) {
andre@0:         /* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */
andre@0:         /* 00000000 00000000 abcdefgh ijklmnop ->
andre@0:            1110abcd 10efghij 10klmnop */
andre@0: 
andre@0:         outBuf[len+0] = 0xE0 | ((inBuf[i+L_2] & 0xF0) >> 4);
andre@0:         outBuf[len+1] = 0x80 | ((inBuf[i+L_2] & 0x0F) << 2)
andre@0:                              | ((inBuf[i+L_3] & 0xC0) >> 6);
andre@0:         outBuf[len+2] = 0x80 | ((inBuf[i+L_3] & 0x3F) >> 0);
andre@0: 
andre@0:         len += 3;
andre@0:       } else if( (inBuf[i+L_2] > 0x00) || (inBuf[i+L_3] >= 0x80) ) {
andre@0:         /* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */
andre@0:         /* 00000000 00000000 00000abc defghijk ->
andre@0:            110abcde 10fghijk */
andre@0: 
andre@0:         outBuf[len+0] = 0xC0 | ((inBuf[i+L_2] & 0x07) << 2)
andre@0:                              | ((inBuf[i+L_3] & 0xC0) >> 6);
andre@0:         outBuf[len+1] = 0x80 | ((inBuf[i+L_3] & 0x3F) >> 0);
andre@0: 
andre@0:         len += 2;
andre@0:       } else {
andre@0:         /* 0000 0000-0000 007F -> 0xxxxxx */
andre@0:         /* 00000000 00000000 00000000 0abcdefg ->
andre@0:            0abcdefg */
andre@0: 
andre@0:         outBuf[len+0] = (inBuf[i+L_3] & 0x7F);
andre@0: 
andre@0:         len += 1;
andre@0:       }
andre@0:     }
andre@0:                             
andre@0:     *outBufLen = len;
andre@0:     return PR_TRUE;
andre@0:   }
andre@0: }
andre@0: 
andre@0: PRBool
andre@0: sec_port_ucs2_utf8_conversion_function
andre@0: (
andre@0:   PRBool toUnicode,
andre@0:   unsigned char *inBuf,
andre@0:   unsigned int inBufLen,
andre@0:   unsigned char *outBuf,
andre@0:   unsigned int maxOutBufLen,
andre@0:   unsigned int *outBufLen
andre@0: )
andre@0: {
andre@0:   PORT_Assert((unsigned int *)NULL != outBufLen);
andre@0: 
andre@0:   if( toUnicode ) {
andre@0:     unsigned int i, len = 0;
andre@0: 
andre@0:     for( i = 0; i < inBufLen; ) {
andre@0:       if( (inBuf[i] & 0x80) == 0x00 ) {
andre@0:         i += 1;
andre@0:         len += 2;
andre@0:       } else if( (inBuf[i] & 0xE0) == 0xC0 ) {
andre@0:         i += 2;
andre@0:         len += 2;
andre@0:       } else if( (inBuf[i] & 0xF0) == 0xE0 ) {
andre@0:         i += 3;
andre@0:         len += 2;
andre@0:       } else if( (inBuf[i] & 0xF8) == 0xF0 ) { 
andre@0:         i += 4;
andre@0:         len += 4;
andre@0:       } else return PR_FALSE;
andre@0:     }
andre@0: 
andre@0:     if( len > maxOutBufLen ) {
andre@0:       *outBufLen = len;
andre@0:       return PR_FALSE;
andre@0:     }
andre@0: 
andre@0:     len = 0;
andre@0: 
andre@0:     for( i = 0; i < inBufLen; ) {
andre@0:       PRUint32 ucs4 = sec_port_read_utf8(&i, inBuf, inBufLen);
andre@0: 
andre@0:       if (ucs4 == BAD_UTF8) return PR_FALSE;
andre@0: 
andre@0:       if( ucs4 < 0x10000) {
andre@0:         outBuf[len+H_0] = (unsigned char)(ucs4 >> 8);
andre@0:         outBuf[len+H_1] = (unsigned char)ucs4;
andre@0:         len += 2;
andre@0:       } else {
andre@0: 	ucs4 -= 0x10000;
andre@0:         outBuf[len+0+H_0] = (unsigned char)(0xD8 | ((ucs4 >> 18) & 0x3));
andre@0:         outBuf[len+0+H_1] = (unsigned char)(ucs4 >> 10);
andre@0:         outBuf[len+2+H_0] = (unsigned char)(0xDC | ((ucs4 >> 8) & 0x3));
andre@0:         outBuf[len+2+H_1] = (unsigned char)ucs4;
andre@0: 	len += 4;
andre@0:       }
andre@0:     }
andre@0: 
andre@0:     *outBufLen = len;
andre@0:     return PR_TRUE;
andre@0:   } else {
andre@0:     unsigned int i, len = 0;
andre@0:     PORT_Assert((inBufLen % 2) == 0);
andre@0:     if ((inBufLen % 2) != 0) {
andre@0:       *outBufLen = 0;
andre@0:       return PR_FALSE;
andre@0:     }
andre@0: 
andre@0:     for( i = 0; i < inBufLen; i += 2 ) {
andre@0:       if( (inBuf[i+H_0] == 0x00) && ((inBuf[i+H_0] & 0x80) == 0x00) ) len += 1;
andre@0:       else if( inBuf[i+H_0] < 0x08 ) len += 2;
andre@0:       else if( ((inBuf[i+0+H_0] & 0xDC) == 0xD8) ) {
andre@0:         if( ((inBuf[i+2+H_0] & 0xDC) == 0xDC) && ((inBufLen - i) > 2) ) {
andre@0:           i += 2;
andre@0:           len += 4;
andre@0:         } else {
andre@0:           return PR_FALSE;
andre@0:         }
andre@0:       }
andre@0:       else len += 3;
andre@0:     }
andre@0: 
andre@0:     if( len > maxOutBufLen ) {
andre@0:       *outBufLen = len;
andre@0:       return PR_FALSE;
andre@0:     }
andre@0: 
andre@0:     len = 0;
andre@0: 
andre@0:     for( i = 0; i < inBufLen; i += 2 ) {
andre@0:       if( (inBuf[i+H_0] == 0x00) && ((inBuf[i+H_1] & 0x80) == 0x00) ) {
andre@0:         /* 0000-007F -> 0xxxxxx */
andre@0:         /* 00000000 0abcdefg -> 0abcdefg */
andre@0: 
andre@0:         outBuf[len] = inBuf[i+H_1] & 0x7F;
andre@0: 
andre@0:         len += 1;
andre@0:       } else if( inBuf[i+H_0] < 0x08 ) {
andre@0:         /* 0080-07FF -> 110xxxxx 10xxxxxx */
andre@0:         /* 00000abc defghijk -> 110abcde 10fghijk */
andre@0: 
andre@0:         outBuf[len+0] = 0xC0 | ((inBuf[i+H_0] & 0x07) << 2) 
andre@0:                              | ((inBuf[i+H_1] & 0xC0) >> 6);
andre@0:         outBuf[len+1] = 0x80 | ((inBuf[i+H_1] & 0x3F) >> 0);
andre@0: 
andre@0:         len += 2;
andre@0:       } else if( (inBuf[i+H_0] & 0xDC) == 0xD8 ) {
andre@0:         int abcde, BCDE;
andre@0: 
andre@0:         PORT_Assert(((inBuf[i+2+H_0] & 0xDC) == 0xDC) && ((inBufLen - i) > 2));
andre@0: 
andre@0:         /* D800-DBFF DC00-DFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
andre@0:         /* 110110BC DEfghijk 110111lm nopqrstu ->
andre@0:            { Let abcde = BCDE + 1 }
andre@0:            11110abc 10defghi 10jklmno 10pqrstu */
andre@0: 
andre@0:         BCDE = ((inBuf[i+H_0] & 0x03) << 2) | ((inBuf[i+H_1] & 0xC0) >> 6);
andre@0:         abcde = BCDE + 1;
andre@0: 
andre@0:         outBuf[len+0] = 0xF0 | ((abcde & 0x1C) >> 2);
andre@0:         outBuf[len+1] = 0x80 | ((abcde & 0x03) << 4) 
andre@0:                              | ((inBuf[i+0+H_1] & 0x3C) >> 2);
andre@0:         outBuf[len+2] = 0x80 | ((inBuf[i+0+H_1] & 0x03) << 4)
andre@0:                              | ((inBuf[i+2+H_0] & 0x03) << 2)
andre@0:                              | ((inBuf[i+2+H_1] & 0xC0) >> 6);
andre@0:         outBuf[len+3] = 0x80 | ((inBuf[i+2+H_1] & 0x3F) >> 0);
andre@0: 
andre@0:         i += 2;
andre@0:         len += 4;
andre@0:       } else {
andre@0:         /* 0800-FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */
andre@0:         /* abcdefgh ijklmnop -> 1110abcd 10efghij 10klmnop */
andre@0: 
andre@0:         outBuf[len+0] = 0xE0 | ((inBuf[i+H_0] & 0xF0) >> 4);
andre@0:         outBuf[len+1] = 0x80 | ((inBuf[i+H_0] & 0x0F) << 2) 
andre@0:                              | ((inBuf[i+H_1] & 0xC0) >> 6);
andre@0:         outBuf[len+2] = 0x80 | ((inBuf[i+H_1] & 0x3F) >> 0);
andre@0: 
andre@0:         len += 3;
andre@0:       }
andre@0:     }
andre@0: 
andre@0:     *outBufLen = len;
andre@0:     return PR_TRUE;
andre@0:   }
andre@0: }
andre@0: 
andre@0: PRBool
andre@0: sec_port_iso88591_utf8_conversion_function
andre@0: (
andre@0:   const unsigned char *inBuf,
andre@0:   unsigned int inBufLen,
andre@0:   unsigned char *outBuf,
andre@0:   unsigned int maxOutBufLen,
andre@0:   unsigned int *outBufLen
andre@0: )
andre@0: {
andre@0:   unsigned int i, len = 0;
andre@0: 
andre@0:   PORT_Assert((unsigned int *)NULL != outBufLen);
andre@0: 
andre@0:   for( i = 0; i < inBufLen; i++) {
andre@0:     if( (inBuf[i] & 0x80) == 0x00 ) len += 1;
andre@0:     else len += 2;
andre@0:   }
andre@0: 
andre@0:   if( len > maxOutBufLen ) {
andre@0:     *outBufLen = len;
andre@0:     return PR_FALSE;
andre@0:   }
andre@0: 
andre@0:   len = 0;
andre@0: 
andre@0:   for( i = 0; i < inBufLen; i++) {
andre@0:     if( (inBuf[i] & 0x80) == 0x00 ) {
andre@0:       /* 00-7F -> 0xxxxxxx */
andre@0:       /* 0abcdefg -> 0abcdefg */
andre@0: 
andre@0:       outBuf[len] = inBuf[i];
andre@0:       len += 1;
andre@0:     } else {
andre@0:       /* 80-FF <- 110xxxxx 10xxxxxx */
andre@0:       /* 00000000 abcdefgh -> 110000ab 10cdefgh */
andre@0: 
andre@0:       outBuf[len+0] = 0xC0 | ((inBuf[i] & 0xC0) >> 6);
andre@0:       outBuf[len+1] = 0x80 | ((inBuf[i] & 0x3F) >> 0);
andre@0: 
andre@0:       len += 2;
andre@0:     }
andre@0:   }
andre@0: 
andre@0:   *outBufLen = len;
andre@0:   return PR_TRUE;
andre@0: }
andre@0: 
andre@0: #ifdef TEST_UTF8
andre@0: 
andre@0: #include <stdio.h>
andre@0: #include <string.h>
andre@0: #include <stdlib.h>
andre@0: #include <netinet/in.h> /* for htonl and htons */
andre@0: 
andre@0: /*
andre@0:  * UCS-4 vectors
andre@0:  */
andre@0: 
andre@0: struct ucs4 {
andre@0:   PRUint32 c;
andre@0:   char *utf8;
andre@0: };
andre@0: 
andre@0: /*
andre@0:  * UCS-2 vectors
andre@0:  */
andre@0: 
andre@0: struct ucs2 {
andre@0:   PRUint16 c;
andre@0:   char *utf8;
andre@0: };
andre@0: 
andre@0: /*
andre@0:  * UTF-16 vectors
andre@0:  */
andre@0: 
andre@0: struct utf16 {
andre@0:   PRUint32 c;
andre@0:   PRUint16 w[2];
andre@0: };
andre@0: 
andre@0: 
andre@0: /*
andre@0:  * UCS-4 vectors
andre@0:  */
andre@0: 
andre@0: struct ucs4 ucs4[] = {
andre@0:   { 0x00000001, "\x01" },
andre@0:   { 0x00000002, "\x02" },
andre@0:   { 0x00000003, "\x03" },
andre@0:   { 0x00000004, "\x04" },
andre@0:   { 0x00000007, "\x07" },
andre@0:   { 0x00000008, "\x08" },
andre@0:   { 0x0000000F, "\x0F" },
andre@0:   { 0x00000010, "\x10" },
andre@0:   { 0x0000001F, "\x1F" },
andre@0:   { 0x00000020, "\x20" },
andre@0:   { 0x0000003F, "\x3F" },
andre@0:   { 0x00000040, "\x40" },
andre@0:   { 0x0000007F, "\x7F" },
andre@0:           
andre@0:   { 0x00000080, "\xC2\x80" },
andre@0:   { 0x00000081, "\xC2\x81" },
andre@0:   { 0x00000082, "\xC2\x82" },
andre@0:   { 0x00000084, "\xC2\x84" },
andre@0:   { 0x00000088, "\xC2\x88" },
andre@0:   { 0x00000090, "\xC2\x90" },
andre@0:   { 0x000000A0, "\xC2\xA0" },
andre@0:   { 0x000000C0, "\xC3\x80" },
andre@0:   { 0x000000FF, "\xC3\xBF" },
andre@0:   { 0x00000100, "\xC4\x80" },
andre@0:   { 0x00000101, "\xC4\x81" },
andre@0:   { 0x00000102, "\xC4\x82" },
andre@0:   { 0x00000104, "\xC4\x84" },
andre@0:   { 0x00000108, "\xC4\x88" },
andre@0:   { 0x00000110, "\xC4\x90" },
andre@0:   { 0x00000120, "\xC4\xA0" },
andre@0:   { 0x00000140, "\xC5\x80" },
andre@0:   { 0x00000180, "\xC6\x80" },
andre@0:   { 0x000001FF, "\xC7\xBF" },
andre@0:   { 0x00000200, "\xC8\x80" },
andre@0:   { 0x00000201, "\xC8\x81" },
andre@0:   { 0x00000202, "\xC8\x82" },
andre@0:   { 0x00000204, "\xC8\x84" },
andre@0:   { 0x00000208, "\xC8\x88" },
andre@0:   { 0x00000210, "\xC8\x90" },
andre@0:   { 0x00000220, "\xC8\xA0" },
andre@0:   { 0x00000240, "\xC9\x80" },
andre@0:   { 0x00000280, "\xCA\x80" },
andre@0:   { 0x00000300, "\xCC\x80" },
andre@0:   { 0x000003FF, "\xCF\xBF" },
andre@0:   { 0x00000400, "\xD0\x80" },
andre@0:   { 0x00000401, "\xD0\x81" },
andre@0:   { 0x00000402, "\xD0\x82" },
andre@0:   { 0x00000404, "\xD0\x84" },
andre@0:   { 0x00000408, "\xD0\x88" },
andre@0:   { 0x00000410, "\xD0\x90" },
andre@0:   { 0x00000420, "\xD0\xA0" },
andre@0:   { 0x00000440, "\xD1\x80" },
andre@0:   { 0x00000480, "\xD2\x80" },
andre@0:   { 0x00000500, "\xD4\x80" },
andre@0:   { 0x00000600, "\xD8\x80" },
andre@0:   { 0x000007FF, "\xDF\xBF" },
andre@0:           
andre@0:   { 0x00000800, "\xE0\xA0\x80" },
andre@0:   { 0x00000801, "\xE0\xA0\x81" },
andre@0:   { 0x00000802, "\xE0\xA0\x82" },
andre@0:   { 0x00000804, "\xE0\xA0\x84" },
andre@0:   { 0x00000808, "\xE0\xA0\x88" },
andre@0:   { 0x00000810, "\xE0\xA0\x90" },
andre@0:   { 0x00000820, "\xE0\xA0\xA0" },
andre@0:   { 0x00000840, "\xE0\xA1\x80" },
andre@0:   { 0x00000880, "\xE0\xA2\x80" },
andre@0:   { 0x00000900, "\xE0\xA4\x80" },
andre@0:   { 0x00000A00, "\xE0\xA8\x80" },
andre@0:   { 0x00000C00, "\xE0\xB0\x80" },
andre@0:   { 0x00000FFF, "\xE0\xBF\xBF" },
andre@0:   { 0x00001000, "\xE1\x80\x80" },
andre@0:   { 0x00001001, "\xE1\x80\x81" },
andre@0:   { 0x00001002, "\xE1\x80\x82" },
andre@0:   { 0x00001004, "\xE1\x80\x84" },
andre@0:   { 0x00001008, "\xE1\x80\x88" },
andre@0:   { 0x00001010, "\xE1\x80\x90" },
andre@0:   { 0x00001020, "\xE1\x80\xA0" },
andre@0:   { 0x00001040, "\xE1\x81\x80" },
andre@0:   { 0x00001080, "\xE1\x82\x80" },
andre@0:   { 0x00001100, "\xE1\x84\x80" },
andre@0:   { 0x00001200, "\xE1\x88\x80" },
andre@0:   { 0x00001400, "\xE1\x90\x80" },
andre@0:   { 0x00001800, "\xE1\xA0\x80" },
andre@0:   { 0x00001FFF, "\xE1\xBF\xBF" },
andre@0:   { 0x00002000, "\xE2\x80\x80" },
andre@0:   { 0x00002001, "\xE2\x80\x81" },
andre@0:   { 0x00002002, "\xE2\x80\x82" },
andre@0:   { 0x00002004, "\xE2\x80\x84" },
andre@0:   { 0x00002008, "\xE2\x80\x88" },
andre@0:   { 0x00002010, "\xE2\x80\x90" },
andre@0:   { 0x00002020, "\xE2\x80\xA0" },
andre@0:   { 0x00002040, "\xE2\x81\x80" },
andre@0:   { 0x00002080, "\xE2\x82\x80" },
andre@0:   { 0x00002100, "\xE2\x84\x80" },
andre@0:   { 0x00002200, "\xE2\x88\x80" },
andre@0:   { 0x00002400, "\xE2\x90\x80" },
andre@0:   { 0x00002800, "\xE2\xA0\x80" },
andre@0:   { 0x00003000, "\xE3\x80\x80" },
andre@0:   { 0x00003FFF, "\xE3\xBF\xBF" },
andre@0:   { 0x00004000, "\xE4\x80\x80" },
andre@0:   { 0x00004001, "\xE4\x80\x81" },
andre@0:   { 0x00004002, "\xE4\x80\x82" },
andre@0:   { 0x00004004, "\xE4\x80\x84" },
andre@0:   { 0x00004008, "\xE4\x80\x88" },
andre@0:   { 0x00004010, "\xE4\x80\x90" },
andre@0:   { 0x00004020, "\xE4\x80\xA0" },
andre@0:   { 0x00004040, "\xE4\x81\x80" },
andre@0:   { 0x00004080, "\xE4\x82\x80" },
andre@0:   { 0x00004100, "\xE4\x84\x80" },
andre@0:   { 0x00004200, "\xE4\x88\x80" },
andre@0:   { 0x00004400, "\xE4\x90\x80" },
andre@0:   { 0x00004800, "\xE4\xA0\x80" },
andre@0:   { 0x00005000, "\xE5\x80\x80" },
andre@0:   { 0x00006000, "\xE6\x80\x80" },
andre@0:   { 0x00007FFF, "\xE7\xBF\xBF" },
andre@0:   { 0x00008000, "\xE8\x80\x80" },
andre@0:   { 0x00008001, "\xE8\x80\x81" },
andre@0:   { 0x00008002, "\xE8\x80\x82" },
andre@0:   { 0x00008004, "\xE8\x80\x84" },
andre@0:   { 0x00008008, "\xE8\x80\x88" },
andre@0:   { 0x00008010, "\xE8\x80\x90" },
andre@0:   { 0x00008020, "\xE8\x80\xA0" },
andre@0:   { 0x00008040, "\xE8\x81\x80" },
andre@0:   { 0x00008080, "\xE8\x82\x80" },
andre@0:   { 0x00008100, "\xE8\x84\x80" },
andre@0:   { 0x00008200, "\xE8\x88\x80" },
andre@0:   { 0x00008400, "\xE8\x90\x80" },
andre@0:   { 0x00008800, "\xE8\xA0\x80" },
andre@0:   { 0x00009000, "\xE9\x80\x80" },
andre@0:   { 0x0000A000, "\xEA\x80\x80" },
andre@0:   { 0x0000C000, "\xEC\x80\x80" },
andre@0:   { 0x0000FFFF, "\xEF\xBF\xBF" },
andre@0:           
andre@0:   { 0x00010000, "\xF0\x90\x80\x80" },
andre@0:   { 0x00010001, "\xF0\x90\x80\x81" },
andre@0:   { 0x00010002, "\xF0\x90\x80\x82" },
andre@0:   { 0x00010004, "\xF0\x90\x80\x84" },
andre@0:   { 0x00010008, "\xF0\x90\x80\x88" },
andre@0:   { 0x00010010, "\xF0\x90\x80\x90" },
andre@0:   { 0x00010020, "\xF0\x90\x80\xA0" },
andre@0:   { 0x00010040, "\xF0\x90\x81\x80" },
andre@0:   { 0x00010080, "\xF0\x90\x82\x80" },
andre@0:   { 0x00010100, "\xF0\x90\x84\x80" },
andre@0:   { 0x00010200, "\xF0\x90\x88\x80" },
andre@0:   { 0x00010400, "\xF0\x90\x90\x80" },
andre@0:   { 0x00010800, "\xF0\x90\xA0\x80" },
andre@0:   { 0x00011000, "\xF0\x91\x80\x80" },
andre@0:   { 0x00012000, "\xF0\x92\x80\x80" },
andre@0:   { 0x00014000, "\xF0\x94\x80\x80" },
andre@0:   { 0x00018000, "\xF0\x98\x80\x80" },
andre@0:   { 0x0001FFFF, "\xF0\x9F\xBF\xBF" },
andre@0:   { 0x00020000, "\xF0\xA0\x80\x80" },
andre@0:   { 0x00020001, "\xF0\xA0\x80\x81" },
andre@0:   { 0x00020002, "\xF0\xA0\x80\x82" },
andre@0:   { 0x00020004, "\xF0\xA0\x80\x84" },
andre@0:   { 0x00020008, "\xF0\xA0\x80\x88" },
andre@0:   { 0x00020010, "\xF0\xA0\x80\x90" },
andre@0:   { 0x00020020, "\xF0\xA0\x80\xA0" },
andre@0:   { 0x00020040, "\xF0\xA0\x81\x80" },
andre@0:   { 0x00020080, "\xF0\xA0\x82\x80" },
andre@0:   { 0x00020100, "\xF0\xA0\x84\x80" },
andre@0:   { 0x00020200, "\xF0\xA0\x88\x80" },
andre@0:   { 0x00020400, "\xF0\xA0\x90\x80" },
andre@0:   { 0x00020800, "\xF0\xA0\xA0\x80" },
andre@0:   { 0x00021000, "\xF0\xA1\x80\x80" },
andre@0:   { 0x00022000, "\xF0\xA2\x80\x80" },
andre@0:   { 0x00024000, "\xF0\xA4\x80\x80" },
andre@0:   { 0x00028000, "\xF0\xA8\x80\x80" },
andre@0:   { 0x00030000, "\xF0\xB0\x80\x80" },
andre@0:   { 0x0003FFFF, "\xF0\xBF\xBF\xBF" },
andre@0:   { 0x00040000, "\xF1\x80\x80\x80" },
andre@0:   { 0x00040001, "\xF1\x80\x80\x81" },
andre@0:   { 0x00040002, "\xF1\x80\x80\x82" },
andre@0:   { 0x00040004, "\xF1\x80\x80\x84" },
andre@0:   { 0x00040008, "\xF1\x80\x80\x88" },
andre@0:   { 0x00040010, "\xF1\x80\x80\x90" },
andre@0:   { 0x00040020, "\xF1\x80\x80\xA0" },
andre@0:   { 0x00040040, "\xF1\x80\x81\x80" },
andre@0:   { 0x00040080, "\xF1\x80\x82\x80" },
andre@0:   { 0x00040100, "\xF1\x80\x84\x80" },
andre@0:   { 0x00040200, "\xF1\x80\x88\x80" },
andre@0:   { 0x00040400, "\xF1\x80\x90\x80" },
andre@0:   { 0x00040800, "\xF1\x80\xA0\x80" },
andre@0:   { 0x00041000, "\xF1\x81\x80\x80" },
andre@0:   { 0x00042000, "\xF1\x82\x80\x80" },
andre@0:   { 0x00044000, "\xF1\x84\x80\x80" },
andre@0:   { 0x00048000, "\xF1\x88\x80\x80" },
andre@0:   { 0x00050000, "\xF1\x90\x80\x80" },
andre@0:   { 0x00060000, "\xF1\xA0\x80\x80" },
andre@0:   { 0x0007FFFF, "\xF1\xBF\xBF\xBF" },
andre@0:   { 0x00080000, "\xF2\x80\x80\x80" },
andre@0:   { 0x00080001, "\xF2\x80\x80\x81" },
andre@0:   { 0x00080002, "\xF2\x80\x80\x82" },
andre@0:   { 0x00080004, "\xF2\x80\x80\x84" },
andre@0:   { 0x00080008, "\xF2\x80\x80\x88" },
andre@0:   { 0x00080010, "\xF2\x80\x80\x90" },
andre@0:   { 0x00080020, "\xF2\x80\x80\xA0" },
andre@0:   { 0x00080040, "\xF2\x80\x81\x80" },
andre@0:   { 0x00080080, "\xF2\x80\x82\x80" },
andre@0:   { 0x00080100, "\xF2\x80\x84\x80" },
andre@0:   { 0x00080200, "\xF2\x80\x88\x80" },
andre@0:   { 0x00080400, "\xF2\x80\x90\x80" },
andre@0:   { 0x00080800, "\xF2\x80\xA0\x80" },
andre@0:   { 0x00081000, "\xF2\x81\x80\x80" },
andre@0:   { 0x00082000, "\xF2\x82\x80\x80" },
andre@0:   { 0x00084000, "\xF2\x84\x80\x80" },
andre@0:   { 0x00088000, "\xF2\x88\x80\x80" },
andre@0:   { 0x00090000, "\xF2\x90\x80\x80" },
andre@0:   { 0x000A0000, "\xF2\xA0\x80\x80" },
andre@0:   { 0x000C0000, "\xF3\x80\x80\x80" },
andre@0:   { 0x000FFFFF, "\xF3\xBF\xBF\xBF" },
andre@0:   { 0x00100000, "\xF4\x80\x80\x80" },
andre@0:   { 0x00100001, "\xF4\x80\x80\x81" },
andre@0:   { 0x00100002, "\xF4\x80\x80\x82" },
andre@0:   { 0x00100004, "\xF4\x80\x80\x84" },
andre@0:   { 0x00100008, "\xF4\x80\x80\x88" },
andre@0:   { 0x00100010, "\xF4\x80\x80\x90" },
andre@0:   { 0x00100020, "\xF4\x80\x80\xA0" },
andre@0:   { 0x00100040, "\xF4\x80\x81\x80" },
andre@0:   { 0x00100080, "\xF4\x80\x82\x80" },
andre@0:   { 0x00100100, "\xF4\x80\x84\x80" },
andre@0:   { 0x00100200, "\xF4\x80\x88\x80" },
andre@0:   { 0x00100400, "\xF4\x80\x90\x80" },
andre@0:   { 0x00100800, "\xF4\x80\xA0\x80" },
andre@0:   { 0x00101000, "\xF4\x81\x80\x80" },
andre@0:   { 0x00102000, "\xF4\x82\x80\x80" },
andre@0:   { 0x00104000, "\xF4\x84\x80\x80" },
andre@0:   { 0x00108000, "\xF4\x88\x80\x80" },
andre@0:   { 0x0010FFFF, "\xF4\x8F\xBF\xBF" },
andre@0: };
andre@0: 
andre@0: /*
andre@0:  * UCS-2 vectors
andre@0:  */
andre@0: 
andre@0: struct ucs2 ucs2[] = {
andre@0:   { 0x0001, "\x01" },
andre@0:   { 0x0002, "\x02" },
andre@0:   { 0x0003, "\x03" },
andre@0:   { 0x0004, "\x04" },
andre@0:   { 0x0007, "\x07" },
andre@0:   { 0x0008, "\x08" },
andre@0:   { 0x000F, "\x0F" },
andre@0:   { 0x0010, "\x10" },
andre@0:   { 0x001F, "\x1F" },
andre@0:   { 0x0020, "\x20" },
andre@0:   { 0x003F, "\x3F" },
andre@0:   { 0x0040, "\x40" },
andre@0:   { 0x007F, "\x7F" },
andre@0:           
andre@0:   { 0x0080, "\xC2\x80" },
andre@0:   { 0x0081, "\xC2\x81" },
andre@0:   { 0x0082, "\xC2\x82" },
andre@0:   { 0x0084, "\xC2\x84" },
andre@0:   { 0x0088, "\xC2\x88" },
andre@0:   { 0x0090, "\xC2\x90" },
andre@0:   { 0x00A0, "\xC2\xA0" },
andre@0:   { 0x00C0, "\xC3\x80" },
andre@0:   { 0x00FF, "\xC3\xBF" },
andre@0:   { 0x0100, "\xC4\x80" },
andre@0:   { 0x0101, "\xC4\x81" },
andre@0:   { 0x0102, "\xC4\x82" },
andre@0:   { 0x0104, "\xC4\x84" },
andre@0:   { 0x0108, "\xC4\x88" },
andre@0:   { 0x0110, "\xC4\x90" },
andre@0:   { 0x0120, "\xC4\xA0" },
andre@0:   { 0x0140, "\xC5\x80" },
andre@0:   { 0x0180, "\xC6\x80" },
andre@0:   { 0x01FF, "\xC7\xBF" },
andre@0:   { 0x0200, "\xC8\x80" },
andre@0:   { 0x0201, "\xC8\x81" },
andre@0:   { 0x0202, "\xC8\x82" },
andre@0:   { 0x0204, "\xC8\x84" },
andre@0:   { 0x0208, "\xC8\x88" },
andre@0:   { 0x0210, "\xC8\x90" },
andre@0:   { 0x0220, "\xC8\xA0" },
andre@0:   { 0x0240, "\xC9\x80" },
andre@0:   { 0x0280, "\xCA\x80" },
andre@0:   { 0x0300, "\xCC\x80" },
andre@0:   { 0x03FF, "\xCF\xBF" },
andre@0:   { 0x0400, "\xD0\x80" },
andre@0:   { 0x0401, "\xD0\x81" },
andre@0:   { 0x0402, "\xD0\x82" },
andre@0:   { 0x0404, "\xD0\x84" },
andre@0:   { 0x0408, "\xD0\x88" },
andre@0:   { 0x0410, "\xD0\x90" },
andre@0:   { 0x0420, "\xD0\xA0" },
andre@0:   { 0x0440, "\xD1\x80" },
andre@0:   { 0x0480, "\xD2\x80" },
andre@0:   { 0x0500, "\xD4\x80" },
andre@0:   { 0x0600, "\xD8\x80" },
andre@0:   { 0x07FF, "\xDF\xBF" },
andre@0:           
andre@0:   { 0x0800, "\xE0\xA0\x80" },
andre@0:   { 0x0801, "\xE0\xA0\x81" },
andre@0:   { 0x0802, "\xE0\xA0\x82" },
andre@0:   { 0x0804, "\xE0\xA0\x84" },
andre@0:   { 0x0808, "\xE0\xA0\x88" },
andre@0:   { 0x0810, "\xE0\xA0\x90" },
andre@0:   { 0x0820, "\xE0\xA0\xA0" },
andre@0:   { 0x0840, "\xE0\xA1\x80" },
andre@0:   { 0x0880, "\xE0\xA2\x80" },
andre@0:   { 0x0900, "\xE0\xA4\x80" },
andre@0:   { 0x0A00, "\xE0\xA8\x80" },
andre@0:   { 0x0C00, "\xE0\xB0\x80" },
andre@0:   { 0x0FFF, "\xE0\xBF\xBF" },
andre@0:   { 0x1000, "\xE1\x80\x80" },
andre@0:   { 0x1001, "\xE1\x80\x81" },
andre@0:   { 0x1002, "\xE1\x80\x82" },
andre@0:   { 0x1004, "\xE1\x80\x84" },
andre@0:   { 0x1008, "\xE1\x80\x88" },
andre@0:   { 0x1010, "\xE1\x80\x90" },
andre@0:   { 0x1020, "\xE1\x80\xA0" },
andre@0:   { 0x1040, "\xE1\x81\x80" },
andre@0:   { 0x1080, "\xE1\x82\x80" },
andre@0:   { 0x1100, "\xE1\x84\x80" },
andre@0:   { 0x1200, "\xE1\x88\x80" },
andre@0:   { 0x1400, "\xE1\x90\x80" },
andre@0:   { 0x1800, "\xE1\xA0\x80" },
andre@0:   { 0x1FFF, "\xE1\xBF\xBF" },
andre@0:   { 0x2000, "\xE2\x80\x80" },
andre@0:   { 0x2001, "\xE2\x80\x81" },
andre@0:   { 0x2002, "\xE2\x80\x82" },
andre@0:   { 0x2004, "\xE2\x80\x84" },
andre@0:   { 0x2008, "\xE2\x80\x88" },
andre@0:   { 0x2010, "\xE2\x80\x90" },
andre@0:   { 0x2020, "\xE2\x80\xA0" },
andre@0:   { 0x2040, "\xE2\x81\x80" },
andre@0:   { 0x2080, "\xE2\x82\x80" },
andre@0:   { 0x2100, "\xE2\x84\x80" },
andre@0:   { 0x2200, "\xE2\x88\x80" },
andre@0:   { 0x2400, "\xE2\x90\x80" },
andre@0:   { 0x2800, "\xE2\xA0\x80" },
andre@0:   { 0x3000, "\xE3\x80\x80" },
andre@0:   { 0x3FFF, "\xE3\xBF\xBF" },
andre@0:   { 0x4000, "\xE4\x80\x80" },
andre@0:   { 0x4001, "\xE4\x80\x81" },
andre@0:   { 0x4002, "\xE4\x80\x82" },
andre@0:   { 0x4004, "\xE4\x80\x84" },
andre@0:   { 0x4008, "\xE4\x80\x88" },
andre@0:   { 0x4010, "\xE4\x80\x90" },
andre@0:   { 0x4020, "\xE4\x80\xA0" },
andre@0:   { 0x4040, "\xE4\x81\x80" },
andre@0:   { 0x4080, "\xE4\x82\x80" },
andre@0:   { 0x4100, "\xE4\x84\x80" },
andre@0:   { 0x4200, "\xE4\x88\x80" },
andre@0:   { 0x4400, "\xE4\x90\x80" },
andre@0:   { 0x4800, "\xE4\xA0\x80" },
andre@0:   { 0x5000, "\xE5\x80\x80" },
andre@0:   { 0x6000, "\xE6\x80\x80" },
andre@0:   { 0x7FFF, "\xE7\xBF\xBF" },
andre@0:   { 0x8000, "\xE8\x80\x80" },
andre@0:   { 0x8001, "\xE8\x80\x81" },
andre@0:   { 0x8002, "\xE8\x80\x82" },
andre@0:   { 0x8004, "\xE8\x80\x84" },
andre@0:   { 0x8008, "\xE8\x80\x88" },
andre@0:   { 0x8010, "\xE8\x80\x90" },
andre@0:   { 0x8020, "\xE8\x80\xA0" },
andre@0:   { 0x8040, "\xE8\x81\x80" },
andre@0:   { 0x8080, "\xE8\x82\x80" },
andre@0:   { 0x8100, "\xE8\x84\x80" },
andre@0:   { 0x8200, "\xE8\x88\x80" },
andre@0:   { 0x8400, "\xE8\x90\x80" },
andre@0:   { 0x8800, "\xE8\xA0\x80" },
andre@0:   { 0x9000, "\xE9\x80\x80" },
andre@0:   { 0xA000, "\xEA\x80\x80" },
andre@0:   { 0xC000, "\xEC\x80\x80" },
andre@0:   { 0xFFFF, "\xEF\xBF\xBF" }
andre@0: 
andre@0: };
andre@0: 
andre@0: /*
andre@0:  * UTF-16 vectors
andre@0:  */
andre@0: 
andre@0: struct utf16 utf16[] = {
andre@0:   { 0x00010000, { 0xD800, 0xDC00 } },
andre@0:   { 0x00010001, { 0xD800, 0xDC01 } },
andre@0:   { 0x00010002, { 0xD800, 0xDC02 } },
andre@0:   { 0x00010003, { 0xD800, 0xDC03 } },
andre@0:   { 0x00010004, { 0xD800, 0xDC04 } },
andre@0:   { 0x00010007, { 0xD800, 0xDC07 } },
andre@0:   { 0x00010008, { 0xD800, 0xDC08 } },
andre@0:   { 0x0001000F, { 0xD800, 0xDC0F } },
andre@0:   { 0x00010010, { 0xD800, 0xDC10 } },
andre@0:   { 0x0001001F, { 0xD800, 0xDC1F } },
andre@0:   { 0x00010020, { 0xD800, 0xDC20 } },
andre@0:   { 0x0001003F, { 0xD800, 0xDC3F } },
andre@0:   { 0x00010040, { 0xD800, 0xDC40 } },
andre@0:   { 0x0001007F, { 0xD800, 0xDC7F } },
andre@0:   { 0x00010080, { 0xD800, 0xDC80 } },
andre@0:   { 0x00010081, { 0xD800, 0xDC81 } },
andre@0:   { 0x00010082, { 0xD800, 0xDC82 } },
andre@0:   { 0x00010084, { 0xD800, 0xDC84 } },
andre@0:   { 0x00010088, { 0xD800, 0xDC88 } },
andre@0:   { 0x00010090, { 0xD800, 0xDC90 } },
andre@0:   { 0x000100A0, { 0xD800, 0xDCA0 } },
andre@0:   { 0x000100C0, { 0xD800, 0xDCC0 } },
andre@0:   { 0x000100FF, { 0xD800, 0xDCFF } },
andre@0:   { 0x00010100, { 0xD800, 0xDD00 } },
andre@0:   { 0x00010101, { 0xD800, 0xDD01 } },
andre@0:   { 0x00010102, { 0xD800, 0xDD02 } },
andre@0:   { 0x00010104, { 0xD800, 0xDD04 } },
andre@0:   { 0x00010108, { 0xD800, 0xDD08 } },
andre@0:   { 0x00010110, { 0xD800, 0xDD10 } },
andre@0:   { 0x00010120, { 0xD800, 0xDD20 } },
andre@0:   { 0x00010140, { 0xD800, 0xDD40 } },
andre@0:   { 0x00010180, { 0xD800, 0xDD80 } },
andre@0:   { 0x000101FF, { 0xD800, 0xDDFF } },
andre@0:   { 0x00010200, { 0xD800, 0xDE00 } },
andre@0:   { 0x00010201, { 0xD800, 0xDE01 } },
andre@0:   { 0x00010202, { 0xD800, 0xDE02 } },
andre@0:   { 0x00010204, { 0xD800, 0xDE04 } },
andre@0:   { 0x00010208, { 0xD800, 0xDE08 } },
andre@0:   { 0x00010210, { 0xD800, 0xDE10 } },
andre@0:   { 0x00010220, { 0xD800, 0xDE20 } },
andre@0:   { 0x00010240, { 0xD800, 0xDE40 } },
andre@0:   { 0x00010280, { 0xD800, 0xDE80 } },
andre@0:   { 0x00010300, { 0xD800, 0xDF00 } },
andre@0:   { 0x000103FF, { 0xD800, 0xDFFF } },
andre@0:   { 0x00010400, { 0xD801, 0xDC00 } },
andre@0:   { 0x00010401, { 0xD801, 0xDC01 } },
andre@0:   { 0x00010402, { 0xD801, 0xDC02 } },
andre@0:   { 0x00010404, { 0xD801, 0xDC04 } },
andre@0:   { 0x00010408, { 0xD801, 0xDC08 } },
andre@0:   { 0x00010410, { 0xD801, 0xDC10 } },
andre@0:   { 0x00010420, { 0xD801, 0xDC20 } },
andre@0:   { 0x00010440, { 0xD801, 0xDC40 } },
andre@0:   { 0x00010480, { 0xD801, 0xDC80 } },
andre@0:   { 0x00010500, { 0xD801, 0xDD00 } },
andre@0:   { 0x00010600, { 0xD801, 0xDE00 } },
andre@0:   { 0x000107FF, { 0xD801, 0xDFFF } },
andre@0:   { 0x00010800, { 0xD802, 0xDC00 } },
andre@0:   { 0x00010801, { 0xD802, 0xDC01 } },
andre@0:   { 0x00010802, { 0xD802, 0xDC02 } },
andre@0:   { 0x00010804, { 0xD802, 0xDC04 } },
andre@0:   { 0x00010808, { 0xD802, 0xDC08 } },
andre@0:   { 0x00010810, { 0xD802, 0xDC10 } },
andre@0:   { 0x00010820, { 0xD802, 0xDC20 } },
andre@0:   { 0x00010840, { 0xD802, 0xDC40 } },
andre@0:   { 0x00010880, { 0xD802, 0xDC80 } },
andre@0:   { 0x00010900, { 0xD802, 0xDD00 } },
andre@0:   { 0x00010A00, { 0xD802, 0xDE00 } },
andre@0:   { 0x00010C00, { 0xD803, 0xDC00 } },
andre@0:   { 0x00010FFF, { 0xD803, 0xDFFF } },
andre@0:   { 0x00011000, { 0xD804, 0xDC00 } },
andre@0:   { 0x00011001, { 0xD804, 0xDC01 } },
andre@0:   { 0x00011002, { 0xD804, 0xDC02 } },
andre@0:   { 0x00011004, { 0xD804, 0xDC04 } },
andre@0:   { 0x00011008, { 0xD804, 0xDC08 } },
andre@0:   { 0x00011010, { 0xD804, 0xDC10 } },
andre@0:   { 0x00011020, { 0xD804, 0xDC20 } },
andre@0:   { 0x00011040, { 0xD804, 0xDC40 } },
andre@0:   { 0x00011080, { 0xD804, 0xDC80 } },
andre@0:   { 0x00011100, { 0xD804, 0xDD00 } },
andre@0:   { 0x00011200, { 0xD804, 0xDE00 } },
andre@0:   { 0x00011400, { 0xD805, 0xDC00 } },
andre@0:   { 0x00011800, { 0xD806, 0xDC00 } },
andre@0:   { 0x00011FFF, { 0xD807, 0xDFFF } },
andre@0:   { 0x00012000, { 0xD808, 0xDC00 } },
andre@0:   { 0x00012001, { 0xD808, 0xDC01 } },
andre@0:   { 0x00012002, { 0xD808, 0xDC02 } },
andre@0:   { 0x00012004, { 0xD808, 0xDC04 } },
andre@0:   { 0x00012008, { 0xD808, 0xDC08 } },
andre@0:   { 0x00012010, { 0xD808, 0xDC10 } },
andre@0:   { 0x00012020, { 0xD808, 0xDC20 } },
andre@0:   { 0x00012040, { 0xD808, 0xDC40 } },
andre@0:   { 0x00012080, { 0xD808, 0xDC80 } },
andre@0:   { 0x00012100, { 0xD808, 0xDD00 } },
andre@0:   { 0x00012200, { 0xD808, 0xDE00 } },
andre@0:   { 0x00012400, { 0xD809, 0xDC00 } },
andre@0:   { 0x00012800, { 0xD80A, 0xDC00 } },
andre@0:   { 0x00013000, { 0xD80C, 0xDC00 } },
andre@0:   { 0x00013FFF, { 0xD80F, 0xDFFF } },
andre@0:   { 0x00014000, { 0xD810, 0xDC00 } },
andre@0:   { 0x00014001, { 0xD810, 0xDC01 } },
andre@0:   { 0x00014002, { 0xD810, 0xDC02 } },
andre@0:   { 0x00014004, { 0xD810, 0xDC04 } },
andre@0:   { 0x00014008, { 0xD810, 0xDC08 } },
andre@0:   { 0x00014010, { 0xD810, 0xDC10 } },
andre@0:   { 0x00014020, { 0xD810, 0xDC20 } },
andre@0:   { 0x00014040, { 0xD810, 0xDC40 } },
andre@0:   { 0x00014080, { 0xD810, 0xDC80 } },
andre@0:   { 0x00014100, { 0xD810, 0xDD00 } },
andre@0:   { 0x00014200, { 0xD810, 0xDE00 } },
andre@0:   { 0x00014400, { 0xD811, 0xDC00 } },
andre@0:   { 0x00014800, { 0xD812, 0xDC00 } },
andre@0:   { 0x00015000, { 0xD814, 0xDC00 } },
andre@0:   { 0x00016000, { 0xD818, 0xDC00 } },
andre@0:   { 0x00017FFF, { 0xD81F, 0xDFFF } },
andre@0:   { 0x00018000, { 0xD820, 0xDC00 } },
andre@0:   { 0x00018001, { 0xD820, 0xDC01 } },
andre@0:   { 0x00018002, { 0xD820, 0xDC02 } },
andre@0:   { 0x00018004, { 0xD820, 0xDC04 } },
andre@0:   { 0x00018008, { 0xD820, 0xDC08 } },
andre@0:   { 0x00018010, { 0xD820, 0xDC10 } },
andre@0:   { 0x00018020, { 0xD820, 0xDC20 } },
andre@0:   { 0x00018040, { 0xD820, 0xDC40 } },
andre@0:   { 0x00018080, { 0xD820, 0xDC80 } },
andre@0:   { 0x00018100, { 0xD820, 0xDD00 } },
andre@0:   { 0x00018200, { 0xD820, 0xDE00 } },
andre@0:   { 0x00018400, { 0xD821, 0xDC00 } },
andre@0:   { 0x00018800, { 0xD822, 0xDC00 } },
andre@0:   { 0x00019000, { 0xD824, 0xDC00 } },
andre@0:   { 0x0001A000, { 0xD828, 0xDC00 } },
andre@0:   { 0x0001C000, { 0xD830, 0xDC00 } },
andre@0:   { 0x0001FFFF, { 0xD83F, 0xDFFF } },
andre@0:   { 0x00020000, { 0xD840, 0xDC00 } },
andre@0:   { 0x00020001, { 0xD840, 0xDC01 } },
andre@0:   { 0x00020002, { 0xD840, 0xDC02 } },
andre@0:   { 0x00020004, { 0xD840, 0xDC04 } },
andre@0:   { 0x00020008, { 0xD840, 0xDC08 } },
andre@0:   { 0x00020010, { 0xD840, 0xDC10 } },
andre@0:   { 0x00020020, { 0xD840, 0xDC20 } },
andre@0:   { 0x00020040, { 0xD840, 0xDC40 } },
andre@0:   { 0x00020080, { 0xD840, 0xDC80 } },
andre@0:   { 0x00020100, { 0xD840, 0xDD00 } },
andre@0:   { 0x00020200, { 0xD840, 0xDE00 } },
andre@0:   { 0x00020400, { 0xD841, 0xDC00 } },
andre@0:   { 0x00020800, { 0xD842, 0xDC00 } },
andre@0:   { 0x00021000, { 0xD844, 0xDC00 } },
andre@0:   { 0x00022000, { 0xD848, 0xDC00 } },
andre@0:   { 0x00024000, { 0xD850, 0xDC00 } },
andre@0:   { 0x00028000, { 0xD860, 0xDC00 } },
andre@0:   { 0x0002FFFF, { 0xD87F, 0xDFFF } },
andre@0:   { 0x00030000, { 0xD880, 0xDC00 } },
andre@0:   { 0x00030001, { 0xD880, 0xDC01 } },
andre@0:   { 0x00030002, { 0xD880, 0xDC02 } },
andre@0:   { 0x00030004, { 0xD880, 0xDC04 } },
andre@0:   { 0x00030008, { 0xD880, 0xDC08 } },
andre@0:   { 0x00030010, { 0xD880, 0xDC10 } },
andre@0:   { 0x00030020, { 0xD880, 0xDC20 } },
andre@0:   { 0x00030040, { 0xD880, 0xDC40 } },
andre@0:   { 0x00030080, { 0xD880, 0xDC80 } },
andre@0:   { 0x00030100, { 0xD880, 0xDD00 } },
andre@0:   { 0x00030200, { 0xD880, 0xDE00 } },
andre@0:   { 0x00030400, { 0xD881, 0xDC00 } },
andre@0:   { 0x00030800, { 0xD882, 0xDC00 } },
andre@0:   { 0x00031000, { 0xD884, 0xDC00 } },
andre@0:   { 0x00032000, { 0xD888, 0xDC00 } },
andre@0:   { 0x00034000, { 0xD890, 0xDC00 } },
andre@0:   { 0x00038000, { 0xD8A0, 0xDC00 } },
andre@0:   { 0x0003FFFF, { 0xD8BF, 0xDFFF } },
andre@0:   { 0x00040000, { 0xD8C0, 0xDC00 } },
andre@0:   { 0x00040001, { 0xD8C0, 0xDC01 } },
andre@0:   { 0x00040002, { 0xD8C0, 0xDC02 } },
andre@0:   { 0x00040004, { 0xD8C0, 0xDC04 } },
andre@0:   { 0x00040008, { 0xD8C0, 0xDC08 } },
andre@0:   { 0x00040010, { 0xD8C0, 0xDC10 } },
andre@0:   { 0x00040020, { 0xD8C0, 0xDC20 } },
andre@0:   { 0x00040040, { 0xD8C0, 0xDC40 } },
andre@0:   { 0x00040080, { 0xD8C0, 0xDC80 } },
andre@0:   { 0x00040100, { 0xD8C0, 0xDD00 } },
andre@0:   { 0x00040200, { 0xD8C0, 0xDE00 } },
andre@0:   { 0x00040400, { 0xD8C1, 0xDC00 } },
andre@0:   { 0x00040800, { 0xD8C2, 0xDC00 } },
andre@0:   { 0x00041000, { 0xD8C4, 0xDC00 } },
andre@0:   { 0x00042000, { 0xD8C8, 0xDC00 } },
andre@0:   { 0x00044000, { 0xD8D0, 0xDC00 } },
andre@0:   { 0x00048000, { 0xD8E0, 0xDC00 } },
andre@0:   { 0x0004FFFF, { 0xD8FF, 0xDFFF } },
andre@0:   { 0x00050000, { 0xD900, 0xDC00 } },
andre@0:   { 0x00050001, { 0xD900, 0xDC01 } },
andre@0:   { 0x00050002, { 0xD900, 0xDC02 } },
andre@0:   { 0x00050004, { 0xD900, 0xDC04 } },
andre@0:   { 0x00050008, { 0xD900, 0xDC08 } },
andre@0:   { 0x00050010, { 0xD900, 0xDC10 } },
andre@0:   { 0x00050020, { 0xD900, 0xDC20 } },
andre@0:   { 0x00050040, { 0xD900, 0xDC40 } },
andre@0:   { 0x00050080, { 0xD900, 0xDC80 } },
andre@0:   { 0x00050100, { 0xD900, 0xDD00 } },
andre@0:   { 0x00050200, { 0xD900, 0xDE00 } },
andre@0:   { 0x00050400, { 0xD901, 0xDC00 } },
andre@0:   { 0x00050800, { 0xD902, 0xDC00 } },
andre@0:   { 0x00051000, { 0xD904, 0xDC00 } },
andre@0:   { 0x00052000, { 0xD908, 0xDC00 } },
andre@0:   { 0x00054000, { 0xD910, 0xDC00 } },
andre@0:   { 0x00058000, { 0xD920, 0xDC00 } },
andre@0:   { 0x00060000, { 0xD940, 0xDC00 } },
andre@0:   { 0x00070000, { 0xD980, 0xDC00 } },
andre@0:   { 0x0007FFFF, { 0xD9BF, 0xDFFF } },
andre@0:   { 0x00080000, { 0xD9C0, 0xDC00 } },
andre@0:   { 0x00080001, { 0xD9C0, 0xDC01 } },
andre@0:   { 0x00080002, { 0xD9C0, 0xDC02 } },
andre@0:   { 0x00080004, { 0xD9C0, 0xDC04 } },
andre@0:   { 0x00080008, { 0xD9C0, 0xDC08 } },
andre@0:   { 0x00080010, { 0xD9C0, 0xDC10 } },
andre@0:   { 0x00080020, { 0xD9C0, 0xDC20 } },
andre@0:   { 0x00080040, { 0xD9C0, 0xDC40 } },
andre@0:   { 0x00080080, { 0xD9C0, 0xDC80 } },
andre@0:   { 0x00080100, { 0xD9C0, 0xDD00 } },
andre@0:   { 0x00080200, { 0xD9C0, 0xDE00 } },
andre@0:   { 0x00080400, { 0xD9C1, 0xDC00 } },
andre@0:   { 0x00080800, { 0xD9C2, 0xDC00 } },
andre@0:   { 0x00081000, { 0xD9C4, 0xDC00 } },
andre@0:   { 0x00082000, { 0xD9C8, 0xDC00 } },
andre@0:   { 0x00084000, { 0xD9D0, 0xDC00 } },
andre@0:   { 0x00088000, { 0xD9E0, 0xDC00 } },
andre@0:   { 0x0008FFFF, { 0xD9FF, 0xDFFF } },
andre@0:   { 0x00090000, { 0xDA00, 0xDC00 } },
andre@0:   { 0x00090001, { 0xDA00, 0xDC01 } },
andre@0:   { 0x00090002, { 0xDA00, 0xDC02 } },
andre@0:   { 0x00090004, { 0xDA00, 0xDC04 } },
andre@0:   { 0x00090008, { 0xDA00, 0xDC08 } },
andre@0:   { 0x00090010, { 0xDA00, 0xDC10 } },
andre@0:   { 0x00090020, { 0xDA00, 0xDC20 } },
andre@0:   { 0x00090040, { 0xDA00, 0xDC40 } },
andre@0:   { 0x00090080, { 0xDA00, 0xDC80 } },
andre@0:   { 0x00090100, { 0xDA00, 0xDD00 } },
andre@0:   { 0x00090200, { 0xDA00, 0xDE00 } },
andre@0:   { 0x00090400, { 0xDA01, 0xDC00 } },
andre@0:   { 0x00090800, { 0xDA02, 0xDC00 } },
andre@0:   { 0x00091000, { 0xDA04, 0xDC00 } },
andre@0:   { 0x00092000, { 0xDA08, 0xDC00 } },
andre@0:   { 0x00094000, { 0xDA10, 0xDC00 } },
andre@0:   { 0x00098000, { 0xDA20, 0xDC00 } },
andre@0:   { 0x000A0000, { 0xDA40, 0xDC00 } },
andre@0:   { 0x000B0000, { 0xDA80, 0xDC00 } },
andre@0:   { 0x000C0000, { 0xDAC0, 0xDC00 } },
andre@0:   { 0x000D0000, { 0xDB00, 0xDC00 } },
andre@0:   { 0x000FFFFF, { 0xDBBF, 0xDFFF } },
andre@0:   { 0x0010FFFF, { 0xDBFF, 0xDFFF } }
andre@0: 
andre@0: };
andre@0: 
andre@0: /* illegal utf8 sequences */
andre@0: char *utf8_bad[] = {
andre@0:   "\xC0\x80",
andre@0:   "\xC1\xBF",
andre@0:   "\xE0\x80\x80",
andre@0:   "\xE0\x9F\xBF",
andre@0:   "\xF0\x80\x80\x80",
andre@0:   "\xF0\x8F\xBF\xBF",
andre@0:   "\xF4\x90\x80\x80",
andre@0:   "\xF7\xBF\xBF\xBF",
andre@0:   "\xF8\x80\x80\x80\x80",
andre@0:   "\xF8\x88\x80\x80\x80",
andre@0:   "\xF8\x92\x80\x80\x80",
andre@0:   "\xF8\x9F\xBF\xBF\xBF",
andre@0:   "\xF8\xA0\x80\x80\x80",
andre@0:   "\xF8\xA8\x80\x80\x80",
andre@0:   "\xF8\xB0\x80\x80\x80",
andre@0:   "\xF8\xBF\xBF\xBF\xBF",
andre@0:   "\xF9\x80\x80\x80\x88",
andre@0:   "\xF9\x84\x80\x80\x80",
andre@0:   "\xF9\xBF\xBF\xBF\xBF",
andre@0:   "\xFA\x80\x80\x80\x80",
andre@0:   "\xFA\x90\x80\x80\x80",
andre@0:   "\xFB\xBF\xBF\xBF\xBF",
andre@0:   "\xFC\x84\x80\x80\x80\x81",
andre@0:   "\xFC\x85\x80\x80\x80\x80",
andre@0:   "\xFC\x86\x80\x80\x80\x80",
andre@0:   "\xFC\x87\xBF\xBF\xBF\xBF",
andre@0:   "\xFC\x88\xA0\x80\x80\x80",
andre@0:   "\xFC\x89\x80\x80\x80\x80",
andre@0:   "\xFC\x8A\x80\x80\x80\x80",
andre@0:   "\xFC\x90\x80\x80\x80\x82",
andre@0:   "\xFD\x80\x80\x80\x80\x80",
andre@0:   "\xFD\xBF\xBF\xBF\xBF\xBF",
andre@0:   "\x80",
andre@0:   "\xC3",
andre@0:   "\xC3\xC3\x80",
andre@0:   "\xED\xA0\x80",
andre@0:   "\xED\xBF\x80",
andre@0:   "\xED\xBF\xBF",
andre@0:   "\xED\xA0\x80\xE0\xBF\xBF",
andre@0: };
andre@0: 
andre@0: static void
andre@0: dump_utf8
andre@0: (
andre@0:   char *word,
andre@0:   unsigned char *utf8,
andre@0:   char *end
andre@0: )
andre@0: {
andre@0:   fprintf(stdout, "%s ", word);
andre@0:   for( ; *utf8; utf8++ ) {
andre@0:     fprintf(stdout, "%02.2x ", (unsigned int)*utf8);
andre@0:   }
andre@0:   fprintf(stdout, "%s", end);
andre@0: }
andre@0: 
andre@0: static PRBool
andre@0: test_ucs4_chars
andre@0: (
andre@0:   void
andre@0: )
andre@0: {
andre@0:   PRBool rv = PR_TRUE;
andre@0:   int i;
andre@0: 
andre@0:   for( i = 0; i < sizeof(ucs4)/sizeof(ucs4[0]); i++ ) {
andre@0:     struct ucs4 *e = &ucs4[i];
andre@0:     PRBool result;
andre@0:     unsigned char utf8[8];
andre@0:     unsigned int len = 0;
andre@0:     PRUint32 back = 0;
andre@0: 
andre@0:     (void)memset(utf8, 0, sizeof(utf8));
andre@0:     
andre@0:     result = sec_port_ucs4_utf8_conversion_function(PR_FALSE, 
andre@0:       (unsigned char *)&e->c, sizeof(e->c), utf8, sizeof(utf8), &len);
andre@0: 
andre@0:     if( !result ) {
andre@0:       fprintf(stdout, "Failed to convert UCS-4 0x%08.8x to UTF-8\n", e->c);
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     if( (len >= sizeof(utf8)) ||
andre@0:         (strlen(e->utf8) != len) ||
andre@0:         (utf8[len] = '\0', 0 != strcmp(e->utf8, utf8)) ) {
andre@0:       fprintf(stdout, "Wrong conversion of UCS-4 0x%08.8x to UTF-8: ", e->c);
andre@0:       dump_utf8("expected", e->utf8, ", ");
andre@0:       dump_utf8("received", utf8, "\n");
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     result = sec_port_ucs4_utf8_conversion_function(PR_TRUE,
andre@0:       utf8, len, (unsigned char *)&back, sizeof(back), &len);
andre@0: 
andre@0:     if( !result ) {
andre@0:       dump_utf8("Failed to convert UTF-8", utf8, "to UCS-4\n");
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     if( (sizeof(back) != len) || (e->c != back) ) {
andre@0:       dump_utf8("Wrong conversion of UTF-8", utf8, " to UCS-4:");
andre@0:       fprintf(stdout, "expected 0x%08.8x, received 0x%08.8x\n", e->c, back);
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0:   }
andre@0: 
andre@0:   return rv;
andre@0: }
andre@0: 
andre@0: static PRBool
andre@0: test_ucs2_chars
andre@0: (
andre@0:   void
andre@0: )
andre@0: {
andre@0:   PRBool rv = PR_TRUE;
andre@0:   int i;
andre@0: 
andre@0:   for( i = 0; i < sizeof(ucs2)/sizeof(ucs2[0]); i++ ) {
andre@0:     struct ucs2 *e = &ucs2[i];
andre@0:     PRBool result;
andre@0:     unsigned char utf8[8];
andre@0:     unsigned int len = 0;
andre@0:     PRUint16 back = 0;
andre@0: 
andre@0:     (void)memset(utf8, 0, sizeof(utf8));
andre@0:     
andre@0:     result = sec_port_ucs2_utf8_conversion_function(PR_FALSE,
andre@0:       (unsigned char *)&e->c, sizeof(e->c), utf8, sizeof(utf8), &len);
andre@0: 
andre@0:     if( !result ) {
andre@0:       fprintf(stdout, "Failed to convert UCS-2 0x%04.4x to UTF-8\n", e->c);
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     if( (len >= sizeof(utf8)) ||
andre@0:         (strlen(e->utf8) != len) ||
andre@0:         (utf8[len] = '\0', 0 != strcmp(e->utf8, utf8)) ) {
andre@0:       fprintf(stdout, "Wrong conversion of UCS-2 0x%04.4x to UTF-8: ", e->c);
andre@0:       dump_utf8("expected", e->utf8, ", ");
andre@0:       dump_utf8("received", utf8, "\n");
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     result = sec_port_ucs2_utf8_conversion_function(PR_TRUE,
andre@0:       utf8, len, (unsigned char *)&back, sizeof(back), &len);
andre@0: 
andre@0:     if( !result ) {
andre@0:       dump_utf8("Failed to convert UTF-8", utf8, "to UCS-2\n");
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     if( (sizeof(back) != len) || (e->c != back) ) {
andre@0:       dump_utf8("Wrong conversion of UTF-8", utf8, "to UCS-2:");
andre@0:       fprintf(stdout, "expected 0x%08.8x, received 0x%08.8x\n", e->c, back);
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0:   }
andre@0: 
andre@0:   return rv;
andre@0: }
andre@0: 
andre@0: static PRBool
andre@0: test_utf16_chars
andre@0: (
andre@0:   void
andre@0: )
andre@0: {
andre@0:   PRBool rv = PR_TRUE;
andre@0:   int i;
andre@0: 
andre@0:   for( i = 0; i < sizeof(utf16)/sizeof(utf16[0]); i++ ) {
andre@0:     struct utf16 *e = &utf16[i];
andre@0:     PRBool result;
andre@0:     unsigned char utf8[8];
andre@0:     unsigned int len = 0;
andre@0:     PRUint32 back32 = 0;
andre@0:     PRUint16 back[2];
andre@0: 
andre@0:     (void)memset(utf8, 0, sizeof(utf8));
andre@0:     
andre@0:     result = sec_port_ucs2_utf8_conversion_function(PR_FALSE, 
andre@0:       (unsigned char *)&e->w[0], sizeof(e->w), utf8, sizeof(utf8), &len);
andre@0: 
andre@0:     if( !result ) {
andre@0:       fprintf(stdout, "Failed to convert UTF-16 0x%04.4x 0x%04.4x to UTF-8\n", 
andre@0:               e->w[0], e->w[1]);
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     result = sec_port_ucs4_utf8_conversion_function(PR_TRUE,
andre@0:       utf8, len, (unsigned char *)&back32, sizeof(back32), &len);
andre@0: 
andre@0:     if( 4 != len ) {
andre@0:       fprintf(stdout, "Failed to convert UTF-16 0x%04.4x 0x%04.4x to UTF-8: "
andre@0:               "unexpected len %d\n", e->w[0], e->w[1], len);
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     utf8[len] = '\0'; /* null-terminate for printing */
andre@0: 
andre@0:     if( !result ) {
andre@0:       dump_utf8("Failed to convert UTF-8", utf8, "to UCS-4 (utf-16 test)\n");
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     if( (sizeof(back32) != len) || (e->c != back32) ) {
andre@0:       fprintf(stdout, "Wrong conversion of UTF-16 0x%04.4x 0x%04.4x ", 
andre@0:               e->w[0], e->w[1]);
andre@0:       dump_utf8("to UTF-8", utf8, "and then to UCS-4: ");
andre@0:       if( sizeof(back32) != len ) {
andre@0:         fprintf(stdout, "len is %d\n", len);
andre@0:       } else {
andre@0:         fprintf(stdout, "expected 0x%08.8x, received 0x%08.8x\n", e->c, back32);
andre@0:       }
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     (void)memset(utf8, 0, sizeof(utf8));
andre@0:     back[0] = back[1] = 0;
andre@0: 
andre@0:     result = sec_port_ucs4_utf8_conversion_function(PR_FALSE,
andre@0:       (unsigned char *)&e->c, sizeof(e->c), utf8, sizeof(utf8), &len);
andre@0: 
andre@0:     if( !result ) {
andre@0:       fprintf(stdout, "Failed to convert UCS-4 0x%08.8x to UTF-8 (utf-16 test)\n",
andre@0:               e->c);
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     result = sec_port_ucs2_utf8_conversion_function(PR_TRUE,
andre@0:       utf8, len, (unsigned char *)&back[0], sizeof(back), &len);
andre@0: 
andre@0:     if( 4 != len ) {
andre@0:       fprintf(stdout, "Failed to convert UCS-4 0x%08.8x to UTF-8: "
andre@0:               "unexpected len %d\n", e->c, len);
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     utf8[len] = '\0'; /* null-terminate for printing */
andre@0: 
andre@0:     if( !result ) {
andre@0:       dump_utf8("Failed to convert UTF-8", utf8, "to UTF-16\n");
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     if( (sizeof(back) != len) || (e->w[0] != back[0]) || (e->w[1] != back[1]) ) {
andre@0:       fprintf(stdout, "Wrong conversion of UCS-4 0x%08.8x to UTF-8", e->c);
andre@0:       dump_utf8("", utf8, "and then to UTF-16:");
andre@0:       if( sizeof(back) != len ) {
andre@0:         fprintf(stdout, "len is %d\n", len);
andre@0:       } else {
andre@0:         fprintf(stdout, "expected 0x%04.4x 0x%04.4x, received 0x%04.4x 0x%04.4xx\n",
andre@0:                 e->w[0], e->w[1], back[0], back[1]);
andre@0:       }
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0:   }
andre@0: 
andre@0:   return rv;
andre@0: }
andre@0: 
andre@0: static PRBool
andre@0: test_utf8_bad_chars
andre@0: (
andre@0:   void
andre@0: )
andre@0: {
andre@0:   PRBool rv = PR_TRUE;
andre@0:   int i;
andre@0: 
andre@0:   for( i = 0; i < sizeof(utf8_bad)/sizeof(utf8_bad[0]); i++ ) {
andre@0:     PRBool result;
andre@0:     unsigned char destbuf[30];
andre@0:     unsigned int len = 0;
andre@0: 
andre@0:     result = sec_port_ucs2_utf8_conversion_function(PR_TRUE,
andre@0:       (unsigned char *)utf8_bad[i], strlen(utf8_bad[i]), destbuf, sizeof(destbuf), &len);
andre@0: 
andre@0:     if( result ) {
andre@0:       dump_utf8("Failed to detect bad UTF-8 string converting to UCS2: ", utf8_bad[i], "\n");
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0:     result = sec_port_ucs4_utf8_conversion_function(PR_TRUE,
andre@0:       (unsigned char *)utf8_bad[i], strlen(utf8_bad[i]), destbuf, sizeof(destbuf), &len);
andre@0: 
andre@0:     if( result ) {
andre@0:       dump_utf8("Failed to detect bad UTF-8 string converting to UCS4: ", utf8_bad[i], "\n");
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:   }
andre@0: 
andre@0:   return rv;
andre@0: }
andre@0: 
andre@0: static PRBool
andre@0: test_iso88591_chars
andre@0: (
andre@0:   void
andre@0: )
andre@0: {
andre@0:   PRBool rv = PR_TRUE;
andre@0:   int i;
andre@0: 
andre@0:   for( i = 0; i < sizeof(ucs2)/sizeof(ucs2[0]); i++ ) {
andre@0:     struct ucs2 *e = &ucs2[i];
andre@0:     PRBool result;
andre@0:     unsigned char iso88591;
andre@0:     unsigned char utf8[3];
andre@0:     unsigned int len = 0;
andre@0: 
andre@0:     if (ntohs(e->c) > 0xFF) continue;
andre@0: 
andre@0:     (void)memset(utf8, 0, sizeof(utf8));
andre@0:     iso88591 = ntohs(e->c);
andre@0:     
andre@0:     result = sec_port_iso88591_utf8_conversion_function(&iso88591,
andre@0:       1, utf8, sizeof(utf8), &len);
andre@0: 
andre@0:     if( !result ) {
andre@0:       fprintf(stdout, "Failed to convert ISO-8859-1 0x%02.2x to UTF-8\n", iso88591);
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:     if( (len >= sizeof(utf8)) ||
andre@0:         (strlen(e->utf8) != len) ||
andre@0:         (utf8[len] = '\0', 0 != strcmp(e->utf8, utf8)) ) {
andre@0:       fprintf(stdout, "Wrong conversion of ISO-8859-1 0x%02.2x to UTF-8: ", iso88591);
andre@0:       dump_utf8("expected", e->utf8, ", ");
andre@0:       dump_utf8("received", utf8, "\n");
andre@0:       rv = PR_FALSE;
andre@0:       continue;
andre@0:     }
andre@0: 
andre@0:   }
andre@0: 
andre@0:   return rv;
andre@0: }
andre@0: 
andre@0: static PRBool
andre@0: test_zeroes
andre@0: (
andre@0:   void
andre@0: )
andre@0: {
andre@0:   PRBool rv = PR_TRUE;
andre@0:   PRBool result;
andre@0:   PRUint32 lzero = 0;
andre@0:   PRUint16 szero = 0;
andre@0:   unsigned char utf8[8];
andre@0:   unsigned int len = 0;
andre@0:   PRUint32 lback = 1;
andre@0:   PRUint16 sback = 1;
andre@0: 
andre@0:   (void)memset(utf8, 1, sizeof(utf8));
andre@0: 
andre@0:   result = sec_port_ucs4_utf8_conversion_function(PR_FALSE, 
andre@0:     (unsigned char *)&lzero, sizeof(lzero), utf8, sizeof(utf8), &len);
andre@0: 
andre@0:   if( !result ) {
andre@0:     fprintf(stdout, "Failed to convert UCS-4 0x00000000 to UTF-8\n");
andre@0:     rv = PR_FALSE;
andre@0:   } else if( 1 != len ) {
andre@0:     fprintf(stdout, "Wrong conversion of UCS-4 0x00000000: len = %d\n", len);
andre@0:     rv = PR_FALSE;
andre@0:   } else if( '\0' != *utf8 ) {
andre@0:     fprintf(stdout, "Wrong conversion of UCS-4 0x00000000: expected 00 ,"
andre@0:             "received %02.2x\n", (unsigned int)*utf8);
andre@0:     rv = PR_FALSE;
andre@0:   }
andre@0: 
andre@0:   result = sec_port_ucs4_utf8_conversion_function(PR_TRUE,
andre@0:     "", 1, (unsigned char *)&lback, sizeof(lback), &len);
andre@0: 
andre@0:   if( !result ) {
andre@0:     fprintf(stdout, "Failed to convert UTF-8 00 to UCS-4\n");
andre@0:     rv = PR_FALSE;
andre@0:   } else if( 4 != len ) {
andre@0:     fprintf(stdout, "Wrong conversion of UTF-8 00 to UCS-4: len = %d\n", len);
andre@0:     rv = PR_FALSE;
andre@0:   } else if( 0 != lback ) {
andre@0:     fprintf(stdout, "Wrong conversion of UTF-8 00 to UCS-4: "
andre@0:             "expected 0x00000000, received 0x%08.8x\n", lback);
andre@0:     rv = PR_FALSE;
andre@0:   }
andre@0: 
andre@0:   (void)memset(utf8, 1, sizeof(utf8));
andre@0: 
andre@0:   result = sec_port_ucs2_utf8_conversion_function(PR_FALSE, 
andre@0:     (unsigned char *)&szero, sizeof(szero), utf8, sizeof(utf8), &len);
andre@0: 
andre@0:   if( !result ) {
andre@0:     fprintf(stdout, "Failed to convert UCS-2 0x0000 to UTF-8\n");
andre@0:     rv = PR_FALSE;
andre@0:   } else if( 1 != len ) {
andre@0:     fprintf(stdout, "Wrong conversion of UCS-2 0x0000: len = %d\n", len);
andre@0:     rv = PR_FALSE;
andre@0:   } else if( '\0' != *utf8 ) {
andre@0:     fprintf(stdout, "Wrong conversion of UCS-2 0x0000: expected 00 ,"
andre@0:             "received %02.2x\n", (unsigned int)*utf8);
andre@0:     rv = PR_FALSE;
andre@0:   }
andre@0: 
andre@0:   result = sec_port_ucs2_utf8_conversion_function(PR_TRUE,
andre@0:     "", 1, (unsigned char *)&sback, sizeof(sback), &len);
andre@0: 
andre@0:   if( !result ) {
andre@0:     fprintf(stdout, "Failed to convert UTF-8 00 to UCS-2\n");
andre@0:     rv = PR_FALSE;
andre@0:   } else if( 2 != len ) {
andre@0:     fprintf(stdout, "Wrong conversion of UTF-8 00 to UCS-2: len = %d\n", len);
andre@0:     rv = PR_FALSE;
andre@0:   } else if( 0 != sback ) {
andre@0:     fprintf(stdout, "Wrong conversion of UTF-8 00 to UCS-2: "
andre@0:             "expected 0x0000, received 0x%04.4x\n", sback);
andre@0:     rv = PR_FALSE;
andre@0:   }
andre@0: 
andre@0:   return rv;
andre@0: }
andre@0: 
andre@0: static PRBool
andre@0: test_multichars
andre@0: (
andre@0:   void
andre@0: )
andre@0: {
andre@0:   int i;
andre@0:   unsigned int len, lenout;
andre@0:   PRUint32 *ucs4s;
andre@0:   char *ucs4_utf8;
andre@0:   PRUint16 *ucs2s;
andre@0:   char *ucs2_utf8;
andre@0:   void *tmp;
andre@0:   PRBool result;
andre@0: 
andre@0:   ucs4s = (PRUint32 *)calloc(sizeof(ucs4)/sizeof(ucs4[0]), sizeof(PRUint32));
andre@0:   ucs2s = (PRUint16 *)calloc(sizeof(ucs2)/sizeof(ucs2[0]), sizeof(PRUint16));
andre@0: 
andre@0:   if( ((PRUint32 *)NULL == ucs4s) || ((PRUint16 *)NULL == ucs2s) ) {
andre@0:     fprintf(stderr, "out of memory\n");
andre@0:     exit(1);
andre@0:   }
andre@0: 
andre@0:   len = 0;
andre@0:   for( i = 0; i < sizeof(ucs4)/sizeof(ucs4[0]); i++ ) {
andre@0:     ucs4s[i] = ucs4[i].c;
andre@0:     len += strlen(ucs4[i].utf8);
andre@0:   }
andre@0: 
andre@0:   ucs4_utf8 = (char *)malloc(len);
andre@0: 
andre@0:   len = 0;
andre@0:   for( i = 0; i < sizeof(ucs2)/sizeof(ucs2[0]); i++ ) {
andre@0:     ucs2s[i] = ucs2[i].c;
andre@0:     len += strlen(ucs2[i].utf8);
andre@0:   }
andre@0: 
andre@0:   ucs2_utf8 = (char *)malloc(len);
andre@0: 
andre@0:   if( ((char *)NULL == ucs4_utf8) || ((char *)NULL == ucs2_utf8) ) {
andre@0:     fprintf(stderr, "out of memory\n");
andre@0:     exit(1);
andre@0:   }
andre@0: 
andre@0:   *ucs4_utf8 = '\0';
andre@0:   for( i = 0; i < sizeof(ucs4)/sizeof(ucs4[0]); i++ ) {
andre@0:     strcat(ucs4_utf8, ucs4[i].utf8);
andre@0:   }
andre@0: 
andre@0:   *ucs2_utf8 = '\0';
andre@0:   for( i = 0; i < sizeof(ucs2)/sizeof(ucs2[0]); i++ ) {
andre@0:     strcat(ucs2_utf8, ucs2[i].utf8);
andre@0:   }
andre@0: 
andre@0:   /* UTF-8 -> UCS-4 */
andre@0:   len = sizeof(ucs4)/sizeof(ucs4[0]) * sizeof(PRUint32);
andre@0:   tmp = calloc(len, 1);
andre@0:   if( (void *)NULL == tmp ) {
andre@0:     fprintf(stderr, "out of memory\n");
andre@0:     exit(1);
andre@0:   }
andre@0: 
andre@0:   result = sec_port_ucs4_utf8_conversion_function(PR_TRUE,
andre@0:     ucs4_utf8, strlen(ucs4_utf8), tmp, len, &lenout);
andre@0:   if( !result ) {
andre@0:     fprintf(stdout, "Failed to convert much UTF-8 to UCS-4\n");
andre@0:     goto done;
andre@0:   }
andre@0: 
andre@0:   if( lenout != len ) {
andre@0:     fprintf(stdout, "Unexpected length converting much UTF-8 to UCS-4\n");
andre@0:     goto loser;
andre@0:   }
andre@0: 
andre@0:   if( 0 != memcmp(ucs4s, tmp, len) ) {
andre@0:     fprintf(stdout, "Wrong conversion of much UTF-8 to UCS-4\n");
andre@0:     goto loser;
andre@0:   }
andre@0: 
andre@0:   free(tmp); tmp = (void *)NULL;
andre@0: 
andre@0:   /* UCS-4 -> UTF-8 */
andre@0:   len = strlen(ucs4_utf8);
andre@0:   tmp = calloc(len, 1);
andre@0:   if( (void *)NULL == tmp ) {
andre@0:     fprintf(stderr, "out of memory\n");
andre@0:     exit(1);
andre@0:   }
andre@0: 
andre@0:   result = sec_port_ucs4_utf8_conversion_function(PR_FALSE,
andre@0:     (unsigned char *)ucs4s, sizeof(ucs4)/sizeof(ucs4[0]) * sizeof(PRUint32), 
andre@0:     tmp, len, &lenout);
andre@0:   if( !result ) {
andre@0:     fprintf(stdout, "Failed to convert much UCS-4 to UTF-8\n");
andre@0:     goto done;
andre@0:   }
andre@0: 
andre@0:   if( lenout != len ) {
andre@0:     fprintf(stdout, "Unexpected length converting much UCS-4 to UTF-8\n");
andre@0:     goto loser;
andre@0:   }
andre@0: 
andre@0:   if( 0 != strncmp(ucs4_utf8, tmp, len) ) {
andre@0:     fprintf(stdout, "Wrong conversion of much UCS-4 to UTF-8\n");
andre@0:     goto loser;
andre@0:   }
andre@0: 
andre@0:   free(tmp); tmp = (void *)NULL;
andre@0: 
andre@0:   /* UTF-8 -> UCS-2 */
andre@0:   len = sizeof(ucs2)/sizeof(ucs2[0]) * sizeof(PRUint16);
andre@0:   tmp = calloc(len, 1);
andre@0:   if( (void *)NULL == tmp ) {
andre@0:     fprintf(stderr, "out of memory\n");
andre@0:     exit(1);
andre@0:   }
andre@0: 
andre@0:   result = sec_port_ucs2_utf8_conversion_function(PR_TRUE,
andre@0:     ucs2_utf8, strlen(ucs2_utf8), tmp, len, &lenout);
andre@0:   if( !result ) {
andre@0:     fprintf(stdout, "Failed to convert much UTF-8 to UCS-2\n");
andre@0:     goto done;
andre@0:   }
andre@0: 
andre@0:   if( lenout != len ) {
andre@0:     fprintf(stdout, "Unexpected length converting much UTF-8 to UCS-2\n");
andre@0:     goto loser;
andre@0:   }
andre@0: 
andre@0:   if( 0 != memcmp(ucs2s, tmp, len) ) {
andre@0:     fprintf(stdout, "Wrong conversion of much UTF-8 to UCS-2\n");
andre@0:     goto loser;
andre@0:   }
andre@0: 
andre@0:   free(tmp); tmp = (void *)NULL;
andre@0: 
andre@0:   /* UCS-2 -> UTF-8 */
andre@0:   len = strlen(ucs2_utf8);
andre@0:   tmp = calloc(len, 1);
andre@0:   if( (void *)NULL == tmp ) {
andre@0:     fprintf(stderr, "out of memory\n");
andre@0:     exit(1);
andre@0:   }
andre@0: 
andre@0:   result = sec_port_ucs2_utf8_conversion_function(PR_FALSE,
andre@0:     (unsigned char *)ucs2s, sizeof(ucs2)/sizeof(ucs2[0]) * sizeof(PRUint16), 
andre@0:     tmp, len, &lenout);
andre@0:   if( !result ) {
andre@0:     fprintf(stdout, "Failed to convert much UCS-2 to UTF-8\n");
andre@0:     goto done;
andre@0:   }
andre@0: 
andre@0:   if( lenout != len ) {
andre@0:     fprintf(stdout, "Unexpected length converting much UCS-2 to UTF-8\n");
andre@0:     goto loser;
andre@0:   }
andre@0: 
andre@0:   if( 0 != strncmp(ucs2_utf8, tmp, len) ) {
andre@0:     fprintf(stdout, "Wrong conversion of much UCS-2 to UTF-8\n");
andre@0:     goto loser;
andre@0:   }
andre@0: 
andre@0:   /* implement UTF16 */
andre@0: 
andre@0:   result = PR_TRUE;
andre@0:   goto done;
andre@0: 
andre@0:  loser:
andre@0:   result = PR_FALSE;
andre@0:  done:
andre@0:   free(ucs4s);
andre@0:   free(ucs4_utf8);
andre@0:   free(ucs2s);
andre@0:   free(ucs2_utf8);
andre@0:   if( (void *)NULL != tmp ) free(tmp);
andre@0:   return result;
andre@0: }
andre@0: 
andre@0: void
andre@0: byte_order
andre@0: (
andre@0:   void
andre@0: )
andre@0: {
andre@0:   /*
andre@0:    * The implementation (now) expects the 16- and 32-bit characters
andre@0:    * to be in network byte order, not host byte order.  Therefore I
andre@0:    * have to byteswap all those test vectors above.  hton[ls] may be
andre@0:    * functions, so I have to do this dynamically.  If you want to 
andre@0:    * use this code to do host byte order conversions, just remove
andre@0:    * the call in main() to this function.
andre@0:    */
andre@0: 
andre@0:   int i;
andre@0: 
andre@0:   for( i = 0; i < sizeof(ucs4)/sizeof(ucs4[0]); i++ ) {
andre@0:     struct ucs4 *e = &ucs4[i];
andre@0:     e->c = htonl(e->c);
andre@0:   }
andre@0: 
andre@0:   for( i = 0; i < sizeof(ucs2)/sizeof(ucs2[0]); i++ ) {
andre@0:     struct ucs2 *e = &ucs2[i];
andre@0:     e->c = htons(e->c);
andre@0:   }
andre@0: 
andre@0:   for( i = 0; i < sizeof(utf16)/sizeof(utf16[0]); i++ ) {
andre@0:     struct utf16 *e = &utf16[i];
andre@0:     e->c = htonl(e->c);
andre@0:     e->w[0] = htons(e->w[0]);
andre@0:     e->w[1] = htons(e->w[1]);
andre@0:   }
andre@0: 
andre@0:   return;
andre@0: }
andre@0: 
andre@0: int
andre@0: main
andre@0: (
andre@0:   int argc,
andre@0:   char *argv[]
andre@0: )
andre@0: {
andre@0:   byte_order();
andre@0: 
andre@0:   if( test_ucs4_chars() &&
andre@0:       test_ucs2_chars() &&
andre@0:       test_utf16_chars() &&
andre@0:       test_utf8_bad_chars() &&
andre@0:       test_iso88591_chars() &&
andre@0:       test_zeroes() &&
andre@0:       test_multichars() &&
andre@0:       PR_TRUE ) {
andre@0:     fprintf(stderr, "PASS\n");
andre@0:     return 1;
andre@0:   } else {
andre@0:     fprintf(stderr, "FAIL\n");
andre@0:     return 0;
andre@0:   }
andre@0: }
andre@0: 
andre@0: #endif /* TEST_UTF8 */