1 /*

     2  * Copyright (C) 2007 Apple Inc.  All rights reserved.

     3  *

     4  * Redistribution and use in source and binary forms, with or without

     5  * modification, are permitted provided that the following conditions

     6  * are met:

     7  * 1. Redistributions of source code must retain the above copyright

     8  *    notice, this list of conditions and the following disclaimer.

     9  * 2. Redistributions in binary form must reproduce the above copyright

    10  *    notice, this list of conditions and the following disclaimer in the

    11  *    documentation and/or other materials provided with the distribution.

    12  *

    13  * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY

    14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

    15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

    16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR

    17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,

    18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,

    19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR

    20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY

    21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

    22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

    23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 

    24  */

    25 

    26 #include "config.h"

    27 #include "UTF8.h"

    28 

    29 namespace WTF {

    30 namespace Unicode {

    31 

    32 inline int inlineUTF8SequenceLengthNonASCII(char b0)

    33 {

    34     if ((b0 & 0xC0) != 0xC0)

    35         return 0;

    36     if ((b0 & 0xE0) == 0xC0)

    37         return 2;

    38     if ((b0 & 0xF0) == 0xE0)

    39         return 3;

    40     if ((b0 & 0xF8) == 0xF0)

    41         return 4;

    42     return 0;

    43 }

    44 

    45 inline int inlineUTF8SequenceLength(char b0)

    46 {

    47     return (b0 & 0x80) == 0 ? 1 : inlineUTF8SequenceLengthNonASCII(b0);

    48 }

    49 

    50 int UTF8SequenceLength(char b0)

    51 {

    52     return (b0 & 0x80) == 0 ? 1 : inlineUTF8SequenceLengthNonASCII(b0);

    53 }

    54 

    55 int decodeUTF8Sequence(const char* sequence)

    56 {

    57     // Handle 0-byte sequences (never valid).

    58     const unsigned char b0 = sequence[0];

    59     const int length = inlineUTF8SequenceLength(b0);

    60     if (length == 0)

    61         return -1;

    62 

    63     // Handle 1-byte sequences (plain ASCII).

    64     const unsigned char b1 = sequence[1];

    65     if (length == 1) {

    66         if (b1)

    67             return -1;

    68         return b0;

    69     }

    70 

    71     // Handle 2-byte sequences.

    72     if ((b1 & 0xC0) != 0x80)

    73         return -1;

    74     const unsigned char b2 = sequence[2];

    75     if (length == 2) {

    76         if (b2)

    77             return -1;

    78         const int c = ((b0 & 0x1F) << 6) | (b1 & 0x3F);

    79         if (c < 0x80)

    80             return -1;

    81         return c;

    82     }

    83 

    84     // Handle 3-byte sequences.

    85     if ((b2 & 0xC0) != 0x80)

    86         return -1;

    87     const unsigned char b3 = sequence[3];

    88     if (length == 3) {

    89         if (b3)

    90             return -1;

    91         const int c = ((b0 & 0xF) << 12) | ((b1 & 0x3F) << 6) | (b2 & 0x3F);

    92         if (c < 0x800)

    93             return -1;

    94         // UTF-16 surrogates should never appear in UTF-8 data.

    95         if (c >= 0xD800 && c <= 0xDFFF)

    96             return -1;

    97         return c;

    98     }

    99 

   100     // Handle 4-byte sequences.

   101     if ((b3 & 0xC0) != 0x80)

   102         return -1;

   103     const unsigned char b4 = sequence[4];

   104     if (length == 4) {

   105         if (b4)

   106             return -1;

   107         const int c = ((b0 & 0x7) << 18) | ((b1 & 0x3F) << 12) | ((b2 & 0x3F) << 6) | (b3 & 0x3F);

   108         if (c < 0x10000 || c > 0x10FFFF)

   109             return -1;

   110         return c;

   111     }

   112 

   113     return -1;

   114 }

   115 

   116 // Once the bits are split out into bytes of UTF-8, this is a mask OR-ed

   117 // into the first byte, depending on how many bytes follow.  There are

   118 // as many entries in this table as there are UTF-8 sequence types.

   119 // (I.e., one byte sequence, two byte... etc.). Remember that sequencs

   120 // for *legal* UTF-8 will be 4 or fewer bytes total.

   121 static const unsigned char firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };

   122 

   123 ConversionResult convertUTF16ToUTF8(

   124     const UChar** sourceStart, const UChar* sourceEnd, 

   125     char** targetStart, char* targetEnd, bool strict)

   126 {

   127     ConversionResult result = conversionOK;

   128     const UChar* source = *sourceStart;

   129     char* target = *targetStart;

   130     while (source < sourceEnd) {

   131         UChar32 ch;

   132         unsigned short bytesToWrite = 0;

   133         const UChar32 byteMask = 0xBF;

   134         const UChar32 byteMark = 0x80; 

   135         const UChar* oldSource = source; // In case we have to back up because of target overflow.

   136         ch = static_cast<unsigned short>(*source++);

   137         // If we have a surrogate pair, convert to UChar32 first.

   138         if (ch >= 0xD800 && ch <= 0xDBFF) {

   139             // If the 16 bits following the high surrogate are in the source buffer...

   140             if (source < sourceEnd) {

   141                 UChar32 ch2 = static_cast<unsigned short>(*source);

   142                 // If it's a low surrogate, convert to UChar32.

   143                 if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) {

   144                     ch = ((ch - 0xD800) << 10) + (ch2 - 0xDC00) + 0x0010000;

   145                     ++source;

   146                 } else if (strict) { // it's an unpaired high surrogate

   147                     --source; // return to the illegal value itself

   148                     result = sourceIllegal;

   149                     break;

   150                 }

   151             } else { // We don't have the 16 bits following the high surrogate.

   152                 --source; // return to the high surrogate

   153                 result = sourceExhausted;

   154                 break;

   155             }

   156         } else if (strict) {

   157             // UTF-16 surrogate values are illegal in UTF-32

   158             if (ch >= 0xDC00 && ch <= 0xDFFF) {

   159                 --source; // return to the illegal value itself

   160                 result = sourceIllegal;

   161                 break;

   162             }

   163         }

   164         // Figure out how many bytes the result will require

   165         if (ch < (UChar32)0x80) {

   166             bytesToWrite = 1;

   167         } else if (ch < (UChar32)0x800) {

   168             bytesToWrite = 2;

   169         } else if (ch < (UChar32)0x10000) {

   170             bytesToWrite = 3;

   171         } else if (ch < (UChar32)0x110000) {

   172             bytesToWrite = 4;

   173         } else {

   174             bytesToWrite = 3;

   175             ch = 0xFFFD;

   176         }

   177 

   178         target += bytesToWrite;

   179         if (target > targetEnd) {

   180             source = oldSource; // Back up source pointer!

   181             target -= bytesToWrite;

   182             result = targetExhausted;

   183             break;

   184         }

   185         switch (bytesToWrite) { // note: everything falls through.

   186             case 4: *--target = (char)((ch | byteMark) & byteMask); ch >>= 6;

   187             case 3: *--target = (char)((ch | byteMark) & byteMask); ch >>= 6;

   188             case 2: *--target = (char)((ch | byteMark) & byteMask); ch >>= 6;

   189             case 1: *--target =  (char)(ch | firstByteMark[bytesToWrite]);

   190         }

   191         target += bytesToWrite;

   192     }

   193     *sourceStart = source;

   194     *targetStart = target;

   195     return result;

   196 }

   197 

   198 // This must be called with the length pre-determined by the first byte.

   199 // If presented with a length > 4, this returns false.  The Unicode

   200 // definition of UTF-8 goes up to 4-byte sequences.

   201 static bool isLegalUTF8(const unsigned char* source, int length)

   202 {

   203     unsigned char a;

   204     const unsigned char* srcptr = source + length;

   205     switch (length) {

   206         default: return false;

   207         // Everything else falls through when "true"...

   208         case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;

   209         case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;

   210         case 2: if ((a = (*--srcptr)) > 0xBF) return false;

   211 

   212         switch (*source) {

   213             // no fall-through in this inner switch

   214             case 0xE0: if (a < 0xA0) return false; break;

   215             case 0xED: if (a > 0x9F) return false; break;

   216             case 0xF0: if (a < 0x90) return false; break;

   217             case 0xF4: if (a > 0x8F) return false; break;

   218             default:   if (a < 0x80) return false;

   219         }

   220 

   221         case 1: if (*source >= 0x80 && *source < 0xC2) return false;

   222     }

   223     if (*source > 0xF4)

   224         return false;

   225     return true;

   226 }

   227 

   228 // Magic values subtracted from a buffer value during UTF8 conversion.

   229 // This table contains as many values as there might be trailing bytes

   230 // in a UTF-8 sequence.

   231 static const UChar32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 

   232             0x03C82080UL, 0xFA082080UL, 0x82082080UL };

   233 

   234 ConversionResult convertUTF8ToUTF16(

   235     const char** sourceStart, const char* sourceEnd, 

   236     UChar** targetStart, UChar* targetEnd, bool strict)

   237 {

   238     ConversionResult result = conversionOK;

   239     const char* source = *sourceStart;

   240     UChar* target = *targetStart;

   241     while (source < sourceEnd) {

   242         UChar32 ch = 0;

   243         int extraBytesToRead = UTF8SequenceLength(*source) - 1;

   244         if (source + extraBytesToRead >= sourceEnd) {

   245             result = sourceExhausted;

   246             break;

   247         }

   248         // Do this check whether lenient or strict

   249         if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(source), extraBytesToRead + 1)) {

   250             result = sourceIllegal;

   251             break;

   252         }

   253         // The cases all fall through.

   254         switch (extraBytesToRead) {

   255             case 5: ch += static_cast<unsigned char>(*source++); ch <<= 6; // remember, illegal UTF-8

   256             case 4: ch += static_cast<unsigned char>(*source++); ch <<= 6; // remember, illegal UTF-8

   257             case 3: ch += static_cast<unsigned char>(*source++); ch <<= 6;

   258             case 2: ch += static_cast<unsigned char>(*source++); ch <<= 6;

   259             case 1: ch += static_cast<unsigned char>(*source++); ch <<= 6;

   260             case 0: ch += static_cast<unsigned char>(*source++);

   261         }

   262         ch -= offsetsFromUTF8[extraBytesToRead];

   263 

   264         if (target >= targetEnd) {

   265             source -= (extraBytesToRead + 1); // Back up source pointer!

   266             result = targetExhausted; break;

   267         }

   268         if (ch <= 0xFFFF) {

   269             // UTF-16 surrogate values are illegal in UTF-32

   270             if (ch >= 0xD800 && ch <= 0xDFFF) {

   271                 if (strict) {

   272                     source -= (extraBytesToRead + 1); // return to the illegal value itself

   273                     result = sourceIllegal;

   274                     break;

   275                 } else

   276                     *target++ = 0xFFFD;

   277             } else

   278                 *target++ = (UChar)ch; // normal case

   279         } else if (ch > 0x10FFFF) {

   280             if (strict) {

   281                 result = sourceIllegal;

   282                 source -= (extraBytesToRead + 1); // return to the start

   283                 break; // Bail out; shouldn't continue

   284             } else

   285                 *target++ = 0xFFFD;

   286         } else {

   287             // target is a character in range 0xFFFF - 0x10FFFF

   288             if (target + 1 >= targetEnd) {

   289                 source -= (extraBytesToRead + 1); // Back up source pointer!

   290                 result = targetExhausted;

   291                 break;

   292             }

   293             ch -= 0x0010000UL;

   294             *target++ = (UChar)((ch >> 10) + 0xD800);

   295             *target++ = (UChar)((ch & 0x03FF) + 0xDC00);

   296         }

   297     }

   298     *sourceStart = source;

   299     *targetStart = target;

   300     return result;

   301 }

   302 

   303 }

   304 }