TYPO3 Core

<?php

/***************************************************************

/**

* Class for conversion between charsets.

*

*

*

* 136: class t3lib_cs

*

*

*

*

*

*

*

* TOTAL FUNCTIONS: 35

* (This index is automatically created/updated by the extension "extdeveval")

*/

/**

* Notes on UTF-8

*

* @subpackage t3lib

*/

class t3lib_cs {

// This is the array where parsed conversion tables are stored (cached)

// An array where case folding data will be stored (cached)

// An array where charset-to-ASCII mappings are stored (cached)

// This tells the converter which charsets has two bytes per char:

);

// This tells the converter which charsets has four bytes per char:

);

// This tells the converter which charsets use a scheme like the Extended Unix Code:

);

// see http://developer.apple.com/documentation/macos8/TextIntlSvcs/TextEncodingConversionManager/TEC1.5/TEC.b0.html

// http://czyborra.com/charsets/iso8859.html

'us' => 'ascii',

'cp819' => 'iso-8859-1',

'ibm819' => 'iso-8859-1',

'iso-ir-100' => 'iso-8859-1',

);

// mapping of iso-639-1 language codes to script names

// iso-639-1 language codes, see http://www.loc.gov/standards/iso639-2/php/code_list.php

'ar' => 'arabic',

'et' => 'estonian',

'gr' => 'greek',

'ja' => 'japanese',

'ko' => 'korean',

'lt' => 'lithuanian',

'th' => 'thai',

'vi' => 'vietnamese',

'zh' => 'chinese',

// MS language codes, see http://msdn.microsoft.com/library/default.asp?url=/library/en-us/vclib/html/_crt_language_strings.asp

// http://msdn.microsoft.com/library/default.asp?url=/library/en-us/wceinternational5/html/wce50conLanguageIdentifiersandLocales.asp

'ara' => 'arabic',

'chs' => 'simpl_chinese',

'cht' => 'trad_chinese',

'ell' => 'greek',

'heb' => 'hebrew',

'jpn' => 'japanese',

'kor' => 'korean',

'lth' => 'lithuanian',

'tha' => 'thai',

'trk' => 'turkish',

// English language names

'albanian' => 'albanian',

'arabic' => 'arabic',

);

// mapping of language (family) names to charsets on Unix

'west_european' => 'iso-8859-1',

'estonian' => 'iso-8859-1',

'east_european' => 'iso-8859-2',

);

// mapping of language (family) names to charsets on Windows

'east_european' => 'windows-1250',

'cyrillic' => 'windows-1251',

'west_european' => 'windows-1252',

);

// mapping of locale names to charsets

'japanese.euc' => 'euc-jp',

'ja_jp.ujis' => 'euc-jp',

'korean.euc' => 'euc-kr',

* @return string Normalized charset

* @author Martin Kutschker <martin.t.kutschker@blackbox.net>

*/

$charset = trim(strtolower($charset));

return $charset;

}

/**

* Get the charset of a locale.

*

* ln_CN.cs@mod language / country / charset / modifier

*

* @param string Locale string

* @return string Charset resolved for locale string

* @author Martin Kutschker <martin.t.kutschker@blackbox.net>

*/

$locale = strtolower($locale);

// exact locale specific charset?

// get modifier

// locale contains charset: use it

// modifier is 'euro' (after charset check, because of xx.utf-8@euro)

// get language

$cs = $this->script_to_charset_windows[$script] ? $this->script_to_charset_windows[$script] : 'windows-1252';

} else {

$cs = $this->script_to_charset_unix[$script] ? $this->script_to_charset_unix[$script] : 'iso-8859-1';

}

/********************************************

*

* Charset Conversion functions

* @return string Converted string

* @see convArray()

*/

// PHP-libs don't support fallback to SGML entities, but UTF-8 handles everything

}

// fallback to TYPO3 conversion

}

return $str;

}

* @return void

* @see conv()

*/

} elseif (is_string($array[$key])) {

}

}

}

* @param string Charset, lowercase. Must be found in csconvtbl/ folder.

* @return string Output string, converted to UTF-8

*/

// Charset is case-insensitive.

$strLen = strlen($str);

$a++;

$a++;

}

}

}

return $outStr;

}

* @param boolean If set, then characters that are not available in the destination character set will be encoded as numeric entities

* @return string Output string, converted to local charset

*/

if ($charset === 'utf-8') {

return $str;

}

// Charset is case-insensitive.

$strLen = strlen($str);

}

return $outStr;

}

* @param string Input string

* @return string Output string

*/

$strLen = strlen($str);

}

return $outStr;

* @param boolean If set, then all string-HTML entities (like &amp; or &pound; will be converted as well)

* @return string Output string

*/

}

$token = md5(microtime());

$parts = explode($token, preg_replace('/(&([#[:alnum:]]*);)/', $token . '${2}' . $token, $str));

} else {

}

}

}

}

}

/**

* @param boolean If set, then instead of integer numbers the real UTF-8 char is returned.

* @return array Output array with the char numbers

*/

// If entities must be registered as well...:

}

// Do conversion:

$strLen = strlen($str);

}

return $outArr;

* The binary representation of the character's integer value is thus simply spread across the bytes and the number of high bits set in the lead byte announces the number of bytes in the multibyte sequence:

*

* bytes | bits | representation

*

* @param integer UNICODE integer

* @return string UTF-8 multibyte character string

* @see utf8CharToUnumber()

*/

if ($cbyte < 0x80) {

return $str;

}

* @return integer UNICODE integer

* @see UnumberToChar()

*/

$int = bindec($binBuf);

}

/********************************************

*

* Init functions

* @return integer Returns '1' if already loaded. Returns FALSE if charset conversion table was not found. Returns '2' if the charset conversion table was found and parsed.

* @access private

*/

// Only process if the charset is not yet loaded:

// Conversion table filename:

// If the conversion table is found:

// Cache file for charsets:

// Caching brought parsing time for gb2312 down from 2400 ms to 150 ms. For other charsets we are talking 11 ms down to zero.

} else {

// Parse conversion table into lines:

// Initialize the internal variable holding the conv. table:

// traverse the lines:

// Detect type if not done yet: (Done on first real line)

// The "whitespaced" type is on the syntax "0x0A 0x000A #LINE FEED" while "ms-token" is like "B9 = U+00B9 : SUPERSCRIPT ONE"

list($hexbyte, $utf8) = preg_split('/[=:]/', $value, 3);

$hexbyte = $regA[1];

}

$decval = hexdec(trim($hexbyte));

}

}

}

}

}

return 2;

/**

* This function initializes all UTF-8 character data tables.

* @return integer Returns FALSE on error, a TRUE value on success: 1 table already loaded, 2, cached version, 3 table parsed (and cached).

* @access private

*/

// cache files

$cacheFileCase = t3lib_div::getFileAbsFileName('typo3temp/cs/cscase_utf-8.tbl');

$cacheFileASCII = t3lib_div::getFileAbsFileName('typo3temp/cs/csascii_utf-8.tbl');

// Only process if the tables are not yet loaded

case 'case':

// Use cached version if possible

$this->caseFolding['utf-8'] = unserialize(t3lib_div::getUrl($cacheFileCase));

return 2;

}

break;

case 'ascii':

// Use cached version if possible

$this->toASCII['utf-8'] = unserialize(t3lib_div::getUrl($cacheFileASCII));

return 2;

}

}

// process main Unicode data file

// key = utf8 char (single codepoint), value = utf8 string (codepoint sequence)

// note: we use the UTF-8 characters here and not the Unicode numbers to avoid conversion roundtrip in utf8_strtolower/-upper)

$utf8CaseFolding['toLower'] = array();

$utf8CaseFolding['toTitle'] = array();

// has a lot of info

$ord = hexdec($char);

$utf8_char = $this->UnumberToChar($ord);

// store "title" only when different from "upper" (only a few)

$mark["U+$char"] = 1;

break;

// accented Latin letters without "official" decomposition

$match = array();

$c = ord($match[2]);

$decomposition["U+$char"] = array(dechex($c));

continue;

}

$match = array();

break;

break;

break;

// ignore Arabic and vertical layout presentation decomposition

fclose($fh);

// process additional Unicode data for casing (allow folded characters to expand into a sequence)

$utf8_char = $this->UnumberToChar(hexdec($char));

$arr = explode(' ', $lower);

}

$arr = explode(' ', $title);

}

}

}

}

}

// process custom decompositions

$decomposition["U+$char"] = explode(' ', $translit);

}

}

// decompose and remove marks; inspired by unac (Loic Dachary <loic@senga.org>)

$code_decomp = array();

array_unshift($to, $cv);

}

array_push($code_decomp, $code_value);

}

}

$decomposition[$from] = $code_decomp;

} else {

unset($decomposition[$from]);

$this->toASCII['utf-8'] = array();

$ascii =& $this->toASCII['utf-8'];

$code_decomp = array();

$ord = hexdec($code_value);

else

}

// add numeric decompositions

$utf8_char = $this->UnumberToChar(hexdec($from));

$ascii[$utf8_char] = $to;

}

}

}

}

return 3;

* @return integer Returns FALSE on error, a TRUE value on success: 1 table already loaded, 2, cached version, 3 table parsed (and cached).

* @access private

*/

// Only process if the case table is not yet loaded:

// Use cached version if possible

$this->caseFolding[$charset] = unserialize(t3lib_div::getUrl($cacheFile));

return 2;

}

// init UTF-8 conversion for this charset