Index: t3lib/class.t3lib_div.php =================================================================== --- t3lib/class.t3lib_div.php (revision 3054) +++ t3lib/class.t3lib_div.php (working copy) @@ -3662,54 +3662,146 @@ /** * Includes a locallang file and returns the $LOCAL_LANG array found inside. * - * @param string Input is a file-reference (see t3lib_div::getFileAbsFileName) which, if exists, is included. That file is expected to be a 'local_lang' file containing a $LOCAL_LANG array. + * @param string Input is a file-reference (see t3lib_div::getFileAbsFileName). That file is expected to be a 'locallang.php' file containing a $LOCAL_LANG array (will be included!) or a 'locallang.xml' file conataining a valid XML TYPO3 language structure. * @param string Language key + * @param string Character set (option); if not set, determined by the language key * @return array Value of $LOCAL_LANG found in the included file. If that array is found it's returned. Otherwise an empty array */ - public static function readLLfile($fileRef,$langKey) { + public static function readLLfile($fileRef, $langKey, $charset='') { $file = t3lib_div::getFileAbsFileName($fileRef); if ($file) { - $baseFile = ereg_replace('\.(php|xml)$', '', $file); + $baseFile = preg_replace('/\.(php|xml)$/', '', $file); - if (@is_file($baseFile.'.xml')) { - $LOCAL_LANG = t3lib_div::readLLXMLfile($baseFile.'.xml', $langKey); - } elseif (@is_file($baseFile.'.php')) { - include($baseFile.'.php'); - } else die('Filereference, "'.$file.'", not found!'); + if (@is_file($baseFile.'.xml')) { + $LOCAL_LANG = t3lib_div::readLLXMLfile($baseFile.'.xml', $langKey, $charset); + } elseif (@is_file($baseFile.'.php')) { + if ($GLOBALS['TYPO3_CONF_VARS']['BE']['forceCharset'] || $charset) { + $LOCAL_LANG = t3lib_div::readLLPHPfile($baseFile.'.php', $langKey, $charset); + } else { + include($baseFile.'.php'); + if (is_array($LOCAL_LANG)) { + $LOCAL_LANG = array('default'=>$LOCAL_LANG['default'], $langKey=>$LOCAL_LANG[$langKey]); } + } + } else { + die('File "'.$file.'" not found!'); + } } - return is_array($LOCAL_LANG)?$LOCAL_LANG:array(); + return is_array($LOCAL_LANG) ? $LOCAL_LANG : array(); } /** + * Includes a locallang-php file and returns the $LOCAL_LANG array + * Works only when the frontend or backend has been initialized with a charset conversion object. See first code lines. + * + * @param string Absolute reference to locallang-PHP file + * @param string TYPO3 language key, eg. "dk" or "de" or "default" + * @param string Character set (optional) + * @return array LOCAL_LANG array in return. + */ + function readLLPHPfile($fileRef, $langKey, $charset='') { + + if (is_object($GLOBALS['LANG'])) { + $csConvObj = &$GLOBALS['LANG']->csConvObj; + } elseif (is_object($GLOBALS['TSFE'])) { + $csConvObj = &$GLOBALS['TSFE']->csConvObj; + } else { + $csConvObj = t3lib_div::makeInstance('t3lib_cs'); + } + + if (@is_file($fileRef) && $langKey) { + + // Set charsets: + $sourceCharset = $csConvObj->parse_charset($csConvObj->charSetArray[$langKey] ? $csConvObj->charSetArray[$langKey] : 'iso-8859-1'); + if ($charset) { + $targetCharset = $csConvObj->parse_charset($charset); + } elseif ($GLOBALS['TYPO3_CONF_VARS']['BE']['forceCharset']) { + // when forceCharset is set, we store ALL labels in this charset!!! + $targetCharset = $csConvObj->parse_charset($GLOBALS['TYPO3_CONF_VARS']['BE']['forceCharset']); + } else { + $targetCharset = $csConvObj->parse_charset($csConvObj->charSetArray[$langKey] ? $csConvObj->charSetArray[$langKey] : 'iso-8859-1'); + } + + // Cache file name: + $hashSource = substr($fileRef,strlen(PATH_site)).'|'.date('d-m-Y H:i:s',filemtime($fileRef)).'|version=2.3'; + $cacheFileName = PATH_site.'typo3temp/llxml/'. + substr(basename($fileRef),10,15). + '_'.t3lib_div::shortMD5($hashSource).'.'.$langKey.'.'.$targetCharset.'.cache'; + // Check if cache file exists... + if (!@is_file($cacheFileName)) { // ... if it doesn't, create content and write it: + + // Get PHP data + include($fileRef); + if (!is_array($LOCAL_LANG)) { + die('\''.$fileRef.'\' is no TYPO3 language file)!'); + } + + // converting the default language (English) + // this needs to be done for a few accented loan words and extension names + if (is_array($LOCAL_LANG['default'] && $targetCharset!='iso-8859-1')) { + foreach($LOCAL_LANG['default'] as $labelKey => $labelValue) { + $LOCAL_LANG['default'][$labelKey] = $csConvObj->conv($labelValue,'iso-8859-1',$targetCharset); + } + } + + if ($langKey!='default' && is_array($LOCAL_LANG[$langKey]) && $sourceCharset!=$targetCharset) { + foreach($LOCAL_LANG[$langKey] as $labelKey => $labelValue) { + $LOCAL_LANG[$langKey][$labelKey] = $csConvObj->conv($labelValue,$sourceCharset,$targetCharset); + } + } + + // Cache the content now: + $serContent = array('origFile'=>$hashSource, 'LOCAL_LANG'=>array('default'=>$LOCAL_LANG['default'], $langKey=>$LOCAL_LANG[$langKey])); + $res = t3lib_div::writeFileToTypo3tempDir($cacheFileName, serialize($serContent)); + if ($res) die('ERROR: '.$res); + } else { + // Get content from cache: + $serContent = unserialize(t3lib_div::getUrl($cacheFileName)); + $LOCAL_LANG = $serContent['LOCAL_LANG']; + } + + return $LOCAL_LANG; + } + } + + /** * Includes a locallang-xml file and returns the $LOCAL_LANG array * Works only when the frontend or backend has been initialized with a charset conversion object. See first code lines. * * @param string Absolute reference to locallang-XML file * @param string TYPO3 language key, eg. "dk" or "de" or "default" + * @param string Character set (optional) * @return array LOCAL_LANG array in return. */ - public static function readLLXMLfile($fileRef,$langKey) { + public static function readLLXMLfile($fileRef, $langKey, $charset='') { if (is_object($GLOBALS['LANG'])) { $csConvObj = &$GLOBALS['LANG']->csConvObj; } elseif (is_object($GLOBALS['TSFE'])) { $csConvObj = &$GLOBALS['TSFE']->csConvObj; - } else $csConvObj = NULL; + } else { + $csConvObj = t3lib_div::makeInstance('t3lib_cs'); + } - if (@is_file($fileRef) && $langKey && is_object($csConvObj)) { + $LOCAL_LANG = NULL; + if (@is_file($fileRef) && $langKey) { // Set charset: - $origCharset = $csConvObj->parse_charset($csConvObj->charSetArray[$langKey] ? $csConvObj->charSetArray[$langKey] : 'iso-8859-1'); + if ($charset) { + $targetCharset = $csConvObj->parse_charset($charset); + } elseif ($GLOBALS['TYPO3_CONF_VARS']['BE']['forceCharset']) { + // when forceCharset is set, we store ALL labels in this charset!!! + $targetCharset = $csConvObj->parse_charset($GLOBALS['TYPO3_CONF_VARS']['BE']['forceCharset']); + } else { + $targetCharset = $csConvObj->parse_charset($csConvObj->charSetArray[$langKey] ? $csConvObj->charSetArray[$langKey] : 'iso-8859-1'); + } // Cache file name: $hashSource = substr($fileRef,strlen(PATH_site)).'|'.date('d-m-Y H:i:s',filemtime($fileRef)).'|version=2.3'; $cacheFileName = PATH_site.'typo3temp/llxml/'. - #str_replace('_','',ereg_replace('^.*\/','',dirname($fileRef))). - #'_'.basename($fileRef). - substr(basename($fileRef),10,15). - '_'.t3lib_div::shortMD5($hashSource).'.'.$langKey.'.'.$origCharset.'.cache'; + substr(basename($fileRef),10,15). + '_'.t3lib_div::shortMD5($hashSource).'.'.$langKey.'.'.$targetCharset.'.cache'; // Check if cache file exists... if (!@is_file($cacheFileName)) { // ... if it doesn't, create content and write it: @@ -3718,26 +3810,24 @@ $xmlString = t3lib_div::getUrl($fileRef); $xmlContent = t3lib_div::xml2array($xmlString); if (!is_array($xmlContent)) { - die($fileRef.' was not XML!: '.$xmlContent); + die('\''.$fileRef.'\' is no TYPO3 language file)!'); } // Set default LOCAL_LANG array content: $LOCAL_LANG = array(); $LOCAL_LANG['default'] = $xmlContent['data']['default']; - // Converting charset of default language from utf-8 to iso-8859-1 (since that is what the system would expect for default langauge in the core due to historical reasons) - // This conversion is unneccessary for 99,99% of all default labels since they are in english, therefore ASCII. - // However, an extension like TemplaVoila uses an extended character in its name, even in Default language. To accommodate that (special chars for default) this conversion must be made. - // Since the output from this function is probably always cached it is considered insignificant to do this conversion. - // - kasper - if (is_array($LOCAL_LANG['default'])) { + // converting the default language (English) + // this needs to be done for a few accented loan words and extension names + // NOTE: no conversion is done when in UTF-8 mode! + if (is_array($LOCAL_LANG['default'] && $targetCharset!='utf-8')) { foreach($LOCAL_LANG['default'] as $labelKey => $labelValue) { - $LOCAL_LANG['default'][$labelKey] = $csConvObj->utf8_decode($labelValue,'iso-8859-1'); + $LOCAL_LANG['default'][$labelKey] = $csConvObj->utf8_decode($labelValue,$targetCharset); } } - // Specific language, convert from utf-8 to backend language charset: - // NOTICE: Converting from utf-8 back to "native" language may be a temporary solution until we can totally discard "locallang.php" files altogether (and use utf-8 for everything). But doing this conversion is the quickest way to migrate now and the source is in utf-8 anyway which is the main point. + // converting other languages to their "native" charsets + // NOTE: no conversion is done when in UTF-8 mode! if ($langKey!='default') { // If no entry is found for the language key, then force a value depending on meta-data setting. By default an automated filename will be used: @@ -3748,15 +3838,15 @@ } // Checking if charset should be converted. - if (is_array($LOCAL_LANG[$langKey]) && $origCharset!='utf-8') { + if (is_array($LOCAL_LANG[$langKey]) && $targetCharset!='utf-8') { foreach($LOCAL_LANG[$langKey] as $labelKey => $labelValue) { - $LOCAL_LANG[$langKey][$labelKey] = $csConvObj->utf8_decode($labelValue,$origCharset); + $LOCAL_LANG[$langKey][$labelKey] = $csConvObj->utf8_decode($labelValue,$targetCharset); } } } // Cache the content now: - $serContent = array('origFile'=>$hashSource, 'LOCAL_LANG'=>$LOCAL_LANG); + $serContent = array('origFile'=>$hashSource, 'LOCAL_LANG'=>array('default'=>$LOCAL_LANG['default'], $langKey=>$LOCAL_LANG[$langKey])); $res = t3lib_div::writeFileToTypo3tempDir($cacheFileName, serialize($serContent)); if ($res) die('ERROR: '.$res); } else { @@ -3773,10 +3863,10 @@ if ($localized_file && @is_file($localized_file)) { // Cache file name: - $hashSource = substr($localized_file,strlen(PATH_site)).'|'.date('d-m-Y H:i:s',filemtime($localized_file)); - $cacheFileName = PATH_site.'typo3temp/llxml/ext_'. + $hashSource = substr($localized_file,strlen(PATH_site)).'|'.date('d-m-Y H:i:s',filemtime($localized_file)).'|version=2.3'; + $cacheFileName = PATH_site.'typo3temp/llxml/EXT_'. substr(basename($localized_file),10,15). - '_'.t3lib_div::shortMD5($hashSource).'.'.$langKey.'.'.$origCharset.'.cache'; + '_'.t3lib_div::shortMD5($hashSource).'.'.$langKey.'.'.$targetCharset.'.cache'; // Check if cache file exists... if (!@is_file($cacheFileName)) { // ... if it doesn't, create content and write it: @@ -3787,14 +3877,14 @@ $LOCAL_LANG[$langKey] = is_array($local_xmlContent['data'][$langKey]) ? $local_xmlContent['data'][$langKey] : array(); // Checking if charset should be converted. - if (is_array($LOCAL_LANG[$langKey]) && $origCharset!='utf-8') { + if (is_array($LOCAL_LANG[$langKey]) && $targetCharset!='utf-8') { foreach($LOCAL_LANG[$langKey] as $labelKey => $labelValue) { - $LOCAL_LANG[$langKey][$labelKey] = $csConvObj->utf8_decode($labelValue,$origCharset); + $LOCAL_LANG[$langKey][$labelKey] = $csConvObj->utf8_decode($labelValue,$targetCharset); } } // Cache the content now: - $serContent = array('extlang'=>$langKey, 'origFile'=>$LOCAL_LANG[$langKey], 'EXT_DATA'=>$LOCAL_LANG[$langKey]); + $serContent = array('extlang'=>$langKey, 'origFile'=>$hashSource, 'EXT_DATA'=>$LOCAL_LANG[$langKey]); $res = t3lib_div::writeFileToTypo3tempDir($cacheFileName, serialize($serContent)); if ($res) die('ERROR: '.$res); } else { @@ -3845,7 +3935,9 @@ $file_extKey.'/'. ($file_extPath?$file_extPath.'/':''). $language.'.'.$file_fileName; - } else return NULL; + } else { + return NULL; + } } Index: typo3/sysext/lang/lang.php =================================================================== --- typo3/sysext/lang/lang.php (revision 3054) +++ typo3/sysext/lang/lang.php (working copy) @@ -112,9 +112,6 @@ var $LL_files_cache=array(); // Internal cache for read LL-files var $LL_labels_cache=array(); // Internal cache for ll-labels (filled as labels are requested) - // Internal charset conversion: - var $origCharSet=''; // If set, then it means that the this->charSet is set to a forced, common value for the WHOLE backend regardless of user language. And THIS variable will contain the original charset for the language labels. With ->csConvObj we must then convert the original charset to the charset used in the backend from now on. - /** * instance of the "t3lib_cs" class. May be used by any application. * @@ -164,17 +161,12 @@ // If a forced charset is used and different from the charset otherwise used: if ($GLOBALS['TYPO3_CONF_VARS']['BE']['forceCharset'] && $GLOBALS['TYPO3_CONF_VARS']['BE']['forceCharset']!=$this->charSet) { // Set the forced charset: - $this->origCharSet = $this->charSet; $this->charSet = $GLOBALS['TYPO3_CONF_VARS']['BE']['forceCharset']; if ($this->charSet!='utf-8' && !$this->csConvObj->initCharset($this->charSet)) { t3lib_BEfunc::typo3PrintError ('The forced character set "'.$this->charSet.'" was not found in t3lib/csconvtbl/','Forced charset not found'); exit; } - if ($this->origCharSet!='utf-8' && !$this->csConvObj->initCharset($this->origCharSet)) { - t3lib_BEfunc::typo3PrintError ('The original character set "'.$this->origCharSet.'" was not found in t3lib/csconvtbl/','Forced charset not found'); - exit; - } } } @@ -214,9 +206,9 @@ */ function hscAndCharConv($lStr,$hsc) { $lStr = $hsc ? htmlspecialchars($lStr) : $lStr; - if ($this->origCharSet) { - $lStr = $this->csConvObj->conv($lStr,$this->origCharSet,$this->charSet,1); - } + + // labels returned from a locallang file used to be in the language of the charset. Since TYPO3 4.1 they are always in the charset of the BE. + return $lStr; } @@ -445,7 +437,7 @@ * @return array Value of $LOCAL_LANG found in the included file. If that array is found it's returned. Otherwise an empty array */ function readLLfile($fileRef) { - return t3lib_div::readLLfile($fileRef,$this->lang); + return t3lib_div::readLLfile($fileRef, $this->lang, $this->charSet); } /** @@ -465,4 +457,4 @@ if (defined('TYPO3_MODE') && $TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/lang/lang.php']) { include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/lang/lang.php']); } -?> +?> \ No newline at end of file Index: typo3/sysext/cms/tslib/class.tslib_fe.php =================================================================== --- typo3/sysext/cms/tslib/class.tslib_fe.php (revision 3054) +++ typo3/sysext/cms/tslib/class.tslib_fe.php (working copy) @@ -386,8 +386,6 @@ // LANG: var $lang=''; // Set to the system language key (used on the site) var $langSplitIndex=0; // Set to the index number of the language key - var $labelsCharset=''; // Charset of the labels from locallang (based on $this->lang) - var $convCharsetToFrom=''; // Set to the charsets to convert from/to IF there are any difference. Otherwise this stays a string var $LL_labels_cache=array(); var $LL_files_cache=array(); @@ -4075,11 +4073,11 @@ $extPrfx='EXT:'; } $parts = explode(':',$restStr); - $parts[0]=$extPrfx.$parts[0]; + $parts[0] = $extPrfx.$parts[0]; if (!isset($this->LL_files_cache[$parts[0]])) { // Getting data if not cached $this->LL_files_cache[$parts[0]] = $this->readLLfile($parts[0]); } - $this->LL_labels_cache[$this->lang][$input] = $this->csConv($this->getLLL($parts[1],$this->LL_files_cache[$parts[0]])); + $this->LL_labels_cache[$this->lang][$input] = $this->getLLL($parts[1],$this->LL_files_cache[$parts[0]]); } return $this->LL_labels_cache[$this->lang][$input]; } @@ -4092,7 +4090,7 @@ * @return array Returns the $LOCAL_LANG array found in the file. If no array found, returns empty array. */ function readLLfile($fileRef) { - return t3lib_div::readLLfile($fileRef,$this->lang); + return t3lib_div::readLLfile($fileRef, $this->lang, $this->renderCharset); } /** @@ -4102,7 +4100,7 @@ * @param array The locallang array in which to search * @return string Label value of $index key. */ - function getLLL($index,$LOCAL_LANG) { + function getLLL($index, &$LOCAL_LANG) { if (strcmp($LOCAL_LANG[$this->lang][$index],'')) { return $LOCAL_LANG[$this->lang][$index]; } else { @@ -4129,19 +4127,11 @@ // Setting charsets: $this->renderCharset = $this->csConvObj->parse_charset($this->config['config']['renderCharset'] ? $this->config['config']['renderCharset'] : ($this->TYPO3_CONF_VARS['BE']['forceCharset'] ? $this->TYPO3_CONF_VARS['BE']['forceCharset'] : $this->defaultCharSet)); // Rendering charset of HTML page. $this->metaCharset = $this->csConvObj->parse_charset($this->config['config']['metaCharset'] ? $this->config['config']['metaCharset'] : $this->renderCharset); // Output charset of HTML page. - $this->labelsCharset = $this->csConvObj->parse_charset($this->csConvObj->charSetArray[$this->lang] ? $this->csConvObj->charSetArray[$this->lang] : 'iso-8859-1'); - if ($this->renderCharset != $this->labelsCharset) { - $this->convCharsetToFrom = array( - 'from' => $this->labelsCharset, - 'to' => $this->renderCharset - ); - } } /** * Converts the charset of the input string if applicable. - * The "from" charset is determined by the TYPO3 system charset for the current language key ($this->lang) - * The "to" charset is determined by the currently used charset for the page which is "iso-8859-1" by default or set by $GLOBALS['TSFE']->config['config']['metaCharset'] + * The "to" charset is determined by the currently used charset for the page which is "iso-8859-1" by default or set by $GLOBALS['TSFE']->config['config']['renderCharset'] * Only if there is a difference between the two charsets will a conversion be made * The conversion is done real-time - no caching for performance at this point! * @@ -4154,8 +4144,6 @@ if ($from) { $output = $this->csConvObj->conv($str,$this->csConvObj->parse_charset($from),$this->renderCharset,1); return $output ? $output : $str; - } elseif (is_array($this->convCharsetToFrom)) { - return $this->csConvObj->conv($str,$this->convCharsetToFrom['from'],$this->convCharsetToFrom['to'],1); } else { return $str; } Index: typo3/sysext/cms/tslib/class.tslib_pibase.php =================================================================== --- typo3/sysext/cms/tslib/class.tslib_pibase.php (revision 3054) +++ typo3/sysext/cms/tslib/class.tslib_pibase.php (working copy) @@ -2,7 +2,7 @@ /*************************************************************** * Copyright notice * -* (c) 1999-2005 Kasper Skaarhoj (kasperYYYY@typo3.com) +* (c) 1999-2008 Kasper Skaarhoj (kasperYYYY@typo3.com) * All rights reserved * * This script is part of the TYPO3 project. The TYPO3 project is @@ -945,10 +945,11 @@ * @return string The value from LOCAL_LANG. */ function pi_getLL($key,$alt='',$hsc=FALSE) { + // The "from" charset of csConv() is only set for strings from TypoScript via _LOCAL_LANG if (isset($this->LOCAL_LANG[$this->LLkey][$key])) { - $word = $GLOBALS['TSFE']->csConv($this->LOCAL_LANG[$this->LLkey][$key], $this->LOCAL_LANG_charset[$this->LLkey][$key]); // The "from" charset is normally empty and thus it will convert from the charset of the system language, but if it is set (see ->pi_loadLL()) it will be used. + $word = $GLOBALS['TSFE']->csConv($this->LOCAL_LANG[$this->LLkey][$key], $this->LOCAL_LANG_charset[$this->LLkey][$key]); } elseif ($this->altLLkey && isset($this->LOCAL_LANG[$this->altLLkey][$key])) { - $word = $GLOBALS['TSFE']->csConv($this->LOCAL_LANG[$this->altLLkey][$key], $this->LOCAL_LANG_charset[$this->altLLkey][$key]); // The "from" charset is normally empty and thus it will convert from the charset of the system language, but if it is set (see ->pi_loadLL()) it will be used. + $word = $GLOBALS['TSFE']->csConv($this->LOCAL_LANG[$this->altLLkey][$key], $this->LOCAL_LANG_charset[$this->altLLkey][$key]); } elseif (isset($this->LOCAL_LANG['default'][$key])) { $word = $this->LOCAL_LANG['default'][$key]; // No charset conversion because default is english and thereby ASCII } else { @@ -971,9 +972,8 @@ if (!$this->LOCAL_LANG_loaded && $this->scriptRelPath) { $basePath = t3lib_extMgm::extPath($this->extKey).dirname($this->scriptRelPath).'/locallang.php'; - // php or xml as source: In any case the charset will be that of the system language. - // However, this function guarantees only return output for default language plus the specified language (which is different from how 3.7.0 dealt with it) - $this->LOCAL_LANG = t3lib_div::readLLfile($basePath,$this->LLkey); + // Read the strings in the required charset (since TYPO3 4.2) + $this->LOCAL_LANG = t3lib_div::readLLfile($basePath,$this->LLkey,$GLOBALS['TSFE']->renderCharset); if ($this->altLLkey) { $tempLOCAL_LANG = t3lib_div::readLLfile($basePath,$this->altLLkey); $this->LOCAL_LANG = array_merge(is_array($this->LOCAL_LANG) ? $this->LOCAL_LANG : array(),$tempLOCAL_LANG); @@ -988,9 +988,8 @@ foreach($lA as $llK => $llV) { if (!is_array($llV)) { $this->LOCAL_LANG[$k][$llK] = $llV; - if ($k != 'default') { - $this->LOCAL_LANG_charset[$k][$llK] = $GLOBALS['TYPO3_CONF_VARS']['BE']['forceCharset']; // For labels coming from the TypoScript (database) the charset is assumed to be "forceCharset" and if that is not set, assumed to be that of the individual system languages (thus no conversion) - } + // For labels coming from the TypoScript (database) the charset is assumed to be "forceCharset" and if that is not set, assumed to be that of the individual system languages + $this->LOCAL_LANG_charset[$k][$llK] = $GLOBALS['TYPO3_CONF_VARS']['BE']['forceCharset'] ? $GLOBALS['TYPO3_CONF_VARS']['BE']['forceCharset'] : $GLOBALS['TSFE']->csConvObj->charSetArray[$k]; } } }