TYPO3 Core

2008-04-01 Michael Stucki <michael@typo3.org>

* Fixed bug #7980: Fix wrong TypoScript code in plugin template

var $freqRange = 32000;

var $freqMax = 0.1;

// Objects:

/**

* Charset class object

$this->maxExternalFiles = t3lib_div::intInRange($this->indexerConfig['maxExternalFiles'],0,1000,5);

$this->flagBitMask = t3lib_div::intInRange($this->indexerConfig['flagBitMask'],0,255);

// Initialize external document parsers:

// Example configuration, see ext_localconf.php of this file!

if ($this->conf['index_externals']) {

// Initialize metaphone hook:

// Example configuration (localconf.php) for this hook: $TYPO3_CONF_VARS['EXTCONF']['indexed_search']['metaphone'] = 'EXT:indexed_search/class.doublemetaphone.php:&user_DoubleMetaPhone';

$this->metaphoneObj = &t3lib_div::getUserObj($TYPO3_CONF_VARS['EXTCONF']['indexed_search']['metaphone']);

$this->metaphoneObj->pObj = &$this;

}

$this->log_pull();

// Calculating a hash over what is to be the actual page content. Maybe this hash should not include title,description and keywords? The bodytext is the primary concern. (on the other hand a changed page-title would make no difference then, so dont!)

// This function checks if there is already a page (with gr_list = 0,-1) indexed and if that page has the very same contentHash.

// If the contentHash is the same, then we can rest assured that this page is already indexed and regardless of mtime and origContent we don't need to do anything more.

// Check words and submit to word list if not there

$this->log_push('Check word list and submit words','');

$this->log_pull();

// Set parsetime

// Check words and submit to word list if not there

$this->log_push('Check word list and submit words','');

$this->log_pull();

// Set parsetime

$this->analyzeHeaderinfo($indexArr,$content,'description',5);

$this->analyzeBody($indexArr,$content);

}

/**

* @return void

*/

function analyzeHeaderinfo(&$retArr,$content,$key,$offset) {

$retArr[$val]['cmp'] = $retArr[$val]['cmp']|pow(2,$offset);

$this->wordcount++;

}

}

* @return void

*/

function analyzeBody(&$retArr,$content) {

$retArr[$val]['first'] = $key;

}

$this->wordcount++;

}

}

if (is_object($this->metaphoneObj)) {

$tmp = $this->metaphoneObj->metaphone($word, $this->conf['sys_language_uid']);

} else {

$tmp = metaphone($word);

}

return $ret;

}

'freeIndexSetId' => intval($this->conf['freeIndexSetId']),

);

// PROCESSING index_section

$this->submit_section($this->hash['phash'],$this->hash['phash']);

// PROCESSING index_fulltext

$fields = array(

'phash' => $this->hash['phash'],

);

if ($this->indexerConfig['fullTextDataLength']>0) {

$fields['fulltextdata'] = substr($fields['fulltextdata'],0,$this->indexerConfig['fullTextDataLength']);

}

// PROCESSING index_debug

if ($this->indexerConfig['debugMode']) {

'lexer' => $this->lexerObj->debugString,

))

);

}

}

'hash_gr_list' => $this->md5inthash($this->conf['gr_list']),

'gr_list' => $this->conf['gr_list']

);

}

/**

$this->getRootLineFields($fields);

}

/**

function removeOldIndexedPages($phash) {

// Removing old registrations for all tables. Because the pages are TYPO3 pages there can be nothing else than 1-1 relations here.

$tableArr = explode(',','index_phash,index_section,index_grlist,index_fulltext,index_debug');

}

// Removing all index_section records with hash_t3 set to this hash (this includes such records set for external media on the page as well!). The re-insert of these records are done in indexRegularDocument($file).

}

'tstamp' => time(),

'crdate' => time(),

'gr_list' => $this->conf['gr_list'],

);

// PROCESSING index_fulltext

$fields = array(

'phash' => $hash['phash'],

);

if ($this->indexerConfig['fullTextDataLength']>0) {

$fields['fulltextdata'] = substr($fields['fulltextdata'],0,$this->indexerConfig['fullTextDataLength']);

}

// PROCESSING index_debug

if ($this->indexerConfig['debugMode']) {

'lexer' => $this->lexerObj->debugString,

))

);

}

}

*/

function submitFile_grlist($hash) {

// Testing if there is a gr_list record for a non-logged in user and if so, there is no need to place another one.

$this->submit_grlist($hash,$hash);

}

}

* @return void

*/

function submitFile_section($hash) {

$this->submit_section($hash,$this->hash['phash']);

}

}

* @return void

*/

function removeOldIndexedFiles($phash) {

// Removing old registrations for tables.

$tableArr = explode(',','index_phash,index_grlist,index_fulltext,index_debug');

}

}

* @return integer Result integer: Generally: <0 = No indexing, >0 = Do indexing (see $this->reasons): -2) Min age was NOT exceeded and so indexing cannot occur. -1) mtime matched so no need to reindex page. 0) N/A 1) Max age exceeded, page must be indexed again. 2) mtime of indexed page doesn't match mtime given for current content and we must index page. 3) No mtime was set, so we will index... 4) No indexed page found, so of course we will index.

*/

function checkMtimeTstamp($mtime,$phash) {

// Select indexed page:

// If there was an indexing of the page...:

if ($this->tstamp_maxAge && ($row['tstamp']+$this->tstamp_maxAge) < time()) { // If max age is exceeded, index the page

$out = 1; // The configured max-age was exceeded for the document and thus it's indexed.

} else {

*/

function checkContentHash() {

// With this query the page will only be indexed if it's content is different from the same "phash_grouping" -page.

return $row;

}

return 1;

* @return boolean Returns true if the document needs to be indexed (that is, there was no result)

*/

function checkExternalDocContentHash($hashGr,$content_md5h) {

return 0;

}

return 1;

* @return void

*/

function is_grlist_set($phash_x) {

}

/**

* @see submit_grlist()

*/

function update_grlist($phash,$phash_x) {

$this->submit_grlist($phash,$phash_x);

$this->log_setTSlogMessage("Inserted gr_list '".$this->conf['gr_list']."' for phash '".$phash."'",1);

}

);

if ($mtime) { $updateFields['item_mtime'] = intval($mtime); }

}

/**

'freeIndexSetId' => intval($this->conf['freeIndexSetId'])

);

}

/**

'parsetime' => intval($parsetime)

);

}

/**

$updateFields = array();

$this->getRootLineFields($updateFields);

}

/**

* @return void

*/

function removeLoginpagesWithContentHash() {

A.phash=B.phash

AND A.phash_grouping='.intval($this->hash['phash_grouping']).'

AND B.hash_gr_list!='.$this->md5inthash($this->defaultGrList).'

AND A.contentHash='.intval($this->content_md5h));

}

}

/**

require_once(t3lib_extMgm::extPath('crawler').'class.tx_crawler_lib.php');

}

/********************************

*

* SQL; Submitting words

}

if (count($phashArr)) {

$cwl = implode(',',$phashArr);

$this->log_setTSlogMessage('Inserting words: '.(count($wl)-$GLOBALS['TYPO3_DB']->sql_num_rows($res)),1);

unset($wl[$row['baseword']]);

}

reset($wl);

$insertFields = array(

'wid' => $val['hash'],

'baseword' => $key,

'metaphone' => $val['metaphone']

);

// A duplicate-key error will occur here if a word is NOT unset in the unset() line. However as long as the words in $wl are NOT longer as 60 chars (the baseword varchar is 60 characters...) this is not a problem.

}

}

}

* @return void

*/

function submitWords($wl,$phash) {

$insertFields = array(

'phash' => $phash,

'wid' => $val['hash'],

'flags' => ($val['cmp'] & $this->flagBitMask)

);

}

}

CREATE TABLE index_fulltext (

phash int(11) DEFAULT '0' NOT NULL,

fulltextdata mediumtext,

PRIMARY KEY (phash)

) ENGINE=InnoDB;

PRIMARY KEY (uid)

) ENGINE=InnoDB;

#

# Table structure for table 'index_stat_word'

#

word varchar(30) DEFAULT '' NOT NULL,

index_stat_search_id int(11) DEFAULT '0' NOT NULL,

tstamp int(11) DEFAULT '0' NOT NULL,

PRIMARY KEY (uid),

KEY tstamp (tstamp,word)

) ENGINE=InnoDB;

PRIMARY KEY (uid),

KEY parent (pid)

);

A full documentation manual for the indexed search extension can be found in the extension "doc_indexed_search" in the TER.

See http://typo3.org/documentation/document-library/extension-manuals/doc_indexed_search/current/view/

// TYPO3: Had to change name to "user_DoubleMetaPhone" from just "DoubleMetaPhone" because TYPO3 requires a user class to be prefixed so:

class user_DoubleMetaPhone

{

'tif' => 'EXT:indexed_search/class.external_parser.php:&tx_indexed_search_extparse',

);

// EXAMPLE configuration of hooks:

/*

$TYPO3_CONF_VARS['EXTCONF']['indexed_search']['pi1_hooks'] = array (

MAILS about:

If you are an SQL wizard, you may be able to help me here.

- index_words which contains all the words indexed

- index_pages which represents a link to a page id or external url

So searching an OR search for "content" and "management" could be done like this:

index_phash AS IP

(IW.baseword = 'content' OR IW.baseword = 'management')

[... and here comes some GROUP BY, ORDER BY and LIMIT]

Therefore I tought of a little trick to do it:

index_phash AS IP

(IW.baseword = 'content' and IW2.baseword = 'management')

[... and here comes some GROUP BY, ORDER BY and LIMIT]

Can anyone help me?

1) Select the word-ids (SQL-query 1)

2) If both words were found, make another query for all linking-table entries matching the words and group by the word-id. The count(*) statement shows the number equal to the number of searchwords if they were both found. So select only records which delivers this. Then you have the document ids.... (SQL 2)

OK, I am not an sql-wizard. Just some hints and wishes, which I hope are

helpful.

> 1) To the AND question:

> Maybe it's best to make a search for each word; After getting the total list

> of page-ids from first search, this is included as a condition in the next

> search, which generates a new list which is included in the next search,

I have to say a really elaborate, fast and multiformat (pdf's !) search

engine is really one the single most important things for every 100+

website.

> 2) Search query syntax

> Any suggestions to a search query syntax.

> - Search for "content management" is by default AND search

*****************************************************************************************************************

var $cache_rl = array(); // Caching of root line data

var $fe_groups_required = array(); // Required fe_groups memberships for display of a result.

var $domain_records = array(); // Domain records (?)

var $resultSections = array(); // Page tree sections for search result.

var $external_parsers = array(); // External parser objects

var $iconFileNameCache = array(); // Storage of icons....

/**

* Lexer object

*

*/

var $lexerObj;

/**

$this->pi_loadLL();

$this->pi_setPiVarDefaults();

// Initialize:

$this->initialize();

$this->printRules().

$content;

/**

* Initialize internal variables, especially selector box values for the search form and search words

function initialize() {

global $TYPO3_CONF_VARS;

// Initialize external document parsers for icon display and other soft operations

if (is_array($TYPO3_CONF_VARS['EXTCONF']['indexed_search']['external_parsers'])) {

foreach ($TYPO3_CONF_VARS['EXTCONF']['indexed_search']['external_parsers'] as $extension => $_objRef) {

)

);

// Free Index Uid:

if ($this->conf['search.']['defaultFreeIndexUidList']) {

$uidList = t3lib_div::intExplode(',', $this->conf['search.']['defaultFreeIndexUidList']);

// Add search languages:

$res = $GLOBALS['TYPO3_DB']->exec_SELECTquery('*', 'sys_language', '1=1'.$this->cObj->enableFields('sys_language'));

}

// Calling hook for modification of initialized content

}

/**

*

* Only words with 2 or more characters are accepted

* Max 200 chars total

* AND, OR and NOT are prefix words, overruling the default operator

* +/|/- equals AND, OR and NOT as operators.

* All search words are converted to lowercase.

$inSW = $GLOBALS['TSFE']->csConvObj->utf8_encode($inSW, $GLOBALS['TSFE']->metaCharset);

$inSW = $GLOBALS['TSFE']->csConvObj->entities_to_utf8($inSW,TRUE);

if ($hookObj = &$this->hookRequest('getSearchWords')) {

} else {

} else {

$search = t3lib_div::makeInstance('tslib_search');

$search->default_operator = $defOp==1 ? 'OR' : 'AND';

$search->register_and_explode_search_string($inSW);

if (is_array($search->sword_array)) {

}

}

}

}

/**

// Getting SQL result pointer:

$GLOBALS['TT']->push('Searching result');

$GLOBALS['TT']->pull();

// Organize and process result:

// Each row should contain the fields from 'ISEC.*, IP.*' combined + artificial fields "show_resume" (boolean) and "result_number" (counter)

while($row = $GLOBALS['TYPO3_DB']->sql_fetch_assoc($res)) {

// Set first row:

if (!$c) {

$firstRow = $row;

$c++; // Increase the result pointer

// All rows for display is put into resultRows[]

$row['result_number'] = $c;

$resultRows[] = $row;

}

} else {

$count--; // Skip this row if the user cannot view it (missing permission)

}

}

return array(

} else { // No results found:

return FALSE;

}

* @return pointer

*/

function getResultRows_SQLpointer($sWArr,$freeIndexUid=-1) {

$list = $this->getPhashList($sWArr);

// Perform SQL Search / collection of result rows array:

// Initialize variables:

$c=0;

$totalHashList = array(); // This array accumulates the phash-values

// Traverse searchwords; for each, select all phash integers and merge/diff/intersect them with previous word (based on operator)

foreach ($sWArr as $k => $v) {

// Making the query for a single search word based on the search-type

$sWord = $v['sword']; // $GLOBALS['TSFE']->csConvObj->conv_case('utf-8',$v['sword'],'toLower'); // lower-case all of them...

$theType = (string)$this->piVars['type'];

$GLOBALS['TT']->push('SearchWord "'.$sWord.'" - $theType='.$theType);

// Perform search for word:

case '1': // Part of word

break;

case '2': // First part of word

break;

case '3': // Last part of word

break;

case '10': // Sounds like

break;

case '20': // Sentence