TYPO3CMS  8
 All Classes Namespaces Files Functions Variables Pages
indexed_search/Classes/Lexer.php
Go to the documentation of this file.
1 <?php
2 namespace TYPO3\CMS\IndexedSearch;
3 
4 /*
5  * This file is part of the TYPO3 CMS project.
6  *
7  * It is free software; you can redistribute it and/or modify it under
8  * the terms of the GNU General Public License, either version 2
9  * of the License, or any later version.
10  *
11  * For the full copyright and license information, please read the
12  * LICENSE.txt file that was distributed with this source code.
13  *
14  * The TYPO3 project - inspiring people to share!
15  */
16 
21 class Lexer
22 {
28  public $debug = false;
29 
35  public $debugString = '';
36 
42  public $csObj;
43 
49  public $lexerConf = [
50  //Characters: . - _ : / '
51  'printjoins' => [46, 45, 95, 58, 47, 39],
52  'casesensitive' => false,
53  // Set, if case sensitive indexing is wanted.
54  'removeChars' => [45]
55  ];
56 
61  public function __construct()
62  {
63  $this->csObj = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Core\Charset\CharsetConverter::class);
64  }
65 
73  public function split2Words($wordString)
74  {
75  // Reset debug string:
76  $this->debugString = '';
77  // Then convert the string to lowercase:
78  if (!$this->lexerConf['casesensitive']) {
79  $wordString = mb_strtolower($wordString, 'utf-8');
80  }
81  // Now, splitting words:
82  $len = 0;
83  $start = 0;
84  $pos = 0;
85  $words = [];
86  $this->debugString = '';
87  while (1) {
88  list($start, $len) = $this->get_word($wordString, $pos);
89  if ($len) {
90  $this->addWords($words, $wordString, $start, $len);
91  if ($this->debug) {
92  $this->debugString .= '<span style="color:red">' . htmlspecialchars(substr($wordString, $pos, ($start - $pos))) . '</span>' . htmlspecialchars(substr($wordString, $start, $len));
93  }
94  $pos = $start + $len;
95  } else {
96  break;
97  }
98  }
99  return $words;
100  }
101 
102  /**********************************
103  *
104  * Helper functions
105  *
106  ********************************/
117  public function addWords(&$words, &$wordString, $start, $len)
118  {
119  // Get word out of string:
120  $theWord = substr($wordString, $start, $len);
121  // Get next chars unicode number and find type:
122  $bc = 0;
123  $cp = $this->utf8_ord($theWord, $bc);
124  list($cType) = $this->charType($cp);
125  // If string is a CJK sequence we follow this algorithm:
126  /*
127  DESCRIPTION OF (CJK) ALGORITHMContinuous letters and numbers make up words. Spaces and symbols
128  separate letters and numbers into words. This is sufficient for
129  all western text.CJK doesn't use spaces or separators to separate words, so the only
130  way to really find out what constitutes a word would be to have a
131  dictionary and advanced heuristics. Instead, we form pairs from
132  consecutive characters, in such a way that searches will find only
133  characters that appear more-or-less the right sequence. For example:ABCDE => AB BC CD DEThis works okay since both the index and the search query is split
134  in the same manner, and since the set of characters is huge so the
135  extra matches are not significant.(Hint taken from ZOPEs chinese user group)[Kasper: As far as I can see this will only work well with or-searches!]
136  */
137  if ($cType == 'cjk') {
138  // Find total string length:
139  $strlen = mb_strlen($theWord, 'utf-8');
140  // Traverse string length and add words as pairs of two chars:
141  for ($a = 0; $a < $strlen; $a++) {
142  if ($strlen == 1 || $a < $strlen - 1) {
143  $words[] = mb_substr($theWord, $a, 2, 'utf-8');
144  }
145  }
146  } else {
147  // Normal "single-byte" chars:
148  // Remove chars:
149  foreach ($this->lexerConf['removeChars'] as $skipJoin) {
150  $theWord = str_replace($this->csObj->UnumberToChar($skipJoin), '', $theWord);
151  }
152  // Add word:
153  $words[] = $theWord;
154  }
155  }
156 
164  public function get_word(&$str, $pos = 0)
165  {
166  $len = 0;
167  // If return is TRUE, a word was found starting at this position, so returning position and length:
168  if ($this->utf8_is_letter($str, $len, $pos)) {
169  return [$pos, $len];
170  }
171  // If the return value was FALSE it means a sequence of non-word chars were found (or blank string) - so we will start another search for the word:
172  $pos += $len;
173  if ($str[$pos] == '') {
174  // Check end of string before looking for word of course.
175  return false;
176  }
177  $this->utf8_is_letter($str, $len, $pos);
178  return [$pos, $len];
179  }
180 
189  public function utf8_is_letter(&$str, &$len, $pos = 0)
190  {
191  $len = 0;
192  $bc = 0;
193  $cp = 0;
194  $printJoinLgd = 0;
195  $cType = ($cType_prev = false);
196  // Letter type
197  $letter = true;
198  // looking for a letter?
199  if ($str[$pos] == '') {
200  // Return FALSE on end-of-string at this stage
201  return false;
202  }
203  while (1) {
204  // If characters has been obtained we will know whether the string starts as a sequence of letters or not:
205  if ($len) {
206  if ($letter) {
207  // We are in a sequence of words
208  if (
209  !$cType
210  || $cType_prev == 'cjk' && ($cType === 'num' || $cType === 'alpha')
211  || $cType == 'cjk' && ($cType_prev === 'num' || $cType_prev === 'alpha')
212  ) {
213  // Check if the non-letter char is NOT a print-join char because then it signifies the end of the word.
214  if (!in_array($cp, $this->lexerConf['printjoins'])) {
215  // If a printjoin start length has been recorded, set that back now so the length is right (filtering out multiple end chars)
216  if ($printJoinLgd) {
217  $len = $printJoinLgd;
218  }
219  return true;
220  } else {
221  // If a printJoin char is found, record the length if it has not been recorded already:
222  if (!$printJoinLgd) {
223  $printJoinLgd = $len;
224  }
225  }
226  } else {
227  // When a true letter is found, reset printJoinLgd counter:
228  $printJoinLgd = 0;
229  }
230  } elseif (!$letter && $cType) {
231  // end of non-word reached
232  return false;
233  }
234  }
235  $len += $bc;
236  // add byte-length of last found character
237  if ($str[$pos] == '') {
238  // End of string; return status of string till now
239  return $letter;
240  }
241  // Get next chars unicode number:
242  $cp = $this->utf8_ord($str, $bc, $pos);
243  $pos += $bc;
244  // Determine the type:
245  $cType_prev = $cType;
246  list($cType) = $this->charType($cp);
247  if ($cType) {
248  continue;
249  }
250  // Setting letter to FALSE if the first char was not a letter!
251  if (!$len) {
252  $letter = false;
253  }
254  }
255  return false;
256  }
257 
264  public function charType($cp)
265  {
266  // Numeric?
267  if ($cp >= 48 && $cp <= 57) {
268  return ['num'];
269  }
270  // LOOKING for Alpha chars (Latin, Cyrillic, Greek, Hebrew and Arabic):
271  if ($cp >= 65 && $cp <= 90 || $cp >= 97 && $cp <= 122 || $cp >= 192 && $cp <= 255 && $cp != 215 && $cp != 247 || $cp >= 256 && $cp < 640 || ($cp == 902 || $cp >= 904 && $cp < 1024) || ($cp >= 1024 && $cp < 1154 || $cp >= 1162 && $cp < 1328) || ($cp >= 1424 && $cp < 1456 || $cp >= 1488 && $cp < 1523) || ($cp >= 1569 && $cp <= 1624 || $cp >= 1646 && $cp <= 1747) || $cp >= 7680 && $cp < 8192) {
272  return ['alpha'];
273  }
274  // Looking for CJK (Chinese / Japanese / Korean)
275  // Ranges are not certain - deducted from the translation tables in typo3/sysext/core/Resources/Private/Charsets/csconvtbl/
276  // Verified with http://www.unicode.org/charts/ (16/2) - may still not be complete.
277  if ($cp >= 12352 && $cp <= 12543 || $cp >= 12592 && $cp <= 12687 || $cp >= 13312 && $cp <= 19903 || $cp >= 19968 && $cp <= 40879 || $cp >= 44032 && $cp <= 55215 || $cp >= 131072 && $cp <= 195103) {
278  return ['cjk'];
279  }
280  }
281 
291  public function utf8_ord(&$str, &$len, $pos = 0, $hex = false)
292  {
293  $ord = ord($str[$pos]);
294  $len = 1;
295  if ($ord > 128) {
296  for ($bc = -1, $mbs = $ord; $mbs & 128; $mbs = $mbs << 1) {
297  // calculate number of extra bytes
298  $bc++;
299  }
300  $len += $bc;
301  $ord = $ord & (1 << 6 - $bc) - 1;
302  // mask utf-8 lead-in bytes
303  // "bring in" data bytes
304  for ($i = $pos + 1; $bc; $bc--, $i++) {
305  $ord = $ord << 6 | ord($str[$i]) & 63;
306  }
307  }
308  return $hex ? 'x' . dechex($ord) : $ord;
309  }
310 }
addWords(&$words, &$wordString, $start, $len)
static makeInstance($className,...$constructorArguments)
debug($variable= '', $name= '*variable *', $line= '*line *', $file= '*file *', $recursiveDepth=3, $debugLevel=E_DEBUG)
utf8_ord(&$str, &$len, $pos=0, $hex=false)