Indexer.php

Go to the documentation of this file.

<?php
namespace ‪TYPO3\CMS\IndexedSearch;
 
/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */
 
use Psr\Http\Message\ServerRequestInterface;
use ‪TYPO3\CMS\Core\Compatibility\PublicPropertyDeprecationTrait;
use ‪TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
use ‪TYPO3\CMS\Core\Context\Context;
use ‪TYPO3\CMS\Core\Context\LanguageAspect;
use ‪TYPO3\CMS\Core\Core\Environment;
use ‪TYPO3\CMS\Core\Database\Connection;
use ‪TYPO3\CMS\Core\Database\ConnectionPool;
use ‪TYPO3\CMS\Core\Routing\PageArguments;
use ‪TYPO3\CMS\Core\TimeTracker\TimeTracker;
use ‪TYPO3\CMS\Core\Utility\GeneralUtility;
use ‪TYPO3\CMS\Core\Utility\HttpUtility;
use ‪TYPO3\CMS\Core\Utility\MathUtility;
use ‪TYPO3\CMS\Core\Utility\PathUtility;
use ‪TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
use ‪TYPO3\CMS\IndexedSearch\Utility\IndexedSearchUtility;
 
class ‪Indexer
{
    use ‪PublicPropertyDeprecationTrait;
 
    protected ‪$deprecatedPublicProperties = [
        'csObj' => 'Using $csObj within Indexing is discouraged, the property will be removed in TYPO3 v10.0 - if needed instantiate CharsetConverter yourself.',
    ];
 
    public ‪$reasons = [
        -1 => 'mtime matched the document, so no changes detected and no content updated',
        -2 => 'The minimum age was not exceeded',
        1 => 'The configured max-age was exceeded for the document and thus it\'s indexed.',
        2 => 'The minimum age was exceed and mtime was set and the mtime was different, so the page was indexed.',
        3 => 'The minimum age was exceed, but mtime was not set, so the page was indexed.',
        4 => 'Page has never been indexed (is not represented in the index_phash table).'
    ];
 
    public ‪$excludeSections = 'script,style';
 
    public ‪$external_parsers = [];
 
    public ‪$defaultGrList = '0,-1';
 
    public ‪$tstamp_maxAge = 0;
 
    public ‪$tstamp_minAge = 0;
 
    public ‪$maxExternalFiles = 0;
 
    public ‪$forceIndexing = false;
 
    public ‪$crawlerActive = false;
 
    public ‪$defaultContentArray = [
        'title' => '',
        'description' => '',
        'keywords' => '',
        'body' => ''
    ];
 
    public ‪$wordcount = 0;
 
    public ‪$externalFileCounter = 0;
 
    public ‪$conf = [];
 
    public ‪$indexerConfig = [];
 
    public ‪$hash = [];
 
    public ‪$file_phash_arr = [];
 
    public ‪$contentParts = [];
 
    public ‪$content_md5h = '';
 
    public ‪$internal_log = [];
 
    public ‪$indexExternalUrl_content = '';
 
    public ‪$cHashParams = [];
 
    public ‪$freqRange = 32000;
 
    public ‪$freqMax = 0.1;
 
    public ‪$enableMetaphoneSearch = false;
 
    public ‪$storeMetaphoneInfoAsWords;
 
    public ‪$metaphoneContent = '';
 
    protected ‪$csObj;
 
    public ‪$metaphoneObj;
 
    public ‪$lexerObj;
 
    public ‪$flagBitMask;
 
    protected ‪$timeTracker;
 
    public function ‪__construct()
    {
        $this->timeTracker = GeneralUtility::makeInstance(TimeTracker::class);
    }
 
    public function ‪hook_indexContent(&$pObj)
    {
        // Indexer configuration from Extension Manager interface:
        $disableFrontendIndexing = (bool)GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('indexed_search', 'disableFrontendIndexing');
        // Crawler activation:
        // Requirements are that the crawler is loaded, a crawler session is running and re-indexing requested as processing instruction:
        if (\‪TYPO3\CMS\Core\Utility\ExtensionManagementUtility::isLoaded('crawler') && $pObj->applicationData['tx_crawler']['running'] && in_array('tx_indexedsearch_reindex', $pObj->applicationData['tx_crawler']['parameters']['procInstructions'])) {
            // Setting simple log message:
            $pObj->applicationData['tx_crawler']['log'][] = 'Forced Re-indexing enabled';
            // Setting variables:
            $this->crawlerActive = true;
            // Crawler active flag
            $this->forceIndexing = true;
        }
        // Determine if page should be indexed, and if so, configure and initialize indexer
        if ($pObj->config['config']['index_enable']) {
            $this->‪log_push('Index page', '');
            if (!$disableFrontendIndexing || $this->crawlerActive) {
                if (!$pObj->page['no_search']) {
                    if (!$pObj->no_cache) {
                        $languageAspect = GeneralUtility::makeInstance(Context::class)->getAspect('language');
                        if ($languageAspect->getId() === $languageAspect->getContentId()) {
                            // Setting up internal configuration from config array:
                            $this->conf = [];
                            // Information about page for which the indexing takes place
                            $this->conf['id'] = $pObj->id;
                            // Page id
                            $this->conf['type'] = $pObj->type;
                            // Page type
                            $this->conf['sys_language_uid'] = $languageAspect->getId();
                            // sys_language UID of the language of the indexing.
                            $this->conf['MP'] = $pObj->MP;
                            // MP variable, if any (Mount Points)
                            // Group list
                            $this->conf['gr_list'] = implode(',', GeneralUtility::makeInstance(Context::class)->getPropertyFromAspect('frontend.user', 'groupIds', [0, -1]));
                            // cHash string for additional parameters
                            $this->conf['cHash'] = $pObj->cHash;
                            // cHash array with additional parameters
                            $this->conf['cHash_array'] = $pObj->cHash_array;
                            // page arguments array
                            $this->conf['staticPageArguments'] = [];
                            if (‪$GLOBALS['TYPO3_REQUEST'] instanceof ServerRequestInterface) {
                                $pageArguments = ‪$GLOBALS['TYPO3_REQUEST']->getAttribute('routing', null);
                                if ($pageArguments instanceof PageArguments) {
                                    $this->conf['staticPageArguments'] = $pageArguments->getStaticArguments();
                                }
                            }
                            // Array of the additional parameters
                            $this->conf['crdate'] = $pObj->page['crdate'];
                            // The creation date of the TYPO3 page
 
                            // reg1 of the caching table. Not known what practical use this has.
                            // @deprecated since TYPO3 v9, will be removed in TYPO3 v10.0. Remove along with database field data_page_reg1
                            $this->conf['page_cache_reg1'] = $pObj->page_cache_reg1;
 
                            // Root line uids
                            $this->conf['rootline_uids'] = [];
                            foreach ($pObj->config['rootLine'] as $rlkey => $rldat) {
                                $this->conf['rootline_uids'][$rlkey] = $rldat['uid'];
                            }
                            // Content of page:
                            $this->conf['content'] = $pObj->content;
                            // Content string (HTML of TYPO3 page)
                            $this->conf['indexedDocTitle'] = $pObj->convOutputCharset($pObj->indexedDocTitle);
                            // Alternative title for indexing
                            $this->conf['metaCharset'] = $pObj->metaCharset;
                            // Character set of content (will be converted to utf-8 during indexing)
                            $this->conf['mtime'] = $pObj->register['SYS_LASTCHANGED'] ?? $pObj->page['SYS_LASTCHANGED'];
                            // Most recent modification time (seconds) of the content on the page. Used to evaluate whether it should be re-indexed.
                            // Configuration of behavior:
                            $this->conf['index_externals'] = $pObj->config['config']['index_externals'];
                            // Whether to index external documents like PDF, DOC etc. (if possible)
                            $this->conf['index_descrLgd'] = $pObj->config['config']['index_descrLgd'];
                            // Length of description text (max 250, default 200)
                            $this->conf['index_metatags'] = $pObj->config['config']['index_metatags'] ?? true;
                            // Set to zero:
                            $this->conf['recordUid'] = 0;
                            $this->conf['freeIndexUid'] = 0;
                            $this->conf['freeIndexSetId'] = 0;
                            // Init and start indexing:
                            $this->‪init();
                            $this->‪indexTypo3PageContent();
                        } else {
                            $this->‪log_setTSlogMessage('Index page? No, languageId was different from contentId which indicates that the page contains fall-back content and that would be falsely indexed as localized content.');
                        }
                    } else {
                        $this->‪log_setTSlogMessage('Index page? No, page was set to "no_cache" and so cannot be indexed.');
                    }
                } else {
                    $this->‪log_setTSlogMessage('Index page? No, The "No Search" flag has been set in the page properties!');
                }
            } else {
                $this->‪log_setTSlogMessage('Index page? No, Ordinary Frontend indexing during rendering is disabled.');
            }
            $this->‪log_pull();
        }
    }
 
    /****************************
     *
     * Backend API
     *
     ****************************/
    public function ‪backend_initIndexer($id, $type, $sys_language_uid, $MP, $uidRL, $cHash_array = [], $createCHash = false)
    {
        // Setting up internal configuration from config array:
        $this->conf = [];
        // Information about page for which the indexing takes place
        $this->conf['id'] = $id;
        // Page id  (int)
        $this->conf['type'] = $type;
        // Page type (int)
        $this->conf['sys_language_uid'] = $sys_language_uid;
        // sys_language UID of the language of the indexing (int)
        $this->conf['MP'] = $MP;
        // MP variable, if any (Mount Points) (string)
        $this->conf['gr_list'] = '0,-1';
        // Group list (hardcoded for now...)
        // cHash values:
        if ($createCHash) {
            $cHash_array['id'] = $id;
            /* @var \TYPO3\CMS\Frontend\Page\CacheHashCalculator $cacheHash */
            $cacheHash = GeneralUtility::makeInstance(\‪TYPO3\CMS\Frontend\Page\CacheHashCalculator::class);
            $this->conf['cHash'] = $cacheHash->generateForParameters(‪HttpUtility::buildQueryString($cHash_array));
        } else {
            $this->conf['cHash'] = '';
        }
        // cHash string for additional parameters
        $this->conf['cHash_array'] = $cHash_array;
        // Array of the additional parameters
        // Set to defaults
        $this->conf['freeIndexUid'] = 0;
        $this->conf['freeIndexSetId'] = 0;
 
        // @deprecated since TYPO3 v9, will be removed in TYPO3 v10.0. Remove along with database field data_page_reg1
        $this->conf['page_cache_reg1'] = 0;
 
        // Root line uids
        $this->conf['rootline_uids'] = $uidRL;
        // Configuration of behavior:
        $this->conf['index_externals'] = 1;
        // Whether to index external documents like PDF, DOC etc. (if possible)
        $this->conf['index_descrLgd'] = 200;
        // Length of description text (max 250, default 200)
        $this->conf['index_metatags'] = true;
        // Whether to index document keywords and description (if present)
        // Init and start indexing:
        $this->‪init();
    }
 
    public function ‪backend_setFreeIndexUid($freeIndexUid, $freeIndexSetId = 0)
    {
        $this->conf['freeIndexUid'] = $freeIndexUid;
        $this->conf['freeIndexSetId'] = $freeIndexSetId;
    }
 
    public function ‪backend_indexAsTYPO3Page($title, $keywords, $description, $content, $charset, $mtime, $crdate = 0, $recordUid = 0)
    {
        // Content of page:
        $this->conf['mtime'] = $mtime;
        // Most recent modification time (seconds) of the content
        $this->conf['crdate'] = $crdate;
        // The creation date of the TYPO3 content
        $this->conf['recordUid'] = $recordUid;
        // UID of the record, if applicable
        // Construct fake HTML for parsing:
        $this->conf['content'] = '
        <html>
            <head>
                <title>' . htmlspecialchars($title) . '</title>
                <meta name="keywords" content="' . htmlspecialchars($keywords) . '" />
                <meta name="description" content="' . htmlspecialchars($description) . '" />
            </head>
            <body>
                ' . htmlspecialchars($content) . '
            </body>
        </html>';
        // Content string (HTML of TYPO3 page)
        // Initializing charset:
        $this->conf['metaCharset'] = $charset;
        // Character set of content (will be converted to utf-8 during indexing)
        $this->conf['indexedDocTitle'] = '';
        // Alternative title for indexing
        // Index content as if it was a TYPO3 page:
        $this->‪indexTypo3PageContent();
    }
 
    /********************************
     *
     * Initialization
     *
     *******************************/
    public function ‪init()
    {
        // Initializing:
        $this->cHashParams = $this->conf['cHash_array'];
        if (is_array($this->cHashParams) && !empty($this->cHashParams)) {
            if ($this->conf['cHash']) {
                // Add this so that URL's come out right...
                $this->cHashParams['cHash'] = $this->conf['cHash'];
            }
            unset($this->cHashParams['encryptionKey']);
        }
        // Setting phash / phash_grouping which identifies the indexed page based on some of these variables:
        $this->‪setT3Hashes();
        // Indexer configuration from Extension Manager interface:
        $this->indexerConfig = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('indexed_search');
        $this->tstamp_minAge = ‪MathUtility::forceIntegerInRange((int)($this->indexerConfig['minAge'] ?? 0) * 3600, 0);
        $this->tstamp_maxAge = ‪MathUtility::forceIntegerInRange((int)($this->indexerConfig['maxAge'] ?? 0) * 3600, 0);
        $this->maxExternalFiles = ‪MathUtility::forceIntegerInRange($this->indexerConfig['maxExternalFiles'], 0, 1000, 5);
        $this->flagBitMask = ‪MathUtility::forceIntegerInRange($this->indexerConfig['flagBitMask'], 0, 255);
        // Workaround: If the extension configuration was not updated yet, the value is not existing
        $this->enableMetaphoneSearch = !isset($this->indexerConfig['enableMetaphoneSearch']) || $this->indexerConfig['enableMetaphoneSearch'];
        $this->storeMetaphoneInfoAsWords = !‪IndexedSearchUtility::isTableUsed('index_words') && ‪$this->enableMetaphoneSearch;
        // Initialize external document parsers:
        // Example configuration, see ext_localconf.php of this file!
        if ($this->conf['index_externals']) {
            $this->‪initializeExternalParsers();
        }
        // Initialize lexer (class that deconstructs the text into words):
        $lexerObjectClassName = ‪$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['lexer'] ?: Lexer::class;
        $this->lexerObj = GeneralUtility::makeInstance($lexerObjectClassName);
        $this->lexerObj->debug = $this->indexerConfig['debugMode'];
        // Initialize metaphone hook:
        // Make sure that the hook is loaded _after_ indexed_search as this may overwrite the hook depending on the configuration.
        if ($this->enableMetaphoneSearch && ‪$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['metaphone']) {
            $this->metaphoneObj = GeneralUtility::makeInstance(‪$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['metaphone']);
            $this->metaphoneObj->pObj = $this;
        }
        // Init charset class:
        $this->csObj = GeneralUtility::makeInstance(\‪TYPO3\CMS\Core\Charset\CharsetConverter::class);
    }
 
    public function ‪initializeExternalParsers()
    {
        foreach (‪$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['external_parsers'] ?? [] as $extension => $className) {
            $this->external_parsers[$extension] = GeneralUtility::makeInstance($className);
            $this->external_parsers[$extension]->pObj = $this;
            // Init parser and if it returns FALSE, unset its entry again:
            if (!$this->external_parsers[$extension]->initParser($extension)) {
                unset($this->external_parsers[$extension]);
            }
        }
    }
 
    /********************************
     *
     * Indexing; TYPO3 pages (HTML content)
     *
     *******************************/
    public function ‪indexTypo3PageContent()
    {
        $check = $this->‪checkMtimeTstamp($this->conf['mtime'], $this->hash['phash']);
        $is_grlist = $this->‪is_grlist_set($this->hash['phash']);
        if ($check > 0 || !$is_grlist || $this->forceIndexing) {
            // Setting message:
            if ($this->forceIndexing) {
                $this->‪log_setTSlogMessage('Indexing needed, reason: Forced', 1);
            } elseif ($check > 0) {
                $this->‪log_setTSlogMessage('Indexing needed, reason: ' . $this->reasons[$check], 1);
            } else {
                $this->‪log_setTSlogMessage('Indexing needed, reason: Updates gr_list!', 1);
            }
            // Divide into title,keywords,description and body:
            $this->‪log_push('Split content', '');
            $this->contentParts = $this->‪splitHTMLContent($this->conf['content']);
            if ($this->conf['indexedDocTitle']) {
                $this->contentParts['title'] = $this->conf['indexedDocTitle'];
            }
            $this->‪log_pull();
            // Calculating a hash over what is to be the actual page content. Maybe this hash should not include title,description and keywords? The bodytext is the primary concern. (on the other hand a changed page-title would make no difference then, so don't!)
            $this->content_md5h = ‪IndexedSearchUtility::md5inthash(implode('', $this->contentParts));
            // This function checks if there is already a page (with gr_list = 0,-1) indexed and if that page has the very same contentHash.
            // If the contentHash is the same, then we can rest assured that this page is already indexed and regardless of mtime and origContent we don't need to do anything more.
            // This will also prevent pages from being indexed if a fe_users has logged in and it turns out that the page content is not changed anyway. fe_users logged in should always search with hash_gr_list = "0,-1" OR "[their_group_list]". This situation will be prevented only if the page has been indexed with no user login on before hand. Else the page will be indexed by users until that event. However that does not present a serious problem.
            $checkCHash = $this->‪checkContentHash();
            if (!is_array($checkCHash) || $check === 1) {
                $Pstart = GeneralUtility::milliseconds();
                $this->‪log_push('Converting charset of content (' . $this->conf['metaCharset'] . ') to utf-8', '');
                $this->‪charsetEntity2utf8($this->contentParts, $this->conf['metaCharset']);
                $this->‪log_pull();
                // Splitting words
                $this->‪log_push('Extract words from content', '');
                $splitInWords = $this->‪processWordsInArrays($this->contentParts);
                $this->‪log_pull();
                // Analyze the indexed words.
                $this->‪log_push('Analyze the extracted words', '');
                $indexArr = $this->‪indexAnalyze($splitInWords);
                $this->‪log_pull();
                // Submitting page (phash) record
                $this->‪log_push('Submitting page', '');
                $this->‪submitPage();
                $this->‪log_pull();
                // Check words and submit to word list if not there
                $this->‪log_push('Check word list and submit words', '');
                if (‪IndexedSearchUtility::isTableUsed('index_words')) {
                    $this->‪checkWordList($indexArr);
                    $this->‪submitWords($indexArr, $this->hash['phash']);
                }
                $this->‪log_pull();
                // Set parsetime
                $this->‪updateParsetime($this->hash['phash'], GeneralUtility::milliseconds() - $Pstart);
                // Checking external files if configured for.
                $this->‪log_push('Checking external files', '');
                if ($this->conf['index_externals']) {
                    $this->‪extractLinks($this->conf['content']);
                }
                $this->‪log_pull();
            } else {
                // Update the timestamp
                $this->‪updateTstamp($this->hash['phash'], $this->conf['mtime']);
                $this->‪updateSetId($this->hash['phash']);
                // $checkCHash['phash'] is the phash of the result row that is similar to the current phash regarding the content hash.
                $this->‪update_grlist($checkCHash['phash'], $this->hash['phash']);
                $this->‪updateRootline();
                $this->‪log_setTSlogMessage('Indexing not needed, the contentHash, ' . $this->content_md5h . ', has not changed. Timestamp, grlist and rootline updated if necessary.');
            }
        } else {
            $this->‪log_setTSlogMessage('Indexing not needed, reason: ' . $this->reasons[$check]);
        }
    }
 
    public function ‪splitHTMLContent($content)
    {
        // divide head from body ( u-ouh :) )
        $contentArr = ‪$this->defaultContentArray;
        $contentArr['body'] = stristr($content, '<body');
        $headPart = substr($content, 0, -strlen($contentArr['body']));
        // get title
        $this->‪embracingTags($headPart, 'TITLE', $contentArr['title'], $dummy2, $dummy);
        $titleParts = explode(':', $contentArr['title'], 2);
        $contentArr['title'] = trim($titleParts[1] ?? $titleParts[0]);
        // get keywords and description metatags
        if ($this->conf['index_metatags']) {
            $meta = [];
            $i = 0;
            while ($this->‪embracingTags($headPart, 'meta', $dummy, $headPart, $meta[$i])) {
                $i++;
            }
            // @todo The code below stops at first unset tag. Is that correct?
            for ($i = 0; isset($meta[$i]); $i++) {
                // decode HTML entities, meta tag content needs to be encoded later
                $meta[$i] = GeneralUtility::get_tag_attributes($meta[$i], true);
                if (stristr($meta[$i]['name'], 'keywords')) {
                    $contentArr['keywords'] .= ',' . $this->‪addSpacesToKeywordList($meta[$i]['content']);
                }
                if (stristr($meta[$i]['name'], 'description')) {
                    $contentArr['description'] .= ',' . $meta[$i]['content'];
                }
            }
        }
        // Process <!--TYPO3SEARCH_begin--> or <!--TYPO3SEARCH_end--> tags:
        $this->‪typoSearchTags($contentArr['body']);
        // Get rid of unwanted sections (ie. scripting and style stuff) in body
        $tagList = explode(',', $this->excludeSections);
        foreach ($tagList as $tag) {
            while ($this->‪embracingTags($contentArr['body'], $tag, $dummy, $contentArr['body'], $dummy2)) {
            }
        }
        // remove tags, but first make sure we don't concatenate words by doing it
        $contentArr['body'] = str_replace('<', ' <', $contentArr['body']);
        $contentArr['body'] = trim(strip_tags($contentArr['body']));
        $contentArr['keywords'] = trim($contentArr['keywords']);
        $contentArr['description'] = trim($contentArr['description']);
        // Return array
        return $contentArr;
    }
 
    public function ‪getHTMLcharset($content)
    {
        if (preg_match('/<meta[[:space:]]+[^>]*http-equiv[[:space:]]*=[[:space:]]*["\']CONTENT-TYPE["\'][^>]*>/i', $content, $reg)) {
            if (preg_match('/charset[[:space:]]*=[[:space:]]*([[:alnum:]-]+)/i', $reg[0], $reg2)) {
                return $reg2[1];
            }
        }
    }
 
    public function ‪convertHTMLToUtf8($content, $charset = '')
    {
        // Find charset:
        $charset = $charset ?: $this->‪getHTMLcharset($content);
        $charset = trim(strtolower($charset));
        // Convert charset:
        if ($charset && $charset !== 'utf-8') {
            $content = mb_convert_encoding($content, 'utf-8', $charset);
        }
        // Convert entities, assuming document is now UTF-8:
        return html_entity_decode($content);
    }
 
    public function ‪embracingTags($string, $tagName, &$tagContent, &$stringAfter, &$paramList)
    {
        $endTag = '</' . $tagName . '>';
        $startTag = '<' . $tagName;
        // stristr used because we want a case-insensitive search for the tag.
        $isTagInText = stristr($string, $startTag);
        // if the tag was not found, return FALSE
        if (!$isTagInText) {
            return false;
        }
        list($paramList, $isTagInText) = explode('>', substr($isTagInText, strlen($startTag)), 2);
        $afterTagInText = stristr($isTagInText, $endTag);
        if ($afterTagInText) {
            $stringBefore = substr($string, 0, strpos(strtolower($string), strtolower($startTag)));
            $tagContent = substr($isTagInText, 0, strlen($isTagInText) - strlen($afterTagInText));
            $stringAfter = $stringBefore . substr($afterTagInText, strlen($endTag));
        } else {
            $tagContent = '';
            $stringAfter = $isTagInText;
        }
        return true;
    }
 
    public function ‪typoSearchTags(&$body)
    {
        $expBody = preg_split('/\\<\\!\\-\\-[\\s]?TYPO3SEARCH_/', $body);
        if (count($expBody) > 1) {
            $body = '';
            foreach ($expBody as $val) {
                $part = explode('-->', $val, 2);
                if (trim($part[0]) === 'begin') {
                    $body .= $part[1];
                    $prev = '';
                } elseif (trim($part[0]) === 'end') {
                    $body .= $prev;
                } else {
                    $prev = $val;
                }
            }
            return true;
        }
        return false;
    }
 
    public function ‪extractLinks($content)
    {
        // Get links:
        $list = $this->‪extractHyperLinks($content);
        if ($this->indexerConfig['useCrawlerForExternalFiles'] && \‪TYPO3\CMS\Core\Utility\ExtensionManagementUtility::isLoaded('crawler')) {
            $crawler = GeneralUtility::makeInstance(\tx_crawler_lib::class);
        }
        // Traverse links:
        foreach ($list as $linkInfo) {
            // Decode entities:
            if ($linkInfo['localPath']) {
                // localPath means: This file is sent by a download script. While the indexed URL has to point to $linkInfo['href'], the absolute path to the file is specified here!
                $linkSource = htmlspecialchars_decode($linkInfo['localPath']);
            } else {
                $linkSource = htmlspecialchars_decode($linkInfo['href']);
            }
            // Parse URL:
            $qParts = parse_url($linkSource);
            // Check for jumpurl (TYPO3 specific thing...)
            if ($qParts['query'] && strstr($qParts['query'], 'jumpurl=')) {
                parse_str($qParts['query'], $getP);
                $linkSource = $getP['jumpurl'];
                $qParts = parse_url($linkSource);
            }
            if (!$linkInfo['localPath'] && $qParts['scheme']) {
                if ($this->indexerConfig['indexExternalURLs']) {
                    // Index external URL (http or otherwise)
                    $this->‪indexExternalUrl($linkSource);
                }
            } elseif (!$qParts['query']) {
                $linkSource = urldecode($linkSource);
                if (GeneralUtility::isAllowedAbsPath($linkSource)) {
                    $localFile = $linkSource;
                } else {
                    $localFile = GeneralUtility::getFileAbsFileName(‪Environment::getPublicPath() . '/' . $linkSource);
                }
                if ($localFile && @is_file($localFile)) {
                    // Index local file:
                    if ($linkInfo['localPath']) {
                        $fI = pathinfo($linkSource);
                        $ext = strtolower($fI['extension']);
                        if (is_object($crawler)) {
                            $params = [
                                'document' => $linkSource,
                                'alturl' => $linkInfo['href'],
                                'conf' => ‪$this->conf
                            ];
                            unset($params['conf']['content']);
                            $crawler->addQueueEntry_callBack(0, $params, Hook\CrawlerFilesHook::class, $this->conf['id']);
                            $this->‪log_setTSlogMessage('media "' . $params['document'] . '" added to "crawler" queue.', 1);
                        } else {
                            $this->‪indexRegularDocument($linkInfo['href'], false, $linkSource, $ext);
                        }
                    } else {
                        if (is_object($crawler)) {
                            $params = [
                                'document' => $linkSource,
                                'conf' => ‪$this->conf
                            ];
                            unset($params['conf']['content']);
                            $crawler->addQueueEntry_callBack(0, $params, Hook\CrawlerFilesHook::class, $this->conf['id']);
                            $this->‪log_setTSlogMessage('media "' . $params['document'] . '" added to "crawler" queue.', 1);
                        } else {
                            $this->‪indexRegularDocument($linkSource);
                        }
                    }
                }
            }
        }
    }
 
    public function ‪extractHyperLinks($html)
    {
        $htmlParser = GeneralUtility::makeInstance(\‪TYPO3\CMS\Core\Html\HtmlParser::class);
        $htmlParts = $htmlParser->splitTags('a', $html);
        $hyperLinksData = [];
        foreach ($htmlParts as $index => $tagData) {
            if ($index % 2 !== 0) {
                $tagAttributes = $htmlParser->get_tag_attributes($tagData, true);
                $firstTagName = $htmlParser->getFirstTagName($tagData);
                if (strtolower($firstTagName) === 'a') {
                    if ($tagAttributes[0]['href'] && $tagAttributes[0]['href'][0] !== '#') {
                        $hyperLinksData[] = [
                            'tag' => $tagData,
                            'href' => $tagAttributes[0]['href'],
                            'localPath' => $this->‪createLocalPath(urldecode($tagAttributes[0]['href']))
                        ];
                    }
                }
            }
        }
        return $hyperLinksData;
    }
 
    public function ‪extractBaseHref($html)
    {
        $href = '';
        $htmlParser = GeneralUtility::makeInstance(\‪TYPO3\CMS\Core\Html\HtmlParser::class);
        $htmlParts = $htmlParser->splitTags('base', $html);
        foreach ($htmlParts as $index => $tagData) {
            if ($index % 2 !== 0) {
                $tagAttributes = $htmlParser->get_tag_attributes($tagData, true);
                $firstTagName = $htmlParser->getFirstTagName($tagData);
                if (strtolower($firstTagName) === 'base') {
                    $href = $tagAttributes[0]['href'];
                    if ($href) {
                        break;
                    }
                }
            }
        }
        return $href;
    }
 
    /******************************************
     *
     * Indexing; external URL
     *
     ******************************************/
    public function ‪indexExternalUrl($externalUrl)
    {
        // Get headers:
        $urlHeaders = $this->‪getUrlHeaders($externalUrl);
        if (stristr($urlHeaders['Content-Type'], 'text/html')) {
            $content = ($this->indexExternalUrl_content = GeneralUtility::getUrl($externalUrl));
            if ((string)$content !== '') {
                // Create temporary file:
                $tmpFile = GeneralUtility::tempnam('EXTERNAL_URL');
                if ($tmpFile) {
                    GeneralUtility::writeFile($tmpFile, $content);
                    // Index that file:
                    $this->‪indexRegularDocument($externalUrl, true, $tmpFile, 'html');
                    // Using "TRUE" for second parameter to force indexing of external URLs (mtime doesn't make sense, does it?)
                    unlink($tmpFile);
                }
            }
        }
    }
 
    public function ‪getUrlHeaders($url)
    {
        // Try to get the headers only
        $content = GeneralUtility::getUrl($url, 2);
        if ((string)$content !== '') {
            // Compile headers:
            $headers = GeneralUtility::trimExplode(LF, $content, true);
            $retVal = [];
            foreach ($headers as $line) {
                if (trim($line) === '') {
                    break;
                }
                list($headKey, $headValue) = explode(':', $line, 2);
                $retVal[$headKey] = $headValue;
            }
            return $retVal;
        }
    }
 
    protected function ‪createLocalPath($sourcePath)
    {
        $localPath = '';
        $pathFunctions = [
            'createLocalPathFromT3vars',
            'createLocalPathUsingAbsRefPrefix',
            'createLocalPathUsingDomainURL',
            'createLocalPathFromAbsoluteURL',
            'createLocalPathFromRelativeURL'
        ];
        foreach ($pathFunctions as $functionName) {
            $localPath = $this->{$functionName}($sourcePath);
            if ($localPath != '') {
                break;
            }
        }
        return $localPath;
    }
 
    protected function ‪createLocalPathFromT3vars($sourcePath)
    {
        $localPath = '';
        $indexLocalFiles = ‪$GLOBALS['T3_VAR']['ext']['indexed_search']['indexLocalFiles'] ?? null;
        if (is_array($indexLocalFiles)) {
            $md5 = GeneralUtility::shortMD5($sourcePath);
            // Note: not using self::isAllowedLocalFile here because this method
            // is allowed to index files outside of the web site (for example,
            // protected downloads)
            if (isset($indexLocalFiles[$md5]) && is_file($indexLocalFiles[$md5])) {
                $localPath = $indexLocalFiles[$md5];
            }
        }
        return $localPath;
    }
 
    protected function ‪createLocalPathUsingDomainURL($sourcePath)
    {
        $localPath = '';
        $baseURL = GeneralUtility::getIndpEnv('TYPO3_SITE_URL');
        $baseURLLength = strlen($baseURL);
        if (strpos($sourcePath, $baseURL) === 0) {
            $sourcePath = substr($sourcePath, $baseURLLength);
            $localPath = ‪Environment::getPublicPath() . '/' . $sourcePath;
            if (!self::isAllowedLocalFile($localPath)) {
                $localPath = '';
            }
        }
        return $localPath;
    }
 
    protected function ‪createLocalPathUsingAbsRefPrefix($sourcePath)
    {
        $localPath = '';
        if (isset(‪$GLOBALS['TSFE']) && ‪$GLOBALS['TSFE'] instanceof ‪TypoScriptFrontendController) {
            $absRefPrefix = ‪$GLOBALS['TSFE']->config['config']['absRefPrefix'];
            $absRefPrefixLength = strlen($absRefPrefix);
            if ($absRefPrefixLength > 0 && strpos($sourcePath, $absRefPrefix) === 0) {
                $sourcePath = substr($sourcePath, $absRefPrefixLength);
                $localPath = ‪Environment::getPublicPath() . '/' . $sourcePath;
                if (!self::isAllowedLocalFile($localPath)) {
                    $localPath = '';
                }
            }
        }
        return $localPath;
    }
 
    protected function ‪createLocalPathFromAbsoluteURL($sourcePath)
    {
        $localPath = '';
        if ($sourcePath[0] === '/') {
            $sourcePath = substr($sourcePath, 1);
            $localPath = ‪Environment::getPublicPath() . '/' . $sourcePath;
            if (!self::isAllowedLocalFile($localPath)) {
                $localPath = '';
            }
        }
        return $localPath;
    }
 
    protected function ‪createLocalPathFromRelativeURL($sourcePath)
    {
        $localPath = '';
        if (self::isRelativeURL($sourcePath)) {
            $localPath = ‪Environment::getPublicPath() . '/' . $sourcePath;
            if (!self::isAllowedLocalFile($localPath)) {
                $localPath = '';
            }
        }
        return $localPath;
    }
 
    protected static function ‪isRelativeURL($url)
    {
        $urlParts = @parse_url($url);
        return (!isset($urlParts['scheme']) || $urlParts['scheme'] === '') && $urlParts['path'][0] !== '/';
    }
 
    protected static function ‪isAllowedLocalFile($filePath)
    {
        $filePath = GeneralUtility::resolveBackPath($filePath);
        $insideWebPath = strpos($filePath, ‪Environment::getPublicPath()) === 0;
        $isFile = is_file($filePath);
        return $insideWebPath && $isFile;
    }
 
    /******************************************
     *
     * Indexing; external files (PDF, DOC, etc)
     *
     ******************************************/
    public function ‪indexRegularDocument($file, $force = false, $contentTmpFile = '', $altExtension = '')
    {
        // Init
        $fI = pathinfo($file);
        $ext = $altExtension ?: strtolower($fI['extension']);
        // Create abs-path:
        if (!$contentTmpFile) {
            if (!GeneralUtility::isAbsPath($file)) {
                // Relative, prepend public web path:
                $absFile = GeneralUtility::getFileAbsFileName(‪Environment::getPublicPath() . '/' . $file);
            } else {
                // Absolute, pass-through:
                $absFile = $file;
            }
            $absFile = GeneralUtility::isAllowedAbsPath($absFile) ? $absFile : '';
        } else {
            $absFile = $contentTmpFile;
        }
        // Indexing the document:
        if ($absFile && @is_file($absFile)) {
            if ($this->external_parsers[$ext]) {
                $fileInfo = stat($absFile);
                $cParts = $this->‪fileContentParts($ext, $absFile);
                foreach ($cParts as $cPKey) {
                    $this->internal_log = [];
                    $this->‪log_push('Index: ' . str_replace('.', '_', ‪PathUtility::basename($file)) . ($cPKey ? '#' . $cPKey : ''), '');
                    $Pstart = GeneralUtility::milliseconds();
                    $subinfo = ['key' => $cPKey];
                    // Setting page range. This is "0" (zero) when no division is made, otherwise a range like "1-3"
                    $phash_arr = ($this->file_phash_arr = $this->‪setExtHashes($file, $subinfo));
                    $check = $this->‪checkMtimeTstamp($fileInfo['mtime'], $phash_arr['phash']);
                    if ($check > 0 || $force) {
                        if ($check > 0) {
                            $this->‪log_setTSlogMessage('Indexing needed, reason: ' . $this->reasons[$check], 1);
                        } else {
                            $this->‪log_setTSlogMessage('Indexing forced by flag', 1);
                        }
                        // Check external file counter:
                        if ($this->externalFileCounter < $this->maxExternalFiles || $force) {
                            // Divide into title,keywords,description and body:
                            $this->‪log_push('Split content', '');
                            ‪$contentParts = $this->‪readFileContent($ext, $absFile, $cPKey);
                            $this->‪log_pull();
                            if (is_array(‪$contentParts)) {
                                // Calculating a hash over what is to be the actual content. (see indexTypo3PageContent())
                                ‪$content_md5h = ‪IndexedSearchUtility::md5inthash(implode('', ‪$contentParts));
                                if ($this->‪checkExternalDocContentHash($phash_arr['phash_grouping'], ‪$content_md5h) || $force) {
                                    // Increment counter:
                                    $this->externalFileCounter++;
                                    // Splitting words
                                    $this->‪log_push('Extract words from content', '');
                                    $splitInWords = $this->‪processWordsInArrays(‪$contentParts);
                                    $this->‪log_pull();
                                    // Analyze the indexed words.
                                    $this->‪log_push('Analyze the extracted words', '');
                                    $indexArr = $this->‪indexAnalyze($splitInWords);
                                    $this->‪log_pull();
                                    // Submitting page (phash) record
                                    $this->‪log_push('Submitting page', '');
                                    // Unfortunately I cannot determine WHEN a file is originally made - so I must return the modification time...
                                    $this->‪submitFilePage($phash_arr, $file, $subinfo, $ext, $fileInfo['mtime'], $fileInfo['ctime'], $fileInfo['size'], ‪$content_md5h, ‪$contentParts);
                                    $this->‪log_pull();
                                    // Check words and submit to word list if not there
                                    $this->‪log_push('Check word list and submit words', '');
                                    if (‪IndexedSearchUtility::isTableUsed('index_words')) {
                                        $this->‪checkWordList($indexArr);
                                        $this->‪submitWords($indexArr, $phash_arr['phash']);
                                    }
                                    $this->‪log_pull();
                                    // Set parsetime
                                    $this->‪updateParsetime($phash_arr['phash'], GeneralUtility::milliseconds() - $Pstart);
                                } else {
                                    // Update the timestamp
                                    $this->‪updateTstamp($phash_arr['phash'], $fileInfo['mtime']);
                                    $this->‪log_setTSlogMessage('Indexing not needed, the contentHash, ' . ‪$content_md5h . ', has not changed. Timestamp updated.');
                                }
                            } else {
                                $this->‪log_setTSlogMessage('Could not index file! Unsupported extension.');
                            }
                        } else {
                            $this->‪log_setTSlogMessage('The limit of ' . $this->maxExternalFiles . ' has already been exceeded, so no indexing will take place this time.');
                        }
                    } else {
                        $this->‪log_setTSlogMessage('Indexing not needed, reason: ' . $this->reasons[$check]);
                    }
                    // Checking and setting sections:
                    $this->‪submitFile_section($phash_arr['phash']);
                    // Setting a section-record for the file. This is done also if the file is not indexed. Notice that section records are deleted when the page is indexed.
                    $this->‪log_pull();
                }
            } else {
                $this->‪log_setTSlogMessage('Indexing not possible; The extension "' . $ext . '" was not supported.');
            }
        } else {
            $this->‪log_setTSlogMessage('Indexing not possible; File "' . $absFile . '" not found or valid.');
        }
    }
 
    public function ‪readFileContent($fileExtension, $absoluteFileName, $sectionPointer)
    {
        $contentArray = null;
        // Consult relevant external document parser:
        if (is_object($this->external_parsers[$fileExtension])) {
            $contentArray = $this->external_parsers[$fileExtension]->readFileContent($fileExtension, $absoluteFileName, $sectionPointer);
        }
        return $contentArray;
    }
 
    public function ‪fileContentParts($ext, $absFile)
    {
        $cParts = [0];
        // Consult relevant external document parser:
        if (is_object($this->external_parsers[$ext])) {
            $cParts = $this->external_parsers[$ext]->fileContentParts($ext, $absFile);
        }
        return $cParts;
    }
 
    public function ‪splitRegularContent($content)
    {
        $contentArr = ‪$this->defaultContentArray;
        $contentArr['body'] = $content;
        return $contentArr;
    }
 
    /**********************************
     *
     * Analysing content, Extracting words
     *
     **********************************/
    public function ‪charsetEntity2utf8(&$contentArr, $charset)
    {
        // Convert charset if necessary
        foreach ($contentArr as $key => $value) {
            if ((string)$contentArr[$key] !== '') {
                if ($charset !== 'utf-8') {
                    $contentArr[$key] = mb_convert_encoding($contentArr[$key], 'utf-8', $charset);
                }
                // decode all numeric / html-entities in the string to real characters:
                $contentArr[$key] = html_entity_decode($contentArr[$key]);
            }
        }
    }
 
    public function ‪processWordsInArrays($contentArr)
    {
        // split all parts to words
        foreach ($contentArr as $key => $value) {
            $contentArr[$key] = $this->lexerObj->split2Words($contentArr[$key]);
        }
        // For title, keywords, and description we don't want duplicates:
        $contentArr['title'] = array_unique($contentArr['title']);
        $contentArr['keywords'] = array_unique($contentArr['keywords']);
        $contentArr['description'] = array_unique($contentArr['description']);
        // Return modified array:
        return $contentArr;
    }
 
    public function ‪bodyDescription($contentArr)
    {
        // Setting description
        $maxL = ‪MathUtility::forceIntegerInRange($this->conf['index_descrLgd'], 0, 255, 200);
        if ($maxL) {
            $bodyDescription = preg_replace('/\s+/u', ' ', $contentArr['body']);
            // Shorten the string:
            $bodyDescription = mb_strcut($bodyDescription, 0, $maxL, 'utf-8');
        }
        return $bodyDescription;
    }
 
    public function ‪indexAnalyze($content)
    {
        $indexArr = [];
        $this->‪analyzeHeaderinfo($indexArr, $content, 'title', 7);
        $this->‪analyzeHeaderinfo($indexArr, $content, 'keywords', 6);
        $this->‪analyzeHeaderinfo($indexArr, $content, 'description', 5);
        $this->‪analyzeBody($indexArr, $content);
        return $indexArr;
    }
 
    public function ‪analyzeHeaderinfo(&$retArr, $content, $key, $offset)
    {
        foreach ($content[$key] as $val) {
            $val = substr($val, 0, 60);
            // Cut after 60 chars because the index_words.baseword varchar field has this length. This MUST be the same.
            if (!isset($retArr[$val])) {
                // Word ID (wid)
                $retArr[$val]['hash'] = ‪IndexedSearchUtility::md5inthash($val);
                // Metaphone value is also 60 only chars long
                $metaphone = $this->enableMetaphoneSearch ? substr($this->‪metaphone($val, $this->storeMetaphoneInfoAsWords), 0, 60) : '';
                $retArr[$val]['metaphone'] = $metaphone;
            }
            // Build metaphone fulltext string (can be used for fulltext indexing)
            if ($this->storeMetaphoneInfoAsWords) {
                $this->metaphoneContent .= ' ' . $retArr[$val]['metaphone'];
            }
            // Priority used for flagBitMask feature (see extension configuration)
            $retArr[$val]['cmp'] = $retArr[$val]['cmp'] | pow(2, $offset);
            // Increase number of occurrences
            $retArr[$val]['count']++;
            $this->wordcount++;
        }
    }
 
    public function ‪analyzeBody(&$retArr, $content)
    {
        foreach ($content['body'] as $key => $val) {
            $val = substr($val, 0, 60);
            // Cut after 60 chars because the index_words.baseword varchar field has this length. This MUST be the same.
            if (!isset($retArr[$val])) {
                // First occurrence (used for ranking results)
                $retArr[$val]['first'] = $key;
                // Word ID (wid)
                $retArr[$val]['hash'] = ‪IndexedSearchUtility::md5inthash($val);
                // Metaphone value is also only 60 chars long
                $metaphone = $this->enableMetaphoneSearch ? substr($this->‪metaphone($val, $this->storeMetaphoneInfoAsWords), 0, 60) : '';
                $retArr[$val]['metaphone'] = $metaphone;
            }
            // Build metaphone fulltext string (can be used for fulltext indexing)
            if ($this->storeMetaphoneInfoAsWords) {
                $this->metaphoneContent .= ' ' . $retArr[$val]['metaphone'];
            }
            // Increase number of occurrences
            $retArr[$val]['count']++;
            $this->wordcount++;
        }
    }
 
    public function ‪metaphone($word, $returnRawMetaphoneValue = false)
    {
        if (is_object($this->metaphoneObj)) {
            $metaphoneRawValue = $this->metaphoneObj->metaphone($word, $this->conf['sys_language_uid']);
        } else {
            // Use native PHP function instead of advanced doubleMetaphone class
            $metaphoneRawValue = ‪metaphone($word);
        }
        if ($returnRawMetaphoneValue) {
            $result = $metaphoneRawValue;
        } elseif ($metaphoneRawValue !== '') {
            // Create hash and return integer
            $result = ‪IndexedSearchUtility::md5inthash($metaphoneRawValue);
        } else {
            $result = 0;
        }
        return $result;
    }
 
    /********************************
     *
     * SQL; TYPO3 Pages
     *
     *******************************/
    public function ‪submitPage()
    {
        // Remove any current data for this phash:
        $this->‪removeOldIndexedPages($this->hash['phash']);
        // setting new phash_row
        ‪$fields = [
            'phash' => $this->hash['phash'],
            'phash_grouping' => $this->hash['phash_grouping'],
            'cHashParams' => serialize($this->cHashParams),
            'static_page_arguments' => is_array($this->conf['staticPageArguments']) ? json_encode($this->conf['staticPageArguments']) : null,
            'contentHash' => $this->content_md5h,
            'data_page_id' => $this->conf['id'],
            // @deprecated since TYPO3 v9, will be removed in TYPO3 v10.0. Remove along with database field data_page_reg1
            'data_page_reg1' => $this->conf['page_cache_reg1'],
            'data_page_type' => $this->conf['type'],
            'data_page_mp' => $this->conf['MP'],
            'gr_list' => $this->conf['gr_list'],
            'item_type' => 0,
            // TYPO3 page
            'item_title' => $this->contentParts['title'],
            'item_description' => $this->‪bodyDescription($this->contentParts),
            'item_mtime' => (int)$this->conf['mtime'],
            'item_size' => strlen($this->conf['content']),
            'tstamp' => ‪$GLOBALS['EXEC_TIME'],
            'crdate' => ‪$GLOBALS['EXEC_TIME'],
            'item_crdate' => $this->conf['crdate'],
            // Creation date of page
            'sys_language_uid' => $this->conf['sys_language_uid'],
            // Sys language uid of the page. Should reflect which language it DOES actually display!
            'externalUrl' => 0,
            'recordUid' => (int)$this->conf['recordUid'],
            'freeIndexUid' => (int)$this->conf['freeIndexUid'],
            'freeIndexSetId' => (int)$this->conf['freeIndexSetId']
        ];
        if (‪IndexedSearchUtility::isTableUsed('index_phash')) {
            $connection = GeneralUtility::makeInstance(ConnectionPool::class)
                ->getConnectionForTable('index_phash');
            $connection->insert(
                'index_phash',
                ‪$fields,
                ['cHashParams' => ‪Connection::PARAM_LOB]
            );
        }
        // PROCESSING index_section
        $this->‪submit_section($this->hash['phash'], $this->hash['phash']);
        // PROCESSING index_grlist
        $this->‪submit_grlist($this->hash['phash'], $this->hash['phash']);
        // PROCESSING index_fulltext
        ‪$fields = [
            'phash' => $this->hash['phash'],
            'fulltextdata' => implode(' ', $this->contentParts),
            'metaphonedata' => ‪$this->metaphoneContent
        ];
        if ($this->indexerConfig['fullTextDataLength'] > 0) {
            ‪$fields['fulltextdata'] = substr(‪$fields['fulltextdata'], 0, $this->indexerConfig['fullTextDataLength']);
        }
        if (‪IndexedSearchUtility::isTableUsed('index_fulltext')) {
            $connection = GeneralUtility::makeInstance(ConnectionPool::class)
                ->getConnectionForTable('index_fulltext');
            $connection->insert('index_fulltext', ‪$fields);
        }
        // PROCESSING index_debug
        if ($this->indexerConfig['debugMode']) {
            ‪$fields = [
                'phash' => $this->hash['phash'],
                'debuginfo' => serialize([
                    'cHashParams' => $this->cHashParams,
                    'external_parsers initialized' => array_keys($this->external_parsers),
                    'conf' => array_merge($this->conf, ['content' => substr($this->conf['content'], 0, 1000)]),
                    'contentParts' => array_merge($this->contentParts, ['body' => substr($this->contentParts['body'], 0, 1000)]),
                    'logs' => $this->internal_log,
                    'lexer' => $this->lexerObj->debugString
                ])
            ];
            if (‪IndexedSearchUtility::isTableUsed('index_debug')) {
                $connection = GeneralUtility::makeInstance(ConnectionPool::class)
                    ->getConnectionForTable('index_debug');
                $connection->insert('index_debug', ‪$fields);
            }
        }
    }
 
    public function ‪submit_grlist(‪$hash, $phash_x)
    {
        // Setting the gr_list record
        ‪$fields = [
            'phash' => ‪$hash,
            'phash_x' => $phash_x,
            'hash_gr_list' => ‪IndexedSearchUtility::md5inthash($this->conf['gr_list']),
            'gr_list' => $this->conf['gr_list']
        ];
        if (‪IndexedSearchUtility::isTableUsed('index_grlist')) {
            $connection = GeneralUtility::makeInstance(ConnectionPool::class)
                ->getConnectionForTable('index_grlist');
            $connection->insert('index_grlist', ‪$fields);
        }
    }
 
    public function ‪submit_section(‪$hash, $hash_t3)
    {
        ‪$fields = [
            'phash' => ‪$hash,
            'phash_t3' => $hash_t3,
            'page_id' => (int)$this->conf['id']
        ];
        $this->‪getRootLineFields(‪$fields);
        if (‪IndexedSearchUtility::isTableUsed('index_section')) {
            $connection = GeneralUtility::makeInstance(ConnectionPool::class)
                ->getConnectionForTable('index_section');
            $connection->insert('index_section', ‪$fields);
        }
    }
 
    public function ‪removeOldIndexedPages($phash)
    {
        // Removing old registrations for all tables. Because the pages are TYPO3 pages
        // there can be nothing else than 1-1 relations here.
        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
        $tableArray = ['index_phash', 'index_section', 'index_grlist', 'index_fulltext', 'index_debug'];
        foreach ($tableArray as $table) {
            if (‪IndexedSearchUtility::isTableUsed($table)) {
                $connectionPool->getConnectionForTable($table)->delete($table, ['phash' => (int)$phash]);
            }
        }
 
        // Removing all index_section records with hash_t3 set to this hash (this includes such
        // records set for external media on the page as well!). The re-insert of these records
        // are done in indexRegularDocument($file).
        if (‪IndexedSearchUtility::isTableUsed('index_section')) {
            $connectionPool->getConnectionForTable('index_section')
                ->delete('index_section', ['phash_t3' => (int)$phash]);
        }
    }
 
    /********************************
     *
     * SQL; External media
     *
     *******************************/
    public function ‪submitFilePage(‪$hash, $file, $subinfo, $ext, $mtime, $ctime, $size, ‪$content_md5h, ‪$contentParts)
    {
        // Find item Type:
        $storeItemType = $this->external_parsers[$ext]->ext2itemtype_map[$ext];
        $storeItemType = $storeItemType ?: $ext;
        // Remove any current data for this phash:
        $this->‪removeOldIndexedFiles(‪$hash['phash']);
        // Split filename:
        $fileParts = parse_url($file);
        // Setting new
        ‪$fields = [
            'phash' => ‪$hash['phash'],
            'phash_grouping' => ‪$hash['phash_grouping'],
            'cHashParams' => serialize($subinfo),
            'contentHash' => ‪$content_md5h,
            'data_filename' => $file,
            'item_type' => $storeItemType,
            'item_title' => trim(‪$contentParts['title']) ?: ‪PathUtility::basename($file),
            'item_description' => $this->‪bodyDescription(‪$contentParts),
            'item_mtime' => $mtime,
            'item_size' => $size,
            'item_crdate' => $ctime,
            'tstamp' => ‪$GLOBALS['EXEC_TIME'],
            'crdate' => ‪$GLOBALS['EXEC_TIME'],
            'gr_list' => $this->conf['gr_list'],
            'externalUrl' => $fileParts['scheme'] ? 1 : 0,
            'recordUid' => (int)$this->conf['recordUid'],
            'freeIndexUid' => (int)$this->conf['freeIndexUid'],
            'freeIndexSetId' => (int)$this->conf['freeIndexSetId'],
            'sys_language_uid' => (int)$this->conf['sys_language_uid']
        ];
        if (‪IndexedSearchUtility::isTableUsed('index_phash')) {
            $connection = GeneralUtility::makeInstance(ConnectionPool::class)
                ->getConnectionForTable('index_phash');
            $connection->insert(
                'index_phash',
                ‪$fields,
                ['cHashParams' => ‪Connection::PARAM_LOB]
            );
        }
        // PROCESSING index_fulltext
        ‪$fields = [
            'phash' => ‪$hash['phash'],
            'fulltextdata' => implode(' ', ‪$contentParts),
            'metaphonedata' => ‪$this->metaphoneContent
        ];
        if ($this->indexerConfig['fullTextDataLength'] > 0) {
            ‪$fields['fulltextdata'] = substr(‪$fields['fulltextdata'], 0, $this->indexerConfig['fullTextDataLength']);
        }
        if (‪IndexedSearchUtility::isTableUsed('index_fulltext')) {
            $connection = GeneralUtility::makeInstance(ConnectionPool::class)
                ->getConnectionForTable('index_fulltext');
            $connection->insert('index_fulltext', ‪$fields);
        }
        // PROCESSING index_debug
        if ($this->indexerConfig['debugMode']) {
            ‪$fields = [
                'phash' => ‪$hash['phash'],
                'debuginfo' => serialize([
                    'cHashParams' => $subinfo,
                    'contentParts' => array_merge(‪$contentParts, ['body' => substr(‪$contentParts['body'], 0, 1000)]),
                    'logs' => $this->internal_log,
                    'lexer' => $this->lexerObj->debugString
                ])
            ];
            if (‪IndexedSearchUtility::isTableUsed('index_debug')) {
                $connection = GeneralUtility::makeInstance(ConnectionPool::class)
                    ->getConnectionForTable('index_debug');
                $connection->insert('index_debug', ‪$fields);
            }
        }
    }
 
    public function ‪submitFile_grlist(‪$hash)
    {
        // Testing if there is a gr_list record for a non-logged in user and if so, there is no need to place another one.
        if (!‪IndexedSearchUtility::isTableUsed('index_grlist')) {
            return;
        }
 
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
            ->getQueryBuilderForTable('index_grlist');
        $count = (int)$queryBuilder->count('*')
            ->from('index_grlist')
            ->where(
                $queryBuilder->expr()->eq(
                    'phash',
                    $queryBuilder->createNamedParameter(‪$hash, \PDO::PARAM_INT)
                ),
                $queryBuilder->expr()->orX(
                    $queryBuilder->expr()->eq(
                        'hash_gr_list',
                        $queryBuilder->createNamedParameter(
                            ‪IndexedSearchUtility::md5inthash($this->defaultGrList),
                            \PDO::PARAM_INT
                        )
                    ),
                    $queryBuilder->expr()->eq(
                        'hash_gr_list',
                        $queryBuilder->createNamedParameter(
                            ‪IndexedSearchUtility::md5inthash($this->conf['gr_list']),
                            \PDO::PARAM_INT
                        )
                    )
                )
            )
            ->execute()
            ->fetchColumn();
 
        if ($count === 0) {
            $this->‪submit_grlist(‪$hash, ‪$hash);
        }
    }
 
    public function ‪submitFile_section(‪$hash)
    {
        // Testing if there is already a section
        if (!‪IndexedSearchUtility::isTableUsed('index_section')) {
            return;
        }
 
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
            ->getQueryBuilderForTable('index_section');
        $count = (int)$queryBuilder->count('phash')
            ->from('index_section')
            ->where(
                $queryBuilder->expr()->eq(
                    'phash',
                    $queryBuilder->createNamedParameter(‪$hash, \PDO::PARAM_INT)
                ),
                $queryBuilder->expr()->eq(
                    'page_id',
                    $queryBuilder->createNamedParameter($this->conf['id'], \PDO::PARAM_INT)
                )
            )
            ->execute()
            ->fetchColumn();
 
        if ($count === 0) {
            $this->‪submit_section(‪$hash, $this->hash['phash']);
        }
    }
 
    public function ‪removeOldIndexedFiles($phash)
    {
        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
        // Removing old registrations for tables.
        $tableArray = ['index_phash', 'index_grlist', 'index_fulltext', 'index_debug'];
        foreach ($tableArray as $table) {
            if (!‪IndexedSearchUtility::isTableUsed($table)) {
                continue;
            }
            $connectionPool->getConnectionForTable($table)->delete($table, ['phash' => (int)$phash]);
        }
    }
 
    /********************************
     *
     * SQL Helper functions
     *
     *******************************/
    public function ‪checkMtimeTstamp($mtime, $phash)
    {
        if (!‪IndexedSearchUtility::isTableUsed('index_phash')) {
            // Not indexed (not in index_phash)
            $result = 4;
        } else {
            $row = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('index_phash')
                ->select(
                    ['item_mtime', 'tstamp'],
                    'index_phash',
                    ['phash' => (int)$phash],
                    [],
                    [],
                    1
                )
                ->fetch();
            // If there was an indexing of the page...:
            if (!empty($row)) {
                if ($this->tstamp_maxAge && $row['tstamp'] + $this->tstamp_maxAge < ‪$GLOBALS['EXEC_TIME']) {
                    // If max age is exceeded, index the page
                    // The configured max-age was exceeded for the document and thus it's indexed.
                    $result = 1;
                } else {
                    if (!$this->tstamp_minAge || $row['tstamp'] + $this->tstamp_minAge < ‪$GLOBALS['EXEC_TIME']) {
                        // if minAge is not set or if minAge is exceeded, consider at mtime
                        if ($mtime) {
                            // It mtime is set, then it's tested. If not, the page must clearly be indexed.
                            if ($row['item_mtime'] != $mtime) {
                                // And if mtime is different from the index_phash mtime, it's about time to re-index.
                                // The minimum age was exceed and mtime was set and the mtime was different, so the page was indexed.
                                $result = 2;
                            } else {
                                // mtime matched the document, so no changes detected and no content updated
                                $result = -1;
                                if ($this->tstamp_maxAge) {
                                    $this->‪log_setTSlogMessage('mtime matched, timestamp NOT updated because a maxAge is set (' . ($row['tstamp'] + $this->tstamp_maxAge - ‪$GLOBALS['EXEC_TIME']) . ' seconds to expire time).', 1);
                                } else {
                                    $this->‪updateTstamp($phash);
                                    $this->‪log_setTSlogMessage('mtime matched, timestamp updated.', 1);
                                }
                            }
                        } else {
                            // The minimum age was exceed, but mtime was not set, so the page was indexed.
                            $result = 3;
                        }
                    } else {
                        // The minimum age was not exceeded
                        $result = -2;
                    }
                }
            } else {
                // Page has never been indexed (is not represented in the index_phash table).
                $result = 4;
            }
        }
        return $result;
    }
 
    public function ‪checkContentHash()
    {
        // With this query the page will only be indexed if it's content is different from the same "phash_grouping" -page.
        $result = true;
        if (‪IndexedSearchUtility::isTableUsed('index_phash')) {
            $row = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('index_phash')
                ->select(
                    ['phash'],
                    'index_phash',
                    [
                        'phash_grouping' => (int)$this->hash['phash_grouping'],
                        'contentHash' => (int)$this->content_md5h
                    ],
                    [],
                    [],
                    1
                )
                ->fetch();
 
            if (!empty($row)) {
                $result = $row;
            }
        }
        return $result;
    }
 
    public function ‪checkExternalDocContentHash($hashGr, ‪$content_md5h)
    {
        $result = true;
        if (‪IndexedSearchUtility::isTableUsed('index_phash')) {
            $count = (int)GeneralUtility::makeInstance(ConnectionPool::class)
                ->getConnectionForTable('index_phash')
                ->count(
                    '*',
                    'index_phash',
                    [
                        'phash_grouping' => (int)$hashGr,
                        'contentHash' => (int)‪$content_md5h
                    ]
                );
 
            $result = $count === 0;
        }
        return $result;
    }
 
    public function ‪is_grlist_set($phash_x)
    {
        $result = false;
        if (‪IndexedSearchUtility::isTableUsed('index_grlist')) {
            $count = (int)GeneralUtility::makeInstance(ConnectionPool::class)
                ->getConnectionForTable('index_grlist')
                ->count(
                    'phash_x',
                    'index_grlist',
                    ['phash_x' => (int)$phash_x]
                );
 
            $result = $count > 0;
        }
        return $result;
    }
 
    public function ‪update_grlist($phash, $phash_x)
    {
        if (‪IndexedSearchUtility::isTableUsed('index_grlist')) {
            $count = (int)GeneralUtility::makeInstance(ConnectionPool::class)
                ->getConnectionForTable('index_grlist')
                ->count(
                    'phash',
                    'index_grlist',
                    [
                        'phash' => (int)$phash,
                        'hash_gr_list' => ‪IndexedSearchUtility::md5inthash($this->conf['gr_list'])
                    ]
                );
 
            if ($count === 0) {
                $this->‪submit_grlist($phash, $phash_x);
                $this->‪log_setTSlogMessage('Inserted gr_list \'' . $this->conf['gr_list'] . '\' for phash \'' . $phash . '\'', 1);
            }
        }
    }
 
    public function ‪updateTstamp($phash, $mtime = 0)
    {
        if (!‪IndexedSearchUtility::isTableUsed('index_phash')) {
            return;
        }
 
        $updateFields = [
            'tstamp' => ‪$GLOBALS['EXEC_TIME']
        ];
 
        if ($mtime) {
            $updateFields['item_mtime'] = (int)$mtime;
        }
 
        GeneralUtility::makeInstance(ConnectionPool::class)
            ->getConnectionForTable('index_phash')
            ->update(
                'index_phash',
                $updateFields,
                [
                    'phash' => (int)$phash
                ]
            );
    }
 
    public function ‪updateSetId($phash)
    {
        if (!‪IndexedSearchUtility::isTableUsed('index_phash')) {
            return;
        }
 
        GeneralUtility::makeInstance(ConnectionPool::class)
            ->getConnectionForTable('index_phash')
            ->update(
                'index_phash',
                [
                    'freeIndexSetId' => (int)$this->conf['freeIndexSetId']
                ],
                [
                    'phash' => (int)$phash
                ]
            );
    }
 
    public function ‪updateParsetime($phash, $parsetime)
    {
        if (!‪IndexedSearchUtility::isTableUsed('index_phash')) {
            return;
        }
 
        GeneralUtility::makeInstance(ConnectionPool::class)
            ->getConnectionForTable('index_phash')
            ->update(
                'index_phash',
                [
                    'parsetime' => (int)$parsetime
                ],
                [
                    'phash' => (int)$phash
                ]
            );
    }
 
    public function ‪updateRootline()
    {
        if (!‪IndexedSearchUtility::isTableUsed('index_section')) {
            return;
        }
 
        $updateFields = [];
        $this->‪getRootLineFields($updateFields);
 
        GeneralUtility::makeInstance(ConnectionPool::class)
            ->getConnectionForTable('index_section')
            ->update(
                'index_section',
                $updateFields,
                [
                    'page_id' => (int)$this->conf['id']
                ]
            );
    }
 
    public function ‪getRootLineFields(array &$fieldArray)
    {
        $fieldArray['rl0'] = (int)$this->conf['rootline_uids'][0];
        $fieldArray['rl1'] = (int)$this->conf['rootline_uids'][1];
        $fieldArray['rl2'] = (int)$this->conf['rootline_uids'][2];
        foreach (‪$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['addRootLineFields'] ?? [] as $fieldName => $rootLineLevel) {
            $fieldArray[$fieldName] = (int)$this->conf['rootline_uids'][$rootLineLevel];
        }
    }
 
    /********************************
     *
     * SQL; Submitting words
     *
     *******************************/
    public function ‪checkWordList($wordListArray)
    {
        if (!‪IndexedSearchUtility::isTableUsed('index_words') || empty($wordListArray)) {
            return;
        }
 
        $wordListArrayCount = count($wordListArray);
        $phashArray = array_map('intval', array_column($wordListArray, 'hash'));
 
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('index_words');
        $count = (int)$queryBuilder->count('baseword')
            ->from('index_words')
            ->where(
                $queryBuilder->expr()->in(
                    'wid',
                    $queryBuilder->createNamedParameter($phashArray, Connection::PARAM_INT_ARRAY)
                )
            )
            ->execute()
            ->fetchColumn();
 
        if ($count !== $wordListArrayCount) {
            $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('index_words');
            $queryBuilder = $connection->createQueryBuilder();
 
            $result = $queryBuilder->select('baseword')
                ->from('index_words')
                ->where(
                    $queryBuilder->expr()->in(
                        'wid',
                        $queryBuilder->createNamedParameter($phashArray, Connection::PARAM_INT_ARRAY)
                    )
                )
                ->execute();
 
            $this->‪log_setTSlogMessage('Inserting words: ' . ($wordListArrayCount - $count), 1);
            while ($row = $result->fetch()) {
                unset($wordListArray[$row['baseword']]);
            }
 
            foreach ($wordListArray as $key => $val) {
                // A duplicate-key error will occur here if a word is NOT unset in the unset() line. However as
                // long as the words in $wl are NOT longer as 60 chars (the baseword varchar is 60 characters...)
                // this is not a problem.
                $connection->insert(
                    'index_words',
                    [
                        'wid' => $val['hash'],
                        'baseword' => $key,
                        'metaphone' => $val['metaphone']
                    ]
                );
            }
        }
    }
 
    public function ‪submitWords($wordList, $phash)
    {
        if (!‪IndexedSearchUtility::isTableUsed('index_rel')) {
            return;
        }
        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
        $queryBuilder = $connectionPool->getQueryBuilderForTable('index_words');
        $result = $queryBuilder->select('wid')
            ->from('index_words')
            ->where(
                $queryBuilder->expr()->neq('is_stopword', $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT))
            )
            ->groupBy('wid')
            ->execute();
 
        $stopWords = [];
        while ($row = $result->fetch()) {
            $stopWords[$row['wid']] = $row;
        }
 
        $connectionPool->getConnectionForTable('index_rel')->delete('index_rel', ['phash' => (int)$phash]);
 
        ‪$fields = ['phash', 'wid', 'count', 'first', 'freq', 'flags'];
        $rows = [];
        foreach ($wordList as $val) {
            if (isset($stopWords[$val['hash']])) {
                continue;
            }
            $rows[] = [
                (int)$phash,
                (int)$val['hash'],
                (int)$val['count'],
                (int)$val['first'],
                $this->‪freqMap($val['count'] / $this->wordcount),
                $val['cmp'] & ‪$this->flagBitMask
            ];
        }
 
        if (!empty($rows)) {
            $connectionPool->getConnectionForTable('index_rel')->bulkInsert('index_rel', $rows, ‪$fields);
        }
    }
 
    public function ‪freqMap($freq)
    {
        $mapFactor = $this->freqMax * 100 * ‪$this->freqRange;
        if ($freq <= 1) {
            $newFreq = $freq * $mapFactor;
            $newFreq = $newFreq > $this->freqRange ? $this->freqRange : $newFreq;
        } else {
            $newFreq = $freq / $mapFactor;
        }
        return (int)$newFreq;
    }
 
    /********************************
     *
     * Hashing
     *
     *******************************/
    public function ‪setT3Hashes()
    {
        //  Set main array:
        $hArray = [
            'id' => (int)$this->conf['id'],
            'type' => (int)$this->conf['type'],
            'sys_lang' => (int)$this->conf['sys_language_uid'],
            'MP' => (string)$this->conf['MP'],
            'cHash' => ‪$this->cHashParams,
            'staticPageArguments' => is_array($this->conf['staticPageArguments']) ? json_encode($this->conf['staticPageArguments']) : null,
        ];
        // Set grouping hash (Identifies a "page" combined of id, type, language, mountpoint and cHash parameters):
        $this->hash['phash_grouping'] = ‪IndexedSearchUtility::md5inthash(serialize($hArray));
        // Add gr_list and set plain phash (Subdivision where special page composition based on login is taken into account as well. It is expected that such pages are normally similar regardless of the login.)
        $hArray['gr_list'] = (string)$this->conf['gr_list'];
        $this->hash['phash'] = ‪IndexedSearchUtility::md5inthash(serialize($hArray));
    }
 
    public function ‪setExtHashes($file, $subinfo = [])
    {
        //  Set main array:
        ‪$hash = [];
        $hArray = [
            'file' => $file
        ];
        // Set grouping hash:
        ‪$hash['phash_grouping'] = ‪IndexedSearchUtility::md5inthash(serialize($hArray));
        // Add subinfo
        $hArray['subinfo'] = $subinfo;
        ‪$hash['phash'] = ‪IndexedSearchUtility::md5inthash(serialize($hArray));
        return ‪$hash;
    }
 
    /*********************************
     *
     * Internal logging functions
     *
     *********************************/
    public function ‪log_push($msg, $key)
    {
        $this->timeTracker->push($msg, $key);
    }
 
    public function ‪log_pull()
    {
        $this->timeTracker->pull();
    }
 
    public function ‪log_setTSlogMessage($msg, $errorNum = 0)
    {
        $this->timeTracker->setTSlogMessage($msg, $errorNum);
        $this->internal_log[] = $msg;
    }
 
    protected function ‪addSpacesToKeywordList($keywordList)
    {
        $keywords = GeneralUtility::trimExplode(',', $keywordList);
        return ' ' . implode(', ', $keywords) . ' ';
    }
}