CrawlerHook.php

Go to the documentation of this file.

<?php
 
/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */
 
namespace ‪TYPO3\CMS\IndexedSearch\Hook;
 
use ‪TYPO3\CMS\Backend\Utility\BackendUtility;
use ‪TYPO3\CMS\Core\Core\Environment;
use ‪TYPO3\CMS\Core\Database\Connection;
use ‪TYPO3\CMS\Core\Database\ConnectionPool;
use ‪TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
use ‪TYPO3\CMS\Core\DataHandling\DataHandler;
use ‪TYPO3\CMS\Core\Exception\Page\RootLineException;
use ‪TYPO3\CMS\Core\TypoScript\ExtendedTemplateService;
use ‪TYPO3\CMS\Core\Utility\GeneralUtility;
use ‪TYPO3\CMS\Core\Utility\MathUtility;
use ‪TYPO3\CMS\Core\Utility\PathUtility;
use ‪TYPO3\CMS\Core\Utility\RootlineUtility;
use ‪TYPO3\CMS\IndexedSearch\Indexer;
 
class ‪CrawlerHook
{
    public ‪$secondsPerExternalUrl = 3;
 
    public ‪$instanceCounter = 0;
 
    public ‪$callBack = self::class;
 
    private ‪$pObj;
 
    public function ‪crawler_init(&‪$pObj)
    {
        $this->pObj = ‪$pObj;
 
        $message = null;
        // Select all indexing configuration which are waiting to be activated:
        $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('index_config');
        $queryBuilder = $connection->createQueryBuilder();
 
        $result = $queryBuilder->select('*')
            ->from('index_config')
            ->where(
                $queryBuilder->expr()->lt(
                    'timer_next_indexing',
                    $queryBuilder->createNamedParameter(‪$GLOBALS['EXEC_TIME'], \PDO::PARAM_INT)
                ),
                $queryBuilder->expr()->eq('set_id', $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT))
            )
            ->execute();
 
        // For each configuration, check if it should be executed and if so, start:
        while ($cfgRec = $result->fetch()) {
            // Generate a unique set-ID:
            $setId = GeneralUtility::md5int(microtime());
            // Get next time:
            $nextTime = $this->‪generateNextIndexingTime($cfgRec);
            // Start process by updating index-config record:
            $connection->update(
                'index_config',
                [
                    'set_id' => $setId,
                    'timer_next_indexing' => $nextTime,
                    'session_data' => ''
                ],
                [
                    'uid' => (int)$cfgRec['uid']
                ]
            );
            // Based on configuration type:
            switch ($cfgRec['type']) {
                case 1:
                    // RECORDS:
                    // Parameters:
                    $params = [
                        'indexConfigUid' => $cfgRec['uid'],
                        'procInstructions' => ['[Index Cfg UID#' . $cfgRec['uid'] . ']'],
                        'url' => 'Records (start)'
                    ];
                    //
                    ‪$pObj->addQueueEntry_callBack($setId, $params, $this->callBack, $cfgRec['pid']);
                    break;
                case 2:
                    // FILES:
                    // Parameters:
                    $params = [
                        'indexConfigUid' => $cfgRec['uid'],
                        // General
                        'procInstructions' => ['[Index Cfg UID#' . $cfgRec['uid'] . ']'],
                        // General
                        'url' => $cfgRec['filepath'],
                        // Partly general... (for URL and file types)
                        'depth' => 0
                    ];
                    ‪$pObj->addQueueEntry_callBack($setId, $params, $this->callBack, $cfgRec['pid']);
                    break;
                case 3:
                    // External URL:
                    // Parameters:
                    $params = [
                        'indexConfigUid' => $cfgRec['uid'],
                        // General
                        'procInstructions' => ['[Index Cfg UID#' . $cfgRec['uid'] . ']'],
                        // General
                        'url' => $cfgRec['externalUrl'],
                        // Partly general... (for URL and file types)
                        'depth' => 0
                    ];
                    ‪$pObj->addQueueEntry_callBack($setId, $params, $this->callBack, $cfgRec['pid']);
                    break;
                case 4:
                    // Page tree
                    // Parameters:
                    $params = [
                        'indexConfigUid' => $cfgRec['uid'],
                        // General
                        'procInstructions' => ['[Index Cfg UID#' . $cfgRec['uid'] . ']'],
                        // General
                        'url' => (int)$cfgRec['alternative_source_pid'],
                        // Partly general... (for URL and file types and page tree (root))
                        'depth' => 0
                    ];
                    ‪$pObj->addQueueEntry_callBack($setId, $params, $this->callBack, $cfgRec['pid']);
                    break;
                case 5:
                    // Meta configuration, nothing to do:
                    // NOOP
                    break;
                default:
                    if (‪$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['crawler'][$cfgRec['type']]) {
                        $hookObj = GeneralUtility::makeInstance(‪$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['crawler'][$cfgRec['type']]);
                        // Parameters:
                        $params = [
                            'indexConfigUid' => $cfgRec['uid'],
                            // General
                            'procInstructions' => ['[Index Cfg UID#' . $cfgRec['uid'] . '/CUSTOM]'],
                            // General
                            'url' => $hookObj->initMessage($message)
                        ];
                        ‪$pObj->addQueueEntry_callBack($setId, $params, $this->callBack, $cfgRec['pid']);
                    }
            }
        }
        // Finally, look up all old index configurations which are finished and needs to be reset and done.
        $this->‪cleanUpOldRunningConfigurations();
    }
 
    public function ‪crawler_execute($params, &‪$pObj)
    {
        // Indexer configuration ID must exist:
        if ($params['indexConfigUid']) {
            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
                ->getQueryBuilderForTable('index_config');
            $queryBuilder->getRestrictions()->removeAll();
            // Load the indexing configuration record:
            $cfgRec = $queryBuilder
                ->select('*')
                ->from('index_config')
                ->where(
                    $queryBuilder->expr()->eq(
                        'uid',
                        $queryBuilder->createNamedParameter($params['indexConfigUid'], \PDO::PARAM_INT)
                    )
                )
                ->execute()
                ->fetch();
            if (is_array($cfgRec)) {
                // Unpack session data:
                $session_data = unserialize($cfgRec['session_data']);
                // Select which type:
                switch ($cfgRec['type']) {
                    case 1:
                        // Records:
                        $this->‪crawler_execute_type1($cfgRec, $session_data, $params, ‪$pObj);
                        break;
                    case 2:
                        // Files
                        $this->‪crawler_execute_type2($cfgRec, $session_data, $params, ‪$pObj);
                        break;
                    case 3:
                        // External URL:
                        $this->‪crawler_execute_type3($cfgRec, $session_data, $params, ‪$pObj);
                        break;
                    case 4:
                        // Page tree:
                        $this->‪crawler_execute_type4($cfgRec, $session_data, $params, ‪$pObj);
                        break;
                    case 5:
                        // Meta
                        // NOOP (should never enter here!)
                        break;
                    default:
                        if (‪$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['crawler'][$cfgRec['type']]) {
                            $hookObj = GeneralUtility::makeInstance(‪$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['crawler'][$cfgRec['type']]);
                            $this->pObj = ‪$pObj;
                            // For addQueueEntryForHook()
                            $ref = $this; // introduced for phpstan to not lose type information when passing $this into callUserFunction
                            $hookObj->indexOperation($cfgRec, $session_data, $params, $ref);
                        }
                }
                // Save process data which might be modified:
                GeneralUtility::makeInstance(ConnectionPool::class)
                    ->getConnectionForTable('index_config')
                    ->update(
                        'index_config',
                        ['session_data' => serialize($session_data)],
                        ['uid' => (int)$cfgRec['uid']]
                    );
            }
        }
        return ['log' => $params];
    }
 
    public function ‪crawler_execute_type1($cfgRec, &$session_data, $params, &‪$pObj)
    {
        if ($cfgRec['table2index'] && isset(‪$GLOBALS['TCA'][$cfgRec['table2index']])) {
            // Init session data array if not already:
            if (!is_array($session_data)) {
                $session_data = [
                    'uid' => 0
                ];
            }
            // Init:
            $pid = (int)$cfgRec['alternative_source_pid'] ?: $cfgRec['pid'];
            $numberOfRecords = $cfgRec['recordsbatch']
                ? ‪MathUtility::forceIntegerInRange($cfgRec['recordsbatch'], 1)
                : 100;
 
            // Get root line:
            $rootLine = $this->‪getUidRootLineForClosestTemplate($cfgRec['pid']);
            // Select
            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
                ->getQueryBuilderForTable($cfgRec['table2index']);
 
            $baseQueryBuilder = $queryBuilder->select('*')
                ->from($cfgRec['table2index'])
                ->where(
                    $queryBuilder->expr()->eq(
                        'pid',
                        $queryBuilder->createNamedParameter($pid, \PDO::PARAM_INT)
                    ),
                    $queryBuilder->expr()->gt(
                        'uid',
                        $queryBuilder->createNamedParameter($session_data['uid'], \PDO::PARAM_INT)
                    )
                );
            $result = $baseQueryBuilder
                ->setMaxResults($numberOfRecords)
                ->orderBy('uid')
                ->execute();
 
            // Traverse:
            while ($row = $result->fetch()) {
                // Index single record:
                $this->‪indexSingleRecord($row, $cfgRec, $rootLine);
                // Update the UID we last processed:
                $session_data['uid'] = $row['uid'];
            }
 
            $rowCount = $baseQueryBuilder->count('uid')->execute()->fetchColumn(0);
            // Finally, set entry for next indexing of batch of records:
            if ($rowCount) {
                $nparams = [
                    'indexConfigUid' => $cfgRec['uid'],
                    'url' => 'Records from UID#' . ($session_data['uid'] + 1) . '-?',
                    'procInstructions' => ['[Index Cfg UID#' . $cfgRec['uid'] . ']']
                ];
                ‪$pObj->addQueueEntry_callBack($cfgRec['set_id'], $nparams, $this->callBack, $cfgRec['pid']);
            }
        }
    }
 
    public function ‪crawler_execute_type2($cfgRec, &$session_data, $params, &‪$pObj)
    {
        // Prepare path, making it absolute and checking:
        $readpath = $params['url'];
        if (!GeneralUtility::isAbsPath($readpath)) {
            $readpath = GeneralUtility::getFileAbsFileName($readpath);
        }
        if (GeneralUtility::isAllowedAbsPath($readpath)) {
            if (@is_file($readpath)) {
                // If file, index it!
                // Get root line (need to provide this when indexing external files)
                $rl = $this->‪getUidRootLineForClosestTemplate($cfgRec['pid']);
                // (Re)-Indexing file on page.
                $indexerObj = $this->‪initializeIndexer($cfgRec['pid'], 0, 0, '', $rl, $cfgRec['uid'], $cfgRec['set_id']);
                $indexerObj->hash['phash'] = -1;
                // EXPERIMENT - but to avoid phash_t3 being written to file sections (otherwise they are removed when page is reindexed!!!)
                // Index document:
                $indexerObj->indexRegularDocument(‪PathUtility::stripPathSitePrefix($readpath), true);
            } elseif (@is_dir($readpath)) {
                // If dir, read content and create new pending items for log:
                // Select files and directories in path:
                $extList = implode(',', ‪GeneralUtility::trimExplode(',', $cfgRec['extensions'], true));
                $fileArr = [];
                $files = GeneralUtility::getAllFilesAndFoldersInPath($fileArr, $readpath, $extList, false, 0);
                $directoryList = ‪GeneralUtility::get_dirs($readpath);
                if (is_array($directoryList) && $params['depth'] < $cfgRec['depth']) {
                    foreach ($directoryList as $subdir) {
                        if ((string)$subdir != '') {
                            $files[] = $readpath . $subdir . '/';
                        }
                    }
                }
                $files = GeneralUtility::removePrefixPathFromList($files, ‪Environment::getPublicPath() . '/');
                // traverse the items and create log entries:
                foreach ($files as $path) {
                    $this->instanceCounter++;
                    if ($path !== $params['url']) {
                        // Parameters:
                        $nparams = [
                            'indexConfigUid' => $cfgRec['uid'],
                            'url' => $path,
                            'procInstructions' => ['[Index Cfg UID#' . $cfgRec['uid'] . ']'],
                            'depth' => $params['depth'] + 1
                        ];
                        ‪$pObj->addQueueEntry_callBack($cfgRec['set_id'], $nparams, $this->callBack, $cfgRec['pid'], ‪$GLOBALS['EXEC_TIME'] + $this->instanceCounter * $this->secondsPerExternalUrl);
                    }
                }
            }
        }
    }
 
    public function ‪crawler_execute_type3($cfgRec, &$session_data, $params, &‪$pObj)
    {
        // Init session data array if not already:
        if (!is_array($session_data)) {
            $session_data = [
                'urlLog' => [$params['url']]
            ];
        }
        // Index the URL:
        $rl = $this->‪getUidRootLineForClosestTemplate($cfgRec['pid']);
        $subUrls = $this->‪indexExtUrl($params['url'], $cfgRec['pid'], $rl, $cfgRec['uid'], $cfgRec['set_id']);
        // Add more elements to log now:
        if ($params['depth'] < $cfgRec['depth']) {
            foreach ($subUrls as $url) {
                if ($url = $this->‪checkUrl($url, $session_data['urlLog'], $cfgRec['externalUrl'])) {
                    if (!$this->‪checkDeniedSuburls($url, $cfgRec['url_deny'])) {
                        $this->instanceCounter++;
                        $session_data['urlLog'][] = $url;
                        // Parameters:
                        $nparams = [
                            'indexConfigUid' => $cfgRec['uid'],
                            'url' => $url,
                            'procInstructions' => ['[Index Cfg UID#' . $cfgRec['uid'] . ']'],
                            'depth' => $params['depth'] + 1
                        ];
                        ‪$pObj->addQueueEntry_callBack($cfgRec['set_id'], $nparams, $this->callBack, $cfgRec['pid'], ‪$GLOBALS['EXEC_TIME'] + $this->instanceCounter * $this->secondsPerExternalUrl);
                    }
                }
            }
        }
    }
 
    public function ‪crawler_execute_type4($cfgRec, &$session_data, $params, &‪$pObj)
    {
        // Base page uid:
        $pageUid = (int)$params['url'];
        // Get array of URLs from page:
        $pageRow = ‪BackendUtility::getRecord('pages', $pageUid);
        $res = ‪$pObj->getUrlsForPageRow($pageRow);
        $duplicateTrack = [];
        // Registry for duplicates
        $downloadUrls = [];
        // Dummy.
        // Submit URLs:
        if (!empty($res)) {
            foreach ($res as $paramSetKey => $vv) {
                ‪$pObj->urlListFromUrlArray($vv, $pageRow, ‪$GLOBALS['EXEC_TIME'], 30, 1, 0, $duplicateTrack, $downloadUrls, ['tx_indexedsearch_reindex']);
            }
        }
        // Add subpages to log now:
        if ($params['depth'] < $cfgRec['depth']) {
            // Subpages selected
            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('pages');
            $queryBuilder->getRestrictions()
                ->removeAll()
                ->add(GeneralUtility::makeInstance(DeletedRestriction::class));
            $result = $queryBuilder->select('uid', 'title')
                ->from('pages')
                ->where(
                    $queryBuilder->expr()->eq(
                        'pid',
                        $queryBuilder->createNamedParameter($pageUid, \PDO::PARAM_INT)
                    )
                )
                ->execute();
            // Traverse subpages and add to queue:
            while ($row = $result->fetch()) {
                $this->instanceCounter++;
                $url = 'pages:' . $row['uid'] . ': ' . $row['title'];
                $session_data['urlLog'][] = $url;
                // Parameters:
                $nparams = [
                    'indexConfigUid' => $cfgRec['uid'],
                    'url' => $row['uid'],
                    'procInstructions' => ['[Index Cfg UID#' . $cfgRec['uid'] . ']'],
                    'depth' => $params['depth'] + 1
                ];
                ‪$pObj->addQueueEntry_callBack(
                    $cfgRec['set_id'],
                    $nparams,
                    $this->callBack,
                    $cfgRec['pid'],
                    ‪$GLOBALS['EXEC_TIME'] + $this->instanceCounter * $this->secondsPerExternalUrl
                );
            }
        }
    }
 
    public function ‪cleanUpOldRunningConfigurations()
    {
        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
        // List of tables that store information related to the phash value
        $tablesToClean = [
            'index_phash',
            'index_rel',
            'index_section',
            'index_grlist',
            'index_fulltext',
            'index_debug'
        ];
 
        $queryBuilder = $connectionPool->getQueryBuilderForTable('index_config');
        $queryBuilder->getRestrictions()
            ->removeAll()
            ->add(GeneralUtility::makeInstance(DeletedRestriction::class));
 
        // Lookup running index configurations:
        $runningIndexingConfigurations = $queryBuilder->select('*')
            ->from('index_config')
            ->where($queryBuilder->expr()->neq('set_id', $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)))
            ->execute()
            ->fetchAll();
        // For each running configuration, look up how many log entries there are which are scheduled
        // for execution and if none, clear the "set_id" (means; Processing was DONE)
        foreach ($runningIndexingConfigurations as $cfgRec) {
            // Look for ended processes:
            $queued_items = $connectionPool->getConnectionForTable('tx_crawler_queue')
                ->count(
                    '*',
                    'tx_crawler_queue',
                    [
                        'set_id' => (int)$cfgRec['set_id'],
                        'exec_time' => 0
                    ]
                );
            if (!$queued_items) {
                // Lookup old phash rows:
                $queryBuilder = $connectionPool->getQueryBuilderForTable('index_phash');
                $oldPhashRows = $queryBuilder
                    ->select('phash')
                    ->from('index_phash')
                    ->where(
                        $queryBuilder->expr()->eq(
                            'freeIndexUid',
                            $queryBuilder->createNamedParameter($cfgRec['uid'], \PDO::PARAM_INT)
                        ),
                        $queryBuilder->expr()->neq(
                            'freeIndexSetId',
                            $queryBuilder->createNamedParameter($cfgRec['set_id'], \PDO::PARAM_INT)
                        )
                    )
                    ->execute()
                    ->fetchAll();
 
                // Removing old registrations for all tables
                foreach ($tablesToClean as $table) {
                    $queryBuilder = $connectionPool->getQueryBuilderForTable($table);
                    $queryBuilder->delete($table)
                        ->where(
                            $queryBuilder->expr()->in(
                                'phash',
                                $queryBuilder->createNamedParameter(
                                    array_column($oldPhashRows, 'phash'),
                                    Connection::PARAM_INT_ARRAY
                                )
                            )
                        )
                        ->execute();
                }
 
                // End process by updating index-config record:
                $connectionPool->getConnectionForTable('index_config')
                    ->update(
                        'index_config',
                        [
                            'set_id' => 0,
                            'session_data' => ''
                        ],
                        ['uid' => (int)$cfgRec['uid']]
                    );
            }
        }
    }
 
    /*****************************************
     *
     * Helper functions
     *
     *****************************************/
    public function ‪checkUrl($url, $urlLog, $baseUrl)
    {
        $url = (string)preg_replace('/\\/\\/$/', '/', $url);
        [$url] = explode('#', $url);
        if (strpos($url, '../') === false) {
            if (GeneralUtility::isFirstPartOfStr($url, $baseUrl)) {
                if (!in_array($url, $urlLog)) {
                    return $url;
                }
            }
        }
 
        return '';
    }
 
    public function ‪indexExtUrl($url, $pageId, $rl, $cfgUid, $setId)
    {
        // Index external URL:
        $indexerObj = $this->‪initializeIndexer($pageId, 0, 0, '', $rl, [], $cfgUid, $setId);
        $indexerObj->hash['phash'] = -1;
        // To avoid phash_t3 being written to file sections (otherwise they are removed when page is reindexed!!!)
        $indexerObj->indexExternalUrl($url);
        $url_qParts = parse_url($url);
        $baseAbsoluteHref = $url_qParts['scheme'] . '://' . $url_qParts['host'];
        $baseHref = $indexerObj->extractBaseHref($indexerObj->indexExternalUrl_content);
        if (!$baseHref) {
            // Extract base href from current URL
            $baseHref = $baseAbsoluteHref;
            $baseHref .= substr($url_qParts['path'], 0, (int)strrpos($url_qParts['path'], '/'));
        }
        $baseHref = rtrim($baseHref, '/');
        // Get URLs on this page:
        $subUrls = [];
        $list = $indexerObj->extractHyperLinks($indexerObj->indexExternalUrl_content);
        // Traverse links:
        foreach ($list as $count => $linkInfo) {
            // Decode entities:
            $subUrl = htmlspecialchars_decode($linkInfo['href']);
            $qParts = parse_url($subUrl);
            if (!$qParts['scheme']) {
                $relativeUrl = GeneralUtility::resolveBackPath($subUrl);
                if ($relativeUrl[0] === '/') {
                    $subUrl = $baseAbsoluteHref . $relativeUrl;
                } else {
                    $subUrl = $baseHref . '/' . $relativeUrl;
                }
            }
            $subUrls[] = $subUrl;
        }
        return $subUrls;
    }
 
    public function ‪indexSingleRecord($r, $cfgRec, $rl = null)
    {
        $rl = is_array($rl) ? $rl : $this->‪getUidRootLineForClosestTemplate($cfgRec['pid']);
        $fieldList = ‪GeneralUtility::trimExplode(',', $cfgRec['fieldlist'], true);
        $languageField = ‪$GLOBALS['TCA'][$cfgRec['table2index']]['ctrl']['languageField'];
        $sys_language_uid = $languageField ? $r[$languageField] : 0;
        parse_str(str_replace('###UID###', $r['uid'], $cfgRec['get_params']), $GETparams);
        // (Re)-Indexing a row from a table
        $indexerObj = $this->‪initializeIndexer($cfgRec['pid'], 0, $sys_language_uid, '', $rl, $GETparams, $cfgRec['uid'], $cfgRec['set_id']);
        $indexerObj->forceIndexing = true;
        $theContent = '';
        $theTitle = '';
        foreach ($fieldList as $k => $v) {
            if (!$k) {
                $theTitle = $r[$v];
            } else {
                $theContent .= $r[$v] . ' ';
            }
        }
        // Indexing the record as a page (but with parameters set)
        $this->‪indexAsTYPO3Page(
            $indexerObj,
            strip_tags(str_replace('<', ' <', $theTitle)),
            strip_tags(str_replace('<', ' <', $theContent)),
            $r[‪$GLOBALS['TCA'][$cfgRec['table2index']]['ctrl']['tstamp']],
            $r[‪$GLOBALS['TCA'][$cfgRec['table2index']]['ctrl']['crdate']],
            $r['uid']
        );
    }
 
    public function ‪getUidRootLineForClosestTemplate($id)
    {
        $rootLineUids = [];
        try {
            // Gets the rootLine
            $rootLine = GeneralUtility::makeInstance(RootlineUtility::class, $id)->get();
            // This generates the constants/config + hierarchy info for the template.
            $tmpl = GeneralUtility::makeInstance(ExtendedTemplateService::class);
            $tmpl->runThroughTemplates($rootLine);
            // Root line uids
            foreach ($tmpl->rootLine as $rlkey => $rldat) {
                $rootLineUids[$rlkey] = $rldat['uid'];
            }
        } catch (RootLineException $e) {
            // do nothing
        }
        return $rootLineUids;
    }
 
    public function ‪generateNextIndexingTime($cfgRec)
    {
        $currentTime = ‪$GLOBALS['EXEC_TIME'];
        // Now, find a midnight time to use for offset calculation. This has to differ depending on whether we have frequencies within a day or more than a day; Less than a day, we don't care which day to use for offset, more than a day we want to respect the currently entered day as offset regardless of when the script is run - thus the day-of-week used in case "Weekly" is selected will be respected
        if ($cfgRec['timer_frequency'] <= 24 * 3600) {
            $aMidNight = mktime(0, 0, 0) - 1 * 24 * 3600;
        } else {
            $lastTime = $cfgRec['timer_next_indexing'] ?: ‪$GLOBALS['EXEC_TIME'];
            $aMidNight = mktime(0, 0, 0, (int)date('m', $lastTime), (int)date('d', $lastTime), (int)date('y', $lastTime));
        }
        // Find last offset time plus frequency in seconds:
        $lastSureOffset = $aMidNight + ‪MathUtility::forceIntegerInRange($cfgRec['timer_offset'], 0, 86400);
        $frequencySeconds = ‪MathUtility::forceIntegerInRange($cfgRec['timer_frequency'], 1);
        // Now, find out how many blocks of the length of frequency there is until the next time:
        $frequencyBlocksUntilNextTime = ceil(($currentTime - $lastSureOffset) / $frequencySeconds);
        // Set next time to the offset + the frequencyblocks multiplied with the frequency length in seconds.
        return $lastSureOffset + $frequencyBlocksUntilNextTime * $frequencySeconds;
    }
 
    public function ‪checkDeniedSuburls($url, $url_deny)
    {
        if (trim($url_deny)) {
            $url_denyArray = ‪GeneralUtility::trimExplode(LF, $url_deny, true);
            foreach ($url_denyArray as $testurl) {
                if (GeneralUtility::isFirstPartOfStr($url, $testurl)) {
                    return true;
                }
            }
        }
        return false;
    }
 
    public function ‪addQueueEntryForHook($cfgRec, $title)
    {
        $nparams = [
            'indexConfigUid' => $cfgRec['uid'],
            // This must ALWAYS be the cfgRec uid!
            'url' => $title,
            'procInstructions' => ['[Index Cfg UID#' . $cfgRec['uid'] . ']']
        ];
        $this->pObj->addQueueEntry_callBack($cfgRec['set_id'], $nparams, $this->callBack, $cfgRec['pid']);
    }
 
    public function ‪deleteFromIndex($id)
    {
        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
 
        // Lookup old phash rows:
 
        $queryBuilder = $connectionPool->getQueryBuilderForTable('index_section');
        $oldPhashRows = $queryBuilder->select('phash')
            ->from('index_section')
            ->where(
                $queryBuilder->expr()->eq(
                    'page_id',
                    $queryBuilder->createNamedParameter($id, \PDO::PARAM_INT)
                )
            )
            ->execute()
            ->fetchAll();
 
        if (empty($oldPhashRows)) {
            return;
        }
 
        $tables = [
            'index_debug',
            'index_fulltext',
            'index_grlist',
            'index_phash',
            'index_rel',
            'index_section',
        ];
        foreach ($tables as $table) {
            $queryBuilder = $connectionPool->getQueryBuilderForTable($table);
            $queryBuilder->delete($table)
                ->where(
                    $queryBuilder->expr()->in(
                        'phash',
                        $queryBuilder->createNamedParameter(
                            array_column($oldPhashRows, 'phash'),
                            Connection::PARAM_INT_ARRAY
                        )
                    )
                )
                ->execute();
        }
    }
 
    /*************************
     *
     * Hook functions for DataHandler (indexing of records)
     *
     *************************/
    public function ‪processCmdmap_preProcess($command, $table, $id, $value, ‪$pObj)
    {
        // Clean up the index
        if ($command === 'delete' && $table === 'pages') {
            $this->‪deleteFromIndex((int)$id);
        }
    }
 
    public function ‪processDatamap_afterDatabaseOperations($status, $table, $id, $fieldArray, ‪$pObj)
    {
        // Check if any fields are actually updated:
        if (empty($fieldArray)) {
            return;
        }
        // Translate new ids.
        if ($status === 'new') {
            $id = ‪$pObj->substNEWwithIDs[$id];
        } elseif ($table === 'pages' && $status === 'update' && (array_key_exists('hidden', $fieldArray) && $fieldArray['hidden'] == 1 || array_key_exists('no_search', $fieldArray) && $fieldArray['no_search'] == 1)) {
            // If the page should be hidden or not indexed after update, delete index for this page
            $this->‪deleteFromIndex((int)$id);
        }
        // Get full record and if exists, search for indexing configurations:
        $currentRecord = ‪BackendUtility::getRecord($table, $id);
        if (is_array($currentRecord)) {
            // Select all (not running) indexing configurations of type "record" (1) and
            // which points to this table and is located on the same page as the record
            // or pointing to the right source PID
            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
                ->getQueryBuilderForTable('index_config');
            $result = $queryBuilder->select('*')
                ->from('index_config')
                ->where(
                    $queryBuilder->expr()->eq('set_id', $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)),
                    $queryBuilder->expr()->eq('type', $queryBuilder->createNamedParameter(1, \PDO::PARAM_INT)),
                    $queryBuilder->expr()->eq(
                        'table2index',
                        $queryBuilder->createNamedParameter($table, \PDO::PARAM_STR)
                    ),
                    $queryBuilder->expr()->orX(
                        $queryBuilder->expr()->andX(
                            $queryBuilder->expr()->eq(
                                'alternative_source_pid',
                                $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)
                            ),
                            $queryBuilder->expr()->eq(
                                'pid',
                                $queryBuilder->createNamedParameter($currentRecord['pid'], \PDO::PARAM_INT)
                            )
                        ),
                        $queryBuilder->expr()->eq(
                            'alternative_source_pid',
                            $queryBuilder->createNamedParameter($currentRecord['pid'], \PDO::PARAM_INT)
                        )
                    ),
                    $queryBuilder->expr()->eq(
                        'records_indexonchange',
                        $queryBuilder->createNamedParameter(1, \PDO::PARAM_INT)
                    )
                )
                ->execute();
 
            while ($cfgRec = $result->fetch()) {
                $this->‪indexSingleRecord($currentRecord, $cfgRec);
            }
        }
    }
 
    protected function ‪initializeIndexer($id, $type, $sys_language_uid, $MP, $uidRL, $queryArguments = [], $freeIndexUid = 0, $freeIndexSetId = 0): Indexer
    {
        $indexerObj = GeneralUtility::makeInstance(Indexer::class);
        // Setting up internal configuration from config array:
        // Information about page for which the indexing takes place
        $configuration = [
            // Page id  (int)
            'id' => $id,
            // Page type (int)
            'type' => $type,
            // sys_language UID of the language of the indexing (int)
            'sys_language_uid' => $sys_language_uid,
            // MP variable, if any (Mount Points) (string)
            'MP' => $MP,
            // Group list (hardcoded for now...)
            'gr_list' => '0,-1',
            'staticPageArguments' => $queryArguments,
            // Set to defaults
            'freeIndexUid' => $freeIndexUid,
            'freeIndexSetId' => $freeIndexSetId,
            // Root line uids
            'rootline_uids' => $uidRL,
 
            // Configuration of behavior
            // Whether to index external documents like PDF, DOC etc. (if possible)
            'index_externals' => 1,
            // Length of description text (max 250, default 200)
            'index_descrLgd' => 200,
            // Whether to index document keywords and description (if present)
            'index_metatags' => true
        ];
        $indexerObj->init($configuration);
        return $indexerObj;
    }
 
    protected function ‪indexAsTYPO3Page(Indexer $indexer, $title, $content, $mtime, $crdate = 0, $recordUid = 0)
    {
        // Content of page:
        $indexer->conf['mtime'] = $mtime;
        // Most recent modification time (seconds) of the content
        $indexer->conf['crdate'] = $crdate;
        // The creation date of the TYPO3 content
        $indexer->conf['recordUid'] = $recordUid;
        // UID of the record, if applicable
        // Construct fake HTML for parsing:
        $indexer->conf['content'] = '
        <html>
            <head>
                <title>' . htmlspecialchars($title) . '</title>
            </head>
            <body>
                ' . htmlspecialchars($content) . '
            </body>
        </html>';
        // Content string (HTML of TYPO3 page)
        // Initializing charset:
        $indexer->conf['metaCharset'] = 'utf-8';
        // Character set of content (will be converted to utf-8 during indexing)
        $indexer->conf['indexedDocTitle'] = '';
        // Alternative title for indexing
        // Index content as if it was a TYPO3 page:
        $indexer->indexTypo3PageContent();
    }
}