ReferenceIndex.php

Go to the documentation of this file.

<?php
namespace TYPO3\CMS\Core\Database;

/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

use Doctrine\DBAL\DBALException;
use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\Cache\CacheManager;
use TYPO3\CMS\Core\Configuration\FlexForm\FlexFormTools;
use TYPO3\CMS\Core\Database\Platform\PlatformInformation;
use TYPO3\CMS\Core\DataHandling\DataHandler;
use TYPO3\CMS\Core\Messaging\FlashMessage;
use TYPO3\CMS\Core\Messaging\FlashMessageService;
use TYPO3\CMS\Core\Registry;
use TYPO3\CMS\Core\Resource\File;
use TYPO3\CMS\Core\Resource\Folder;
use TYPO3\CMS\Core\Resource\ResourceFactory;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\PathUtility;
use TYPO3\CMS\Extbase\SignalSlot\Dispatcher;

class ReferenceIndex
{
    protected static $excludedTables = [
        'sys_log' => true,
        'sys_history' => true,
        'tx_extensionmanager_domain_model_extension' => true
    ];

    protected static $excludedColumns = [
        'uid' => true,
        'perms_userid' => true,
        'perms_groupid' => true,
        'perms_user' => true,
        'perms_group' => true,
        'perms_everybody' => true,
        'pid' => true
    ];

    protected static $cachePrefixTableRelationFields = 'core-refidx-tblRelFields-';

    public $temp_flexRelations = [];

    public $WSOL = false;

    public $relations = [];

    protected $recordCache = [];

    public $hashVersion = 1;

    protected $workspaceId = 0;

    protected $runtimeCache = null;

    protected $useRuntimeCache = false;

    public function __construct()
    {
        $this->runtimeCache = GeneralUtility::makeInstance(CacheManager::class)->getCache('cache_runtime');
    }

    public function setWorkspaceId($workspaceId)
    {
        $this->workspaceId = (int)$workspaceId;
    }

    public function getWorkspaceId()
    {
        return $this->workspaceId;
    }

    public function updateRefIndexTable($tableName, $uid, $testOnly = false)
    {
        // First, secure that the index table is not updated with workspace tainted relations:
        $this->WSOL = false;

        // Init:
        $result = [
            'keptNodes' => 0,
            'deletedNodes' => 0,
            'addedNodes' => 0
        ];

        $uid = $uid ? (int)$uid : 0;
        if (!$uid) {
            return $result;
        }

        // If this table cannot contain relations, skip it
        if ($this->shouldExcludeTableFromReferenceIndex($tableName)) {
            return $result;
        }

        // Fetch tableRelationFields and save them in cache if not there yet
        $cacheId = static::$cachePrefixTableRelationFields . $tableName;
        if (!$this->useRuntimeCache || !$this->runtimeCache->has($cacheId)) {
            $tableRelationFields = $this->fetchTableRelationFields($tableName);
            $this->runtimeCache->set($cacheId, $tableRelationFields);
        } else {
            $tableRelationFields = $this->runtimeCache->get($cacheId);
        }

        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
        $connection = $connectionPool->getConnectionForTable('sys_refindex');

        // Get current index from Database with hash as index using $uidIndexField
        // no restrictions are needed, since sys_refindex is not a TCA table
        $queryBuilder = $connection->createQueryBuilder();
        $queryBuilder->getRestrictions()->removeAll();
        $queryResult = $queryBuilder->select('hash')->from('sys_refindex')->where(
            $queryBuilder->expr()->eq('tablename', $queryBuilder->createNamedParameter($tableName, \PDO::PARAM_STR)),
            $queryBuilder->expr()->eq('recuid', $queryBuilder->createNamedParameter($uid, \PDO::PARAM_INT)),
            $queryBuilder->expr()->eq(
                'workspace',
                $queryBuilder->createNamedParameter($this->getWorkspaceId(), \PDO::PARAM_INT)
            )
        )->execute();
        $currentRelationHashes = [];
        while ($relation = $queryResult->fetch()) {
            $currentRelationHashes[$relation['hash']] = true;
        }

        // If the table has fields which could contain relations and the record does exist (including deleted-flagged)
        if ($tableRelationFields !== '') {
            $existingRecord = $this->getRecordRawCached($tableName, $uid);
            if ($existingRecord) {
                // Table has relation fields and record exists - get relations
                $this->relations = [];
                $relations = $this->generateDataUsingRecord($tableName, $existingRecord);
                if (!is_array($relations)) {
                    return $result;
                }
                // Traverse the generated index:
                foreach ($relations as &$relation) {
                    if (!is_array($relation)) {
                        continue;
                    }
                    $relation['hash'] = md5(implode('
                    // First, check if already indexed and if so, unset that row (so in the end we know which rows to remove!)
                    if (isset($currentRelationHashes[$relation['hash']])) {
                        unset($currentRelationHashes[$relation['hash']]);
                        $result['keptNodes']++;
                        $relation['_ACTION'] = 'KEPT';
                    } else {
                        // If new, add it:
                        if (!$testOnly) {
                            $connection->insert('sys_refindex', $relation);
                        }
                        $result['addedNodes']++;
                        $relation['_ACTION'] = 'ADDED';
                    }
                }
                $result['relations'] = $relations;
            }
        }

        // If any old are left, remove them:
        if (!empty($currentRelationHashes)) {
            $hashList = array_keys($currentRelationHashes);
            if (!empty($hashList)) {
                $result['deletedNodes'] = count($hashList);
                $result['deletedNodes_hashList'] = implode(',', $hashList);
                if (!$testOnly) {
                    $maxBindParameters = PlatformInformation::getMaxBindParameters($connection->getDatabasePlatform());
                    foreach (array_chunk($hashList, $maxBindParameters - 10, true) as $chunk) {
                        if (empty($chunk)) {
                            continue;
                        }
                        $queryBuilder = $connection->createQueryBuilder();
                        $queryBuilder
                            ->delete('sys_refindex')
                            ->where(
                                $queryBuilder->expr()->in(
                                    'hash',
                                    $queryBuilder->createNamedParameter($chunk, Connection::PARAM_STR_ARRAY)
                                )
                            )
                            ->execute();
                    }
                }
            }
        }

        return $result;
    }

    public function generateRefIndexData($tableName, $uid)
    {
        if (!isset($GLOBALS['TCA'][$tableName])) {
            return null;
        }

        $this->relations = [];

        $record = null;
        $uid = $uid ? (int)$uid : 0;
        if ($uid) {
            // Get raw record from DB
            $record = $this->getRecordRawCached($tableName, $uid);
        }

        if (!is_array($record)) {
            return null;
        }

        return $this->generateDataUsingRecord($tableName, $record);
    }

    protected function generateDataUsingRecord(string $tableName, array $record): array
    {
        $this->relations = [];
        $deleteField = $GLOBALS['TCA'][$tableName]['ctrl']['delete'];

        // Is the record deleted?
        $deleted = $deleteField && $record[$deleteField] ? 1 : 0;

        // Get all relations from record:
        $recordRelations = $this->getRelations($tableName, $record);
        // Traverse those relations, compile records to insert in table:
        foreach ($recordRelations as $fieldName => $fieldRelations) {
            // Based on type
            switch ((string)$fieldRelations['type']) {
                case 'db':
                    $this->createEntryDataForDatabaseRelationsUsingRecord($tableName, $record, $fieldName, '', $deleted, $fieldRelations['itemArray']);
                    break;
                case 'file_reference':
                    // not used (see getRelations()), but fallback to file
                case 'file':
                    $this->createEntryDataForFileRelationsUsingRecord($tableName, $record, $fieldName, '', $deleted, $fieldRelations['newValueFiles']);
                    break;
                case 'flex':
                    // DB references in FlexForms
                    if (is_array($fieldRelations['flexFormRels']['db'])) {
                        foreach ($fieldRelations['flexFormRels']['db'] as $flexPointer => $subList) {
                            $this->createEntryDataForDatabaseRelationsUsingRecord($tableName, $record, $fieldName, $flexPointer, $deleted, $subList);
                        }
                    }
                    // File references in FlexForms
                    // @todo #65463 Test correct handling of file references in FlexForms
                    if (is_array($fieldRelations['flexFormRels']['file'])) {
                        foreach ($fieldRelations['flexFormRels']['file'] as $flexPointer => $subList) {
                            $this->createEntryDataForFileRelationsUsingRecord($tableName, $record, $fieldName, $flexPointer, $deleted, $subList);
                        }
                    }
                    // Soft references in FlexForms
                    // @todo #65464 Test correct handling of soft references in FlexForms
                    if (is_array($fieldRelations['flexFormRels']['softrefs'])) {
                        foreach ($fieldRelations['flexFormRels']['softrefs'] as $flexPointer => $subList) {
                            $this->createEntryDataForSoftReferencesUsingRecord($tableName, $record, $fieldName, $flexPointer, $deleted, $subList['keys']);
                        }
                    }
                    break;
            }
            // Soft references in the field
            if (is_array($fieldRelations['softrefs'])) {
                $this->createEntryDataForSoftReferencesUsingRecord($tableName, $record, $fieldName, '', $deleted, $fieldRelations['softrefs']['keys']);
            }
        }

        return array_filter($this->relations);
    }

    public function createEntryData($table, $uid, $field, $flexPointer, $deleted, $ref_table, $ref_uid, $ref_string = '', $sort = -1, $softref_key = '', $softref_id = '')
    {
        $uid = $uid ? (int)$uid : 0;
        if (!$uid) {
            return null;
        }
        return $this->createEntryDataUsingRecord(
            (string)$table,
            $this->getRecordRawCached($table, $uid),
            (string)$field,
            (string)$flexPointer,
            $deleted ? (int)$deleted : 0,
            (string)$ref_table,
            $ref_uid ? (int)$ref_uid : 0,
            (string)$ref_string,
            $sort ? (int)$sort : 0,
            (string)$softref_key,
            (string)$softref_id
        );
    }

    protected function createEntryDataUsingRecord(string $tableName, array $record, string $fieldName, string $flexPointer, int $deleted, string $referencedTable, int $referencedUid, string $referenceString = '', int $sort = -1, string $softReferenceKey = '', string $softReferenceId = '')
    {
        $workspaceId = 0;
        if (BackendUtility::isTableWorkspaceEnabled($tableName)) {
            $workspaceId = $this->getWorkspaceId();
            if (isset($record['t3ver_wsid']) && (int)$record['t3ver_wsid'] !== $workspaceId) {
                // The given record is workspace-enabled but doesn't live in the selected workspace => don't add index as it's not actually there
                return false;
            }
        }
        return [
            'tablename' => $tableName,
            'recuid' => $record['uid'],
            'field' => $fieldName,
            'flexpointer' => $flexPointer,
            'softref_key' => $softReferenceKey,
            'softref_id' => $softReferenceId,
            'sorting' => $sort,
            'deleted' => (int)$deleted,
            'workspace' => $workspaceId,
            'ref_table' => $referencedTable,
            'ref_uid' => $referencedUid,
            'ref_string' => mb_substr($referenceString, 0, 1024)
        ];
    }

    public function createEntryData_dbRels($table, $uid, $fieldName, $flexPointer, $deleted, $items)
    {
        $uid = $uid ? (int)$uid : 0;
        if (!$uid) {
            return;
        }
        $this->createEntryDataForDatabaseRelationsUsingRecord(
            (string)$table,
            $this->getRecordRawCached($table, $uid),
            (string)$fieldName,
            (string)$flexPointer,
            $deleted ? (int)$deleted : 0,
            (array)$items
        );
    }

    protected function createEntryDataForDatabaseRelationsUsingRecord(string $tableName, array $record, string $fieldName, string $flexPointer, int $deleted, array $items)
    {
        foreach ($items as $sort => $i) {
            $this->relations[] = $this->createEntryDataUsingRecord($tableName, $record, $fieldName, $flexPointer, $deleted, $i['table'], $i['id'], '', $sort);
        }
    }

    public function createEntryData_fileRels($table, $uid, $fieldName, $flexPointer, $deleted, $items)
    {
        $uid = $uid ? (int)$uid : 0;
        if (!$uid) {
            return;
        }
        $this->createEntryDataForFileRelationsUsingRecord(
            (string)$table,
            $this->getRecordRawCached($table, $uid),
            (string)$fieldName,
            (string)$flexPointer,
            $deleted ? (int)$deleted : 0,
            (array)$items
        );
    }

    protected function createEntryDataForFileRelationsUsingRecord(string $tableName, array $record, string $fieldName, string $flexPointer, int $deleted, array $items)
    {
        foreach ($items as $sort => $i) {
            $filePath = $i['ID_absFile'];
            if (GeneralUtility::isFirstPartOfStr($filePath, PATH_site)) {
                $filePath = PathUtility::stripPathSitePrefix($filePath);
            }
            $this->relations[] = $this->createEntryDataUsingRecord(
                $tableName,
                $record,
                $fieldName,
                $flexPointer,
                $deleted,
                '_FILE',
                0,
                $filePath,
                $sort
            );
        }
    }

    public function createEntryData_softreferences($table, $uid, $fieldName, $flexPointer, $deleted, $keys)
    {
        $uid = $uid ? (int)$uid : 0;
        if (!$uid || !is_array($keys)) {
            return;
        }
        $this->createEntryDataForSoftReferencesUsingRecord(
            (string)$table,
            $this->getRecordRawCached($table, $uid),
            (string)$fieldName,
            (string)$flexPointer,
            $deleted ? (int)$deleted : 0,
            (array)$keys
        );
    }

    protected function createEntryDataForSoftReferencesUsingRecord(string $tableName, array $record, string $fieldName, string $flexPointer, int $deleted, array $keys)
    {
        foreach ($keys as $spKey => $elements) {
            if (is_array($elements)) {
                foreach ($elements as $subKey => $el) {
                    if (is_array($el['subst'])) {
                        switch ((string)$el['subst']['type']) {
                            case 'db':
                                list($referencedTable, $referencedUid) = explode(':', $el['subst']['recordRef']);
                                $this->relations[] = $this->createEntryDataUsingRecord($tableName, $record, $fieldName, $flexPointer, $deleted, $referencedTable, $referencedUid, '', -1, $spKey, $subKey);
                                break;
                            case 'file_reference':
                                // not used (see getRelations()), but fallback to file
                            case 'file':
                                $this->relations[] = $this->createEntryDataUsingRecord($tableName, $record, $fieldName, $flexPointer, $deleted, '_FILE', 0, $el['subst']['relFileName'], -1, $spKey, $subKey);
                                break;
                            case 'string':
                                $this->relations[] = $this->createEntryDataUsingRecord($tableName, $record, $fieldName, $flexPointer, $deleted, '_STRING', 0, $el['subst']['tokenValue'], -1, $spKey, $subKey);
                                break;
                        }
                    }
                }
            }
        }
    }

    /*******************************
     *
     * Get relations from table row
     *
     *******************************/

    public function getRelations($table, $row, $onlyField = '')
    {
        // Initialize:
        $uid = $row['uid'];
        $outRow = [];
        foreach ($row as $field => $value) {
            if ($this->shouldExcludeTableColumnFromReferenceIndex($table, $field, $onlyField) === false) {
                $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
                // Add files
                $resultsFromFiles = $this->getRelations_procFiles($value, $conf, $uid);
                if (!empty($resultsFromFiles)) {
                    // We have to fill different arrays here depending on the result.
                    // internal_type file is still a relation of type file and
                    // since http://forge.typo3.org/issues/49538 internal_type file_reference
                    // is a database relation to a sys_file record
                    $fileResultsFromFiles = [];
                    $dbResultsFromFiles = [];
                    foreach ($resultsFromFiles as $resultFromFiles) {
                        if (isset($resultFromFiles['table']) && $resultFromFiles['table'] === 'sys_file') {
                            $dbResultsFromFiles[] = $resultFromFiles;
                        } else {
                            // Creates an entry for the field with all the files:
                            $fileResultsFromFiles[] = $resultFromFiles;
                        }
                    }
                    if (!empty($fileResultsFromFiles)) {
                        $outRow[$field] = [
                            'type' => 'file',
                            'newValueFiles' => $fileResultsFromFiles
                        ];
                    }
                    if (!empty($dbResultsFromFiles)) {
                        $outRow[$field] = [
                            'type' => 'db',
                            'itemArray' => $dbResultsFromFiles
                        ];
                    }
                }
                // Add a softref definition for link fields if the TCA does not specify one already
                if ($conf['type'] === 'input' && $conf['renderType'] === 'inputLink' && empty($conf['softref'])) {
                    $conf['softref'] = 'typolink';
                }
                // Add DB:
                $resultsFromDatabase = $this->getRelations_procDB($value, $conf, $uid, $table);
                if (!empty($resultsFromDatabase)) {
                    // Create an entry for the field with all DB relations:
                    $outRow[$field] = [
                        'type' => 'db',
                        'itemArray' => $resultsFromDatabase
                    ];
                }
                // For "flex" fieldtypes we need to traverse the structure looking for file and db references of course!
                if ($conf['type'] === 'flex') {
                    // Get current value array:
                    // NOTICE: failure to resolve Data Structures can lead to integrity problems with the reference index. Please look up
                    // the note in the JavaDoc documentation for the function FlexFormTools->getDataStructureIdentifier()
                    $currentValueArray = GeneralUtility::xml2array($value);
                    // Traversing the XML structure, processing files:
                    if (is_array($currentValueArray)) {
                        $this->temp_flexRelations = [
                            'db' => [],
                            'file' => [],
                            'softrefs' => []
                        ];
                        // Create and call iterator object:
                        $flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
                        $flexFormTools->traverseFlexFormXMLData($table, $field, $row, $this, 'getRelations_flexFormCallBack');
                        // Create an entry for the field:
                        $outRow[$field] = [
                            'type' => 'flex',
                            'flexFormRels' => $this->temp_flexRelations
                        ];
                    }
                }
                // Soft References:
                if ((string)$value !== '') {
                    $softRefValue = $value;
                    if (!empty($conf['softref'])) {
                        $softRefs = BackendUtility::explodeSoftRefParserList($conf['softref']);
                        foreach ($softRefs as $spKey => $spParams) {
                            $softRefObj = BackendUtility::softRefParserObj($spKey);
                            if (is_object($softRefObj)) {
                                $resultArray = $softRefObj->findRef($table, $field, $uid, $softRefValue, $spKey, $spParams);
                                if (is_array($resultArray)) {
                                    $outRow[$field]['softrefs']['keys'][$spKey] = $resultArray['elements'];
                                    if ((string)$resultArray['content'] !== '') {
                                        $softRefValue = $resultArray['content'];
                                    }
                                }
                            }
                        }
                    }
                    if (!empty($outRow[$field]['softrefs']) && (string)$value !== (string)$softRefValue && strpos($softRefValue, '{softref:') !== false) {
                        $outRow[$field]['softrefs']['tokenizedContent'] = $softRefValue;
                    }
                }
            }
        }
        return $outRow;
    }

    public function getRelations_flexFormCallBack($dsArr, $dataValue, $PA, $structurePath)
    {
        // Removing "data/" in the beginning of path (which points to location in data array)
        $structurePath = substr($structurePath, 5) . '/';
        $dsConf = $dsArr['TCEforms']['config'];
        // Implode parameter values:
        list($table, $uid, $field) = [
            $PA['table'],
            $PA['uid'],
            $PA['field']
        ];
        // Add files
        $resultsFromFiles = $this->getRelations_procFiles($dataValue, $dsConf, $uid);
        if (!empty($resultsFromFiles)) {
            // We have to fill different arrays here depending on the result.
            // internal_type file is still a relation of type file and
            // since http://forge.typo3.org/issues/49538 internal_type file_reference
            // is a database relation to a sys_file record
            $fileResultsFromFiles = [];
            $dbResultsFromFiles = [];
            foreach ($resultsFromFiles as $resultFromFiles) {
                if (isset($resultFromFiles['table']) && $resultFromFiles['table'] === 'sys_file') {
                    $dbResultsFromFiles[] = $resultFromFiles;
                } else {
                    $fileResultsFromFiles[] = $resultFromFiles;
                }
            }
            if (!empty($fileResultsFromFiles)) {
                $this->temp_flexRelations['file'][$structurePath] = $fileResultsFromFiles;
            }
            if (!empty($dbResultsFromFiles)) {
                $this->temp_flexRelations['db'][$structurePath] = $dbResultsFromFiles;
            }
        }
        // Add a softref definition for link fields if the TCA does not specify one already
        if ($dsConf['type'] === 'input' && $dsConf['renderType'] === 'inputLink' && empty($dsConf['softref'])) {
            $dsConf['softref'] = 'typolink';
        }
        // Add DB:
        $resultsFromDatabase = $this->getRelations_procDB($dataValue, $dsConf, $uid, $table);
        if (!empty($resultsFromDatabase)) {
            // Create an entry for the field with all DB relations:
            $this->temp_flexRelations['db'][$structurePath] = $resultsFromDatabase;
        }
        // Soft References:
        if (is_array($dataValue) || (string)$dataValue !== '') {
            $softRefValue = $dataValue;
            $softRefs = BackendUtility::explodeSoftRefParserList($dsConf['softref']);
            if ($softRefs !== false) {
                foreach ($softRefs as $spKey => $spParams) {
                    $softRefObj = BackendUtility::softRefParserObj($spKey);
                    if (is_object($softRefObj)) {
                        $resultArray = $softRefObj->findRef($table, $field, $uid, $softRefValue, $spKey, $spParams, $structurePath);
                        if (is_array($resultArray) && is_array($resultArray['elements'])) {
                            $this->temp_flexRelations['softrefs'][$structurePath]['keys'][$spKey] = $resultArray['elements'];
                            if ((string)$resultArray['content'] !== '') {
                                $softRefValue = $resultArray['content'];
                            }
                        }
                    }
                }
            }
            if (!empty($this->temp_flexRelations['softrefs']) && (string)$dataValue !== (string)$softRefValue) {
                $this->temp_flexRelations['softrefs'][$structurePath]['tokenizedContent'] = $softRefValue;
            }
        }
    }

    public function getRelations_procFiles($value, $conf, $uid)
    {
        if ($conf['type'] !== 'group' || ($conf['internal_type'] !== 'file' && $conf['internal_type'] !== 'file_reference')) {
            return false;
        }

        // Collect file values in array:
        if ($conf['MM']) {
            $theFileValues = [];
            $dbAnalysis = GeneralUtility::makeInstance(RelationHandler::class);
            $dbAnalysis->start('', 'files', $conf['MM'], $uid);
            foreach ($dbAnalysis->itemArray as $someval) {
                if ($someval['id']) {
                    $theFileValues[] = $someval['id'];
                }
            }
        } else {
            $theFileValues = explode(',', $value);
        }
        // Traverse the files and add them:
        $uploadFolder = $conf['internal_type'] === 'file' ? $conf['uploadfolder'] : '';
        $destinationFolder = $this->destPathFromUploadFolder($uploadFolder);
        $newValueFiles = [];
        foreach ($theFileValues as $file) {
            if (trim($file)) {
                $realFile = $destinationFolder . '/' . trim($file);
                $newValueFile = [
                    'filename' => basename($file),
                    'ID' => md5($realFile),
                    'ID_absFile' => $realFile
                ];
                // Set sys_file and id for referenced files
                if ($conf['internal_type'] === 'file_reference') {
                    try {
                        $file = ResourceFactory::getInstance()->retrieveFileOrFolderObject($file);
                        if ($file instanceof File || $file instanceof Folder) {
                            // For setting this as sys_file relation later, the keys filename, ID and ID_absFile
                            // have not to be included, because the are not evaluated for db relations.
                            $newValueFile = [
                                'table' => 'sys_file',
                                'id' => $file->getUid()
                            ];
                        }
                    } catch (\Exception $e) {
                    }
                }
                $newValueFiles[] = $newValueFile;
            }
        }
        return $newValueFiles;
    }

    public function getRelations_procDB($value, $conf, $uid, $table = '')
    {
        // Get IRRE relations
        if (empty($conf)) {
            return false;
        }
        if ($conf['type'] === 'inline' && !empty($conf['foreign_table']) && empty($conf['MM'])) {
            $dbAnalysis = GeneralUtility::makeInstance(RelationHandler::class);
            $dbAnalysis->setUseLiveReferenceIds(false);
            $dbAnalysis->start($value, $conf['foreign_table'], '', $uid, $table, $conf);
            return $dbAnalysis->itemArray;
            // DB record lists:
        }
        if ($this->isDbReferenceField($conf)) {
            $allowedTables = $conf['type'] === 'group' ? $conf['allowed'] : $conf['foreign_table'];
            if ($conf['MM_opposite_field']) {
                return [];
            }
            $dbAnalysis = GeneralUtility::makeInstance(RelationHandler::class);
            $dbAnalysis->start($value, $allowedTables, $conf['MM'], $uid, $table, $conf);
            return $dbAnalysis->itemArray;
        }
        return false;
    }

    /*******************************
     *
     * Setting values
     *
     *******************************/

    public function setReferenceValue($hash, $newValue, $returnDataArray = false, $bypassWorkspaceAdminCheck = false)
    {
        $backendUser = $this->getBackendUser();
        if ($backendUser->workspace === 0 && $backendUser->isAdmin() || $bypassWorkspaceAdminCheck) {
            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_refindex');
            $queryBuilder->getRestrictions()->removeAll();

            // Get current index from Database
            $referenceRecord = $queryBuilder
                ->select('*')
                ->from('sys_refindex')
                ->where(
                    $queryBuilder->expr()->eq('hash', $queryBuilder->createNamedParameter($hash, \PDO::PARAM_STR))
                )
                ->setMaxResults(1)
                ->execute()
                ->fetch();

            // Check if reference existed.
            if (!is_array($referenceRecord)) {
                return 'ERROR: No reference record with hash="' . $hash . '" was found!';
            }

            if (empty($GLOBALS['TCA'][$referenceRecord['tablename']])) {
                return 'ERROR: Table "' . $referenceRecord['tablename'] . '" was not in TCA!';
            }

            // Get that record from database
            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
                ->getQueryBuilderForTable($referenceRecord['tablename']);
            $queryBuilder->getRestrictions()->removeAll();
            $record = $queryBuilder
                ->select('*')
                ->from($referenceRecord['tablename'])
                ->where(
                    $queryBuilder->expr()->eq(
                        'uid',
                        $queryBuilder->createNamedParameter($referenceRecord['recuid'], \PDO::PARAM_INT)
                    )
                )
                ->setMaxResults(1)
                ->execute()
                ->fetch();

            if (is_array($record)) {
                // Get relation for single field from record
                $recordRelations = $this->getRelations($referenceRecord['tablename'], $record, $referenceRecord['field']);
                if ($fieldRelation = $recordRelations[$referenceRecord['field']]) {
                    // Initialize data array that is to be sent to DataHandler afterwards:
                    $dataArray = [];
                    // Based on type
                    switch ((string)$fieldRelation['type']) {
                        case 'db':
                            $error = $this->setReferenceValue_dbRels($referenceRecord, $fieldRelation['itemArray'], $newValue, $dataArray);
                            if ($error) {
                                return $error;
                            }
                            break;
                        case 'file_reference':
                            // not used (see getRelations()), but fallback to file
                        case 'file':
                            $error = $this->setReferenceValue_fileRels($referenceRecord, $fieldRelation['newValueFiles'], $newValue, $dataArray);
                            if ($error) {
                                return $error;
                            }
                            break;
                        case 'flex':
                            // DB references in FlexForms
                            if (is_array($fieldRelation['flexFormRels']['db'][$referenceRecord['flexpointer']])) {
                                $error = $this->setReferenceValue_dbRels($referenceRecord, $fieldRelation['flexFormRels']['db'][$referenceRecord['flexpointer']], $newValue, $dataArray, $referenceRecord['flexpointer']);
                                if ($error) {
                                    return $error;
                                }
                            }
                            // File references in FlexForms
                            if (is_array($fieldRelation['flexFormRels']['file'][$referenceRecord['flexpointer']])) {
                                $error = $this->setReferenceValue_fileRels($referenceRecord, $fieldRelation['flexFormRels']['file'][$referenceRecord['flexpointer']], $newValue, $dataArray, $referenceRecord['flexpointer']);
                                if ($error) {
                                    return $error;
                                }
                            }
                            // Soft references in FlexForms
                            if ($referenceRecord['softref_key'] && is_array($fieldRelation['flexFormRels']['softrefs'][$referenceRecord['flexpointer']]['keys'][$referenceRecord['softref_key']])) {
                                $error = $this->setReferenceValue_softreferences($referenceRecord, $fieldRelation['flexFormRels']['softrefs'][$referenceRecord['flexpointer']], $newValue, $dataArray, $referenceRecord['flexpointer']);
                                if ($error) {
                                    return $error;
                                }
                            }
                            break;
                    }
                    // Soft references in the field:
                    if ($referenceRecord['softref_key'] && is_array($fieldRelation['softrefs']['keys'][$referenceRecord['softref_key']])) {
                        $error = $this->setReferenceValue_softreferences($referenceRecord, $fieldRelation['softrefs'], $newValue, $dataArray);
                        if ($error) {
                            return $error;
                        }
                    }
                    // Data Array, now ready to be sent to DataHandler
                    if ($returnDataArray) {
                        return $dataArray;
                    }
                    // Execute CMD array:
                    $dataHandler = GeneralUtility::makeInstance(DataHandler::class);
                    $dataHandler->dontProcessTransformations = true;
                    $dataHandler->bypassWorkspaceRestrictions = true;
                    $dataHandler->bypassFileHandling = true;
                    // Otherwise this cannot update things in deleted records...
                    $dataHandler->bypassAccessCheckForRecords = true;
                    // Check has been done previously that there is a backend user which is Admin and also in live workspace
                    $dataHandler->start($dataArray, []);
                    $dataHandler->process_datamap();
                    // Return errors if any:
                    if (!empty($dataHandler->errorLog)) {
                        return LF . 'DataHandler:' . implode((LF . 'DataHandler:'), $dataHandler->errorLog);
                    }
                }
            }
        } else {
            return 'ERROR: BE_USER object is not admin OR not in workspace 0 (Live)';
        }

        return false;
    }

    public function setReferenceValue_dbRels($refRec, $itemArray, $newValue, &$dataArray, $flexPointer = '')
    {
        if ((int)$itemArray[$refRec['sorting']]['id'] === (int)$refRec['ref_uid'] && (string)$itemArray[$refRec['sorting']]['table'] === (string)$refRec['ref_table']) {
            // Setting or removing value:
            // Remove value:
            if ($newValue === null) {
                unset($itemArray[$refRec['sorting']]);
            } else {
                list($itemArray[$refRec['sorting']]['table'], $itemArray[$refRec['sorting']]['id']) = explode(':', $newValue);
            }
            // Traverse and compile new list of records:
            $saveValue = [];
            foreach ($itemArray as $pair) {
                $saveValue[] = $pair['table'] . '_' . $pair['id'];
            }
            // Set in data array:
            if ($flexPointer) {
                $flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
                $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']]['data'] = [];
                $flexFormTools->setArrayValueByPath(substr($flexPointer, 0, -1), $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']]['data'], implode(',', $saveValue));
            } else {
                $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']] = implode(',', $saveValue);
            }
        } else {
            return 'ERROR: table:id pair "' . $refRec['ref_table'] . ':' . $refRec['ref_uid'] . '" did not match that of the record ("' . $itemArray[$refRec['sorting']]['table'] . ':' . $itemArray[$refRec['sorting']]['id'] . '") in sorting index "' . $refRec['sorting'] . '"';
        }

        return false;
    }

    public function setReferenceValue_fileRels($refRec, $itemArray, $newValue, &$dataArray, $flexPointer = '')
    {
        $ID_absFile = PathUtility::stripPathSitePrefix($itemArray[$refRec['sorting']]['ID_absFile']);
        if ($ID_absFile === (string)$refRec['ref_string'] && $refRec['ref_table'] === '_FILE') {
            // Setting or removing value:
            // Remove value:
            if ($newValue === null) {
                unset($itemArray[$refRec['sorting']]);
            } else {
                $itemArray[$refRec['sorting']]['filename'] = $newValue;
            }
            // Traverse and compile new list of records:
            $saveValue = [];
            foreach ($itemArray as $fileInfo) {
                $saveValue[] = $fileInfo['filename'];
            }
            // Set in data array:
            if ($flexPointer) {
                $flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
                $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']]['data'] = [];
                $flexFormTools->setArrayValueByPath(substr($flexPointer, 0, -1), $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']]['data'], implode(',', $saveValue));
            } else {
                $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']] = implode(',', $saveValue);
            }
        } else {
            return 'ERROR: either "' . $refRec['ref_table'] . '" was not "_FILE" or file PATH_site+"' . $refRec['ref_string'] . '" did not match that of the record ("' . $itemArray[$refRec['sorting']]['ID_absFile'] . '") in sorting index "' . $refRec['sorting'] . '"';
        }

        return false;
    }

    public function setReferenceValue_softreferences($refRec, $softref, $newValue, &$dataArray, $flexPointer = '')
    {
        if (!is_array($softref['keys'][$refRec['softref_key']][$refRec['softref_id']])) {
            return 'ERROR: Soft reference parser key "' . $refRec['softref_key'] . '" or the index "' . $refRec['softref_id'] . '" was not found.';
        }

        // Set new value:
        $softref['keys'][$refRec['softref_key']][$refRec['softref_id']]['subst']['tokenValue'] = '' . $newValue;
        // Traverse softreferences and replace in tokenized content to rebuild it with new value inside:
        foreach ($softref['keys'] as $sfIndexes) {
            foreach ($sfIndexes as $data) {
                $softref['tokenizedContent'] = str_replace('{softref:' . $data['subst']['tokenID'] . '}', $data['subst']['tokenValue'], $softref['tokenizedContent']);
            }
        }
        // Set in data array:
        if (!strstr($softref['tokenizedContent'], '{softref:')) {
            if ($flexPointer) {
                $flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
                $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']]['data'] = [];
                $flexFormTools->setArrayValueByPath(substr($flexPointer, 0, -1), $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']]['data'], $softref['tokenizedContent']);
            } else {
                $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']] = $softref['tokenizedContent'];
            }
        } else {
            return 'ERROR: After substituting all found soft references there were still soft reference tokens in the text. (theoretically this does not have to be an error if the string "{softref:" happens to be in the field for another reason.)';
        }

        return false;
    }

    /*******************************
     *
     * Helper functions
     *
     *******************************/

    protected function isDbReferenceField(array $configuration)
    {
        return
            ($configuration['type'] === 'group' && $configuration['internal_type'] === 'db')
            || (
                ($configuration['type'] === 'select' || $configuration['type'] === 'inline')
                && !empty($configuration['foreign_table'])
            )
            ;
    }

    public function isReferenceField(array $configuration)
    {
        return
            $this->isDbReferenceField($configuration)
            ||
            ($configuration['type'] === 'group' && ($configuration['internal_type'] === 'file' || $configuration['internal_type'] === 'file_reference')) // getRelations_procFiles
            ||
            ($configuration['type'] === 'input' && $configuration['renderType'] === 'inputLink') // getRelations_procDB
            ||
            $configuration['type'] === 'flex'
            ||
            isset($configuration['softref'])
            ;
    }

    protected function fetchTableRelationFields($tableName)
    {
        if (!isset($GLOBALS['TCA'][$tableName]['columns'])) {
            return '';
        }

        $fields = [];

        foreach ($GLOBALS['TCA'][$tableName]['columns'] as $field => $fieldDefinition) {
            if (is_array($fieldDefinition['config'])) {
                // Check for flex field
                if (isset($fieldDefinition['config']['type']) && $fieldDefinition['config']['type'] === 'flex') {
                    // Fetch all fields if the is a field of type flex in the table definition because the complete row is passed to
                    // FlexFormTools->getDataStructureIdentifier() in the end and might be needed in ds_pointerField or a hook
                    return '*';
                }
                // Only fetch this field if it can contain a reference
                if ($this->isReferenceField($fieldDefinition['config'])) {
                    $fields[] = $field;
                }
            }
        }

        return implode(',', $fields);
    }

    public function destPathFromUploadFolder($folder)
    {
        if (!$folder) {
            return substr(PATH_site, 0, -1);
        }
        return PATH_site . $folder;
    }

    public function updateIndex($testOnly, $cli_echo = false)
    {
        $errors = [];
        $tableNames = [];
        $recCount = 0;
        $tableCount = 0;
        $headerContent = $testOnly ? 'Reference Index being TESTED (nothing written, remove the "--check" argument)' : 'Reference Index being Updated';
        if ($cli_echo) {
            echo '*******************************************' . LF . $headerContent . LF . '*******************************************' . LF;
        }
        // Traverse all tables:
        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
        $refIndexConnectionName = empty($GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping']['sys_refindex'])
                ? ConnectionPool::DEFAULT_CONNECTION_NAME
                : $GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping']['sys_refindex'];

        foreach ($GLOBALS['TCA'] as $tableName => $cfg) {
            if ($this->shouldExcludeTableFromReferenceIndex($tableName)) {
                continue;
            }
            $tableConnectionName = empty($GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'][$tableName])
                ? ConnectionPool::DEFAULT_CONNECTION_NAME
                : $GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'][$tableName];

            $fields = ['uid'];
            if (BackendUtility::isTableWorkspaceEnabled($tableName)) {
                $fields[] = 't3ver_wsid';
            }
            // Traverse all records in tables, including deleted records
            $queryBuilder = $connectionPool->getQueryBuilderForTable($tableName);
            $queryBuilder->getRestrictions()->removeAll();
            try {
                $queryResult = $queryBuilder
                    ->select(...$fields)
                    ->from($tableName)
                    ->execute();
            } catch (DBALException $e) {
                // Table exists in $TCA but does not exist in the database
                // @todo: improve / change message and add actual sql error?
                GeneralUtility::sysLog(sprintf('Table "%s" exists in $TCA but does not exist in the database. You should run the Database Analyzer in the Install Tool to fix this.', $tableName), 'core', GeneralUtility::SYSLOG_SEVERITY_ERROR);
                continue;
            }

            $tableNames[] = $tableName;
            $tableCount++;
            while ($record = $queryResult->fetch()) {
                $refIndexObj = GeneralUtility::makeInstance(self::class);
                if (isset($record['t3ver_wsid'])) {
                    $refIndexObj->setWorkspaceId($record['t3ver_wsid']);
                }
                $result = $refIndexObj->updateRefIndexTable($tableName, $record['uid'], $testOnly);
                $recCount++;
                if ($result['addedNodes'] || $result['deletedNodes']) {
                    $error = 'Record ' . $tableName . ':' . $record['uid'] . ' had ' . $result['addedNodes'] . ' added indexes and ' . $result['deletedNodes'] . ' deleted indexes';
                    $errors[] = $error;
                    if ($cli_echo) {
                        echo $error . LF;
                    }
                }
            }

            // Subselect based queries only work on the same connection
            if ($refIndexConnectionName !== $tableConnectionName) {
                GeneralUtility::sysLog(
                    sprintf(
                        'Not checking table "%s" for lost indexes, "sys_refindex" table uses a different connection',
                        $tableName
                    ),
                    'core',
                    GeneralUtility::SYSLOG_SEVERITY_ERROR
                );
                continue;
            }

            // Searching for lost indexes for this table
            // Build sub-query to find lost records
            $subQueryBuilder = $connectionPool->getQueryBuilderForTable($tableName);
            $subQueryBuilder->getRestrictions()->removeAll();
            $subQueryBuilder
                ->select('uid')
                ->from($tableName, 'sub_' . $tableName)
                ->where(
                    $subQueryBuilder->expr()->eq(
                        'sub_' . $tableName . '.uid',
                        $queryBuilder->quoteIdentifier('sys_refindex.recuid')
                    )
                );

            // Main query to find lost records
            $queryBuilder = $connectionPool->getQueryBuilderForTable('sys_refindex');
            $queryBuilder->getRestrictions()->removeAll();
            $lostIndexes = $queryBuilder
                ->count('hash')
                ->from('sys_refindex')
                ->where(
                    $queryBuilder->expr()->eq(
                        'tablename',
                        $queryBuilder->createNamedParameter($tableName, \PDO::PARAM_STR)
                    ),
                    'NOT EXISTS (' . $subQueryBuilder->getSQL() . ')'
                )
                ->execute()
                ->fetchColumn(0);

            if ($lostIndexes > 0) {
                $error = 'Table ' . $tableName . ' has ' . $lostIndexes . ' lost indexes which are now deleted';
                $errors[] = $error;
                if ($cli_echo) {
                    echo $error . LF;
                }
                if (!$testOnly) {
                    $queryBuilder = $connectionPool->getQueryBuilderForTable('sys_refindex');
                    $queryBuilder->delete('sys_refindex')
                        ->where(
                            $queryBuilder->expr()->eq(
                                'tablename',
                                $queryBuilder->createNamedParameter($tableName, \PDO::PARAM_STR)
                            ),
                            'NOT EXISTS (' . $subQueryBuilder->getSQL() . ')'
                        )
                        ->execute();
                }
            }
        }

        // Searching lost indexes for non-existing tables
        $queryBuilder = $connectionPool->getQueryBuilderForTable('sys_refindex');
        $queryBuilder->getRestrictions()->removeAll();
        $lostTables = $queryBuilder
            ->count('hash')
            ->from('sys_refindex')
            ->where(
                $queryBuilder->expr()->notIn(
                    'tablename',
                    $queryBuilder->createNamedParameter($tableNames, Connection::PARAM_STR_ARRAY)
                )
            )->execute()
            ->fetchColumn(0);

        if ($lostTables > 0) {
            $error = 'Index table hosted ' . $lostTables . ' indexes for non-existing tables, now removed';
            $errors[] = $error;
            if ($cli_echo) {
                echo $error . LF;
            }
            if (!$testOnly) {
                $queryBuilder = $connectionPool->getQueryBuilderForTable('sys_refindex');
                $queryBuilder->delete('sys_refindex')
                    ->where(
                        $queryBuilder->expr()->notIn(
                            'tablename',
                            $queryBuilder->createNamedParameter($tableNames, Connection::PARAM_STR_ARRAY)
                        )
                    )->execute();
            }
        }
        $errorCount = count($errors);
        $recordsCheckedString = $recCount . ' records from ' . $tableCount . ' tables were checked/updated.' . LF;
        $flashMessage = GeneralUtility::makeInstance(
            FlashMessage::class,
            $errorCount ? implode('##LF##', $errors) : 'Index Integrity was perfect!',
            $recordsCheckedString,
            $errorCount ? FlashMessage::ERROR : FlashMessage::OK
        );
        $flashMessageService = GeneralUtility::makeInstance(FlashMessageService::class);
        $defaultFlashMessageQueue = $flashMessageService->getMessageQueueByIdentifier();
        $defaultFlashMessageQueue->enqueue($flashMessage);
        $bodyContent = $defaultFlashMessageQueue->renderFlashMessages();
        if ($cli_echo) {
            echo $recordsCheckedString . ($errorCount ? 'Updates: ' . $errorCount : 'Index Integrity was perfect!') . LF;
        }
        if (!$testOnly) {
            $registry = GeneralUtility::makeInstance(Registry::class);
            $registry->set('core', 'sys_refindex_lastUpdate', $GLOBALS['EXEC_TIME']);
        }
        return [$headerContent, $bodyContent, $errorCount];
    }

    protected function getRecordRawCached(string $tableName, int $uid)
    {
        $recordCacheId = $tableName . ':' . $uid;
        if (!$this->useRuntimeCache || !isset($this->recordCache[$recordCacheId])) {

            // Fetch fields of the table which might contain relations
            $cacheId = static::$cachePrefixTableRelationFields . $tableName;
            if (!$this->useRuntimeCache || !$this->runtimeCache->has($cacheId)) {
                $tableRelationFields = $this->fetchTableRelationFields($tableName);
                if ($this->useRuntimeCache) {
                    $this->runtimeCache->set($cacheId, $tableRelationFields);
                }
            } else {
                $tableRelationFields = $this->runtimeCache->get($cacheId);
            }

            // Return if there are no fields which could contain relations
            if ($tableRelationFields === '') {
                return $this->relations;
            }

            if ($tableRelationFields === '*') {
                // If one field of a record is of type flex, all fields have to be fetched to be passed to FlexFormTools->getDataStructureIdentifier()
                $selectFields = '*';
            } else {
                // otherwise only fields that might contain relations are fetched
                $selectFields = 'uid,' . $tableRelationFields;
                $deleteField = $GLOBALS['TCA'][$tableName]['ctrl']['delete'];
                if ($deleteField) {
                    $selectFields .= ',' . $deleteField;
                }
                if (BackendUtility::isTableWorkspaceEnabled($tableName)) {
                    $selectFields .= ',t3ver_wsid';
                }
            }

            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
                ->getQueryBuilderForTable($tableName);
            $queryBuilder->getRestrictions()->removeAll();
            $row = $queryBuilder
                ->select(...GeneralUtility::trimExplode(',', $selectFields, true))
                ->from($tableName)
                ->where(
                    $queryBuilder->expr()->eq(
                        'uid',
                        $queryBuilder->createNamedParameter($uid, \PDO::PARAM_INT)
                    )
                )
                ->execute()
                ->fetch();

            $this->recordCache[$recordCacheId] = $row;
        }
        return $this->recordCache[$recordCacheId];
    }

    protected function shouldExcludeTableFromReferenceIndex($tableName)
    {
        if (isset(static::$excludedTables[$tableName])) {
            return static::$excludedTables[$tableName];
        }

        // Only exclude tables from ReferenceIndex which do not contain any relations and never did since existing references won't be deleted!
        // There is no need to add tables without a definition in $GLOBALS['TCA] since ReferenceIndex only handles those.
        $excludeTable = false;
        $signalSlotDispatcher = GeneralUtility::makeInstance(Dispatcher::class);
        $signalSlotDispatcher->dispatch(__CLASS__, 'shouldExcludeTableFromReferenceIndex', [$tableName, &$excludeTable]);

        static::$excludedTables[$tableName] = $excludeTable;

        return static::$excludedTables[$tableName];
    }

    protected function shouldExcludeTableColumnFromReferenceIndex($tableName, $column, $onlyColumn)
    {
        if (isset(static::$excludedColumns[$column])) {
            return true;
        }

        if (is_array($GLOBALS['TCA'][$tableName]['columns'][$column]) && (!$onlyColumn || $onlyColumn === $column)) {
            return false;
        }

        return true;
    }

    public function enableRuntimeCache()
    {
        $this->useRuntimeCache = true;
    }

    public function disableRuntimeCache()
    {
        $this->useRuntimeCache = false;
    }

    protected function getBackendUser()
    {
        return $GLOBALS['BE_USER'];
    }
}