ReferenceIndex.php

Go to the documentation of this file.

<?php
 
/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */
 
namespace ‪TYPO3\CMS\Core\Database;
 
use Doctrine\DBAL\DBALException;
use Psr\EventDispatcher\EventDispatcherInterface;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerAwareTrait;
use Psr\Log\LogLevel;
use ‪TYPO3\CMS\Backend\Utility\BackendUtility;
use ‪TYPO3\CMS\Backend\View\ProgressListenerInterface;
use ‪TYPO3\CMS\Core\Cache\CacheManager;
use ‪TYPO3\CMS\Core\Configuration\FlexForm\FlexFormTools;
use ‪TYPO3\CMS\Core\Database\Platform\PlatformInformation;
use ‪TYPO3\CMS\Core\DataHandling\DataHandler;
use ‪TYPO3\CMS\Core\DataHandling\Event\IsTableExcludedFromReferenceIndexEvent;
use ‪TYPO3\CMS\Core\Registry;
use ‪TYPO3\CMS\Core\Utility\GeneralUtility;
use ‪TYPO3\CMS\Core\Versioning\VersionState;
 
class ‪ReferenceIndex implements LoggerAwareInterface
{
    use LoggerAwareTrait;
 
    protected static ‪$excludedTables = [
        'sys_log' => true,
        'tx_extensionmanager_domain_model_extension' => true
    ];
 
    protected static ‪$excludedColumns = [
        'uid' => true,
        'perms_userid' => true,
        'perms_groupid' => true,
        'perms_user' => true,
        'perms_group' => true,
        'perms_everybody' => true,
        'pid' => true
    ];
 
    protected static ‪$cachePrefixTableRelationFields = 'core-refidx-tblRelFields-';
 
    public ‪$temp_flexRelations = [];
 
    public ‪$relations = [];
 
    protected ‪$recordCache = [];
 
    public ‪$hashVersion = 1;
 
    protected ‪$workspaceId = 0;
 
    protected ‪$runtimeCache;
 
    protected ‪$useRuntimeCache = false;
 
    protected ‪$eventDispatcher;
 
    public function ‪__construct(EventDispatcherInterface ‪$eventDispatcher = null)
    {
        $this->eventDispatcher = ‪$eventDispatcher ?? GeneralUtility::getContainer()->get(EventDispatcherInterface::class);
        $this->runtimeCache = GeneralUtility::makeInstance(CacheManager::class)->getCache('runtime');
    }
 
    public function ‪setWorkspaceId(‪$workspaceId)
    {
        $this->workspaceId = (int)‪$workspaceId;
    }
 
    public function ‪getWorkspaceId()
    {
        return ‪$this->workspaceId;
    }
 
    public function ‪updateRefIndexTable($tableName, $uid, $testOnly = false)
    {
        $result = [
            'keptNodes' => 0,
            'deletedNodes' => 0,
            'addedNodes' => 0
        ];
 
        $uid = $uid ? (int)$uid : 0;
        if (!$uid) {
            return $result;
        }
 
        // If this table cannot contain relations, skip it
        if ($this->‪shouldExcludeTableFromReferenceIndex($tableName)) {
            return $result;
        }
 
        // Fetch tableRelationFields and save them in cache if not there yet
        $cacheId = static::$cachePrefixTableRelationFields . $tableName;
        $tableRelationFields = $this->useRuntimeCache ? $this->runtimeCache->get($cacheId) : false;
        if ($tableRelationFields === false) {
            $tableRelationFields = $this->‪fetchTableRelationFields($tableName);
            if ($this->useRuntimeCache) {
                $this->runtimeCache->set($cacheId, $tableRelationFields);
            }
        }
 
        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
        $connection = $connectionPool->getConnectionForTable('sys_refindex');
 
        // Get current index from Database with hash as index using $uidIndexField
        // no restrictions are needed, since sys_refindex is not a TCA table
        $queryBuilder = $connection->createQueryBuilder();
        $queryBuilder->getRestrictions()->removeAll();
        $queryResult = $queryBuilder->select('hash')->from('sys_refindex')->where(
            $queryBuilder->expr()->eq('tablename', $queryBuilder->createNamedParameter($tableName, \PDO::PARAM_STR)),
            $queryBuilder->expr()->eq('recuid', $queryBuilder->createNamedParameter($uid, \PDO::PARAM_INT)),
            $queryBuilder->expr()->eq(
                'workspace',
                $queryBuilder->createNamedParameter($this->getWorkspaceId(), \PDO::PARAM_INT)
            )
        )->execute();
        $currentRelationHashes = [];
        while ($relation = $queryResult->fetch()) {
            $currentRelationHashes[$relation['hash']] = true;
        }
 
        // If the table has fields which could contain relations and the record does exist (including deleted-flagged)
        if ($tableRelationFields !== '') {
            $existingRecord = $this->‪getRecordRawCached($tableName, $uid);
            if ($existingRecord) {
                // Table has relation fields and record exists - get relations
                $this->relations = [];
                ‪$relations = $this->‪generateDataUsingRecord($tableName, $existingRecord);
                if (!is_array(‪$relations)) {
                    return $result;
                }
                // Traverse the generated index:
                foreach (‪$relations as &$relation) {
                    if (!is_array($relation)) {
                        continue;
                    }
                    // Exclude any relations TO a specific table
                    if (($relation['ref_table'] ?? '') && $this->‪shouldExcludeTableFromReferenceIndex($relation['ref_table'])) {
                        continue;
                    }
                    $relation['hash'] = md5(implode('
                    // First, check if already indexed and if so, unset that row (so in the end we know which rows to remove!)
                    if (isset($currentRelationHashes[$relation['hash']])) {
                        unset($currentRelationHashes[$relation['hash']]);
                        $result['keptNodes']++;
                        $relation['_ACTION'] = 'KEPT';
                    } else {
                        // If new, add it:
                        if (!$testOnly) {
                            $connection->insert('sys_refindex', $relation);
                        }
                        $result['addedNodes']++;
                        $relation['_ACTION'] = 'ADDED';
                    }
                }
                $result['relations'] = $relations;
            }
        }
 
        // If any old are left, remove them:
        if (!empty($currentRelationHashes)) {
            $hashList = array_keys($currentRelationHashes);
            if (!empty($hashList)) {
                $result['deletedNodes'] = count($hashList);
                $result['deletedNodes_hashList'] = implode(',', $hashList);
                if (!$testOnly) {
                    $maxBindParameters = PlatformInformation::getMaxBindParameters($connection->getDatabasePlatform());
                    foreach (array_chunk($hashList, $maxBindParameters - 10, true) as $chunk) {
                        if (empty($chunk)) {
                            continue;
                        }
                        $queryBuilder = $connection->createQueryBuilder();
                        $queryBuilder
                            ->delete('sys_refindex')
                            ->where(
                                $queryBuilder->expr()->in(
                                    'hash',
                                    $queryBuilder->createNamedParameter($chunk, Connection::PARAM_STR_ARRAY)
                                )
                            )
                            ->execute();
                    }
                }
            }
        }
 
        return $result;
    }
 
    public function generateRefIndexData($tableName, $uid)
    {
        if (!isset($GLOBALS['TCA'][$tableName])) {
            return null;
        }
 
        $this->relations = [];
 
        $record = null;
        $uid = $uid ? (int)$uid : 0;
        if ($uid) {
            // Get raw record from DB
            $record = $this->getRecordRawCached($tableName, $uid);
        }
 
        if (!is_array($record)) {
            return null;
        }
 
        return $this->generateDataUsingRecord($tableName, $record);
    }
 
    public function getNumberOfReferencedRecords(string $tableName, int $uid): int
    {
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_refindex');
        return (int)$queryBuilder
            ->count('*')->from('sys_refindex')
            ->where(
                $queryBuilder->expr()->eq(
                    'ref_table',
                    $queryBuilder->createNamedParameter($tableName, \PDO::PARAM_STR)
                ),
                $queryBuilder->expr()->eq(
                    'ref_uid',
                    $queryBuilder->createNamedParameter($uid, \PDO::PARAM_INT)
                ),
                $queryBuilder->expr()->eq(
                    'deleted',
                    $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)
                )
            )->execute()->fetchColumn(0);
    }
 
    protected function generateDataUsingRecord(string $tableName, array $record): array
    {
        $this->relations = [];
 
        if (BackendUtility::isTableWorkspaceEnabled($tableName)) {
            // Never write relations for t3ver_state = 1 and t3ver_state = 3 placeholder records
            $versionState = VersionState::cast($record['t3ver_state']);
            if ($versionState->equals(VersionState::NEW_PLACEHOLDER) || $versionState->equals(VersionState::MOVE_PLACEHOLDER)) {
                return [];
            }
        }
 
        // Is the record deleted?
        $deleteField = $GLOBALS['TCA'][$tableName]['ctrl']['delete'];
        $deleted = $deleteField && $record[$deleteField] ? 1 : 0;
 
        // Get all relations from record:
        $recordRelations = $this->getRelations($tableName, $record);
        // Traverse those relations, compile records to insert in table:
        foreach ($recordRelations as $fieldName => $fieldRelations) {
            // Based on type
            switch ((string)$fieldRelations['type']) {
                case 'db':
                    $this->createEntryDataForDatabaseRelationsUsingRecord($tableName, $record, $fieldName, '', $deleted, $fieldRelations['itemArray']);
                    break;
                case 'flex':
                    // DB references in FlexForms
                    if (is_array($fieldRelations['flexFormRels']['db'])) {
                        foreach ($fieldRelations['flexFormRels']['db'] as $flexPointer => $subList) {
                            $this->createEntryDataForDatabaseRelationsUsingRecord($tableName, $record, $fieldName, $flexPointer, $deleted, $subList);
                        }
                    }
                    // Soft references in FlexForms
                    // @todo #65464 Test correct handling of soft references in FlexForms
                    if (is_array($fieldRelations['flexFormRels']['softrefs'])) {
                        foreach ($fieldRelations['flexFormRels']['softrefs'] as $flexPointer => $subList) {
                            $this->createEntryDataForSoftReferencesUsingRecord($tableName, $record, $fieldName, $flexPointer, $deleted, $subList['keys']);
                        }
                    }
                    break;
            }
            // Soft references in the field
            if (is_array($fieldRelations['softrefs'])) {
                $this->createEntryDataForSoftReferencesUsingRecord($tableName, $record, $fieldName, '', $deleted, $fieldRelations['softrefs']['keys']);
            }
        }
 
        return array_filter($this->relations);
    }
 
    public function createEntryData($table, $uid, $field, $flexPointer, $deleted, $ref_table, $ref_uid, $ref_string = '', $sort = -1, $softref_key = '', $softref_id = '')
    {
        $uid = $uid ? (int)$uid : 0;
        if (!$uid) {
            return null;
        }
        return $this->createEntryDataUsingRecord(
            (string)$table,
            $this->getRecordRawCached($table, $uid) ?: [],
            (string)$field,
            (string)$flexPointer,
            $deleted ? (int)$deleted : 0,
            (string)$ref_table,
            $ref_uid ? (int)$ref_uid : 0,
            (string)$ref_string,
            $sort ? (int)$sort : 0,
            (string)$softref_key,
            (string)$softref_id
        );
    }
 
    protected function createEntryDataUsingRecord(string $tableName, array $record, string $fieldName, string $flexPointer, int $deleted, string $referencedTable, int $referencedUid, string $referenceString = '', int $sort = -1, string $softReferenceKey = '', string $softReferenceId = '')
    {
        $workspaceId = 0;
        if (BackendUtility::isTableWorkspaceEnabled($tableName)) {
            $workspaceId = $this->getWorkspaceId();
            if (isset($record['t3ver_wsid']) && (int)$record['t3ver_wsid'] !== $workspaceId) {
                // The given record is workspace-enabled but doesn't live in the selected workspace => don't add index as it's not actually there
                return false;
            }
        }
        return [
            'tablename' => $tableName,
            'recuid' => $record['uid'],
            'field' => $fieldName,
            'flexpointer' => $flexPointer,
            'softref_key' => $softReferenceKey,
            'softref_id' => $softReferenceId,
            'sorting' => $sort,
            'deleted' => (int)$deleted,
            'workspace' => ‪$workspaceId,
            'ref_table' => $referencedTable,
            'ref_uid' => $referencedUid,
            'ref_string' => mb_substr($referenceString, 0, 1024)
        ];
    }
 
    public function ‪createEntryData_dbRels($table, $uid, $fieldName, $flexPointer, $deleted, $items)
    {
        $uid = $uid ? (int)$uid : 0;
        if (!$uid) {
            return;
        }
        $this->‪createEntryDataForDatabaseRelationsUsingRecord(
            (string)$table,
            $this->‪getRecordRawCached($table, $uid) ?: [],
            (string)$fieldName,
            (string)$flexPointer,
            $deleted ? (int)$deleted : 0,
            (array)$items
        );
    }
 
    protected function ‪createEntryDataForDatabaseRelationsUsingRecord(string $tableName, array $record, string $fieldName, string $flexPointer, int $deleted, array $items)
    {
        foreach ($items as $sort => $i) {
            $this->relations[] = $this->‪createEntryDataUsingRecord($tableName, $record, $fieldName, $flexPointer, $deleted, $i['table'], (int)$i['id'], '', $sort);
        }
    }
 
    public function ‪createEntryData_softreferences($table, $uid, $fieldName, $flexPointer, $deleted, $keys)
    {
        $uid = $uid ? (int)$uid : 0;
        if (!$uid || !is_array($keys)) {
            return;
        }
        $this->‪createEntryDataForSoftReferencesUsingRecord(
            (string)$table,
            $this->‪getRecordRawCached($table, $uid) ?: [],
            (string)$fieldName,
            (string)$flexPointer,
            $deleted ? (int)$deleted : 0,
            (array)$keys
        );
    }
 
    protected function ‪createEntryDataForSoftReferencesUsingRecord(string $tableName, array $record, string $fieldName, string $flexPointer, int $deleted, array $keys)
    {
        foreach ($keys as $spKey => $elements) {
            if (is_array($elements)) {
                foreach ($elements as $subKey => $el) {
                    if (is_array($el['subst'])) {
                        switch ((string)$el['subst']['type']) {
                            case 'db':
                                [$referencedTable, $referencedUid] = explode(':', $el['subst']['recordRef']);
                                $this->relations[] = $this->‪createEntryDataUsingRecord($tableName, $record, $fieldName, $flexPointer, $deleted, $referencedTable, (int)$referencedUid, '', -1, $spKey, $subKey);
                                break;
                            case 'string':
                                $this->relations[] = $this->‪createEntryDataUsingRecord($tableName, $record, $fieldName, $flexPointer, $deleted, '_STRING', 0, $el['subst']['tokenValue'], -1, $spKey, $subKey);
                                break;
                        }
                    }
                }
            }
        }
    }
 
    /*******************************
     *
     * Get relations from table row
     *
     *******************************/
 
    public function ‪getRelations($table, $row, $onlyField = '')
    {
        // Initialize:
        $uid = $row['uid'];
        $outRow = [];
        foreach ($row as $field => $value) {
            if ($this->‪shouldExcludeTableColumnFromReferenceIndex($table, $field, $onlyField) === false) {
                $conf = ‪$GLOBALS['TCA'][$table]['columns'][$field]['config'];
                // Add a softref definition for link fields if the TCA does not specify one already
                if ($conf['type'] === 'input' && $conf['renderType'] === 'inputLink' && empty($conf['softref'])) {
                    $conf['softref'] = 'typolink';
                }
                // Add DB:
                $resultsFromDatabase = $this->‪getRelations_procDB($value, $conf, $uid, $table);
                if (!empty($resultsFromDatabase)) {
                    // Create an entry for the field with all DB relations:
                    $outRow[$field] = [
                        'type' => 'db',
                        'itemArray' => $resultsFromDatabase
                    ];
                }
                // For "flex" fieldtypes we need to traverse the structure looking for db references of course!
                if ($conf['type'] === 'flex') {
                    // Get current value array:
                    // NOTICE: failure to resolve Data Structures can lead to integrity problems with the reference index. Please look up
                    // the note in the JavaDoc documentation for the function FlexFormTools->getDataStructureIdentifier()
                    $currentValueArray = ‪GeneralUtility::xml2array($value);
                    // Traversing the XML structure, processing:
                    if (is_array($currentValueArray)) {
                        $this->temp_flexRelations = [
                            'db' => [],
                            'softrefs' => []
                        ];
                        // Create and call iterator object:
                        $flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
                        $flexFormTools->traverseFlexFormXMLData($table, $field, $row, $this, 'getRelations_flexFormCallBack');
                        // Create an entry for the field:
                        $outRow[$field] = [
                            'type' => 'flex',
                            'flexFormRels' => ‪$this->temp_flexRelations
                        ];
                    }
                }
                // Soft References:
                if ((string)$value !== '') {
                    $softRefValue = $value;
                    if (!empty($conf['softref'])) {
                        $softRefs = ‪BackendUtility::explodeSoftRefParserList($conf['softref']);
                        foreach ($softRefs as $spKey => $spParams) {
                            $softRefObj = ‪BackendUtility::softRefParserObj($spKey);
                            if (is_object($softRefObj)) {
                                $resultArray = $softRefObj->findRef($table, $field, $uid, $softRefValue, $spKey, $spParams);
                                if (is_array($resultArray)) {
                                    $outRow[$field]['softrefs']['keys'][$spKey] = $resultArray['elements'];
                                    if ((string)$resultArray['content'] !== '') {
                                        $softRefValue = $resultArray['content'];
                                    }
                                }
                            }
                        }
                    }
                    if (!empty($outRow[$field]['softrefs']) && (string)$value !== (string)$softRefValue && strpos($softRefValue, '{softref:') !== false) {
                        $outRow[$field]['softrefs']['tokenizedContent'] = $softRefValue;
                    }
                }
            }
        }
        return $outRow;
    }
 
    public function ‪getRelations_flexFormCallBack($dsArr, $dataValue, $PA, $structurePath)
    {
        // Removing "data/" in the beginning of path (which points to location in data array)
        $structurePath = substr($structurePath, 5) . '/';
        $dsConf = $dsArr['TCEforms']['config'];
        // Implode parameter values:
        [$table, $uid, $field] = [
            $PA['table'],
            $PA['uid'],
            $PA['field']
        ];
        // Add a softref definition for link fields if the TCA does not specify one already
        if ($dsConf['type'] === 'input' && $dsConf['renderType'] === 'inputLink' && empty($dsConf['softref'])) {
            $dsConf['softref'] = 'typolink';
        }
        // Add DB:
        $resultsFromDatabase = $this->‪getRelations_procDB($dataValue, $dsConf, $uid, $table);
        if (!empty($resultsFromDatabase)) {
            // Create an entry for the field with all DB relations:
            $this->temp_flexRelations['db'][$structurePath] = $resultsFromDatabase;
        }
        // Soft References:
        if (is_array($dataValue) || (string)$dataValue !== '') {
            $softRefValue = $dataValue;
            $softRefs = ‪BackendUtility::explodeSoftRefParserList($dsConf['softref']);
            if ($softRefs !== false) {
                foreach ($softRefs as $spKey => $spParams) {
                    $softRefObj = ‪BackendUtility::softRefParserObj($spKey);
                    if (is_object($softRefObj)) {
                        $resultArray = $softRefObj->findRef($table, $field, $uid, $softRefValue, $spKey, $spParams, $structurePath);
                        if (is_array($resultArray) && is_array($resultArray['elements'])) {
                            $this->temp_flexRelations['softrefs'][$structurePath]['keys'][$spKey] = $resultArray['elements'];
                            if ((string)$resultArray['content'] !== '') {
                                $softRefValue = $resultArray['content'];
                            }
                        }
                    }
                }
            }
            if (!empty($this->temp_flexRelations['softrefs']) && (string)$dataValue !== (string)$softRefValue) {
                $this->temp_flexRelations['softrefs'][$structurePath]['tokenizedContent'] = $softRefValue;
            }
        }
    }
 
    public function ‪getRelations_procDB($value, $conf, $uid, $table = '')
    {
        // Get IRRE relations
        if (empty($conf)) {
            return false;
        }
        if ($conf['type'] === 'inline' && !empty($conf['foreign_table']) && empty($conf['MM'])) {
            $dbAnalysis = GeneralUtility::makeInstance(RelationHandler::class);
            $dbAnalysis->setUseLiveReferenceIds(false);
            $dbAnalysis->setWorkspaceId($this->‪getWorkspaceId());
            $dbAnalysis->start($value, $conf['foreign_table'], '', $uid, $table, $conf);
            return $dbAnalysis->itemArray;
            // DB record lists:
        }
        if ($this->‪isDbReferenceField($conf)) {
            $allowedTables = $conf['type'] === 'group' ? $conf['allowed'] : $conf['foreign_table'];
            if ($conf['MM_opposite_field']) {
                return [];
            }
            $dbAnalysis = GeneralUtility::makeInstance(RelationHandler::class);
            $dbAnalysis->start($value, $allowedTables, $conf['MM'], $uid, $table, $conf);
            return $dbAnalysis->itemArray;
        }
        return false;
    }
 
    /*******************************
     *
     * Setting values
     *
     *******************************/
 
    public function ‪setReferenceValue($hash, $newValue, $returnDataArray = false, $bypassWorkspaceAdminCheck = false)
    {
        $backendUser = $this->‪getBackendUser();
        if ($backendUser->workspace === 0 && $backendUser->isAdmin() || $bypassWorkspaceAdminCheck) {
            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_refindex');
            $queryBuilder->getRestrictions()->removeAll();
 
            // Get current index from Database
            $referenceRecord = $queryBuilder
                ->select('*')
                ->from('sys_refindex')
                ->where(
                    $queryBuilder->expr()->eq('hash', $queryBuilder->createNamedParameter($hash, \PDO::PARAM_STR))
                )
                ->setMaxResults(1)
                ->execute()
                ->fetch();
 
            // Check if reference existed.
            if (!is_array($referenceRecord)) {
                return 'ERROR: No reference record with hash="' . $hash . '" was found!';
            }
 
            if (empty(‪$GLOBALS['TCA'][$referenceRecord['tablename']])) {
                return 'ERROR: Table "' . $referenceRecord['tablename'] . '" was not in TCA!';
            }
 
            // Get that record from database
            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
                ->getQueryBuilderForTable($referenceRecord['tablename']);
            $queryBuilder->getRestrictions()->removeAll();
            $record = $queryBuilder
                ->select('*')
                ->from($referenceRecord['tablename'])
                ->where(
                    $queryBuilder->expr()->eq(
                        'uid',
                        $queryBuilder->createNamedParameter($referenceRecord['recuid'], \PDO::PARAM_INT)
                    )
                )
                ->setMaxResults(1)
                ->execute()
                ->fetch();
 
            if (is_array($record)) {
                // Get relation for single field from record
                $recordRelations = $this->‪getRelations($referenceRecord['tablename'], $record, $referenceRecord['field']);
                if ($fieldRelation = $recordRelations[$referenceRecord['field']]) {
                    // Initialize data array that is to be sent to DataHandler afterwards:
                    $dataArray = [];
                    // Based on type
                    switch ((string)$fieldRelation['type']) {
                        case 'db':
                            $error = $this->‪setReferenceValue_dbRels($referenceRecord, $fieldRelation['itemArray'], $newValue, $dataArray);
                            if ($error) {
                                return $error;
                            }
                            break;
                        case 'flex':
                            // DB references in FlexForms
                            if (is_array($fieldRelation['flexFormRels']['db'][$referenceRecord['flexpointer']])) {
                                $error = $this->‪setReferenceValue_dbRels($referenceRecord, $fieldRelation['flexFormRels']['db'][$referenceRecord['flexpointer']], $newValue, $dataArray, $referenceRecord['flexpointer']);
                                if ($error) {
                                    return $error;
                                }
                            }
                            // Soft references in FlexForms
                            if ($referenceRecord['softref_key'] && is_array($fieldRelation['flexFormRels']['softrefs'][$referenceRecord['flexpointer']]['keys'][$referenceRecord['softref_key']])) {
                                $error = $this->‪setReferenceValue_softreferences($referenceRecord, $fieldRelation['flexFormRels']['softrefs'][$referenceRecord['flexpointer']], $newValue, $dataArray, $referenceRecord['flexpointer']);
                                if ($error) {
                                    return $error;
                                }
                            }
                            break;
                    }
                    // Soft references in the field:
                    if ($referenceRecord['softref_key'] && is_array($fieldRelation['softrefs']['keys'][$referenceRecord['softref_key']])) {
                        $error = $this->‪setReferenceValue_softreferences($referenceRecord, $fieldRelation['softrefs'], $newValue, $dataArray);
                        if ($error) {
                            return $error;
                        }
                    }
                    // Data Array, now ready to be sent to DataHandler
                    if ($returnDataArray) {
                        return $dataArray;
                    }
                    // Execute CMD array:
                    $dataHandler = GeneralUtility::makeInstance(DataHandler::class);
                    $dataHandler->dontProcessTransformations = true;
                    $dataHandler->bypassWorkspaceRestrictions = true;
                    // Otherwise this cannot update things in deleted records...
                    $dataHandler->bypassAccessCheckForRecords = true;
                    // Check has been done previously that there is a backend user which is Admin and also in live workspace
                    $dataHandler->start($dataArray, []);
                    $dataHandler->process_datamap();
                    // Return errors if any:
                    if (!empty($dataHandler->errorLog)) {
                        return LF . 'DataHandler:' . implode(LF . 'DataHandler:', $dataHandler->errorLog);
                    }
                }
            }
        } else {
            return 'ERROR: BE_USER object is not admin OR not in workspace 0 (Live)';
        }
 
        return false;
    }
 
    public function ‪setReferenceValue_dbRels($refRec, $itemArray, $newValue, &$dataArray, $flexPointer = '')
    {
        if ((int)$itemArray[$refRec['sorting']]['id'] === (int)$refRec['ref_uid'] && (string)$itemArray[$refRec['sorting']]['table'] === (string)$refRec['ref_table']) {
            // Setting or removing value:
            // Remove value:
            if ($newValue === null) {
                unset($itemArray[$refRec['sorting']]);
            } else {
                [$itemArray[$refRec['sorting']]['table'], $itemArray[$refRec['sorting']]['id']] = explode(':', $newValue);
            }
            // Traverse and compile new list of records:
            $saveValue = [];
            foreach ($itemArray as $pair) {
                $saveValue[] = $pair['table'] . '_' . $pair['id'];
            }
            // Set in data array:
            if ($flexPointer) {
                $flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
                $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']]['data'] = [];
                $flexFormTools->setArrayValueByPath(substr($flexPointer, 0, -1), $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']]['data'], implode(',', $saveValue));
            } else {
                $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']] = implode(',', $saveValue);
            }
        } else {
            return 'ERROR: table:id pair "' . $refRec['ref_table'] . ':' . $refRec['ref_uid'] . '" did not match that of the record ("' . $itemArray[$refRec['sorting']]['table'] . ':' . $itemArray[$refRec['sorting']]['id'] . '") in sorting index "' . $refRec['sorting'] . '"';
        }
 
        return false;
    }
 
    public function ‪setReferenceValue_softreferences($refRec, $softref, $newValue, &$dataArray, $flexPointer = '')
    {
        if (!is_array($softref['keys'][$refRec['softref_key']][$refRec['softref_id']])) {
            return 'ERROR: Soft reference parser key "' . $refRec['softref_key'] . '" or the index "' . $refRec['softref_id'] . '" was not found.';
        }
 
        // Set new value:
        $softref['keys'][$refRec['softref_key']][$refRec['softref_id']]['subst']['tokenValue'] = '' . $newValue;
        // Traverse softreferences and replace in tokenized content to rebuild it with new value inside:
        foreach ($softref['keys'] as $sfIndexes) {
            foreach ($sfIndexes as $data) {
                $softref['tokenizedContent'] = str_replace('{softref:' . $data['subst']['tokenID'] . '}', $data['subst']['tokenValue'], $softref['tokenizedContent']);
            }
        }
        // Set in data array:
        if (strpos($softref['tokenizedContent'], '{softref:') === false) {
            if ($flexPointer) {
                $flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
                $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']]['data'] = [];
                $flexFormTools->setArrayValueByPath(substr($flexPointer, 0, -1), $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']]['data'], $softref['tokenizedContent']);
            } else {
                $dataArray[$refRec['tablename']][$refRec['recuid']][$refRec['field']] = $softref['tokenizedContent'];
            }
        } else {
            return 'ERROR: After substituting all found soft references there were still soft reference tokens in the text. (theoretically this does not have to be an error if the string "{softref:" happens to be in the field for another reason.)';
        }
 
        return false;
    }
 
    /*******************************
     *
     * Helper functions
     *
     *******************************/
 
    protected function ‪isDbReferenceField(array $configuration)
    {
        return
            ($configuration['type'] === 'group' && $configuration['internal_type'] === 'db')
            || (
                ($configuration['type'] === 'select' || $configuration['type'] === 'inline')
                && !empty($configuration['foreign_table'])
            )
            ;
    }
 
    public function ‪isReferenceField(array $configuration)
    {
        return
            $this->‪isDbReferenceField($configuration)
            ||
            ($configuration['type'] === 'input' && $configuration['renderType'] === 'inputLink') // getRelations_procDB
            ||
            $configuration['type'] === 'flex'
            ||
            isset($configuration['softref'])
            ;
    }
 
    protected function ‪fetchTableRelationFields($tableName)
    {
        if (!isset(‪$GLOBALS['TCA'][$tableName]['columns'])) {
            return '';
        }
 
        ‪$fields = [];
 
        foreach (‪$GLOBALS['TCA'][$tableName]['columns'] as $field => $fieldDefinition) {
            if (is_array($fieldDefinition['config'])) {
                // Check for flex field
                if (isset($fieldDefinition['config']['type']) && $fieldDefinition['config']['type'] === 'flex') {
                    // Fetch all fields if the is a field of type flex in the table definition because the complete row is passed to
                    // FlexFormTools->getDataStructureIdentifier() in the end and might be needed in ds_pointerField or a hook
                    return '*';
                }
                // Only fetch this field if it can contain a reference
                if ($this->‪isReferenceField($fieldDefinition['config'])) {
                    ‪$fields[] = $field;
                }
            }
        }
 
        return implode(',', ‪$fields);
    }
 
    public function ‪updateIndex($testOnly, $cli_echo = null)
    {
        $progressListener = null;
        if ($cli_echo instanceof ‪ProgressListenerInterface) {
            $progressListener = $cli_echo;
            $cli_echo = null;
        }
        if ($cli_echo !== null) {
            trigger_error('The second argument of ReferenceIndex->updateIndex() will not work in TYPO3 v11 anymore. Use the ProgressListener to show detailed results', E_USER_DEPRECATED);
        } else {
            // default value for now
            $cli_echo = false;
        }
        ‪$errors = [];
        $tableNames = [];
        $recCount = 0;
        $headerContent = $testOnly ? 'Reference Index being TESTED (nothing written, remove the "--check" argument)' : 'Reference Index being Updated';
        if ($cli_echo) {
            echo '*******************************************' . LF . $headerContent . LF . '*******************************************' . LF;
        }
        // Traverse all tables:
        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
        $refIndexConnectionName = empty(‪$GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping']['sys_refindex'])
                ? ‪ConnectionPool::DEFAULT_CONNECTION_NAME
                : ‪$GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping']['sys_refindex'];
 
        foreach (‪$GLOBALS['TCA'] as $tableName => $cfg) {
            if ($this->‪shouldExcludeTableFromReferenceIndex($tableName)) {
                continue;
            }
            $tableConnectionName = empty(‪$GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'][$tableName])
                ? ‪ConnectionPool::DEFAULT_CONNECTION_NAME
                : ‪$GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'][$tableName];
 
            ‪$fields = ['uid'];
            if (‪BackendUtility::isTableWorkspaceEnabled($tableName)) {
                ‪$fields[] = 't3ver_wsid';
            }
            // Traverse all records in tables, including deleted records
            $queryBuilder = $connectionPool->getQueryBuilderForTable($tableName);
            $queryBuilder->getRestrictions()->removeAll();
            try {
                $queryResult = $queryBuilder
                    ->select(...‪$fields)
                    ->from($tableName)
                    ->orderBy('uid')
                    ->execute();
            } catch (DBALException $e) {
                // Table exists in TCA but does not exist in the database
                $msg = 'Table "' .
                        $tableName .
                        '" exists in TCA but does not exist in the database. You should run the Database Analyzer in the Install Tool to fix this.';
                $this->logger->error($msg, ['exception' => $e]);
                continue;
            }
 
            if ($progressListener) {
                $progressListener->start($queryResult->rowCount(), $tableName);
            }
            $tableNames[] = $tableName;
            while ($record = $queryResult->fetch()) {
                if ($progressListener) {
                    $progressListener->advance();
                }
                $refIndexObj = GeneralUtility::makeInstance(self::class);
                if (isset($record['t3ver_wsid'])) {
                    $refIndexObj->setWorkspaceId($record['t3ver_wsid']);
                }
                $result = $refIndexObj->updateRefIndexTable($tableName, $record['uid'], $testOnly);
                $recCount++;
                if ($result['addedNodes'] || $result['deletedNodes']) {
                    $error = 'Record ' . $tableName . ':' . $record['uid'] . ' had ' . $result['addedNodes'] . ' added indexes and ' . $result['deletedNodes'] . ' deleted indexes';
                    ‪$errors[] = $error;
                    if ($progressListener) {
                        $progressListener->log($error, LogLevel::WARNING);
                    }
                    if ($cli_echo) {
                        echo $error . LF;
                    }
                }
            }
            if ($progressListener) {
                $progressListener->finish();
            }
 
            // Subselect based queries only work on the same connection
            if ($refIndexConnectionName !== $tableConnectionName) {
                $this->logger->error('Not checking table "' . $tableName . '" for lost indexes, "sys_refindex" table uses a different connection');
                continue;
            }
 
            // Searching for lost indexes for this table
            // Build sub-query to find lost records
            $subQueryBuilder = $connectionPool->getQueryBuilderForTable($tableName);
            $subQueryBuilder->getRestrictions()->removeAll();
            $subQueryBuilder
                ->select('uid')
                ->from($tableName, 'sub_' . $tableName)
                ->where(
                    $subQueryBuilder->expr()->eq(
                        'sub_' . $tableName . '.uid',
                        $queryBuilder->quoteIdentifier('sys_refindex.recuid')
                    )
                );
 
            // Main query to find lost records
            $queryBuilder = $connectionPool->getQueryBuilderForTable('sys_refindex');
            $queryBuilder->getRestrictions()->removeAll();
            $lostIndexes = $queryBuilder
                ->count('hash')
                ->from('sys_refindex')
                ->where(
                    $queryBuilder->expr()->eq(
                        'tablename',
                        $queryBuilder->createNamedParameter($tableName, \PDO::PARAM_STR)
                    ),
                    'NOT EXISTS (' . $subQueryBuilder->getSQL() . ')'
                )
                ->execute()
                ->fetchColumn(0);
 
            if ($lostIndexes > 0) {
                $error = 'Table ' . $tableName . ' has ' . $lostIndexes . ' lost indexes which are now deleted';
                ‪$errors[] = $error;
                if ($progressListener) {
                    $progressListener->log($error, LogLevel::WARNING);
                }
                if ($cli_echo) {
                    echo $error . LF;
                }
                if (!$testOnly) {
                    $queryBuilder = $connectionPool->getQueryBuilderForTable('sys_refindex');
                    $queryBuilder->delete('sys_refindex')
                        ->where(
                            $queryBuilder->expr()->eq(
                                'tablename',
                                $queryBuilder->createNamedParameter($tableName, \PDO::PARAM_STR)
                            ),
                            'NOT EXISTS (' . $subQueryBuilder->getSQL() . ')'
                        )
                        ->execute();
                }
            }
        }
 
        // Searching lost indexes for non-existing tables
        $lostTables = $this->‪getAmountOfUnusedTablesInReferenceIndex($tableNames);
        if ($lostTables > 0) {
            $error = 'Index table hosted ' . $lostTables . ' indexes for non-existing tables, now removed';
            ‪$errors[] = $error;
            if ($progressListener) {
                $progressListener->log($error, LogLevel::WARNING);
            }
            if ($cli_echo) {
                echo $error . LF;
            }
            if (!$testOnly) {
                $this->‪removeReferenceIndexDataFromUnusedDatabaseTables($tableNames);
            }
        }
        $errorCount = count(‪$errors);
        $recordsCheckedString = $recCount . ' records from ' . count($tableNames) . ' tables were checked/updated.' . LF;
        if ($progressListener) {
            if ($errorCount) {
                $progressListener->log($recordsCheckedString . 'Updates: ' . $errorCount, LogLevel::WARNING);
            } else {
                $progressListener->log($recordsCheckedString . 'Index Integrity was perfect!', LogLevel::INFO);
            }
        }
        if ($cli_echo) {
            echo $recordsCheckedString . ($errorCount ? 'Updates: ' . $errorCount : 'Index Integrity was perfect!') . LF;
        }
        if (!$testOnly) {
            $registry = GeneralUtility::makeInstance(Registry::class);
            $registry->set('core', 'sys_refindex_lastUpdate', ‪$GLOBALS['EXEC_TIME']);
        }
        return [$headerContent, trim($recordsCheckedString), $errorCount, ‪$errors];
    }
 
    protected function ‪getAmountOfUnusedTablesInReferenceIndex(array $tableNames): int
    {
        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
        $queryBuilder = $connectionPool->getQueryBuilderForTable('sys_refindex');
        $queryBuilder->getRestrictions()->removeAll();
        $lostTables = $queryBuilder
            ->count('hash')
            ->from('sys_refindex')
            ->where(
                $queryBuilder->expr()->notIn(
                    'tablename',
                    $queryBuilder->createNamedParameter($tableNames, Connection::PARAM_STR_ARRAY)
                )
            )->execute()
            ->fetchColumn(0);
        return (int)$lostTables;
    }
 
    protected function ‪removeReferenceIndexDataFromUnusedDatabaseTables(array $tableNames): void
    {
        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
        $queryBuilder = $connectionPool->getQueryBuilderForTable('sys_refindex');
        $queryBuilder->delete('sys_refindex')
            ->where(
                $queryBuilder->expr()->notIn(
                    'tablename',
                    $queryBuilder->createNamedParameter($tableNames, Connection::PARAM_STR_ARRAY)
                )
            )->execute();
    }
 
    protected function ‪getRecordRawCached(string $tableName, int $uid)
    {
        $recordCacheId = $tableName . ':' . $uid;
        if (!$this->useRuntimeCache || !isset($this->recordCache[$recordCacheId])) {
 
            // Fetch fields of the table which might contain relations
            $cacheId = static::$cachePrefixTableRelationFields . $tableName;
            $tableRelationFields = $this->useRuntimeCache ? $this->runtimeCache->get($cacheId) : false;
            if ($tableRelationFields === false) {
                $tableRelationFields = $this->‪fetchTableRelationFields($tableName);
                if ($this->useRuntimeCache) {
                    $this->runtimeCache->set($cacheId, $tableRelationFields);
                }
            }
 
            // Return if there are no fields which could contain relations
            if ($tableRelationFields === '') {
                return ‪$this->relations;
            }
 
            if ($tableRelationFields === '*') {
                // If one field of a record is of type flex, all fields have to be fetched to be passed to FlexFormTools->getDataStructureIdentifier()
                $selectFields = '*';
            } else {
                // otherwise only fields that might contain relations are fetched
                $selectFields = 'uid,' . $tableRelationFields;
                $deleteField = ‪$GLOBALS['TCA'][$tableName]['ctrl']['delete'];
                if ($deleteField) {
                    $selectFields .= ',' . $deleteField;
                }
                if (‪BackendUtility::isTableWorkspaceEnabled($tableName)) {
                    $selectFields .= ',t3ver_wsid,t3ver_state';
                }
            }
 
            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
                ->getQueryBuilderForTable($tableName);
            $queryBuilder->getRestrictions()->removeAll();
            $row = $queryBuilder
                ->select(...‪GeneralUtility::trimExplode(',', $selectFields, true))
                ->from($tableName)
                ->where(
                    $queryBuilder->expr()->eq(
                        'uid',
                        $queryBuilder->createNamedParameter($uid, \PDO::PARAM_INT)
                    )
                )
                ->execute()
                ->fetch();
 
            $this->recordCache[$recordCacheId] = $row;
        }
        return $this->recordCache[$recordCacheId];
    }
 
    protected function ‪shouldExcludeTableFromReferenceIndex($tableName)
    {
        if (isset(static::$excludedTables[$tableName])) {
            return static::$excludedTables[$tableName];
        }
 
        // Only exclude tables from ReferenceIndex which do not contain any relations and never
        // did since existing references won't be deleted!
        $event = new IsTableExcludedFromReferenceIndexEvent($tableName);
        $event = $this->eventDispatcher->dispatch($event);
        static::$excludedTables[$tableName] = $event->isTableExcluded();
 
        return static::$excludedTables[$tableName];
    }
 
    protected function ‪shouldExcludeTableColumnFromReferenceIndex($tableName, $column, $onlyColumn)
    {
        if (isset(static::$excludedColumns[$column])) {
            return true;
        }
 
        if (is_array(‪$GLOBALS['TCA'][$tableName]['columns'][$column]) && (!$onlyColumn || $onlyColumn === $column)) {
            return false;
        }
 
        return true;
    }
 
    public function ‪enableRuntimeCache()
    {
        $this->useRuntimeCache = true;
    }
 
    public function ‪disableRuntimeCache()
    {
        $this->useRuntimeCache = false;
    }
 
    protected function ‪getBackendUser()
    {
        return ‪$GLOBALS['BE_USER'];
    }
}