DatabaseIntegrityCheck.php

Go to the documentation of this file.

<?php
namespace TYPO3\CMS\Core\Integrity;

/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

use Doctrine\DBAL\Types\Type;
use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\Database\Connection;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\Expression\ExpressionBuilder;
use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
use TYPO3\CMS\Core\Database\RelationHandler;
use TYPO3\CMS\Core\Utility\GeneralUtility;

class DatabaseIntegrityCheck
{
    public $genTree_includeDeleted = true;

    public $genTree_includeVersions = true;

    public $genTree_includeRecords = false;

    public $page_idArray = [];

    public $rec_idArray = [];

    public $checkFileRefs = [];

    public $checkSelectDBRefs = [];

    public $checkGroupDBRefs = [];

    public $recStats = [
        'allValid' => [],
        'published_versions' => [],
        'deleted' => []
    ];

    public $lRecords = [];

    public $lostPagesList = '';

    public function genTree($theID, $depthData = '', $versions = false)
    {
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('pages');
        $queryBuilder->getRestrictions()->removeAll();
        if (!$this->genTree_includeDeleted) {
            $queryBuilder->getRestrictions()->add(GeneralUtility::makeInstance(DeletedRestriction::class));
        }
        $queryBuilder->select('uid', 'title', 'doktype', 'deleted', 'hidden')
            ->from('pages')
            ->orderBy('sorting');
        if ($versions) {
            $queryBuilder->addSelect('t3ver_wsid', 't3ver_id', 't3ver_count');
            $queryBuilder->where(
                $queryBuilder->expr()->eq('pid', $queryBuilder->createNamedParameter(-1, \PDO::PARAM_INT)),
                $queryBuilder->expr()->eq('t3ver_oid', $queryBuilder->createNamedParameter($theID, \PDO::PARAM_INT))
            );
        } else {
            $queryBuilder->where(
                $queryBuilder->expr()->eq('pid', $queryBuilder->createNamedParameter($theID, \PDO::PARAM_INT))
            );
        }
        $result = $queryBuilder->execute();
        // Traverse the records selected
        while ($row = $result->fetch()) {
            $newID = $row['uid'];
            // Register various data for this item:
            $this->page_idArray[$newID] = $row;
            $this->recStats['all_valid']['pages'][$newID] = $newID;
            if ($row['deleted']) {
                $this->recStats['deleted']['pages'][$newID] = $newID;
            }
            if ($versions && $row['t3ver_count'] >= 1) {
                $this->recStats['published_versions']['pages'][$newID] = $newID;
            }
            if ($row['deleted']) {
                $this->recStats['deleted']++;
            }
            if ($row['hidden']) {
                $this->recStats['hidden']++;
            }
            $this->recStats['doktype'][$row['doktype']]++;
            // If all records should be shown, do so:
            if ($this->genTree_includeRecords) {
                foreach ($GLOBALS['TCA'] as $tableName => $cfg) {
                    if ($tableName !== 'pages') {
                        $this->genTree_records($newID, '', $tableName);
                    }
                }
            }
            // Add sub pages:
            $this->genTree($newID);
            // If versions are included in the tree, add those now:
            if ($this->genTree_includeVersions) {
                $this->genTree($newID, '', true);
            }
        }
    }

    public function genTree_records($theID, $_ = '', $table = '', $versions = false)
    {
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table);
        $queryBuilder->getRestrictions()->removeAll();
        if (!$this->genTree_includeDeleted) {
            $queryBuilder->getRestrictions()->add(GeneralUtility::makeInstance(DeletedRestriction::class));
        }
        $queryBuilder
            ->select(...explode(',', BackendUtility::getCommonSelectFields($table)))
            ->from($table);

        // Select all records from table pointing to this page
        if ($versions) {
            $queryBuilder->where(
                $queryBuilder->expr()->eq('pid', $queryBuilder->createNamedParameter(-1, \PDO::PARAM_INT)),
                $queryBuilder->expr()->eq('t3ver_oid', $queryBuilder->createNamedParameter($theID, \PDO::PARAM_INT))
            );
        } else {
            $queryBuilder->where(
                $queryBuilder->expr()->eq('pid', $queryBuilder->createNamedParameter($theID, \PDO::PARAM_INT))
            );
        }
        $queryResult = $queryBuilder->execute();
        // Traverse selected
        while ($row = $queryResult->fetch()) {
            $newID = $row['uid'];
            // Register various data for this item:
            $this->rec_idArray[$table][$newID] = $row;
            $this->recStats['all_valid'][$table][$newID] = $newID;
            if ($row['deleted']) {
                $this->recStats['deleted'][$table][$newID] = $newID;
            }
            if ($versions && $row['t3ver_count'] >= 1 && $row['t3ver_wsid'] == 0) {
                $this->recStats['published_versions'][$table][$newID] = $newID;
            }
            // Select all versions of this record:
            if ($this->genTree_includeVersions && $GLOBALS['TCA'][$table]['ctrl']['versioningWS']) {
                $this->genTree_records($newID, '', $table, true);
            }
        }
    }

    public function lostRecords($pid_list)
    {
        $this->lostPagesList = '';
        $pageIds = GeneralUtility::intExplode(',', $pid_list);
        if (is_array($pageIds)) {
            foreach ($GLOBALS['TCA'] as $table => $tableConf) {
                $pageIdsForTable = $pageIds;
                // Remove preceding "-1," for non-versioned tables
                if (!BackendUtility::isTableWorkspaceEnabled($table)) {
                    $pageIdsForTable = array_combine($pageIdsForTable, $pageIdsForTable);
                    unset($pageIdsForTable[-1]);
                }
                $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table);
                $queryBuilder->getRestrictions()->removeAll();
                $selectFields = ['uid', 'pid'];
                if (!empty($GLOBALS['TCA'][$table]['ctrl']['label'])) {
                    $selectFields[] = $GLOBALS['TCA'][$table]['ctrl']['label'];
                }
                $queryResult = $queryBuilder->select(...$selectFields)
                    ->from($table)
                    ->where(
                        $queryBuilder->expr()->notIn(
                            'pid',
                            $queryBuilder->createNamedParameter($pageIdsForTable, Connection::PARAM_INT_ARRAY)
                        )
                    )
                    ->execute();
                $lostIdList = [];
                while ($row = $queryResult->fetch()) {
                    $this->lRecords[$table][$row['uid']] = [
                        'uid' => $row['uid'],
                        'pid' => $row['pid'],
                        'title' => strip_tags(BackendUtility::getRecordTitle($table, $row))
                    ];
                    $lostIdList[] = $row['uid'];
                }
                if ($table === 'pages') {
                    $this->lostPagesList = implode(',', $lostIdList);
                }
            }
        }
    }

    public function fixLostRecord($table, $uid)
    {
        if ($table && $GLOBALS['TCA'][$table] && $uid && is_array($this->lRecords[$table][$uid]) && $GLOBALS['BE_USER']->user['admin']) {
            $updateFields = [
                'pid' => 0
            ];
            // If possible a lost record restored is hidden as default
            if ($GLOBALS['TCA'][$table]['ctrl']['enablecolumns']['disabled']) {
                $updateFields[$GLOBALS['TCA'][$table]['ctrl']['enablecolumns']['disabled']] = 1;
            }
            GeneralUtility::makeInstance(ConnectionPool::class)
                ->getConnectionForTable($table)
                ->update($table, $updateFields, ['uid' => (int)$uid]);
            return true;
        }
        return false;
    }

    public function countRecords($pid_list)
    {
        $list = [];
        $list_n = [];
        $pageIds = GeneralUtility::intExplode(',', $pid_list);
        if (!empty($pageIds)) {
            foreach ($GLOBALS['TCA'] as $table => $tableConf) {
                $pageIdsForTable = $pageIds;
                // Remove preceding "-1," for non-versioned tables
                if (!BackendUtility::isTableWorkspaceEnabled($table)) {
                    $pageIdsForTable = array_combine($pageIdsForTable, $pageIdsForTable);
                    unset($pageIdsForTable[-1]);
                }
                $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table);
                $queryBuilder->getRestrictions()->removeAll();
                $count = $queryBuilder->count('uid')
                    ->from($table)
                    ->where(
                        $queryBuilder->expr()->in(
                            'pid',
                            $queryBuilder->createNamedParameter($pageIds, Connection::PARAM_INT_ARRAY)
                        )
                    )
                    ->execute()
                    ->fetchColumn(0);
                if ($count) {
                    $list[$table] = $count;
                }

                // same query excluding all deleted records
                $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table);
                $queryBuilder->getRestrictions()
                    ->removeAll()
                    ->add(GeneralUtility::makeInstance(DeletedRestriction::class));
                $count = $queryBuilder->count('uid')
                    ->from($table)
                    ->where(
                        $queryBuilder->expr()->in(
                            'pid',
                            $queryBuilder->createNamedParameter($pageIdsForTable, Connection::PARAM_INT_ARRAY)
                        )
                    )
                    ->execute()
                    ->fetchColumn(0);
                if ($count) {
                    $list_n[$table] = $count;
                }
            }
        }
        return ['all' => $list, 'non_deleted' => $list_n];
    }

    public function getGroupFields($mode)
    {
        $result = [];
        foreach ($GLOBALS['TCA'] as $table => $tableConf) {
            $cols = $GLOBALS['TCA'][$table]['columns'];
            foreach ($cols as $field => $config) {
                if ($config['config']['type'] === 'group') {
                    if ((!$mode || $mode === 'file') && $config['config']['internal_type'] === 'file' || (!$mode || $mode === 'db') && $config['config']['internal_type'] === 'db') {
                        $result[$table][] = $field;
                    }
                }
                if ((!$mode || $mode === 'db') && $config['config']['type'] === 'select' && $config['config']['foreign_table']) {
                    $result[$table][] = $field;
                }
            }
            if ($result[$table]) {
                $result[$table] = implode(',', $result[$table]);
            }
        }
        return $result;
    }

    public function getFileFields($uploadfolder)
    {
        $result = [];
        foreach ($GLOBALS['TCA'] as $table => $tableConf) {
            $cols = $GLOBALS['TCA'][$table]['columns'];
            foreach ($cols as $field => $config) {
                if ($config['config']['type'] === 'group' && $config['config']['internal_type'] === 'file' && $config['config']['uploadfolder'] == $uploadfolder) {
                    $result[] = [$table, $field];
                }
            }
        }
        return $result;
    }

    public function getDBFields($theSearchTable)
    {
        $result = [];
        foreach ($GLOBALS['TCA'] as $table => $tableConf) {
            $cols = $GLOBALS['TCA'][$table]['columns'];
            foreach ($cols as $field => $config) {
                if ($config['config']['type'] === 'group' && $config['config']['internal_type'] === 'db') {
                    if (trim($config['config']['allowed']) === '*' || strstr($config['config']['allowed'], $theSearchTable)) {
                        $result[] = [$table, $field];
                    }
                } elseif ($config['config']['type'] === 'select' && $config['config']['foreign_table'] == $theSearchTable) {
                    $result[] = [$table, $field];
                }
            }
        }
        return $result;
    }

    public function selectNonEmptyRecordsWithFkeys($fkey_arrays)
    {
        if (is_array($fkey_arrays)) {
            $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
            foreach ($fkey_arrays as $table => $field_list) {
                if ($GLOBALS['TCA'][$table] && trim($field_list)) {
                    $connection = $connectionPool->getConnectionForTable($table);
                    $schemaManager = $connection->getSchemaManager();
                    $tableColumns = $schemaManager->listTableColumns($table);

                    $queryBuilder = $connectionPool->getQueryBuilderForTable($table);
                    $queryBuilder->getRestrictions()->removeAll();

                    $fields = GeneralUtility::trimExplode(',', $field_list, true);

                    $queryBuilder->select('uid')
                        ->from($table);
                    $whereClause = [];

                    foreach ($fields as $fieldName) {
                        // The array index of $tableColumns is the lowercased column name!
                        // It is quoted for keywords
                        $column = $tableColumns[strtolower($fieldName)]
                            ?? $tableColumns[$connection->quoteIdentifier(strtolower($fieldName))];
                        if (!$column) {
                            // Throw meaningful exception if field does not exist in DB - 'none' is not filtered here since the
                            // method is only called with type=group fields
                            throw new \RuntimeException(
                                'Field ' . $fieldName . ' for table ' . $table . ' has been defined in TCA, but does not exist in DB',
                                1536248936
                            );
                        }
                        $fieldType = $column->getType()->getName();
                        if (in_array(
                            $fieldType,
                            [Type::BIGINT, Type::INTEGER, Type::SMALLINT, Type::DECIMAL, Type::FLOAT],
                            true
                        )) {
                            $whereClause[] = $queryBuilder->expr()->andX(
                                $queryBuilder->expr()->isNotNull($fieldName),
                                $queryBuilder->expr()->neq(
                                    $fieldName,
                                    $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)
                                )
                            );
                        } elseif (in_array($fieldType, [Type::STRING, Type::TEXT], true)) {
                            $whereClause[] = $queryBuilder->expr()->andX(
                                $queryBuilder->expr()->isNotNull($fieldName),
                                $queryBuilder->expr()->neq(
                                    $fieldName,
                                    $queryBuilder->createNamedParameter('', \PDO::PARAM_STR)
                                )
                            );
                        } elseif (in_array($fieldType, [Type::BLOB], true)) {
                            $whereClause[] = $queryBuilder->expr()->andX(
                                $queryBuilder->expr()->isNotNull($fieldName),
                                $queryBuilder->expr()
                                    ->comparison(
                                        $queryBuilder->expr()->length($fieldName),
                                        ExpressionBuilder::GT,
                                        $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)
                                    )
                            );
                        }
                    }
                    $queryResult = $queryBuilder->orWhere(...$whereClause)->execute();

                    while ($row = $queryResult->fetch()) {
                        foreach ($fields as $field) {
                            if (trim($row[$field])) {
                                $fieldConf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
                                if ($fieldConf['type'] === 'group') {
                                    if ($fieldConf['internal_type'] === 'file') {
                                        // Files...
                                        if ($fieldConf['MM']) {
                                            $tempArr = [];
                                            $dbAnalysis = GeneralUtility::makeInstance(RelationHandler::class);
                                            $dbAnalysis->start('', 'files', $fieldConf['MM'], $row['uid']);
                                            foreach ($dbAnalysis->itemArray as $somekey => $someval) {
                                                if ($someval['id']) {
                                                    $tempArr[] = $someval['id'];
                                                }
                                            }
                                        } else {
                                            $tempArr = explode(',', trim($row[$field]));
                                        }
                                        foreach ($tempArr as $file) {
                                            $file = trim($file);
                                            if ($file) {
                                                $this->checkFileRefs[$fieldConf['uploadfolder']][$file] += 1;
                                            }
                                        }
                                    }
                                    if ($fieldConf['internal_type'] === 'db') {
                                        $dbAnalysis = GeneralUtility::makeInstance(RelationHandler::class);
                                        $dbAnalysis->start(
                                            $row[$field],
                                            $fieldConf['allowed'],
                                            $fieldConf['MM'],
                                            $row['uid'],
                                            $table,
                                            $fieldConf
                                        );
                                        foreach ($dbAnalysis->itemArray as $tempArr) {
                                            $this->checkGroupDBRefs[$tempArr['table']][$tempArr['id']] += 1;
                                        }
                                    }
                                }
                                if ($fieldConf['type'] === 'select' && $fieldConf['foreign_table']) {
                                    $dbAnalysis = GeneralUtility::makeInstance(RelationHandler::class);
                                    $dbAnalysis->start(
                                        $row[$field],
                                        $fieldConf['foreign_table'],
                                        $fieldConf['MM'],
                                        $row['uid'],
                                        $table,
                                        $fieldConf
                                    );
                                    foreach ($dbAnalysis->itemArray as $tempArr) {
                                        if ($tempArr['id'] > 0) {
                                            $this->checkGroupDBRefs[$fieldConf['foreign_table']][$tempArr['id']] += 1;
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }

    public function testFileRefs()
    {
        $output = [];
        // Handle direct references with upload folder setting (workaround)
        $newCheckFileRefs = [];
        foreach ($this->checkFileRefs as $folder => $files) {
            // Only direct references without a folder setting
            if ($folder !== '') {
                $newCheckFileRefs[$folder] = $files;
                continue;
            }
            foreach ($files as $file => $references) {
                // Direct file references have often many references (removes occurrences in the moreReferences section of the result array)
                if ($references > 1) {
                    $references = 1;
                }
                // The directory must be empty (prevents checking of the root directory)
                $directory = dirname($file);
                if ($directory !== '') {
                    $newCheckFileRefs[$directory][basename($file)] = $references;
                }
            }
        }
        $this->checkFileRefs = $newCheckFileRefs;
        foreach ($this->checkFileRefs as $folder => $fileArr) {
            $path = PATH_site . $folder;
            if (@is_dir($path) && @is_readable($path)) {
                $d = dir($path);
                while ($entry = $d->read()) {
                    if (@is_file(($path . '/' . $entry))) {
                        if (isset($fileArr[$entry])) {
                            if ($fileArr[$entry] > 1) {
                                $temp = $this->whereIsFileReferenced($folder, $entry);
                                $tempList = '';
                                foreach ($temp as $inf) {
                                    $tempList .= '[' . $inf['table'] . '][' . $inf['uid'] . '][' . $inf['field'] . '] (pid:' . $inf['pid'] . ') - ';
                                }
                                $output['moreReferences'][] = [$path, $entry, $fileArr[$entry], $tempList];
                            }
                            unset($fileArr[$entry]);
                        } else {
                            // Contains workaround for direct references
                            if (!strstr($entry, 'index.htm') && !preg_match(('/^' . preg_quote($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') . '/'), $folder)) {
                                $output['noReferences'][] = [$path, $entry];
                            }
                        }
                    }
                }
                $d->close();
                $tempCounter = 0;
                foreach ($fileArr as $file => $value) {
                    // Workaround for direct file references
                    if (preg_match('/^' . preg_quote($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') . '/', $folder)) {
                        $file = $folder . '/' . $file;
                        $folder = '';
                        $path = substr(PATH_site, 0, -1);
                    }
                    $temp = $this->whereIsFileReferenced($folder, $file);
                    $tempList = '';
                    foreach ($temp as $inf) {
                        $tempList .= '[' . $inf['table'] . '][' . $inf['uid'] . '][' . $inf['field'] . '] (pid:' . $inf['pid'] . ') - ';
                    }
                    $tempCounter++;
                    $output['noFile'][substr($path, -3) . '_' . substr($file, 0, 3) . '_' . $tempCounter] = [$path, $file, $tempList];
                }
            } else {
                $output['error'][] = [$path];
            }
        }
        return $output;
    }

    public function testDBRefs($theArray)
    {
        $result = '';
        foreach ($theArray as $table => $dbArr) {
            if ($GLOBALS['TCA'][$table]) {
                $ids = array_keys($dbArr);
                if (!empty($ids)) {
                    $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
                        ->getQueryBuilderForTable($table);
                    $queryBuilder->getRestrictions()
                        ->removeAll()
                        ->add(GeneralUtility::makeInstance(DeletedRestriction::class));
                    $queryResult = $queryBuilder
                        ->select('uid')
                        ->from($table)
                        ->where(
                            $queryBuilder->expr()->in(
                                'uid',
                                $queryBuilder->createNamedParameter($ids, Connection::PARAM_INT_ARRAY)
                            )
                        )
                        ->execute();
                    while ($row = $queryResult->fetch()) {
                        if (isset($dbArr[$row['uid']])) {
                            unset($dbArr[$row['uid']]);
                        } else {
                            $result .= 'Strange Error. ...<br />';
                        }
                    }
                    foreach ($dbArr as $theId => $theC) {
                        $result .= 'There are ' . $theC . ' records pointing to this missing or deleted record; [' . $table . '][' . $theId . ']<br />';
                    }
                }
            } else {
                $result .= 'Codeerror. Table is not a table...<br />';
            }
        }
        return $result;
    }

    public function whereIsRecordReferenced($searchTable, $id)
    {
        // Gets tables / Fields that reference to files
        $fileFields = $this->getDBFields($searchTable);
        $theRecordList = [];
        foreach ($fileFields as $info) {
            list($table, $field) = $info;
            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table);
            $queryBuilder->getRestrictions()->removeAll();
            $queryResult = $queryBuilder
                ->select('uid', 'pid', $GLOBALS['TCA'][$table]['ctrl']['label'], $field)
                ->from($table)
                ->where(
                    $queryBuilder->expr()->like(
                        $field,
                        $queryBuilder->createNamedParameter('%' . $queryBuilder->escapeLikeWildcards($id) . '%')
                    )
                )
                ->execute();

            while ($row = $queryResult->fetch()) {
                // Now this is the field, where the reference COULD come from.
                // But we're not guaranteed, so we must carefully examine the data.
                $fieldConf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
                $allowedTables = $fieldConf['type'] === 'group' ? $fieldConf['allowed'] : $fieldConf['foreign_table'];
                $dbAnalysis = GeneralUtility::makeInstance(RelationHandler::class);
                $dbAnalysis->start($row[$field], $allowedTables, $fieldConf['MM'], $row['uid'], $table, $fieldConf);
                foreach ($dbAnalysis->itemArray as $tempArr) {
                    if ($tempArr['table'] == $searchTable && $tempArr['id'] == $id) {
                        $theRecordList[] = [
                            'table' => $table,
                            'uid' => $row['uid'],
                            'field' => $field,
                            'pid' => $row['pid']
                        ];
                    }
                }
            }
        }
        return $theRecordList;
    }

    public function whereIsFileReferenced($uploadFolder, $filename)
    {
        // Gets tables / Fields that reference to files
        $fileFields = $this->getFileFields($uploadFolder);
        $theRecordList = [];
        foreach ($fileFields as $info) {
            list($table, $field) = $info;
            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table);
            $queryBuilder->getRestrictions()->removeAll();
            $queryResult = $queryBuilder
                ->select('uid', 'pid', $GLOBALS['TCA'][$table]['ctrl']['label'], $field)
                ->from($table)
                ->where(
                    $queryBuilder->expr()->like(
                        $field,
                        $queryBuilder->createNamedParameter('%' . $queryBuilder->escapeLikeWildcards($filename) . '%')
                    )
                )
                ->execute();
            while ($row = $queryResult->fetch()) {
                // Now this is the field, where the reference COULD come from.
                // But we're not guaranteed, so we must carefully examine the data.
                $tempArr = explode(',', trim($row[$field]));
                foreach ($tempArr as $file) {
                    $file = trim($file);
                    if ($file == $filename) {
                        $theRecordList[] = [
                            'table' => $table,
                            'uid' => $row['uid'],
                            'field' => $field,
                            'pid' => $row['pid']
                        ];
                    }
                }
            }
        }
        return $theRecordList;
    }
}