Indexer.php

Go to the documentation of this file.

<?php
namespace TYPO3\CMS\Core\Resource\Index;

use TYPO3\CMS\Core\Resource\ResourceStorage;
use TYPO3\CMS\Core\Resource\File;
use TYPO3\CMS\Core\Utility\GeneralUtility;

class Indexer {

    protected $filesToUpdate = array();

    protected $identifiedFileUids = array();

    protected $storage = NULL;

    public function __construct(ResourceStorage $storage) {
        $this->storage = $storage;
    }

    public function createIndexEntry($identifier) {
        if (!isset($identifier) || !is_string($identifier) || $identifier === '') {
            throw new \InvalidArgumentException('Invalid file identifier given. It must be of type string and not empty. "' . gettype($identifier) . '" given.', 1401732565);
        }
        $fileProperties = $this->gatherFileInformationArray($identifier);
        $record = $this->getFileIndexRepository()->addRaw($fileProperties);
        $fileObject = $this->getResourceFactory()->getFileObject($record['uid'], $record);
        $this->extractRequiredMetaData($fileObject);
        return $fileObject;
    }

    public function updateIndexEntry(File $fileObject) {
        $updatedInformation = $this->gatherFileInformationArray($fileObject->getIdentifier());
        $fileObject->updateProperties($updatedInformation);
        $this->getFileIndexRepository()->update($fileObject);
        $this->extractRequiredMetaData($fileObject);
    }

    public function processChangesInStorages() {
        // get all file-identifiers from the storage
        $availableFiles = $this->storage->getFileIdentifiersInFolder($this->storage->getRootLevelFolder(FALSE)->getIdentifier(), TRUE, TRUE);
        $this->detectChangedFilesInStorage($availableFiles);
        $this->processChangedAndNewFiles();

        $this->detectMissingFiles();
    }

    public function runMetaDataExtraction($maximumFileCount = -1) {
        $fileIndexRecords = $this->getFileIndexRepository()->findInStorageWithIndexOutstanding($this->storage, $maximumFileCount);

        $extractionServices = $this->getExtractorRegistry()->getExtractorsWithDriverSupport($this->storage->getDriverType());
        foreach ($fileIndexRecords as $indexRecord) {
            $fileObject = $this->getResourceFactory()->getFileObject($indexRecord['uid'], $indexRecord);

            $newMetaData = array(
                0 => $fileObject->_getMetaData()
            );
            foreach ($extractionServices as $service) {
                if ($service->canProcess($fileObject)) {
                    $newMetaData[$service->getPriority()] = $service->extractMetaData($fileObject, $newMetaData);
                }
            }
            ksort($newMetaData);
            $metaData = array();
            foreach ($newMetaData as $data) {
                $metaData = array_merge($metaData, $data);
            }
            $fileObject->_updateMetaDataProperties($metaData);
            $this->getMetaDataRepository()->update($fileObject->getUid(), $metaData);
            $this->getFileIndexRepository()->updateIndexingTime($fileObject->getUid());
        }
    }

    protected function detectMissingFiles() {
        $indexedNotExistentFiles = $this->getFileIndexRepository()->findInStorageAndNotInUidList(
            $this->storage,
            $this->identifiedFileUids
        );

        foreach ($indexedNotExistentFiles as $record) {
            if (!$this->storage->hasFile($record['identifier'])) {
                $this->getFileIndexRepository()->markFileAsMissing($record['uid']);
            }
        }
    }

    protected function detectChangedFilesInStorage(array $fileIdentifierArray) {
        foreach ($fileIdentifierArray as $fileIdentifier) {
            // skip processed files
            if ($this->storage->isWithinProcessingFolder($fileIdentifier)) {
                continue;
            }
            // Get the modification time for file-identifier from the storage
            $modificationTime = $this->storage->getFileInfoByIdentifier($fileIdentifier, array('mtime'));
            // Look if the the modification time in FS is higher than the one in database (key needed on timestamps)
            $indexRecord = $this->getFileIndexRepository()->findOneByStorageUidAndIdentifier($this->storage->getUid(), $fileIdentifier);

            if ($indexRecord !== FALSE) {
                $this->identifiedFileUids[] = $indexRecord['uid'];

                if ($indexRecord['modification_date'] < $modificationTime['mtime'] || $indexRecord['missing']) {
                    $this->filesToUpdate[$fileIdentifier] = $indexRecord;
                }
            } else {
                $this->filesToUpdate[$fileIdentifier] = NULL;
            }
        }
    }

    protected function processChangedAndNewFiles() {
        foreach ($this->filesToUpdate AS $identifier => $data) {
            if ($data == NULL) {
                // search for files with same content hash in indexed storage
                $fileHash = $this->storage->hashFileByIdentifier($identifier, 'sha1');
                $files = $this->getFileIndexRepository()->findByContentHash($fileHash);
                $fileObject = NULL;
                if (!empty($files)) {
                    foreach ($files as $fileIndexEntry) {
                        // check if file is missing then we assume it's moved/renamed
                        if (!$this->storage->hasFile($fileIndexEntry['identifier'])) {
                            $fileObject = $this->getResourceFactory()->getFileObject($fileIndexEntry['uid'], $fileIndexEntry);
                            $fileObject->updateProperties(array(
                                'identifier' => $identifier
                            ));
                            $this->updateIndexEntry($fileObject);
                            $this->identifiedFileUids[] = $fileObject->getUid();
                            break;
                        }
                    }
                }
                // create new index when no missing file with same content hash is found
                if ($fileObject === NULL) {
                    $fileObject = $this->createIndexEntry($identifier);
                    $this->identifiedFileUids[] = $fileObject->getUid();
                }
            } else {
                // update existing file
                $fileObject = $this->getResourceFactory()->getFileObject($data['uid'], $data);
                $this->updateIndexEntry($fileObject);
            }
        }
    }

    protected function extractRequiredMetaData(File $fileObject) {
        // since the core desperately needs image sizes in metadata table do this manually
        // prevent doing this for remote storages, remote storages must provide the data with extractors
        if ($fileObject->getType() == File::FILETYPE_IMAGE && $this->storage->getDriverType() === 'Local') {
            $rawFileLocation = $fileObject->getForLocalProcessing(FALSE);
            $imageInfo = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Type\\File\\ImageInfo', $rawFileLocation);
            $metaData = array(
                'width' => $imageInfo->getWidth(),
                'height' => $imageInfo->getHeight(),
            );
            $this->getMetaDataRepository()->update($fileObject->getUid(), $metaData);
            $fileObject->_updateMetaDataProperties($metaData);
        }
    }

    /****************************
     *
     *         UTILITY
     *
     ****************************/

    protected function gatherFileInformationArray($identifier) {
        $fileInfo = $this->storage->getFileInfoByIdentifier($identifier);
        $fileInfo = $this->transformFromDriverFileInfoArrayToFileObjectFormat($fileInfo);
        $fileInfo['type'] = $this->getFileType($fileInfo['mime_type']);
        $fileInfo['sha1'] = $this->storage->hashFileByIdentifier($identifier, 'sha1');
        $fileInfo['extension'] = \TYPO3\CMS\Core\Utility\PathUtility::pathinfo($fileInfo['name'], PATHINFO_EXTENSION);
        $fileInfo['missing'] = 0;

        return $fileInfo;
    }

    protected function getFileType($mimeType) {
        list($fileType) = explode('/', $mimeType);
        switch (strtolower($fileType)) {
            case 'text':
                $type = File::FILETYPE_TEXT;
                break;
            case 'image':
                $type = File::FILETYPE_IMAGE;
                break;
            case 'audio':
                $type = File::FILETYPE_AUDIO;
                break;
            case 'video':
                $type = File::FILETYPE_VIDEO;
                break;
            case 'application':
            case 'software':
                $type = File::FILETYPE_APPLICATION;
                break;
            default:
                $type = File::FILETYPE_UNKNOWN;
        }
        return $type;
    }

    protected function transformFromDriverFileInfoArrayToFileObjectFormat(array $fileInfo) {
        $mappingInfo = array(
            // 'driverKey' => 'fileProperty' Key is from the driver, value is for the property in the file
            'size' => 'size',
            'atime' => NULL,
            'mtime' => 'modification_date',
            'ctime' => 'creation_date',
            'mimetype' => 'mime_type'
        );
        $mappedFileInfo = array();
        foreach ($fileInfo as $key => $value) {
            if (array_key_exists($key, $mappingInfo)) {
                if ($mappingInfo[$key] !== NULL) {
                    $mappedFileInfo[$mappingInfo[$key]] = $value;
                }
            } else {
                $mappedFileInfo[$key] = $value;
            }
        }
        return $mappedFileInfo;
    }


    protected function getFileIndexRepository() {
        return FileIndexRepository::getInstance();
    }

    protected function getMetaDataRepository() {
        return MetaDataRepository::getInstance();
    }

    protected function getResourceFactory() {
        return \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance();
    }

    protected function getExtractorRegistry() {
        return ExtractorRegistry::getInstance();
    }
}