Import.php

Go to the documentation of this file.

<?php
namespace TYPO3\CMS\Impexp;

/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\Configuration\FlexForm\FlexFormTools;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\DataHandling\DataHandler;
use TYPO3\CMS\Core\Exception;
use TYPO3\CMS\Core\Resource\Exception\InsufficientFolderAccessPermissionsException;
use TYPO3\CMS\Core\Resource\File;
use TYPO3\CMS\Core\Resource\FileInterface;
use TYPO3\CMS\Core\Resource\ResourceFactory;
use TYPO3\CMS\Core\Resource\ResourceStorage;
use TYPO3\CMS\Core\Resource\StorageRepository;
use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\MathUtility;
use TYPO3\CMS\Core\Utility\PathUtility;
use TYPO3\CMS\Core\Utility\StringUtility;

class Import extends ImportExport
{
    public $suggestedInsertUids = [];

    public $enableLogging = false;

    public $import_newId = [];

    public $import_newId_pids = [];

    public $import_data = [];

    protected $storageObjects = [];

    protected $filesPathForImport = null;

    protected $unlinkFiles = [];

    protected $alternativeFileName = [];

    protected $alternativeFilePath = [];

    protected $filePathMap = [];

    /**************************
     * Initialize
     *************************/

    public function init()
    {
        parent::init();
        $this->mode = 'import';
    }

    /***********************
     * Import
     ***********************/

    protected function initializeImport()
    {
        // Set this flag to indicate that an import is being/has been done.
        $this->doesImport = 1;
        // Initialize:
        // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
        $this->import_mapId = [];
        $this->import_newId = [];
        $this->import_newId_pids = [];
        // Temporary files stack initialized:
        $this->unlinkFiles = [];
        $this->alternativeFileName = [];
        $this->alternativeFilePath = [];

        $this->initializeStorageObjects();
    }

    protected function initializeStorageObjects()
    {
        $storageRepository = GeneralUtility::makeInstance(StorageRepository::class);
        $this->storageObjects = $storageRepository->findAll();
    }

    public function importData($pid)
    {
        $this->initializeImport();

        // Write sys_file_storages first
        $this->writeSysFileStorageRecords();
        // Write sys_file records and write the binary file data
        $this->writeSysFileRecords();
        // Write records, first pages, then the rest
        // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
        $this->writeRecords_pages($pid);
        $this->writeRecords_records($pid);
        // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
        // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
        $this->setRelations();
        // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
        $this->setFlexFormRelations();
        // Unlink temporary files:
        $this->unlinkTempFiles();
        // Finally, traverse all records and process softreferences with substitution attributes.
        $this->processSoftReferences();
    }

    protected function writeSysFileStorageRecords()
    {
        if (!isset($this->dat['header']['records']['sys_file_storage'])) {
            return;
        }
        $sysFileStorageUidsToBeResetToDefaultStorage = [];
        foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
            $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
            // continue with Local, writable and online storage only
            if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
                foreach ($this->storageObjects as $localStorage) {
                    if ($this->isEquivalentObjectStorage($localStorage, $storageRecord)) {
                        $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $localStorage->getUid();
                        break;
                    }
                }

                if (!isset($this->import_mapId['sys_file_storage'][$sysFileStorageUid])) {
                    // Local, writable and online storage. Is allowed to be used to later write files in.
                    // Does currently not exist so add the record.
                    $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
                }
            } else {
                // Storage with non Local drivers could be imported but must not be used to saves files in, because you
                // could not be sure, that this is supported. The default storage will be used in this case.
                // It could happen that non writable and non online storage will be created as dupes because you could not
                // check the detailed configuration options at this point
                $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
                $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
            }
        }

        // Importing the added ones
        $tce = $this->getNewTCE();
        // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
        $tce->reverseOrder = 1;
        $tce->isImporting = true;
        $tce->start($this->import_data, []);
        $tce->process_datamap();
        $this->addToMapId($tce->substNEWwithIDs);

        $defaultStorageUid = null;
        // get default storage
        $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
        if ($defaultStorage !== null) {
            $defaultStorageUid = $defaultStorage->getUid();
        }
        foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
            $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
        }

        // unset the sys_file_storage records to prevent an import in writeRecords_records
        unset($this->dat['header']['records']['sys_file_storage']);
    }

    protected function isEquivalentObjectStorage(ResourceStorage $storageObject, array $storageRecord)
    {
        // compare the properties: driver, writable and online
        if ($storageObject->getDriverType() === $storageRecord['driver']
            && (bool)$storageObject->isWritable() === (bool)$storageRecord['is_writable']
            && (bool)$storageObject->isOnline() === (bool)$storageRecord['is_online']
        ) {
            $storageRecordConfiguration = ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
            $storageObjectConfiguration = $storageObject->getConfiguration();
            // compare the properties: pathType and basePath
            if ($storageRecordConfiguration['pathType'] === $storageObjectConfiguration['pathType']
                && $storageRecordConfiguration['basePath'] === $storageObjectConfiguration['basePath']
            ) {
                return true;
            }
        }
        return false;
    }

    public function checkImportPrerequisites()
    {
        $messages = [];

        // Check #1: Extension dependencies
        $extKeysToInstall = [];
        foreach ($this->dat['header']['extensionDependencies'] as $extKey) {
            if (!empty($extKey) && !ExtensionManagementUtility::isLoaded($extKey)) {
                $extKeysToInstall[] = $extKey;
            }
        }
        if (!empty($extKeysToInstall)) {
            $messages['missingExtensions'] = 'Before you can install this T3D file you need to install the extensions "'
                . implode('", "', $extKeysToInstall) . '".';
        }

        // Check #2: If the path for every local storage object exists.
        // Else files can't get moved into a newly imported storage.
        if (!empty($this->dat['header']['records']['sys_file_storage'])) {
            foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
                $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
                // continue with Local, writable and online storage only
                if ($storageRecord['driver'] === 'Local'
                    && $storageRecord['is_writable']
                    && $storageRecord['is_online']
                ) {
                    foreach ($this->storageObjects as $localStorage) {
                        if ($this->isEquivalentObjectStorage($localStorage, $storageRecord)) {
                            // There is already an existing storage
                            break;
                        }

                        // The storage from the import does not have an equivalent storage
                        // in the current instance (same driver, same path, etc.). Before
                        // the storage record can get inserted later on take care the path
                        // it points to really exists and is accessible.
                        $storageRecordUid = $storageRecord['uid'];
                        // Unset the storage record UID when trying to create the storage object
                        // as the record does not already exist in DB. The constructor of the
                        // storage object will check whether the target folder exists and set the
                        // isOnline flag depending on the outcome.
                        $storageRecord['uid'] = 0;
                        $resourceStorage = ResourceFactory::getInstance()->createStorageObject($storageRecord);
                        if (!$resourceStorage->isOnline()) {
                            $configuration = $resourceStorage->getConfiguration();
                            $messages['resourceStorageFolderMissing_' . $storageRecordUid] =
                                'The resource storage "'
                                . $resourceStorage->getName()
                                . $configuration['basePath']
                                . '" does not exist. Please create the directory prior to starting the import!';
                        }
                    }
                }
            }
        }

        return $messages;
    }

    protected function writeSysFileRecords()
    {
        if (!isset($this->dat['header']['records']['sys_file'])) {
            return;
        }
        $this->addGeneralErrorsByTable('sys_file');

        // fetch fresh storage records from database
        $storageRecords = $this->fetchStorageRecords();

        $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();

        $sanitizedFolderMappings = [];

        foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
            $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];

            $temporaryFile = null;
            // check if there is the right file already in the local folder
            if ($this->filesPathForImport !== null) {
                if (is_file($this->filesPathForImport . '/' . $fileRecord['sha1']) && sha1_file($this->filesPathForImport . '/' . $fileRecord['sha1']) === $fileRecord['sha1']) {
                    $temporaryFile = $this->filesPathForImport . '/' . $fileRecord['sha1'];
                }
            }

            // save file to disk
            if ($temporaryFile === null) {
                $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
                $temporaryFile = $this->writeTemporaryFileFromData($fileId);
                if ($temporaryFile === null) {
                    // error on writing the file. Error message was already added
                    continue;
                }
            }

            $originalStorageUid = $fileRecord['storage'];
            $useStorageFromStorageRecords = false;

            // replace storage id, if an alternative one was registered
            if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
                $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
                $useStorageFromStorageRecords = true;
            }

            if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {
                // no storage for the file is defined, mostly because of a missing default storage.
                $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
                continue;
            }

            // using a storage from the local storage is only allowed, if the uid is present in the
            // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
            if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
                $storage = ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
            } elseif ($this->isFallbackStorage($fileRecord['storage'])) {
                $storage = ResourceFactory::getInstance()->getStorageObject(0);
            } elseif ($defaultStorage !== null) {
                $storage = $defaultStorage;
            } else {
                $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
                continue;
            }

            $newFile = null;

            // check, if there is an identical file
            try {
                if ($storage->hasFile($fileRecord['identifier'])) {
                    $file = $storage->getFile($fileRecord['identifier']);
                    if ($file->getSha1() === $fileRecord['sha1']) {
                        $newFile = $file;
                    }
                }
            } catch (Exception $e) {
            }

            if ($newFile === null) {
                $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
                if (in_array($folderName, $sanitizedFolderMappings)) {
                    $folderName = $sanitizedFolderMappings[$folderName];
                }
                if (!$storage->hasFolder($folderName)) {
                    try {
                        $importFolder = $storage->createFolder($folderName);
                        if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
                            $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
                        }
                    } catch (Exception $e) {
                        $this->error('Error: Folder "' . $folderName . '" could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
                        continue;
                    }
                } else {
                    $importFolder = $storage->getFolder($folderName);
                }

                $this->callHook('before_addSysFileRecord', [
                    'fileRecord' => $fileRecord,
                    'importFolder' => $importFolder,
                    'temporaryFile' => $temporaryFile
                ]);

                try {
                    $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
                } catch (Exception $e) {
                    $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
                    continue;
                }

                if ($newFile->getSha1() !== $fileRecord['sha1']) {
                    $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
                }
            }

            // save the new uid in the import id map
            $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
            $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
        }

        // unset the sys_file records to prevent an import in writeRecords_records
        unset($this->dat['header']['records']['sys_file']);
        // remove all sys_file_reference records that point to file records which are unknown
        // in the system to prevent exceptions
        $this->removeSysFileReferenceRecordsFromImportDataWithRelationToMissingFile();
    }

    protected function removeSysFileReferenceRecordsFromImportDataWithRelationToMissingFile()
    {
        if (!isset($this->dat['header']['records']['sys_file_reference'])) {
            return;
        }

        foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
            $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
            if (!in_array($fileReferenceRecord['uid_local'], $this->import_mapId['sys_file'])) {
                unset($this->dat['header']['records']['sys_file_reference'][$sysFileReferenceUid]);
                unset($this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]);
                $this->error(
                    'Error: sys_file_reference record ' . (int)$sysFileReferenceUid
                             . ' with relation to sys_file record ' . (int)$fileReferenceRecord['uid_local']
                             . ', which is not part of the import data, was not imported.'
                );
            }
        }
    }

    protected function isFallbackStorage($storageId)
    {
        return $storageId === 0 || $storageId === '0';
    }

    protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid)
    {
        if (!isset($this->dat['header']['records']['sys_file_reference'])) {
            return;
        }

        foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
            if (!isset($this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['hasBeenMapped'])
                && $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data']['uid_local'] == $oldFileUid
            ) {
                $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['hasBeenMapped'] = true;
                $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data']['uid_local'] = $newFileUid;
            }
        }
    }

    protected function fetchStorageRecords()
    {
        $result = GeneralUtility::makeInstance(ConnectionPool::class)
            ->getQueryBuilderForTable('sys_file_storage')
            ->select('*')
            ->from('sys_file_storage')
            ->orderBy('uid')
            ->execute();
        $rows = [];
        while ($row = $result->fetch()) {
            $rows[$row['uid']] = $row;
        }
        return $rows;
    }

    protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal')
    {
        $temporaryFilePath = null;
        if (is_array($this->dat[$dataKey][$fileId])) {
            $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
            GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
            clearstatcache();
            if (@is_file($temporaryFilePathInternal)) {
                $this->unlinkFiles[] = $temporaryFilePathInternal;
                if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
                    $temporaryFilePath = $temporaryFilePathInternal;
                } else {
                    $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')');
                }
            } else {
                $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!');
            }
        } else {
            $this->error('Error: No file found for ID ' . $fileId);
        }
        return $temporaryFilePath;
    }

    public function writeRecords_pages($pid)
    {
        // First, write page structure if any:
        if (is_array($this->dat['header']['records']['pages'])) {
            $this->addGeneralErrorsByTable('pages');
            // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
            $pageRecords = $this->dat['header']['records']['pages'];
            $this->import_data = [];
            // First add page tree if any
            if (is_array($this->dat['header']['pagetree'])) {
                $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
                foreach ($pagesFromTree as $uid) {
                    $thisRec = $this->dat['header']['records']['pages'][$uid];
                    // PID: Set the main $pid, unless a NEW-id is found
                    $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
                    $this->addSingle('pages', $uid, $setPid);
                    unset($pageRecords[$uid]);
                }
            }
            // Then add all remaining pages not in tree on root level:
            if (!empty($pageRecords)) {
                $remainingPageUids = array_keys($pageRecords);
                foreach ($remainingPageUids as $pUid) {
                    $this->addSingle('pages', $pUid, $pid);
                }
            }
            // Now write to database:
            $tce = $this->getNewTCE();
            $tce->isImporting = true;
            $this->callHook('before_writeRecordsPages', [
                'tce' => &$tce,
                'data' => &$this->import_data
            ]);
            $tce->suggestedInsertUids = $this->suggestedInsertUids;
            $tce->start($this->import_data, []);
            $tce->process_datamap();
            $this->callHook('after_writeRecordsPages', [
                'tce' => &$tce
            ]);
            // post-processing: Registering new ids (end all DataHandler sessions with this)
            $this->addToMapId($tce->substNEWwithIDs);
            // In case of an update, order pages from the page tree correctly:
            if ($this->update && is_array($this->dat['header']['pagetree'])) {
                $this->writeRecords_pages_order();
            }
        }
    }

    public function writeRecords_pages_order()
    {
        $cmd_data = [];
        // Get uid-pid relations and traverse them in order to map to possible new IDs
        $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
        foreach ($pidsFromTree as $origPid => $newPid) {
            if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
                // If the page had a new id (because it was created) use that instead!
                if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
                    if ($this->import_mapId['pages'][$origPid]) {
                        $mappedPid = $this->import_mapId['pages'][$origPid];
                        $cmd_data['pages'][$mappedPid]['move'] = $newPid;
                    }
                } else {
                    $cmd_data['pages'][$origPid]['move'] = $newPid;
                }
            }
        }
        // Execute the move commands if any:
        if (!empty($cmd_data)) {
            $tce = $this->getNewTCE();
            $this->callHook('before_writeRecordsPagesOrder', [
                'tce' => &$tce,
                'data' => &$cmd_data
            ]);
            $tce->start([], $cmd_data);
            $tce->process_cmdmap();
            $this->callHook('after_writeRecordsPagesOrder', [
                'tce' => &$tce
            ]);
        }
    }

    public function flatInversePageTree_pid($idH, $a = [], $pid = -1)
    {
        if (is_array($idH)) {
            $idH = array_reverse($idH);
            foreach ($idH as $v) {
                $a[$v['uid']] = $pid;
                if (is_array($v['subrow'])) {
                    $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
                }
            }
        }
        return $a;
    }

    public function writeRecords_records($pid)
    {
        // Write the rest of the records
        $this->import_data = [];
        if (is_array($this->dat['header']['records'])) {
            foreach ($this->dat['header']['records'] as $table => $recs) {
                $this->addGeneralErrorsByTable($table);
                if ($table !== 'pages') {
                    foreach ($recs as $uid => $thisRec) {
                        // PID: Set the main $pid, unless a NEW-id is found
                        $setPid = isset($this->import_mapId['pages'][$thisRec['pid']])
                            ? (int)$this->import_mapId['pages'][$thisRec['pid']]
                            : (int)$pid;
                        if (is_array($GLOBALS['TCA'][$table]) && isset($GLOBALS['TCA'][$table]['ctrl']['rootLevel'])) {
                            $rootLevelSetting = (int)$GLOBALS['TCA'][$table]['ctrl']['rootLevel'];
                            if ($rootLevelSetting === 1) {
                                $setPid = 0;
                            } elseif ($rootLevelSetting === 0 && $setPid === 0) {
                                $this->error('Error: Record type ' . $table . ' is not allowed on pid 0');
                                continue;
                            }
                        }
                        // Add record:
                        $this->addSingle($table, $uid, $setPid);
                    }
                }
            }
        } else {
            $this->error('Error: No records defined in internal data array.');
        }
        // Now write to database:
        $tce = $this->getNewTCE();
        $this->callHook('before_writeRecordsRecords', [
            'tce' => &$tce,
            'data' => &$this->import_data
        ]);
        $tce->suggestedInsertUids = $this->suggestedInsertUids;
        // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
        $tce->reverseOrder = 1;
        $tce->isImporting = true;
        $tce->start($this->import_data, []);
        $tce->process_datamap();
        $this->callHook('after_writeRecordsRecords', [
            'tce' => &$tce
        ]);
        // post-processing: Removing files and registering new ids (end all DataHandler sessions with this)
        $this->addToMapId($tce->substNEWwithIDs);
        // In case of an update, order pages from the page tree correctly:
        if ($this->update) {
            $this->writeRecords_records_order($pid);
        }
    }

    public function writeRecords_records_order($mainPid)
    {
        $cmd_data = [];
        if (is_array($this->dat['header']['pagetree'])) {
            $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
        } else {
            $pagesFromTree = [];
        }
        if (is_array($this->dat['header']['pid_lookup'])) {
            foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
                $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
                if (MathUtility::canBeInterpretedAsInteger($newPid)) {
                    foreach ($recList as $tableName => $uidList) {
                        // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
                        // (they will not be in the page tree!)
                        if (($tableName !== 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
                            $uidList = array_reverse(array_keys($uidList));
                            foreach ($uidList as $uid) {
                                if ($this->dontIgnorePid($tableName, $uid)) {
                                    $cmd_data[$tableName][$uid]['move'] = $newPid;
                                }
                            }
                        }
                    }
                }
            }
        }
        // Execute the move commands if any:
        if (!empty($cmd_data)) {
            $tce = $this->getNewTCE();
            $this->callHook('before_writeRecordsRecordsOrder', [
                'tce' => &$tce,
                'data' => &$cmd_data
            ]);
            $tce->start([], $cmd_data);
            $tce->process_cmdmap();
            $this->callHook('after_writeRecordsRecordsOrder', [
                'tce' => &$tce
            ]);
        }
    }

    public function addSingle($table, $uid, $pid)
    {
        if ($this->import_mode[$table . ':' . $uid] === 'exclude') {
            return;
        }
        $record = $this->dat['records'][$table . ':' . $uid]['data'];
        if (is_array($record)) {
            if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
                $ID = $uid;
            } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
                // on adding sys_file records the belonging sys_file_metadata record was also created
                // if there is one the record need to be overwritten instead of creating a new one.
                $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
                    ->getQueryBuilderForTable('sys_file_metadata');
                $recordInDatabase = $queryBuilder->select('uid')
                    ->from('sys_file_metadata')
                    ->where(
                        $queryBuilder->expr()->eq(
                            'file',
                            $queryBuilder->createNamedParameter(
                                $this->import_mapId['sys_file'][$record['file']],
                                \PDO::PARAM_INT
                            )
                        ),
                        $queryBuilder->expr()->eq(
                            'sys_language_uid',
                            $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)
                        ),
                        $queryBuilder->expr()->eq(
                            'pid',
                            $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)
                        )
                    )
                    ->execute()
                    ->fetch();
                // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
                // to a file, that was already there, thus a new metadata record should be created
                if (is_array($recordInDatabase)) {
                    $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
                    $ID = $recordInDatabase['uid'];
                } else {
                    $ID = StringUtility::getUniqueId('NEW');
                }
            } else {
                $ID = StringUtility::getUniqueId('NEW');
            }
            $this->import_newId[$table . ':' . $ID] = ['table' => $table, 'uid' => $uid];
            if ($table === 'pages') {
                $this->import_newId_pids[$uid] = $ID;
            }
            // Set main record data:
            $this->import_data[$table][$ID] = $record;
            $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
            // Reset permission data:
            if ($table === 'pages') {
                // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
                unset($this->import_data[$table][$ID]['perms_userid']);
                unset($this->import_data[$table][$ID]['perms_groupid']);
            }
            // PID and UID:
            unset($this->import_data[$table][$ID]['uid']);
            // Updates:
            if (MathUtility::canBeInterpretedAsInteger($ID)) {
                unset($this->import_data[$table][$ID]['pid']);
            } else {
                // Inserts:
                $this->import_data[$table][$ID]['pid'] = $pid;
                if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $this->getBackendUser()->isAdmin()) {
                    $this->import_data[$table][$ID]['uid'] = $uid;
                    $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
                }
            }
            // Setting db/file blank:
            foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
                switch ((string)$config['type']) {
                    case 'db':

                    case 'file':
                        // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
                        // In the meantime we set NO values for relations.
                        //
                        // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
                        // because the value is already the uid of the right imported sys_file record.
                        // @see fixUidLocalInSysFileReferenceRecords()
                        // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
                        // delete the reference record if the file extension of the related record doesn't match.
                        if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
                            $this->import_data[$table][$ID][$field] = '';
                        }
                        break;
                    case 'flex':
                        // Fixed later in setFlexFormRelations()
                        // In the meantime we set NO value for flexforms - this is mainly because file references
                        // inside will not be processed properly; In fact references will point to no file
                        // or existing files (in which case there will be double-references which is a big problem of course!)
                        $this->import_data[$table][$ID][$field] = '';
                        break;
                }
            }
        } elseif ($table . ':' . $uid != 'pages:0') {
            // On root level we don't want this error message.
            $this->error('Error: no record was found in data array!');
        }
    }

    public function addToMapId($substNEWwithIDs)
    {
        foreach ($this->import_data as $table => $recs) {
            foreach ($recs as $id => $value) {
                $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
                if (isset($substNEWwithIDs[$id])) {
                    $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
                } elseif ($this->update) {
                    // Map same ID to same ID....
                    $this->import_mapId[$table][$old_uid] = $id;
                } else {
                    // if $this->import_mapId contains already the right mapping, skip the error msg.
                    // See special handling of sys_file_metadata in addSingle() => nothing to do
                    if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {
                        $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!');
                    }
                }
            }
        }
    }

    public function getNewTCE()
    {
        $tce = GeneralUtility::makeInstance(DataHandler::class);
        $tce->dontProcessTransformations = 1;
        $tce->enableLogging = $this->enableLogging;
        $tce->alternativeFileName = $this->alternativeFileName;
        $tce->alternativeFilePath = $this->alternativeFilePath;
        return $tce;
    }

    public function unlinkTempFiles()
    {
        foreach ($this->unlinkFiles as $fileName) {
            if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
                GeneralUtility::unlink_tempfile($fileName);
                clearstatcache();
                if (is_file($fileName)) {
                    $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!');
                }
            } else {
                $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!');
            }
        }
        $this->unlinkFiles = [];
    }

    /***************************
     * Import / Relations setting
     ***************************/

    public function setRelations()
    {
        $updateData = [];
        // import_newId contains a register of all records that was in the import memorys "records" key
        foreach ($this->import_newId as $nId => $dat) {
            $table = $dat['table'];
            $uid = $dat['uid'];
            // original UID - NOT the new one!
            // If the record has been written and received a new id, then proceed:
            if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
                $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
                if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
                    // Traverse relation fields of each record
                    foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
                        // uid_local of sys_file_reference needs no update because the correct reference uid was already written
                        // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
                        if ($table === 'sys_file_reference' && $field === 'uid_local') {
                            continue;
                        }
                        switch ((string)$config['type']) {
                            case 'db':
                                if (is_array($config['itemArray']) && !empty($config['itemArray'])) {
                                    $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
                                    $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
                                    $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
                                }
                                break;
                            case 'file':
                                if (is_array($config['newValueFiles']) && !empty($config['newValueFiles'])) {
                                    $valArr = [];
                                    foreach ($config['newValueFiles'] as $fI) {
                                        $valArr[] = $this->import_addFileNameToBeCopied($fI);
                                    }
                                    $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
                                }
                                break;
                        }
                    }
                } else {
                    $this->error('Error: no record was found in data array!');
                }
            } else {
                $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')');
            }
        }
        if (!empty($updateData)) {
            $tce = $this->getNewTCE();
            $tce->isImporting = true;
            $this->callHook('before_setRelation', [
                'tce' => &$tce,
                'data' => &$updateData
            ]);
            $tce->start($updateData, []);
            $tce->process_datamap();
            $this->callHook('after_setRelations', [
                'tce' => &$tce
            ]);
        }
    }

    public function setRelations_db($itemArray, $itemConfig)
    {
        $valArray = [];
        foreach ($itemArray as $relDat) {
            if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
                // Since non FAL file relation type group internal_type file_reference are handled as reference to
                // sys_file records Datahandler requires the value as uid of the the related sys_file record only
                if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
                    $value = $this->import_mapId[$relDat['table']][$relDat['id']];
                } elseif ($itemConfig['type'] === 'input' && isset($itemConfig['wizards']['link'])) {
                    // If an input field has a relation to a sys_file record this need to be converted back to
                    // the public path. But use getPublicUrl here, because could normally only be a local file path.
                    $fileUid = $this->import_mapId[$relDat['table']][$relDat['id']];
                    // Fallback value
                    $value = 'file:' . $fileUid;
                    try {
                        $file = ResourceFactory::getInstance()->retrieveFileOrFolderObject($fileUid);
                    } catch (\Exception $e) {
                        $file = null;
                    }
                    if ($file instanceof FileInterface) {
                        $value = $file->getPublicUrl();
                    }
                } else {
                    $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
                }
                $valArray[] = $value;
            } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
                // Checking for less than zero because some select types could contain negative values,
                // eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
                $valArray[] = $relDat['table'] . '_' . $relDat['id'];
            } else {
                $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id']);
            }
        }
        return $valArray;
    }

    public function import_addFileNameToBeCopied($fI)
    {
        if (is_array($this->dat['files'][$fI['ID']])) {
            $tmpFile = null;
            // check if there is the right file already in the local folder
            if ($this->filesPathForImport !== null) {
                if (is_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) &&
                    md5_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) === $this->dat['files'][$fI['ID']]['content_md5']) {
                    $tmpFile = $this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5'];
                }
            }
            if ($tmpFile === null) {
                $tmpFile = GeneralUtility::tempnam('import_temp_');
                GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
            }
            clearstatcache();
            if (@is_file($tmpFile)) {
                $this->unlinkFiles[] = $tmpFile;
                if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
                    $this->alternativeFileName[$tmpFile] = $fI['filename'];
                    $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
                    return $tmpFile;
                }
                $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')');
            } else {
                $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!');
            }
        } else {
            $this->error('Error: No file found for ID ' . $fI['ID']);
        }
        return null;
    }

    public function setFlexFormRelations()
    {
        $updateData = [];
        // import_newId contains a register of all records that was in the import memorys "records" key
        foreach ($this->import_newId as $nId => $dat) {
            $table = $dat['table'];
            $uid = $dat['uid'];
            // original UID - NOT the new one!
            // If the record has been written and received a new id, then proceed:
            if (!isset($this->import_mapId[$table][$uid])) {
                $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')');
                continue;
            }

            if (!is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
                $this->error('Error: no record was found in data array!');
                continue;
            }
            $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
            // Traverse relation fields of each record
            foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
                switch ((string)$config['type']) {
                    case 'flex':
                        // Get XML content and set as default value (string, non-processed):
                        $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
                        // If there has been registered relations inside the flex form field, run processing on the content:
                        if (!empty($config['flexFormRels']['db']) || !empty($config['flexFormRels']['file'])) {
                            $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
                            // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
                            $fieldTca = $GLOBALS['TCA'][$table]['columns'][$field];
                            if (is_array($origRecordRow) && is_array($fieldTca['config']) && $fieldTca['config']['type'] === 'flex') {
                                // Get current data structure and value array:
                                $flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
                                $dataStructureIdentifier = $flexFormTools->getDataStructureIdentifier(
                                    $fieldTca,
                                    $table,
                                    $field,
                                    $origRecordRow
                                );
                                $dataStructureArray = $flexFormTools->parseDataStructureByIdentifier($dataStructureIdentifier);
                                $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
                                // Do recursive processing of the XML data:
                                $iteratorObj = GeneralUtility::makeInstance(DataHandler::class);
                                $iteratorObj->callBackObj = $this;
                                $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData(
                                    $currentValueArray['data'],
                                    [],
                                    [],
                                    $dataStructureArray,
                                    [$table, $thisNewUid, $field, $config],
                                    'remapListedDBRecords_flexFormCallBack'
                                );
                                // The return value is set as an array which means it will be processed by DataHandler for file and DB references!
                                if (is_array($currentValueArray['data'])) {
                                    $updateData[$table][$thisNewUid][$field] = $currentValueArray;
                                }
                            }
                        }
                        break;
                }
            }
        }
        if (!empty($updateData)) {
            $tce = $this->getNewTCE();
            $tce->isImporting = true;
            $this->callHook('before_setFlexFormRelations', [
                'tce' => &$tce,
                'data' => &$updateData
            ]);
            $tce->start($updateData, []);
            $tce->process_datamap();
            $this->callHook('after_setFlexFormRelations', [
                'tce' => &$tce
            ]);
        }
    }

    public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
    {
        // Extract parameters:
        list(, , , $config) = $pParams;
        // In case the $path is used as index without a trailing slash we will remove that
        if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
            $path = rtrim($path, '/');
        }
        if (is_array($config['flexFormRels']['db'][$path])) {
            $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
            $dataValue = implode(',', $valArray);
        }
        if (is_array($config['flexFormRels']['file'][$path])) {
            $valArr = [];
            foreach ($config['flexFormRels']['file'][$path] as $fI) {
                $valArr[] = $this->import_addFileNameToBeCopied($fI);
            }
            $dataValue = implode(',', $valArr);
        }
        return ['value' => $dataValue];
    }

    /**************************
     * Import / Soft References
     *************************/

    public function processSoftReferences()
    {
        // Initialize:
        $inData = [];
        // Traverse records:
        if (is_array($this->dat['header']['records'])) {
            foreach ($this->dat['header']['records'] as $table => $recs) {
                foreach ($recs as $uid => $thisRec) {
                    // If there are soft references defined, traverse those:
                    if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
                        // First traversal is to collect softref configuration and split them up based on fields.
                        // This could probably also have been done with the "records" key instead of the header.
                        $fieldsIndex = [];
                        foreach ($thisRec['softrefs'] as $softrefDef) {
                            // If a substitution token is set:
                            if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
                                $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
                            }
                        }
                        // The new id:
                        $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
                        // Now, if there are any fields that require substitution to be done, lets go for that:
                        foreach ($fieldsIndex as $field => $softRefCfgs) {
                            if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
                                if ($GLOBALS['TCA'][$table]['columns'][$field]['config']['type'] === 'flex') {
                                    // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
                                    $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
                                    if (is_array($origRecordRow)) {
                                        // Get current data structure and value array:
                                        $flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
                                        $dataStructureIdentifier = $flexFormTools->getDataStructureIdentifier(
                                            $GLOBALS['TCA'][$table]['columns'][$field],
                                            $table,
                                            $field,
                                            $origRecordRow
                                        );
                                        $dataStructureArray = $flexFormTools->parseDataStructureByIdentifier($dataStructureIdentifier);
                                        $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
                                        // Do recursive processing of the XML data:
                                        $iteratorObj = GeneralUtility::makeInstance(DataHandler::class);
                                        $iteratorObj->callBackObj = $this;
                                        $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], [], [], $dataStructureArray, [$table, $uid, $field, $softRefCfgs], 'processSoftReferences_flexFormCallBack');
                                        // The return value is set as an array which means it will be processed by DataHandler for file and DB references!
                                        if (is_array($currentValueArray['data'])) {
                                            $inData[$table][$thisNewUid][$field] = $currentValueArray;
                                        }
                                    }
                                } else {
                                    // Get tokenizedContent string and proceed only if that is not blank:
                                    $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
                                    if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
                                        $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
        // Now write to database:
        $tce = $this->getNewTCE();
        $tce->isImporting = true;
        $this->callHook('before_processSoftReferences', [
            'tce' => $tce,
            'data' => &$inData
        ]);
        $tce->enableLogging = true;
        $tce->start($inData, []);
        $tce->process_datamap();
        $this->callHook('after_processSoftReferences', [
            'tce' => $tce
        ]);
    }

    public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
    {
        // Extract parameters:
        list($table, $origUid, $field, $softRefCfgs) = $pParams;
        if (is_array($softRefCfgs)) {
            // First, find all soft reference configurations for this structure path (they are listed flat in the header):
            $thisSoftRefCfgList = [];
            foreach ($softRefCfgs as $sK => $sV) {
                if ($sV['structurePath'] === $path) {
                    $thisSoftRefCfgList[$sK] = $sV;
                }
            }
            // If any was found, do processing:
            if (!empty($thisSoftRefCfgList)) {
                // Get tokenizedContent string and proceed only if that is not blank:
                $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
                if (strlen($tokenizedContent)) {
                    $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
                }
            }
        }
        // Return
        return ['value' => $dataValue];
    }

    public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid)
    {
        // traverse each softref type for this field:
        foreach ($softRefCfgs as $cfg) {
            // Get token ID:
            $tokenID = $cfg['subst']['tokenID'];
            // Default is current token value:
            $insertValue = $cfg['subst']['tokenValue'];
            // Based on mode:
            switch ((string)$this->softrefCfg[$tokenID]['mode']) {
                case 'exclude':
                    // Exclude is a simple passthrough of the value
                    break;
                case 'editable':
                    // Editable always picks up the value from this input array:
                    $insertValue = $this->softrefInputValues[$tokenID];
                    break;
                default:
                    // Mapping IDs/creating files: Based on type, look up new value:
                    switch ((string)$cfg['subst']['type']) {
                        case 'file':
                            // Create / Overwrite file:
                            $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
                            break;
                        case 'db':
                        default:
                            // Trying to map database element if found in the mapID array:
                            list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
                            if (isset($this->import_mapId[$tempTable][$tempUid])) {
                                $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
                                // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
                                if ($tempTable === 'pages' && !MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
                                    $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
                                    if ($recWithUniqueValue['alias']) {
                                        $insertValue = $recWithUniqueValue['alias'];
                                    }
                                } elseif (strpos($cfg['subst']['tokenValue'], ':') !== false) {
                                    list($tokenKey) = explode(':', $cfg['subst']['tokenValue']);
                                    $insertValue = $tokenKey . ':' . $insertValue;
                                }
                            }
                    }
            }
            // Finally, swap the soft reference token in tokenized content with the insert value:
            $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
        }
        return $tokenizedContent;
    }

    public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid)
    {
        if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
            // Initialize; Get directory prefix for file and find possible RTE filename
            $dirPrefix = PathUtility::dirname($relFileName) . '/';
            $rteOrigName = $this->getRTEoriginalFilename(PathUtility::basename($relFileName));
            // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
            if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
                // RTE:
                // First, find unique RTE file name:
                if (@is_dir((PATH_site . $dirPrefix))) {
                    // From the "original" RTE filename, produce a new "original" destination filename which is unused.
                    // Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
                    $fileProcObj = $this->getFileProcObj();
                    $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
                    // Create copy file name:
                    $pI = pathinfo($relFileName);
                    $copyDestName = PathUtility::dirname($origDestName) . '/RTEmagicC_' . substr(PathUtility::basename($origDestName), 10) . '.' . $pI['extension'];
                    if (
                        !@is_file($copyDestName) && !@is_file($origDestName)
                        && $origDestName === GeneralUtility::getFileAbsFileName($origDestName)
                        && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)
                    ) {
                        if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
                            // Write the copy and original RTE file to the respective filenames:
                            $this->writeFileVerify($copyDestName, $cfg['file_ID'], true);
                            $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], true);
                            // Return the relative path of the copy file name:
                            return PathUtility::stripPathSitePrefix($copyDestName);
                        }
                        $this->error('ERROR: Could not find original file ID');
                    } else {
                        $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
                    }
                } else {
                    $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
                }
            } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
                // File in fileadmin/ folder:
                // Create file (and possible resources)
                $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
                if (strlen($newFileName)) {
                    $relFileName = $newFileName;
                } else {
                    $this->error('ERROR: No new file created for "' . $relFileName . '"');
                }
            } else {
                $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
            }
        } else {
            $this->error('ERROR: Could not find file ID in header.');
        }
        // Return (new) filename relative to PATH_site:
        return $relFileName;
    }

    public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid)
    {
        // If the fileID map contains an entry for this fileID then just return the relative filename of that entry;
        // we don't want to write another unique filename for this one!
        if (isset($this->fileIDMap[$fileID])) {
            return PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
        }
        // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
        $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
        if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
            $fileHeaderInfo = $this->dat['header']['files'][$fileID];
            $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
            // Create new name for file:
            // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".

            // Write main file:
            if ($updMode) {
                $newName = PATH_site . $dirPrefix . $fileName;
            } else {
                // Create unique filename:
                $fileProcObj = $this->getFileProcObj();
                $newName = $fileProcObj->getUniqueName($fileName, PATH_site . $dirPrefix);
            }
            if ($this->writeFileVerify($newName, $fileID)) {
                // If the resource was an HTML/CSS file with resources attached, we will write those as well!
                if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
                    $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
                    $tokenSubstituted = false;
                    $fileProcObj = $this->getFileProcObj();
                    if ($updMode) {
                        foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
                            if ($this->dat['files'][$res_fileID]['filename']) {
                                // Resolve original filename:
                                $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
                                $absResourceFileName = GeneralUtility::resolveBackPath(PATH_site . $origDirPrefix . $relResourceFileName);
                                $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
                                if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, PATH_site . $this->fileadminFolderName . '/')) {
                                    $destDir = PathUtility::stripPathSitePrefix(PathUtility::dirname($absResourceFileName) . '/');
                                    if ($this->verifyFolderAccess($destDir, true) && $this->checkOrCreateDir($destDir)) {
                                        $this->writeFileVerify($absResourceFileName, $res_fileID);
                                    } else {
                                        $this->error('ERROR: Could not create file in directory "' . $destDir . '"');
                                    }
                                } else {
                                    $this->error('ERROR: Could not resolve path for "' . $relResourceFileName . '"');
                                }
                                $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
                                $tokenSubstituted = true;
                            }
                        }
                    } else {
                        // Create the resouces directory name (filename without extension, suffixed "_FILES")
                        $resourceDir = PathUtility::dirname($newName) . '/' . preg_replace('/\\.[^.]*$/', '', PathUtility::basename($newName)) . '_FILES';
                        if (GeneralUtility::mkdir($resourceDir)) {
                            foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
                                if ($this->dat['files'][$res_fileID]['filename']) {
                                    $absResourceFileName = $fileProcObj->getUniqueName($this->dat['files'][$res_fileID]['filename'], $resourceDir);
                                    $relResourceFileName = substr($absResourceFileName, strlen(PathUtility::dirname($resourceDir)) + 1);
                                    $this->writeFileVerify($absResourceFileName, $res_fileID);
                                    $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
                                    $tokenSubstituted = true;
                                }
                            }
                        }
                    }
                    // If substitutions has been made, write the content to the file again:
                    if ($tokenSubstituted) {
                        GeneralUtility::writeFile($newName, $tokenizedContent);
                    }
                }
                return PathUtility::stripPathSitePrefix($newName);
            }
        }
        return null;
    }

    public function writeFileVerify($fileName, $fileID, $bypassMountCheck = false)
    {
        $fileProcObj = $this->getFileProcObj();
        if (!$fileProcObj->actionPerms['addFile']) {
            $this->error('ERROR: You did not have sufficient permissions to write the file "' . $fileName . '"');
            return false;
        }
        // Just for security, check again. Should actually not be necessary.
        if (!$bypassMountCheck) {
            try {
                ResourceFactory::getInstance()->getFolderObjectFromCombinedIdentifier(dirname($fileName));
            } catch (InsufficientFolderAccessPermissionsException $e) {
                $this->error('ERROR: Filename "' . $fileName . '" was not allowed in destination path!');
                return false;
            }
        }
        $fI = GeneralUtility::split_fileref($fileName);
        if (!$fileProcObj->checkIfAllowed($fI['fileext'], $fI['path'], $fI['file']) && (!$this->allowPHPScripts || !$this->getBackendUser()->isAdmin())) {
            $this->error('ERROR: Filename "' . $fileName . '" failed against extension check or deny-pattern!');
            return false;
        }
        if (!GeneralUtility::getFileAbsFileName($fileName)) {
            $this->error('ERROR: Filename "' . $fileName . '" was not a valid relative file path!');
            return false;
        }
        if (!$this->dat['files'][$fileID]) {
            $this->error('ERROR: File ID "' . $fileID . '" could not be found');
            return false;
        }
        GeneralUtility::writeFile($fileName, $this->dat['files'][$fileID]['content']);
        $this->fileIDMap[$fileID] = $fileName;
        if (md5(file_get_contents($fileName)) == $this->dat['files'][$fileID]['content_md5']) {
            return true;
        }
        $this->error('ERROR: File content "' . $fileName . '" was corrupted');
        return false;
    }

    public function checkOrCreateDir($dirPrefix)
    {
        // Split dir path and remove first directory (which should be "fileadmin")
        $filePathParts = explode('/', $dirPrefix);
        $firstDir = array_shift($filePathParts);
        if ($firstDir === $this->fileadminFolderName && GeneralUtility::getFileAbsFileName($dirPrefix)) {
            $pathAcc = '';
            foreach ($filePathParts as $dirname) {
                $pathAcc .= '/' . $dirname;
                if (strlen($dirname)) {
                    if (!@is_dir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
                        if (!GeneralUtility::mkdir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
                            $this->error('ERROR: Directory could not be created....B');
                            return false;
                        }
                    }
                } elseif ($dirPrefix === $this->fileadminFolderName . $pathAcc) {
                    return true;
                } else {
                    $this->error('ERROR: Directory could not be created....A');
                }
            }
        }
        return false;
    }

    /**************************
     * File Input
     *************************/

    public function loadFile($filename, $all = false)
    {
        if (!@is_file($filename)) {
            $this->error('Filename not found: ' . $filename);
            return false;
        }
        $fI = pathinfo($filename);
        if (@is_dir($filename . '.files')) {
            if (GeneralUtility::isAllowedAbsPath($filename . '.files')) {
                // copy the folder lowlevel to typo3temp, because the files would be deleted after import
                $temporaryFolderName = $this->getTemporaryFolderName();
                GeneralUtility::copyDirectory($filename . '.files', $temporaryFolderName);
                $this->filesPathForImport = $temporaryFolderName;
            } else {
                $this->error('External import files for the given import source is currently not supported.');
            }
        }
        if (strtolower($fI['extension']) === 'xml') {
            // XML:
            $xmlContent = file_get_contents($filename);
            if (strlen($xmlContent)) {
                $this->dat = GeneralUtility::xml2array($xmlContent, '', true);
                if (is_array($this->dat)) {
                    if ($this->dat['_DOCUMENT_TAG'] === 'T3RecordDocument' && is_array($this->dat['header']) && is_array($this->dat['records'])) {
                        $this->loadInit();
                        return true;
                    }
                    $this->error('XML file did not contain proper XML for TYPO3 Import');
                } else {
                    $this->error('XML could not be parsed: ' . $this->dat);
                }
            } else {
                $this->error('Error opening file: ' . $filename);
            }
        } else {
            // T3D
            if ($fd = fopen($filename, 'rb')) {
                $this->dat['header'] = $this->getNextFilePart($fd, 1, 'header');
                if ($all) {
                    $this->dat['records'] = $this->getNextFilePart($fd, 1, 'records');
                    $this->dat['files'] = $this->getNextFilePart($fd, 1, 'files');
                    $this->dat['files_fal'] = $this->getNextFilePart($fd, 1, 'files_fal');
                }
                $this->loadInit();
                return true;
            }
            $this->error('Error opening file: ' . $filename);

            fclose($fd);
        }
        return false;
    }

    public function getNextFilePart($fd, $unserialize = false, $name = '')
    {
        $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
        // Getting header data
        $initStr = fread($fd, $initStrLen);
        if (empty($initStr)) {
            $this->error('File does not contain data for "' . $name . '"');
            return null;
        }
        $initStrDat = explode(':', $initStr);
        if (strstr($initStrDat[0], 'Warning')) {
            $this->error('File read error: Warning message in file. (' . $initStr . fgets($fd) . ')');
            return null;
        }
        if ((string)$initStrDat[3] !== '') {
            $this->error('File read error: InitString had a wrong length. (' . $name . ')');
            return null;
        }
        $datString = fread($fd, (int)$initStrDat[2]);
        fread($fd, 1);
        if (md5($datString) === $initStrDat[0]) {
            if ($initStrDat[1]) {
                if ($this->compress) {
                    $datString = gzuncompress($datString);
                } else {
                    $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.');
                    return null;
                }
            }
            return $unserialize ? unserialize($datString, ['allowed_classes' => false]) : $datString;
        }
        $this->error('MD5 check failed (' . $name . ')');

        return null;
    }

    public function loadContent($filecontent)
    {
        $pointer = 0;
        $this->dat['header'] = $this->getNextContentPart($filecontent, $pointer, 1, 'header');
        $this->dat['records'] = $this->getNextContentPart($filecontent, $pointer, 1, 'records');
        $this->dat['files'] = $this->getNextContentPart($filecontent, $pointer, 1, 'files');
        $this->loadInit();
    }

    public function getNextContentPart($filecontent, &$pointer, $unserialize = false, $name = '')
    {
        $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
        // getting header data
        $initStr = substr($filecontent, $pointer, $initStrLen);
        $pointer += $initStrLen;
        $initStrDat = explode(':', $initStr);
        if ((string)$initStrDat[3] !== '') {
            $this->error('Content read error: InitString had a wrong length. (' . $name . ')');
            return null;
        }
        $datString = substr($filecontent, $pointer, (int)$initStrDat[2]);
        $pointer += (int)$initStrDat[2] + 1;
        if (md5($datString) === $initStrDat[0]) {
            if ($initStrDat[1]) {
                if ($this->compress) {
                    $datString = gzuncompress($datString);
                    return $unserialize ? unserialize($datString, ['allowed_classes' => false]) : $datString;
                }
                $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.');
            }
        } else {
            $this->error('MD5 check failed (' . $name . ')');
        }
        return null;
    }

    public function loadInit()
    {
        $this->relStaticTables = (array)$this->dat['header']['relStaticTables'];
        $this->excludeMap = (array)$this->dat['header']['excludeMap'];
        $this->softrefCfg = (array)$this->dat['header']['softrefCfg'];
    }
}