ImportExport.php

Go to the documentation of this file.

<?php
namespace TYPO3\CMS\Impexp;

use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\Exception;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\PathUtility;

class ImportExport {

    public $showStaticRelations = FALSE;

    public $fileadminFolderName = '';

    public $mode = '';

    public $update = FALSE;

    public $doesImport = FALSE;

    public $display_import_pid_record = '';

    public $suggestedInsertUids = array();

    public $import_mode = array();

    public $global_ignore_pid = FALSE;

    public $force_all_UIDS = FALSE;

    public $showDiff = FALSE;

    public $allowPHPScripts = FALSE;

    public $enableLogging = FALSE;

    public $softrefInputValues = array();

    public $fileIDMap = array();

    public $maxFileSize = 1000000;

    public $maxRecordSize = 1000000;

    public $maxExportSize = 10000000;

    public $relOnlyTables = array();

    public $relStaticTables = array();

    public $excludeMap = array();

    public $softrefCfg = array();

    public $extensionDependencies = array();

    public $dontCompress = 0;

    public $includeExtFileResources = 0;

    public $extFileResourceExtensions = 'html,htm,css';

    public $import_mapId = array();

    public $import_newId = array();

    public $import_newId_pids = array();

    public $import_data = array();

    public $errorLog = array();

    public $cache_getRecordPath = array();

    public $checkPID_cache = array();

    public $compress = 0;

    public $dat = array();

    public $fileProcObj = '';

    protected $recordTypesIncludeFields = array();

    protected $defaultRecordIncludeFields = array('uid', 'pid');

    protected $storageObjects = array();

    protected $legacyImport = FALSE;

    protected $legacyImportFolder = NULL;

    protected $legacyImportTargetPath = '_imported/';

    protected $legacyImportMigrationTables = array(
        'tt_content' => array(
            'image' => array(
                'titleTexts' => 'titleText',
                'description' => 'imagecaption',
                'links' => 'image_link',
                'alternativeTexts' => 'altText'
            ),
            'media' => array(
                'description' => 'imagecaption',
            )
        ),
        'pages' => array(
            'media' => array()
        ),
        'pages_language_overlay' => array(
            'media' => array()
        )
    );


    protected $legacyImportMigrationRecords = array();


    /**************************
     * Initialize
     *************************/

    public function init($dontCompress = 0, $mode = '') {
        $this->compress = function_exists('gzcompress');
        $this->dontCompress = $dontCompress;
        $this->mode = $mode;
        $this->fileadminFolderName = !empty($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir']) ? rtrim($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') : 'fileadmin';
    }

    /**************************
     * Export / Init + Meta Data
     *************************/

    public function setHeaderBasics() {
        // Initializing:
        if (is_array($this->softrefCfg)) {
            foreach ($this->softrefCfg as $key => $value) {
                if (!strlen($value['mode'])) {
                    unset($this->softrefCfg[$key]);
                }
            }
        }
        // Setting in header memory:
        // Version of file format
        $this->dat['header']['XMLversion'] = '1.0';
        // Initialize meta data array (to put it in top of file)
        $this->dat['header']['meta'] = array();
        // Add list of tables to consider static
        $this->dat['header']['relStaticTables'] = $this->relStaticTables;
        // The list of excluded records
        $this->dat['header']['excludeMap'] = $this->excludeMap;
        // Soft Reference mode for elements
        $this->dat['header']['softrefCfg'] = $this->softrefCfg;
        // List of extensions the import depends on.
        $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
    }

    public function setCharset($charset) {
        $this->dat['header']['charset'] = $charset;
    }

    public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email) {
        $this->dat['header']['meta'] = array(
            'title' => $title,
            'description' => $description,
            'notes' => $notes,
            'packager_username' => $packager_username,
            'packager_name' => $packager_name,
            'packager_email' => $packager_email,
            'TYPO3_version' => TYPO3_version,
            'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
        );
    }

    public function addThumbnail($imgFilepath) {
        if (@is_file($imgFilepath)) {
            $imgInfo = @getimagesize($imgFilepath);
            if (is_array($imgInfo)) {
                $fileContent = GeneralUtility::getUrl($imgFilepath);
                $this->dat['header']['thumbnail'] = array(
                    'imgInfo' => $imgInfo,
                    'content' => $fileContent,
                    'filesize' => strlen($fileContent),
                    'filemtime' => filemtime($imgFilepath),
                    'filename' => PathUtility::basename($imgFilepath)
                );
            }
        }
    }

    /**************************
     * Export / Init Page tree
     *************************/

    public function setPageTree($idH) {
        $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
        return $this->flatInversePageTree($this->dat['header']['pagetree']);
    }

    public function unsetExcludedSections($idH) {
        if (is_array($idH)) {
            foreach ($idH as $k => $v) {
                if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
                    unset($idH[$k]);
                } elseif (is_array($idH[$k]['subrow'])) {
                    $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
                }
            }
        }
        return $idH;
    }

    public function flatInversePageTree($idH, $a = array()) {
        if (is_array($idH)) {
            $idH = array_reverse($idH);
            foreach ($idH as $k => $v) {
                $a[$v['uid']] = $v['uid'];
                if (is_array($v['subrow'])) {
                    $a = $this->flatInversePageTree($v['subrow'], $a);
                }
            }
        }
        return $a;
    }

    public function flatInversePageTree_pid($idH, $a = array(), $pid = -1) {
        if (is_array($idH)) {
            $idH = array_reverse($idH);
            foreach ($idH as $k => $v) {
                $a[$v['uid']] = $pid;
                if (is_array($v['subrow'])) {
                    $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
                }
            }
        }
        return $a;
    }

    /**************************
     * Export
     *************************/

    public function setRecordTypesIncludeFields(array $recordTypesIncludeFields) {
        foreach ($recordTypesIncludeFields as $table => $fields) {
            if (!is_array($fields)) {
                throw new \TYPO3\CMS\Core\Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
            }
            $this->setRecordTypeIncludeFields($table, $fields);
        }
    }

    public function setRecordTypeIncludeFields($table, array $fields) {
        $this->recordTypesIncludeFields[$table] = $fields;
    }

    public function export_addRecord($table, $row, $relationLevel = 0) {
        BackendUtility::workspaceOL($table, $row);
        if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[($table . ':' . $row['uid'])]) {
            if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
                if (!isset($this->dat['records'][($table . ':' . $row['uid'])])) {
                    // Prepare header info:
                    $row = $this->filterRecordFields($table, $row);
                    $headerInfo = array();
                    $headerInfo['uid'] = $row['uid'];
                    $headerInfo['pid'] = $row['pid'];
                    $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
                    $headerInfo['size'] = strlen(serialize($row));
                    if ($relationLevel) {
                        $headerInfo['relationLevel'] = $relationLevel;
                    }
                    // If record content is not too large in size, set the header content and add the rest:
                    if ($headerInfo['size'] < $this->maxRecordSize) {
                        // Set the header summary:
                        $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
                        // Create entry in the PID lookup:
                        $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
                        // Initialize reference index object:
                        $refIndexObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Database\\ReferenceIndex');
                        // Yes to workspace overlays for exporting....
                        $refIndexObj->WSOL = TRUE;
                        $relations = $refIndexObj->getRelations($table, $row);
                        $relations = $this->fixFileIDsInRelations($relations);
                        $relations = $this->removeSoftrefsHavingTheSameDatabaseRelation($relations);
                        // Data:
                        $this->dat['records'][$table . ':' . $row['uid']] = array();
                        $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
                        $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;
                        $this->errorLog = array_merge($this->errorLog, $refIndexObj->errorLog);
                        // Merge error logs.
                        // Add information about the relations in the record in the header:
                        $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
                        // Add information about the softrefs to header:
                        $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
                    } else {
                        $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');
                    }
                } else {
                    $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
                }
            } else {
                $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
            }
        }
    }

    protected function fixFileIDsInRelations(array $relations) {
        foreach ($relations as $field => $relation) {
            if (isset($relation['type']) && $relation['type'] === 'file') {
                foreach ($relation['newValueFiles'] as $key => $fileRelationData) {
                    $absoluteFilePath = $fileRelationData['ID_absFile'];
                    if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
                        $relatedFilePath = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($absoluteFilePath);
                        $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);
                    }
                }
            }
            if ($relation['type'] == 'flex') {
                if (is_array($relation['flexFormRels']['file'])) {
                    foreach ($relation['flexFormRels']['file'] as $key => $subList) {
                        foreach ($subList as $subKey => $fileRelationData) {
                            $absoluteFilePath = $fileRelationData['ID_absFile'];
                            if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
                                $relatedFilePath = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($absoluteFilePath);
                                $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);
                            }
                        }
                    }
                }
            }
        }
        return $relations;
    }

    protected function removeSoftrefsHavingTheSameDatabaseRelation($relations) {
        $fixedRelations = array();
        foreach ($relations as $field => $relation) {
            $newRelation = $relation;
            if (isset($newRelation['type']) && $newRelation['type'] === 'db') {
                foreach ($newRelation['itemArray'] as $key => $dbRelationData) {
                    if ($dbRelationData['table'] === 'sys_file') {
                        if (isset($newRelation['softrefs']['keys']['typolink'])) {
                            foreach ($newRelation['softrefs']['keys']['typolink'] as $softrefKey => $softRefData) {
                                if ($softRefData['subst']['type'] === 'file') {
                                    $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->retrieveFileOrFolderObject($softRefData['subst']['relFileName']);
                                    if ($file instanceof \TYPO3\CMS\Core\Resource\File) {
                                        if ($file->getUid() == $dbRelationData['id']) {
                                            unset($newRelation['softrefs']['keys']['typolink'][$softrefKey]);
                                        }
                                    }
                                }
                            }
                            if (empty($newRelation['softrefs']['keys']['typolink'])) {
                                unset($newRelation['softrefs']);
                            }
                        }
                    }
                }
            }
            $fixedRelations[$field] = $newRelation;
        }
        return $fixedRelations;
    }


    public function export_addDBRelations($relationLevel = 0) {
        // Initialize:
        $addR = array();
        // Traverse all "rels" registered for "records"
        if (is_array($this->dat['records'])) {
            foreach ($this->dat['records'] as $k => $value) {
                if (is_array($this->dat['records'][$k])) {
                    foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
                        // For all DB types of relations:
                        if ($vR['type'] == 'db') {
                            foreach ($vR['itemArray'] as $fI) {
                                $this->export_addDBRelations_registerRelation($fI, $addR);
                            }
                        }
                        // For all flex/db types of relations:
                        if ($vR['type'] == 'flex') {
                            // DB relations in flex form fields:
                            if (is_array($vR['flexFormRels']['db'])) {
                                foreach ($vR['flexFormRels']['db'] as $subList) {
                                    foreach ($subList as $fI) {
                                        $this->export_addDBRelations_registerRelation($fI, $addR);
                                    }
                                }
                            }
                            // DB oriented soft references in flex form fields:
                            if (is_array($vR['flexFormRels']['softrefs'])) {
                                foreach ($vR['flexFormRels']['softrefs'] as $subList) {
                                    foreach ($subList['keys'] as $spKey => $elements) {
                                        foreach ($elements as $el) {
                                            if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
                                                list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
                                                $fI = array(
                                                    'table' => $tempTable,
                                                    'id' => $tempUid
                                                );
                                                $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
                                            }
                                        }
                                    }
                                }
                            }
                        }
                        // In any case, if there are soft refs:
                        if (is_array($vR['softrefs']['keys'])) {
                            foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
                                foreach ($elements as $el) {
                                    if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
                                        list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
                                        $fI = array(
                                            'table' => $tempTable,
                                            'id' => $tempUid
                                        );
                                        $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
                                    }
                                }
                            }
                        }
                    }
                }
            }
        } else {
            $this->error('There were no records available.');
        }
        // Now, if there were new records to add, do so:
        if (count($addR)) {
            foreach ($addR as $fI) {
                // Get and set record:
                $row = BackendUtility::getRecord($fI['table'], $fI['id']);
                if (is_array($row)) {
                    $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
                }
                // Set status message
                // Relation pointers always larger than zero except certain "select" types with negative values pointing to uids - but that is not supported here.
                if ($fI['id'] > 0) {
                    $rId = $fI['table'] . ':' . $fI['id'];
                    if (!isset($this->dat['records'][$rId])) {
                        $this->dat['records'][$rId] = 'NOT_FOUND';
                        $this->error('Relation record ' . $rId . ' was not found!');
                    }
                }
            }
        }
        // Return overview of relations found and added
        return $addR;
    }

    public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '') {
        $rId = $fI['table'] . ':' . $fI['id'];
        if (isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id']) && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])) {
            if (!isset($this->dat['records'][$rId])) {
                // Set this record to be included since it is not already.
                $addR[$rId] = $fI;
            }
        }
    }

    public function export_addFilesFromRelations() {
        // Traverse all "rels" registered for "records"
        if (is_array($this->dat['records'])) {
            foreach ($this->dat['records'] as $k => $value) {
                if (isset($this->dat['records'][$k]['rels']) && is_array($this->dat['records'][$k]['rels'])) {
                    foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
                        // For all file type relations:
                        if ($vR['type'] == 'file') {
                            foreach ($vR['newValueFiles'] as $key => $fI) {
                                $this->export_addFile($fI, $k, $fieldname);
                                // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
                                unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
                            }
                        }
                        // For all flex type relations:
                        if ($vR['type'] == 'flex') {
                            if (is_array($vR['flexFormRels']['file'])) {
                                foreach ($vR['flexFormRels']['file'] as $key => $subList) {
                                    foreach ($subList as $subKey => $fI) {
                                        $this->export_addFile($fI, $k, $fieldname);
                                        // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
                                        unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
                                    }
                                }
                            }
                            // DB oriented soft references in flex form fields:
                            if (is_array($vR['flexFormRels']['softrefs'])) {
                                foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
                                    foreach ($subList['keys'] as $spKey => $elements) {
                                        foreach ($elements as $subKey => $el) {
                                            if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
                                                // Create abs path and ID for file:
                                                $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
                                                $ID = md5($el['subst']['relFileName']);
                                                if ($ID_absFile) {
                                                    if (!$this->dat['files'][$ID]) {
                                                        $fI = array(
                                                            'filename' => PathUtility::basename($ID_absFile),
                                                            'ID_absFile' => $ID_absFile,
                                                            'ID' => $ID,
                                                            'relFileName' => $el['subst']['relFileName']
                                                        );
                                                        $this->export_addFile($fI, '_SOFTREF_');
                                                    }
                                                    $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                        // In any case, if there are soft refs:
                        if (is_array($vR['softrefs']['keys'])) {
                            foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
                                foreach ($elements as $subKey => $el) {
                                    if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
                                        // Create abs path and ID for file:
                                        $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
                                        $ID = md5($el['subst']['relFileName']);
                                        if ($ID_absFile) {
                                            if (!$this->dat['files'][$ID]) {
                                                $fI = array(
                                                    'filename' => PathUtility::basename($ID_absFile),
                                                    'ID_absFile' => $ID_absFile,
                                                    'ID' => $ID,
                                                    'relFileName' => $el['subst']['relFileName']
                                                );
                                                $this->export_addFile($fI, '_SOFTREF_');
                                            }
                                            $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
        } else {
            $this->error('There were no records available.');
        }
    }

    public function export_addFilesFromSysFilesRecords() {
        if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
            return;
        }
        foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
            $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
            $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->createFileObject($recordData);
            $this->export_addSysFile($file);
        }
    }

    public function export_addSysFile(\TYPO3\CMS\Core\Resource\File $file) {
        if ($file->getProperty('size') >= $this->maxFileSize) {
            $this->error('File ' . $file->getPublicUrl() . ' was larger (' . GeneralUtility::formatSize($file->getProperty('size')) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
            return;
        }
        try {
            $fileContent = $file->getContents();
        } catch (\Exception $e) {
            $this->error('Error when trying to add file ' . $file->getCombinedIdentifier() . ': ' . $e->getMessage());
            return;
        }
        $fileUid = $file->getUid();
        $fileInfo = $file->getStorage()->getFileInfo($file);
        // we sadly have to cast it to string here, because the size property is also returning a string
        $fileSize = (string)$fileInfo['size'];
        if ($fileSize !== $file->getProperty('size')) {
            $this->error('File size of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added with current size.');
            $this->dat['records']['sys_file:' . $fileUid]['data']['size'] = $fileSize;
        }
        $fileSha1 = $file->getStorage()->hashFile($file, 'sha1');
        if ($fileSha1 !== $file->getProperty('sha1')) {
            $this->error('File sha1 hash of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added on current sha1.');
            $this->dat['records']['sys_file:' . $fileUid]['data']['sha1'] = $fileSha1;
        }

        $fileRec = array();
        $fileRec['filesize'] = $fileSize;
        $fileRec['filename'] = $file->getProperty('name');
        $fileRec['filemtime'] = $file->getProperty('modification_date');

        // build unique id based on the storage and the file identifier
        $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));

        // Setting this data in the header
        $this->dat['header']['files_fal'][$fileId] = $fileRec;

        // ... and finally add the heavy stuff:
        $fileRec['content'] = $fileContent;
        $fileRec['content_sha1'] = $fileSha1;

        $this->dat['files_fal'][$fileId] = $fileRec;
    }


    public function export_addFile($fI, $recordRef = '', $fieldname = '') {
        if (@is_file($fI['ID_absFile'])) {
            if (filesize($fI['ID_absFile']) < $this->maxFileSize) {
                $fileInfo = stat($fI['ID_absFile']);
                $fileRec = array();
                $fileRec['filesize'] = $fileInfo['size'];
                $fileRec['filename'] = PathUtility::basename($fI['ID_absFile']);
                $fileRec['filemtime'] = $fileInfo['mtime'];
                //for internal type file_reference
                $fileRec['relFileRef'] = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($fI['ID_absFile']);
                if ($recordRef) {
                    $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
                }
                if ($fI['relFileName']) {
                    $fileRec['relFileName'] = $fI['relFileName'];
                }
                // Setting this data in the header
                $this->dat['header']['files'][$fI['ID']] = $fileRec;
                // ... and for the recordlisting, why not let us know WHICH relations there was...
                if ($recordRef && $recordRef !== '_SOFTREF_') {
                    $refParts = explode(':', $recordRef, 2);
                    if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
                        $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = array();
                    }
                    $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
                }
                // ... and finally add the heavy stuff:
                $fileRec['content'] = GeneralUtility::getUrl($fI['ID_absFile']);
                $fileRec['content_md5'] = md5($fileRec['content']);
                $this->dat['files'][$fI['ID']] = $fileRec;
                // For soft references, do further processing:
                if ($recordRef === '_SOFTREF_') {
                    // RTE files?
                    if ($RTEoriginal = $this->getRTEoriginalFilename(PathUtility::basename($fI['ID_absFile']))) {
                        $RTEoriginal_absPath = PathUtility::dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
                        if (@is_file($RTEoriginal_absPath)) {
                            $RTEoriginal_ID = md5($RTEoriginal_absPath);
                            $fileInfo = stat($RTEoriginal_absPath);
                            $fileRec = array();
                            $fileRec['filesize'] = $fileInfo['size'];
                            $fileRec['filename'] = PathUtility::basename($RTEoriginal_absPath);
                            $fileRec['filemtime'] = $fileInfo['mtime'];
                            $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
                            $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
                            // Setting this data in the header
                            $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
                            // ... and finally add the heavy stuff:
                            $fileRec['content'] = GeneralUtility::getUrl($RTEoriginal_absPath);
                            $fileRec['content_md5'] = md5($fileRec['content']);
                            $this->dat['files'][$RTEoriginal_ID] = $fileRec;
                        } else {
                            $this->error('RTE original file "' . \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
                        }
                    }
                    // Files with external media?
                    // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
                    $html_fI = pathinfo(PathUtility::basename($fI['ID_absFile']));
                    if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
                        $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
                        if (strtolower($html_fI['extension']) === 'css') {
                            $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
                        } else {
                            // html, htm:
                            $htmlParser = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Html\\HtmlParser');
                            $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], array(), $uniquePrefix));
                        }
                        $htmlResourceCaptured = FALSE;
                        foreach ($prefixedMedias as $k => $v) {
                            if ($k % 2) {
                                $EXTres_absPath = GeneralUtility::resolveBackPath(PathUtility::dirname($fI['ID_absFile']) . '/' . $v);
                                $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
                                if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
                                    $htmlResourceCaptured = TRUE;
                                    $EXTres_ID = md5($EXTres_absPath);
                                    $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
                                    $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
                                    // Add file to memory if it is not set already:
                                    if (!isset($this->dat['header']['files'][$EXTres_ID])) {
                                        $fileInfo = stat($EXTres_absPath);
                                        $fileRec = array();
                                        $fileRec['filesize'] = $fileInfo['size'];
                                        $fileRec['filename'] = PathUtility::basename($EXTres_absPath);
                                        $fileRec['filemtime'] = $fileInfo['mtime'];
                                        $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
                                        // Media relative to the HTML file.
                                        $fileRec['parentRelFileName'] = $v;
                                        // Setting this data in the header
                                        $this->dat['header']['files'][$EXTres_ID] = $fileRec;
                                        // ... and finally add the heavy stuff:
                                        $fileRec['content'] = GeneralUtility::getUrl($EXTres_absPath);
                                        $fileRec['content_md5'] = md5($fileRec['content']);
                                        $this->dat['files'][$EXTres_ID] = $fileRec;
                                    }
                                }
                            }
                        }
                        if ($htmlResourceCaptured) {
                            $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
                        }
                    }
                }
            } else {
                $this->error($fI['ID_absFile'] . ' was larger (' . GeneralUtility::formatSize(filesize($fI['ID_absFile'])) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
            }
        } else {
            $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
        }
    }

    public function flatDBrels($dbrels) {
        $list = array();
        foreach ($dbrels as $dat) {
            if ($dat['type'] == 'db') {
                foreach ($dat['itemArray'] as $i) {
                    $list[$i['table'] . ':' . $i['id']] = $i;
                }
            }
            if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['db'])) {
                foreach ($dat['flexFormRels']['db'] as $subList) {
                    foreach ($subList as $i) {
                        $list[$i['table'] . ':' . $i['id']] = $i;
                    }
                }
            }
        }
        return $list;
    }

    public function flatSoftRefs($dbrels) {
        $list = array();
        foreach ($dbrels as $field => $dat) {
            if (is_array($dat['softrefs']['keys'])) {
                foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
                    if (is_array($elements)) {
                        foreach ($elements as $subKey => $el) {
                            $lKey = $field . ':' . $spKey . ':' . $subKey;
                            $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey), $el);
                            // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!
                            if ($el['subst'] && $el['subst']['relFileName']) {
                                $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
                            }
                        }
                    }
                }
            }
            if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
                foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
                    if (is_array($subSoftrefs['keys'])) {
                        foreach ($subSoftrefs['keys'] as $spKey => $elements) {
                            foreach ($elements as $subKey => $el) {
                                $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
                                $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath), $el);
                                // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!
                                if ($el['subst'] && $el['subst']['relFileName']) {
                                    $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
                                }
                            }
                        }
                    }
                }
            }
        }
        return $list;
    }

    protected function filterRecordFields($table, array $row) {
        if (isset($this->recordTypesIncludeFields[$table])) {
            $includeFields = array_unique(array_merge(
                $this->recordTypesIncludeFields[$table],
                $this->defaultRecordIncludeFields
            ));
            $newRow = array();
            foreach ($row as $key => $value) {
                if (in_array($key, $includeFields)) {
                    $newRow[$key] = $value;
                }
            }
        } else {
            $newRow = $row;
        }
        return $newRow;
    }


    /**************************
     * File Output
     *************************/

    public function compileMemoryToFileContent($type = '') {
        if ($type == 'xml') {
            $out = $this->createXML();
        } else {
            $compress = $this->doOutputCompress();
            $out = '';
            // adding header:
            $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
            // adding records:
            $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
            // adding files:
            $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
            // adding files_fal:
            $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
        }
        return $out;
    }

    public function createXML() {
        // Options:
        $options = array(
            'alt_options' => array(
                '/header' => array(
                    'disableTypeAttrib' => TRUE,
                    'clearStackPath' => TRUE,
                    'parentTagMap' => array(
                        'files' => 'file',
                        'files_fal' => 'file',
                        'records' => 'table',
                        'table' => 'rec',
                        'rec:rels' => 'relations',
                        'relations' => 'element',
                        'filerefs' => 'file',
                        'pid_lookup' => 'page_contents',
                        'header:relStaticTables' => 'static_tables',
                        'static_tables' => 'tablename',
                        'excludeMap' => 'item',
                        'softrefCfg' => 'softrefExportMode',
                        'extensionDependencies' => 'extkey',
                        'softrefs' => 'softref_element'
                    ),
                    'alt_options' => array(
                        '/pagetree' => array(
                            'disableTypeAttrib' => TRUE,
                            'useIndexTagForNum' => 'node',
                            'parentTagMap' => array(
                                'node:subrow' => 'node'
                            )
                        ),
                        '/pid_lookup/page_contents' => array(
                            'disableTypeAttrib' => TRUE,
                            'parentTagMap' => array(
                                'page_contents' => 'table'
                            ),
                            'grandParentTagMap' => array(
                                'page_contents/table' => 'item'
                            )
                        )
                    )
                ),
                '/records' => array(
                    'disableTypeAttrib' => TRUE,
                    'parentTagMap' => array(
                        'records' => 'tablerow',
                        'tablerow:data' => 'fieldlist',
                        'tablerow:rels' => 'related',
                        'related' => 'field',
                        'field:itemArray' => 'relations',
                        'field:newValueFiles' => 'filerefs',
                        'field:flexFormRels' => 'flexform',
                        'relations' => 'element',
                        'filerefs' => 'file',
                        'flexform:db' => 'db_relations',
                        'flexform:file' => 'file_relations',
                        'flexform:softrefs' => 'softref_relations',
                        'softref_relations' => 'structurePath',
                        'db_relations' => 'path',
                        'file_relations' => 'path',
                        'path' => 'element',
                        'keys' => 'softref_key',
                        'softref_key' => 'softref_element'
                    ),
                    'alt_options' => array(
                        '/records/tablerow/fieldlist' => array(
                            'useIndexTagForAssoc' => 'field'
                        )
                    )
                ),
                '/files' => array(
                    'disableTypeAttrib' => TRUE,
                    'parentTagMap' => array(
                        'files' => 'file'
                    )
                ),
                '/files_fal' => array(
                    'disableTypeAttrib' => TRUE,
                    'parentTagMap' => array(
                        'files_fal' => 'file'
                    )
                )
            )
        );
        // Creating XML file from $outputArray:
        $charset = $this->dat['header']['charset'] ?: 'utf-8';
        $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
        $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
        return $XML;
    }

    public function doOutputCompress() {
        return $this->compress && !$this->dontCompress;
    }

    public function addFilePart($data, $compress = FALSE) {
        if ($compress) {
            $data = gzcompress($data);
        }
        return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
    }

    /***********************
     * Import
     ***********************/

    protected function initializeImport() {
        // Set this flag to indicate that an import is being/has been done.
        $this->doesImport = 1;
        // Initialize:
        // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
        $this->import_mapId = array();
        $this->import_newId = array();
        $this->import_newId_pids = array();
        // Temporary files stack initialized:
        $this->unlinkFiles = array();
        $this->alternativeFileName = array();
        $this->alternativeFilePath = array();

        $this->initializeStorageObjects();
    }

    protected function initializeStorageObjects() {
        $storageRepository = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Resource\\StorageRepository');
        $this->storageObjects = $storageRepository->findAll();
    }


    public function importData($pid) {

        $this->initializeImport();

        // Write sys_file_storages first
        $this->writeSysFileStorageRecords();
        // Write sys_file records and write the binary file data
        $this->writeSysFileRecords();
        // Write records, first pages, then the rest
        // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
        $this->writeRecords_pages($pid);
        $this->writeRecords_records($pid);
        // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
        // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
        $this->setRelations();
        // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
        $this->setFlexFormRelations();
        // Unlink temporary files:
        $this->unlinkTempFiles();
        // Finally, traverse all records and process softreferences with substitution attributes.
        $this->processSoftReferences();
        // After all migrate records using sys_file_reference now
        if ($this->legacyImport) {
            $this->migrateLegacyImportRecords();
        }
    }

    protected function writeSysFileStorageRecords() {
        if (!isset($this->dat['header']['records']['sys_file_storage'])) {
            return;
        }
        $sysFileStorageUidsToBeResetToDefaultStorage = array();
        foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
            $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
            // continue with Local, writable and online storage only
            if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
                $useThisStorageUidInsteadOfTheOneInImport = 0;
                foreach ($this->storageObjects as $localStorage) {
                    // check the available storage for Local, writable and online ones
                    if ($localStorage->getDriverType() === 'Local' && $localStorage->isWritable() && $localStorage->isOnline()) {
                        // check if there is already a identical storage present (same pathType and basePath)
                        $storageRecordConfiguration = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
                        $localStorageRecordConfiguration = $localStorage->getConfiguration();
                        if ($storageRecordConfiguration['pathType'] == $localStorageRecordConfiguration['pathType'] && $storageRecordConfiguration['basePath'] == $localStorageRecordConfiguration['basePath']) {
                            // same storage is already present
                            $useThisStorageUidInsteadOfTheOneInImport = $localStorage->getUid();
                            break;
                        }
                    }
                }
                if ($useThisStorageUidInsteadOfTheOneInImport > 0) {
                    // same storage is already present; map the to be imported one to the present one
                    $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $useThisStorageUidInsteadOfTheOneInImport;
                } else {
                    // Local, writable and online storage. Is allowed to be used to later write files in.
                    $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
                }
            } else {
                // Storage with non Local drivers could be imported but must not be used to saves files in, because you
                // could not be sure, that this is supported. The default storage will be used in this case.
                // It could happen that non writable and non online storage will be created as dupes because you could not
                // check the detailed configuration options at this point
                $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
                $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
            }

        }

        // Importing the added ones
        $tce = $this->getNewTCE();
        // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
        $tce->reverseOrder = 1;
        $tce->isImporting = TRUE;
        $tce->start($this->import_data, array());
        $tce->process_datamap();
        $this->addToMapId($tce->substNEWwithIDs);

        $defaultStorageUid = NULL;
        // get default storage
        $defaultStorage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getDefaultStorage();
        if ($defaultStorage !== NULL) {
            $defaultStorageUid = $defaultStorage->getUid();
        }
        foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
            $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
        }

        // unset the sys_file_storage records to prevent a import in writeRecords_records
        unset($this->dat['header']['records']['sys_file_storage']);
    }

    protected function writeSysFileRecords() {
        if (!isset($this->dat['header']['records']['sys_file'])) {
            return;
        }
        $this->addGeneralErrorsByTable('sys_file');

        // fetch fresh storage records from database
        $storageRecords = $this->fetchStorageRecords();

        $defaultStorage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getDefaultStorage();

        $sanitizedFolderMappings = array();

        foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
            $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];

            // save file to disk
            $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
            $temporaryFile = $this->writeTemporaryFileFromData($fileId);
            if ($temporaryFile === NULL) {
                // error on writing the file. Error message was already added
                continue;
            }

            $originalStorageUid = $fileRecord['storage'];
            $useStorageFromStorageRecords = FALSE;

            // replace storage id, if a alternative one was registered
            if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
                $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
                $useStorageFromStorageRecords = TRUE;
            }

            if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {
                // no storage for the file is defined, mostly because of a missing default storage.
                $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
                continue;
            }

            // using a storage from the local storage is only allowed, if the uid is present in the
            // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
            if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
                $storage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
            } elseif ($this->isFallbackStorage($fileRecord['storage'])) {
                $storage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getStorageObject(0);
            } elseif ($defaultStorage !== NULL) {
                $storage = $defaultStorage;
            } else {
                $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
                continue;
            }

            $newFile = NULL;

            // check, if there is a identical file
            try {
                if ($storage->hasFile($fileRecord['identifier'])) {
                        $file = $storage->getFile($fileRecord['identifier']);
                    if ($file->getSha1() === $fileRecord['sha1']) {
                        $newFile = $file;
                    }
                }
            } catch (Exception $e) {}

            if ($newFile === NULL) {

                $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
                if (in_array($folderName, $sanitizedFolderMappings)) {
                    $folderName = $sanitizedFolderMappings[$folderName];
                }
                if (!$storage->hasFolder($folderName)) {
                    try {
                        $importFolder = $storage->createFolder($folderName);
                        if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
                            $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
                        }
                    } catch (Exception $e) {
                        $this->error('Error: Folder could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
                        continue;
                    }
                } else {
                    $importFolder = $storage->getFolder($folderName);
                }

                try {
                    $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
                } catch (Exception $e) {
                    $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
                    continue;
                }

                if ($newFile->getSha1() !== $fileRecord['sha1']) {
                    $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
                }
            }

            // save the new uid in the import id map
            $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
            $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());

        }

        // unset the sys_file records to prevent a import in writeRecords_records
        unset($this->dat['header']['records']['sys_file']);
        // remove all sys_file_reference records that point to file records which are unknown
        // in the system to prevent exceptions
        $this->removeSysFileReferenceRecordsFromImportDataWithRelationToMissingFile();
    }

    protected function removeSysFileReferenceRecordsFromImportDataWithRelationToMissingFile()
    {
        if (!isset($this->dat['header']['records']['sys_file_reference'])) {
            return;
        }

        foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
            $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
            if (!in_array($fileReferenceRecord['uid_local'], $this->import_mapId['sys_file'])) {
                unset($this->dat['header']['records']['sys_file_reference'][$sysFileReferenceUid]);
                unset($this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]);
                $this->error('Error: sys_file_reference record ' . (int)$sysFileReferenceUid
                             . ' with relation to sys_file record ' . (int)$fileReferenceRecord['uid_local']
                             . ', which is not part of the import data, was not imported.'
                );
            }
        }
    }

    protected function isFallbackStorage($storageId) {
        return $storageId === 0 || $storageId === '0';
    }

    protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid) {
        if (!isset($this->dat['header']['records']['sys_file_reference'])) {
            return;
        }

        foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
            $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
            if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
                $fileReferenceRecord['uid_local'] = $newFileUid;
                $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
            }
        }
    }

    protected function initializeLegacyImportFolder() {
        $folder = $GLOBALS['BE_USER']->getDefaultUploadFolder();
        if ($folder === FALSE) {
            $this->error('Error: the backend users default upload folder is missing! No files will be imported!');
        }
        if (!$folder->hasFolder($this->legacyImportTargetPath)) {
            try {
                $this->legacyImportFolder = $folder->createFolder($this->legacyImportTargetPath);
            } catch (\TYPO3\CMS\Core\Exception $e) {
                $this->error('Error: the import folder in the default upload folder could not be created! No files will be imported!');
            }
        } else {
            $this->legacyImportFolder = $folder->getSubfolder($this->legacyImportTargetPath);
        }

    }

    protected function fetchStorageRecords() {
        $whereClause = \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('sys_file_storage');
        $whereClause .= \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('sys_file_storage');

        $rows = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
            '*',
            'sys_file_storage',
            '1=1' . $whereClause,
            '',
            '',
            '',
            'uid'
        );

        return $rows;
    }

    protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal') {
        $temporaryFilePath = NULL;
        if (is_array($this->dat[$dataKey][$fileId])) {
            $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
            GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
            clearstatcache();
            if (@is_file($temporaryFilePathInternal)) {
                $this->unlinkFiles[] = $temporaryFilePathInternal;
                if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
                    $temporaryFilePath = $temporaryFilePathInternal;
                } else {
                    $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')', 1);
                }
            } else {
                $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!', 1);
            }
        } else {
            $this->error('Error: No file found for ID ' . $fileId, 1);
        }
        return $temporaryFilePath;
    }

    public function writeRecords_pages($pid) {
        // First, write page structure if any:
        if (is_array($this->dat['header']['records']['pages'])) {
            $this->addGeneralErrorsByTable('pages');
            // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
            $pageRecords = $this->dat['header']['records']['pages'];
            $this->import_data = array();
            // First add page tree if any
            if (is_array($this->dat['header']['pagetree'])) {
                $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
                foreach ($pagesFromTree as $uid) {
                    $thisRec = $this->dat['header']['records']['pages'][$uid];
                    // PID: Set the main $pid, unless a NEW-id is found
                    $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
                    $this->addSingle('pages', $uid, $setPid);
                    unset($pageRecords[$uid]);
                }
            }
            // Then add all remaining pages not in tree on root level:
            if (count($pageRecords)) {
                $remainingPageUids = array_keys($pageRecords);
                foreach ($remainingPageUids as $pUid) {
                    $this->addSingle('pages', $pUid, $pid);
                }
            }
            // Now write to database:
            $tce = $this->getNewTCE();
            $tce->isImporting = TRUE;
            $this->callHook('before_writeRecordsPages', array(
                'tce' => &$tce,
                'data' => &$this->import_data
            ));
            $tce->suggestedInsertUids = $this->suggestedInsertUids;
            $tce->start($this->import_data, array());
            $tce->process_datamap();
            $this->callHook('after_writeRecordsPages', array(
                'tce' => &$tce
            ));
            // post-processing: Registering new ids (end all tcemain sessions with this)
            $this->addToMapId($tce->substNEWwithIDs);
            // In case of an update, order pages from the page tree correctly:
            if ($this->update && is_array($this->dat['header']['pagetree'])) {
                $this->writeRecords_pages_order($pid);
            }
        }
    }

    public function writeRecords_pages_order($pid) {
        $cmd_data = array();
        // Get uid-pid relations and traverse them in order to map to possible new IDs
        $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
        foreach ($pidsFromTree as $origPid => $newPid) {
            if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
                // If the page had a new id (because it was created) use that instead!
                if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
                    if ($this->import_mapId['pages'][$origPid]) {
                        $mappedPid = $this->import_mapId['pages'][$origPid];
                        $cmd_data['pages'][$mappedPid]['move'] = $newPid;
                    }
                } else {
                    $cmd_data['pages'][$origPid]['move'] = $newPid;
                }
            }
        }
        // Execute the move commands if any:
        if (count($cmd_data)) {
            $tce = $this->getNewTCE();
            $this->callHook('before_writeRecordsPagesOrder', array(
                'tce' => &$tce,
                'data' => &$cmd_data
            ));
            $tce->start(array(), $cmd_data);
            $tce->process_cmdmap();
            $this->callHook('after_writeRecordsPagesOrder', array(
                'tce' => &$tce
            ));
        }
    }

    public function writeRecords_records($pid) {
        // Write the rest of the records
        $this->import_data = array();
        if (is_array($this->dat['header']['records'])) {
            foreach ($this->dat['header']['records'] as $table => $recs) {
                $this->addGeneralErrorsByTable($table);
                if ($table != 'pages') {
                    foreach ($recs as $uid => $thisRec) {
                        // PID: Set the main $pid, unless a NEW-id is found
                        $setPid = isset($this->import_mapId['pages'][$thisRec['pid']])
                            ? (int)$this->import_mapId['pages'][$thisRec['pid']]
                            : (int)$pid;
                        if (is_array($GLOBALS['TCA'][$table]) && isset($GLOBALS['TCA'][$table]['ctrl']['rootLevel'])) {
                            $rootLevelSetting = (int)$GLOBALS['TCA'][$table]['ctrl']['rootLevel'];
                            if ($rootLevelSetting === 1) {
                                $setPid = 0;
                            } elseif ($rootLevelSetting === 0 && $setPid === 0) {
                                $this->error('Error: Record type ' . $table . ' is not allowed on pid 0');
                                continue;
                            }
                        }
                        // Add record:
                        $this->addSingle($table, $uid, $setPid);
                    }
                }
            }
        } else {
            $this->error('Error: No records defined in internal data array.');
        }
        // Now write to database:
        $tce = $this->getNewTCE();
        $this->callHook('before_writeRecordsRecords', array(
            'tce' => &$tce,
            'data' => &$this->import_data
        ));
        $tce->suggestedInsertUids = $this->suggestedInsertUids;
        // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
        $tce->reverseOrder = 1;
        $tce->isImporting = TRUE;
        $tce->start($this->import_data, array());
        $tce->process_datamap();
        $this->callHook('after_writeRecordsRecords', array(
            'tce' => &$tce
        ));
        // post-processing: Removing files and registering new ids (end all tcemain sessions with this)
        $this->addToMapId($tce->substNEWwithIDs);
        // In case of an update, order pages from the page tree correctly:
        if ($this->update) {
            $this->writeRecords_records_order($pid);
        }
    }

    public function writeRecords_records_order($mainPid) {
        $cmd_data = array();
        if (is_array($this->dat['header']['pagetree'])) {
            $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
        } else {
            $pagesFromTree = array();
        }
        if (is_array($this->dat['header']['pid_lookup'])) {
            foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
                $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
                if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($newPid)) {
                    foreach ($recList as $tableName => $uidList) {
                        // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
                        // (they will not be in the page tree!)
                        if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
                            $uidList = array_reverse(array_keys($uidList));
                            foreach ($uidList as $uid) {
                                if ($this->dontIgnorePid($tableName, $uid)) {
                                    $cmd_data[$tableName][$uid]['move'] = $newPid;
                                } else {

                                }
                            }
                        }
                    }
                }
            }
        }
        // Execute the move commands if any:
        if (count($cmd_data)) {
            $tce = $this->getNewTCE();
            $this->callHook('before_writeRecordsRecordsOrder', array(
                'tce' => &$tce,
                'data' => &$cmd_data
            ));
            $tce->start(array(), $cmd_data);
            $tce->process_cmdmap();
            $this->callHook('after_writeRecordsRecordsOrder', array(
                'tce' => &$tce
            ));
        }
    }

    public function addSingle($table, $uid, $pid) {
        if ($this->import_mode[$table . ':' . $uid] !== 'exclude') {
            $record = $this->dat['records'][$table . ':' . $uid]['data'];
            if (is_array($record)) {
                if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
                    $ID = $uid;
                } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
                    // on adding sys_file records the belonging sys_file_metadata record was also created
                    // if there is one the record need to be overwritten instead of creating a new one.
                    $recordInDatabase = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow(
                        'uid',
                        'sys_file_metadata',
                        'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'
                    );
                    // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
                    // to a file, that was already there, thus a new metadata record should be created
                    if (is_array($recordInDatabase)) {
                        $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
                        $ID = $recordInDatabase['uid'];
                    } else {
                        $ID = uniqid('NEW', TRUE);
                    }

                } else {
                    $ID = uniqid('NEW', TRUE);
                }
                $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);
                if ($table == 'pages') {
                    $this->import_newId_pids[$uid] = $ID;
                }
                // Set main record data:
                $this->import_data[$table][$ID] = $record;
                $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
                // Reset permission data:
                if ($table === 'pages') {
                    // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
                    unset($this->import_data[$table][$ID]['perms_userid']);
                    unset($this->import_data[$table][$ID]['perms_groupid']);
                }
                // PID and UID:
                unset($this->import_data[$table][$ID]['uid']);
                // Updates:
                if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($ID)) {
                    unset($this->import_data[$table][$ID]['pid']);
                } else {
                    // Inserts:
                    $this->import_data[$table][$ID]['pid'] = $pid;
                    if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $GLOBALS['BE_USER']->isAdmin()) {
                        $this->import_data[$table][$ID]['uid'] = $uid;
                        $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
                    }
                }
                // Setting db/file blank:
                foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
                    switch ((string) $config['type']) {
                        case 'db':

                        case 'file':
                            // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
                            // In the meantime we set NO values for relations.
                            //
                            // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
                            // because the value is already the uid of the right imported sys_file record.
                            // @see fixUidLocalInSysFileReferenceRecords()
                            // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
                            // delete the reference record if the file extension of the related record doesn't match.
                            if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
                                $this->import_data[$table][$ID][$field] = '';
                            }
                            break;
                        case 'flex':
                            // Fixed later in setFlexFormRelations()
                            // In the meantime we set NO value for flexforms - this is mainly because file references inside will not be processed properly; In fact references will point to no file or existing files (in which case there will be double-references which is a big problem of course!)
                            $this->import_data[$table][$ID][$field] = '';
                            break;
                    }
                }
            } elseif ($table . ':' . $uid != 'pages:0') {
                // On root level we don't want this error message.
                $this->error('Error: no record was found in data array!', 1);
            }
        }
    }

    public function addToMapId($substNEWwithIDs) {
        foreach ($this->import_data as $table => $recs) {
            foreach ($recs as $id => $value) {
                $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
                if (isset($substNEWwithIDs[$id])) {
                    $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
                } elseif ($this->update) {
                    // Map same ID to same ID....
                    $this->import_mapId[$table][$old_uid] = $id;
                } else {
                    // if $this->import_mapId contains already the right mapping, skip the error msg. See special handling of sys_file_metadata in addSingle() => nothing to do
                    if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {
                        $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!', 1);
                    }

                }
            }
        }
    }

    public function getNewTCE() {
        $tce = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
        $tce->stripslashes_values = 0;
        $tce->dontProcessTransformations = 1;
        $tce->enableLogging = $this->enableLogging;
        $tce->alternativeFileName = $this->alternativeFileName;
        $tce->alternativeFilePath = $this->alternativeFilePath;
        return $tce;
    }

    public function unlinkTempFiles() {
        foreach ($this->unlinkFiles as $fileName) {
            if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
                GeneralUtility::unlink_tempfile($fileName);
                clearstatcache();
                if (is_file($fileName)) {
                    $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!', 1);
                }
            } else {
                $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!', 1);
            }
        }
        $this->unlinkFiles = array();
    }

    /***************************
     * Import / Relations setting
     ***************************/

    public function setRelations() {
        $updateData = array();
        // import_newId contains a register of all records that was in the import memorys "records" key
        foreach ($this->import_newId as $nId => $dat) {
            $table = $dat['table'];
            $uid = $dat['uid'];
            // original UID - NOT the new one!
            // If the record has been written and received a new id, then proceed:
            if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
                $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
                if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
                    if ($this->legacyImport) {
                        if ($table != 'pages') {
                            $oldPid = $this->dat['records'][$table . ':' . $uid]['data']['pid'];
                            $thisNewPageUid = \TYPO3\CMS\Backend\Utility\BackendUtility::wsMapId($table, $this->import_mapId['pages'][$oldPid]);
                        } else {
                            $thisNewPageUid = $thisNewUid;
                        }
                    }
                    // Traverse relation fields of each record
                    foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
                        // uid_local of sys_file_reference needs no update because the correct reference uid was already written
                        // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
                        if ($table === 'sys_file_reference' && $field === 'uid_local') {
                            continue;
                        }
                        switch ((string) $config['type']) {
                            case 'db':
                                if (is_array($config['itemArray']) && count($config['itemArray'])) {
                                    $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
                                    $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
                                    $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
                                }
                                break;
                            case 'file':
                                if (is_array($config['newValueFiles']) && count($config['newValueFiles'])) {
                                    $valArr = array();
                                    foreach ($config['newValueFiles'] as $fI) {
                                        $valArr[] = $this->import_addFileNameToBeCopied($fI);
                                    }
                                    if ($this->legacyImport && $this->legacyImportFolder === NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
                                        // Do nothing - the legacy import folder is missing
                                    } elseif ($this->legacyImport && $this->legacyImportFolder !== NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
                                        $refIds = array();
                                        foreach ($valArr as $tempFile) {
                                            $fileName = $this->alternativeFileName[$tempFile];
                                            $fileObject = NULL;

                                            try {
                                                // check, if there is alreay the same file in the folder
                                                if ($this->legacyImportFolder->hasFile($fileName)) {
                                                    $fileStorage = $this->legacyImportFolder->getStorage();
                                                    $file = $fileStorage->getFile($this->legacyImportFolder->getIdentifier() . $fileName);
                                                    if ($file->getSha1() === sha1_file($tempFile)) {
                                                        $fileObject = $file;
                                                    }
                                                }
                                            } catch (Exception $e) {}

                                            if ($fileObject === NULL) {
                                                try {
                                                    $fileObject = $this->legacyImportFolder->addFile($tempFile, $fileName, 'changeName');
                                                } catch (\TYPO3\CMS\Core\Exception $e) {
                                                    $this->error('Error: no file could be added to the storage for file name' . $this->alternativeFileName[$tempFile]);
                                                }
                                            }
                                            if ($fileObject !== NULL) {
                                                $refId = uniqid('NEW', TRUE);
                                                $refIds[] = $refId;
                                                $updateData['sys_file_reference'][$refId] = array(
                                                    'uid_local' => $fileObject->getUid(),
                                                    'uid_foreign' => $thisNewUid, // uid of your content record
                                                    'tablenames' => $table,
                                                    'fieldname' => $field,
                                                    'pid' => $thisNewPageUid, // parent id of the parent page
                                                    'table_local' => 'sys_file',
                                                );
                                            }
                                        }
                                        $updateData[$table][$thisNewUid][$field] = implode(',', $refIds);
                                        if (!empty($this->legacyImportMigrationTables[$table][$field])) {
                                            $this->legacyImportMigrationRecords[$table][$thisNewUid][$field] = $refIds;
                                        }
                                    } else {
                                        $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
                                    }
                                }
                                break;
                        }
                    }
                } else {
                    $this->error('Error: no record was found in data array!', 1);
                }
            } else {
                $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
            }
        }
        if (count($updateData)) {
            $tce = $this->getNewTCE();
            $tce->isImporting = TRUE;
            $this->callHook('before_setRelation', array(
                'tce' => &$tce,
                'data' => &$updateData
            ));
            $tce->start($updateData, array());
            $tce->process_datamap();
            // Replace the temporary "NEW" ids with the final ones.
            foreach ($this->legacyImportMigrationRecords as $table => $records) {
                foreach ($records as $uid => $fields) {
                    foreach ($fields as $field => $referenceIds) {
                        foreach ($referenceIds as $key => $referenceId) {
                            $this->legacyImportMigrationRecords[$table][$uid][$field][$key] = $tce->substNEWwithIDs[$referenceId];
                        }
                    }
                }
            }
            $this->callHook('after_setRelations', array(
                'tce' => &$tce
            ));
        }
    }

    public function setRelations_db($itemArray, $itemConfig) {
        $valArray = array();
        foreach ($itemArray as $relDat) {
            if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
                // Since non FAL file relation type group internal_type file_reference are handled as reference to
                // sys_file records Datahandler requires the value as uid of the the related sys_file record only
                if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
                    $value = $this->import_mapId[$relDat['table']][$relDat['id']];
                } elseif ($itemConfig['type'] === 'input' && isset($itemConfig['wizards']['link'])) {
                    // If an input field has a relation to a sys_file record this need to be converted back to
                    // the public path. But use getPublicUrl here, because could normally only be a local file path.
                    $fileUid = $this->import_mapId[$relDat['table']][$relDat['id']];
                    // Fallback value
                    $value = 'file:' . $fileUid;
                    try {
                        $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->retrieveFileOrFolderObject($fileUid);
                    } catch (\Exception $e) {}
                    if ($file instanceof \TYPO3\CMS\Core\Resource\FileInterface) {
                        $value = $file->getPublicUrl();
                    }
                } else {
                    $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
                }
                $valArray[] = $value;
            } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
                // Checking for less than zero because some select types could contain negative values, eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
                $valArray[] = $relDat['table'] . '_' . $relDat['id'];
            } else {
                $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id'], 1);
            }
        }
        return $valArray;
    }

    public function import_addFileNameToBeCopied($fI) {
        if (is_array($this->dat['files'][$fI['ID']])) {
            $tmpFile = GeneralUtility::tempnam('import_temp_');
            GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
            clearstatcache();
            if (@is_file($tmpFile)) {
                $this->unlinkFiles[] = $tmpFile;
                if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
                    $this->alternativeFileName[$tmpFile] = $fI['filename'];
                    $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
                    return $tmpFile;
                } else {
                    $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')', 1);
                }
            } else {
                $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!', 1);
            }
        } else {
            $this->error('Error: No file found for ID ' . $fI['ID'], 1);
        }
    }

    public function setFlexFormRelations() {
        $updateData = array();
        // import_newId contains a register of all records that was in the import memorys "records" key
        foreach ($this->import_newId as $nId => $dat) {
            $table = $dat['table'];
            $uid = $dat['uid'];
            // original UID - NOT the new one!
            // If the record has been written and received a new id, then proceed:
            if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
                $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
                if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
                    // Traverse relation fields of each record
                    foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
                        switch ((string) $config['type']) {
                            case 'flex':
                                // Get XML content and set as default value (string, non-processed):
                                $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
                                // If there has been registered relations inside the flex form field, run processing on the content:
                                if (count($config['flexFormRels']['db']) || count($config['flexFormRels']['file'])) {
                                    $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
                                    // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
                                    $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
                                    if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {
                                        // Get current data structure and value array:
                                        $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
                                        $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
                                        // Do recursive processing of the XML data:
                                        $iteratorObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
                                        $iteratorObj->callBackObj = $this;
                                        $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $thisNewUid, $field, $config), 'remapListedDBRecords_flexFormCallBack');
                                        // The return value is set as an array which means it will be processed by tcemain for file and DB references!
                                        if (is_array($currentValueArray['data'])) {
                                            $updateData[$table][$thisNewUid][$field] = $currentValueArray;
                                        }
                                    }
                                }
                                break;
                        }
                    }
                } else {
                    $this->error('Error: no record was found in data array!', 1);
                }
            } else {
                $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
            }
        }
        if (count($updateData)) {
            $tce = $this->getNewTCE();
            $tce->isImporting = TRUE;
            $this->callHook('before_setFlexFormRelations', array(
                'tce' => &$tce,
                'data' => &$updateData
            ));
            $tce->start($updateData, array());
            $tce->process_datamap();
            $this->callHook('after_setFlexFormRelations', array(
                'tce' => &$tce
            ));
        }
    }

    public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
        // Extract parameters:
        list($table, $uid, $field, $config) = $pParams;
        // In case the $path is used as index without a trailing slash we will remove that
        if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
            $path = rtrim($path, '/');
        }
        if (is_array($config['flexFormRels']['db'][$path])) {
            $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
            $dataValue = implode(',', $valArray);
        }
        if (is_array($config['flexFormRels']['file'][$path])) {
            foreach ($config['flexFormRels']['file'][$path] as $fI) {
                $valArr[] = $this->import_addFileNameToBeCopied($fI);
            }
            $dataValue = implode(',', $valArr);
        }
        return array('value' => $dataValue);
    }

    /**************************
     * Import / Soft References
     *************************/

    public function processSoftReferences() {
        // Initialize:
        $inData = array();
        // Traverse records:
        if (is_array($this->dat['header']['records'])) {
            foreach ($this->dat['header']['records'] as $table => $recs) {
                foreach ($recs as $uid => $thisRec) {
                    // If there are soft references defined, traverse those:
                    if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
                        // First traversal is to collect softref configuration and split them up based on fields. This could probably also have been done with the "records" key instead of the header.
                        $fieldsIndex = array();
                        foreach ($thisRec['softrefs'] as $softrefDef) {
                            // If a substitution token is set:
                            if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
                                $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
                            }
                        }
                        // The new id:
                        $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
                        // Now, if there are any fields that require substitution to be done, lets go for that:
                        foreach ($fieldsIndex as $field => $softRefCfgs) {
                            if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
                                $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
                                if ($conf['type'] === 'flex') {
                                    // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
                                    $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
                                    if (is_array($origRecordRow)) {
                                        // Get current data structure and value array:
                                        $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
                                        $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
                                        // Do recursive processing of the XML data:
                                        $iteratorObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
                                        $iteratorObj->callBackObj = $this;
                                        $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $uid, $field, $softRefCfgs), 'processSoftReferences_flexFormCallBack');
                                        // The return value is set as an array which means it will be processed by tcemain for file and DB references!
                                        if (is_array($currentValueArray['data'])) {
                                            $inData[$table][$thisNewUid][$field] = $currentValueArray;
                                        }
                                    }
                                } else {
                                    // Get tokenizedContent string and proceed only if that is not blank:
                                    $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
                                    if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
                                        $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
        // Now write to database:
        $tce = $this->getNewTCE();
        $tce->isImporting = TRUE;
        $this->callHook('before_processSoftReferences', array(
            'tce' => &$tce,
            'data' => &$inData
        ));
        $tce->enableLogging = TRUE;
        $tce->start($inData, array());
        $tce->process_datamap();
        $this->callHook('after_processSoftReferences', array(
            'tce' => &$tce
        ));
    }

    public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
        // Extract parameters:
        list($table, $origUid, $field, $softRefCfgs) = $pParams;
        if (is_array($softRefCfgs)) {
            // First, find all soft reference configurations for this structure path (they are listed flat in the header):
            $thisSoftRefCfgList = array();
            foreach ($softRefCfgs as $sK => $sV) {
                if ($sV['structurePath'] === $path) {
                    $thisSoftRefCfgList[$sK] = $sV;
                }
            }
            // If any was found, do processing:
            if (count($thisSoftRefCfgList)) {
                // Get tokenizedContent string and proceed only if that is not blank:
                $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
                if (strlen($tokenizedContent)) {
                    $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
                }
            }
        }
        // Return
        return array('value' => $dataValue);
    }

    public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid) {
        // traverse each softref type for this field:
        foreach ($softRefCfgs as $cfg) {
            // Get token ID:
            $tokenID = $cfg['subst']['tokenID'];
            // Default is current token value:
            $insertValue = $cfg['subst']['tokenValue'];
            // Based on mode:
            switch ((string) $this->softrefCfg[$tokenID]['mode']) {
                case 'exclude':
                    // Exclude is a simple passthrough of the value
                    break;
                case 'editable':
                    // Editable always picks up the value from this input array:
                    $insertValue = $this->softrefInputValues[$tokenID];
                    break;
                default:
                    // Mapping IDs/creating files: Based on type, look up new value:
                    switch ((string) $cfg['subst']['type']) {
                        case 'file':
                            // Create / Overwrite file:
                            $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
                            break;
                        case 'db':
                        default:
                            // Trying to map database element if found in the mapID array:
                            list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
                            if (isset($this->import_mapId[$tempTable][$tempUid])) {
                                $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
                                // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
                                if ($tempTable === 'pages' && !\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
                                    $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
                                    if ($recWithUniqueValue['alias']) {
                                        $insertValue = $recWithUniqueValue['alias'];
                                    }
                                } elseif (strpos($cfg['subst']['tokenValue'], ':') !== FALSE) {
                                    list($tokenKey, $tokenId) = explode(':', $cfg['subst']['tokenValue']);
                                    $insertValue = $tokenKey . ':' . $insertValue;
                                }
                            }
                    }
            }
            // Finally, swap the soft reference token in tokenized content with the insert value:
            $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
        }
        return $tokenizedContent;
    }

    public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid) {
        if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
            // Initialize; Get directory prefix for file and find possible RTE filename
            $dirPrefix = PathUtility::dirname($relFileName) . '/';
            $rteOrigName = $this->getRTEoriginalFilename(PathUtility::basename($relFileName));
            // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
            if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
                // RTE:
                // First, find unique RTE file name:
                if (@is_dir((PATH_site . $dirPrefix))) {
                    // From the "original" RTE filename, produce a new "original" destination filename which is unused. Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
                    $fileProcObj = $this->getFileProcObj();
                    $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
                    // Create copy file name:
                    $pI = pathinfo($relFileName);
                    $copyDestName = PathUtility::dirname($origDestName) . '/RTEmagicC_' . substr(PathUtility::basename($origDestName), 10) . '.' . $pI['extension'];
                    if (!@is_file($copyDestName) && !@is_file($origDestName) && $origDestName === GeneralUtility::getFileAbsFileName($origDestName) && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)) {
                        if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
                            if ($this->legacyImport) {
                                $fileName = PathUtility::basename($copyDestName);
                                $this->writeSysFileResourceForLegacyImport($fileName, $cfg['file_ID']);
                                $relFileName = $this->filePathMap[$cfg['file_ID']] . '" data-htmlarea-file-uid="' . $fileName . '" data-htmlarea-file-table="sys_file';
                                // Also save the original file
                                $originalFileName = PathUtility::basename($origDestName);
                                $this->writeSysFileResourceForLegacyImport($originalFileName, $fileHeaderInfo['RTE_ORIG_ID']);
                            } else {
                                // Write the copy and original RTE file to the respective filenames:
                                $this->writeFileVerify($copyDestName, $cfg['file_ID'], TRUE);
                                $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], TRUE);
                                // Return the relative path of the copy file name:
                                return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($copyDestName);
                            }
                        } else {
                            $this->error('ERROR: Could not find original file ID');
                        }
                    } else {
                        $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
                    }
                } else {
                    $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
                }
            } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
                // File in fileadmin/ folder:
                // Create file (and possible resources)
                $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
                if (strlen($newFileName)) {
                    $relFileName = $newFileName;
                } else {
                    $this->error('ERROR: No new file created for "' . $relFileName . '"');
                }
            } else {
                $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
            }
        } else {
            $this->error('ERROR: Could not find file ID in header.');
        }
        // Return (new) filename relative to PATH_site:
        return $relFileName;
    }

    public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid) {
        // If the fileID map contains an entry for this fileID then just return the relative filename of that entry; we don't want to write another unique filename for this one!
        if (isset($this->fileIDMap[$fileID])) {
            return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
        }
        if ($this->legacyImport) {
            // set dirPrefix to fileadmin because the right target folder is set and checked for permissions later
            $dirPrefix = $this->fileadminFolderName . '/';
        } else {
            // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
            $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
        }
        if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
            $fileHeaderInfo = $this->dat['header']['files'][$fileID];
            $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
            // Create new name for file:
            // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".

            // Write main file:
            if ($this->legacyImport) {
                $fileWritten = $this->writeSysFileResourceForLegacyImport($fileName, $fileID);
                if ($fileWritten) {
                    $newName = 'file:' . $fileName;
                    return $newName;
                    // no support for HTML/CSS file resources attached ATM - see below
                }
            } else {
                if ($updMode) {
                    $newName = PATH_site . $dirPrefix . $fileName;
                } else {
                    // Create unique filename:
                    $fileProcObj = $this->getFileProcObj();
                    $newName = $fileProcObj->getUniqueName($fileName, PATH_site . $dirPrefix);
                }
                if ($this->writeFileVerify($newName, $fileID)) {
                    // If the resource was an HTML/CSS file with resources attached, we will write those as well!
                    if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
                        $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
                        $tokenSubstituted = FALSE;
                        $fileProcObj = $this->getFileProcObj();
                        if ($updMode) {
                            foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
                                if ($this->dat['files'][$res_fileID]['filename']) {
                                    // Resolve original filename:
                                    $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
                                    $absResourceFileName = GeneralUtility::resolveBackPath(PATH_site . $origDirPrefix . $relResourceFileName);
                                    $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
                                    if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, PATH_site . $this->fileadminFolderName . '/')) {
                                        $destDir = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix(PathUtility::dirname($absResourceFileName) . '/');
                                        if ($this->verifyFolderAccess($destDir, TRUE) && $this->checkOrCreateDir($destDir)) {
                                            $this->writeFileVerify($absResourceFileName, $res_fileID);
                                        } else {
                                            $this->error('ERROR: Could not create file in directory "' . $destDir . '"');
                                        }
                                    } else {
                                        $this->error('ERROR: Could not resolve path for "' . $relResourceFileName . '"');
                                    }
                                    $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
                                    $tokenSubstituted = TRUE;
                                }
                            }
                        } else {
                            // Create the resouces directory name (filename without extension, suffixed "_FILES")
                            $resourceDir = PathUtility::dirname($newName) . '/' . preg_replace('/\\.[^.]*$/', '', PathUtility::basename($newName)) . '_FILES';
                            if (GeneralUtility::mkdir($resourceDir)) {
                                foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
                                    if ($this->dat['files'][$res_fileID]['filename']) {
                                        $absResourceFileName = $fileProcObj->getUniqueName($this->dat['files'][$res_fileID]['filename'], $resourceDir);
                                        $relResourceFileName = substr($absResourceFileName, strlen(PathUtility::dirname($resourceDir)) + 1);
                                        $this->writeFileVerify($absResourceFileName, $res_fileID);
                                        $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
                                        $tokenSubstituted = TRUE;
                                    }
                                }
                            }
                        }
                        // If substitutions has been made, write the content to the file again:
                        if ($tokenSubstituted) {
                            GeneralUtility::writeFile($newName, $tokenizedContent);
                        }
                    }
                    return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($newName);
                }
            }
        }
    }

    public function writeFileVerify($fileName, $fileID, $bypassMountCheck = FALSE) {
        $fileProcObj = $this->getFileProcObj();
        if ($fileProcObj->actionPerms['addFile']) {
            // Just for security, check again. Should actually not be necessary.
            if ($fileProcObj->checkPathAgainstMounts($fileName) || $bypassMountCheck) {
                $fI = GeneralUtility::split_fileref($fileName);
                if ($fileProcObj->checkIfAllowed($fI['fileext'], $fI['path'], $fI['file']) || $this->allowPHPScripts && $GLOBALS['BE_USER']->isAdmin()) {
                    if (GeneralUtility::getFileAbsFileName($fileName)) {
                        if ($this->dat['files'][$fileID]) {
                            GeneralUtility::writeFile($fileName, $this->dat['files'][$fileID]['content']);
                            $this->fileIDMap[$fileID] = $fileName;
                            if (md5(GeneralUtility::getUrl($fileName)) == $this->dat['files'][$fileID]['content_md5']) {
                                return TRUE;
                            } else {
                                $this->error('ERROR: File content "' . $fileName . '" was corrupted');
                            }
                        } else {
                            $this->error('ERROR: File ID "' . $fileID . '" could not be found');
                        }
                    } else {
                        $this->error('ERROR: Filename "' . $fileName . '" was not a valid relative file path!');
                    }
                } else {
                    $this->error('ERROR: Filename "' . $fileName . '" failed against extension check or deny-pattern!');
                }
            } else {
                $this->error('ERROR: Filename "' . $fileName . '" was not allowed in destination path!');
            }
        } else {
            $this->error('ERROR: You did not have sufficient permissions to write the file "' . $fileName . '"');
        }
    }


    protected function writeSysFileResourceForLegacyImport(&$fileName, $fileId) {
        if ($this->legacyImportFolder === NULL) {
            return FALSE;
        }

        if (!isset($this->dat['files'][$fileId])) {
            $this->error('ERROR: File ID "' . $fileId . '" could not be found');
            return FALSE;
        }

        $temporaryFile = $this->writeTemporaryFileFromData($fileId, 'files');
        if ($temporaryFile === NULL) {
            // error on writing the file. Error message was already added
            return FALSE;
        }

        $importFolder = $this->legacyImportFolder;

        if (isset($this->dat['files'][$fileId]['relFileName'])) {
            $relativeFilePath = PathUtility::dirname($this->dat['files'][$fileId]['relFileName']);

            if (!$this->legacyImportFolder->hasFolder($relativeFilePath)) {
                $this->legacyImportFolder->createFolder($relativeFilePath);
            }
            $importFolder = $this->legacyImportFolder->getSubfolder($relativeFilePath);
        }

        $fileObject = NULL;

        try {
            // check, if there is alreay the same file in the folder
            if ($importFolder->hasFile($fileName)) {
                $fileStorage = $importFolder->getStorage();
                $file = $fileStorage->getFile($importFolder->getIdentifier() . $fileName);
                if ($file->getSha1() === sha1_file($temporaryFile)) {
                    $fileObject = $file;
                }
            }
        } catch (Exception $e) {}

        if ($fileObject === NULL) {
            try {
                $fileObject = $importFolder->addFile($temporaryFile, $fileName, 'changeName');
            } catch (\TYPO3\CMS\Core\Exception $e) {
                $this->error('Error: no file could be added to the storage for file name ' . $this->alternativeFileName[$temporaryFile]);
            }
        }

        if (md5_file(PATH_site . $fileObject->getPublicUrl()) == $this->dat['files'][$fileId]['content_md5']) {
            $fileName = $fileObject->getUid();
            $this->fileIDMap[$fileId] = $fileName;
            $this->filePathMap[$fileId] = $fileObject->getPublicUrl();
            return TRUE;
        } else {
            $this->error('ERROR: File content "' . $this->dat['files'][$fileId]['relFileName'] . '" was corrupted');
        }

        return FALSE;
    }

    protected function migrateLegacyImportRecords() {
        $updateData= array();

        foreach ($this->legacyImportMigrationRecords as $table => $records) {
            foreach ($records as $uid => $fields) {

                $row = BackendUtility::getRecord($table, $uid);
                if (empty($row)) {
                    continue;
                }

                foreach ($fields as $field => $referenceIds) {

                    $fieldConfiguration = $this->legacyImportMigrationTables[$table][$field];

                    if (isset($fieldConfiguration['titleTexts'])) {
                        $titleTextField = $fieldConfiguration['titleTexts'];
                        if ($row[$titleTextField] !== '') {
                            $titleTextContents = explode(LF, $row[$titleTextField]);
                            $updateData[$table][$uid][$titleTextField] = '';
                        }
                    }

                    if (isset($fieldConfiguration['alternativeTexts'])) {
                        $alternativeTextField = $fieldConfiguration['alternativeTexts'];
                        if ($row[$alternativeTextField] !== '') {
                            $alternativeTextContents = explode(LF, $row[$alternativeTextField]);
                            $updateData[$table][$uid][$alternativeTextField] = '';
                        }
                    }
                    if (isset($fieldConfiguration['description'])) {
                        $descriptionField = $fieldConfiguration['description'];
                        if ($row[$descriptionField] !== '') {
                            $descriptionContents = explode(LF, $row[$descriptionField]);
                            $updateData[$table][$uid][$descriptionField] = '';
                        }
                    }
                    if (isset($fieldConfiguration['links'])) {
                        $linkField = $fieldConfiguration['links'];
                        if ($row[$linkField] !== '') {
                            $linkContents = explode(LF, $row[$linkField]);
                            $updateData[$table][$uid][$linkField] = '';
                        }
                    }

                    foreach ($referenceIds as $key => $referenceId) {
                        if (isset($titleTextContents[$key])) {
                            $updateData['sys_file_reference'][$referenceId]['title'] = trim($titleTextContents[$key]);
                        }
                        if (isset($alternativeTextContents[$key])) {
                            $updateData['sys_file_reference'][$referenceId]['alternative'] = trim($alternativeTextContents[$key]);
                        }
                        if (isset($descriptionContents[$key])) {
                            $updateData['sys_file_reference'][$referenceId]['description'] = trim($descriptionContents[$key]);
                        }
                        if (isset($linkContents[$key])) {
                            $updateData['sys_file_reference'][$referenceId]['link'] = trim($linkContents[$key]);
                        }
                    }
                }
            }
        }

        // update
        $tce = $this->getNewTCE();
        $tce->isImporting = TRUE;
        $tce->start($updateData, array());
        $tce->process_datamap();

    }

    public function checkOrCreateDir($dirPrefix) {
        // Split dir path and remove first directory (which should be "fileadmin")
        $filePathParts = explode('/', $dirPrefix);
        $firstDir = array_shift($filePathParts);
        if ($firstDir === $this->fileadminFolderName && GeneralUtility::getFileAbsFileName($dirPrefix)) {
            $pathAcc = '';
            foreach ($filePathParts as $dirname) {
                $pathAcc .= '/' . $dirname;
                if (strlen($dirname)) {
                    if (!@is_dir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
                        if (!GeneralUtility::mkdir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
                            $this->error('ERROR: Directory could not be created....B');
                            return FALSE;
                        }
                    }
                } elseif ($dirPrefix === $this->fileadminFolderName . $pathAcc) {
                    return TRUE;
                } else {
                    $this->error('ERROR: Directory could not be created....A');
                }
            }
        }
    }

    public function verifyFolderAccess($dirPrefix, $noAlternative = FALSE) {
        $fileProcObj = $this->getFileProcObj();
        // Check, if dirPrefix is inside a valid Filemount for user:
        $result = $fileProcObj->checkPathAgainstMounts(PATH_site . $dirPrefix);
        // If not, try to find another relative filemount and use that instead:
        if (!$result) {
            if ($noAlternative) {
                return FALSE;
            }
            // Find first web folder:
            $result = $fileProcObj->findFirstWebFolder();
            // If that succeeded, return the path to it:
            if ($result) {
                // Remove the "fileadmin/" prefix of input path - and append the rest to the return value:
                if (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
                    $dirPrefix = substr($dirPrefix, strlen($this->fileadminFolderName . '/'));
                }
                return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($fileProcObj->mounts[$result]['path'] . $dirPrefix);
            }
        } else {
            return $dirPrefix;
        }
    }

    /**************************
     * File Input
     *************************/

    public function loadFile($filename, $all = 0) {
        if (@is_file($filename)) {
            $fI = pathinfo($filename);
            if (strtolower($fI['extension']) == 'xml') {
                // XML:
                $xmlContent = GeneralUtility::getUrl($filename);
                if (strlen($xmlContent)) {
                    $this->dat = GeneralUtility::xml2array($xmlContent, '', TRUE);
                    if (is_array($this->dat)) {
                        if ($this->dat['_DOCUMENT_TAG'] === 'T3RecordDocument' && is_array($this->dat['header']) && is_array($this->dat['records'])) {
                            $this->loadInit();
                            return TRUE;
                        } else {
                            $this->error('XML file did not contain proper XML for TYPO3 Import');
                        }
                    } else {
                        $this->error('XML could not be parsed: ' . $this->dat);
                    }
                } else {
                    $this->error('Error opening file: ' . $filename);
                }
            } else {
                // T3D
                if ($fd = fopen($filename, 'rb')) {
                    $this->dat['header'] = $this->getNextFilePart($fd, 1, 'header');
                    if ($all) {
                        $this->dat['records'] = $this->getNextFilePart($fd, 1, 'records');
                        $this->dat['files'] = $this->getNextFilePart($fd, 1, 'files');
                        $this->dat['files_fal'] = $this->getNextFilePart($fd, 1, 'files_fal');
                    }
                    $this->loadInit();
                    return TRUE;
                } else {
                    $this->error('Error opening file: ' . $filename);
                }
                fclose($fd);
            }
        } else {
            $this->error('Filename not found: ' . $filename);
        }
        return FALSE;
    }

    public function getNextFilePart($fd, $unserialize = 0, $name = '') {
        $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
        // Getting header data
        $initStr = fread($fd, $initStrLen);
        if (empty($initStr)) {
            $this->error('File does not contain data for "' . $name . '"');
            return NULL;
        }
        $initStrDat = explode(':', $initStr);
        if (strstr($initStrDat[0], 'Warning') == FALSE) {
            if ((string)$initStrDat[3] === '') {
                $datString = fread($fd, (int)$initStrDat[2]);
                fread($fd, 1);
                if (md5($datString) === $initStrDat[0]) {
                    if ($initStrDat[1]) {
                        if ($this->compress) {
                            $datString = gzuncompress($datString);
                        } else {
                            $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.', 1);
                        }
                    }
                    return $unserialize ? unserialize($datString) : $datString;
                } else {
                    $this->error('MD5 check failed (' . $name . ')');
                }
            } else {
                $this->error('File read error: InitString had a wrong length. (' . $name . ')');
            }
        } else {
            $this->error('File read error: Warning message in file. (' . $initStr . fgets($fd) . ')');
        }
    }

    public function loadContent($filecontent) {
        $pointer = 0;
        $this->dat['header'] = $this->getNextContentPart($filecontent, $pointer, 1, 'header');
        $this->dat['records'] = $this->getNextContentPart($filecontent, $pointer, 1, 'records');
        $this->dat['files'] = $this->getNextContentPart($filecontent, $pointer, 1, 'files');
        $this->loadInit();
    }

    public function getNextContentPart($filecontent, &$pointer, $unserialize = 0, $name = '') {
        $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
        // getting header data
        $initStr = substr($filecontent, $pointer, $initStrLen);
        $pointer += $initStrLen;
        $initStrDat = explode(':', $initStr);
        if ((string)$initStrDat[3] === '') {
            $datString = substr($filecontent, $pointer, (int)$initStrDat[2]);
            $pointer += (int)$initStrDat[2] + 1;
            if (md5($datString) === $initStrDat[0]) {
                if ($initStrDat[1]) {
                    if ($this->compress) {
                        $datString = gzuncompress($datString);
                    } else {
                        $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.', 1);
                    }
                }
                return $unserialize ? unserialize($datString) : $datString;
            } else {
                $this->error('MD5 check failed (' . $name . ')');
            }
        } else {
            $this->error('Content read error: InitString had a wrong length. (' . $name . ')');
        }
    }

    public function loadInit() {
        $this->relStaticTables = (array) $this->dat['header']['relStaticTables'];
        $this->excludeMap = (array) $this->dat['header']['excludeMap'];
        $this->softrefCfg = (array) $this->dat['header']['softrefCfg'];
        $this->extensionDependencies = (array) $this->dat['header']['extensionDependencies'];
        $this->fixCharsets();
        if (isset($this->dat['header']['meta']['TYPO3_version']) && \TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger($this->dat['header']['meta']['TYPO3_version']) < 6000000){
            $this->legacyImport = TRUE;
            $this->initializeLegacyImportFolder();
        }
    }

    public function fixCharsets() {
        global $LANG;
        $importCharset = $this->dat['header']['charset'];
        if ($importCharset) {
            if ($importCharset !== $LANG->charSet) {
                $this->error('CHARSET: Converting charset of input file (' . $importCharset . ') to the system charset (' . $LANG->charSet . ')');
                // Convert meta data:
                if (is_array($this->dat['header']['meta'])) {
                    $LANG->csConvObj->convArray($this->dat['header']['meta'], $importCharset, $LANG->charSet);
                }
                // Convert record headers:
                if (is_array($this->dat['header']['records'])) {
                    $LANG->csConvObj->convArray($this->dat['header']['records'], $importCharset, $LANG->charSet);
                }
                // Convert records themselves:
                if (is_array($this->dat['records'])) {
                    $LANG->csConvObj->convArray($this->dat['records'], $importCharset, $LANG->charSet);
                }
            }
        } else {
            $this->error('CHARSET: No charset found in import file!');
        }
    }

    /********************************************************
     * Visual rendering of import/export memory, $this->dat
     ********************************************************/

    public function displayContentOverview() {
        global $LANG;
        // Check extension dependencies:
        if (is_array($this->dat['header']['extensionDependencies'])) {
            foreach ($this->dat['header']['extensionDependencies'] as $extKey) {
                if (!empty($extKey) && !\TYPO3\CMS\Core\Utility\ExtensionManagementUtility::isLoaded($extKey)) {
                    $this->error('DEPENDENCY: The extension with key "' . $extKey . '" must be installed!');
                }
            }
        }
        // Probably this is done to save memory space?
        unset($this->dat['files']);
        // Traverse header:
        $this->remainHeader = $this->dat['header'];
        if (is_array($this->remainHeader)) {
            // If there is a page tree set, show that:
            if (is_array($this->dat['header']['pagetree'])) {
                reset($this->dat['header']['pagetree']);
                $lines = array();
                $this->traversePageTree($this->dat['header']['pagetree'], $lines);
                $rows = array();
                $rows[] = '
                <tr class="bgColor5 tableheader">
                    <td>' . $LANG->getLL('impexpcore_displaycon_controls', TRUE) . '</td>
                    <td>' . $LANG->getLL('impexpcore_displaycon_title', TRUE) . '</td>
                    <td>' . $LANG->getLL('impexpcore_displaycon_size', TRUE) . '</td>
                    <td>' . $LANG->getLL('impexpcore_displaycon_message', TRUE) . '</td>
                    ' . ($this->update ? '<td>' . $LANG->getLL('impexpcore_displaycon_updateMode', TRUE) . '</td>' : '') . '
                    ' . ($this->update ? '<td>' . $LANG->getLL('impexpcore_displaycon_currentPath', TRUE) . '</td>' : '') . '
                    ' . ($this->showDiff ? '<td>' . $LANG->getLL('impexpcore_displaycon_result', TRUE) . '</td>' : '') . '
                </tr>';
                foreach ($lines as $r) {
                    $rows[] = '
                    <tr class="' . $r['class'] . '">
                        <td>' . $this->renderControls($r) . '</td>
                        <td nowrap="nowrap">' . $r['preCode'] . $r['title'] . '</td>
                        <td nowrap="nowrap">' . GeneralUtility::formatSize($r['size']) . '</td>
                        <td nowrap="nowrap">' . ($r['msg'] && !$this->doesImport ? '<span class="typo3-red">' . htmlspecialchars($r['msg']) . '</span>' : '') . '</td>
                        ' . ($this->update ? '<td nowrap="nowrap">' . $r['updateMode'] . '</td>' : '') . '
                        ' . ($this->update ? '<td nowrap="nowrap">' . $r['updatePath'] . '</td>' : '') . '
                        ' . ($this->showDiff ? '<td>' . $r['showDiffContent'] . '</td>' : '') . '
                    </tr>';
                }
                $out = '
                    <strong>' . $LANG->getLL('impexpcore_displaycon_insidePagetree', TRUE) . '</strong>
                    <br /><br />
                    <table border="0" cellpadding="0" cellspacing="1">' . implode('', $rows) . '</table>
                    <br /><br />';
            }
            // Print remaining records that were not contained inside the page tree:
            $lines = array();
            if (is_array($this->remainHeader['records'])) {
                if (is_array($this->remainHeader['records']['pages'])) {
                    $this->traversePageRecords($this->remainHeader['records']['pages'], $lines);
                }
                $this->traverseAllRecords($this->remainHeader['records'], $lines);
                if (count($lines)) {
                    $rows = array();
                    $rows[] = '
                    <tr class="bgColor5 tableheader">
                        <td>' . $LANG->getLL('impexpcore_displaycon_controls', TRUE) . '</td>
                        <td>' . $LANG->getLL('impexpcore_displaycon_title', TRUE) . '</td>
                        <td>' . $LANG->getLL('impexpcore_displaycon_size', TRUE) . '</td>
                        <td>' . $LANG->getLL('impexpcore_displaycon_message', TRUE) . '</td>
                        ' . ($this->update ? '<td>' . $LANG->getLL('impexpcore_displaycon_updateMode', TRUE) . '</td>' : '') . '
                        ' . ($this->update ? '<td>' . $LANG->getLL('impexpcore_displaycon_currentPath', TRUE) . '</td>' : '') . '
                        ' . ($this->showDiff ? '<td>' . $LANG->getLL('impexpcore_displaycon_result', TRUE) . '</td>' : '') . '
                    </tr>';
                    foreach ($lines as $r) {
                        $rows[] = '<tr class="' . $r['class'] . '">
                            <td>' . $this->renderControls($r) . '</td>
                            <td nowrap="nowrap">' . $r['preCode'] . $r['title'] . '</td>
                            <td nowrap="nowrap">' . GeneralUtility::formatSize($r['size']) . '</td>
                            <td nowrap="nowrap">' . ($r['msg'] && !$this->doesImport ? '<span class="typo3-red">' . htmlspecialchars($r['msg']) . '</span>' : '') . '</td>
                            ' . ($this->update ? '<td nowrap="nowrap">' . $r['updateMode'] . '</td>' : '') . '
                            ' . ($this->update ? '<td nowrap="nowrap">' . $r['updatePath'] . '</td>' : '') . '
                            ' . ($this->showDiff ? '<td>' . $r['showDiffContent'] . '</td>' : '') . '
                        </tr>';
                    }
                    $out .= '
                        <strong>' . $LANG->getLL('impexpcore_singlereco_outsidePagetree', TRUE) . '</strong>
                        <br /><br />
                        <table border="0" cellpadding="0" cellspacing="1">' . implode('', $rows) . '</table>';
                }
            }
        }
        return $out;
    }

    public function traversePageTree($pT, &$lines, $preCode = '') {
        foreach ($pT as $k => $v) {
            // Add this page:
            $this->singleRecordLines('pages', $k, $lines, $preCode);
            // Subrecords:
            if (is_array($this->dat['header']['pid_lookup'][$k])) {
                foreach ($this->dat['header']['pid_lookup'][$k] as $t => $recUidArr) {
                    if ($t != 'pages') {
                        foreach ($recUidArr as $ruid => $value) {
                            $this->singleRecordLines($t, $ruid, $lines, $preCode . '&nbsp;&nbsp;&nbsp;&nbsp;');
                        }
                    }
                }
                unset($this->remainHeader['pid_lookup'][$k]);
            }
            // Subpages, called recursively:
            if (is_array($v['subrow'])) {
                $this->traversePageTree($v['subrow'], $lines, $preCode . '&nbsp;&nbsp;&nbsp;&nbsp;');
            }
        }
    }

    public function traversePageRecords($pT, &$lines) {
        foreach ($pT as $k => $rHeader) {
            $this->singleRecordLines('pages', $k, $lines, '', 1);
            // Subrecords:
            if (is_array($this->dat['header']['pid_lookup'][$k])) {
                foreach ($this->dat['header']['pid_lookup'][$k] as $t => $recUidArr) {
                    if ($t != 'pages') {
                        foreach ($recUidArr as $ruid => $value) {
                            $this->singleRecordLines($t, $ruid, $lines, '&nbsp;&nbsp;&nbsp;&nbsp;');
                        }
                    }
                }
                unset($this->remainHeader['pid_lookup'][$k]);
            }
        }
    }

    public function traverseAllRecords($pT, &$lines) {
        foreach ($pT as $t => $recUidArr) {
            $this->addGeneralErrorsByTable($t);
            if ($t != 'pages') {
                $preCode = '';
                foreach ($recUidArr as $ruid => $value) {
                    $this->singleRecordLines($t, $ruid, $lines, $preCode, 1);
                }
            }
        }
    }

    protected function addGeneralErrorsByTable($table) {
        if ($this->update && $table === 'sys_file') {
            $this->error('Updating sys_file records is not supported! They will be imported as new records!');
        }
        if ($this->force_all_UIDS && $table === 'sys_file') {
            $this->error('Forcing uids of sys_file records is not supported! They will be imported as new records!');
        }
    }

    public function singleRecordLines($table, $uid, &$lines, $preCode, $checkImportInPidRecord = 0) {
        global $LANG;
        // Get record:
        $record = $this->dat['header']['records'][$table][$uid];
        unset($this->remainHeader['records'][$table][$uid]);
        if (!is_array($record) && !($table === 'pages' && !$uid)) {
            $this->error('MISSING RECORD: ' . $table . ':' . $uid, 1);
        }
        // Begin to create the line arrays information record, pInfo:
        $pInfo = array();
        $pInfo['ref'] = $table . ':' . $uid;
        // Unknown table name:
        if ($table === '_SOFTREF_') {
            $pInfo['preCode'] = $preCode;
            $pInfo['title'] = '<em>' . $GLOBALS['LANG']->getLL('impexpcore_singlereco_softReferencesFiles', TRUE) . '</em>';
        } elseif (!isset($GLOBALS['TCA'][$table])) {
            // Unknown table name:
            $pInfo['preCode'] = $preCode;
            $pInfo['msg'] = 'UNKNOWN TABLE \'' . $pInfo['ref'] . '\'';
            $pInfo['title'] = '<em>' . htmlspecialchars($record['title']) . '</em>';
        } else {
            // Otherwise, set table icon and title.
            // Import Validation (triggered by $this->display_import_pid_record) will show messages if import is not possible of various items.
            if (is_array($this->display_import_pid_record)) {
                if ($checkImportInPidRecord) {
                    if (!$GLOBALS['BE_USER']->doesUserHaveAccess($this->display_import_pid_record, ($table === 'pages' ? 8 : 16))) {
                        $pInfo['msg'] .= '\'' . $pInfo['ref'] . '\' cannot be INSERTED on this page! ';
                    }
                    if (!$this->checkDokType($table, $this->display_import_pid_record['doktype']) && !$GLOBALS['TCA'][$table]['ctrl']['rootLevel']) {
                        $pInfo['msg'] .= '\'' . $table . '\' cannot be INSERTED on this page type (change page type to \'Folder\'.) ';
                    }
                }
                if (!$GLOBALS['BE_USER']->check('tables_modify', $table)) {
                    $pInfo['msg'] .= 'You are not allowed to CREATE \'' . $table . '\' tables! ';
                }
                if ($GLOBALS['TCA'][$table]['ctrl']['readOnly']) {
                    $pInfo['msg'] .= 'TABLE \'' . $table . '\' is READ ONLY! ';
                }
                if ($GLOBALS['TCA'][$table]['ctrl']['adminOnly'] && !$GLOBALS['BE_USER']->isAdmin()) {
                    $pInfo['msg'] .= 'TABLE \'' . $table . '\' is ADMIN ONLY! ';
                }
                if ($GLOBALS['TCA'][$table]['ctrl']['is_static']) {
                    $pInfo['msg'] .= 'TABLE \'' . $table . '\' is a STATIC TABLE! ';
                }
                if ((int)$GLOBALS['TCA'][$table]['ctrl']['rootLevel'] === 1) {
                    $pInfo['msg'] .= 'TABLE \'' . $table . '\' will be inserted on ROOT LEVEL! ';
                }
                $diffInverse = FALSE;
                if ($this->update) {
                    // In case of update-PREVIEW we swap the diff-sources.
                    $diffInverse = TRUE;
                    $recInf = $this->doesRecordExist($table, $uid, $this->showDiff ? '*' : '');
                    $pInfo['updatePath'] = $recInf ? htmlspecialchars($this->getRecordPath($recInf['pid'])) : '<strong>NEW!</strong>';
                    // Mode selector:
                    $optValues = array();
                    $optValues[] = $recInf ? $LANG->getLL('impexpcore_singlereco_update') : $LANG->getLL('impexpcore_singlereco_insert');
                    if ($recInf) {
                        $optValues['as_new'] = $LANG->getLL('impexpcore_singlereco_importAsNew');
                    }
                    if ($recInf) {
                        if (!$this->global_ignore_pid) {
                            $optValues['ignore_pid'] = $LANG->getLL('impexpcore_singlereco_ignorePid');
                        } else {
                            $optValues['respect_pid'] = $LANG->getLL('impexpcore_singlereco_respectPid');
                        }
                    }
                    if (!$recInf && $GLOBALS['BE_USER']->isAdmin()) {
                        $optValues['force_uid'] = sprintf($LANG->getLL('impexpcore_singlereco_forceUidSAdmin'), $uid);
                    }
                    $optValues['exclude'] = $LANG->getLL('impexpcore_singlereco_exclude');
                    if ($table === 'sys_file') {
                        $pInfo['updateMode'] = '';
                    } else {
                        $pInfo['updateMode'] = $this->renderSelectBox('tx_impexp[import_mode][' . $table . ':' . $uid . ']', $this->import_mode[$table . ':' . $uid], $optValues);
                    }
                }
                // Diff vieiw:
                if ($this->showDiff) {
                    // For IMPORTS, get new id:
                    if ($newUid = $this->import_mapId[$table][$uid]) {
                        $diffInverse = FALSE;
                        $recInf = $this->doesRecordExist($table, $newUid, '*');
                        BackendUtility::workspaceOL($table, $recInf);
                    }
                    if (is_array($recInf)) {
                        $pInfo['showDiffContent'] = $this->compareRecords($recInf, $this->dat['records'][$table . ':' . $uid]['data'], $table, $diffInverse);
                    }
                }
            }
            $pInfo['preCode'] = $preCode . \TYPO3\CMS\Backend\Utility\IconUtility::getSpriteIconForRecord($table, (array) $this->dat['records'][($table . ':' . $uid)]['data'], array('title' => htmlspecialchars(($table . ':' . $uid))));
            $pInfo['title'] = htmlspecialchars($record['title']);
            // View page:
            if ($table === 'pages') {
                $viewID = $this->mode === 'export' ? $uid : ($this->doesImport ? $this->import_mapId['pages'][$uid] : 0);
                if ($viewID) {
                    $pInfo['title'] = '<a href="#" onclick="' . htmlspecialchars(BackendUtility::viewOnClick($viewID, $GLOBALS['BACK_PATH'])) . 'return false;">' . $pInfo['title'] . '</a>';
                }
            }
        }
        $pInfo['class'] = $table == 'pages' ? 'bgColor4-20' : 'bgColor4';
        $pInfo['type'] = 'record';
        $pInfo['size'] = $record['size'];
        $lines[] = $pInfo;
        // File relations:
        if (is_array($record['filerefs'])) {
            $this->addFiles($record['filerefs'], $lines, $preCode);
        }
        // DB relations
        if (is_array($record['rels'])) {
            $this->addRelations($record['rels'], $lines, $preCode);
        }
        // Soft ref
        if (count($record['softrefs'])) {
            $preCode_A = $preCode . '&nbsp;&nbsp;&nbsp;&nbsp;';
            $preCode_B = $preCode . '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;';
            foreach ($record['softrefs'] as $info) {
                $pInfo = array();
                $pInfo['preCode'] = $preCode_A . \TYPO3\CMS\Backend\Utility\IconUtility::getSpriteIcon('status-status-reference-soft');
                $pInfo['title'] = '<em>' . $info['field'] . ', "' . $info['spKey'] . '" </em>: <span title="' . htmlspecialchars($info['matchString']) . '">' . htmlspecialchars(GeneralUtility::fixed_lgd_cs($info['matchString'], 60)) . '</span>';
                if ($info['subst']['type']) {
                    if (strlen($info['subst']['title'])) {
                        $pInfo['title'] .= '<br/>' . $preCode_B . '<strong>' . $LANG->getLL('impexpcore_singlereco_title', TRUE) . '</strong> ' . htmlspecialchars(GeneralUtility::fixed_lgd_cs($info['subst']['title'], 60));
                    }
                    if (strlen($info['subst']['description'])) {
                        $pInfo['title'] .= '<br/>' . $preCode_B . '<strong>' . $LANG->getLL('impexpcore_singlereco_descr', TRUE) . '</strong> ' . htmlspecialchars(GeneralUtility::fixed_lgd_cs($info['subst']['description'], 60));
                    }
                    $pInfo['title'] .= '<br/>' . $preCode_B . ($info['subst']['type'] == 'file' ? $LANG->getLL('impexpcore_singlereco_filename', TRUE) . ' <strong>' . $info['subst']['relFileName'] . '</strong>' : '') . ($info['subst']['type'] == 'string' ? $LANG->getLL('impexpcore_singlereco_value', TRUE) . ' <strong>' . $info['subst']['tokenValue'] . '</strong>' : '') . ($info['subst']['type'] == 'db' ? $LANG->getLL('impexpcore_softrefsel_record', TRUE) . ' <strong>' . $info['subst']['recordRef'] . '</strong>' : '');
                }
                $pInfo['ref'] = 'SOFTREF';
                $pInfo['size'] = '';
                $pInfo['class'] = 'bgColor3';
                $pInfo['type'] = 'softref';
                $pInfo['_softRefInfo'] = $info;
                $pInfo['type'] = 'softref';
                if ($info['error'] && !GeneralUtility::inList('editable,exclude', $this->softrefCfg[$info['subst']['tokenID']]['mode'])) {
                    $pInfo['msg'] .= $info['error'];
                }
                $lines[] = $pInfo;
                // Add relations:
                if ($info['subst']['type'] == 'db') {
                    list($tempTable, $tempUid) = explode(':', $info['subst']['recordRef']);
                    $this->addRelations(array(array('table' => $tempTable, 'id' => $tempUid, 'tokenID' => $info['subst']['tokenID'])), $lines, $preCode_B, array(), '');
                }
                // Add files:
                if ($info['subst']['type'] == 'file') {
                    $this->addFiles(array($info['file_ID']), $lines, $preCode_B, '', $info['subst']['tokenID']);
                }
            }
        }
    }

    public function addRelations($rels, &$lines, $preCode, $recurCheck = array(), $htmlColorClass = '') {
        foreach ($rels as $dat) {
            $table = $dat['table'];
            $uid = $dat['id'];
            $pInfo = array();
            $pInfo['ref'] = $table . ':' . $uid;
            if (in_array($pInfo['ref'], $recurCheck)) {
                continue;
            }
            $Iprepend = '';
            $staticFixed = FALSE;
            $record = NULL;
            if ($uid > 0) {
                $record = $this->dat['header']['records'][$table][$uid];
                if (!is_array($record)) {
                    if ($this->isTableStatic($table) || $this->isExcluded($table, $uid) || $dat['tokenID'] && !$this->includeSoftref($dat['tokenID'])) {
                        $pInfo['title'] = htmlspecialchars('STATIC: ' . $pInfo['ref']);
                        $Iprepend = '_static';
                        $staticFixed = TRUE;
                    } else {
                        $doesRE = $this->doesRecordExist($table, $uid);
                        $lostPath = $this->getRecordPath($table === 'pages' ? $doesRE['uid'] : $doesRE['pid']);
                        $pInfo['title'] = htmlspecialchars($pInfo['ref']);
                        $pInfo['title'] = '<span title="' . htmlspecialchars($lostPath) . '">' . $pInfo['title'] . '</span>';
                        $pInfo['msg'] = 'LOST RELATION' . (!$doesRE ? ' (Record not found!)' : ' (Path: ' . $lostPath . ')');
                        $Iprepend = '_lost';
                    }
                } else {
                    $pInfo['title'] = htmlspecialchars($record['title']);
                    $pInfo['title'] = '<span title="' . htmlspecialchars($this->getRecordPath(($table === 'pages' ? $record['uid'] : $record['pid']))) . '">' . $pInfo['title'] . '</span>';
                }
            } else {
                // Negative values in relation fields. This is typically sys_language fields, fe_users fields etc. They are static values. They CAN theoretically be negative pointers to uids in other tables but this is so rarely used that it is not supported
                $pInfo['title'] = htmlspecialchars('FIXED: ' . $pInfo['ref']);
                $staticFixed = TRUE;
            }
            $pInfo['preCode'] = $preCode . '&nbsp;&nbsp;&nbsp;&nbsp;<img' . \TYPO3\CMS\Backend\Utility\IconUtility::skinImg($GLOBALS['BACK_PATH'], ('gfx/rel_db' . $Iprepend . '.gif'), 'width="13" height="12"') . ' align="top" title="' . htmlspecialchars($pInfo['ref']) . '" alt="" />';
            $pInfo['class'] = $htmlColorClass ?: 'bgColor3';
            $pInfo['type'] = 'rel';
            if (!$staticFixed || $this->showStaticRelations) {
                $lines[] = $pInfo;
                if (is_array($record) && is_array($record['rels'])) {
                    $this->addRelations($record['rels'], $lines, $preCode . '&nbsp;&nbsp;', array_merge($recurCheck, array($pInfo['ref'])), $htmlColorClass);
                }
            }
        }
    }

    public function addFiles($rels, &$lines, $preCode, $htmlColorClass = '', $tokenID = '') {
        foreach ($rels as $ID) {
            // Process file:
            $pInfo = array();
            $fI = $this->dat['header']['files'][$ID];
            if (!is_array($fI)) {
                if (!$tokenID || $this->includeSoftref($tokenID)) {
                    $pInfo['msg'] = 'MISSING FILE: ' . $ID;
                    $this->error('MISSING FILE: ' . $ID, 1);
                } else {
                    return;
                }
            }
            $pInfo['preCode'] = $preCode . '&nbsp;&nbsp;&nbsp;&nbsp;' . \TYPO3\CMS\Backend\Utility\IconUtility::getSpriteIcon('status-status-reference-hard');
            $pInfo['title'] = htmlspecialchars($fI['filename']);
            $pInfo['ref'] = 'FILE';
            $pInfo['size'] = $fI['filesize'];
            $pInfo['class'] = $htmlColorClass ?: 'bgColor3';
            $pInfo['type'] = 'file';
            // If import mode and there is a non-RTE softreference, check the destination directory:
            if ($this->mode === 'import' && $tokenID && !$fI['RTE_ORIG_ID']) {
                if (isset($fI['parentRelFileName'])) {
                    $pInfo['msg'] = 'Seems like this file is already referenced from within an HTML/CSS file. That takes precedence. ';
                } else {
                    $testDirPrefix = PathUtility::dirname($fI['relFileName']) . '/';
                    $testDirPrefix2 = $this->verifyFolderAccess($testDirPrefix);
                    if (!$testDirPrefix2) {
                        $pInfo['msg'] = 'ERROR: There are no available filemounts to write file in! ';
                    } elseif ($testDirPrefix !== $testDirPrefix2) {
                        $pInfo['msg'] = 'File will be attempted written to "' . $testDirPrefix2 . '". ';
                    }
                }
                // Check if file exists:
                if (file_exists(PATH_site . $fI['relFileName'])) {
                    if ($this->update) {
                        $pInfo['updatePath'] .= 'File exists.';
                    } else {
                        $pInfo['msg'] .= 'File already exists! ';
                    }
                }
                // Check extension:
                $fileProcObj = $this->getFileProcObj();
                if ($fileProcObj->actionPerms['addFile']) {
                    $testFI = GeneralUtility::split_fileref(PATH_site . $fI['relFileName']);
                    if (!$this->allowPHPScripts && !$fileProcObj->checkIfAllowed($testFI['fileext'], $testFI['path'], $testFI['file'])) {
                        $pInfo['msg'] .= 'File extension was not allowed!';
                    }
                } else {
                    $pInfo['msg'] = 'You user profile does not allow you to create files on the server!';
                }
            }
            $pInfo['showDiffContent'] = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($this->fileIDMap[$ID]);
            $lines[] = $pInfo;
            unset($this->remainHeader['files'][$ID]);
            // RTE originals:
            if ($fI['RTE_ORIG_ID']) {
                $ID = $fI['RTE_ORIG_ID'];
                $pInfo = array();
                $fI = $this->dat['header']['files'][$ID];
                if (!is_array($fI)) {
                    $pInfo['msg'] = 'MISSING RTE original FILE: ' . $ID;
                    $this->error('MISSING RTE original FILE: ' . $ID, 1);
                }
                $pInfo['showDiffContent'] = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($this->fileIDMap[$ID]);
                $pInfo['preCode'] = $preCode . '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;' . \TYPO3\CMS\Backend\Utility\IconUtility::getSpriteIcon('actions-reference-file');
                $pInfo['title'] = htmlspecialchars($fI['filename']) . ' <em>(Original)</em>';
                $pInfo['ref'] = 'FILE';
                $pInfo['size'] = $fI['filesize'];
                $pInfo['class'] = $htmlColorClass ?: 'bgColor3';
                $pInfo['type'] = 'file';
                $lines[] = $pInfo;
                unset($this->remainHeader['files'][$ID]);
            }
            // External resources:
            if (is_array($fI['EXT_RES_ID'])) {
                foreach ($fI['EXT_RES_ID'] as $ID) {
                    $pInfo = array();
                    $fI = $this->dat['header']['files'][$ID];
                    if (!is_array($fI)) {
                        $pInfo['msg'] = 'MISSING External Resource FILE: ' . $ID;
                        $this->error('MISSING External Resource FILE: ' . $ID, 1);
                    } else {
                        $pInfo['updatePath'] = $fI['parentRelFileName'];
                    }
                    $pInfo['showDiffContent'] = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($this->fileIDMap[$ID]);
                    $pInfo['preCode'] = $preCode . '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;' . \TYPO3\CMS\Backend\Utility\IconUtility::getSpriteIcon('actions-insert-reference');
                    $pInfo['title'] = htmlspecialchars($fI['filename']) . ' <em>(Resource)</em>';
                    $pInfo['ref'] = 'FILE';
                    $pInfo['size'] = $fI['filesize'];
                    $pInfo['class'] = $htmlColorClass ?: 'bgColor3';
                    $pInfo['type'] = 'file';
                    $lines[] = $pInfo;
                    unset($this->remainHeader['files'][$ID]);
                }
            }
        }
    }

    public function checkDokType($checkTable, $doktype) {
        global $PAGES_TYPES;
        $allowedTableList = isset($PAGES_TYPES[$doktype]['allowedTables']) ? $PAGES_TYPES[$doktype]['allowedTables'] : $PAGES_TYPES['default']['allowedTables'];
        $allowedArray = GeneralUtility::trimExplode(',', $allowedTableList, TRUE);
        // If all tables or the table is listed as a allowed type, return TRUE
        if (strstr($allowedTableList, '*') || in_array($checkTable, $allowedArray)) {
            return TRUE;
        }
    }

    public function renderControls($r) {
        global $LANG;
        if ($this->mode === 'export') {
            return $r['type'] == 'record' ? '<input type="checkbox" name="tx_impexp[exclude][' . $r['ref'] . ']" id="checkExclude' . $r['ref'] . '" value="1" /> <label for="checkExclude' . $r['ref'] . '">' . $LANG->getLL('impexpcore_singlereco_exclude', TRUE) . '</label>' : ($r['type'] == 'softref' ? $this->softrefSelector($r['_softRefInfo']) : '');
        } else {
            // During import
            // For softreferences with editable fields:
            if ($r['type'] == 'softref' && is_array($r['_softRefInfo']['subst']) && $r['_softRefInfo']['subst']['tokenID']) {
                $tokenID = $r['_softRefInfo']['subst']['tokenID'];
                $cfg = $this->softrefCfg[$tokenID];
                if ($cfg['mode'] === 'editable') {
                    return (strlen($cfg['title']) ? '<strong>' . htmlspecialchars($cfg['title']) . '</strong><br/>' : '') . htmlspecialchars($cfg['description']) . '<br/>
                        <input type="text" name="tx_impexp[softrefInputValues][' . $tokenID . ']" value="' . htmlspecialchars((isset($this->softrefInputValues[$tokenID]) ? $this->softrefInputValues[$tokenID] : $cfg['defValue'])) . '" />';
                }
            }
        }
    }

    public function softrefSelector($cfg) {
        global $LANG;
        // Looking for file ID if any:
        $fI = $cfg['file_ID'] ? $this->dat['header']['files'][$cfg['file_ID']] : array();
        // Substitution scheme has to be around and RTE images MUST be exported.
        if (is_array($cfg['subst']) && $cfg['subst']['tokenID'] && !$fI['RTE_ORIG_ID']) {
            // Create options:
            $optValues = array();
            $optValues[''] = '';
            $optValues['editable'] = $LANG->getLL('impexpcore_softrefsel_editable');
            $optValues['exclude'] = $LANG->getLL('impexpcore_softrefsel_exclude');
            // Get current value:
            $value = $this->softrefCfg[$cfg['subst']['tokenID']]['mode'];
            // Render options selector:
            $selectorbox = $this->renderSelectBox(('tx_impexp[softrefCfg][' . $cfg['subst']['tokenID'] . '][mode]'), $value, $optValues) . '<br/>';
            if ($value === 'editable') {
                $descriptionField = '';
                // Title:
                if (strlen($cfg['subst']['title'])) {
                    $descriptionField .= '
                    <input type="hidden" name="tx_impexp[softrefCfg][' . $cfg['subst']['tokenID'] . '][title]" value="' . htmlspecialchars($cfg['subst']['title']) . '" />
                    <strong>' . htmlspecialchars($cfg['subst']['title']) . '</strong><br/>';
                }
                // Description:
                if (!strlen($cfg['subst']['description'])) {
                    $descriptionField .= '
                    ' . $LANG->getLL('impexpcore_printerror_description', TRUE) . '<br/>
                    <input type="text" name="tx_impexp[softrefCfg][' . $cfg['subst']['tokenID'] . '][description]" value="' . htmlspecialchars($this->softrefCfg[$cfg['subst']['tokenID']]['description']) . '" />';
                } else {
                    $descriptionField .= '

                    <input type="hidden" name="tx_impexp[softrefCfg][' . $cfg['subst']['tokenID'] . '][description]" value="' . htmlspecialchars($cfg['subst']['description']) . '" />' . htmlspecialchars($cfg['subst']['description']);
                }
                // Default Value:
                $descriptionField .= '<input type="hidden" name="tx_impexp[softrefCfg][' . $cfg['subst']['tokenID'] . '][defValue]" value="' . htmlspecialchars($cfg['subst']['tokenValue']) . '" />';
            } else {
                $descriptionField = '';
            }
            return $selectorbox . $descriptionField;
        }
    }

    /*****************************
     * Helper functions of kinds
     *****************************/

    public function isTableStatic($table) {
        if (is_array($GLOBALS['TCA'][$table])) {
            return $GLOBALS['TCA'][$table]['ctrl']['is_static'] || in_array($table, $this->relStaticTables) || in_array('_ALL', $this->relStaticTables);
        }
    }

    public function inclRelation($table) {
        if (is_array($GLOBALS['TCA'][$table])) {
            return (in_array($table, $this->relOnlyTables) || in_array('_ALL', $this->relOnlyTables)) && $GLOBALS['BE_USER']->check('tables_select', $table);
        }
    }

    public function isExcluded($table, $uid) {
        return $this->excludeMap[$table . ':' . $uid] ? TRUE : FALSE;
    }

    public function includeSoftref($tokenID) {
        return $tokenID && !GeneralUtility::inList('exclude,editable', $this->softrefCfg[$tokenID]['mode']);
    }

    public function checkPID($pid) {
        if (!isset($this->checkPID_cache[$pid])) {
            $this->checkPID_cache[$pid] = (bool) $GLOBALS['BE_USER']->isInWebMount($pid);
        }
        return $this->checkPID_cache[$pid];
    }

    public function dontIgnorePid($table, $uid) {
        return $this->import_mode[$table . ':' . $uid] !== 'ignore_pid' && (!$this->global_ignore_pid || $this->import_mode[$table . ':' . $uid] === 'respect_pid');
    }

    public function doesRecordExist($table, $uid, $fields = '') {
        return BackendUtility::getRecord($table, $uid, $fields ? $fields : 'uid,pid');
    }

    public function getRecordPath($pid) {
        if (!isset($this->cache_getRecordPath[$pid])) {
            $clause = $GLOBALS['BE_USER']->getPagePermsClause(1);
            $this->cache_getRecordPath[$pid] = (string) BackendUtility::getRecordPath($pid, $clause, 20);
        }
        return $this->cache_getRecordPath[$pid];
    }

    public function renderSelectBox($prefix, $value, $optValues) {
        $opt = array();
        $isSelFlag = 0;
        foreach ($optValues as $k => $v) {
            $sel = (string)$k === (string)$value ? ' selected="selected"' : '';
            if ($sel) {
                $isSelFlag++;
            }
            $opt[] = '<option value="' . htmlspecialchars($k) . '"' . $sel . '>' . htmlspecialchars($v) . '</option>';
        }
        if (!$isSelFlag && (string)$value !== '') {
            $opt[] = '<option value="' . htmlspecialchars($value) . '" selected="selected">' . htmlspecialchars(('[\'' . $value . '\']')) . '</option>';
        }
        return '<select name="' . $prefix . '">' . implode('', $opt) . '</select>';
    }

    public function compareRecords($databaseRecord, $importRecord, $table, $inverseDiff = FALSE) {
        global $LANG;
        // Initialize:
        $output = array();
        $t3lib_diff_Obj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Utility\\DiffUtility');
        // Check if both inputs are records:
        if (is_array($databaseRecord) && is_array($importRecord)) {
            // Traverse based on database record
            foreach ($databaseRecord as $fN => $value) {
                if (is_array($GLOBALS['TCA'][$table]['columns'][$fN]) && $GLOBALS['TCA'][$table]['columns'][$fN]['config']['type'] != 'passthrough') {
                    if (isset($importRecord[$fN])) {
                        if (trim($databaseRecord[$fN]) !== trim($importRecord[$fN])) {
                            // Create diff-result:
                            $output[$fN] = $t3lib_diff_Obj->makeDiffDisplay(BackendUtility::getProcessedValue($table, $fN, !$inverseDiff ? $importRecord[$fN] : $databaseRecord[$fN], 0, 1, 1), BackendUtility::getProcessedValue($table, $fN, !$inverseDiff ? $databaseRecord[$fN] : $importRecord[$fN], 0, 1, 1));
                        }
                        unset($importRecord[$fN]);
                    }
                }
            }
            // Traverse remaining in import record:
            foreach ($importRecord as $fN => $value) {
                if (is_array($GLOBALS['TCA'][$table]['columns'][$fN]) && $GLOBALS['TCA'][$table]['columns'][$fN]['config']['type'] !== 'passthrough') {
                    $output[$fN] = '<strong>Field missing</strong> in database';
                }
            }
            // Create output:
            if (count($output)) {
                $tRows = array();
                foreach ($output as $fN => $state) {
                    $tRows[] = '
                        <tr>
                            <td class="bgColor5">' . $GLOBALS['LANG']->sL($GLOBALS['TCA'][$table]['columns'][$fN]['label'], TRUE) . ' (' . htmlspecialchars($fN) . ')</td>
                            <td class="bgColor4">' . $state . '</td>
                        </tr>
                    ';
                }
                $output = '<table border="0" cellpadding="0" cellspacing="1">' . implode('', $tRows) . '</table>';
            } else {
                $output = 'Match';
            }
            return '<strong class="nobr">[' . htmlspecialchars(($table . ':' . $importRecord['uid'] . ' => ' . $databaseRecord['uid'])) . ']:</strong> ' . $output;
        }
        return 'ERROR: One of the inputs were not an array!';
    }

    public function getRTEoriginalFilename($string) {
        // If "magic image":
        if (GeneralUtility::isFirstPartOfStr($string, 'RTEmagicC_')) {
            // Find original file:
            $pI = pathinfo(substr($string, strlen('RTEmagicC_')));
            $filename = substr($pI['basename'], 0, -strlen(('.' . $pI['extension'])));
            $origFilePath = 'RTEmagicP_' . $filename;
            return $origFilePath;
        }
    }

    public function getFileProcObj() {
        if (!is_object($this->fileProcObj)) {
            $this->fileProcObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Utility\\File\\ExtendedFileUtility');
            $this->fileProcObj->init(array(), $GLOBALS['TYPO3_CONF_VARS']['BE']['fileExtensions']);
            $this->fileProcObj->setActionPermissions();
        }
        return $this->fileProcObj;
    }

    public function callHook($name, $params) {
        if (is_array($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/impexp/class.tx_impexp.php'][$name])) {
            foreach ($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/impexp/class.tx_impexp.php'][$name] as $hook) {
                GeneralUtility::callUserFunction($hook, $params, $this);
            }
        }
    }

    /*****************************
     * Error handling
     *****************************/

    public function error($msg) {
        $this->errorLog[] = $msg;
    }

    public function printErrorLog() {
        return count($this->errorLog) ? \TYPO3\CMS\Core\Utility\DebugUtility::viewArray($this->errorLog) : '';
    }

}