1 <?php

2 namespace TYPO3\CMS\Impexp;

3 

17 use TYPO3\CMS\Backend\Utility\BackendUtility;

18 use TYPO3\CMS\Core\Exception;

19 use TYPO3\CMS\Core\Utility\GeneralUtility;

20 use TYPO3\CMS\Core\Utility\PathUtility;

21 

56 class ImportExport {

57 

63  public $showStaticRelations = FALSE;

64 

70  public $fileadminFolderName = '';

71 

77  public $mode = '';

78 

84  public $update = FALSE;

85 

91  public $doesImport = FALSE;

92 

100  public $display_import_pid_record = '';

101 

107  public $suggestedInsertUids = array();

108 

114  public $import_mode = array();

115 

121  public $global_ignore_pid = FALSE;

122 

128  public $force_all_UIDS = FALSE;

129 

135  public $showDiff = FALSE;

136 

142  public $allowPHPScripts = FALSE;

143 

149  public $enableLogging = FALSE;

150 

156  public $softrefInputValues = array();

157 

163  public $fileIDMap = array();

164 

170  public $maxFileSize = 1000000;

171 

177  public $maxRecordSize = 1000000;

178 

184  public $maxExportSize = 10000000;

185 

191  public $relOnlyTables = array();

192 

198  public $relStaticTables = array();

199 

205  public $excludeMap = array();

206 

212  public $softrefCfg = array();

213 

219  public $extensionDependencies = array();

220 

226  public $dontCompress = 0;

227 

233  public $includeExtFileResources = 0;

234 

240  public $extFileResourceExtensions = 'html,htm,css';

241 

247  public $import_mapId = array();

248 

255  public $import_newId = array();

256 

262  public $import_newId_pids = array();

263 

269  public $import_data = array();

270 

276  public $errorLog = array();

277 

283  public $cache_getRecordPath = array();

284 

290  public $checkPID_cache = array();

291 

297  public $compress = 0;

298 

304  public $dat = array();

305 

312  public $fileProcObj = '';

313 

320  protected $recordTypesIncludeFields = array();

321 

327  protected $defaultRecordIncludeFields = array('uid', 'pid');

328 

334  protected $storageObjects = array();

335 

341  protected $legacyImport = FALSE;

342 

346  protected $legacyImportFolder = NULL;

347 

353  protected $legacyImportTargetPath = '_imported/';

354 

360  protected $legacyImportMigrationTables = array(

361  'tt_content' => array(

362  'image' => array(

363  'titleTexts' => 'titleText',

364  'description' => 'imagecaption',

365  'links' => 'image_link',

366  'alternativeTexts' => 'altText'

367  ),

368  'media' => array(

369  'description' => 'imagecaption',

370  )

371  ),

372  'pages' => array(

373  'media' => array()

374  ),

375  'pages_language_overlay' => array(

376  'media' => array()

377  )

378  );

379 

380 

387  protected $legacyImportMigrationRecords = array();

388 

389 

390  /**************************

391  * Initialize

392  *************************/

393 

402  public function init($dontCompress = 0, $mode = '') {

403  $this->compress = function_exists('gzcompress');

404  $this->dontCompress = $dontCompress;

405  $this->mode = $mode;

406  $this->fileadminFolderName = !empty($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir']) ? rtrim($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') : 'fileadmin';

407  }

408 

409  /**************************

410  * Export / Init + Meta Data

411  *************************/

412 

419  public function setHeaderBasics() {

420  // Initializing:

421  if (is_array($this->softrefCfg)) {

422  foreach ($this->softrefCfg as $key => $value) {

423  if (!strlen($value['mode'])) {

424  unset($this->softrefCfg[$key]);

425  }

426  }

427  }

428  // Setting in header memory:

429  // Version of file format

430  $this->dat['header']['XMLversion'] = '1.0';

431  // Initialize meta data array (to put it in top of file)

432  $this->dat['header']['meta'] = array();

433  // Add list of tables to consider static

434  $this->dat['header']['relStaticTables'] = $this->relStaticTables;

435  // The list of excluded records

436  $this->dat['header']['excludeMap'] = $this->excludeMap;

437  // Soft Reference mode for elements

438  $this->dat['header']['softrefCfg'] = $this->softrefCfg;

439  // List of extensions the import depends on.

440  $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;

441  }

442 

450  public function setCharset($charset) {

451  $this->dat['header']['charset'] = $charset;

452  }

453 

466  public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email) {

467  $this->dat['header']['meta'] = array(

468  'title' => $title,

469  'description' => $description,

470  'notes' => $notes,

471  'packager_username' => $packager_username,

472  'packager_name' => $packager_name,

473  'packager_email' => $packager_email,

474  'TYPO3_version' => TYPO3_version,

475  'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])

476  );

477  }

478 

486  public function addThumbnail($imgFilepath) {

487  if (@is_file($imgFilepath)) {

488  $imgInfo = @getimagesize($imgFilepath);

489  if (is_array($imgInfo)) {

490  $fileContent = GeneralUtility::getUrl($imgFilepath);

491  $this->dat['header']['thumbnail'] = array(

492  'imgInfo' => $imgInfo,

493  'content' => $fileContent,

494  'filesize' => strlen($fileContent),

495  'filemtime' => filemtime($imgFilepath),

496  'filename' => PathUtility::basename($imgFilepath)

497  );

498  }

499  }

500  }

501 

502  /**************************

503  * Export / Init Page tree

504  *************************/

505 

513  public function setPageTree($idH) {

514  $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);

515  return $this->flatInversePageTree($this->dat['header']['pagetree']);

516  }

517 

527  public function unsetExcludedSections($idH) {

528  if (is_array($idH)) {

529  foreach ($idH as $k => $v) {

530  if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {

531  unset($idH[$k]);

532  } elseif (is_array($idH[$k]['subrow'])) {

533  $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);

534  }

535  }

536  }

537  return $idH;

538  }

539 

549  public function flatInversePageTree($idH, $a = array()) {

550  if (is_array($idH)) {

551  $idH = array_reverse($idH);

552  foreach ($idH as $k => $v) {

553  $a[$v['uid']] = $v['uid'];

554  if (is_array($v['subrow'])) {

555  $a = $this->flatInversePageTree($v['subrow'], $a);

556  }

557  }

558  }

559  return $a;

560  }

561 

572  public function flatInversePageTree_pid($idH, $a = array(), $pid = -1) {

573  if (is_array($idH)) {

574  $idH = array_reverse($idH);

575  foreach ($idH as $k => $v) {

576  $a[$v['uid']] = $pid;

577  if (is_array($v['subrow'])) {

578  $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);

579  }

580  }

581  }

582  return $a;

583  }

584 

585  /**************************

586  * Export

587  *************************/

588 

596  public function setRecordTypesIncludeFields(array $recordTypesIncludeFields) {

597  foreach ($recordTypesIncludeFields as $table => $fields) {

598  if (!is_array($fields)) {

599  throw new \TYPO3\CMS\Core\Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);

600  }

601  $this->setRecordTypeIncludeFields($table, $fields);

602  }

603  }

604 

612  public function setRecordTypeIncludeFields($table, array $fields) {

613  $this->recordTypesIncludeFields[$table] = $fields;

614  }

615 

626  public function export_addRecord($table, $row, $relationLevel = 0) {

627  BackendUtility::workspaceOL($table, $row);

628  if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[($table . ':' . $row['uid'])]) {

629  if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {

630  if (!isset($this->dat['records'][($table . ':' . $row['uid'])])) {

631  // Prepare header info:

632  $row = $this->filterRecordFields($table, $row);

633  $headerInfo = array();

634  $headerInfo['uid'] = $row['uid'];

635  $headerInfo['pid'] = $row['pid'];

636  $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);

637  $headerInfo['size'] = strlen(serialize($row));

638  if ($relationLevel) {

639  $headerInfo['relationLevel'] = $relationLevel;

640  }

641  // If record content is not too large in size, set the header content and add the rest:

642  if ($headerInfo['size'] < $this->maxRecordSize) {

643  // Set the header summary:

644  $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;

645  // Create entry in the PID lookup:

646  $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;

647  // Initialize reference index object:

648  $refIndexObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Database\\ReferenceIndex');

649  // Yes to workspace overlays for exporting....

650  $refIndexObj->WSOL = TRUE;

651  $relations = $refIndexObj->getRelations($table, $row);

652  $relations = $this->fixFileIDsInRelations($relations);

653  $relations = $this->removeSoftrefsHavingTheSameDatabaseRelation($relations);

654  // Data:

655  $this->dat['records'][$table . ':' . $row['uid']] = array();

656  $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;

657  $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;

658  $this->errorLog = array_merge($this->errorLog, $refIndexObj->errorLog);

659  // Merge error logs.

660  // Add information about the relations in the record in the header:

661  $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);

662  // Add information about the softrefs to header:

663  $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);

664  } else {

665  $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');

666  }

667  } else {

668  $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');

669  }

670  } else {

671  $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');

672  }

673  }

674  }

675 

683  protected function fixFileIDsInRelations(array $relations) {

684  foreach ($relations as $field => $relation) {

685  if (isset($relation['type']) && $relation['type'] === 'file') {

686  foreach ($relation['newValueFiles'] as $key => $fileRelationData) {

687  $absoluteFilePath = $fileRelationData['ID_absFile'];

688  if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {

689  $relatedFilePath = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($absoluteFilePath);

690  $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);

691  }

692  }

693  }

694  if ($relation['type'] == 'flex') {

695  if (is_array($relation['flexFormRels']['file'])) {

696  foreach ($relation['flexFormRels']['file'] as $key => $subList) {

697  foreach ($subList as $subKey => $fileRelationData) {

698  $absoluteFilePath = $fileRelationData['ID_absFile'];

699  if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {

700  $relatedFilePath = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($absoluteFilePath);

701  $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);

702  }

703  }

704  }

705  }

706  }

707  }

708  return $relations;

709  }

710 

720  protected function removeSoftrefsHavingTheSameDatabaseRelation($relations) {

721  $fixedRelations = array();

722  foreach ($relations as $field => $relation) {

723  $newRelation = $relation;

724  if (isset($newRelation['type']) && $newRelation['type'] === 'db') {

725  foreach ($newRelation['itemArray'] as $key => $dbRelationData) {

726  if ($dbRelationData['table'] === 'sys_file') {

727  if (isset($newRelation['softrefs']['keys']['typolink'])) {

728  foreach ($newRelation['softrefs']['keys']['typolink'] as $softrefKey => $softRefData) {

729  if ($softRefData['subst']['type'] === 'file') {

730  $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->retrieveFileOrFolderObject($softRefData['subst']['relFileName']);

731  if ($file instanceof \TYPO3\CMS\Core\Resource\File) {

732  if ($file->getUid() == $dbRelationData['id']) {

733  unset($newRelation['softrefs']['keys']['typolink'][$softrefKey]);

734  }

735  }

736  }

737  }

738  if (empty($newRelation['softrefs']['keys']['typolink'])) {

739  unset($newRelation['softrefs']);

740  }

741  }

742  }

743  }

744  }

745  $fixedRelations[$field] = $newRelation;

746  }

747  return $fixedRelations;

748  }

749 

750 

761  public function export_addDBRelations($relationLevel = 0) {

762  // Initialize:

763  $addR = array();

764  // Traverse all "rels" registered for "records"

765  if (is_array($this->dat['records'])) {

766  foreach ($this->dat['records'] as $k => $value) {

767  if (is_array($this->dat['records'][$k])) {

768  foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {

769  // For all DB types of relations:

770  if ($vR['type'] == 'db') {

771  foreach ($vR['itemArray'] as $fI) {

772  $this->export_addDBRelations_registerRelation($fI, $addR);

773  }

774  }

775  // For all flex/db types of relations:

776  if ($vR['type'] == 'flex') {

777  // DB relations in flex form fields:

778  if (is_array($vR['flexFormRels']['db'])) {

779  foreach ($vR['flexFormRels']['db'] as $subList) {

780  foreach ($subList as $fI) {

781  $this->export_addDBRelations_registerRelation($fI, $addR);

782  }

783  }

784  }

785  // DB oriented soft references in flex form fields:

786  if (is_array($vR['flexFormRels']['softrefs'])) {

787  foreach ($vR['flexFormRels']['softrefs'] as $subList) {

788  foreach ($subList['keys'] as $spKey => $elements) {

789  foreach ($elements as $el) {

790  if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {

791  list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);

792  $fI = array(

793  'table' => $tempTable,

794  'id' => $tempUid

795  );

796  $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);

797  }

798  }

799  }

800  }

801  }

802  }

803  // In any case, if there are soft refs:

804  if (is_array($vR['softrefs']['keys'])) {

805  foreach ($vR['softrefs']['keys'] as $spKey => $elements) {

806  foreach ($elements as $el) {

807  if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {

808  list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);

809  $fI = array(

810  'table' => $tempTable,

811  'id' => $tempUid

812  );

813  $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);

814  }

815  }

816  }

817  }

818  }

819  }

820  }

821  } else {

822  $this->error('There were no records available.');

823  }

824  // Now, if there were new records to add, do so:

825  if (count($addR)) {

826  foreach ($addR as $fI) {

827  // Get and set record:

828  $row = BackendUtility::getRecord($fI['table'], $fI['id']);

829  if (is_array($row)) {

830  $this->export_addRecord($fI['table'], $row, $relationLevel + 1);

831  }

832  // Set status message

833  // Relation pointers always larger than zero except certain "select" types with negative values pointing to uids - but that is not supported here.

834  if ($fI['id'] > 0) {

835  $rId = $fI['table'] . ':' . $fI['id'];

836  if (!isset($this->dat['records'][$rId])) {

837  $this->dat['records'][$rId] = 'NOT_FOUND';

838  $this->error('Relation record ' . $rId . ' was not found!');

839  }

840  }

841  }

842  }

843  // Return overview of relations found and added

844  return $addR;

845  }

846 

857  public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '') {

858  $rId = $fI['table'] . ':' . $fI['id'];

859  if (isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id']) && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])) {

860  if (!isset($this->dat['records'][$rId])) {

861  // Set this record to be included since it is not already.

862  $addR[$rId] = $fI;

863  }

864  }

865  }

866 

875  public function export_addFilesFromRelations() {

876  // Traverse all "rels" registered for "records"

877  if (is_array($this->dat['records'])) {

878  foreach ($this->dat['records'] as $k => $value) {

879  if (isset($this->dat['records'][$k]['rels']) && is_array($this->dat['records'][$k]['rels'])) {

880  foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {

881  // For all file type relations:

882  if ($vR['type'] == 'file') {

883  foreach ($vR['newValueFiles'] as $key => $fI) {

884  $this->export_addFile($fI, $k, $fieldname);

885  // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:

886  unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);

887  }

888  }

889  // For all flex type relations:

890  if ($vR['type'] == 'flex') {

891  if (is_array($vR['flexFormRels']['file'])) {

892  foreach ($vR['flexFormRels']['file'] as $key => $subList) {

893  foreach ($subList as $subKey => $fI) {

894  $this->export_addFile($fI, $k, $fieldname);

895  // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:

896  unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);

897  }

898  }

899  }

900  // DB oriented soft references in flex form fields:

901  if (is_array($vR['flexFormRels']['softrefs'])) {

902  foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {

903  foreach ($subList['keys'] as $spKey => $elements) {

904  foreach ($elements as $subKey => $el) {

905  if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {

906  // Create abs path and ID for file:

907  $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);

908  $ID = md5($el['subst']['relFileName']);

909  if ($ID_absFile) {

910  if (!$this->dat['files'][$ID]) {

911  $fI = array(

912  'filename' => PathUtility::basename($ID_absFile),

913  'ID_absFile' => $ID_absFile,

914  'ID' => $ID,

915  'relFileName' => $el['subst']['relFileName']

916  );

917  $this->export_addFile($fI, '_SOFTREF_');

918  }

919  $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;

920  }

921  }

922  }

923  }

924  }

925  }

926  }

927  // In any case, if there are soft refs:

928  if (is_array($vR['softrefs']['keys'])) {

929  foreach ($vR['softrefs']['keys'] as $spKey => $elements) {

930  foreach ($elements as $subKey => $el) {

931  if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {

932  // Create abs path and ID for file:

933  $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);

934  $ID = md5($el['subst']['relFileName']);

935  if ($ID_absFile) {

936  if (!$this->dat['files'][$ID]) {

937  $fI = array(

938  'filename' => PathUtility::basename($ID_absFile),

939  'ID_absFile' => $ID_absFile,

940  'ID' => $ID,

941  'relFileName' => $el['subst']['relFileName']

942  );

943  $this->export_addFile($fI, '_SOFTREF_');

944  }

945  $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;

946  }

947  }

948  }

949  }

950  }

951  }

952  }

953  }

954  } else {

955  $this->error('There were no records available.');

956  }

957  }

958 

964  public function export_addFilesFromSysFilesRecords() {

965  if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {

966  return;

967  }

968  foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {

969  $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];

970  $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->createFileObject($recordData);

971  $this->export_addSysFile($file);

972  }

973  }

974 

981  public function export_addSysFile(\TYPO3\CMS\Core\Resource\File $file) {

982  if ($file->getProperty('size') >= $this->maxFileSize) {

983  $this->error('File ' . $file->getPublicUrl() . ' was larger (' . GeneralUtility::formatSize($file->getProperty('size')) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');

984  return;

985  }

986  try {

987  $fileContent = $file->getContents();

988  } catch (\Exception $e) {

989  $this->error('Error when trying to add file ' . $file->getCombinedIdentifier() . ': ' . $e->getMessage());

990  return;

991  }

992  $fileUid = $file->getUid();

993  $fileInfo = $file->getStorage()->getFileInfo($file);

994  // we sadly have to cast it to string here, because the size property is also returning a string

995  $fileSize = (string)$fileInfo['size'];

996  if ($fileSize !== $file->getProperty('size')) {

997  $this->error('File size of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added with current size.');

998  $this->dat['records']['sys_file:' . $fileUid]['data']['size'] = $fileSize;

999  }

1000  $fileSha1 = $file->getStorage()->hashFile($file, 'sha1');

1001  if ($fileSha1 !== $file->getProperty('sha1')) {

1002  $this->error('File sha1 hash of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added on current sha1.');

1003  $this->dat['records']['sys_file:' . $fileUid]['data']['sha1'] = $fileSha1;

1004  }

1005 

1006  $fileRec = array();

1007  $fileRec['filesize'] = $fileSize;

1008  $fileRec['filename'] = $file->getProperty('name');

1009  $fileRec['filemtime'] = $file->getProperty('modification_date');

1010 

1011  // build unique id based on the storage and the file identifier

1012  $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));

1013 

1014  // Setting this data in the header

1015  $this->dat['header']['files_fal'][$fileId] = $fileRec;

1016 

1017  // ... and finally add the heavy stuff:

1018  $fileRec['content'] = $fileContent;

1019  $fileRec['content_sha1'] = $fileSha1;

1020 

1021  $this->dat['files_fal'][$fileId] = $fileRec;

1022  }

1023 

1024 

1034  public function export_addFile($fI, $recordRef = '', $fieldname = '') {

1035  if (@is_file($fI['ID_absFile'])) {

1036  if (filesize($fI['ID_absFile']) < $this->maxFileSize) {

1037  $fileInfo = stat($fI['ID_absFile']);

1038  $fileRec = array();

1039  $fileRec['filesize'] = $fileInfo['size'];

1040  $fileRec['filename'] = PathUtility::basename($fI['ID_absFile']);

1041  $fileRec['filemtime'] = $fileInfo['mtime'];

1042  //for internal type file_reference

1043  $fileRec['relFileRef'] = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($fI['ID_absFile']);

1044  if ($recordRef) {

1045  $fileRec['record_ref'] = $recordRef . '/' . $fieldname;

1046  }

1047  if ($fI['relFileName']) {

1048  $fileRec['relFileName'] = $fI['relFileName'];

1049  }

1050  // Setting this data in the header

1051  $this->dat['header']['files'][$fI['ID']] = $fileRec;

1052  // ... and for the recordlisting, why not let us know WHICH relations there was...

1053  if ($recordRef && $recordRef !== '_SOFTREF_') {

1054  $refParts = explode(':', $recordRef, 2);

1055  if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {

1056  $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = array();

1057  }

1058  $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];

1059  }

1060  // ... and finally add the heavy stuff:

1061  $fileRec['content'] = GeneralUtility::getUrl($fI['ID_absFile']);

1062  $fileRec['content_md5'] = md5($fileRec['content']);

1063  $this->dat['files'][$fI['ID']] = $fileRec;

1064  // For soft references, do further processing:

1065  if ($recordRef === '_SOFTREF_') {

1066  // RTE files?

1067  if ($RTEoriginal = $this->getRTEoriginalFilename(PathUtility::basename($fI['ID_absFile']))) {

1068  $RTEoriginal_absPath = PathUtility::dirname($fI['ID_absFile']) . '/' . $RTEoriginal;

1069  if (@is_file($RTEoriginal_absPath)) {

1070  $RTEoriginal_ID = md5($RTEoriginal_absPath);

1071  $fileInfo = stat($RTEoriginal_absPath);

1072  $fileRec = array();

1073  $fileRec['filesize'] = $fileInfo['size'];

1074  $fileRec['filename'] = PathUtility::basename($RTEoriginal_absPath);

1075  $fileRec['filemtime'] = $fileInfo['mtime'];

1076  $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];

1077  $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;

1078  // Setting this data in the header

1079  $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;

1080  // ... and finally add the heavy stuff:

1081  $fileRec['content'] = GeneralUtility::getUrl($RTEoriginal_absPath);

1082  $fileRec['content_md5'] = md5($fileRec['content']);

1083  $this->dat['files'][$RTEoriginal_ID] = $fileRec;

1084  } else {

1085  $this->error('RTE original file "' . \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');

1086  }

1087  }

1088  // Files with external media?

1089  // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.

1090  $html_fI = pathinfo(PathUtility::basename($fI['ID_absFile']));

1091  if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {

1092  $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';

1093  if (strtolower($html_fI['extension']) === 'css') {

1094  $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));

1095  } else {

1096  // html, htm:

1097  $htmlParser = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Html\\HtmlParser');

1098  $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], array(), $uniquePrefix));

1099  }

1100  $htmlResourceCaptured = FALSE;

1101  foreach ($prefixedMedias as $k => $v) {

1102  if ($k % 2) {

1103  $EXTres_absPath = GeneralUtility::resolveBackPath(PathUtility::dirname($fI['ID_absFile']) . '/' . $v);

1104  $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);

1105  if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {

1106  $htmlResourceCaptured = TRUE;

1107  $EXTres_ID = md5($EXTres_absPath);

1108  $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;

1109  $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';

1110  // Add file to memory if it is not set already:

1111  if (!isset($this->dat['header']['files'][$EXTres_ID])) {

1112  $fileInfo = stat($EXTres_absPath);

1113  $fileRec = array();

1114  $fileRec['filesize'] = $fileInfo['size'];

1115  $fileRec['filename'] = PathUtility::basename($EXTres_absPath);

1116  $fileRec['filemtime'] = $fileInfo['mtime'];

1117  $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];

1118  // Media relative to the HTML file.

1119  $fileRec['parentRelFileName'] = $v;

1120  // Setting this data in the header

1121  $this->dat['header']['files'][$EXTres_ID] = $fileRec;

1122  // ... and finally add the heavy stuff:

1123  $fileRec['content'] = GeneralUtility::getUrl($EXTres_absPath);

1124  $fileRec['content_md5'] = md5($fileRec['content']);

1125  $this->dat['files'][$EXTres_ID] = $fileRec;

1126  }

1127  }

1128  }

1129  }

1130  if ($htmlResourceCaptured) {

1131  $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);

1132  }

1133  }

1134  }

1135  } else {

1136  $this->error($fI['ID_absFile'] . ' was larger (' . GeneralUtility::formatSize(filesize($fI['ID_absFile'])) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');

1137  }

1138  } else {

1139  $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');

1140  }

1141  }

1142 

1151  public function flatDBrels($dbrels) {

1152  $list = array();

1153  foreach ($dbrels as $dat) {

1154  if ($dat['type'] == 'db') {

1155  foreach ($dat['itemArray'] as $i) {

1156  $list[$i['table'] . ':' . $i['id']] = $i;

1157  }

1158  }

1159  if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['db'])) {

1160  foreach ($dat['flexFormRels']['db'] as $subList) {

1161  foreach ($subList as $i) {

1162  $list[$i['table'] . ':' . $i['id']] = $i;

1163  }

1164  }

1165  }

1166  }

1167  return $list;

1168  }

1169 

1177  public function flatSoftRefs($dbrels) {

1178  $list = array();

1179  foreach ($dbrels as $field => $dat) {

1180  if (is_array($dat['softrefs']['keys'])) {

1181  foreach ($dat['softrefs']['keys'] as $spKey => $elements) {

1182  if (is_array($elements)) {

1183  foreach ($elements as $subKey => $el) {

1184  $lKey = $field . ':' . $spKey . ':' . $subKey;

1185  $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey), $el);

1186  // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!

1187  if ($el['subst'] && $el['subst']['relFileName']) {

1188  $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);

1189  }

1190  }

1191  }

1192  }

1193  }

1194  if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['softrefs'])) {

1195  foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {

1196  if (is_array($subSoftrefs['keys'])) {

1197  foreach ($subSoftrefs['keys'] as $spKey => $elements) {

1198  foreach ($elements as $subKey => $el) {

1199  $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;

1200  $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath), $el);

1201  // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!

1202  if ($el['subst'] && $el['subst']['relFileName']) {

1203  $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);

1204  }

1205  }

1206  }

1207  }

1208  }

1209  }

1210  }

1211  return $list;

1212  }

1213 

1222  protected function filterRecordFields($table, array $row) {

1223  if (isset($this->recordTypesIncludeFields[$table])) {

1224  $includeFields = array_unique(array_merge(

1225  $this->recordTypesIncludeFields[$table],

1226  $this->defaultRecordIncludeFields

1227  ));

1228  $newRow = array();

1229  foreach ($row as $key => $value) {

1230  if (in_array($key, $includeFields)) {

1231  $newRow[$key] = $value;

1232  }

1233  }

1234  } else {

1235  $newRow = $row;

1236  }

1237  return $newRow;

1238  }

1239 

1240 

1241  /**************************

1242  * File Output

1243  *************************/

1244 

1252  public function compileMemoryToFileContent($type = '') {

1253  if ($type == 'xml') {

1254  $out = $this->createXML();

1255  } else {

1256  $compress = $this->doOutputCompress();

1257  $out = '';

1258  // adding header:

1259  $out .= $this->addFilePart(serialize($this->dat['header']), $compress);

1260  // adding records:

1261  $out .= $this->addFilePart(serialize($this->dat['records']), $compress);

1262  // adding files:

1263  $out .= $this->addFilePart(serialize($this->dat['files']), $compress);

1264  // adding files_fal:

1265  $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);

1266  }

1267  return $out;

1268  }

1269 

1276  public function createXML() {

1277  // Options:

1278  $options = array(

1279  'alt_options' => array(

1280  '/header' => array(

1281  'disableTypeAttrib' => TRUE,

1282  'clearStackPath' => TRUE,

1283  'parentTagMap' => array(

1284  'files' => 'file',

1285  'files_fal' => 'file',

1286  'records' => 'table',

1287  'table' => 'rec',

1288  'rec:rels' => 'relations',

1289  'relations' => 'element',

1290  'filerefs' => 'file',

1291  'pid_lookup' => 'page_contents',

1292  'header:relStaticTables' => 'static_tables',

1293  'static_tables' => 'tablename',

1294  'excludeMap' => 'item',

1295  'softrefCfg' => 'softrefExportMode',

1296  'extensionDependencies' => 'extkey',

1297  'softrefs' => 'softref_element'

1298  ),

1299  'alt_options' => array(

1300  '/pagetree' => array(

1301  'disableTypeAttrib' => TRUE,

1302  'useIndexTagForNum' => 'node',

1303  'parentTagMap' => array(

1304  'node:subrow' => 'node'

1305  )

1306  ),

1307  '/pid_lookup/page_contents' => array(

1308  'disableTypeAttrib' => TRUE,

1309  'parentTagMap' => array(

1310  'page_contents' => 'table'

1311  ),

1312  'grandParentTagMap' => array(

1313  'page_contents/table' => 'item'

1314  )

1315  )

1316  )

1317  ),

1318  '/records' => array(

1319  'disableTypeAttrib' => TRUE,

1320  'parentTagMap' => array(

1321  'records' => 'tablerow',

1322  'tablerow:data' => 'fieldlist',

1323  'tablerow:rels' => 'related',

1324  'related' => 'field',

1325  'field:itemArray' => 'relations',

1326  'field:newValueFiles' => 'filerefs',

1327  'field:flexFormRels' => 'flexform',

1328  'relations' => 'element',

1329  'filerefs' => 'file',

1330  'flexform:db' => 'db_relations',

1331  'flexform:file' => 'file_relations',

1332  'flexform:softrefs' => 'softref_relations',

1333  'softref_relations' => 'structurePath',

1334  'db_relations' => 'path',

1335  'file_relations' => 'path',

1336  'path' => 'element',

1337  'keys' => 'softref_key',

1338  'softref_key' => 'softref_element'

1339  ),

1340  'alt_options' => array(

1341  '/records/tablerow/fieldlist' => array(

1342  'useIndexTagForAssoc' => 'field'

1343  )

1344  )

1345  ),

1346  '/files' => array(

1347  'disableTypeAttrib' => TRUE,

1348  'parentTagMap' => array(

1349  'files' => 'file'

1350  )

1351  ),

1352  '/files_fal' => array(

1353  'disableTypeAttrib' => TRUE,

1354  'parentTagMap' => array(

1355  'files_fal' => 'file'

1356  )

1357  )

1358  )

1359  );

1360  // Creating XML file from $outputArray:

1361  $charset = $this->dat['header']['charset'] ?: 'utf-8';

1362  $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;

1363  $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);

1364  return $XML;

1365  }

1366 

1373  public function doOutputCompress() {

1374  return $this->compress && !$this->dontCompress;

1375  }

1376 

1385  public function addFilePart($data, $compress = FALSE) {

1386  if ($compress) {

1387  $data = gzcompress($data);

1388  }

1389  return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';

1390  }

1391 

1392  /***********************

1393  * Import

1394  ***********************/

1395 

1401  protected function initializeImport() {

1402  // Set this flag to indicate that an import is being/has been done.

1403  $this->doesImport = 1;

1404  // Initialize:

1405  // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.

1406  $this->import_mapId = array();

1407  $this->import_newId = array();

1408  $this->import_newId_pids = array();

1409  // Temporary files stack initialized:

1410  $this->unlinkFiles = array();

1411  $this->alternativeFileName = array();

1412  $this->alternativeFilePath = array();

1413 

1414  $this->initializeStorageObjects();

1415  }

1416 

1422  protected function initializeStorageObjects() {

1424  $storageRepository = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Resource\\StorageRepository');

1425  $this->storageObjects = $storageRepository->findAll();

1426  }

1427 

1428 

1435  public function importData($pid) {

1436 

1437  $this->initializeImport();

1438 

1439  // Write sys_file_storages first

1440  $this->writeSysFileStorageRecords();

1441  // Write sys_file records and write the binary file data

1442  $this->writeSysFileRecords();

1443  // Write records, first pages, then the rest

1444  // Fields with "hard" relations to database, files and flexform fields are kept empty during this run

1445  $this->writeRecords_pages($pid);

1446  $this->writeRecords_records($pid);

1447  // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:

1448  // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.

1449  $this->setRelations();

1450  // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):

1451  $this->setFlexFormRelations();

1452  // Unlink temporary files:

1453  $this->unlinkTempFiles();

1454  // Finally, traverse all records and process softreferences with substitution attributes.

1455  $this->processSoftReferences();

1456  // After all migrate records using sys_file_reference now

1457  if ($this->legacyImport) {

1458  $this->migrateLegacyImportRecords();

1459  }

1460  }

1461 

1467  protected function writeSysFileStorageRecords() {

1468  if (!isset($this->dat['header']['records']['sys_file_storage'])) {

1469  return;

1470  }

1471  $sysFileStorageUidsToBeResetToDefaultStorage = array();

1472  foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {

1473  $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];

1474  // continue with Local, writable and online storage only

1475  if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {

1476  $useThisStorageUidInsteadOfTheOneInImport = 0;

1478  foreach ($this->storageObjects as $localStorage) {

1479  // check the available storage for Local, writable and online ones

1480  if ($localStorage->getDriverType() === 'Local' && $localStorage->isWritable() && $localStorage->isOnline()) {

1481  // check if there is already a identical storage present (same pathType and basePath)

1482  $storageRecordConfiguration = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);

1483  $localStorageRecordConfiguration = $localStorage->getConfiguration();

1484  if ($storageRecordConfiguration['pathType'] == $localStorageRecordConfiguration['pathType'] && $storageRecordConfiguration['basePath'] == $localStorageRecordConfiguration['basePath']) {

1485  // same storage is already present

1486  $useThisStorageUidInsteadOfTheOneInImport = $localStorage->getUid();

1487  break;

1488  }

1489  }

1490  }

1491  if ($useThisStorageUidInsteadOfTheOneInImport > 0) {

1492  // same storage is already present; map the to be imported one to the present one

1493  $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $useThisStorageUidInsteadOfTheOneInImport;

1494  } else {

1495  // Local, writable and online storage. Is allowed to be used to later write files in.

1496  $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);

1497  }

1498  } else {

1499  // Storage with non Local drivers could be imported but must not be used to saves files in, because you

1500  // could not be sure, that this is supported. The default storage will be used in this case.

1501  // It could happen that non writable and non online storage will be created as dupes because you could not

1502  // check the detailed configuration options at this point

1503  $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);

1504  $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;

1505  }

1506 

1507  }

1508 

1509  // Importing the added ones

1510  $tce = $this->getNewTCE();

1511  // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.

1512  $tce->reverseOrder = 1;

1513  $tce->isImporting = TRUE;

1514  $tce->start($this->import_data, array());

1515  $tce->process_datamap();

1516  $this->addToMapId($tce->substNEWwithIDs);

1517 

1518  $defaultStorageUid = NULL;

1519  // get default storage

1520  $defaultStorage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getDefaultStorage();

1521  if ($defaultStorage !== NULL) {

1522  $defaultStorageUid = $defaultStorage->getUid();

1523  }

1524  foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {

1525  $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;

1526  }

1527 

1528  // unset the sys_file_storage records to prevent a import in writeRecords_records

1529  unset($this->dat['header']['records']['sys_file_storage']);

1530  }

1531 

1537  protected function writeSysFileRecords() {

1538  if (!isset($this->dat['header']['records']['sys_file'])) {

1539  return;

1540  }

1541  $this->addGeneralErrorsByTable('sys_file');

1542 

1543  // fetch fresh storage records from database

1544  $storageRecords = $this->fetchStorageRecords();

1545 

1546  $defaultStorage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getDefaultStorage();

1547 

1548  $sanitizedFolderMappings = array();

1549 

1550  foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {

1551  $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];

1552 

1553  // save file to disk

1554  $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);

1555  $temporaryFile = $this->writeTemporaryFileFromData($fileId);

1556  if ($temporaryFile === NULL) {

1557  // error on writing the file. Error message was already added

1558  continue;

1559  }

1560 

1561  $originalStorageUid = $fileRecord['storage'];

1562  $useStorageFromStorageRecords = FALSE;

1563 

1564  // replace storage id, if a alternative one was registered

1565  if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {

1566  $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];

1567  $useStorageFromStorageRecords = TRUE;

1568  }

1569 

1570  if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {

1571  // no storage for the file is defined, mostly because of a missing default storage.

1572  $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');

1573  continue;

1574  }

1575 

1576  // using a storage from the local storage is only allowed, if the uid is present in the

1577  // mapping. Only in this case we could be sure, that it's a local, online and writable storage.

1578  if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {

1580  $storage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);

1581  } elseif ($this->isFallbackStorage($fileRecord['storage'])) {

1582  $storage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getStorageObject(0);

1583  } elseif ($defaultStorage !== NULL) {

1584  $storage = $defaultStorage;

1585  } else {

1586  $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');

1587  continue;

1588  }

1589 

1590  $newFile = NULL;

1591 

1592  // check, if there is a identical file

1593  try {

1594  if ($storage->hasFile($fileRecord['identifier'])) {

1595  $file = $storage->getFile($fileRecord['identifier']);

1596  if ($file->getSha1() === $fileRecord['sha1']) {

1597  $newFile = $file;

1598  }

1599  }

1600  } catch (Exception $e) {}

1601 

1602  if ($newFile === NULL) {

1603 

1604  $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));

1605  if (in_array($folderName, $sanitizedFolderMappings)) {

1606  $folderName = $sanitizedFolderMappings[$folderName];

1607  }

1608  if (!$storage->hasFolder($folderName)) {

1609  try {

1610  $importFolder = $storage->createFolder($folderName);

1611  if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {

1612  $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();

1613  }

1614  } catch (Exception $e) {

1615  $this->error('Error: Folder could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');

1616  continue;

1617  }

1618  } else {

1619  $importFolder = $storage->getFolder($folderName);

1620  }

1621 

1622  try {

1624  $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);

1625  } catch (Exception $e) {

1626  $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');

1627  continue;

1628  }

1629 

1630  if ($newFile->getSha1() !== $fileRecord['sha1']) {

1631  $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');

1632  }

1633  }

1634 

1635  // save the new uid in the import id map

1636  $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();

1637  $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());

1638 

1639  }

1640 

1641  // unset the sys_file records to prevent a import in writeRecords_records

1642  unset($this->dat['header']['records']['sys_file']);

1643  // remove all sys_file_reference records that point to file records which are unknown

1644  // in the system to prevent exceptions

1645  $this->removeSysFileReferenceRecordsFromImportDataWithRelationToMissingFile();

1646  }

1647 

1654  protected function removeSysFileReferenceRecordsFromImportDataWithRelationToMissingFile()

1655  {

1656  if (!isset($this->dat['header']['records']['sys_file_reference'])) {

1657  return;

1658  }

1659 

1660  foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {

1661  $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];

1662  if (!in_array($fileReferenceRecord['uid_local'], $this->import_mapId['sys_file'])) {

1663  unset($this->dat['header']['records']['sys_file_reference'][$sysFileReferenceUid]);

1664  unset($this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]);

1665  $this->error('Error: sys_file_reference record ' . (int)$sysFileReferenceUid

1666  . ' with relation to sys_file record ' . (int)$fileReferenceRecord['uid_local']

1667  . ', which is not part of the import data, was not imported.'

1668  );

1669  }

1670  }

1671  }

1672 

1679  protected function isFallbackStorage($storageId) {

1680  return $storageId === 0 || $storageId === '0';

1681  }

1682 

1699  protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid) {

1700  if (!isset($this->dat['header']['records']['sys_file_reference'])) {

1701  return;

1702  }

1703 

1704  foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {

1705  $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];

1706  if ($fileReferenceRecord['uid_local'] == $oldFileUid) {

1707  $fileReferenceRecord['uid_local'] = $newFileUid;

1708  $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;

1709  }

1710  }

1711  }

1712 

1718  protected function initializeLegacyImportFolder() {

1720  $folder = $GLOBALS['BE_USER']->getDefaultUploadFolder();

1721  if ($folder === FALSE) {

1722  $this->error('Error: the backend users default upload folder is missing! No files will be imported!');

1723  }

1724  if (!$folder->hasFolder($this->legacyImportTargetPath)) {

1725  try {

1726  $this->legacyImportFolder = $folder->createFolder($this->legacyImportTargetPath);

1727  } catch (\TYPO3\CMS\Core\Exception $e) {

1728  $this->error('Error: the import folder in the default upload folder could not be created! No files will be imported!');

1729  }

1730  } else {

1731  $this->legacyImportFolder = $folder->getSubfolder($this->legacyImportTargetPath);

1732  }

1733 

1734  }

1735 

1742  protected function fetchStorageRecords() {

1743  $whereClause = \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('sys_file_storage');

1744  $whereClause .= \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('sys_file_storage');

1745 

1746  $rows = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(

1747  '*',

1748  'sys_file_storage',

1749  '1=1' . $whereClause,

1750  '',

1751  '',

1752  '',

1753  'uid'

1754  );

1755 

1756  return $rows;

1757  }

1758 

1766  protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal') {

1767  $temporaryFilePath = NULL;

1768  if (is_array($this->dat[$dataKey][$fileId])) {

1769  $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');

1770  GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);

1771  clearstatcache();

1772  if (@is_file($temporaryFilePathInternal)) {

1773  $this->unlinkFiles[] = $temporaryFilePathInternal;

1774  if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {

1775  $temporaryFilePath = $temporaryFilePathInternal;

1776  } else {

1777  $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')', 1);

1778  }

1779  } else {

1780  $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!', 1);

1781  }

1782  } else {

1783  $this->error('Error: No file found for ID ' . $fileId, 1);

1784  }

1785  return $temporaryFilePath;

1786  }

1787 

1796  public function writeRecords_pages($pid) {

1797  // First, write page structure if any:

1798  if (is_array($this->dat['header']['records']['pages'])) {

1799  $this->addGeneralErrorsByTable('pages');

1800  // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.

1801  $pageRecords = $this->dat['header']['records']['pages'];

1802  $this->import_data = array();

1803  // First add page tree if any

1804  if (is_array($this->dat['header']['pagetree'])) {

1805  $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);

1806  foreach ($pagesFromTree as $uid) {

1807  $thisRec = $this->dat['header']['records']['pages'][$uid];

1808  // PID: Set the main $pid, unless a NEW-id is found

1809  $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;

1810  $this->addSingle('pages', $uid, $setPid);

1811  unset($pageRecords[$uid]);

1812  }

1813  }

1814  // Then add all remaining pages not in tree on root level:

1815  if (count($pageRecords)) {

1816  $remainingPageUids = array_keys($pageRecords);

1817  foreach ($remainingPageUids as $pUid) {

1818  $this->addSingle('pages', $pUid, $pid);

1819  }

1820  }

1821  // Now write to database:

1822  $tce = $this->getNewTCE();

1823  $tce->isImporting = TRUE;

1824  $this->callHook('before_writeRecordsPages', array(

1825  'tce' => &$tce,

1826  'data' => &$this->import_data

1827  ));

1828  $tce->suggestedInsertUids = $this->suggestedInsertUids;

1829  $tce->start($this->import_data, array());

1830  $tce->process_datamap();

1831  $this->callHook('after_writeRecordsPages', array(

1832  'tce' => &$tce

1833  ));

1834  // post-processing: Registering new ids (end all tcemain sessions with this)

1835  $this->addToMapId($tce->substNEWwithIDs);

1836  // In case of an update, order pages from the page tree correctly:

1837  if ($this->update && is_array($this->dat['header']['pagetree'])) {

1838  $this->writeRecords_pages_order($pid);

1839  }

1840  }

1841  }

1842 

1853  public function writeRecords_pages_order($pid) {

1854  $cmd_data = array();

1855  // Get uid-pid relations and traverse them in order to map to possible new IDs

1856  $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);

1857  foreach ($pidsFromTree as $origPid => $newPid) {

1858  if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {

1859  // If the page had a new id (because it was created) use that instead!

1860  if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {

1861  if ($this->import_mapId['pages'][$origPid]) {

1862  $mappedPid = $this->import_mapId['pages'][$origPid];

1863  $cmd_data['pages'][$mappedPid]['move'] = $newPid;

1864  }

1865  } else {

1866  $cmd_data['pages'][$origPid]['move'] = $newPid;

1867  }

1868  }

1869  }

1870  // Execute the move commands if any:

1871  if (count($cmd_data)) {

1872  $tce = $this->getNewTCE();

1873  $this->callHook('before_writeRecordsPagesOrder', array(

1874  'tce' => &$tce,

1875  'data' => &$cmd_data

1876  ));

1877  $tce->start(array(), $cmd_data);

1878  $tce->process_cmdmap();

1879  $this->callHook('after_writeRecordsPagesOrder', array(

1880  'tce' => &$tce

1881  ));

1882  }

1883  }

1884 

1893  public function writeRecords_records($pid) {

1894  // Write the rest of the records

1895  $this->import_data = array();

1896  if (is_array($this->dat['header']['records'])) {

1897  foreach ($this->dat['header']['records'] as $table => $recs) {

1898  $this->addGeneralErrorsByTable($table);

1899  if ($table != 'pages') {

1900  foreach ($recs as $uid => $thisRec) {

1901  // PID: Set the main $pid, unless a NEW-id is found

1902  $setPid = isset($this->import_mapId['pages'][$thisRec['pid']])

1903  ? (int)$this->import_mapId['pages'][$thisRec['pid']]

1904  : (int)$pid;

1905  if (is_array($GLOBALS['TCA'][$table]) && isset($GLOBALS['TCA'][$table]['ctrl']['rootLevel'])) {

1906  $rootLevelSetting = (int)$GLOBALS['TCA'][$table]['ctrl']['rootLevel'];

1907  if ($rootLevelSetting === 1) {

1908  $setPid = 0;

1909  } elseif ($rootLevelSetting === 0 && $setPid === 0) {

1910  $this->error('Error: Record type ' . $table . ' is not allowed on pid 0');

1911  continue;

1912  }

1913  }

1914  // Add record:

1915  $this->addSingle($table, $uid, $setPid);

1916  }

1917  }

1918  }

1919  } else {

1920  $this->error('Error: No records defined in internal data array.');

1921  }

1922  // Now write to database:

1923  $tce = $this->getNewTCE();

1924  $this->callHook('before_writeRecordsRecords', array(

1925  'tce' => &$tce,

1926  'data' => &$this->import_data

1927  ));

1928  $tce->suggestedInsertUids = $this->suggestedInsertUids;

1929  // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.

1930  $tce->reverseOrder = 1;

1931  $tce->isImporting = TRUE;

1932  $tce->start($this->import_data, array());

1933  $tce->process_datamap();

1934  $this->callHook('after_writeRecordsRecords', array(

1935  'tce' => &$tce

1936  ));

1937  // post-processing: Removing files and registering new ids (end all tcemain sessions with this)

1938  $this->addToMapId($tce->substNEWwithIDs);

1939  // In case of an update, order pages from the page tree correctly:

1940  if ($this->update) {

1941  $this->writeRecords_records_order($pid);

1942  }

1943  }

1944 

1955  public function writeRecords_records_order($mainPid) {

1956  $cmd_data = array();

1957  if (is_array($this->dat['header']['pagetree'])) {

1958  $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);

1959  } else {

1960  $pagesFromTree = array();

1961  }

1962  if (is_array($this->dat['header']['pid_lookup'])) {

1963  foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {

1964  $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;

1965  if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($newPid)) {

1966  foreach ($recList as $tableName => $uidList) {

1967  // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!

1968  // (they will not be in the page tree!)

1969  if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {

1970  $uidList = array_reverse(array_keys($uidList));

1971  foreach ($uidList as $uid) {

1972  if ($this->dontIgnorePid($tableName, $uid)) {

1973  $cmd_data[$tableName][$uid]['move'] = $newPid;

1974  } else {

1975 

1976  }

1977  }

1978  }

1979  }

1980  }

1981  }

1982  }

1983  // Execute the move commands if any:

1984  if (count($cmd_data)) {

1985  $tce = $this->getNewTCE();

1986  $this->callHook('before_writeRecordsRecordsOrder', array(

1987  'tce' => &$tce,

1988  'data' => &$cmd_data

1989  ));

1990  $tce->start(array(), $cmd_data);

1991  $tce->process_cmdmap();

1992  $this->callHook('after_writeRecordsRecordsOrder', array(

1993  'tce' => &$tce

1994  ));

1995  }

1996  }

1997 

2009  public function addSingle($table, $uid, $pid) {

2010  if ($this->import_mode[$table . ':' . $uid] !== 'exclude') {

2011  $record = $this->dat['records'][$table . ':' . $uid]['data'];

2012  if (is_array($record)) {

2013  if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {

2014  $ID = $uid;

2015  } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {

2016  // on adding sys_file records the belonging sys_file_metadata record was also created

2017  // if there is one the record need to be overwritten instead of creating a new one.

2018  $recordInDatabase = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow(

2019  'uid',

2020  'sys_file_metadata',

2021  'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'

2022  );

2023  // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing

2024  // to a file, that was already there, thus a new metadata record should be created

2025  if (is_array($recordInDatabase)) {

2026  $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];

2027  $ID = $recordInDatabase['uid'];

2028  } else {

2029  $ID = uniqid('NEW', TRUE);

2030  }

2031 

2032  } else {

2033  $ID = uniqid('NEW', TRUE);

2034  }

2035  $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);

2036  if ($table == 'pages') {

2037  $this->import_newId_pids[$uid] = $ID;

2038  }

2039  // Set main record data:

2040  $this->import_data[$table][$ID] = $record;

2041  $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];

2042  // Reset permission data:

2043  if ($table === 'pages') {

2044  // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!

2045  unset($this->import_data[$table][$ID]['perms_userid']);

2046  unset($this->import_data[$table][$ID]['perms_groupid']);

2047  }

2048  // PID and UID:

2049  unset($this->import_data[$table][$ID]['uid']);

2050  // Updates:

2051  if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($ID)) {

2052  unset($this->import_data[$table][$ID]['pid']);

2053  } else {

2054  // Inserts:

2055  $this->import_data[$table][$ID]['pid'] = $pid;

2056  if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $GLOBALS['BE_USER']->isAdmin()) {

2057  $this->import_data[$table][$ID]['uid'] = $uid;

2058  $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';

2059  }

2060  }

2061  // Setting db/file blank:

2062  foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {

2063  switch ((string) $config['type']) {

2064  case 'db':

2065 

2066  case 'file':

2067  // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]

2068  // In the meantime we set NO values for relations.

2069  //

2070  // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,

2071  // because the value is already the uid of the right imported sys_file record.

2072  // @see fixUidLocalInSysFileReferenceRecords()

2073  // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or

2074  // delete the reference record if the file extension of the related record doesn't match.

2075  if ($table !== 'sys_file_reference' && $field !== 'uid_local') {

2076  $this->import_data[$table][$ID][$field] = '';

2077  }

2078  break;

2079  case 'flex':

2080  // Fixed later in setFlexFormRelations()

2081  // In the meantime we set NO value for flexforms - this is mainly because file references inside will not be processed properly; In fact references will point to no file or existing files (in which case there will be double-references which is a big problem of course!)

2082  $this->import_data[$table][$ID][$field] = '';

2083  break;

2084  }

2085  }

2086  } elseif ($table . ':' . $uid != 'pages:0') {

2087  // On root level we don't want this error message.

2088  $this->error('Error: no record was found in data array!', 1);

2089  }

2090  }

2091  }

2092 

2101  public function addToMapId($substNEWwithIDs) {

2102  foreach ($this->import_data as $table => $recs) {

2103  foreach ($recs as $id => $value) {

2104  $old_uid = $this->import_newId[$table . ':' . $id]['uid'];

2105  if (isset($substNEWwithIDs[$id])) {

2106  $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];

2107  } elseif ($this->update) {

2108  // Map same ID to same ID....

2109  $this->import_mapId[$table][$old_uid] = $id;

2110  } else {

2111  // if $this->import_mapId contains already the right mapping, skip the error msg. See special handling of sys_file_metadata in addSingle() => nothing to do

2112  if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {

2113  $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!', 1);

2114  }

2115 

2116  }

2117  }

2118  }

2119  }

2120 

2127  public function getNewTCE() {

2128  $tce = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');

2129  $tce->stripslashes_values = 0;

2130  $tce->dontProcessTransformations = 1;

2131  $tce->enableLogging = $this->enableLogging;

2132  $tce->alternativeFileName = $this->alternativeFileName;

2133  $tce->alternativeFilePath = $this->alternativeFilePath;

2134  return $tce;

2135  }

2136 

2143  public function unlinkTempFiles() {

2144  foreach ($this->unlinkFiles as $fileName) {

2145  if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {

2146  GeneralUtility::unlink_tempfile($fileName);

2147  clearstatcache();

2148  if (is_file($fileName)) {

2149  $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!', 1);

2150  }

2151  } else {

2152  $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!', 1);

2153  }

2154  }

2155  $this->unlinkFiles = array();

2156  }

2157 

2158  /***************************

2159  * Import / Relations setting

2160  ***************************/

2161 

2171  public function setRelations() {

2172  $updateData = array();

2173  // import_newId contains a register of all records that was in the import memorys "records" key

2174  foreach ($this->import_newId as $nId => $dat) {

2175  $table = $dat['table'];

2176  $uid = $dat['uid'];

2177  // original UID - NOT the new one!

2178  // If the record has been written and received a new id, then proceed:

2179  if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {

2180  $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);

2181  if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {

2182  if ($this->legacyImport) {

2183  if ($table != 'pages') {

2184  $oldPid = $this->dat['records'][$table . ':' . $uid]['data']['pid'];

2185  $thisNewPageUid = \TYPO3\CMS\Backend\Utility\BackendUtility::wsMapId($table, $this->import_mapId['pages'][$oldPid]);

2186  } else {

2187  $thisNewPageUid = $thisNewUid;

2188  }

2189  }

2190  // Traverse relation fields of each record

2191  foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {

2192  // uid_local of sys_file_reference needs no update because the correct reference uid was already written

2193  // @see ImportExport::fixUidLocalInSysFileReferenceRecords()

2194  if ($table === 'sys_file_reference' && $field === 'uid_local') {

2195  continue;

2196  }

2197  switch ((string) $config['type']) {

2198  case 'db':

2199  if (is_array($config['itemArray']) && count($config['itemArray'])) {

2200  $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];

2201  $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);

2202  $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);

2203  }

2204  break;

2205  case 'file':

2206  if (is_array($config['newValueFiles']) && count($config['newValueFiles'])) {

2207  $valArr = array();

2208  foreach ($config['newValueFiles'] as $fI) {

2209  $valArr[] = $this->import_addFileNameToBeCopied($fI);

2210  }

2211  if ($this->legacyImport && $this->legacyImportFolder === NULL && isset($this->legacyImportMigrationTables[$table][$field])) {

2212  // Do nothing - the legacy import folder is missing

2213  } elseif ($this->legacyImport && $this->legacyImportFolder !== NULL && isset($this->legacyImportMigrationTables[$table][$field])) {

2214  $refIds = array();

2215  foreach ($valArr as $tempFile) {

2216  $fileName = $this->alternativeFileName[$tempFile];

2217  $fileObject = NULL;

2218 

2219  try {

2220  // check, if there is alreay the same file in the folder

2221  if ($this->legacyImportFolder->hasFile($fileName)) {

2222  $fileStorage = $this->legacyImportFolder->getStorage();

2223  $file = $fileStorage->getFile($this->legacyImportFolder->getIdentifier() . $fileName);

2224  if ($file->getSha1() === sha1_file($tempFile)) {

2225  $fileObject = $file;

2226  }

2227  }

2228  } catch (Exception $e) {}

2229 

2230  if ($fileObject === NULL) {

2231  try {

2232  $fileObject = $this->legacyImportFolder->addFile($tempFile, $fileName, 'changeName');

2233  } catch (\TYPO3\CMS\Core\Exception $e) {

2234  $this->error('Error: no file could be added to the storage for file name' . $this->alternativeFileName[$tempFile]);

2235  }

2236  }

2237  if ($fileObject !== NULL) {

2238  $refId = uniqid('NEW', TRUE);

2239  $refIds[] = $refId;

2240  $updateData['sys_file_reference'][$refId] = array(

2241  'uid_local' => $fileObject->getUid(),

2242  'uid_foreign' => $thisNewUid, // uid of your content record

2243  'tablenames' => $table,

2244  'fieldname' => $field,

2245  'pid' => $thisNewPageUid, // parent id of the parent page

2246  'table_local' => 'sys_file',

2247  );

2248  }

2249  }

2250  $updateData[$table][$thisNewUid][$field] = implode(',', $refIds);

2251  if (!empty($this->legacyImportMigrationTables[$table][$field])) {

2252  $this->legacyImportMigrationRecords[$table][$thisNewUid][$field] = $refIds;

2253  }

2254  } else {

2255  $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);

2256  }

2257  }

2258  break;

2259  }

2260  }

2261  } else {

2262  $this->error('Error: no record was found in data array!', 1);

2263  }

2264  } else {

2265  $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);

2266  }

2267  }

2268  if (count($updateData)) {

2269  $tce = $this->getNewTCE();

2270  $tce->isImporting = TRUE;

2271  $this->callHook('before_setRelation', array(

2272  'tce' => &$tce,

2273  'data' => &$updateData

2274  ));

2275  $tce->start($updateData, array());

2276  $tce->process_datamap();

2277  // Replace the temporary "NEW" ids with the final ones.

2278  foreach ($this->legacyImportMigrationRecords as $table => $records) {

2279  foreach ($records as $uid => $fields) {

2280  foreach ($fields as $field => $referenceIds) {

2281  foreach ($referenceIds as $key => $referenceId) {

2282  $this->legacyImportMigrationRecords[$table][$uid][$field][$key] = $tce->substNEWwithIDs[$referenceId];

2283  }

2284  }

2285  }

2286  }

2287  $this->callHook('after_setRelations', array(

2288  'tce' => &$tce

2289  ));

2290  }

2291  }

2292 

2301  public function setRelations_db($itemArray, $itemConfig) {

2302  $valArray = array();

2303  foreach ($itemArray as $relDat) {

2304  if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {

2305  // Since non FAL file relation type group internal_type file_reference are handled as reference to

2306  // sys_file records Datahandler requires the value as uid of the the related sys_file record only

2307  if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {

2308  $value = $this->import_mapId[$relDat['table']][$relDat['id']];

2309  } elseif ($itemConfig['type'] === 'input' && isset($itemConfig['wizards']['link'])) {

2310  // If an input field has a relation to a sys_file record this need to be converted back to

2311  // the public path. But use getPublicUrl here, because could normally only be a local file path.

2312  $fileUid = $this->import_mapId[$relDat['table']][$relDat['id']];

2313  // Fallback value

2314  $value = 'file:' . $fileUid;

2315  try {

2316  $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->retrieveFileOrFolderObject($fileUid);

2317  } catch (\Exception $e) {}

2318  if ($file instanceof \TYPO3\CMS\Core\Resource\FileInterface) {

2319  $value = $file->getPublicUrl();

2320  }

2321  } else {

2322  $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];

2323  }

2324  $valArray[] = $value;

2325  } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {

2326  // Checking for less than zero because some select types could contain negative values, eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.

2327  $valArray[] = $relDat['table'] . '_' . $relDat['id'];

2328  } else {

2329  $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id'], 1);

2330  }

2331  }

2332  return $valArray;

2333  }

2334 

2342  public function import_addFileNameToBeCopied($fI) {

2343  if (is_array($this->dat['files'][$fI['ID']])) {

2344  $tmpFile = GeneralUtility::tempnam('import_temp_');

2345  GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);

2346  clearstatcache();

2347  if (@is_file($tmpFile)) {

2348  $this->unlinkFiles[] = $tmpFile;

2349  if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {

2350  $this->alternativeFileName[$tmpFile] = $fI['filename'];

2351  $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];

2352  return $tmpFile;

2353  } else {

2354  $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')', 1);

2355  }

2356  } else {

2357  $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!', 1);

2358  }

2359  } else {

2360  $this->error('Error: No file found for ID ' . $fI['ID'], 1);

2361  }

2362  }

2363 

2372  public function setFlexFormRelations() {

2373  $updateData = array();

2374  // import_newId contains a register of all records that was in the import memorys "records" key

2375  foreach ($this->import_newId as $nId => $dat) {

2376  $table = $dat['table'];

2377  $uid = $dat['uid'];

2378  // original UID - NOT the new one!

2379  // If the record has been written and received a new id, then proceed:

2380  if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {

2381  $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);

2382  if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {

2383  // Traverse relation fields of each record

2384  foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {

2385  switch ((string) $config['type']) {

2386  case 'flex':

2387  // Get XML content and set as default value (string, non-processed):

2388  $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];

2389  // If there has been registered relations inside the flex form field, run processing on the content:

2390  if (count($config['flexFormRels']['db']) || count($config['flexFormRels']['file'])) {

2391  $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');

2392  // This will fetch the new row for the element (which should be updated with any references to data structures etc.)

2393  $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];

2394  if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {

2395  // Get current data structure and value array:

2396  $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);

2397  $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);

2398  // Do recursive processing of the XML data:

2399  $iteratorObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');

2400  $iteratorObj->callBackObj = $this;

2401  $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $thisNewUid, $field, $config), 'remapListedDBRecords_flexFormCallBack');

2402  // The return value is set as an array which means it will be processed by tcemain for file and DB references!

2403  if (is_array($currentValueArray['data'])) {

2404  $updateData[$table][$thisNewUid][$field] = $currentValueArray;

2405  }

2406  }

2407  }

2408  break;

2409  }

2410  }

2411  } else {

2412  $this->error('Error: no record was found in data array!', 1);

2413  }

2414  } else {

2415  $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);

2416  }

2417  }

2418  if (count($updateData)) {

2419  $tce = $this->getNewTCE();

2420  $tce->isImporting = TRUE;

2421  $this->callHook('before_setFlexFormRelations', array(

2422  'tce' => &$tce,

2423  'data' => &$updateData

2424  ));

2425  $tce->start($updateData, array());

2426  $tce->process_datamap();

2427  $this->callHook('after_setFlexFormRelations', array(

2428  'tce' => &$tce

2429  ));

2430  }

2431  }

2432 

2446  public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {

2447  // Extract parameters:

2448  list($table, $uid, $field, $config) = $pParams;

2449  // In case the $path is used as index without a trailing slash we will remove that

2450  if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {

2451  $path = rtrim($path, '/');

2452  }

2453  if (is_array($config['flexFormRels']['db'][$path])) {