TYPO3CMS  8
 All Classes Namespaces Files Functions Variables Pages
Import.php
Go to the documentation of this file.
1 <?php
2 namespace TYPO3\CMS\Impexp;
3 
4 /*
5  * This file is part of the TYPO3 CMS project.
6  *
7  * It is free software; you can redistribute it and/or modify it under
8  * the terms of the GNU General Public License, either version 2
9  * of the License, or any later version.
10  *
11  * For the full copyright and license information, please read the
12  * LICENSE.txt file that was distributed with this source code.
13  *
14  * The TYPO3 project - inspiring people to share!
15  */
16 
33 
37 class Import extends ImportExport
38 {
45  public $suggestedInsertUids = [];
46 
52  public $enableLogging = false;
53 
61  public $import_newId = [];
62 
68  public $import_newId_pids = [];
69 
75  public $import_data = [];
76 
82  protected $storageObjects = [];
83 
87  protected $filesPathForImport = null;
88 
92  protected $unlinkFiles = [];
93 
97  protected $alternativeFileName = [];
98 
102  protected $alternativeFilePath = [];
103 
107  protected $filePathMap = [];
108 
109  /**************************
110  * Initialize
111  *************************/
112 
118  public function init()
119  {
120  parent::init();
121  $this->mode = 'import';
122  }
123 
124  /***********************
125  * Import
126  ***********************/
127 
133  protected function initializeImport()
134  {
135  // Set this flag to indicate that an import is being/has been done.
136  $this->doesImport = 1;
137  // Initialize:
138  // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
139  $this->import_mapId = [];
140  $this->import_newId = [];
141  $this->import_newId_pids = [];
142  // Temporary files stack initialized:
143  $this->unlinkFiles = [];
144  $this->alternativeFileName = [];
145  $this->alternativeFilePath = [];
146 
147  $this->initializeStorageObjects();
148  }
149 
155  protected function initializeStorageObjects()
156  {
158  $storageRepository = GeneralUtility::makeInstance(StorageRepository::class);
159  $this->storageObjects = $storageRepository->findAll();
160  }
161 
168  public function importData($pid)
169  {
170  $this->initializeImport();
171 
172  // Write sys_file_storages first
174  // Write sys_file records and write the binary file data
175  $this->writeSysFileRecords();
176  // Write records, first pages, then the rest
177  // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
178  $this->writeRecords_pages($pid);
179  $this->writeRecords_records($pid);
180  // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
181  // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
182  $this->setRelations();
183  // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
184  $this->setFlexFormRelations();
185  // Unlink temporary files:
186  $this->unlinkTempFiles();
187  // Finally, traverse all records and process softreferences with substitution attributes.
188  $this->processSoftReferences();
189  }
190 
196  protected function writeSysFileStorageRecords()
197  {
198  if (!isset($this->dat['header']['records']['sys_file_storage'])) {
199  return;
200  }
201  $sysFileStorageUidsToBeResetToDefaultStorage = [];
202  foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
203  $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
204  // continue with Local, writable and online storage only
205  if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
206  foreach ($this->storageObjects as $localStorage) {
207  if ($this->isEquivalentObjectStorage($localStorage, $storageRecord)) {
208  $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $localStorage->getUid();
209  break;
210  }
211  }
212 
213  if (!isset($this->import_mapId['sys_file_storage'][$sysFileStorageUid])) {
214  // Local, writable and online storage. Is allowed to be used to later write files in.
215  // Does currently not exist so add the record.
216  $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
217  }
218  } else {
219  // Storage with non Local drivers could be imported but must not be used to saves files in, because you
220  // could not be sure, that this is supported. The default storage will be used in this case.
221  // It could happen that non writable and non online storage will be created as dupes because you could not
222  // check the detailed configuration options at this point
223  $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
224  $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
225  }
226  }
227 
228  // Importing the added ones
229  $tce = $this->getNewTCE();
230  // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
231  $tce->reverseOrder = 1;
232  $tce->isImporting = true;
233  $tce->start($this->import_data, []);
234  $tce->process_datamap();
235  $this->addToMapId($tce->substNEWwithIDs);
236 
237  $defaultStorageUid = null;
238  // get default storage
239  $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
240  if ($defaultStorage !== null) {
241  $defaultStorageUid = $defaultStorage->getUid();
242  }
243  foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
244  $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
245  }
246 
247  // unset the sys_file_storage records to prevent an import in writeRecords_records
248  unset($this->dat['header']['records']['sys_file_storage']);
249  }
250 
259  protected function isEquivalentObjectStorage(ResourceStorage $storageObject, array $storageRecord)
260  {
261  // compare the properties: driver, writable and online
262  if ($storageObject->getDriverType() === $storageRecord['driver']
263  && (bool)$storageObject->isWritable() === (bool)$storageRecord['is_writable']
264  && (bool)$storageObject->isOnline() === (bool)$storageRecord['is_online']
265  ) {
266  $storageRecordConfiguration = ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
267  $storageObjectConfiguration = $storageObject->getConfiguration();
268  // compare the properties: pathType and basePath
269  if ($storageRecordConfiguration['pathType'] === $storageObjectConfiguration['pathType']
270  && $storageRecordConfiguration['basePath'] === $storageObjectConfiguration['basePath']
271  ) {
272  return true;
273  }
274  }
275  return false;
276  }
277 
283  public function checkImportPrerequisites()
284  {
285  $messages = [];
286 
287  // Check #1: Extension dependencies
288  $extKeysToInstall = [];
289  foreach ($this->dat['header']['extensionDependencies'] as $extKey) {
290  if (!empty($extKey) && !ExtensionManagementUtility::isLoaded($extKey)) {
291  $extKeysToInstall[] = $extKey;
292  }
293  }
294  if (!empty($extKeysToInstall)) {
295  $messages['missingExtensions'] = 'Before you can install this T3D file you need to install the extensions "'
296  . implode('", "', $extKeysToInstall) . '".';
297  }
298 
299  // Check #2: If the path for every local storage object exists.
300  // Else files can't get moved into a newly imported storage.
301  if (!empty($this->dat['header']['records']['sys_file_storage'])) {
302  foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
303  $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
304  // continue with Local, writable and online storage only
305  if ($storageRecord['driver'] === 'Local'
306  && $storageRecord['is_writable']
307  && $storageRecord['is_online']
308  ) {
309  foreach ($this->storageObjects as $localStorage) {
310  if ($this->isEquivalentObjectStorage($localStorage, $storageRecord)) {
311  // There is already an existing storage
312  break;
313  }
314 
315  // The storage from the import does not have an equivalent storage
316  // in the current instance (same driver, same path, etc.). Before
317  // the storage record can get inserted later on take care the path
318  // it points to really exists and is accessible.
319  $storageRecordUid = $storageRecord['uid'];
320  // Unset the storage record UID when trying to create the storage object
321  // as the record does not already exist in DB. The constructor of the
322  // storage object will check whether the target folder exists and set the
323  // isOnline flag depending on the outcome.
324  $storageRecord['uid'] = 0;
325  $resourceStorage = ResourceFactory::getInstance()->createStorageObject($storageRecord);
326  if (!$resourceStorage->isOnline()) {
327  $configuration = $resourceStorage->getConfiguration();
328  $messages['resourceStorageFolderMissing_' . $storageRecordUid] =
329  'The resource storage "'
330  . $resourceStorage->getName()
331  . $configuration['basePath']
332  . '" does not exist. Please create the directory prior to starting the import!';
333  }
334  }
335  }
336  }
337  }
338 
339  return $messages;
340  }
341 
347  protected function writeSysFileRecords()
348  {
349  if (!isset($this->dat['header']['records']['sys_file'])) {
350  return;
351  }
352  $this->addGeneralErrorsByTable('sys_file');
353 
354  // fetch fresh storage records from database
355  $storageRecords = $this->fetchStorageRecords();
356 
357  $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
358 
359  $sanitizedFolderMappings = [];
360 
361  foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
362  $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
363 
364  $temporaryFile = null;
365  // check if there is the right file already in the local folder
366  if ($this->filesPathForImport !== null) {
367  if (is_file($this->filesPathForImport . '/' . $fileRecord['sha1']) && sha1_file($this->filesPathForImport . '/' . $fileRecord['sha1']) === $fileRecord['sha1']) {
368  $temporaryFile = $this->filesPathForImport . '/' . $fileRecord['sha1'];
369  }
370  }
371 
372  // save file to disk
373  if ($temporaryFile === null) {
374  $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
375  $temporaryFile = $this->writeTemporaryFileFromData($fileId);
376  if ($temporaryFile === null) {
377  // error on writing the file. Error message was already added
378  continue;
379  }
380  }
381 
382  $originalStorageUid = $fileRecord['storage'];
383  $useStorageFromStorageRecords = false;
384 
385  // replace storage id, if an alternative one was registered
386  if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
387  $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
388  $useStorageFromStorageRecords = true;
389  }
390 
391  if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {
392  // no storage for the file is defined, mostly because of a missing default storage.
393  $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
394  continue;
395  }
396 
397  // using a storage from the local storage is only allowed, if the uid is present in the
398  // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
399  if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
401  $storage = ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
402  } elseif ($this->isFallbackStorage($fileRecord['storage'])) {
403  $storage = ResourceFactory::getInstance()->getStorageObject(0);
404  } elseif ($defaultStorage !== null) {
405  $storage = $defaultStorage;
406  } else {
407  $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
408  continue;
409  }
410 
411  $newFile = null;
412 
413  // check, if there is an identical file
414  try {
415  if ($storage->hasFile($fileRecord['identifier'])) {
416  $file = $storage->getFile($fileRecord['identifier']);
417  if ($file->getSha1() === $fileRecord['sha1']) {
418  $newFile = $file;
419  }
420  }
421  } catch (Exception $e) {
422  }
423 
424  if ($newFile === null) {
425  $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
426  if (in_array($folderName, $sanitizedFolderMappings)) {
427  $folderName = $sanitizedFolderMappings[$folderName];
428  }
429  if (!$storage->hasFolder($folderName)) {
430  try {
431  $importFolder = $storage->createFolder($folderName);
432  if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
433  $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
434  }
435  } catch (Exception $e) {
436  $this->error('Error: Folder "' . $folderName . '" could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
437  continue;
438  }
439  } else {
440  $importFolder = $storage->getFolder($folderName);
441  }
442 
443  try {
445  $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
446  } catch (Exception $e) {
447  $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
448  continue;
449  }
450 
451  if ($newFile->getSha1() !== $fileRecord['sha1']) {
452  $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
453  }
454  }
455 
456  // save the new uid in the import id map
457  $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
458  $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
459  }
460 
461  // unset the sys_file records to prevent an import in writeRecords_records
462  unset($this->dat['header']['records']['sys_file']);
463  // remove all sys_file_reference records that point to file records which are unknown
464  // in the system to prevent exceptions
466  }
467 
475  {
476  if (!isset($this->dat['header']['records']['sys_file_reference'])) {
477  return;
478  }
479 
480  foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
481  $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
482  if (!in_array($fileReferenceRecord['uid_local'], $this->import_mapId['sys_file'])) {
483  unset($this->dat['header']['records']['sys_file_reference'][$sysFileReferenceUid]);
484  unset($this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]);
485  $this->error('Error: sys_file_reference record ' . (int)$sysFileReferenceUid
486  . ' with relation to sys_file record ' . (int)$fileReferenceRecord['uid_local']
487  . ', which is not part of the import data, was not imported.'
488  );
489  }
490  }
491  }
492 
499  protected function isFallbackStorage($storageId)
500  {
501  return $storageId === 0 || $storageId === '0';
502  }
503 
520  protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid)
521  {
522  if (!isset($this->dat['header']['records']['sys_file_reference'])) {
523  return;
524  }
525 
526  foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
527  $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
528  if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
529  $fileReferenceRecord['uid_local'] = $newFileUid;
530  $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
531  }
532  }
533  }
534 
541  protected function fetchStorageRecords()
542  {
543  $result = GeneralUtility::makeInstance(ConnectionPool::class)
544  ->getQueryBuilderForTable('sys_file_storage')
545  ->select('*')
546  ->from('sys_file_storage')
547  ->orderBy('uid')
548  ->execute();
549  $rows = [];
550  while ($row = $result->fetch()) {
551  $rows[$row['uid']] = $row;
552  }
553  return $rows;
554  }
555 
563  protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal')
564  {
565  $temporaryFilePath = null;
566  if (is_array($this->dat[$dataKey][$fileId])) {
567  $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
568  GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
569  clearstatcache();
570  if (@is_file($temporaryFilePathInternal)) {
571  $this->unlinkFiles[] = $temporaryFilePathInternal;
572  if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
573  $temporaryFilePath = $temporaryFilePathInternal;
574  } else {
575  $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')');
576  }
577  } else {
578  $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!');
579  }
580  } else {
581  $this->error('Error: No file found for ID ' . $fileId);
582  }
583  return $temporaryFilePath;
584  }
585 
593  public function writeRecords_pages($pid)
594  {
595  // First, write page structure if any:
596  if (is_array($this->dat['header']['records']['pages'])) {
597  $this->addGeneralErrorsByTable('pages');
598  // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
599  $pageRecords = $this->dat['header']['records']['pages'];
600  $this->import_data = [];
601  // First add page tree if any
602  if (is_array($this->dat['header']['pagetree'])) {
603  $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
604  foreach ($pagesFromTree as $uid) {
605  $thisRec = $this->dat['header']['records']['pages'][$uid];
606  // PID: Set the main $pid, unless a NEW-id is found
607  $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
608  $this->addSingle('pages', $uid, $setPid);
609  unset($pageRecords[$uid]);
610  }
611  }
612  // Then add all remaining pages not in tree on root level:
613  if (!empty($pageRecords)) {
614  $remainingPageUids = array_keys($pageRecords);
615  foreach ($remainingPageUids as $pUid) {
616  $this->addSingle('pages', $pUid, $pid);
617  }
618  }
619  // Now write to database:
620  $tce = $this->getNewTCE();
621  $tce->isImporting = true;
622  $this->callHook('before_writeRecordsPages', [
623  'tce' => &$tce,
624  'data' => &$this->import_data
625  ]);
626  $tce->suggestedInsertUids = $this->suggestedInsertUids;
627  $tce->start($this->import_data, []);
628  $tce->process_datamap();
629  $this->callHook('after_writeRecordsPages', [
630  'tce' => &$tce
631  ]);
632  // post-processing: Registering new ids (end all DataHandler sessions with this)
633  $this->addToMapId($tce->substNEWwithIDs);
634  // In case of an update, order pages from the page tree correctly:
635  if ($this->update && is_array($this->dat['header']['pagetree'])) {
636  $this->writeRecords_pages_order();
637  }
638  }
639  }
640 
649  public function writeRecords_pages_order()
650  {
651  $cmd_data = [];
652  // Get uid-pid relations and traverse them in order to map to possible new IDs
653  $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
654  foreach ($pidsFromTree as $origPid => $newPid) {
655  if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
656  // If the page had a new id (because it was created) use that instead!
657  if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
658  if ($this->import_mapId['pages'][$origPid]) {
659  $mappedPid = $this->import_mapId['pages'][$origPid];
660  $cmd_data['pages'][$mappedPid]['move'] = $newPid;
661  }
662  } else {
663  $cmd_data['pages'][$origPid]['move'] = $newPid;
664  }
665  }
666  }
667  // Execute the move commands if any:
668  if (!empty($cmd_data)) {
669  $tce = $this->getNewTCE();
670  $this->callHook('before_writeRecordsPagesOrder', [
671  'tce' => &$tce,
672  'data' => &$cmd_data
673  ]);
674  $tce->start([], $cmd_data);
675  $tce->process_cmdmap();
676  $this->callHook('after_writeRecordsPagesOrder', [
677  'tce' => &$tce
678  ]);
679  }
680  }
681 
691  public function flatInversePageTree_pid($idH, $a = [], $pid = -1)
692  {
693  if (is_array($idH)) {
694  $idH = array_reverse($idH);
695  foreach ($idH as $v) {
696  $a[$v['uid']] = $pid;
697  if (is_array($v['subrow'])) {
698  $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
699  }
700  }
701  }
702  return $a;
703  }
704 
712  public function writeRecords_records($pid)
713  {
714  // Write the rest of the records
715  $this->import_data = [];
716  if (is_array($this->dat['header']['records'])) {
717  foreach ($this->dat['header']['records'] as $table => $recs) {
718  $this->addGeneralErrorsByTable($table);
719  if ($table != 'pages') {
720  foreach ($recs as $uid => $thisRec) {
721  // PID: Set the main $pid, unless a NEW-id is found
722  $setPid = isset($this->import_mapId['pages'][$thisRec['pid']])
723  ? (int)$this->import_mapId['pages'][$thisRec['pid']]
724  : (int)$pid;
725  if (is_array($GLOBALS['TCA'][$table]) && isset($GLOBALS['TCA'][$table]['ctrl']['rootLevel'])) {
726  $rootLevelSetting = (int)$GLOBALS['TCA'][$table]['ctrl']['rootLevel'];
727  if ($rootLevelSetting === 1) {
728  $setPid = 0;
729  } elseif ($rootLevelSetting === 0 && $setPid === 0) {
730  $this->error('Error: Record type ' . $table . ' is not allowed on pid 0');
731  continue;
732  }
733  }
734  // Add record:
735  $this->addSingle($table, $uid, $setPid);
736  }
737  }
738  }
739  } else {
740  $this->error('Error: No records defined in internal data array.');
741  }
742  // Now write to database:
743  $tce = $this->getNewTCE();
744  $this->callHook('before_writeRecordsRecords', [
745  'tce' => &$tce,
746  'data' => &$this->import_data
747  ]);
748  $tce->suggestedInsertUids = $this->suggestedInsertUids;
749  // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
750  $tce->reverseOrder = 1;
751  $tce->isImporting = true;
752  $tce->start($this->import_data, []);
753  $tce->process_datamap();
754  $this->callHook('after_writeRecordsRecords', [
755  'tce' => &$tce
756  ]);
757  // post-processing: Removing files and registering new ids (end all DataHandler sessions with this)
758  $this->addToMapId($tce->substNEWwithIDs);
759  // In case of an update, order pages from the page tree correctly:
760  if ($this->update) {
761  $this->writeRecords_records_order($pid);
762  }
763  }
764 
774  public function writeRecords_records_order($mainPid)
775  {
776  $cmd_data = [];
777  if (is_array($this->dat['header']['pagetree'])) {
778  $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
779  } else {
780  $pagesFromTree = [];
781  }
782  if (is_array($this->dat['header']['pid_lookup'])) {
783  foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
784  $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
786  foreach ($recList as $tableName => $uidList) {
787  // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
788  // (they will not be in the page tree!)
789  if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
790  $uidList = array_reverse(array_keys($uidList));
791  foreach ($uidList as $uid) {
792  if ($this->dontIgnorePid($tableName, $uid)) {
793  $cmd_data[$tableName][$uid]['move'] = $newPid;
794  } else {
795  }
796  }
797  }
798  }
799  }
800  }
801  }
802  // Execute the move commands if any:
803  if (!empty($cmd_data)) {
804  $tce = $this->getNewTCE();
805  $this->callHook('before_writeRecordsRecordsOrder', [
806  'tce' => &$tce,
807  'data' => &$cmd_data
808  ]);
809  $tce->start([], $cmd_data);
810  $tce->process_cmdmap();
811  $this->callHook('after_writeRecordsRecordsOrder', [
812  'tce' => &$tce
813  ]);
814  }
815  }
816 
828  public function addSingle($table, $uid, $pid)
829  {
830  if ($this->import_mode[$table . ':' . $uid] === 'exclude') {
831  return;
832  }
833  $record = $this->dat['records'][$table . ':' . $uid]['data'];
834  if (is_array($record)) {
835  if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
836  $ID = $uid;
837  } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
838  // on adding sys_file records the belonging sys_file_metadata record was also created
839  // if there is one the record need to be overwritten instead of creating a new one.
840  $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
841  ->getQueryBuilderForTable('sys_file_metadata');
842  $recordInDatabase = $queryBuilder->select('uid')
843  ->from('sys_file_metadata')
844  ->where(
845  $queryBuilder->expr()->eq(
846  'file',
847  $queryBuilder->createNamedParameter(
848  $this->import_mapId['sys_file'][$record['file']],
849  \PDO::PARAM_INT
850  )
851  ),
852  $queryBuilder->expr()->eq(
853  'sys_language_uid',
854  $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)
855  ),
856  $queryBuilder->expr()->eq(
857  'pid',
858  $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)
859  )
860  )
861  ->execute()
862  ->fetch();
863  // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
864  // to a file, that was already there, thus a new metadata record should be created
865  if (is_array($recordInDatabase)) {
866  $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
867  $ID = $recordInDatabase['uid'];
868  } else {
869  $ID = StringUtility::getUniqueId('NEW');
870  }
871  } else {
872  $ID = StringUtility::getUniqueId('NEW');
873  }
874  $this->import_newId[$table . ':' . $ID] = ['table' => $table, 'uid' => $uid];
875  if ($table == 'pages') {
876  $this->import_newId_pids[$uid] = $ID;
877  }
878  // Set main record data:
879  $this->import_data[$table][$ID] = $record;
880  $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
881  // Reset permission data:
882  if ($table === 'pages') {
883  // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
884  unset($this->import_data[$table][$ID]['perms_userid']);
885  unset($this->import_data[$table][$ID]['perms_groupid']);
886  }
887  // PID and UID:
888  unset($this->import_data[$table][$ID]['uid']);
889  // Updates:
891  unset($this->import_data[$table][$ID]['pid']);
892  } else {
893  // Inserts:
894  $this->import_data[$table][$ID]['pid'] = $pid;
895  if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $this->getBackendUser()->isAdmin()) {
896  $this->import_data[$table][$ID]['uid'] = $uid;
897  $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
898  }
899  }
900  // Setting db/file blank:
901  foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
902  switch ((string)$config['type']) {
903  case 'db':
904 
905  case 'file':
906  // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
907  // In the meantime we set NO values for relations.
908  //
909  // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
910  // because the value is already the uid of the right imported sys_file record.
911  // @see fixUidLocalInSysFileReferenceRecords()
912  // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
913  // delete the reference record if the file extension of the related record doesn't match.
914  if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
915  $this->import_data[$table][$ID][$field] = '';
916  }
917  break;
918  case 'flex':
919  // Fixed later in setFlexFormRelations()
920  // In the meantime we set NO value for flexforms - this is mainly because file references
921  // inside will not be processed properly; In fact references will point to no file
922  // or existing files (in which case there will be double-references which is a big problem of course!)
923  $this->import_data[$table][$ID][$field] = '';
924  break;
925  }
926  }
927  } elseif ($table . ':' . $uid != 'pages:0') {
928  // On root level we don't want this error message.
929  $this->error('Error: no record was found in data array!');
930  }
931  }
932 
940  public function addToMapId($substNEWwithIDs)
941  {
942  foreach ($this->import_data as $table => $recs) {
943  foreach ($recs as $id => $value) {
944  $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
945  if (isset($substNEWwithIDs[$id])) {
946  $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
947  } elseif ($this->update) {
948  // Map same ID to same ID....
949  $this->import_mapId[$table][$old_uid] = $id;
950  } else {
951  // if $this->import_mapId contains already the right mapping, skip the error msg.
952  // See special handling of sys_file_metadata in addSingle() => nothing to do
953  if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {
954  $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!');
955  }
956  }
957  }
958  }
959  }
960 
966  public function getNewTCE()
967  {
968  $tce = GeneralUtility::makeInstance(DataHandler::class);
969  $tce->dontProcessTransformations = 1;
970  $tce->enableLogging = $this->enableLogging;
971  $tce->alternativeFileName = $this->alternativeFileName;
972  $tce->alternativeFilePath = $this->alternativeFilePath;
973  return $tce;
974  }
975 
981  public function unlinkTempFiles()
982  {
983  foreach ($this->unlinkFiles as $fileName) {
984  if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
986  clearstatcache();
987  if (is_file($fileName)) {
988  $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!');
989  }
990  } else {
991  $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!');
992  }
993  }
994  $this->unlinkFiles = [];
995  }
996 
997  /***************************
998  * Import / Relations setting
999  ***************************/
1000 
1009  public function setRelations()
1010  {
1011  $updateData = [];
1012  // import_newId contains a register of all records that was in the import memorys "records" key
1013  foreach ($this->import_newId as $nId => $dat) {
1014  $table = $dat['table'];
1015  $uid = $dat['uid'];
1016  // original UID - NOT the new one!
1017  // If the record has been written and received a new id, then proceed:
1018  if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
1019  $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1020  if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
1021  // Traverse relation fields of each record
1022  foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1023  // uid_local of sys_file_reference needs no update because the correct reference uid was already written
1024  // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
1025  if ($table === 'sys_file_reference' && $field === 'uid_local') {
1026  continue;
1027  }
1028  switch ((string)$config['type']) {
1029  case 'db':
1030  if (is_array($config['itemArray']) && !empty($config['itemArray'])) {
1031  $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
1032  $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
1033  $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
1034  }
1035  break;
1036  case 'file':
1037  if (is_array($config['newValueFiles']) && !empty($config['newValueFiles'])) {
1038  $valArr = [];
1039  foreach ($config['newValueFiles'] as $fI) {
1040  $valArr[] = $this->import_addFileNameToBeCopied($fI);
1041  }
1042  $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
1043  }
1044  break;
1045  }
1046  }
1047  } else {
1048  $this->error('Error: no record was found in data array!');
1049  }
1050  } else {
1051  $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')');
1052  }
1053  }
1054  if (!empty($updateData)) {
1055  $tce = $this->getNewTCE();
1056  $tce->isImporting = true;
1057  $this->callHook('before_setRelation', [
1058  'tce' => &$tce,
1059  'data' => &$updateData
1060  ]);
1061  $tce->start($updateData, []);
1062  $tce->process_datamap();
1063  $this->callHook('after_setRelations', [
1064  'tce' => &$tce
1065  ]);
1066  }
1067  }
1068 
1076  public function setRelations_db($itemArray, $itemConfig)
1077  {
1078  $valArray = [];
1079  foreach ($itemArray as $relDat) {
1080  if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
1081  // Since non FAL file relation type group internal_type file_reference are handled as reference to
1082  // sys_file records Datahandler requires the value as uid of the the related sys_file record only
1083  if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
1084  $value = $this->import_mapId[$relDat['table']][$relDat['id']];
1085  } elseif ($itemConfig['type'] === 'input' && isset($itemConfig['wizards']['link'])) {
1086  // If an input field has a relation to a sys_file record this need to be converted back to
1087  // the public path. But use getPublicUrl here, because could normally only be a local file path.
1088  $fileUid = $this->import_mapId[$relDat['table']][$relDat['id']];
1089  // Fallback value
1090  $value = 'file:' . $fileUid;
1091  try {
1092  $file = ResourceFactory::getInstance()->retrieveFileOrFolderObject($fileUid);
1093  } catch (\Exception $e) {
1094  $file = null;
1095  }
1096  if ($file instanceof FileInterface) {
1097  $value = $file->getPublicUrl();
1098  }
1099  } else {
1100  $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
1101  }
1102  $valArray[] = $value;
1103  } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
1104  // Checking for less than zero because some select types could contain negative values,
1105  // eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
1106  $valArray[] = $relDat['table'] . '_' . $relDat['id'];
1107  } else {
1108  $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id']);
1109  }
1110  }
1111  return $valArray;
1112  }
1113 
1120  public function import_addFileNameToBeCopied($fI)
1121  {
1122  if (is_array($this->dat['files'][$fI['ID']])) {
1123  $tmpFile = null;
1124  // check if there is the right file already in the local folder
1125  if ($this->filesPathForImport !== null) {
1126  if (is_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) &&
1127  md5_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) === $this->dat['files'][$fI['ID']]['content_md5']) {
1128  $tmpFile = $this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5'];
1129  }
1130  }
1131  if ($tmpFile === null) {
1132  $tmpFile = GeneralUtility::tempnam('import_temp_');
1133  GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
1134  }
1135  clearstatcache();
1136  if (@is_file($tmpFile)) {
1137  $this->unlinkFiles[] = $tmpFile;
1138  if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
1139  $this->alternativeFileName[$tmpFile] = $fI['filename'];
1140  $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
1141  return $tmpFile;
1142  } else {
1143  $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')');
1144  }
1145  } else {
1146  $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!');
1147  }
1148  } else {
1149  $this->error('Error: No file found for ID ' . $fI['ID']);
1150  }
1151  return null;
1152  }
1153 
1161  public function setFlexFormRelations()
1162  {
1163  $updateData = [];
1164  // import_newId contains a register of all records that was in the import memorys "records" key
1165  foreach ($this->import_newId as $nId => $dat) {
1166  $table = $dat['table'];
1167  $uid = $dat['uid'];
1168  // original UID - NOT the new one!
1169  // If the record has been written and received a new id, then proceed:
1170  if (!isset($this->import_mapId[$table][$uid])) {
1171  $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')');
1172  continue;
1173  }
1174 
1175  if (!is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
1176  $this->error('Error: no record was found in data array!');
1177  continue;
1178  }
1179  $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1180  // Traverse relation fields of each record
1181  foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1182  switch ((string)$config['type']) {
1183  case 'flex':
1184  // Get XML content and set as default value (string, non-processed):
1185  $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
1186  // If there has been registered relations inside the flex form field, run processing on the content:
1187  if (!empty($config['flexFormRels']['db']) || !empty($config['flexFormRels']['file'])) {
1188  $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
1189  // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
1190  $fieldTca = $GLOBALS['TCA'][$table]['columns'][$field];
1191  if (is_array($origRecordRow) && is_array($fieldTca['config']) && $fieldTca['config']['type'] === 'flex') {
1192  // Get current data structure and value array:
1193  $flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
1194  $dataStructureIdentifier = $flexFormTools->getDataStructureIdentifier(
1195  $fieldTca,
1196  $table,
1197  $field,
1198  $origRecordRow
1199  );
1200  $dataStructureArray = $flexFormTools->parseDataStructureByIdentifier($dataStructureIdentifier);
1201  $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
1202  // Do recursive processing of the XML data:
1203  $iteratorObj = GeneralUtility::makeInstance(DataHandler::class);
1204  $iteratorObj->callBackObj = $this;
1205  $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData(
1206  $currentValueArray['data'],
1207  [],
1208  [],
1209  $dataStructureArray,
1210  [$table, $thisNewUid, $field, $config],
1211  'remapListedDBRecords_flexFormCallBack'
1212  );
1213  // The return value is set as an array which means it will be processed by DataHandler for file and DB references!
1214  if (is_array($currentValueArray['data'])) {
1215  $updateData[$table][$thisNewUid][$field] = $currentValueArray;
1216  }
1217  }
1218  }
1219  break;
1220  }
1221  }
1222  }
1223  if (!empty($updateData)) {
1224  $tce = $this->getNewTCE();
1225  $tce->isImporting = true;
1226  $this->callHook('before_setFlexFormRelations', [
1227  'tce' => &$tce,
1228  'data' => &$updateData
1229  ]);
1230  $tce->start($updateData, []);
1231  $tce->process_datamap();
1232  $this->callHook('after_setFlexFormRelations', [
1233  'tce' => &$tce
1234  ]);
1235  }
1236  }
1237 
1250  public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
1251  {
1252  // Extract parameters:
1253  list(, , , $config) = $pParams;
1254  // In case the $path is used as index without a trailing slash we will remove that
1255  if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
1256  $path = rtrim($path, '/');
1257  }
1258  if (is_array($config['flexFormRels']['db'][$path])) {
1259  $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
1260  $dataValue = implode(',', $valArray);
1261  }
1262  if (is_array($config['flexFormRels']['file'][$path])) {
1263  $valArr = [];
1264  foreach ($config['flexFormRels']['file'][$path] as $fI) {
1265  $valArr[] = $this->import_addFileNameToBeCopied($fI);
1266  }
1267  $dataValue = implode(',', $valArr);
1268  }
1269  return ['value' => $dataValue];
1270  }
1271 
1272  /**************************
1273  * Import / Soft References
1274  *************************/
1275 
1281  public function processSoftReferences()
1282  {
1283  // Initialize:
1284  $inData = [];
1285  // Traverse records:
1286  if (is_array($this->dat['header']['records'])) {
1287  foreach ($this->dat['header']['records'] as $table => $recs) {
1288  foreach ($recs as $uid => $thisRec) {
1289  // If there are soft references defined, traverse those:
1290  if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
1291  // First traversal is to collect softref configuration and split them up based on fields.
1292  // This could probably also have been done with the "records" key instead of the header.
1293  $fieldsIndex = [];
1294  foreach ($thisRec['softrefs'] as $softrefDef) {
1295  // If a substitution token is set:
1296  if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
1297  $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
1298  }
1299  }
1300  // The new id:
1301  $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1302  // Now, if there are any fields that require substitution to be done, lets go for that:
1303  foreach ($fieldsIndex as $field => $softRefCfgs) {
1304  if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
1305  if ($GLOBALS['TCA'][$table]['columns'][$field]['config']['type'] === 'flex') {
1306  // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
1307  $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
1308  if (is_array($origRecordRow)) {
1309  // Get current data structure and value array:
1310  $flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
1311  $dataStructureIdentifier = $flexFormTools->getDataStructureIdentifier(
1312  $GLOBALS['TCA'][$table]['columns'][$field],
1313  $table,
1314  $field,
1315  $origRecordRow
1316  );
1317  $dataStructureArray = $flexFormTools->parseDataStructureByIdentifier($dataStructureIdentifier);
1318  $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
1319  // Do recursive processing of the XML data:
1321  $iteratorObj = GeneralUtility::makeInstance(DataHandler::class);
1322  $iteratorObj->callBackObj = $this;
1323  $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], [], [], $dataStructureArray, [$table, $uid, $field, $softRefCfgs], 'processSoftReferences_flexFormCallBack');
1324  // The return value is set as an array which means it will be processed by DataHandler for file and DB references!
1325  if (is_array($currentValueArray['data'])) {
1326  $inData[$table][$thisNewUid][$field] = $currentValueArray;
1327  }
1328  }
1329  } else {
1330  // Get tokenizedContent string and proceed only if that is not blank:
1331  $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
1332  if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
1333  $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
1334  }
1335  }
1336  }
1337  }
1338  }
1339  }
1340  }
1341  }
1342  // Now write to database:
1343  $tce = $this->getNewTCE();
1344  $tce->isImporting = true;
1345  $this->callHook('before_processSoftReferences', [
1346  'tce' => $tce,
1347  'data' => &$inData
1348  ]);
1349  $tce->enableLogging = true;
1350  $tce->start($inData, []);
1351  $tce->process_datamap();
1352  $this->callHook('after_processSoftReferences', [
1353  'tce' => $tce
1354  ]);
1355  }
1356 
1369  public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
1370  {
1371  // Extract parameters:
1372  list($table, $origUid, $field, $softRefCfgs) = $pParams;
1373  if (is_array($softRefCfgs)) {
1374  // First, find all soft reference configurations for this structure path (they are listed flat in the header):
1375  $thisSoftRefCfgList = [];
1376  foreach ($softRefCfgs as $sK => $sV) {
1377  if ($sV['structurePath'] === $path) {
1378  $thisSoftRefCfgList[$sK] = $sV;
1379  }
1380  }
1381  // If any was found, do processing:
1382  if (!empty($thisSoftRefCfgList)) {
1383  // Get tokenizedContent string and proceed only if that is not blank:
1384  $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
1385  if (strlen($tokenizedContent)) {
1386  $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
1387  }
1388  }
1389  }
1390  // Return
1391  return ['value' => $dataValue];
1392  }
1393 
1403  public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid)
1404  {
1405  // traverse each softref type for this field:
1406  foreach ($softRefCfgs as $cfg) {
1407  // Get token ID:
1408  $tokenID = $cfg['subst']['tokenID'];
1409  // Default is current token value:
1410  $insertValue = $cfg['subst']['tokenValue'];
1411  // Based on mode:
1412  switch ((string)$this->softrefCfg[$tokenID]['mode']) {
1413  case 'exclude':
1414  // Exclude is a simple passthrough of the value
1415  break;
1416  case 'editable':
1417  // Editable always picks up the value from this input array:
1418  $insertValue = $this->softrefInputValues[$tokenID];
1419  break;
1420  default:
1421  // Mapping IDs/creating files: Based on type, look up new value:
1422  switch ((string)$cfg['subst']['type']) {
1423  case 'file':
1424  // Create / Overwrite file:
1425  $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
1426  break;
1427  case 'db':
1428  default:
1429  // Trying to map database element if found in the mapID array:
1430  list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
1431  if (isset($this->import_mapId[$tempTable][$tempUid])) {
1432  $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
1433  // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
1434  if ($tempTable === 'pages' && !MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
1435  $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
1436  if ($recWithUniqueValue['alias']) {
1437  $insertValue = $recWithUniqueValue['alias'];
1438  }
1439  } elseif (strpos($cfg['subst']['tokenValue'], ':') !== false) {
1440  list($tokenKey) = explode(':', $cfg['subst']['tokenValue']);
1441  $insertValue = $tokenKey . ':' . $insertValue;
1442  }
1443  }
1444  }
1445  }
1446  // Finally, swap the soft reference token in tokenized content with the insert value:
1447  $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
1448  }
1449  return $tokenizedContent;
1450  }
1451 
1461  public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid)
1462  {
1463  if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
1464  // Initialize; Get directory prefix for file and find possible RTE filename
1465  $dirPrefix = PathUtility::dirname($relFileName) . '/';
1466  $rteOrigName = $this->getRTEoriginalFilename(PathUtility::basename($relFileName));
1467  // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
1468  if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
1469  // RTE:
1470  // First, find unique RTE file name:
1471  if (@is_dir((PATH_site . $dirPrefix))) {
1472  // From the "original" RTE filename, produce a new "original" destination filename which is unused.
1473  // Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
1474  $fileProcObj = $this->getFileProcObj();
1475  $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
1476  // Create copy file name:
1477  $pI = pathinfo($relFileName);
1478  $copyDestName = PathUtility::dirname($origDestName) . '/RTEmagicC_' . substr(PathUtility::basename($origDestName), 10) . '.' . $pI['extension'];
1479  if (
1480  !@is_file($copyDestName) && !@is_file($origDestName)
1481  && $origDestName === GeneralUtility::getFileAbsFileName($origDestName)
1482  && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)
1483  ) {
1484  if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
1485  // Write the copy and original RTE file to the respective filenames:
1486  $this->writeFileVerify($copyDestName, $cfg['file_ID'], true);
1487  $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], true);
1488  // Return the relative path of the copy file name:
1489  return PathUtility::stripPathSitePrefix($copyDestName);
1490  } else {
1491  $this->error('ERROR: Could not find original file ID');
1492  }
1493  } else {
1494  $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
1495  }
1496  } else {
1497  $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
1498  }
1499  } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
1500  // File in fileadmin/ folder:
1501  // Create file (and possible resources)
1502  $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
1503  if (strlen($newFileName)) {
1504  $relFileName = $newFileName;
1505  } else {
1506  $this->error('ERROR: No new file created for "' . $relFileName . '"');
1507  }
1508  } else {
1509  $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
1510  }
1511  } else {
1512  $this->error('ERROR: Could not find file ID in header.');
1513  }
1514  // Return (new) filename relative to PATH_site:
1515  return $relFileName;
1516  }
1517 
1528  public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid)
1529  {
1530  // If the fileID map contains an entry for this fileID then just return the relative filename of that entry;
1531  // we don't want to write another unique filename for this one!
1532  if (isset($this->fileIDMap[$fileID])) {
1533  return PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
1534  }
1535  // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
1536  $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
1537  if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
1538  $fileHeaderInfo = $this->dat['header']['files'][$fileID];
1539  $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
1540  // Create new name for file:
1541  // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".
1542 
1543  // Write main file:
1544  if ($updMode) {
1545  $newName = PATH_site . $dirPrefix . $fileName;
1546  } else {
1547  // Create unique filename:
1548  $fileProcObj = $this->getFileProcObj();
1549  $newName = $fileProcObj->getUniqueName($fileName, PATH_site . $dirPrefix);
1550  }
1551  if ($this->writeFileVerify($newName, $fileID)) {
1552  // If the resource was an HTML/CSS file with resources attached, we will write those as well!
1553  if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
1554  $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
1555  $tokenSubstituted = false;
1556  $fileProcObj = $this->getFileProcObj();
1557  if ($updMode) {
1558  foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
1559  if ($this->dat['files'][$res_fileID]['filename']) {
1560  // Resolve original filename:
1561  $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
1562  $absResourceFileName = GeneralUtility::resolveBackPath(PATH_site . $origDirPrefix . $relResourceFileName);
1563  $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
1564  if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, PATH_site . $this->fileadminFolderName . '/')) {
1565  $destDir = PathUtility::stripPathSitePrefix(PathUtility::dirname($absResourceFileName) . '/');
1566  if ($this->verifyFolderAccess($destDir, true) && $this->checkOrCreateDir($destDir)) {
1567  $this->writeFileVerify($absResourceFileName, $res_fileID);
1568  } else {
1569  $this->error('ERROR: Could not create file in directory "' . $destDir . '"');
1570  }
1571  } else {
1572  $this->error('ERROR: Could not resolve path for "' . $relResourceFileName . '"');
1573  }
1574  $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
1575  $tokenSubstituted = true;
1576  }
1577  }
1578  } else {
1579  // Create the resouces directory name (filename without extension, suffixed "_FILES")
1580  $resourceDir = PathUtility::dirname($newName) . '/' . preg_replace('/\\.[^.]*$/', '', PathUtility::basename($newName)) . '_FILES';
1581  if (GeneralUtility::mkdir($resourceDir)) {
1582  foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
1583  if ($this->dat['files'][$res_fileID]['filename']) {
1584  $absResourceFileName = $fileProcObj->getUniqueName($this->dat['files'][$res_fileID]['filename'], $resourceDir);
1585  $relResourceFileName = substr($absResourceFileName, strlen(PathUtility::dirname($resourceDir)) + 1);
1586  $this->writeFileVerify($absResourceFileName, $res_fileID);
1587  $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
1588  $tokenSubstituted = true;
1589  }
1590  }
1591  }
1592  }
1593  // If substitutions has been made, write the content to the file again:
1594  if ($tokenSubstituted) {
1595  GeneralUtility::writeFile($newName, $tokenizedContent);
1596  }
1597  }
1598  return PathUtility::stripPathSitePrefix($newName);
1599  }
1600  }
1601  return null;
1602  }
1603 
1612  public function writeFileVerify($fileName, $fileID, $bypassMountCheck = false)
1613  {
1614  $fileProcObj = $this->getFileProcObj();
1615  if (!$fileProcObj->actionPerms['addFile']) {
1616  $this->error('ERROR: You did not have sufficient permissions to write the file "' . $fileName . '"');
1617  return false;
1618  }
1619  // Just for security, check again. Should actually not be necessary.
1620  if (!$bypassMountCheck) {
1621  try {
1622  ResourceFactory::getInstance()->getFolderObjectFromCombinedIdentifier(dirname($fileName));
1624  $this->error('ERROR: Filename "' . $fileName . '" was not allowed in destination path!');
1625  return false;
1626  }
1627  }
1628  $fI = GeneralUtility::split_fileref($fileName);
1629  if (!$fileProcObj->checkIfAllowed($fI['fileext'], $fI['path'], $fI['file']) && (!$this->allowPHPScripts || !$this->getBackendUser()->isAdmin())) {
1630  $this->error('ERROR: Filename "' . $fileName . '" failed against extension check or deny-pattern!');
1631  return false;
1632  }
1633  if (!GeneralUtility::getFileAbsFileName($fileName)) {
1634  $this->error('ERROR: Filename "' . $fileName . '" was not a valid relative file path!');
1635  return false;
1636  }
1637  if (!$this->dat['files'][$fileID]) {
1638  $this->error('ERROR: File ID "' . $fileID . '" could not be found');
1639  return false;
1640  }
1641  GeneralUtility::writeFile($fileName, $this->dat['files'][$fileID]['content']);
1642  $this->fileIDMap[$fileID] = $fileName;
1643  if (md5(file_get_contents($fileName)) == $this->dat['files'][$fileID]['content_md5']) {
1644  return true;
1645  } else {
1646  $this->error('ERROR: File content "' . $fileName . '" was corrupted');
1647  return false;
1648  }
1649  }
1650 
1657  public function checkOrCreateDir($dirPrefix)
1658  {
1659  // Split dir path and remove first directory (which should be "fileadmin")
1660  $filePathParts = explode('/', $dirPrefix);
1661  $firstDir = array_shift($filePathParts);
1662  if ($firstDir === $this->fileadminFolderName && GeneralUtility::getFileAbsFileName($dirPrefix)) {
1663  $pathAcc = '';
1664  foreach ($filePathParts as $dirname) {
1665  $pathAcc .= '/' . $dirname;
1666  if (strlen($dirname)) {
1667  if (!@is_dir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
1668  if (!GeneralUtility::mkdir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
1669  $this->error('ERROR: Directory could not be created....B');
1670  return false;
1671  }
1672  }
1673  } elseif ($dirPrefix === $this->fileadminFolderName . $pathAcc) {
1674  return true;
1675  } else {
1676  $this->error('ERROR: Directory could not be created....A');
1677  }
1678  }
1679  }
1680  return false;
1681  }
1682 
1683  /**************************
1684  * File Input
1685  *************************/
1686 
1694  public function loadFile($filename, $all = false)
1695  {
1696  if (!@is_file($filename)) {
1697  $this->error('Filename not found: ' . $filename);
1698  return false;
1699  }
1700  $fI = pathinfo($filename);
1701  if (@is_dir($filename . '.files')) {
1702  if (GeneralUtility::isAllowedAbsPath($filename . '.files')) {
1703  // copy the folder lowlevel to typo3temp, because the files would be deleted after import
1704  $temporaryFolderName = $this->getTemporaryFolderName();
1705  GeneralUtility::copyDirectory($filename . '.files', $temporaryFolderName);
1706  $this->filesPathForImport = $temporaryFolderName;
1707  } else {
1708  $this->error('External import files for the given import source is currently not supported.');
1709  }
1710  }
1711  if (strtolower($fI['extension']) == 'xml') {
1712  // XML:
1713  $xmlContent = file_get_contents($filename);
1714  if (strlen($xmlContent)) {
1715  $this->dat = GeneralUtility::xml2array($xmlContent, '', true);
1716  if (is_array($this->dat)) {
1717  if ($this->dat['_DOCUMENT_TAG'] === 'T3RecordDocument' && is_array($this->dat['header']) && is_array($this->dat['records'])) {
1718  $this->loadInit();
1719  return true;
1720  } else {
1721  $this->error('XML file did not contain proper XML for TYPO3 Import');
1722  }
1723  } else {
1724  $this->error('XML could not be parsed: ' . $this->dat);
1725  }
1726  } else {
1727  $this->error('Error opening file: ' . $filename);
1728  }
1729  } else {
1730  // T3D
1731  if ($fd = fopen($filename, 'rb')) {
1732  $this->dat['header'] = $this->getNextFilePart($fd, 1, 'header');
1733  if ($all) {
1734  $this->dat['records'] = $this->getNextFilePart($fd, 1, 'records');
1735  $this->dat['files'] = $this->getNextFilePart($fd, 1, 'files');
1736  $this->dat['files_fal'] = $this->getNextFilePart($fd, 1, 'files_fal');
1737  }
1738  $this->loadInit();
1739  return true;
1740  } else {
1741  $this->error('Error opening file: ' . $filename);
1742  }
1743  fclose($fd);
1744  }
1745  return false;
1746  }
1747 
1758  public function getNextFilePart($fd, $unserialize = false, $name = '')
1759  {
1760  $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
1761  // Getting header data
1762  $initStr = fread($fd, $initStrLen);
1763  if (empty($initStr)) {
1764  $this->error('File does not contain data for "' . $name . '"');
1765  return null;
1766  }
1767  $initStrDat = explode(':', $initStr);
1768  if (strstr($initStrDat[0], 'Warning')) {
1769  $this->error('File read error: Warning message in file. (' . $initStr . fgets($fd) . ')');
1770  return null;
1771  }
1772  if ((string)$initStrDat[3] !== '') {
1773  $this->error('File read error: InitString had a wrong length. (' . $name . ')');
1774  return null;
1775  }
1776  $datString = fread($fd, (int)$initStrDat[2]);
1777  fread($fd, 1);
1778  if (md5($datString) === $initStrDat[0]) {
1779  if ($initStrDat[1]) {
1780  if ($this->compress) {
1781  $datString = gzuncompress($datString);
1782  } else {
1783  $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.');
1784  return null;
1785  }
1786  }
1787  return $unserialize ? unserialize($datString, ['allowed_classes' => false]) : $datString;
1788  } else {
1789  $this->error('MD5 check failed (' . $name . ')');
1790  }
1791  return null;
1792  }
1793 
1801  public function loadContent($filecontent)
1802  {
1803  $pointer = 0;
1804  $this->dat['header'] = $this->getNextContentPart($filecontent, $pointer, 1, 'header');
1805  $this->dat['records'] = $this->getNextContentPart($filecontent, $pointer, 1, 'records');
1806  $this->dat['files'] = $this->getNextContentPart($filecontent, $pointer, 1, 'files');
1807  $this->loadInit();
1808  }
1809 
1819  public function getNextContentPart($filecontent, &$pointer, $unserialize = false, $name = '')
1820  {
1821  $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
1822  // getting header data
1823  $initStr = substr($filecontent, $pointer, $initStrLen);
1824  $pointer += $initStrLen;
1825  $initStrDat = explode(':', $initStr);
1826  if ((string)$initStrDat[3] !== '') {
1827  $this->error('Content read error: InitString had a wrong length. (' . $name . ')');
1828  return null;
1829  }
1830  $datString = substr($filecontent, $pointer, (int)$initStrDat[2]);
1831  $pointer += (int)$initStrDat[2] + 1;
1832  if (md5($datString) === $initStrDat[0]) {
1833  if ($initStrDat[1]) {
1834  if ($this->compress) {
1835  $datString = gzuncompress($datString);
1836  return $unserialize ? unserialize($datString, ['allowed_classes' => false]) : $datString;
1837  } else {
1838  $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.');
1839  }
1840  }
1841  } else {
1842  $this->error('MD5 check failed (' . $name . ')');
1843  }
1844  return null;
1845  }
1846 
1852  public function loadInit()
1853  {
1854  $this->relStaticTables = (array)$this->dat['header']['relStaticTables'];
1855  $this->excludeMap = (array)$this->dat['header']['excludeMap'];
1856  $this->softrefCfg = (array)$this->dat['header']['softrefCfg'];
1857  }
1858 }
processSoftReferences_saveFile($relFileName, $cfg, $table, $uid)
Definition: Import.php:1461
addSingle($table, $uid, $pid)
Definition: Import.php:828
fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid)
Definition: Import.php:520
processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid)
Definition: Import.php:1403
static copyDirectory($source, $destination)
import_addFileNameToBeCopied($fI)
Definition: Import.php:1120
static isFirstPartOfStr($str, $partStr)
static xml2array($string, $NSprefix= '', $reportDocTag=false)
writeTemporaryFileFromData($fileId, $dataKey= 'files_fal')
Definition: Import.php:563
processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
Definition: Import.php:1369
static split_fileref($fileNameWithPath)
remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
Definition: Import.php:1250
verifyFolderAccess($dirPrefix, $noAlternative=false)
flatInversePageTree_pid($idH, $a=[], $pid=-1)
Definition: Import.php:691
checkOrCreateDir($dirPrefix)
Definition: Import.php:1657
static writeFile($file, $content, $changePermissions=false)
setRelations_db($itemArray, $itemConfig)
Definition: Import.php:1076
removeSysFileReferenceRecordsFromImportDataWithRelationToMissingFile()
Definition: Import.php:474
static getRecord($table, $uid, $fields= '*', $where= '', $useDeleteClause=true)
static unlink_tempfile($uploadedTempFileName)
writeRecords_records_order($mainPid)
Definition: Import.php:774
isFallbackStorage($storageId)
Definition: Import.php:499
static tempnam($filePrefix, $fileSuffix= '')
processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid)
Definition: Import.php:1528
getNextFilePart($fd, $unserialize=false, $name= '')
Definition: Import.php:1758
loadFile($filename, $all=false)
Definition: Import.php:1694
addToMapId($substNEWwithIDs)
Definition: Import.php:940
if(TYPO3_MODE=== 'BE') $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['t3lib/class.t3lib_tsfebeuserauth.php']['frontendEditingController']['default']
writeFileVerify($fileName, $fileID, $bypassMountCheck=false)
Definition: Import.php:1612
static makeInstance($className,...$constructorArguments)
getNextContentPart($filecontent, &$pointer, $unserialize=false, $name= '')
Definition: Import.php:1819
loadContent($filecontent)
Definition: Import.php:1801
static getFileAbsFileName($filename, $_=null, $_2=null)
doesRecordExist($table, $uid, $fields= '')
isEquivalentObjectStorage(ResourceStorage $storageObject, array $storageRecord)
Definition: Import.php:259