TYPO3 CMS  TYPO3_8-7
FilesWithMultipleReferencesCommand.php
Go to the documentation of this file.
1 <?php
2 declare(strict_types = 1);
4 
5 /*
6  * This file is part of the TYPO3 CMS project.
7  *
8  * It is free software; you can redistribute it and/or modify it under
9  * the terms of the GNU General Public License, either version 2
10  * of the License, or any later version.
11  *
12  * For the full copyright and license information, please read the
13  * LICENSE.txt file that was distributed with this source code.
14  *
15  * The TYPO3 project - inspiring people to share!
16  */
17 
30 
35 {
36 
40  public function configure()
41  {
42  $this
43  ->setDescription('Looking for files from TYPO3 managed records which are referenced more than once')
44  ->setHelp('
45 Assumptions:
46 - a perfect integrity of the reference index table (always update the reference index table before using this tool!)
47 - files found in deleted records are included (otherwise you would see a false list of lost files)
48 
49 Files attached to records in TYPO3 using a "group" type configuration in TCA or FlexForm DataStructure are managed exclusively by the system and there must always exist a 1-1 reference between the file and the reference in the record.
50 This tool will expose when such files are referenced from multiple locations which is considered an integrity error.
51 If a multi-reference is found it was typically created because the record was copied or modified outside of DataHandler which will otherwise maintain the relations correctly.
52 Multi-references should be resolved to 1-1 references as soon as possible. The danger of keeping multi-references is that if the file is removed from one of the referring records it will actually be deleted in the file system, leaving missing files for the remaining referers!
53 
54 If the option "--dry-run" is not set, the files that are referenced multiple times are copied with a new name
55 and the references are updated accordingly.
56 Warning: First, make sure those files are not used somewhere TYPO3 does not know about!
57 
58 If you want to get more detailed information, use the --verbose option.')
59  ->addOption(
60  'dry-run',
61  null,
62  InputOption::VALUE_NONE,
63  'If this option is set, the files will not actually be deleted, but just the output which files would be deleted are shown'
64  )
65  ->addOption(
66  'update-refindex',
67  null,
68  InputOption::VALUE_NONE,
69  'Setting this option automatically updates the reference index and does not ask on command line. Alternatively, use -n to avoid the interactive mode'
70  );
71  }
72 
82  protected function execute(InputInterface $input, OutputInterface $output)
83  {
84  // Make sure the _cli_ user is loaded
85  Bootstrap::getInstance()->initializeBackendAuthentication();
86 
87  $io = new SymfonyStyle($input, $output);
88  $io->title($this->getDescription());
89 
90  $dryRun = $input->hasOption('dry-run') && $input->getOption('dry-run') != false ? true : false;
91 
92  $this->updateReferenceIndex($input, $io);
93 
94  // Find files which are referenced multiple times
95  $doubleFiles = $this->findMultipleReferencedFiles();
96 
97  if (count($doubleFiles)) {
98  if (!$io->isQuiet()) {
99  $io->note('Found ' . count($doubleFiles) . ' files that are referenced more than once.');
100  if ($io->isVerbose()) {
101  $io->listing($doubleFiles);
102  }
103  }
104 
105  $this->copyMultipleReferencedFiles($doubleFiles, $dryRun, $io);
106  $io->success('Cleaned up ' . count($doubleFiles) . ' files which have been referenced multiple times.');
107  } else {
108  $io->success('Nothing to do, no files found which are referenced more than once.');
109  }
110  }
111 
121  protected function updateReferenceIndex(InputInterface $input, SymfonyStyle $io)
122  {
123  // Check for reference index to update
124  $io->note('Finding files referenced multiple times in records managed by TYPO3 requires a clean reference index (sys_refindex)');
125  $updateReferenceIndex = false;
126  if ($input->hasOption('update-refindex') && $input->getOption('update-refindex')) {
127  $updateReferenceIndex = true;
128  } elseif ($input->isInteractive()) {
129  $updateReferenceIndex = $io->confirm('Should the reference index be updated right now?', false);
130  }
131 
132  // Update the reference index
133  if ($updateReferenceIndex) {
134  $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
135  $referenceIndex->updateIndex(false, !$io->isQuiet());
136  } else {
137  $io->writeln('Reference index is assumed to be up to date, continuing.');
138  }
139  }
140 
146  protected function findMultipleReferencedFiles(): array
147  {
148  $multipleReferencesList = [];
149 
150  // Select all files in the reference table not found by a soft reference parser (thus TCA configured)
151  $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
152  ->getQueryBuilderForTable('sys_refindex');
153 
154  $result = $queryBuilder
155  ->select('*')
156  ->from('sys_refindex')
157  ->where(
158  $queryBuilder->expr()->eq('ref_table', $queryBuilder->createNamedParameter('_FILE', \PDO::PARAM_STR)),
159  $queryBuilder->expr()->eq('softref_key', $queryBuilder->createNamedParameter('', \PDO::PARAM_STR))
160  )
161  ->execute();
162 
163  // Traverse the files and put into a large table
164  $allReferencesToFiles = [];
165  while ($record = $result->fetch()) {
166  // Compile info string for location of reference
167  $infoString = $this->formatReferenceIndexEntryToString($record);
168  $hash = $record['hash'];
169  $fileName = $record['ref_string'];
170  // Add entry if file has multiple references pointing to it
171  if (isset($allReferencesToFiles[$fileName])) {
172  if (!is_array($multipleReferencesList[$fileName])) {
173  $multipleReferencesList[$fileName] = [];
174  $multipleReferencesList[$fileName][$allReferencesToFiles[$fileName]['hash']] = $allReferencesToFiles[$fileName]['infoString'];
175  }
176  $multipleReferencesList[$fileName][$hash] = $infoString;
177  } else {
178  $allReferencesToFiles[$fileName] = [
179  'infoString' => $infoString,
180  'hash' => $hash
181  ];
182  }
183  }
184 
185  return ArrayUtility::sortByKeyRecursive($multipleReferencesList);
186  }
187 
195  protected function copyMultipleReferencedFiles(array $multipleReferencesToFiles, bool $dryRun, SymfonyStyle $io)
196  {
197  $fileFunc = GeneralUtility::makeInstance(BasicFileUtility::class);
198  $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
199 
200  foreach ($multipleReferencesToFiles as $fileName => $usages) {
201  $absoluteFileName = GeneralUtility::getFileAbsFileName($fileName);
202  if ($absoluteFileName && @is_file($absoluteFileName)) {
203  if ($io->isVeryVerbose()) {
204  $io->writeln('Processing file "' . $absoluteFileName . '"');
205  }
206  $counter = 0;
207  foreach ($usages as $hash => $recReference) {
208  if ($counter++ === 0) {
209  $io->writeln('Keeping "' . $fileName . '" for record "' . $recReference . '"');
210  } else {
211  // Create unique name for file
212  $newName = $fileFunc->getUniqueName(basename($fileName), dirname($absoluteFileName));
213  $io->writeln('Copying "' . $fileName . '" to "' . PathUtility::stripPathSitePrefix($newName) . '" for record "' . $recReference . '"');
214  if (!$dryRun) {
215  GeneralUtility::upload_copy_move($absoluteFileName, $newName);
216  clearstatcache();
217  if (@is_file($newName)) {
218  $error = $referenceIndex->setReferenceValue($hash, basename($newName));
219  if ($error) {
220  $io->error('ReferenceIndex::setReferenceValue() reported "' . $error . '"');
221  }
222  } else {
223  $io->error('File "' . $newName . '" could not be created.');
224  }
225  }
226  }
227  }
228  } else {
229  $io->error('File "' . $absoluteFileName . '" was not found.');
230  }
231  }
232  }
233 
240  protected function formatReferenceIndexEntryToString(array $record): string
241  {
242  return $record['tablename']
243  . ':' . $record['recuid']
244  . ':' . $record['field']
245  . ($record['flexpointer'] ? ':' . $record['flexpointer'] : '')
246  . ($record['softref_key'] ? ':' . $record['softref_key'] . ' (Soft Reference) ' : '')
247  . ($record['deleted'] ? ' (DELETED)' : '');
248  }
249 }
static getFileAbsFileName($filename, $_=null, $_2=null)
static makeInstance($className,... $constructorArguments)
static sortByKeyRecursive(array $array)
copyMultipleReferencedFiles(array $multipleReferencesToFiles, bool $dryRun, SymfonyStyle $io)
static upload_copy_move($source, $destination)