TYPO3 CMS  TYPO3_7-6
LostFilesCommand.php
Go to the documentation of this file.
1 <?php
2 namespace TYPO3\CMS\Lowlevel;
3 
4 /*
5  * This file is part of the TYPO3 CMS project.
6  *
7  * It is free software; you can redistribute it and/or modify it under
8  * the terms of the GNU General Public License, either version 2
9  * of the License, or any later version.
10  *
11  * For the full copyright and license information, please read the
12  * LICENSE.txt file that was distributed with this source code.
13  *
14  * The TYPO3 project - inspiring people to share!
15  */
16 
21 {
25  public $checkRefIndex = true;
26 
30  public function __construct()
31  {
32  parent::__construct();
33  $this->cli_options[] = ['--excludePath [path-list]', 'Comma separated list of paths to exclude. Example: "uploads/[path1],uploads/[path2],..."'];
34  // Setting up help:
35  $this->cli_help['name'] = 'lost_files -- Looking for files in the uploads/ folder which does not have a reference in TYPO3 managed records.';
36  $this->cli_help['description'] = trim('
37 Assumptions:
38 - a perfect integrity of the reference index table (always update the reference index table before using this tool!)
39 - that all contents in the uploads folder are files attached to TCA records and exclusively managed by TCEmain through "group" type fields
40 - exceptions are: index.html and .htaccess files (ignored)
41 - exceptions are: RTEmagic* image files (ignored)
42 - files found in deleted records are included (otherwise you would see a false list of lost files)
43 
44 The assumptions are not requirements by the TYPO3 API but reflects the de facto implementation of most TYPO3 installations and therefore a practical approach to cleaning up the uploads/ folder.
45 Therefore, if all "group" type fields in TCA and flexforms are positioned inside the uploads/ folder and if no files inside are managed manually it should be safe to clean out files with no relations found in the system.
46 Under such circumstances there should theoretically be no lost files in the uploads/ folder since TCEmain should have managed relations automatically including adding and deleting files.
47 However, there is at least one reason known to why files might be found lost and that is when FlexForms are used. In such a case a change of/in the Data Structure XML (or the ability of the system to find the Data Structure definition!) used for the flexform could leave lost files behind. This is not unlikely to happen when records are deleted. More details can be found in a note to the function TYPO3\\CMS\\Backend\\Utility\\BackendUtility::getFlexFormDS()
48 Another scenario could of course be de-installation of extensions which managed files in the uploads/ folders.
49 
50 Automatic Repair of Errors:
51 - Simply delete lost files (Warning: First, make sure those files are not used somewhere TYPO3 does not know about! See the assumptions above).
52 ');
53  $this->cli_help['examples'] = '/.../cli_dispatch.phpsh lowlevel_cleaner lost_files -s -r
54 Will report lost files.';
55  }
56 
67  public function main()
68  {
69  // Initialize result array:
70  $resultArray = [
71  'message' => $this->cli_help['name'] . LF . LF . $this->cli_help['description'],
72  'headers' => [
73  'managedFiles' => ['Files related to TYPO3 records and managed by TCEmain', 'These files you definitely want to keep.', 0],
74  'ignoredFiles' => ['Ignored files (index.html, .htaccess etc.)', 'These files are allowed in uploads/ folder', 0],
75  'RTEmagicFiles' => ['RTE magic images - those found (and ignored)', 'These files are also allowed in some uploads/ folders as RTEmagic images.', 0],
76  'lostFiles' => ['Lost files - those you can delete', 'You can delete these files!', 3],
77  'warnings' => ['Warnings picked up', '', 2]
78  ],
79  'managedFiles' => [],
80  'ignoredFiles' => [],
81  'RTEmagicFiles' => [],
82  'lostFiles' => [],
83  'warnings' => []
84  ];
85  // Get all files:
86  $fileArr = [];
87  $fileArr = \TYPO3\CMS\Core\Utility\GeneralUtility::getAllFilesAndFoldersInPath($fileArr, PATH_site . 'uploads/');
89  $excludePaths = \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',', $this->cli_argValue('--excludePath', 0), true);
90  // Traverse files and for each, look up if its found in the reference index.
91  foreach ($fileArr as $key => $value) {
92  $include = true;
93  foreach ($excludePaths as $exclPath) {
94  if (\TYPO3\CMS\Core\Utility\GeneralUtility::isFirstPartOfStr($value, $exclPath)) {
95  $include = false;
96  }
97  }
98  $shortKey = \TYPO3\CMS\Core\Utility\GeneralUtility::shortmd5($value);
99  if ($include) {
100  // First, allow "index.html", ".htaccess" files since they are often used for good reasons
101  if (substr($value, -11) == '/index.html' || substr($value, -10) == '/.htaccess') {
102  unset($fileArr[$key]);
103  $resultArray['ignoredFiles'][$shortKey] = $value;
104  } else {
105  // Looking for a reference from a field which is NOT a soft reference (thus, only fields with a proper TCA/Flexform configuration)
106  $recs = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows('*', 'sys_refindex', 'ref_table=' . $GLOBALS['TYPO3_DB']->fullQuoteStr('_FILE', 'sys_refindex') . ' AND ref_string=' . $GLOBALS['TYPO3_DB']->fullQuoteStr($value, 'sys_refindex') . ' AND softref_key=' . $GLOBALS['TYPO3_DB']->fullQuoteStr('', 'sys_refindex'), '', 'sorting DESC');
107  // If found, unset entry:
108  if (count($recs)) {
109  unset($fileArr[$key]);
110  $resultArray['managedFiles'][$shortKey] = $value;
111  if (count($recs) > 1) {
112  $resultArray['warnings'][$shortKey] = 'Warning: File "' . $value . '" had ' . count($recs) . ' references from group-fields, should have only one!';
113  }
114  } else {
115  // When here it means the file was not found. So we test if it has a RTEmagic-image name and if so, we allow it:
116  if (preg_match('/^RTEmagic[P|C]_/', basename($value))) {
117  unset($fileArr[$key]);
118  $resultArray['RTEmagicFiles'][$shortKey] = $value;
119  } else {
120  // We conclude that the file is lost...:
121  unset($fileArr[$key]);
122  $resultArray['lostFiles'][$shortKey] = $value;
123  }
124  }
125  }
126  }
127  }
128  asort($resultArray['ignoredFiles']);
129  asort($resultArray['managedFiles']);
130  asort($resultArray['RTEmagicFiles']);
131  asort($resultArray['lostFiles']);
132  asort($resultArray['warnings']);
133  // $fileArr variable should now be empty with all contents transferred to the result array keys.
134  return $resultArray;
135  }
136 
144  public function main_autoFix($resultArray)
145  {
146  foreach ($resultArray['lostFiles'] as $key => $value) {
148  echo 'Deleting file: "' . $absFileName . '": ';
149  if ($bypass = $this->cli_noExecutionCheck($absFileName)) {
150  echo $bypass;
151  } else {
152  if ($absFileName && @is_file($absFileName)) {
153  unlink($absFileName);
154  echo 'DONE';
155  } else {
156  echo ' ERROR: File "' . $absFileName . '" was not found!';
157  }
158  }
159  echo LF;
160  }
161  }
162 }
static isFirstPartOfStr($str, $partStr)
static trimExplode($delim, $string, $removeEmptyValues=false, $limit=0)
static getAllFilesAndFoldersInPath(array $fileArr, $path, $extList='', $regDirs=false, $recursivityLevels=99, $excludePattern='')
static getFileAbsFileName($filename, $onlyRelative=true, $relToTYPO3_mainDir=false)
if(TYPO3_MODE==='BE') $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['t3lib/class.t3lib_tsfebeuserauth.php']['frontendEditingController']['default']
static removePrefixPathFromList(array $fileArr, $prefixToRemove)