‪TYPO3CMS  ‪main
CleanUpLocalProcessedFilesService.php
Go to the documentation of this file.
1 <?php
2 
3 declare(strict_types=1);
4 
5 /*
6  * This file is part of the TYPO3 CMS project.
7  *
8  * It is free software; you can redistribute it and/or modify it under
9  * the terms of the GNU General Public License, either version 2
10  * of the License, or any later version.
11  *
12  * For the full copyright and license information, please read the
13  * LICENSE.txt file that was distributed with this source code.
14  *
15  * The TYPO3 project - inspiring people to share!
16  */
17 
19 
23 use TYPO3\CMS\Core\Database\Query\QueryBuilder;
30 
35 {
36  protected const ‪TABLE_NAME = 'sys_file_processedfile';
37  protected const ‪DRIVER = 'Local';
38 
44  public function ‪getFilesToClean(int $limit = 0, bool $fullReset = false): array
45  {
46  $queryBuilder = $this->‪getQueryBuilderWithoutRestrictions();
47  $localStorages = GeneralUtility::makeInstance(StorageRepository::class)->findByStorageType(self::DRIVER);
48 
49  $queryBuilder
50  ->count('*')
51  ->from(self::TABLE_NAME)
52  ->where(
53  // processed file identifier (placeholder pos 1)
54  $queryBuilder->expr()->eq(
55  'identifier',
56  $queryBuilder->createPositionalParameter('')
57  ),
58  // we need to ensure to search an identifier only for the responsible storage (placeholder pos 2)
59  $queryBuilder->expr()->eq(
60  'storage',
61  $queryBuilder->createPositionalParameter(0, ‪Connection::PARAM_INT)
62  )
63  );
64  $statement = $queryBuilder->prepare();
65 
66  $files = [];
67  foreach ($localStorages as $storage) {
68  $storageBasePath = ‪PathUtility::stripPathSitePrefix($this->‪getAbsoluteBasePath($storage->getConfiguration()));
69  foreach ($storage->getProcessingFolders() as $folder) {
70  foreach ($this->‪getFilesOfFolderRecursive($folder) as $splFileInfo) {
71  if ($splFileInfo->getRealPath() === false) {
72  continue;
73  }
74 
75  // prepare identifier for proper lookup
76  $filePath = '/' . mb_substr(
77  ‪PathUtility::stripPathSitePrefix($splFileInfo->getRealPath()),
78  mb_strlen(trim($storageBasePath, '/') . '/')
79  );
80 
81  // The full reset does not need to check the database, it clears all files.
82  if (!$fullReset) {
83  // reuse prepared statement to find processed files without any processed record entries in matching
84  // storage, using `$filePath` as equal match for field `identifier` and storage uid.
85  $statement->bindValue(1, $filePath, ‪Connection::PARAM_STR);
86  $statement->bindValue(2, $storage->getUid(), ‪Connection::PARAM_INT);
87  if ((int)$statement->executeQuery()->fetchOne() === 0) {
88  $files[] = $splFileInfo;
89  }
90  } else {
91  $files[] = $splFileInfo;
92  }
93  }
94  }
95  }
96 
97  return $files;
98  }
99 
104  public function ‪getRecordsToClean(bool $fullReset = false): array
105  {
106  $storageRepository = GeneralUtility::makeInstance(StorageRepository::class);
107  $queryBuilder = $this->‪getQueryBuilderWithoutRestrictions();
108  $conditions = [
109  $queryBuilder->expr()->eq(
110  'sfs.driver',
111  $queryBuilder->createNamedParameter(self::DRIVER)
112  ),
113  ];
114 
115  if (!$fullReset) {
116  $conditions[] = $queryBuilder->expr()->neq(
117  'sfp.identifier',
118  $queryBuilder->createNamedParameter('')
119  );
120  $storageJoinCondition = $queryBuilder->expr()->eq(
121  'sfp.storage',
122  $queryBuilder->quoteIdentifier('sfs.uid')
123  );
124  } else {
125  // When removing all records, also the fallback storage needs to be cleared.
126  $storageJoinCondition = $queryBuilder->expr()->gte(
127  'sfp.storage',
128  0
129  );
130  }
131 
132  $queryBuilder
133  ->select('sfp.storage', 'sfp.identifier', 'sfp.uid')
134  ->from(self::TABLE_NAME, 'sfp')
135  ->leftJoin(
136  'sfp',
137  'sys_file_storage',
138  'sfs',
139  $storageJoinCondition
140  )
141  ->where(...array_values($conditions));
142 
143  $results = $queryBuilder->executeQuery();
144  $processedToDelete = [];
145  while ($processedFile = $results->fetchAssociative()) {
146  if ($fullReset) {
147  // When removing all records, it does not matter if the storage file exists or not.
148  $processedToDelete[] = $processedFile;
149  continue;
150  }
151 
152  $storage = $storageRepository->findByUid((int)$processedFile['storage']);
153  $processedPathAndFileIdentifier = (string)$processedFile['identifier'];
154 
155  // Storage does no longer have that file => delete entry
156  if ($storage !== null && !$storage->hasFile($processedPathAndFileIdentifier)) {
157  $processedToDelete[] = $processedFile;
158  }
159  }
160 
161  return $processedToDelete;
162  }
163 
170  protected function ‪getFilesOfFolderRecursive(‪Folder $folder): iterable
171  {
172  try {
173  $basePath = $this->‪getAbsoluteBasePath($folder->‪getStorage()->getConfiguration());
174  } catch (‪InvalidPathException | ‪InvalidConfigurationException $invalidPathException) {
175  return [];
176  }
177 
178  $iterator = new \RecursiveIteratorIterator(
179  new \RecursiveDirectoryIterator(
180  $basePath . $folder->‪getIdentifier(),
181  \FilesystemIterator::UNIX_PATHS | \FilesystemIterator::SKIP_DOTS | \FilesystemIterator::FOLLOW_SYMLINKS | \FilesystemIterator::CURRENT_AS_FILEINFO
182  ),
183  \RecursiveIteratorIterator::SELF_FIRST,
184  \RecursiveIteratorIterator::CATCH_GET_CHILD
185  );
186  foreach ($iterator as $splFileInfo) {
187  if ($splFileInfo instanceof \SplFileInfo && $splFileInfo->isFile()) {
188  yield $splFileInfo;
189  }
190  }
191  }
192 
193  protected function ‪getAbsoluteBasePath(array $configuration): string
194  {
195  if (!array_key_exists('basePath', $configuration) || empty($configuration['basePath'])) {
197  'Configuration must contain base path.',
198  1640297535
199  );
200  }
201 
202  $absoluteBasePath = $configuration['basePath'];
203  if (!empty($configuration['pathType']) && $configuration['pathType'] === 'relative') {
204  $relativeBasePath = $configuration['basePath'];
205  $absoluteBasePath = ‪Environment::getPublicPath() . '/' . $relativeBasePath;
206  }
207  $absoluteBasePath = ‪PathUtility::getCanonicalPath($absoluteBasePath);
208  $absoluteBasePath = rtrim($absoluteBasePath, '/') . '/';
209  if (!is_dir($absoluteBasePath)) {
211  'Base path "' . $absoluteBasePath . '" does not exist or is no directory.',
212  1640297526
213  );
214  }
215 
216  return $absoluteBasePath;
217  }
218 
219  public function ‪deleteRecord(array $recordUids): int
220  {
221  $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME);
222  $queryBuilder->getRestrictions()->removeAll();
223 
224  return $queryBuilder->delete(self::TABLE_NAME)
225  ->where(
226  $queryBuilder->expr()->in('uid', $queryBuilder->createNamedParameter($recordUids, ‪Connection::PARAM_INT_ARRAY))
227  )
228  ->executeStatement();
229  }
230 
231  protected function ‪getQueryBuilderWithoutRestrictions(): QueryBuilder
232  {
233  $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME);
234  $queryBuilder->getRestrictions()->removeAll();
235 
236  return $queryBuilder;
237  }
238 }
‪TYPO3\CMS\Core\Utility\PathUtility\getCanonicalPath
‪static string getCanonicalPath(string $path)
Definition: PathUtility.php:364
‪TYPO3\CMS\Core\Utility\PathUtility\stripPathSitePrefix
‪static stripPathSitePrefix(string $path)
Definition: PathUtility.php:428
‪TYPO3\CMS\Lowlevel\Service\CleanUpLocalProcessedFilesService\getAbsoluteBasePath
‪getAbsoluteBasePath(array $configuration)
Definition: CleanUpLocalProcessedFilesService.php:193
‪TYPO3\CMS\Core\Utility\PathUtility
Definition: PathUtility.php:27
‪TYPO3\CMS\Core\Database\Connection\PARAM_INT
‪const PARAM_INT
Definition: Connection.php:52
‪TYPO3\CMS\Lowlevel\Service\CleanUpLocalProcessedFilesService\getQueryBuilderWithoutRestrictions
‪getQueryBuilderWithoutRestrictions()
Definition: CleanUpLocalProcessedFilesService.php:231
‪TYPO3\CMS\Core\Core\Environment\getPublicPath
‪static getPublicPath()
Definition: Environment.php:187
‪TYPO3\CMS\Lowlevel\Service\CleanUpLocalProcessedFilesService\getFilesOfFolderRecursive
‪SplFileInfo[] getFilesOfFolderRecursive(Folder $folder)
Definition: CleanUpLocalProcessedFilesService.php:170
‪TYPO3\CMS\Lowlevel\Service\CleanUpLocalProcessedFilesService\deleteRecord
‪deleteRecord(array $recordUids)
Definition: CleanUpLocalProcessedFilesService.php:219
‪TYPO3\CMS\Lowlevel\Service\CleanUpLocalProcessedFilesService\TABLE_NAME
‪const TABLE_NAME
Definition: CleanUpLocalProcessedFilesService.php:36
‪TYPO3\CMS\Core\Database\Connection\PARAM_STR
‪const PARAM_STR
Definition: Connection.php:57
‪TYPO3\CMS\Lowlevel\Service\CleanUpLocalProcessedFilesService\getRecordsToClean
‪getRecordsToClean(bool $fullReset=false)
Definition: CleanUpLocalProcessedFilesService.php:104
‪TYPO3\CMS\Core\Resource\Exception\InvalidConfigurationException
Definition: InvalidConfigurationException.php:23
‪TYPO3\CMS\Core\Resource\Folder
Definition: Folder.php:38
‪TYPO3\CMS\Core\Resource\StorageRepository
Definition: StorageRepository.php:38
‪TYPO3\CMS\Core\Resource\Folder\getStorage
‪getStorage()
Definition: Folder.php:139
‪TYPO3\CMS\Core\Database\Connection
Definition: Connection.php:41
‪TYPO3\CMS\Lowlevel\Service\CleanUpLocalProcessedFilesService
Definition: CleanUpLocalProcessedFilesService.php:35
‪TYPO3\CMS\Core\Core\Environment
Definition: Environment.php:41
‪TYPO3\CMS\Lowlevel\Service
Definition: CleanUpLocalProcessedFilesService.php:18
‪TYPO3\CMS\Core\Resource\Folder\getIdentifier
‪non empty string getIdentifier()
Definition: Folder.php:150
‪TYPO3\CMS\Core\Database\ConnectionPool
Definition: ConnectionPool.php:46
‪TYPO3\CMS\Core\Utility\GeneralUtility
Definition: GeneralUtility.php:52
‪TYPO3\CMS\Lowlevel\Service\CleanUpLocalProcessedFilesService\DRIVER
‪const DRIVER
Definition: CleanUpLocalProcessedFilesService.php:37
‪TYPO3\CMS\Core\Resource\Exception\InvalidPathException
Definition: InvalidPathException.php:23
‪TYPO3\CMS\Core\Database\Connection\PARAM_INT_ARRAY
‪const PARAM_INT_ARRAY
Definition: Connection.php:72
‪TYPO3\CMS\Lowlevel\Service\CleanUpLocalProcessedFilesService\getFilesToClean
‪getFilesToClean(int $limit=0, bool $fullReset=false)
Definition: CleanUpLocalProcessedFilesService.php:44