‪TYPO3CMS  ‪main
BulkExtensionRepositoryWriter.php
Go to the documentation of this file.
1 <?php
2 
3 declare(strict_types=1);
4 
5 /*
6  * This file is part of the TYPO3 CMS project.
7  *
8  * It is free software; you can redistribute it and/or modify it under
9  * the terms of the GNU General Public License, either version 2
10  * of the License, or any later version.
11  *
12  * For the full copyright and license information, please read the
13  * LICENSE.txt file that was distributed with this source code.
14  *
15  * The TYPO3 project - inspiring people to share!
16  */
17 
19 
20 use Doctrine\DBAL\Exception as DBALException;
21 use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
28 
34 #[Autoconfigure(public: true)]
35 class ‪BulkExtensionRepositoryWriter implements \SplObserver
36 {
40  private const ‪TABLE_NAME = 'tx_extensionmanager_domain_model_extension';
41 
43 
49  protected ‪$sumRecords = 0;
50 
56  protected ‪$arrRows = [];
57 
63  protected static ‪$fieldNames = [
64  'extension_key',
65  'version',
66  'integer_version',
67  'current_version',
68  'alldownloadcounter',
69  'downloadcounter',
70  'title',
71  'ownerusername',
72  'author_name',
73  'author_email',
74  'authorcompany',
75  'last_updated',
76  'md5hash',
77  'remote',
78  'state',
79  'review_state',
80  'category',
81  'description',
82  'serialized_dependencies',
83  'update_comment',
84  'documentation_link',
85  'distribution_image',
86  'distribution_welcome_image',
87  ];
88 
95  protected ‪$maxRowsPerChunk = 50;
96 
102  protected ‪$remoteIdentifier;
103 
107  protected ‪$extensionRepository;
108 
112  protected ‪$extensionModel;
113 
117  protected ‪$connectionPool;
118 
125  protected ‪$minimumDateToImport;
126 
132  public function ‪__construct(
133  ‪ExtensionRepository $repository,
134  ‪Extension $extension,
137  ) {
138  $this->extensionRepository = $repository;
139  $this->extensionModel = $extension;
140  $this->connectionPool = ‪$connectionPool;
141  $this->parser = ‪$parser;
142  $this->parser->‪attach($this);
143 
144  $connection = $this->connectionPool->getConnectionForTable(self::TABLE_NAME);
146  $connection->getDatabasePlatform()
147  );
148  $countOfBindParamsPerRow = count(self::$fieldNames);
149  // flush at least chunks of 50 elements - in case the currently used
150  // database platform does not support that, the threshold is lowered
151  $this->maxRowsPerChunk = (int)min(
152  $this->maxRowsPerChunk,
153  floor($maxBindParameters / $countOfBindParamsPerRow)
154  );
155  // Only import extensions that are compatible with 7.6 or higher.
156  // TER only allows to publish extensions with compatibility if the TYPO3 version has been released
157  // And 7.6 was released on 10th of November 2015.
158  // This effectively reduces the number of extensions imported into this TYPO3 installation
159  // by more than 70%. As long as the extensions.xml from TER includes these files, we need to "hack" this
160  // within TYPO3 Core.
161  // For TYPO3 v11.0, this date could be set to 2018-10-02 (v9 LTS release).
162  // Also see https://decisions.typo3.org/t/reduce-size-of-extension-manager-db-table/329/
163  $this->minimumDateToImport = strtotime('2017-04-04T00:00:00+00:00');
164  }
165 
172  public function import(string $localExtensionListFile, string ‪$remoteIdentifier): void
173  {
174  // Remove all existing entries of this remote from the database
175  $this->connectionPool
176  ->getConnectionForTable(self::TABLE_NAME)
177  ->delete(
178  self::TABLE_NAME,
179  ['remote' => ‪$remoteIdentifier],
181  );
182  $this->remoteIdentifier = ‪$remoteIdentifier;
183  $zlibStream = 'compress.zlib://';
184  $this->sumRecords = 0;
185  $this->parser->parseXml($zlibStream . $localExtensionListFile);
186  // flush last rows to database if existing
187  if (!empty($this->arrRows)) {
188  $this->connectionPool
189  ->getConnectionForTable(self::TABLE_NAME)
190  ->bulkInsert(
191  'tx_extensionmanager_domain_model_extension',
192  $this->arrRows,
193  self::$fieldNames
194  );
195  }
197  }
198 
204  protected function ‪loadIntoDatabase(ExtensionXmlParser $subject): void
205  {
206  if ($this->sumRecords !== 0 && $this->sumRecords % $this->maxRowsPerChunk === 0) {
207  $this->connectionPool
208  ->getConnectionForTable(self::TABLE_NAME)
209  ->bulkInsert(
210  self::TABLE_NAME,
211  $this->arrRows,
212  self::$fieldNames
213  );
214  $this->arrRows = [];
215  }
216  if (!$subject->isValidVersionNumber()) {
217  // Skip in case extension version is not valid
218  return;
219  }
220  $versionRepresentations = VersionNumberUtility::convertVersionStringToArray($subject->getVersion());
221  // order must match that of self::$fieldNames!
222  $this->arrRows[] = [
223  $subject->getExtkey(),
224  $subject->getVersion(),
225  $versionRepresentations['version_int'],
226  // initialize current_version, correct value computed later:
227  0,
228  $subject->getAlldownloadcounter(),
229  $subject->getDownloadcounter(),
230  $subject->getTitle(),
231  $subject->getOwnerusername(),
232  $subject->getAuthorname(),
233  $subject->getAuthoremail(),
234  $subject->getAuthorcompany(),
235  $subject->getLastuploaddate(),
236  $subject->getT3xfilemd5(),
238  $this->extensionModel->getDefaultState($subject->getState() ?: ''),
239  $subject->getReviewstate(),
240  $this->extensionModel->getCategoryIndexFromStringOrNumber($subject->getCategory() ?: ''),
241  $subject->getDescription() ?: '',
242  $subject->getDependencies() ?: '',
243  $subject->getUploadcomment() ?: '',
244  $subject->getDocumentationLink() ?: '',
245  $subject->getDistributionImage() ?: '',
246  $subject->getDistributionWelcomeImage() ?: '',
247  ];
249  }
250 
256  public function ‪update(\SplSubject $subject): void
257  {
258  if ($subject instanceof ‪ExtensionXmlParser) {
259  if ((int)$subject->getLastuploaddate() > $this->minimumDateToImport) {
260  $this->‪loadIntoDatabase($subject);
261  }
262  }
263  }
264 
273  {
274  $uidsOfCurrentVersion = $this->‪fetchMaximalVersionsForAllExtensions($remoteIdentifier);
275  $queryBuilder = $this->connectionPool->getQueryBuilderForTable(self::TABLE_NAME);
276  $connection = $this->connectionPool->getConnectionForTable(self::TABLE_NAME);
278  $connection->getDatabasePlatform()
279  );
280 
281  foreach (array_chunk($uidsOfCurrentVersion, $maxBindParameters - 10) as $chunk) {
282  $queryBuilder
283  ->update(self::TABLE_NAME)
284  ->where(
285  $queryBuilder->expr()->in(
286  'uid',
287  $queryBuilder->createNamedParameter($chunk, ‪Connection::PARAM_INT_ARRAY)
288  )
289  )
290  ->set('current_version', 1)
291  ->executeStatement();
292  }
293  }
294 
300  protected function ‪fetchMaximalVersionsForAllExtensions(string ‪$remoteIdentifier): array
301  {
302  $queryBuilder = $this->connectionPool->getQueryBuilderForTable(self::TABLE_NAME);
303 
304  $queryResult = $queryBuilder
305  ->select('a.uid AS uid')
306  ->from(self::TABLE_NAME, 'a')
307  ->leftJoin(
308  'a',
309  self::TABLE_NAME,
310  'b',
311  $queryBuilder->expr()->and(
312  $queryBuilder->expr()->eq('a.remote', $queryBuilder->quoteIdentifier('b.remote')),
313  $queryBuilder->expr()->eq('a.extension_key', $queryBuilder->quoteIdentifier('b.extension_key')),
314  $queryBuilder->expr()->lt('a.integer_version', $queryBuilder->quoteIdentifier('b.integer_version'))
315  )
316  )
317  ->where(
318  $queryBuilder->expr()->eq(
319  'a.remote',
320  $queryBuilder->createNamedParameter(‪$remoteIdentifier)
321  ),
322  $queryBuilder->expr()->isNull('b.extension_key')
323  )
324  ->orderBy('a.uid')
325  ->executeQuery();
326 
327  $extensionUids = [];
328  while ($row = $queryResult->fetchAssociative()) {
329  $extensionUids[] = $row['uid'];
330  }
331 
332  return $extensionUids;
333  }
334 }
‪TYPO3\CMS\Core\Utility\VersionNumberUtility
Definition: VersionNumberUtility.php:26
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter
Definition: BulkExtensionRepositoryWriter.php:36
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\fetchMaximalVersionsForAllExtensions
‪fetchMaximalVersionsForAllExtensions(string $remoteIdentifier)
Definition: BulkExtensionRepositoryWriter.php:291
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getExtkey
‪getExtkey()
Definition: ExtensionXmlParser.php:362
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\markExtensionWithMaximumVersionAsCurrent
‪markExtensionWithMaximumVersionAsCurrent(string $remoteIdentifier)
Definition: BulkExtensionRepositoryWriter.php:263
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\update
‪update(\SplSubject $subject)
Definition: BulkExtensionRepositoryWriter.php:247
‪TYPO3\CMS\Core\Database\Platform\PlatformInformation\getMaxBindParameters
‪static getMaxBindParameters(DoctrineAbstractPlatform $platform)
Definition: PlatformInformation.php:106
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\attach
‪attach(\SplObserver $observer)
Definition: ExtensionXmlParser.php:267
‪TYPO3\CMS\Extensionmanager\Domain\Model\Extension
Definition: Extension.php:30
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getT3xfilemd5
‪getT3xfilemd5()
Definition: ExtensionXmlParser.php:402
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getCategory
‪getCategory()
Definition: ExtensionXmlParser.php:330
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\$minimumDateToImport
‪int $minimumDateToImport
Definition: BulkExtensionRepositoryWriter.php:116
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getVersion
‪getVersion()
Definition: ExtensionXmlParser.php:426
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getDownloadcounter
‪getDownloadcounter()
Definition: ExtensionXmlParser.php:354
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getAlldownloadcounter
‪getAlldownloadcounter()
Definition: ExtensionXmlParser.php:298
‪TYPO3\CMS\Extensionmanager\Domain\Repository
Definition: BulkExtensionRepositoryWriter.php:18
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getState
‪getState()
Definition: ExtensionXmlParser.php:394
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getAuthorname
‪getAuthorname()
Definition: ExtensionXmlParser.php:322
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getDependencies
‪getDependencies()
Definition: ExtensionXmlParser.php:338
‪TYPO3\CMS\Core\Database\Connection\PARAM_STR
‪const PARAM_STR
Definition: Connection.php:57
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getOwnerusername
‪getOwnerusername()
Definition: ExtensionXmlParser.php:378
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\$maxRowsPerChunk
‪int $maxRowsPerChunk
Definition: BulkExtensionRepositoryWriter.php:91
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\loadIntoDatabase
‪loadIntoDatabase(ExtensionXmlParser $subject)
Definition: BulkExtensionRepositoryWriter.php:195
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getDescription
‪getDescription()
Definition: ExtensionXmlParser.php:346
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\$arrRows
‪array $arrRows
Definition: BulkExtensionRepositoryWriter.php:54
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getAuthoremail
‪getAuthoremail()
Definition: ExtensionXmlParser.php:314
‪TYPO3\CMS\Extensionmanager\Domain\Repository\ExtensionRepository
Definition: ExtensionRepository.php:37
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getTitle
‪getTitle()
Definition: ExtensionXmlParser.php:410
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getUploadcomment
‪getUploadcomment()
Definition: ExtensionXmlParser.php:418
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\$fieldNames
‪static array $fieldNames
Definition: BulkExtensionRepositoryWriter.php:60
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getReviewstate
‪getReviewstate()
Definition: ExtensionXmlParser.php:386
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getAuthorcompany
‪getAuthorcompany()
Definition: ExtensionXmlParser.php:306
‪TYPO3\CMS\Core\Database\Connection
Definition: Connection.php:41
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getDistributionImage
‪getDistributionImage()
Definition: ExtensionXmlParser.php:451
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\isValidVersionNumber
‪isValidVersionNumber()
Definition: ExtensionXmlParser.php:434
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\$extensionRepository
‪ExtensionRepository $extensionRepository
Definition: BulkExtensionRepositoryWriter.php:101
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\$remoteIdentifier
‪string $remoteIdentifier
Definition: BulkExtensionRepositoryWriter.php:97
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser
Definition: ExtensionXmlParser.php:31
‪TYPO3\CMS\Core\Database\Platform\PlatformInformation
Definition: PlatformInformation.php:33
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\$parser
‪ExtensionXmlParser $parser
Definition: BulkExtensionRepositoryWriter.php:42
‪TYPO3\CMS\Core\Database\ConnectionPool
Definition: ConnectionPool.php:46
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\$sumRecords
‪int $sumRecords
Definition: BulkExtensionRepositoryWriter.php:48
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\TABLE_NAME
‪const TABLE_NAME
Definition: BulkExtensionRepositoryWriter.php:40
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getLastuploaddate
‪getLastuploaddate()
Definition: ExtensionXmlParser.php:370
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\$extensionModel
‪Extension $extensionModel
Definition: BulkExtensionRepositoryWriter.php:105
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getDistributionWelcomeImage
‪getDistributionWelcomeImage()
Definition: ExtensionXmlParser.php:459
‪TYPO3\CMS\Core\Database\Connection\PARAM_INT_ARRAY
‪const PARAM_INT_ARRAY
Definition: Connection.php:72
‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser\getDocumentationLink
‪getDocumentationLink()
Definition: ExtensionXmlParser.php:443
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\__construct
‪__construct(ExtensionRepository $repository, Extension $extension, ConnectionPool $connectionPool, ExtensionXmlParser $parser)
Definition: BulkExtensionRepositoryWriter.php:123
‪TYPO3\CMS\Extensionmanager\Domain\Repository\BulkExtensionRepositoryWriter\$connectionPool
‪ConnectionPool $connectionPool
Definition: BulkExtensionRepositoryWriter.php:109