BulkExtensionRepositoryWriter.php

Go to the documentation of this file.

<?php
 
declare(strict_types=1);
 
/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */
 
namespace ‪TYPO3\CMS\Extensionmanager\Domain\Repository;
 
use Doctrine\DBAL\Exception as DBALException;
use ‪TYPO3\CMS\Core\Database\Connection;
use ‪TYPO3\CMS\Core\Database\ConnectionPool;
use ‪TYPO3\CMS\Core\Database\Platform\PlatformInformation;
use ‪TYPO3\CMS\Core\Utility\VersionNumberUtility;
use ‪TYPO3\CMS\Extensionmanager\Domain\Model\Extension;
use ‪TYPO3\CMS\Extensionmanager\Parser\ExtensionXmlParser;
 
class ‪BulkExtensionRepositoryWriter implements \SplObserver
{
    private const ‪TABLE_NAME = 'tx_extensionmanager_domain_model_extension';
 
    protected ‪ExtensionXmlParser ‪$parser;
 
    protected ‪$sumRecords = 0;
 
    protected ‪$arrRows = [];
 
    protected static ‪$fieldNames = [
        'extension_key',
        'version',
        'integer_version',
        'current_version',
        'alldownloadcounter',
        'downloadcounter',
        'title',
        'ownerusername',
        'author_name',
        'author_email',
        'authorcompany',
        'last_updated',
        'md5hash',
        'remote',
        'state',
        'review_state',
        'category',
        'description',
        'serialized_dependencies',
        'update_comment',
        'documentation_link',
        'distribution_image',
        'distribution_welcome_image',
    ];
 
    protected ‪$maxRowsPerChunk = 50;
 
    protected ‪$remoteIdentifier;
 
    protected ‪$extensionRepository;
 
    protected ‪$extensionModel;
 
    protected ‪$connectionPool;
 
    protected ‪$minimumDateToImport;
 
    public function ‪__construct(
        ‪ExtensionRepository $repository,
        ‪Extension $extension,
        ‪ConnectionPool ‪$connectionPool,
        ‪ExtensionXmlParser ‪$parser
    ) {
        $this->extensionRepository = $repository;
        $this->extensionModel = $extension;
        $this->connectionPool = ‪$connectionPool;
        $this->parser = ‪$parser;
        $this->parser->‪attach($this);
 
        $connection = $this->connectionPool->getConnectionForTable(self::TABLE_NAME);
        $maxBindParameters = ‪PlatformInformation::getMaxBindParameters(
            $connection->getDatabasePlatform()
        );
        $countOfBindParamsPerRow = count(self::$fieldNames);
        // flush at least chunks of 50 elements - in case the currently used
        // database platform does not support that, the threshold is lowered
        $this->maxRowsPerChunk = (int)min(
            $this->maxRowsPerChunk,
            floor($maxBindParameters / $countOfBindParamsPerRow)
        );
        // Only import extensions that are compatible with 7.6 or higher.
        // TER only allows to publish extensions with compatibility if the TYPO3 version has been released
        // And 7.6 was released on 10th of November 2015.
        // This effectively reduces the number of extensions imported into this TYPO3 installation
        // by more than 70%. As long as the extensions.xml from TER includes these files, we need to "hack" this
        // within TYPO3 Core.
        // For TYPO3 v11.0, this date could be set to 2018-10-02 (v9 LTS release).
        // Also see https://decisions.typo3.org/t/reduce-size-of-extension-manager-db-table/329/
        $this->minimumDateToImport = strtotime('2017-04-04T00:00:00+00:00');
    }
 
    public function import(string $localExtensionListFile, string ‪$remoteIdentifier): void
    {
        // Remove all existing entries of this remote from the database
        $this->connectionPool
            ->getConnectionForTable(self::TABLE_NAME)
            ->delete(
                self::TABLE_NAME,
                ['remote' => ‪$remoteIdentifier],
                [‪Connection::PARAM_STR]
            );
        $this->remoteIdentifier = ‪$remoteIdentifier;
        $zlibStream = 'compress.zlib://';
        $this->sumRecords = 0;
        $this->parser->parseXml($zlibStream . $localExtensionListFile);
        // flush last rows to database if existing
        if (!empty($this->arrRows)) {
            $this->connectionPool
                ->getConnectionForTable(self::TABLE_NAME)
                ->bulkInsert(
                    'tx_extensionmanager_domain_model_extension',
                    $this->arrRows,
                    self::$fieldNames
                );
        }
        $this->‪markExtensionWithMaximumVersionAsCurrent(‪$remoteIdentifier);
    }
 
    protected function ‪loadIntoDatabase(ExtensionXmlParser $subject): void
    {
        if ($this->sumRecords !== 0 && $this->sumRecords % $this->maxRowsPerChunk === 0) {
            $this->connectionPool
                ->getConnectionForTable(self::TABLE_NAME)
                ->bulkInsert(
                    self::TABLE_NAME,
                    $this->arrRows,
                    self::$fieldNames
                );
            $this->arrRows = [];
        }
        if (!$subject->isValidVersionNumber()) {
            // Skip in case extension version is not valid
            return;
        }
        $versionRepresentations = VersionNumberUtility::convertVersionStringToArray($subject->getVersion());
        // order must match that of self::$fieldNames!
        $this->arrRows[] = [
            $subject->getExtkey(),
            $subject->getVersion(),
            $versionRepresentations['version_int'],
            // initialize current_version, correct value computed later:
            0,
            $subject->getAlldownloadcounter(),
            $subject->getDownloadcounter(),
            $subject->getTitle(),
            $subject->getOwnerusername(),
            $subject->getAuthorname(),
            $subject->getAuthoremail(),
            $subject->getAuthorcompany(),
            $subject->getLastuploaddate(),
            $subject->getT3xfilemd5(),
            ‪$this->remoteIdentifier,
            $this->extensionModel->getDefaultState($subject->getState() ?: ''),
            $subject->getReviewstate(),
            $this->extensionModel->getCategoryIndexFromStringOrNumber($subject->getCategory() ?: ''),
            $subject->getDescription() ?: '',
            $subject->getDependencies() ?: '',
            $subject->getUploadcomment() ?: '',
            $subject->getDocumentationLink() ?: '',
            $subject->getDistributionImage() ?: '',
            $subject->getDistributionWelcomeImage() ?: '',
        ];
        ++‪$this->sumRecords;
    }
 
    public function ‪update(\SplSubject $subject): void
    {
        if ($subject instanceof ‪ExtensionXmlParser) {
            if ((int)$subject->getLastuploaddate() > $this->minimumDateToImport) {
                $this->‪loadIntoDatabase($subject);
            }
        }
    }
 
    protected function ‪markExtensionWithMaximumVersionAsCurrent(string ‪$remoteIdentifier): void
    {
        $uidsOfCurrentVersion = $this->‪fetchMaximalVersionsForAllExtensions($remoteIdentifier);
        $queryBuilder = $this->connectionPool->getQueryBuilderForTable(self::TABLE_NAME);
        $connection = $this->connectionPool->getConnectionForTable(self::TABLE_NAME);
        $maxBindParameters = ‪PlatformInformation::getMaxBindParameters(
            $connection->getDatabasePlatform()
        );
 
        foreach (array_chunk($uidsOfCurrentVersion, $maxBindParameters - 10) as $chunk) {
            $queryBuilder
                ->update(self::TABLE_NAME)
                ->where(
                    $queryBuilder->expr()->in(
                        'uid',
                        $queryBuilder->createNamedParameter($chunk, ‪Connection::PARAM_INT_ARRAY)
                    )
                )
                ->set('current_version', 1)
                ->executeStatement();
        }
    }
 
    protected function ‪fetchMaximalVersionsForAllExtensions(string ‪$remoteIdentifier): array
    {
        $queryBuilder = $this->connectionPool->getQueryBuilderForTable(self::TABLE_NAME);
 
        $queryResult = $queryBuilder
            ->select('a.uid AS uid')
            ->from(self::TABLE_NAME, 'a')
            ->leftJoin(
                'a',
                self::TABLE_NAME,
                'b',
                $queryBuilder->expr()->and(
                    $queryBuilder->expr()->eq('a.remote', $queryBuilder->quoteIdentifier('b.remote')),
                    $queryBuilder->expr()->eq('a.extension_key', $queryBuilder->quoteIdentifier('b.extension_key')),
                    $queryBuilder->expr()->lt('a.integer_version', $queryBuilder->quoteIdentifier('b.integer_version'))
                )
            )
            ->where(
                $queryBuilder->expr()->eq(
                    'a.remote',
                    $queryBuilder->createNamedParameter(‪$remoteIdentifier)
                ),
                $queryBuilder->expr()->isNull('b.extension_key')
            )
            ->orderBy('a.uid')
            ->executeQuery();
 
        $extensionUids = [];
        while ($row = $queryResult->fetchAssociative()) {
            $extensionUids[] = $row['uid'];
        }
 
        return $extensionUids;
    }
}