LossyTokenizer.php

Go to the documentation of this file.

<?php
 
declare(strict_types=1);
 
/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */
 
namespace ‪TYPO3\CMS\Core\TypoScript\Tokenizer;
 
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\BlockCloseLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionElseLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionStopLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierAssignmentLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierBlockOpenLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierCopyLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierFunctionLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierReferenceLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierUnsetLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ImportLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ImportOldLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\ConstantAwareTokenStream;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierToken;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStream;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
 
final class ‪LossyTokenizer implements ‪TokenizerInterface
{
    private ‪LineStream ‪$lineStream;
 
    private ‪IdentifierTokenStream ‪$identifierStream;
    private ‪TokenStreamInterface ‪$valueStream;
 
    private array ‪$lines;
    private int ‪$currentLineNumber;
    private string ‪$currentLineString;
 
    public function ‪tokenize(string $source): ‪LineStream
    {
        $this->lineStream = new ‪LineStream();
        $this->currentLineNumber = -1;
        $this->lines = [];
        $this->‪splitLines($source);
 
        while (true) {
            $this->currentLineNumber++;
            if (!array_key_exists($this->currentLineNumber, $this->lines)) {
                break;
            }
            $this->currentLineString = trim($this->lines[$this->currentLineNumber]['line']);
            $nextChar = substr($this->currentLineString, 0, 1);
            if ($nextChar === '') {
                continue;
            }
            $nextTwoChars = substr($this->currentLineString, 0, 2);
            if ($nextChar === '#' || $nextTwoChars === '//') {
                continue;
            }
            if ($nextTwoChars === '/*') {
                // @todo: This is one of multiple places where multiline "/*" comments are parsed in this tokenizer. Other
                //        places are cluttered in detail methods. It might be more straight to have an early scanning
                //        phase through all lines to remove comments up front, to not wire especially the multiline comment
                //        parsing to single places, and throw away commented lines early. This isn't trivial though, since
                //        for instance "foo = bar /* not a comment */" then needs to be sorted out, too. Having an early
                //        "kick comments" loop however might be quicker in the end and would make the main parsing
                //        methods more concise and probably more bullet proof.
                //        Also note there are currently not-unit-tested edge cases, that will currently not parse as
                //        (maybe) expected. In the example below, "foo2 = bar2" is ignored. This is an issue with the
                //        LosslessTokenizer as well, probably, and we may rather want to declare this as invalid syntax?!
                //          foo = bar /* comment start
                //          comment end */ foo2 = bar2
                $this->‪ignoreUntilEndOfMultilineComment();
                continue;
            }
            if ($nextChar === '[') {
                $this->‪createConditionLine();
            } elseif ($nextChar === '}') {
                $this->lineStream->append((new ‪BlockCloseLine()));
            } elseif (str_starts_with($this->currentLineString, '@import')) {
                $this->‪parseImportLine();
            } elseif (str_starts_with($this->currentLineString, '<INCLUDE_TYPOSCRIPT:')) {
                $this->‪parseImportOld();
            } else {
                $this->‪parseIdentifier();
            }
        }
 
        return ‪$this->lineStream;
    }
 
    private function ‪splitLines($source): void
    {
        $vanillaLines = explode(chr(10), $source);
        $this->lines = array_map(
            fn(int $lineNumber, string $vanillaLine): array => [
                'line' => rtrim($vanillaLine, "\r"),
            ],
            array_keys($vanillaLines),
            $vanillaLines
        );
    }
 
    private function ‪ignoreUntilEndOfMultilineComment(): void
    {
        while (true) {
            if (str_contains($this->currentLineString, '*/')) {
                return;
            }
            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
                return;
            }
            $this->currentLineNumber++;
            $this->currentLineString = trim($this->lines[$this->currentLineNumber]['line']);
        }
    }
 
    private function ‪createConditionLine(): void
    {
        $upperCaseLine = strtoupper($this->currentLineString);
        if (str_starts_with($upperCaseLine, '[ELSE]')) {
            $this->lineStream->append((new ‪ConditionElseLine()));
            $this->currentLineString = trim(substr($this->currentLineString, 6));
            if (str_starts_with($this->currentLineString, '/*')) {
                $this->‪ignoreUntilEndOfMultilineComment();
            }
            return;
        }
        if (str_starts_with($upperCaseLine, '[END]')) {
            $this->lineStream->append((new ‪ConditionStopLine()));
            $this->currentLineString = trim(substr($this->currentLineString, 5));
            if (str_starts_with($this->currentLineString, '/*')) {
                $this->‪ignoreUntilEndOfMultilineComment();
            }
            return;
        }
        if (str_starts_with($upperCaseLine, '[GLOBAL]')) {
            $this->lineStream->append((new ‪ConditionStopLine()));
            $this->currentLineString = trim(substr($this->currentLineString, 8));
            if (str_starts_with($this->currentLineString, '/*')) {
                $this->‪ignoreUntilEndOfMultilineComment();
            }
            return;
        }
        $conditionBody = '';
        $conditionBodyCharCount = 0;
        $conditionBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
        $bracketCount = 1;
        while (true) {
            $nextChar = $conditionBodyChars[$conditionBodyCharCount] ?? null;
            if ($nextChar === null) {
                // end of chars
                return;
            }
            if ($nextChar === '[') {
                $bracketCount++;
                $conditionBody .= $nextChar;
                $conditionBodyCharCount++;
                continue;
            }
            if ($nextChar === ']') {
                $bracketCount--;
                if ($bracketCount === 0) {
                    if ($conditionBodyCharCount) {
                        $conditionBodyToken = new ‪Token(TokenType::T_VALUE, $conditionBody);
                        $this->lineStream->append((new ‪ConditionLine())->setValueToken($conditionBodyToken));
                        $conditionBodyCharCount++;
                        break;
                    }
                    $conditionBodyCharCount++;
                    break;
                }
                $conditionBody .= $nextChar;
                $conditionBodyCharCount++;
                continue;
            }
            $conditionBody .= $nextChar;
            $conditionBodyCharCount++;
        }
        $this->currentLineString = trim(mb_substr($this->currentLineString, $conditionBodyCharCount + 1));
        if (str_starts_with($this->currentLineString, '/*')) {
            $this->‪ignoreUntilEndOfMultilineComment();
        }
    }
 
    private function ‪parseBlockStart(): void
    {
        $this->currentLineString = trim(substr($this->currentLineString, 1));
        if (str_starts_with($this->currentLineString, '}')) {
            // Edge case: foo = { } in one line. Note content within {} is not parsed, everything behind { ends up as comment.
            $this->lineStream->append((new ‪IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream));
            $this->lineStream->append((new ‪BlockCloseLine()));
            return;
        }
        $this->lineStream->append((new ‪IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream));
    }
 
    private function ‪parseImportLine(): void
    {
        $this->currentLineString = trim(substr($this->currentLineString, 7));
 
        // Next char should be the opening tick or doubletick, otherwise we create a comment until end of line
        $nextChar = substr($this->currentLineString, 0, 1);
        if ($nextChar !== '\'' && $nextChar !== '"') {
            return;
        }
 
        $importBody = '';
        $importBodyCharCount = 0;
        $importBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
        while (true) {
            $nextChar = $importBodyChars[$importBodyCharCount] ?? null;
            if ($nextChar === null) {
                // end of chars
                if ($importBodyCharCount) {
                    $importBodyToken = (new ‪Token(TokenType::T_VALUE, $importBody));
                    $this->lineStream->append((new ‪ImportLine())->setValueToken($importBodyToken));
                    return;
                }
                return;
            }
            if ($nextChar === '\'' || $nextChar === '"') {
                if ($importBodyCharCount) {
                    $importBodyToken = new ‪Token(TokenType::T_VALUE, $importBody);
                    $this->lineStream->append((new ‪ImportLine())->setValueToken($importBodyToken));
                    break;
                }
                break;
            }
            $importBody .= $nextChar;
            $importBodyCharCount++;
        }
        $this->currentLineString = trim(mb_substr($this->currentLineString, $importBodyCharCount + 2));
        if (str_starts_with($this->currentLineString, '/*')) {
            $this->‪ignoreUntilEndOfMultilineComment();
        }
    }
 
    private function ‪parseImportOld(): void
    {
        $this->currentLineString = substr($this->currentLineString, 20);
        $importBody = '';
        $importBodyCharCount = 0;
        $importBodyChars = mb_str_split($this->currentLineString, 1, 'UTF-8');
        while (true) {
            $nextChar = $importBodyChars[$importBodyCharCount] ?? null;
            if ($nextChar === null) {
                // end of chars
                if ($importBodyCharCount) {
                    $importBodyToken = (new ‪Token(TokenType::T_VALUE, $importBody));
                    $this->lineStream->append((new ‪ImportOldLine())->setValueToken($importBodyToken));
                    return;
                }
                return;
            }
            if ($nextChar === '>') {
                if ($importBodyCharCount) {
                    $importBodyToken = new ‪Token(TokenType::T_VALUE, $importBody);
                    $this->lineStream->append((new ‪ImportOldLine())->setValueToken($importBodyToken));
                    break;
                }
                break;
            }
            $importBody .= $nextChar;
            $importBodyCharCount++;
        }
        $this->currentLineString = trim(mb_substr($this->currentLineString, $importBodyCharCount + 2));
        if (str_starts_with($this->currentLineString, '/*')) {
            $this->‪ignoreUntilEndOfMultilineComment();
        }
    }
 
    private function ‪parseIdentifier(): void
    {
        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
        $currentPosition = $this->‪parseIdentifierUntilStopChar($splitLine);
        if (!$currentPosition) {
            return;
        }
        $this->currentLineString = trim(substr($this->currentLineString, $currentPosition));
        $nextChar = substr($this->currentLineString, 0, 1);
        $nextTwoChars = $nextChar . substr($this->currentLineString, 1, 1);
        if ($nextTwoChars === '=<') {
            $this->‪parseOperatorReference();
            return;
        }
        if ($nextChar === '=') {
            $this->‪parseOperatorAssignment();
            return;
        }
        if ($nextChar === '{') {
            $this->‪parseBlockStart();
            return;
        }
        if ($nextChar === '>') {
            $this->‪parseOperatorUnset();
            return;
        }
        if ($nextChar === '<') {
            $this->‪parseOperatorCopy();
            return;
        }
        if ($nextChar === '(') {
            $this->‪parseOperatorMultilineAssignment();
            return;
        }
        if ($nextTwoChars === ':=') {
            $this->‪parseOperatorFunction();
        }
        if ($nextTwoChars === '/*') {
            $this->‪ignoreUntilEndOfMultilineComment();
        }
    }
 
    private function ‪parseOperatorUnset(): void
    {
        $this->lineStream->append((new ‪IdentifierUnsetLine())->setIdentifierTokenStream($this->identifierStream));
        $this->currentLineString = trim(trim(trim($this->currentLineString), '>'));
        if (str_starts_with($this->currentLineString, '/*')) {
            $this->‪ignoreUntilEndOfMultilineComment();
        }
    }
 
    private function ‪parseOperatorAssignment(): void
    {
        $this->currentLineString = trim(substr($this->currentLineString, 1));
        $this->valueStream = new ‪TokenStream();
        $this->‪parseValueForConstants();
        $this->lineStream->append((new ‪IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
    }
 
    private function ‪parseOperatorMultilineAssignment(): void
    {
        $this->valueStream = new ‪TokenStream();
        $this->currentLineString = substr($this->currentLineString, 1);
        // True if we're currently in the line with the opening '('
        $isFirstLine = true;
        // True if the first line has a first value token: "foo ( thisIsTheFirstValueToken"
        $valueOnFirstLine = false;
        // True if the line after '(' is parsed
        $isSecondLine = false;
        while (true) {
            if (str_starts_with(ltrim($this->currentLineString), ')')) {
                $this->currentLineString = trim(substr($this->currentLineString, 1));
                if (!$this->valueStream->isEmpty()) {
                    $this->lineStream->append((new ‪IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
                }
                if (str_starts_with($this->currentLineString, '/*')) {
                    $this->‪ignoreUntilEndOfMultilineComment();
                }
                return;
            }
            if ($isFirstLine && str_ends_with($this->currentLineString, ')')) {
                $this->currentLineString = substr($this->currentLineString, 0, -1);
                if (strlen($this->currentLineString) > 1) {
                    $this->‪parseValueForConstants();
                    $this->lineStream->append((new ‪IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
                    return;
                }
                return;
            }
            if ($isFirstLine && strlen($this->currentLineString)) {
                $this->‪parseValueForConstants();
                $valueOnFirstLine = true;
            }
            if (($isFirstLine && $valueOnFirstLine)
                || (!$isFirstLine && !$isSecondLine)
            ) {
                $this->valueStream->append(new ‪Token(TokenType::T_NEWLINE, "\n"));
            }
            if (!$isFirstLine && strlen($this->currentLineString)) {
                $this->‪parseValueForConstants();
            }
            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
                return;
            }
            if ($isFirstLine) {
                $isSecondLine = true;
            } else {
                $isSecondLine = false;
            }
            $isFirstLine = false;
            $valueOnFirstLine = false;
            $this->currentLineNumber++;
            $this->currentLineString = $this->lines[‪$this->currentLineNumber]['line'];
        }
    }
 
    private function ‪parseOperatorCopy(): void
    {
        $this->currentLineString = trim(substr($this->currentLineString, 1));
        ‪$identifierStream = ‪$this->identifierStream;
        $charsHandled = $this->‪parseIdentifierAtEndOfLine();
        $referenceStream = ‪$this->identifierStream;
        if ($referenceStream->isEmpty()) {
            return;
        }
        $this->lineStream->‪append(
            (new ‪IdentifierCopyLine())
                ->setIdentifierTokenStream(‪$identifierStream)
                ->setValueTokenStream($referenceStream)
        );
        $this->currentLineString = trim(mb_substr($this->currentLineString, $charsHandled));
        if (str_starts_with($this->currentLineString, '/*')) {
            $this->‪ignoreUntilEndOfMultilineComment();
        }
    }
 
    private function ‪parseOperatorReference(): void
    {
        $this->currentLineString = trim(substr($this->currentLineString, 2));
        ‪$identifierStream = ‪$this->identifierStream;
        $charsHandled = $this->‪parseIdentifierAtEndOfLine();
        $referenceStream = ‪$this->identifierStream;
        if ($referenceStream->isEmpty()) {
            return;
        }
        $this->lineStream->‪append(
            (new ‪IdentifierReferenceLine())
                ->setIdentifierTokenStream(‪$identifierStream)
                ->setValueTokenStream($referenceStream)
        );
        $this->currentLineString = trim(mb_substr($this->currentLineString, $charsHandled));
        if (str_starts_with($this->currentLineString, '/*')) {
            $this->‪ignoreUntilEndOfMultilineComment();
        }
    }
 
    private function ‪parseIdentifierAtEndOfLine(): int
    {
        $this->identifierStream = new ‪IdentifierTokenStream();
        $isRelative = false;
        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
        $char = $splitLine[0] ?? null;
        if ($char === null) {
            return 0;
        }
        $nextTwoChars = $char . ($splitLine[1] ?? '');
        if ($char === '.') {
            // A relative right side: foo.bar < .foo (note the dot!). we identifierStream->setRelative() and
            // get rid of the dot for the rest of the processing.
            $isRelative = true;
            array_shift($splitLine);
            $this->currentLineString = substr($this->currentLineString, 1);
        }
        if ($char === '#') {
            return 1;
        }
        if ($nextTwoChars === '//') {
            return 2;
        }
        if ($nextTwoChars === '/*') {
            $this->‪ignoreUntilEndOfMultilineComment();
            return 0;
        }
        $charsHandled = $this->‪parseIdentifierUntilStopChar($splitLine, $isRelative);
        return $charsHandled;
    }
 
    private function ‪parseIdentifierUntilStopChar(array $splitLine, bool $isRelative = false): int
    {
        $this->identifierStream = new ‪IdentifierTokenStream();
        if ($isRelative) {
            $this->identifierStream->setRelative();
        }
        $currentPosition = 0;
        $currentIdentifierBody = '';
        $currentIdentifierCharCount = 0;
        while (true) {
            $nextChar = $splitLine[$currentPosition] ?? null;
            if ($nextChar === null) {
                if ($currentIdentifierCharCount) {
                    $identifierToken = new ‪IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody);
                    $this->identifierStream->append($identifierToken);
                }
                return $currentPosition;
            }
            $nextTwoChars = $nextChar . ($splitLine[$currentPosition + 1] ?? null);
            if ($currentPosition > 0
                && ($nextChar === ' ' || $nextChar === "\t" || $nextChar === '=' || $nextChar === '<' || $nextChar === '>' || $nextChar === '{' || $nextTwoChars === ':=' || $nextChar === '(')
            ) {
                if ($currentIdentifierCharCount) {
                    $identifierToken = new ‪IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody);
                    $this->identifierStream->append($identifierToken);
                }
                break;
            }
            if ($nextTwoChars === '\\.') {
                // A quoted dot is part of *this* identifier
                $currentIdentifierBody .= '.';
                $currentPosition += 2;
                $currentIdentifierCharCount++;
            } elseif ($nextChar === '.') {
                if ($currentIdentifierCharCount) {
                    $identifierToken = new ‪IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody);
                    $this->identifierStream->append($identifierToken);
                    $currentIdentifierCharCount = 0;
                    $currentIdentifierBody = '';
                }
                $currentPosition++;
            } else {
                $currentIdentifierBody .= $nextChar;
                $currentIdentifierCharCount++;
                $currentPosition++;
            }
        }
        return $currentPosition;
    }
 
    private function ‪parseOperatorFunction(): void
    {
        $this->currentLineString = trim(substr($this->currentLineString, 2));
        if ($this->currentLineString === '') {
            return;
        }
        $functionName = '';
        $functionNameCharCount = 0;
        $functionChars = mb_str_split($this->currentLineString, 1, 'UTF-8');
        while (true) {
            $nextChar = $functionChars[$functionNameCharCount] ?? null;
            if ($nextChar === null) {
                // end of chars
                return;
            }
            if ($nextChar === '(') {
                if ($functionNameCharCount) {
                    $functionNameToken = new ‪Token(TokenType::T_FUNCTION_NAME, $functionName);
                    $functionNameCharCount++;
                    break;
                }
                return;
            }
            $functionName .= $nextChar;
            $functionNameCharCount++;
        }
        $functionBodyStartPosition = $functionNameCharCount;
        $functionBody = '';
        $functionBodyCharCount = 0;
        $functionValueToken = false;
        while (true) {
            $nextChar = $functionChars[$functionBodyStartPosition + $functionBodyCharCount] ?? null;
            if ($nextChar === null) {
                return;
            }
            if ($nextChar === ')') {
                if ($functionBodyCharCount) {
                    $functionValueToken = new ‪Token(TokenType::T_VALUE, $functionBody);
                    $functionBodyCharCount++;
                }
                break;
            }
            $functionBody .= $nextChar;
            $functionBodyCharCount++;
        }
        $line = (new ‪IdentifierFunctionLine())
            ->setIdentifierTokenStream($this->identifierStream)
            ->setFunctionNameToken($functionNameToken); 
        if ($functionValueToken) {
            $line->setFunctionValueToken($functionValueToken);
        }
        $this->lineStream->append($line);
        // Check for multiline comment
        $this->currentLineString = implode('', array_slice($functionChars, $functionBodyStartPosition + $functionBodyCharCount + 1));
        if (mb_strlen($this->currentLineString) >= 1 && str_starts_with($this->currentLineString, '/*')) {
            $this->‪ignoreUntilEndOfMultilineComment();
        }
    }
 
    private function ‪parseValueForConstants(): void
    {
        if (!str_contains($this->currentLineString, '{$')) {
            $valueToken = new ‪Token(TokenType::T_VALUE, $this->currentLineString);
            $this->valueStream->append($valueToken);
            return;
        }
        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
        $isInConstant = false;
        $currentPosition = 0;
        $currentString = '';
        $currentStringLength = 0;
        while (true) {
            $char = $splitLine[$currentPosition] ?? null;
            if ($char === null) {
                if ($currentStringLength) {
                    $valueToken = new ‪Token(TokenType::T_VALUE, $currentString);
                    $this->valueStream->append($valueToken);
                }
                break;
            }
            $nextTwoChars = $char . ($splitLine[$currentPosition + 1] ?? '');
            if ($nextTwoChars === '{$') {
                $isInConstant = true;
                if ($currentStringLength) {
                    $valueToken = new ‪Token(TokenType::T_VALUE, $currentString);
                    $this->valueStream->append($valueToken);
                }
                $currentString = '{$';
                $currentPosition += 2;
                continue;
            }
            if ($isInConstant && $char === '}') {
                $valueToken = new ‪Token(TokenType::T_CONSTANT, $currentString . '}');
                if (!$this->valueStream instanceof ‪ConstantAwareTokenStream) {
                    $this->valueStream = (new ‪ConstantAwareTokenStream())->setAll($this->valueStream->getAll());
                }
                $this->valueStream->append($valueToken);
                $currentPosition++;
                $currentString = '';
                $currentStringLength = 0;
                $isInConstant = false;
                continue;
            }
            $currentPosition++;
            $currentStringLength++;
            $currentString .= $char;
        }
    }
}