LosslessTokenizer.php

Go to the documentation of this file.

<?php
 
declare(strict_types=1);
 
/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */
 
namespace ‪TYPO3\CMS\Core\TypoScript\Tokenizer;
 
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\BlockCloseLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\CommentLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionElseLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionStopLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\EmptyLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierAssignmentLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierBlockOpenLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierCopyLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierFunctionLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierReferenceLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierUnsetLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ImportLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ImportOldLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\InvalidLine;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\ConstantAwareTokenStream;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierToken;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStream;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
use ‪TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
 
final class ‪LosslessTokenizer implements ‪TokenizerInterface
{
    private ‪LineStream ‪$lineStream;
 
    private ‪TokenStreamInterface ‪$tokenStream;
    private ‪IdentifierTokenStream ‪$identifierStream;
    private ‪TokenStreamInterface ‪$valueStream;
 
    private array ‪$lines;
    private int ‪$currentLineNumber;
    private string ‪$currentLineString;
    private \closure ‪$currentLinebreakCallback;
    private int ‪$currentColumnInLine = 0;
 
    public function ‪tokenize(string $source): ‪LineStream
    {
        $this->lineStream = new ‪LineStream();
        $this->currentLineNumber = -1;
        $this->lines = [];
        $this->‪splitLines($source);
 
        while (true) {
            $this->tokenStream = new ‪TokenStream();
            $this->currentLineNumber++;
            if (!array_key_exists($this->currentLineNumber, $this->lines)) {
                break;
            }
            $this->currentColumnInLine = 0;
            $this->currentLineString = $this->lines[‪$this->currentLineNumber]['line'];
            $this->currentLinebreakCallback = $this->lines[‪$this->currentLineNumber]['linebreakCallback'];
            $this->‪parseTabsAndWhitespaces();
            $nextChar = substr($this->currentLineString, 0, 1);
            if ($nextChar === '') {
                (‪$this->currentLinebreakCallback)();
                if (!$this->tokenStream->isEmpty()) {
                    $this->‪createEmptyLine();
                }
                continue;
            }
            $nextTwoChars = substr($this->currentLineString, 0, 2);
            if ($nextChar === '#') {
                $this->‪createHashCommentLine();
            } elseif ($nextTwoChars === '//') {
                $this->‪createDoubleSlashCommentLine();
            } elseif ($nextTwoChars === '/*') {
                $this->‪createMultilineCommentLine();
            } elseif ($nextChar === '[') {
                $this->‪createConditionLine();
            } elseif ($nextChar === '}') {
                $this->‪createBlockStopLine();
            } elseif (str_starts_with($this->currentLineString, '@import')) {
                $this->‪parseImportLine();
            } elseif (str_starts_with($this->currentLineString, '<INCLUDE_TYPOSCRIPT:')) {
                $this->‪parseImportOld();
            } else {
                $this->‪parseIdentifier();
            }
        }
 
        return ‪$this->lineStream;
    }
 
    private function ‪splitLines($source): void
    {
        $vanillaLines = explode(chr(10), $source);
        $this->lines = array_map(
            fn(int $lineNumber, string $vanillaLine): array => [
                'line' => rtrim($vanillaLine, "\r"),
                'linebreakCallback' => str_ends_with($vanillaLine, "\r")
                    ? fn() => $this->tokenStream->append(new ‪Token(TokenType::T_NEWLINE, "\r\n", $lineNumber, mb_strlen($vanillaLine) - 1))
                    : fn() => $this->tokenStream->append(new ‪Token(TokenType::T_NEWLINE, "\n", $lineNumber, mb_strlen($vanillaLine))),
            ],
            array_keys($vanillaLines),
            $vanillaLines
        );
        // Set the linebreak callback of last line to empty to suppress dangling linebreak tokens
        $this->lines[count($vanillaLines) - 1]['linebreakCallback'] = function () {};
    }
 
    private function ‪createEmptyLine(): void
    {
        $this->lineStream->append((new ‪EmptyLine())->setTokenStream($this->tokenStream));
    }
 
    private function ‪parseTabsAndWhitespaces(): void
    {
        $matches = [];
        if (preg_match('#^(\s+)(.*)$#', $this->currentLineString, $matches)) {
            $this->tokenStream->append(new ‪Token(TokenType::T_BLANK, $matches[1], $this->currentLineNumber, $this->currentColumnInLine));
            $this->currentLineString = $matches[2];
            $this->currentColumnInLine = $this->currentColumnInLine + strlen($matches[1]);
        }
    }
 
    private function ‪makeComment(): void
    {
        $nextChar = substr($this->currentLineString, 0, 1);
        if ($nextChar === '') {
            (‪$this->currentLinebreakCallback)();
            return;
        }
        $nextTwoChars = substr($this->currentLineString, 0, 2);
        if ($nextChar === '#') {
            $this->‪parseHashComment();
        } elseif ($nextTwoChars === '//') {
            $this->‪parseDoubleSlashComment();
        } elseif ($nextTwoChars === '/*') {
            $this->‪parseMultilineComment();
        } else {
            $this->‪parseHashComment();
        }
    }
 
    private function ‪createHashCommentLine(): void
    {
        $this->‪parseHashComment();
        $this->lineStream->append((new ‪CommentLine())->setTokenStream($this->tokenStream));
    }
 
    private function ‪parseHashComment(): void
    {
        $this->tokenStream->append(new ‪Token(TokenType::T_COMMENT_ONELINE_HASH, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
        (‪$this->currentLinebreakCallback)();
    }
 
    private function ‪createDoubleSlashCommentLine(): void
    {
        $this->‪parseDoubleSlashComment();
        $this->lineStream->append((new ‪CommentLine())->setTokenStream($this->tokenStream));
    }
 
    private function ‪parseDoubleSlashComment(): void
    {
        $this->tokenStream->append(new ‪Token(TokenType::T_COMMENT_ONELINE_DOUBLESLASH, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
        (‪$this->currentLinebreakCallback)();
    }
 
    private function ‪createMultilineCommentLine(): void
    {
        $this->‪parseMultilineComment();
        $this->lineStream->append((new ‪CommentLine())->setTokenStream($this->tokenStream));
    }
 
    private function ‪parseMultilineComment(): void
    {
        $this->tokenStream->append(new ‪Token(TokenType::T_COMMENT_MULTILINE_START, '/*', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine += 2;
        $this->currentLineString = substr($this->currentLineString, 2);
        while (true) {
            if (str_ends_with($this->currentLineString, '*/')) {
                if (strlen($this->currentLineString) > 2) {
                    $this->tokenStream->append(new ‪Token(TokenType::T_VALUE, substr($this->currentLineString, 0, -2), $this->currentLineNumber, $this->currentColumnInLine));
                }
                $this->tokenStream->append(new ‪Token(TokenType::T_COMMENT_MULTILINE_STOP, '*/', $this->currentLineNumber, $this->currentColumnInLine + strlen($this->currentLineString) - 2));
                (‪$this->currentLinebreakCallback)();
                return;
            }
            if (strlen($this->currentLineString)) {
                $this->tokenStream->append(new ‪Token(TokenType::T_VALUE, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
            }
            (‪$this->currentLinebreakCallback)();
            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
                return;
            }
            $this->currentLineNumber++;
            $this->currentColumnInLine = 0;
            $this->currentLineString = $this->lines[‪$this->currentLineNumber]['line'];
            $this->currentLinebreakCallback = $this->lines[‪$this->currentLineNumber]['linebreakCallback'];
        }
    }
 
    private function ‪createConditionLine(): void
    {
        $upperCaseLine = strtoupper($this->currentLineString);
        $this->tokenStream->append(new ‪Token(TokenType::T_CONDITION_START, '[', $this->currentLineNumber, $this->currentColumnInLine));
        if (str_starts_with($upperCaseLine, '[ELSE]')) {
            $this->tokenStream->append(new ‪Token(TokenType::T_CONDITION_ELSE, substr($this->currentLineString, 1, 4), $this->currentLineNumber, $this->currentColumnInLine + 1));
            $this->tokenStream->append(new ‪Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 5));
            $this->currentLineString = substr($this->currentLineString, 6);
            $this->currentColumnInLine += 6;
            $this->‪parseTabsAndWhitespaces();
            $this->‪makeComment();
            $this->lineStream->append((new ‪ConditionElseLine())->setTokenStream($this->tokenStream));
            return;
        }
        if (str_starts_with($upperCaseLine, '[END]')) {
            $this->tokenStream->append(new ‪Token(TokenType::T_CONDITION_END, substr($this->currentLineString, 1, 3), $this->currentLineNumber, $this->currentColumnInLine + 1));
            $this->tokenStream->append(new ‪Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 4));
            $this->currentLineString = substr($this->currentLineString, 5);
            $this->currentColumnInLine += 5;
            $this->‪parseTabsAndWhitespaces();
            $this->‪makeComment();
            $this->lineStream->append((new ‪ConditionStopLine())->setTokenStream($this->tokenStream));
            return;
        }
        if (str_starts_with($upperCaseLine, '[GLOBAL]')) {
            $this->tokenStream->append(new ‪Token(TokenType::T_CONDITION_GLOBAL, substr($this->currentLineString, 1, 6), $this->currentLineNumber, $this->currentColumnInLine + 1));
            $this->tokenStream->append(new ‪Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 7));
            $this->currentLineString = substr($this->currentLineString, 8);
            $this->currentColumnInLine += 8;
            $this->‪parseTabsAndWhitespaces();
            $this->‪makeComment();
            $this->lineStream->append((new ‪ConditionStopLine())->setTokenStream($this->tokenStream));
            return;
        }
        $conditionBody = '';
        $conditionBodyStartPosition = $this->currentColumnInLine + 1;
        $conditionBodyCharCount = 0;
        $conditionBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
        $bracketCount = 1;
        while (true) {
            $nextChar = $conditionBodyChars[$conditionBodyCharCount] ?? null;
            if ($nextChar === null) {
                // end of chars
                if ($conditionBodyCharCount) {
                    $this->tokenStream->append(new ‪Token(TokenType::T_VALUE, $conditionBody, $this->currentLineNumber, $conditionBodyStartPosition));
                }
                (‪$this->currentLinebreakCallback)();
                $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            if ($nextChar === '[') {
                $bracketCount++;
                $conditionBody .= $nextChar;
                $conditionBodyCharCount++;
                continue;
            }
            if ($nextChar === ']') {
                $bracketCount--;
                if ($bracketCount === 0) {
                    if ($conditionBodyCharCount) {
                        $conditionBodyToken = new ‪Token(TokenType::T_VALUE, $conditionBody, $this->currentLineNumber, $conditionBodyStartPosition);
                        $this->tokenStream->append($conditionBodyToken);
                        $this->tokenStream->append(new ‪Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + $conditionBodyCharCount + 1));
                        $this->currentLineString = mb_substr($this->currentLineString, $conditionBodyCharCount + 2);
                        $this->currentColumnInLine = $this->currentColumnInLine + $conditionBodyCharCount + 2;
                        $this->‪parseTabsAndWhitespaces();
                        $this->‪makeComment();
                        $this->lineStream->append((new ‪ConditionLine())->setTokenStream($this->tokenStream)->setValueToken($conditionBodyToken));
                        return;
                    }
                    $this->tokenStream->append(new ‪Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + $conditionBodyCharCount + 1));
                    $this->currentLineString = mb_substr($this->currentLineString, $conditionBodyCharCount + 2);
                    $this->currentColumnInLine = $this->currentColumnInLine + $conditionBodyCharCount + 2;
                    $this->‪parseTabsAndWhitespaces();
                    $this->‪makeComment();
                    $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
                    return;
                }
                $conditionBody .= $nextChar;
                $conditionBodyCharCount++;
                continue;
            }
            $conditionBody .= $nextChar;
            $conditionBodyCharCount++;
        }
    }
 
    private function ‪createBlockStopLine(): void
    {
        $this->tokenStream->append(new ‪Token(TokenType::T_BLOCK_STOP, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine++;
        $this->currentLineString = substr($this->currentLineString, 1);
        $this->‪makeComment();
        $this->lineStream->append((new ‪BlockCloseLine())->setTokenStream($this->tokenStream));
    }
 
    private function ‪parseBlockStart(): void
    {
        $this->tokenStream->append(new ‪Token(TokenType::T_BLOCK_START, '{', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine++;
        $this->currentLineString = substr($this->currentLineString, 1);
        $this->‪parseTabsAndWhitespaces();
        if (str_starts_with($this->currentLineString, '}')) {
            // Edge case: foo = { } in one line. Note content within {} is not parsed, everything behind { ends up as comment.
            $this->lineStream->append((new ‪IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream)->setTokenStream($this->tokenStream));
            $this->tokenStream = new ‪TokenStream();
            $this->tokenStream->append(new ‪Token(TokenType::T_BLOCK_STOP, '}', $this->currentLineNumber, $this->currentColumnInLine));
            $this->currentLineString = substr($this->currentLineString, 1);
            $this->currentColumnInLine++;
            $this->‪makeComment();
            $this->lineStream->append((new ‪BlockCloseLine())->setTokenStream($this->tokenStream));
            return;
        }
        $this->‪makeComment();
        $this->lineStream->append((new ‪IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream)->setTokenStream($this->tokenStream));
    }
 
    private function ‪parseImportLine(): void
    {
        $this->tokenStream->append(new ‪Token(TokenType::T_IMPORT_KEYWORD, '@import', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine += 7;
        $this->currentLineString = substr($this->currentLineString, 7);
        $this->‪parseTabsAndWhitespaces();
 
        // Next char should be the opening tick or doubletick, otherwise we create a comment until end of line
        $nextChar = substr($this->currentLineString, 0, 1);
        if ($nextChar !== '\'' && $nextChar !== '"') {
            $this->‪makeComment();
            $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        $this->tokenStream->append(new ‪Token(TokenType::T_IMPORT_START, $nextChar, $this->currentLineNumber, $this->currentColumnInLine));
 
        $importBody = '';
        $importBodyStartPosition = $this->currentColumnInLine + 1;
        $importBodyCharCount = 0;
        $importBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
        while (true) {
            $nextChar = $importBodyChars[$importBodyCharCount] ?? null;
            if ($nextChar === null) {
                // end of chars
                if ($importBodyCharCount) {
                    $importBodyToken = (new ‪Token(TokenType::T_VALUE, $importBody, $this->currentLineNumber, $importBodyStartPosition));
                    $this->tokenStream->append($importBodyToken);
                    (‪$this->currentLinebreakCallback)();
                    $this->lineStream->append((new ‪ImportLine())->setTokenStream($this->tokenStream)->setValueToken($importBodyToken));
                    return;
                }
                (‪$this->currentLinebreakCallback)();
                $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            if ($nextChar === '\'' || $nextChar === '"') {
                if ($importBodyCharCount) {
                    $importBodyToken = new ‪Token(TokenType::T_VALUE, $importBody, $this->currentLineNumber, $importBodyStartPosition);
                    $this->tokenStream->append($importBodyToken);
                    $this->tokenStream->append(new ‪Token(TokenType::T_IMPORT_STOP, $nextChar, $this->currentLineNumber, $this->currentColumnInLine + $importBodyCharCount + 1));
                    $this->currentLineString = mb_substr($this->currentLineString, $importBodyCharCount + 2);
                    $this->currentColumnInLine = $this->currentColumnInLine + $importBodyCharCount + 2;
                    $this->‪parseTabsAndWhitespaces();
                    $this->‪makeComment();
                    $this->lineStream->append((new ‪ImportLine())->setTokenStream($this->tokenStream)->setValueToken($importBodyToken));
                    return;
                }
                $this->tokenStream->append(new ‪Token(TokenType::T_IMPORT_STOP, $nextChar, $this->currentLineNumber, $this->currentColumnInLine + $importBodyCharCount + 1));
                $this->currentLineString = mb_substr($this->currentLineString, $importBodyCharCount + 2);
                $this->currentColumnInLine = $this->currentColumnInLine + $importBodyCharCount + 2;
                $this->‪parseTabsAndWhitespaces();
                $this->‪makeComment();
                $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            $importBody .= $nextChar;
            $importBodyCharCount++;
        }
    }
 
    private function ‪parseImportOld(): void
    {
        $this->tokenStream->append(new ‪Token(TokenType::T_IMPORT_KEYWORD_OLD, '<INCLUDE_TYPOSCRIPT:', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine += 20;
        $this->currentLineString = substr($this->currentLineString, 20);
        $importBody = '';
        $importBodyStartPosition = ‪$this->currentColumnInLine;
        $importBodyCharCount = 0;
        $importBodyChars = mb_str_split($this->currentLineString, 1, 'UTF-8');
        while (true) {
            $nextChar = $importBodyChars[$importBodyCharCount] ?? null;
            if ($nextChar === null) {
                // end of chars
                if ($importBodyCharCount) {
                    $importBodyToken = (new ‪Token(TokenType::T_VALUE, $importBody, $this->currentLineNumber, $importBodyStartPosition));
                    $this->tokenStream->append($importBodyToken);
                    (‪$this->currentLinebreakCallback)();
                    $this->lineStream->append((new ‪ImportOldLine())->setTokenStream($this->tokenStream)->setValueToken($importBodyToken));
                    return;
                }
                (‪$this->currentLinebreakCallback)();
                $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            if ($nextChar === '>') {
                if ($importBodyCharCount) {
                    $importBodyToken = new ‪Token(TokenType::T_VALUE, $importBody, $this->currentLineNumber, $importBodyStartPosition);
                    $this->tokenStream->append($importBodyToken);
                    $this->tokenStream->append(new ‪Token(‪TokenType::T_IMPORT_KEYWORD_OLD_STOP, '>', $this->currentLineNumber, $this->currentColumnInLine + $importBodyCharCount));
                    $this->currentLineString = mb_substr($this->currentLineString, $importBodyCharCount + 1);
                    $this->currentColumnInLine = $this->currentColumnInLine + $importBodyCharCount + 1;
                    $this->‪parseTabsAndWhitespaces();
                    $this->‪makeComment();
                    $this->lineStream->append((new ‪ImportOldLine())->setTokenStream($this->tokenStream)->setValueToken($importBodyToken));
                    return;
                }
                $this->tokenStream->append(new ‪Token(‪TokenType::T_IMPORT_KEYWORD_OLD_STOP, '>', $this->currentLineNumber, $this->currentColumnInLine + $importBodyCharCount));
                $this->currentLineString = mb_substr($this->currentLineString, $importBodyCharCount + 1);
                $this->currentColumnInLine = $this->currentColumnInLine + $importBodyCharCount + 1;
                $this->‪parseTabsAndWhitespaces();
                $this->‪makeComment();
                $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            $importBody .= $nextChar;
            $importBodyCharCount++;
        }
    }
 
    private function ‪parseIdentifier(): void
    {
        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
        $currentPosition = $this->‪parseIdentifierUntilStopChar($splitLine);
        if (!$currentPosition) {
            $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        $this->currentLineString = substr($this->currentLineString, $currentPosition);
        $this->currentColumnInLine = $this->currentColumnInLine + $currentPosition;
        $currentColumnInLineBefore = ‪$this->currentColumnInLine;
        $this->‪parseTabsAndWhitespaces();
        $currentPosition = $currentPosition + $this->currentColumnInLine - $currentColumnInLineBefore;
        $nextChar = $splitLine[$currentPosition] ?? null;
        $nextTwoChars = $nextChar . ($splitLine[$currentPosition + 1] ?? '');
        if ($nextTwoChars === '=<') {
            $this->‪parseOperatorReference();
            return;
        }
        if ($nextChar === '=') {
            $this->‪parseOperatorAssignment();
            return;
        }
        if ($nextChar === '{') {
            $this->‪parseBlockStart();
            return;
        }
        if ($nextChar === '>') {
            $this->‪parseOperatorUnset();
            return;
        }
        if ($nextChar === '<') {
            $this->‪parseOperatorCopy();
            return;
        }
        if ($nextChar === '(') {
            $this->‪parseOperatorMultilineAssignment();
            return;
        }
        if ($nextTwoChars === ':=') {
            $this->‪parseOperatorFunction();
            return;
        }
        if ($nextChar === '#') {
            $this->‪parseHashComment();
            $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        if ($nextTwoChars === '//') {
            $this->‪parseDoubleSlashComment();
            $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        if ($nextTwoChars === '/*') {
            $this->‪parseMultilineComment();
            $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        if ($nextChar === null) {
            (‪$this->currentLinebreakCallback)();
            $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
        }
    }
 
    private function ‪parseOperatorAssignment(): void
    {
        $this->tokenStream->append(new ‪Token(TokenType::T_OPERATOR_ASSIGNMENT, '=', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine++;
        $this->currentLineString = substr($this->currentLineString, 1);
        $this->‪parseTabsAndWhitespaces();
        $this->valueStream = new ‪TokenStream();
        $this->‪parseValueForConstants();
        (‪$this->currentLinebreakCallback)();
        $this->lineStream->append((new ‪IdentifierAssignmentLine())->setTokenStream($this->tokenStream)->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
    }
 
    private function ‪parseOperatorMultilineAssignment(): void
    {
        $this->valueStream = new ‪TokenStream();
        $this->tokenStream->append(new ‪Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_START, '(', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine++;
        $this->currentLineString = substr($this->currentLineString, 1);
        // True if we're currently in the line with the opening '('
        $isFirstLine = true;
        // True if the first line has a first value token: "foo ( thisIsTheFirstValueToken"
        $valueOnFirstLine = false;
        // True if the line after '(' is parsed
        $isSecondLine = false;
        $previousLineCallback = function () {};
        while (true) {
            if (str_starts_with(ltrim($this->currentLineString), ')')) {
                $this->‪parseTabsAndWhitespaces();
                $this->tokenStream->append(new ‪Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine));
                $this->currentLineString = substr($this->currentLineString, 1);
                $this->currentColumnInLine++;
                $this->‪parseTabsAndWhitespaces();
                $this->‪makeComment();
                if ($this->valueStream->isEmpty()) {
                    $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
                } else {
                    $this->lineStream->append((new ‪IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream)->setTokenStream($this->tokenStream));
                }
                return;
            }
            if ($isFirstLine && str_ends_with($this->currentLineString, ')')) {
                // Special case if the ')' is on same line as the opening '('
                $this->currentLineString = substr($this->currentLineString, 0, -1);
                if (strlen($this->currentLineString) > 1) {
                    $this->‪parseValueForConstants();
                    $this->tokenStream->append(new ‪Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + strlen($this->currentLineString)));
                    // Tricky to swap the streams here, but that's the most effective solution I could come up with for the line endings here.
                    (‪$this->currentLinebreakCallback)();
                    $tempStream = ‪$this->tokenStream;
                    $this->tokenStream = ‪$this->valueStream;
                    (‪$this->currentLinebreakCallback)();
                    $this->tokenStream = $tempStream;
                    $this->lineStream->‪append((new ‪IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream)->setTokenStream($this->tokenStream));
                    return;
                }
                $this->tokenStream->append(new ‪Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + strlen($this->currentLineString)));
                (‪$this->currentLinebreakCallback)();
                $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            if ($isFirstLine && strlen($this->currentLineString)) {
                $this->‪parseValueForConstants();
                $valueOnFirstLine = true;
                $previousLineCallback = ‪$this->currentLinebreakCallback;
            }
            if (($isFirstLine && $valueOnFirstLine)
                || (!$isFirstLine && !$isSecondLine)
            ) {
                $tempStream = ‪$this->tokenStream;
                $this->tokenStream = ‪$this->valueStream;
                $previousLineCallback();
                $this->tokenStream = $tempStream;
            }
            if (!$isFirstLine && strlen($this->currentLineString)) {
                $this->‪parseValueForConstants();
            }
            $previousLineCallback = ‪$this->currentLinebreakCallback;
            (‪$this->currentLinebreakCallback)();
            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
                $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            if ($isFirstLine) {
                $isSecondLine = true;
            } else {
                $isSecondLine = false;
            }
            $isFirstLine = false;
            $valueOnFirstLine = false;
            $this->currentLineNumber++;
            $this->currentColumnInLine = 0;
            $this->currentLineString = $this->lines[‪$this->currentLineNumber]['line'];
            $this->currentLinebreakCallback = $this->lines[‪$this->currentLineNumber]['linebreakCallback'];
        }
    }
 
    private function ‪parseOperatorUnset(): void
    {
        $this->tokenStream->append(new ‪Token(TokenType::T_OPERATOR_UNSET, '>', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine++;
        $this->currentLineString = substr($this->currentLineString, 1);
        $this->‪parseTabsAndWhitespaces();
        $this->‪makeComment();
        $this->lineStream->append((new ‪IdentifierUnsetLine())->setTokenStream($this->tokenStream)->setIdentifierTokenStream($this->identifierStream));
    }
 
    private function ‪parseOperatorCopy(): void
    {
        $this->tokenStream->append(new ‪Token(TokenType::T_OPERATOR_COPY, '<', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine++;
        $this->currentLineString = substr($this->currentLineString, 1);
        $this->‪parseTabsAndWhitespaces();
        $identifierStream = ‪$this->identifierStream;
        $this->‪parseIdentifierAtEndOfLine();
        $referenceStream = ‪$this->identifierStream;
        if ($referenceStream->isEmpty()) {
            $this->lineStream->‪append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        $this->lineStream->append(
            (new ‪IdentifierCopyLine())
                ->setIdentifierTokenStream(‪$identifierStream)
                ->setValueTokenStream($referenceStream)
                ->setTokenStream($this->tokenStream)
        );
    }
 
    private function ‪parseOperatorReference(): void
    {
        $this->tokenStream->append(new ‪Token(TokenType::T_OPERATOR_REFERENCE, '=<', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine += 2;
        $this->currentLineString = substr($this->currentLineString, 2);
        $this->‪parseTabsAndWhitespaces();
        $identifierStream = ‪$this->identifierStream;
        $this->‪parseIdentifierAtEndOfLine();
        $referenceStream = ‪$this->identifierStream;
        if ($referenceStream->isEmpty()) {
            $this->lineStream->‪append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        $this->lineStream->append(
            (new ‪IdentifierReferenceLine())
                ->setIdentifierTokenStream(‪$identifierStream)
                ->setValueTokenStream($referenceStream)
                ->setTokenStream($this->tokenStream)
        );
    }
 
    private function ‪parseIdentifierAtEndOfLine(): void
    {
        $this->identifierStream = new ‪IdentifierTokenStream();
        $isRelative = false;
        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
        $char = $splitLine[0] ?? null;
        if ($char === null) {
            return;
        }
        $nextTwoChars = $char . ($splitLine[1] ?? '');
        if ($char === '.') {
            // A relative right side: foo.bar < .foo (note the dot!). we identifierStream->setRelative() and
            // get rid of the dot for the rest of the processing.
            $isRelative = true;
            $this->tokenStream->append((new ‪Token(TokenType::T_DOT, '.', 0, $this->currentColumnInLine)));
            array_shift($splitLine);
            $this->currentColumnInLine++;
            $this->currentLineString = substr($this->currentLineString, 1);
        }
        if ($char === '#') {
            $this->‪parseHashComment();
            return;
        }
        if ($nextTwoChars === '//') {
            $this->‪parseDoubleSlashComment();
            return;
        }
        if ($nextTwoChars === '/*') {
            $this->‪parseMultilineComment();
            return;
        }
        $currentPosition = $this->‪parseIdentifierUntilStopChar($splitLine, $isRelative);
        if (!$currentPosition) {
            return;
        }
        $this->currentLineString = substr($this->currentLineString, $currentPosition);
        $this->currentColumnInLine = $this->currentColumnInLine + $currentPosition;
        $this->‪parseTabsAndWhitespaces();
        $this->‪makeComment();
    }
 
    private function ‪parseIdentifierUntilStopChar(array $splitLine, bool $isRelative = false): ?int
    {
        $this->identifierStream = new ‪IdentifierTokenStream();
        if ($isRelative) {
            $this->identifierStream->setRelative();
        }
        $currentPosition = 0;
        $currentIdentifierStartPosition = ‪$this->currentColumnInLine;
        $currentIdentifierBody = '';
        $currentIdentifierCharCount = 0;
        while (true) {
            $nextChar = $splitLine[$currentPosition] ?? null;
            if ($nextChar === null) {
                if ($currentIdentifierCharCount) {
                    $identifierToken = new ‪IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
                    $this->tokenStream->append($identifierToken);
                    $this->identifierStream->append($identifierToken);
                }
                (‪$this->currentLinebreakCallback)();
                return null;
            }
            $nextTwoChars = $nextChar . ($splitLine[$currentPosition + 1] ?? null);
            if ($currentPosition > 0
                && ($nextChar === ' ' || $nextChar === "\t" || $nextChar === '=' || $nextChar === '<' || $nextChar === '>' || $nextChar === '{' || $nextTwoChars === ':=' || $nextChar === '(')
            ) {
                if ($currentIdentifierCharCount) {
                    $identifierToken = new ‪IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
                    $this->tokenStream->append($identifierToken);
                    $this->identifierStream->append($identifierToken);
                }
                break;
            }
            if ($nextTwoChars === '\\.') {
                // A quoted dot is part of *this* identifier
                $currentIdentifierBody .= '.';
                $currentPosition += 2;
                $currentIdentifierCharCount++;
            } elseif ($nextChar === '.') {
                if ($currentIdentifierCharCount) {
                    $identifierToken = new ‪IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
                    $this->tokenStream->append($identifierToken);
                    $this->identifierStream->append($identifierToken);
                    $currentIdentifierCharCount = 0;
                    $currentIdentifierBody = '';
                }
                $this->tokenStream->append(new ‪Token(TokenType::T_DOT, '.', $this->currentLineNumber, $this->currentColumnInLine + $currentPosition));
                $currentPosition++;
                $currentIdentifierStartPosition = $this->currentColumnInLine + $currentPosition;
            } else {
                $currentIdentifierBody .= $nextChar;
                $currentIdentifierCharCount++;
                $currentPosition++;
            }
        }
        return $currentPosition;
    }
 
    private function ‪parseOperatorFunction(): void
    {
        $this->tokenStream->append(new ‪Token(TokenType::T_OPERATOR_FUNCTION, ':=', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine += 2;
        $this->currentLineString = substr($this->currentLineString, 2);
        $this->‪parseTabsAndWhitespaces();
        if ($this->currentLineString === '') {
            (‪$this->currentLinebreakCallback)();
            $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        $functionName = '';
        $functionNameStartPosition = ‪$this->currentColumnInLine;
        $functionNameCharCount = 0;
        $functionChars = mb_str_split($this->currentLineString, 1, 'UTF-8');
        while (true) {
            $nextChar = $functionChars[$functionNameCharCount] ?? null;
            if ($nextChar === null) {
                // end of chars
                if ($functionNameCharCount) {
                    $this->tokenStream->append(new ‪Token(TokenType::T_FUNCTION_NAME, $functionName, $this->currentLineNumber, $functionNameStartPosition));
                }
                (‪$this->currentLinebreakCallback)();
                $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            if ($nextChar === '(') {
                if ($functionNameCharCount) {
                    $functionNameToken = new ‪Token(TokenType::T_FUNCTION_NAME, $functionName, $this->currentLineNumber, $functionNameStartPosition);
                    $this->tokenStream->append($functionNameToken);
                    $this->tokenStream->append(new ‪Token(TokenType::T_FUNCTION_VALUE_START, '(', $this->currentLineNumber, $this->currentColumnInLine + $functionNameCharCount));
                    $functionNameCharCount++;
                    break;
                }
                $this->‪makeComment();
                $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            $functionName .= $nextChar;
            $functionNameCharCount++;
        }
        $functionBodyStartPosition = $functionNameCharCount;
        $functionBody = '';
        $functionBodyCharCount = 0;
        $functionValueToken = false;
        while (true) {
            $nextChar = $functionChars[$functionBodyStartPosition + $functionBodyCharCount] ?? null;
            if ($nextChar === null) {
                if ($functionBodyCharCount) {
                    $this->tokenStream->append(new ‪Token(TokenType::T_VALUE, $functionBody, $this->currentLineNumber, $functionBodyCharCount));
                }
                (‪$this->currentLinebreakCallback)();
                $this->lineStream->append((new ‪InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            if ($nextChar === ')') {
                if ($functionBodyCharCount) {
                    $functionValueToken = new ‪Token(TokenType::T_VALUE, $functionBody, $this->currentLineNumber, $this->currentColumnInLine + $functionNameCharCount);
                    $this->tokenStream->append($functionValueToken);
                }
                $this->tokenStream->append(new ‪Token(TokenType::T_FUNCTION_VALUE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + $functionNameCharCount + $functionBodyCharCount));
                $functionBodyCharCount++;
                break;
            }
            $functionBody .= $nextChar;
            $functionBodyCharCount++;
        }
        $this->currentColumnInLine = $this->currentColumnInLine + $functionNameCharCount + $functionBodyCharCount;
        $this->currentLineString = substr($this->currentLineString, $functionNameCharCount + $functionBodyCharCount);
        $this->‪parseTabsAndWhitespaces();
        $this->‪makeComment();
        $line = (new ‪IdentifierFunctionLine())
            ->setIdentifierTokenStream($this->identifierStream)
            ->setFunctionNameToken($functionNameToken) 
            ->setTokenStream($this->tokenStream);
        if ($functionValueToken) {
            $line->setFunctionValueToken($functionValueToken);
        }
        $this->lineStream->append($line);
    }
 
    private function ‪parseValueForConstants(): void
    {
        if (!str_contains($this->currentLineString, '{$')) {
            $valueToken = new ‪Token(TokenType::T_VALUE, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
            $this->tokenStream->append($valueToken);
            $this->valueStream->append($valueToken);
            return;
        }
        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
        $isInConstant = false;
        $currentPosition = 0;
        $currentString = '';
        $currentStringLength = 0;
        $lastTokenEndPosition = 0;
        while (true) {
            $char = $splitLine[$currentPosition] ?? null;
            if ($char === null) {
                if ($currentStringLength) {
                    $valueToken = new ‪Token(TokenType::T_VALUE, $currentString, $this->currentLineNumber, $this->currentColumnInLine + $lastTokenEndPosition);
                    $this->tokenStream->append($valueToken);
                    $this->valueStream->append($valueToken);
                }
                break;
            }
            $nextTwoChars = $char . ($splitLine[$currentPosition + 1] ?? '');
            if ($nextTwoChars === '{$') {
                $isInConstant = true;
                if ($currentStringLength) {
                    $valueToken = new ‪Token(TokenType::T_VALUE, $currentString, $this->currentLineNumber, $this->currentColumnInLine + $lastTokenEndPosition);
                    $this->tokenStream->append($valueToken);
                    $this->valueStream->append($valueToken);
                    $lastTokenEndPosition = $currentPosition;
                }
                $currentString = '{$';
                $currentPosition += 2;
                continue;
            }
            if ($isInConstant && $char === '}') {
                $valueToken = new ‪Token(TokenType::T_CONSTANT, $currentString . '}', $this->currentLineNumber, $this->currentColumnInLine + $lastTokenEndPosition);
                $this->tokenStream->append($valueToken);
                if (!$this->valueStream instanceof ‪ConstantAwareTokenStream) {
                    $this->valueStream = (new ‪ConstantAwareTokenStream())->setAll($this->valueStream->getAll());
                }
                $this->valueStream->append($valueToken);
                $currentPosition++;
                $currentString = '';
                $currentStringLength = 0;
                $lastTokenEndPosition = $currentPosition;
                $isInConstant = false;
                continue;
            }
            $currentPosition++;
            $currentStringLength++;
            $currentString .= $char;
        }
    }
}