Parser.php

Go to the documentation of this file.

<?php
declare(strict_types = 1);
 
namespace ‪TYPO3\CMS\Core\Database\Schema\Parser;
 
/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */
 
use Doctrine\DBAL\Schema\Table;
use ‪TYPO3\CMS\Core\Database\Schema\Exception\StatementException;
use ‪TYPO3\CMS\Core\Database\Schema\Parser\AST\CreateTableStatement;
 
class ‪Parser
{
    protected ‪$lexer;
 
    protected ‪$statement;
 
    public function ‪__construct(string ‪$statement)
    {
        $this->statement = ‪$statement;
        $this->lexer = new ‪Lexer(‪$statement);
    }
 
    public function ‪getLexer(): ‪Lexer
    {
        return ‪$this->lexer;
    }
 
    public function ‪getAST(): ‪AST\AbstractCreateStatement
    {
        // Parse & build AST
        return $this->‪queryLanguage();
    }
 
    public function ‪match($token)
    {
        $lookaheadType = $this->lexer->lookahead['type'];
 
        // Short-circuit on first condition, usually types match
        if ($lookaheadType !== $token) {
            // If parameter is not identifier (1-99) must be exact match
            if ($token < ‪Lexer::T_IDENTIFIER) {
                $this->‪syntaxError($this->lexer->getLiteral($token));
            }
 
            // If parameter is keyword (200+) must be exact match
            if ($token > ‪Lexer::T_IDENTIFIER) {
                $this->‪syntaxError($this->lexer->getLiteral($token));
            }
 
            // If parameter is MATCH then FULL, PARTIAL or SIMPLE must follow
            if ($token === ‪Lexer::T_MATCH
                && $lookaheadType !== ‪Lexer::T_FULL
                && $lookaheadType !== ‪Lexer::T_PARTIAL
                && $lookaheadType !== ‪Lexer::T_SIMPLE
            ) {
                $this->‪syntaxError($this->lexer->getLiteral($token));
            }
 
            if ($token === ‪Lexer::T_ON && $lookaheadType !== ‪Lexer::T_DELETE && $lookaheadType !== ‪Lexer::T_UPDATE) {
                $this->‪syntaxError($this->lexer->getLiteral($token));
            }
        }
 
        $this->lexer->moveNext();
    }
 
    public function ‪free($deep = false, $position = 0)
    {
        // WARNING! Use this method with care. It resets the scanner!
        $this->lexer->resetPosition($position);
 
        // Deep = true cleans peek and also any previously defined errors
        if ($deep) {
            $this->lexer->resetPeek();
        }
 
        $this->lexer->token = null;
        $this->lexer->lookahead = null;
    }
 
    public function ‪parse(): array
    {
        $ast = $this->‪getAST();
 
        if (!$ast instanceof ‪CreateTableStatement) {
            return [];
        }
 
        $tableBuilder = new ‪TableBuilder();
        $table = $tableBuilder->create($ast);
 
        return [$table];
    }
 
    public function ‪syntaxError($expected = '', $token = null)
    {
        if ($token === null) {
            $token = $this->lexer->lookahead;
        }
 
        $tokenPos = $token['position'] ?? '-1';
 
        $message = "line 0, col {$tokenPos}: Error: ";
        $message .= ($expected !== '') ? "Expected {$expected}, got " : 'Unexpected ';
        $message .= ($this->lexer->lookahead === null) ? 'end of string.' : "'{$token['value']}'";
 
        throw ‪StatementException::syntaxError($message, ‪StatementException::sqlError($this->statement));
    }
 
    public function ‪semanticalError($message = '', $token = null)
    {
        if ($token === null) {
            $token = $this->lexer->lookahead;
        }
 
        // Minimum exposed chars ahead of token
        $distance = 12;
 
        // Find a position of a final word to display in error string
        $createTableStatement = ‪$this->statement;
        $length = strlen($createTableStatement);
        $pos = $token['position'] + $distance;
        $pos = strpos($createTableStatement, ' ', ($length > $pos) ? $pos : $length);
        $length = ($pos !== false) ? $pos - $token['position'] : $distance;
 
        $tokenPos = array_key_exists('position', $token) && $token['position'] > 0 ? $token['position'] : '-1';
        $tokenStr = substr($createTableStatement, $token['position'], $length);
 
        // Building informative message
        $message = 'line 0, col ' . $tokenPos . " near '" . $tokenStr . "': Error: " . $message;
 
        throw ‪StatementException::semanticalError($message, ‪StatementException::sqlError($this->statement));
    }
 
    protected function ‪peekBeyondClosingParenthesis($resetPeek = true)
    {
        $token = $this->lexer->peek();
        $numUnmatched = 1;
 
        while ($numUnmatched > 0 && $token !== null) {
            switch ($token['type']) {
                case ‪Lexer::T_OPEN_PARENTHESIS:
                    ++$numUnmatched;
                    break;
                case ‪Lexer::T_CLOSE_PARENTHESIS:
                    --$numUnmatched;
                    break;
                default:
                    // Do nothing
            }
 
            $token = $this->lexer->peek();
        }
 
        if ($resetPeek) {
            $this->lexer->resetPeek();
        }
 
        return $token;
    }
 
    public function ‪queryLanguage(): ‪AST\AbstractCreateStatement
    {
        $this->lexer->moveNext();
 
        if ($this->lexer->lookahead['type'] !== ‪Lexer::T_CREATE) {
            $this->‪syntaxError('CREATE');
        }
 
        $statement = $this->‪createStatement();
 
        // Check for end of string
        if ($this->lexer->lookahead !== null) {
            $this->‪syntaxError('end of string');
        }
 
        return ‪$statement;
    }
 
    public function ‪createStatement(): ‪AST\AbstractCreateStatement
    {
        ‪$statement = null;
        $this->‪match(‪Lexer::T_CREATE);
 
        switch ($this->lexer->lookahead['type']) {
            case ‪Lexer::T_TEMPORARY:
                // Intentional fall-through
            case ‪Lexer::T_TABLE:
                ‪$statement = $this->‪createTableStatement();
                break;
            default:
                $this->‪syntaxError('TEMPORARY or TABLE');
                break;
        }
 
        $this->‪match(‪Lexer::T_SEMICOLON);
 
        return ‪$statement;
    }
 
    protected function ‪createTableStatement(): ‪AST\CreateTableStatement
    {
        $createTableStatement = new ‪AST\CreateTableStatement($this->‪createTableClause(), $this->‪createDefinition());
 
        if (!$this->lexer->isNextToken(‪Lexer::T_SEMICOLON)) {
            $createTableStatement->tableOptions = $this->‪tableOptions();
        }
        return $createTableStatement;
    }
 
    protected function ‪createTableClause(): ‪AST\CreateTableClause
    {
        $isTemporary = false;
        // Check for TEMPORARY
        if ($this->lexer->isNextToken(‪Lexer::T_TEMPORARY)) {
            $this->‪match(‪Lexer::T_TEMPORARY);
            $isTemporary = true;
        }
 
        $this->‪match(‪Lexer::T_TABLE);
 
        // Check for IF NOT EXISTS
        if ($this->lexer->isNextToken(‪Lexer::T_IF)) {
            $this->‪match(‪Lexer::T_IF);
            $this->‪match(‪Lexer::T_NOT);
            $this->‪match(‪Lexer::T_EXISTS);
        }
 
        // Process schema object name (table name)
        $tableName = $this->‪schemaObjectName();
 
        return new ‪AST\CreateTableClause($tableName, $isTemporary);
    }
 
    protected function ‪createDefinition(): ‪AST\CreateDefinition
    {
        $createDefinitions = [];
 
        // Process opening parenthesis
        $this->‪match(‪Lexer::T_OPEN_PARENTHESIS);
 
        $createDefinitions[] = $this->‪createDefinitionItem();
 
        while ($this->lexer->isNextToken(‪Lexer::T_COMMA)) {
            $this->‪match(‪Lexer::T_COMMA);
 
            // TYPO3 previously accepted invalid SQL files where a create definition
            // item terminated with a comma before the final closing parenthesis.
            // Silently swallow the extra comma and stop the create definition parsing.
            if ($this->lexer->isNextToken(‪Lexer::T_CLOSE_PARENTHESIS)) {
                break;
            }
 
            $createDefinitions[] = $this->‪createDefinitionItem();
        }
 
        // Process closing parenthesis
        $this->‪match(‪Lexer::T_CLOSE_PARENTHESIS);
 
        return new ‪AST\CreateDefinition($createDefinitions);
    }
 
    protected function ‪createDefinitionItem(): ‪AST\AbstractCreateDefinitionItem
    {
        $definitionItem = null;
 
        switch ($this->lexer->lookahead['type']) {
            case ‪Lexer::T_FULLTEXT:
                // Intentional fall-through
            case ‪Lexer::T_SPATIAL:
                // Intentional fall-through
            case ‪Lexer::T_PRIMARY:
                // Intentional fall-through
            case ‪Lexer::T_UNIQUE:
                // Intentional fall-through
            case ‪Lexer::T_KEY:
                // Intentional fall-through
            case ‪Lexer::T_INDEX:
                $definitionItem = $this->‪createIndexDefinitionItem();
                break;
            case ‪Lexer::T_FOREIGN:
                $definitionItem = $this->‪createForeignKeyDefinitionItem();
                break;
            case ‪Lexer::T_CONSTRAINT:
                $this->‪semanticalError('CONSTRAINT [symbol] index definition part not supported');
                break;
            case ‪Lexer::T_CHECK:
                $this->‪semanticalError('CHECK (expr) create definition not supported');
                break;
            default:
                $definitionItem = $this->‪createColumnDefinitionItem();
        }
 
        return $definitionItem;
    }
 
    protected function ‪createIndexDefinitionItem(): ‪AST\CreateIndexDefinitionItem
    {
        $indexName = null;
        $isPrimary = false;
        $isFulltext = false;
        $isSpatial = false;
        $isUnique = false;
        $indexDefinition = new ‪AST\CreateIndexDefinitionItem();
 
        switch ($this->lexer->lookahead['type']) {
            case ‪Lexer::T_PRIMARY:
                $this->‪match(‪Lexer::T_PRIMARY);
                // KEY is a required keyword for PRIMARY index
                $this->‪match(‪Lexer::T_KEY);
                $isPrimary = true;
                break;
            case ‪Lexer::T_KEY:
                // Plain index, no special configuration
                $this->‪match(‪Lexer::T_KEY);
                break;
            case ‪Lexer::T_INDEX:
                // Plain index, no special configuration
                $this->‪match(‪Lexer::T_INDEX);
                break;
            case ‪Lexer::T_UNIQUE:
                $this->‪match(‪Lexer::T_UNIQUE);
                // INDEX|KEY are optional keywords for UNIQUE index
                if ($this->lexer->isNextTokenAny([‪Lexer::T_INDEX, ‪Lexer::T_KEY])) {
                    $this->lexer->moveNext();
                }
                $isUnique = true;
                break;
            case ‪Lexer::T_FULLTEXT:
                $this->‪match(‪Lexer::T_FULLTEXT);
                // INDEX|KEY are optional keywords for FULLTEXT index
                if ($this->lexer->isNextTokenAny([‪Lexer::T_INDEX, ‪Lexer::T_KEY])) {
                    $this->lexer->moveNext();
                }
                $isFulltext = true;
                break;
            case ‪Lexer::T_SPATIAL:
                $this->‪match(‪Lexer::T_SPATIAL);
                // INDEX|KEY are optional keywords for SPATIAL index
                if ($this->lexer->isNextTokenAny([‪Lexer::T_INDEX, ‪Lexer::T_KEY])) {
                    $this->lexer->moveNext();
                }
                $isSpatial = true;
                break;
            default:
                $this->‪syntaxError('PRIMARY, KEY, INDEX, UNIQUE, FULLTEXT or SPATIAL');
        }
 
        // PRIMARY KEY has no name in MySQL
        if (!$indexDefinition->isPrimary) {
            $indexName = $this->‪indexName();
        }
 
        $indexDefinition = new ‪AST\CreateIndexDefinitionItem(
            $indexName,
            $isPrimary,
            $isUnique,
            $isSpatial,
            $isFulltext
        );
 
        // FULLTEXT and SPATIAL indexes can not have a type definition
        if (!$isFulltext && !$isSpatial) {
            $indexDefinition->indexType = $this->‪indexType();
        }
 
        $this->‪match(‪Lexer::T_OPEN_PARENTHESIS);
 
        $indexDefinition->columnNames[] = $this->‪indexColumnName();
 
        while ($this->lexer->isNextToken(‪Lexer::T_COMMA)) {
            $this->‪match(‪Lexer::T_COMMA);
            $indexDefinition->columnNames[] = $this->‪indexColumnName();
        }
 
        $this->‪match(‪Lexer::T_CLOSE_PARENTHESIS);
 
        $indexDefinition->options = $this->‪indexOptions();
 
        return $indexDefinition;
    }
 
    protected function ‪createForeignKeyDefinitionItem(): ‪AST\CreateForeignKeyDefinitionItem
    {
        $this->‪match(‪Lexer::T_FOREIGN);
        $this->‪match(‪Lexer::T_KEY);
 
        $indexName = $this->‪indexName();
 
        $this->‪match(‪Lexer::T_OPEN_PARENTHESIS);
 
        $indexColumns = [];
        $indexColumns[] = $this->‪indexColumnName();
 
        while ($this->lexer->isNextToken(‪Lexer::T_COMMA)) {
            $this->‪match(‪Lexer::T_COMMA);
            $indexColumns[] = $this->‪indexColumnName();
        }
 
        $this->‪match(‪Lexer::T_CLOSE_PARENTHESIS);
 
        $foreignKeyDefinition = new ‪AST\CreateForeignKeyDefinitionItem(
            $indexName,
            $indexColumns,
            $this->‪referenceDefinition()
        );
 
        return $foreignKeyDefinition;
    }
 
    public function ‪indexName(): ‪AST\Identifier
    {
        $indexName = new ‪AST\Identifier(null);
        if (!$this->lexer->isNextTokenAny([‪Lexer::T_USING, ‪Lexer::T_OPEN_PARENTHESIS])) {
            $indexName = $this->‪schemaObjectName();
        }
 
        return $indexName;
    }
 
    public function ‪indexType(): string
    {
        $indexType = '';
        if (!$this->lexer->isNextToken(‪Lexer::T_USING)) {
            return $indexType;
        }
 
        $this->‪match(‪Lexer::T_USING);
 
        switch ($this->lexer->lookahead['type']) {
            case ‪Lexer::T_BTREE:
                $this->‪match(‪Lexer::T_BTREE);
                $indexType = 'BTREE';
                break;
            case ‪Lexer::T_HASH:
                $this->‪match(‪Lexer::T_HASH);
                $indexType = 'HASH';
                break;
            default:
                $this->‪syntaxError('BTREE or HASH');
        }
 
        return $indexType;
    }
 
    public function ‪indexOptions(): array
    {
        $options = [];
 
        while ($this->lexer->lookahead && !$this->lexer->isNextTokenAny([‪Lexer::T_COMMA, ‪Lexer::T_CLOSE_PARENTHESIS])) {
            switch ($this->lexer->lookahead['type']) {
                case ‪Lexer::T_KEY_BLOCK_SIZE:
                    $this->‪match(‪Lexer::T_KEY_BLOCK_SIZE);
                    if ($this->lexer->isNextToken(‪Lexer::T_EQUALS)) {
                        $this->‪match(‪Lexer::T_EQUALS);
                    }
                    $this->lexer->moveNext();
                    $options['key_block_size'] = (int)$this->lexer->token['value'];
                    break;
                case ‪Lexer::T_USING:
                    $options['index_type'] = $this->indexType();
                    break;
                case ‪Lexer::T_WITH:
                    $this->‪match(‪Lexer::T_WITH);
                    $this->‪match(‪Lexer::T_PARSER);
                    $options['parser'] = $this->‪schemaObjectName();
                    break;
                case ‪Lexer::T_COMMENT:
                    $this->‪match(‪Lexer::T_COMMENT);
                    $this->‪match(‪Lexer::T_STRING);
                    $options['comment'] = $this->lexer->token['value'];
                    break;
                default:
                    $this->‪syntaxError('KEY_BLOCK_SIZE, USING, WITH PARSER or COMMENT');
            }
        }
 
        return $options;
    }
 
    protected function ‪createColumnDefinitionItem(): ‪AST\CreateColumnDefinitionItem
    {
        $columnName = $this->‪schemaObjectName();
        $dataType = $this->‪columnDataType();
 
        $columnDefinitionItem = new ‪AST\CreateColumnDefinitionItem($columnName, $dataType);
 
        while ($this->lexer->lookahead && !$this->lexer->isNextTokenAny([‪Lexer::T_COMMA, ‪Lexer::T_CLOSE_PARENTHESIS])) {
            switch ($this->lexer->lookahead['type']) {
                case ‪Lexer::T_NOT:
                    $columnDefinitionItem->allowNull = false;
                    $this->‪match(‪Lexer::T_NOT);
                    $this->‪match(‪Lexer::T_NULL);
                    break;
                case ‪Lexer::T_NULL:
                    $columnDefinitionItem->null = true;
                    $this->‪match(‪Lexer::T_NULL);
                    break;
                case ‪Lexer::T_DEFAULT:
                    $columnDefinitionItem->hasDefaultValue = true;
                    $columnDefinitionItem->defaultValue = $this->‪columnDefaultValue();
                    break;
                case ‪Lexer::T_AUTO_INCREMENT:
                    $columnDefinitionItem->autoIncrement = true;
                    $this->‪match(‪Lexer::T_AUTO_INCREMENT);
                    break;
                case ‪Lexer::T_UNIQUE:
                    $columnDefinitionItem->unique = true;
                    $this->‪match(‪Lexer::T_UNIQUE);
                    if ($this->lexer->isNextToken(‪Lexer::T_KEY)) {
                        $this->‪match(‪Lexer::T_KEY);
                    }
                    break;
                case ‪Lexer::T_PRIMARY:
                    $columnDefinitionItem->primary = true;
                    $this->‪match(‪Lexer::T_PRIMARY);
                    if ($this->lexer->isNextToken(‪Lexer::T_KEY)) {
                        $this->‪match(‪Lexer::T_KEY);
                    }
                    break;
                case ‪Lexer::T_KEY:
                    $columnDefinitionItem->index = true;
                    $this->‪match(‪Lexer::T_KEY);
                    break;
                case ‪Lexer::T_COMMENT:
                    $this->‪match(‪Lexer::T_COMMENT);
                    if ($this->lexer->isNextToken(‪Lexer::T_STRING)) {
                        $columnDefinitionItem->comment = $this->lexer->lookahead['value'];
                        $this->‪match(‪Lexer::T_STRING);
                    }
                    break;
                case ‪Lexer::T_COLUMN_FORMAT:
                    $this->‪match(‪Lexer::T_COLUMN_FORMAT);
                    if ($this->lexer->isNextToken(‪Lexer::T_FIXED)) {
                        $columnDefinitionItem->columnFormat = 'fixed';
                        $this->‪match(‪Lexer::T_FIXED);
                    } elseif ($this->lexer->isNextToken(‪Lexer::T_DYNAMIC)) {
                        $columnDefinitionItem->columnFormat = 'dynamic';
                        $this->‪match(‪Lexer::T_DYNAMIC);
                    } else {
                        $this->‪match(‪Lexer::T_DEFAULT);
                    }
                    break;
                case ‪Lexer::T_STORAGE:
                    $this->‪match(‪Lexer::T_STORAGE);
                    if ($this->lexer->isNextToken(‪Lexer::T_MEMORY)) {
                        $columnDefinitionItem->storage = 'memory';
                        $this->‪match(‪Lexer::T_MEMORY);
                    } elseif ($this->lexer->isNextToken(‪Lexer::T_DISK)) {
                        $columnDefinitionItem->storage = 'disk';
                        $this->‪match(‪Lexer::T_DISK);
                    } else {
                        $this->‪match(‪Lexer::T_DEFAULT);
                    }
                    break;
                case ‪Lexer::T_REFERENCES:
                    $columnDefinitionItem->reference = $this->‪referenceDefinition();
                    break;
                default:
                    $this->‪syntaxError(
                        'NOT, NULL, DEFAULT, AUTO_INCREMENT, UNIQUE, ' .
                        'PRIMARY, COMMENT, COLUMN_FORMAT, STORAGE or REFERENCES'
                    );
            }
        }
 
        return $columnDefinitionItem;
    }
 
    protected function ‪columnDataType(): ‪AST\DataType\AbstractDataType
    {
        $dataType = null;
 
        switch ($this->lexer->lookahead['type']) {
            case ‪Lexer::T_BIT:
                $this->‪match(‪Lexer::T_BIT);
                $dataType = new ‪AST\DataType\BitDataType(
                    $this->‪dataTypeLength()
                );
                break;
            case ‪Lexer::T_TINYINT:
                $this->‪match(‪Lexer::T_TINYINT);
                $dataType = new ‪AST\DataType\TinyIntDataType(
                    $this->‪dataTypeLength(),
                    $this->‪numericDataTypeOptions()
                );
                break;
            case ‪Lexer::T_SMALLINT:
                $this->‪match(‪Lexer::T_SMALLINT);
                $dataType = new ‪AST\DataType\SmallIntDataType(
                    $this->‪dataTypeLength(),
                    $this->‪numericDataTypeOptions()
                );
                break;
            case ‪Lexer::T_MEDIUMINT:
                $this->‪match(‪Lexer::T_MEDIUMINT);
                $dataType = new ‪AST\DataType\MediumIntDataType(
                    $this->‪dataTypeLength(),
                    $this->‪numericDataTypeOptions()
                );
                break;
            case ‪Lexer::T_INT:
                $this->‪match(‪Lexer::T_INT);
                $dataType = new ‪AST\DataType\IntegerDataType(
                    $this->‪dataTypeLength(),
                    $this->‪numericDataTypeOptions()
                );
                break;
            case ‪Lexer::T_INTEGER:
                $this->‪match(‪Lexer::T_INTEGER);
                $dataType = new ‪AST\DataType\IntegerDataType(
                    $this->‪dataTypeLength(),
                    $this->‪numericDataTypeOptions()
                );
                break;
            case ‪Lexer::T_BIGINT:
                $this->‪match(‪Lexer::T_BIGINT);
                $dataType = new ‪AST\DataType\BigIntDataType(
                    $this->‪dataTypeLength(),
                    $this->‪numericDataTypeOptions()
                );
                break;
            case ‪Lexer::T_REAL:
                $this->‪match(‪Lexer::T_REAL);
                $dataType = new ‪AST\DataType\RealDataType(
                    $this->‪dataTypeDecimals(),
                    $this->‪numericDataTypeOptions()
                );
                break;
            case ‪Lexer::T_DOUBLE:
                $this->‪match(‪Lexer::T_DOUBLE);
                if ($this->lexer->isNextToken(‪Lexer::T_PRECISION)) {
                    $this->‪match(‪Lexer::T_PRECISION);
                }
                $dataType = new ‪AST\DataType\DoubleDataType(
                    $this->‪dataTypeDecimals(),
                    $this->‪numericDataTypeOptions()
                );
                break;
            case ‪Lexer::T_FLOAT:
                $this->‪match(‪Lexer::T_FLOAT);
                $dataType = new ‪AST\DataType\FloatDataType(
                    $this->‪dataTypeDecimals(),
                    $this->‪numericDataTypeOptions()
                );
 
                break;
            case ‪Lexer::T_DECIMAL:
                $this->‪match(‪Lexer::T_DECIMAL);
                $dataType = new ‪AST\DataType\DecimalDataType(
                    $this->‪dataTypeDecimals(),
                    $this->‪numericDataTypeOptions()
                );
                break;
            case ‪Lexer::T_NUMERIC:
                $this->‪match(‪Lexer::T_NUMERIC);
                $dataType = new ‪AST\DataType\NumericDataType(
                    $this->‪dataTypeDecimals(),
                    $this->‪numericDataTypeOptions()
                );
                break;
            case ‪Lexer::T_DATE:
                $this->‪match(‪Lexer::T_DATE);
                $dataType = new ‪AST\DataType\DateDataType();
                break;
            case ‪Lexer::T_TIME:
                $this->‪match(‪Lexer::T_TIME);
                $dataType = new ‪AST\DataType\TimeDataType($this->‪fractionalSecondsPart());
                break;
            case ‪Lexer::T_TIMESTAMP:
                $this->‪match(‪Lexer::T_TIMESTAMP);
                $dataType = new ‪AST\DataType\TimestampDataType($this->‪fractionalSecondsPart());
                break;
            case ‪Lexer::T_DATETIME:
                $this->‪match(‪Lexer::T_DATETIME);
                $dataType = new ‪AST\DataType\DateTimeDataType($this->‪fractionalSecondsPart());
                break;
            case ‪Lexer::T_YEAR:
                $this->‪match(‪Lexer::T_YEAR);
                $dataType = new ‪AST\DataType\YearDataType();
                break;
            case ‪Lexer::T_CHAR:
                $this->‪match(‪Lexer::T_CHAR);
                $dataType = new ‪AST\DataType\CharDataType(
                    $this->‪dataTypeLength(),
                    $this->‪characterDataTypeOptions()
                );
                break;
            case ‪Lexer::T_VARCHAR:
                $this->‪match(‪Lexer::T_VARCHAR);
                $dataType = new ‪AST\DataType\VarCharDataType(
                    $this->‪dataTypeLength(true),
                    $this->‪characterDataTypeOptions()
                );
                break;
            case ‪Lexer::T_BINARY:
                $this->‪match(‪Lexer::T_BINARY);
                $dataType = new ‪AST\DataType\BinaryDataType($this->‪dataTypeLength());
                break;
            case ‪Lexer::T_VARBINARY:
                $this->‪match(‪Lexer::T_VARBINARY);
                $dataType = new ‪AST\DataType\VarBinaryDataType($this->‪dataTypeLength(true));
                break;
            case ‪Lexer::T_TINYBLOB:
                $this->‪match(‪Lexer::T_TINYBLOB);
                $dataType = new ‪AST\DataType\TinyBlobDataType();
                break;
            case ‪Lexer::T_BLOB:
                $this->‪match(‪Lexer::T_BLOB);
                $dataType = new ‪AST\DataType\BlobDataType();
                break;
            case ‪Lexer::T_MEDIUMBLOB:
                $this->‪match(‪Lexer::T_MEDIUMBLOB);
                $dataType = new ‪AST\DataType\MediumBlobDataType();
                break;
            case ‪Lexer::T_LONGBLOB:
                $this->‪match(‪Lexer::T_LONGBLOB);
                $dataType = new ‪AST\DataType\LongBlobDataType();
                break;
            case ‪Lexer::T_TINYTEXT:
                $this->‪match(‪Lexer::T_TINYTEXT);
                $dataType = new ‪AST\DataType\TinyTextDataType($this->‪characterDataTypeOptions());
                break;
            case ‪Lexer::T_TEXT:
                $this->‪match(‪Lexer::T_TEXT);
                $dataType = new ‪AST\DataType\TextDataType($this->‪characterDataTypeOptions());
                break;
            case ‪Lexer::T_MEDIUMTEXT:
                $this->‪match(‪Lexer::T_MEDIUMTEXT);
                $dataType = new ‪AST\DataType\MediumTextDataType($this->‪characterDataTypeOptions());
                break;
            case ‪Lexer::T_LONGTEXT:
                $this->‪match(‪Lexer::T_LONGTEXT);
                $dataType = new ‪AST\DataType\LongTextDataType($this->‪characterDataTypeOptions());
                break;
            case ‪Lexer::T_ENUM:
                $this->‪match(‪Lexer::T_ENUM);
                $dataType = new ‪AST\DataType\EnumDataType($this->‪valueList(), $this->‪enumerationDataTypeOptions());
                break;
            case ‪Lexer::T_SET:
                $this->‪match(‪Lexer::T_SET);
                $dataType = new ‪AST\DataType\SetDataType($this->‪valueList(), $this->‪enumerationDataTypeOptions());
                break;
            case ‪Lexer::T_JSON:
                $this->‪match(‪Lexer::T_JSON);
                $dataType = new ‪AST\DataType\JsonDataType();
                break;
            default:
                $this->‪syntaxError(
                    'BIT, TINYINT, SMALLINT, MEDIUMINT, INT, INTEGER, BIGINT, REAL, DOUBLE, FLOAT, DECIMAL, NUMERIC, ' .
                    'DATE, TIME, TIMESTAMP, DATETIME, YEAR, CHAR, VARCHAR, BINARY, VARBINARY, TINYBLOB, BLOB, ' .
                    'MEDIUMBLOB, LONGBLOB, TINYTEXT, TEXT, MEDIUMTEXT, LONGTEXT, ENUM, SET, or JSON'
                );
        }
 
        return $dataType;
    }
 
    protected function ‪columnDefaultValue()
    {
        $this->‪match(‪Lexer::T_DEFAULT);
        $value = null;
 
        switch ($this->lexer->lookahead['type']) {
            case ‪Lexer::T_INTEGER:
                $value = (int)$this->lexer->lookahead['value'];
                break;
            case ‪Lexer::T_FLOAT:
                $value = (float)$this->lexer->lookahead['value'];
                break;
            case ‪Lexer::T_STRING:
                $value = (string)$this->lexer->lookahead['value'];
                break;
            case ‪Lexer::T_CURRENT_TIMESTAMP:
                $value = 'CURRENT_TIMESTAMP';
                break;
            case ‪Lexer::T_NULL:
                $value = null;
                break;
            default:
                $this->syntaxError('String, Integer, Float, NULL or CURRENT_TIMESTAMP');
        }
 
        $this->lexer->moveNext();
 
        return $value;
    }
 
    protected function ‪dataTypeLength(bool $required = false): int
    {
        $length = 0;
        if (!$this->lexer->isNextToken(‪Lexer::T_OPEN_PARENTHESIS)) {
            if ($required) {
                $this->‪semanticalError('The current data type requires a field length definition.');
            }
            return $length;
        }
 
        $this->‪match(‪Lexer::T_OPEN_PARENTHESIS);
        $length = (int)$this->lexer->lookahead['value'];
        $this->match(‪Lexer::T_INTEGER);
        $this->‪match(‪Lexer::T_CLOSE_PARENTHESIS);
 
        return $length;
    }
 
    private function ‪dataTypeDecimals(): array
    {
        $options = [];
        if (!$this->lexer->isNextToken(‪Lexer::T_OPEN_PARENTHESIS)) {
            return $options;
        }
 
        $this->‪match(‪Lexer::T_OPEN_PARENTHESIS);
        $options['length'] = (int)$this->lexer->lookahead['value'];
        $this->match(‪Lexer::T_INTEGER);
 
        if ($this->lexer->isNextToken(‪Lexer::T_COMMA)) {
            $this->‪match(‪Lexer::T_COMMA);
            $options['decimals'] = (int)$this->lexer->lookahead['value'];
            $this->match(‪Lexer::T_INTEGER);
        }
 
        $this->‪match(‪Lexer::T_CLOSE_PARENTHESIS);
 
        return $options;
    }
 
    protected function ‪numericDataTypeOptions(): array
    {
        $options = ['unsigned' => false, 'zerofill' => false];
 
        if (!$this->lexer->isNextTokenAny([‪Lexer::T_UNSIGNED, ‪Lexer::T_ZEROFILL])) {
            return $options;
        }
 
        while ($this->lexer->isNextTokenAny([‪Lexer::T_UNSIGNED, ‪Lexer::T_ZEROFILL])) {
            switch ($this->lexer->lookahead['type']) {
                case ‪Lexer::T_UNSIGNED:
                    $this->‪match(‪Lexer::T_UNSIGNED);
                    $options['unsigned'] = true;
                    break;
                case ‪Lexer::T_ZEROFILL:
                    $this->‪match(‪Lexer::T_ZEROFILL);
                    $options['zerofill'] = true;
                    break;
                default:
                    $this->‪syntaxError('USIGNED or ZEROFILL');
            }
        }
 
        return $options;
    }
 
    protected function ‪fractionalSecondsPart(): int
    {
        $fractionalSecondsPart = $this->‪dataTypeLength();
        if ($fractionalSecondsPart < 0) {
            $this->‪semanticalError('the fractional seconds part for TIME, DATETIME or TIMESTAMP columns must >= 0');
        }
        if ($fractionalSecondsPart > 6) {
            $this->‪semanticalError('the fractional seconds part for TIME, DATETIME or TIMESTAMP columns must <= 6');
        }
 
        return $fractionalSecondsPart;
    }
 
    protected function ‪characterDataTypeOptions(): array
    {
        $options = ['binary' => false, 'charset' => null, 'collation' => null];
 
        if (!$this->lexer->isNextTokenAny([‪Lexer::T_CHARACTER, ‪Lexer::T_COLLATE, ‪Lexer::T_BINARY])) {
            return $options;
        }
 
        while ($this->lexer->isNextTokenAny([‪Lexer::T_CHARACTER, ‪Lexer::T_COLLATE, ‪Lexer::T_BINARY])) {
            switch ($this->lexer->lookahead['type']) {
                case ‪Lexer::T_BINARY:
                    $this->‪match(‪Lexer::T_BINARY);
                    $options['binary'] = true;
                    break;
                case ‪Lexer::T_CHARACTER:
                    $this->‪match(‪Lexer::T_CHARACTER);
                    $this->‪match(‪Lexer::T_SET);
                    $this->‪match(‪Lexer::T_STRING);
                    $options['charset'] = $this->lexer->token['value'];
                    break;
                case ‪Lexer::T_COLLATE:
                    $this->‪match(‪Lexer::T_COLLATE);
                    $this->‪match(‪Lexer::T_STRING);
                    $options['collation'] = $this->lexer->token['value'];
                    break;
                default:
                    $this->‪syntaxError('BINARY, CHARACTER SET or COLLATE');
            }
        }
 
        return $options;
    }
 
    protected function ‪enumerationDataTypeOptions(): array
    {
        $options = ['charset' => null, 'collation' => null];
 
        if (!$this->lexer->isNextTokenAny([‪Lexer::T_CHARACTER, ‪Lexer::T_COLLATE])) {
            return $options;
        }
 
        while ($this->lexer->isNextTokenAny([‪Lexer::T_CHARACTER, ‪Lexer::T_COLLATE])) {
            switch ($this->lexer->lookahead['type']) {
                case ‪Lexer::T_CHARACTER:
                    $this->‪match(‪Lexer::T_CHARACTER);
                    $this->‪match(‪Lexer::T_SET);
                    $this->‪match(‪Lexer::T_STRING);
                    $options['charset'] = $this->lexer->token['value'];
                    break;
                case ‪Lexer::T_COLLATE:
                    $this->‪match(‪Lexer::T_COLLATE);
                    $this->‪match(‪Lexer::T_STRING);
                    $options['collation'] = $this->lexer->token['value'];
                    break;
                default:
                    $this->‪syntaxError('CHARACTER SET or COLLATE');
            }
        }
 
        return $options;
    }
 
    protected function ‪valueList(): array
    {
        $this->‪match(‪Lexer::T_OPEN_PARENTHESIS);
 
        $values = [];
        $values[] = $this->‪valueListItem();
 
        while ($this->lexer->isNextToken(‪Lexer::T_COMMA)) {
            $this->‪match(‪Lexer::T_COMMA);
            $values[] = $this->‪valueListItem();
        }
 
        $this->‪match(‪Lexer::T_CLOSE_PARENTHESIS);
 
        return $values;
    }
 
    protected function ‪valueListItem(): string
    {
        $this->‪match(‪Lexer::T_STRING);
 
        return (string)$this->lexer->token['value'];
    }
 
    protected function ‪referenceDefinition(): ‪AST\ReferenceDefinition
    {
        $this->‪match(‪Lexer::T_REFERENCES);
        $tableName = $this->‪schemaObjectName();
        $this->‪match(‪Lexer::T_OPEN_PARENTHESIS);
 
        $referenceColumns = [];
        $referenceColumns[] = $this->‪indexColumnName();
 
        while ($this->lexer->isNextToken(‪Lexer::T_COMMA)) {
            $this->‪match(‪Lexer::T_COMMA);
            $referenceColumns[] = $this->‪indexColumnName();
        }
 
        $this->‪match(‪Lexer::T_CLOSE_PARENTHESIS);
 
        $referenceDefinition = new ‪AST\ReferenceDefinition($tableName, $referenceColumns);
 
        while (!$this->lexer->isNextTokenAny([‪Lexer::T_COMMA, ‪Lexer::T_CLOSE_PARENTHESIS])) {
            switch ($this->lexer->lookahead['type']) {
                case ‪Lexer::T_MATCH:
                    $this->‪match(‪Lexer::T_MATCH);
                    $referenceDefinition->match = $this->lexer->lookahead['value'];
                    $this->lexer->moveNext();
                    break;
                case ‪Lexer::T_ON:
                    $this->‪match(‪Lexer::T_ON);
                    if ($this->lexer->isNextToken(‪Lexer::T_DELETE)) {
                        $this->‪match(‪Lexer::T_DELETE);
                        $referenceDefinition->onDelete = $this->‪referenceOption();
                    } else {
                        $this->‪match(‪Lexer::T_UPDATE);
                        $referenceDefinition->onUpdate = $this->‪referenceOption();
                    }
                    break;
                default:
                    $this->‪syntaxError('MATCH, ON DELETE or ON UPDATE');
            }
        }
 
        return $referenceDefinition;
    }
 
    protected function ‪indexColumnName(): ‪AST\IndexColumnName
    {
        $columnName = $this->‪schemaObjectName();
        $length = $this->‪dataTypeLength();
        $direction = null;
 
        if ($this->lexer->isNextToken(‪Lexer::T_ASC)) {
            $this->‪match(‪Lexer::T_ASC);
            $direction = 'ASC';
        } elseif ($this->lexer->isNextToken(‪Lexer::T_DESC)) {
            $this->‪match(‪Lexer::T_DESC);
            $direction = 'DESC';
        }
 
        return new ‪AST\IndexColumnName($columnName, $length, $direction);
    }
 
    protected function ‪referenceOption(): string
    {
        $action = null;
 
        switch ($this->lexer->lookahead['type']) {
            case ‪Lexer::T_RESTRICT:
                $this->‪match(‪Lexer::T_RESTRICT);
                $action = 'RESTRICT';
                break;
            case ‪Lexer::T_CASCADE:
                $this->‪match(‪Lexer::T_CASCADE);
                $action = 'CASCADE';
                break;
            case ‪Lexer::T_SET:
                $this->‪match(‪Lexer::T_SET);
                $this->‪match(‪Lexer::T_NULL);
                $action = 'SET NULL';
                break;
            case ‪Lexer::T_NO:
                $this->‪match(‪Lexer::T_NO);
                $this->‪match(‪Lexer::T_ACTION);
                $action = 'NO ACTION';
                break;
            default:
                $this->‪syntaxError('RESTRICT, CASCADE, SET NULL or NO ACTION');
        }
 
        return $action;
    }
 
    protected function ‪tableOptions(): array
    {
        $options = [];
 
        while ($this->lexer->lookahead && !$this->lexer->isNextToken(‪Lexer::T_SEMICOLON)) {
            switch ($this->lexer->lookahead['type']) {
                case ‪Lexer::T_DEFAULT:
                    // DEFAULT prefix is optional for COLLATE/CHARACTER SET, do nothing
                    $this->‪match(‪Lexer::T_DEFAULT);
                    break;
                case ‪Lexer::T_ENGINE:
                    $this->‪match(‪Lexer::T_ENGINE);
                    $options['engine'] = (string)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_AUTO_INCREMENT:
                    $this->‪match(‪Lexer::T_AUTO_INCREMENT);
                    $options['auto_increment'] = (int)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_AVG_ROW_LENGTH:
                    $this->‪match(‪Lexer::T_AVG_ROW_LENGTH);
                    $options['average_row_length'] = (int)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_CHARACTER:
                    $this->‪match(‪Lexer::T_CHARACTER);
                    $this->‪match(‪Lexer::T_SET);
                    $options['character_set'] = (string)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_CHECKSUM:
                    $this->‪match(‪Lexer::T_CHECKSUM);
                    $options['checksum'] = (int)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_COLLATE:
                    $this->‪match(‪Lexer::T_COLLATE);
                    $options['collation'] = (string)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_COMMENT:
                    $this->‪match(‪Lexer::T_COMMENT);
                    $options['comment'] = (string)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_COMPRESSION:
                    $this->‪match(‪Lexer::T_COMPRESSION);
                    $options['compression'] = strtoupper((string)$this->‪tableOptionValue());
                    if (!in_array($options['compression'], ['ZLIB', 'LZ4', 'NONE'], true)) {
                        $this->‪syntaxError('ZLIB, LZ4 or NONE', $this->lexer->token);
                    }
                    break;
                case ‪Lexer::T_CONNECTION:
                    $this->‪match(‪Lexer::T_CONNECTION);
                    $options['connection'] = (string)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_DATA:
                    $this->‪match(‪Lexer::T_DATA);
                    $this->‪match(‪Lexer::T_DIRECTORY);
                    $options['data_directory'] = (string)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_DELAY_KEY_WRITE:
                    $this->‪match(‪Lexer::T_DELAY_KEY_WRITE);
                    $options['delay_key_write'] = (int)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_ENCRYPTION:
                    $this->‪match(‪Lexer::T_ENCRYPTION);
                    $options['encryption'] = strtoupper((string)$this->‪tableOptionValue());
                    if (!in_array($options['encryption'], ['Y', 'N'], true)) {
                        $this->‪syntaxError('Y or N', $this->lexer->token);
                    }
                    break;
                case ‪Lexer::T_INDEX:
                    $this->‪match(‪Lexer::T_INDEX);
                    $this->‪match(‪Lexer::T_DIRECTORY);
                    $options['index_directory'] = (string)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_INSERT_METHOD:
                    $this->‪match(‪Lexer::T_INSERT_METHOD);
                    $options['insert_method'] = strtoupper((string)$this->‪tableOptionValue());
                    if (!in_array($options['insert_method'], ['NO', 'FIRST', 'LAST'], true)) {
                        $this->‪syntaxError('NO, FIRST or LAST', $this->lexer->token);
                    }
                    break;
                case ‪Lexer::T_KEY_BLOCK_SIZE:
                    $this->‪match(‪Lexer::T_KEY_BLOCK_SIZE);
                    $options['key_block_size'] = (int)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_MAX_ROWS:
                    $this->‪match(‪Lexer::T_MAX_ROWS);
                    $options['max_rows'] = (int)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_MIN_ROWS:
                    $this->‪match(‪Lexer::T_MIN_ROWS);
                    $options['min_rows'] = (int)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_PACK_KEYS:
                    $this->‪match(‪Lexer::T_PACK_KEYS);
                    $options['pack_keys'] = strtoupper((string)$this->‪tableOptionValue());
                    if (!in_array($options['pack_keys'], ['0', '1', 'DEFAULT'], true)) {
                        $this->‪syntaxError('0, 1 or DEFAULT', $this->lexer->token);
                    }
                    break;
                case ‪Lexer::T_PASSWORD:
                    $this->‪match(‪Lexer::T_PASSWORD);
                    $options['password'] = (string)$this->‪tableOptionValue();
                    break;
                case ‪Lexer::T_ROW_FORMAT:
                    $this->‪match(‪Lexer::T_ROW_FORMAT);
                    $options['row_format'] = (string)$this->‪tableOptionValue();
                    $validRowFormats = ['DEFAULT', 'DYNAMIC', 'FIXED', 'COMPRESSED', 'REDUNDANT', 'COMPACT'];
                    if (!in_array($options['row_format'], $validRowFormats, true)) {
                        $this->‪syntaxError(
                            'DEFAULT, DYNAMIC, FIXED, COMPRESSED, REDUNDANT, COMPACT',
                            $this->lexer->token
                        );
                    }
                    break;
                case ‪Lexer::T_STATS_AUTO_RECALC:
                    $this->‪match(‪Lexer::T_STATS_AUTO_RECALC);
                    $options['stats_auto_recalc'] = strtoupper((string)$this->‪tableOptionValue());
                    if (!in_array($options['stats_auto_recalc'], ['0', '1', 'DEFAULT'], true)) {
                        $this->‪syntaxError('0, 1 or DEFAULT', $this->lexer->token);
                    }
                    break;
                case ‪Lexer::T_STATS_PERSISTENT:
                    $this->‪match(‪Lexer::T_STATS_PERSISTENT);
                    $options['stats_persistent'] = strtoupper((string)$this->‪tableOptionValue());
                    if (!in_array($options['stats_persistent'], ['0', '1', 'DEFAULT'], true)) {
                        $this->‪syntaxError('0, 1 or DEFAULT', $this->lexer->token);
                    }
                    break;
                case ‪Lexer::T_STATS_SAMPLE_PAGES:
                    $this->‪match(‪Lexer::T_STATS_SAMPLE_PAGES);
                    $options['stats_sample_pages'] = strtoupper((string)$this->‪tableOptionValue());
                    if (!in_array($options['stats_sample_pages'], ['0', '1', 'DEFAULT'], true)) {
                        $this->‪syntaxError('0, 1 or DEFAULT', $this->lexer->token);
                    }
                    break;
                case ‪Lexer::T_TABLESPACE:
                    $this->‪match(‪Lexer::T_TABLESPACE);
                    $options['tablespace'] = (string)$this->‪tableOptionValue();
                    break;
                default:
                    $this->‪syntaxError(
                        'DEFAULT, ENGINE, AUTO_INCREMENT, AVG_ROW_LENGTH, CHARACTER SET, ' .
                        'CHECKSUM, COLLATE, COMMENT, COMPRESSION, CONNECTION, DATA DIRECTORY, ' .
                        'DELAY_KEY_WRITE, ENCRYPTION, INDEX DIRECTORY, INSERT_METHOD, KEY_BLOCK_SIZE, ' .
                        'MAX_ROWS, MIN_ROWS, PACK_KEYS, PASSWORD, ROW_FORMAT, STATS_AUTO_RECALC, ' .
                        'STATS_PERSISTENT, STATS_SAMPLE_PAGES or TABLESPACE'
                    );
            }
        }
 
        return $options;
    }
 
    protected function ‪tableOptionValue()
    {
        // Skip the optional equals sign
        if ($this->lexer->isNextToken(‪Lexer::T_EQUALS)) {
            $this->‪match(‪Lexer::T_EQUALS);
        }
        $this->lexer->moveNext();
 
        return $this->lexer->token['value'];
    }
 
    protected function ‪schemaObjectName()
    {
        $schemaObjectName = $this->lexer->lookahead['value'];
        $this->lexer->moveNext();
 
        return new ‪AST\Identifier((string)$schemaObjectName);
    }
}