TASK: Reform all parsers to use new Lexer

Author: grebaldi

src/Language/AST/Node/BinaryOperation/BinaryOperator.php

@@ -24,6 +24,8 @@
 
 enum BinaryOperator: string
 {
+    case NULLISH_COALESCE = 'NULLISH_COALESCE';
+
     case AND = 'AND';
     case OR = 'OR';
 

src/Language/AST/Node/TemplateLiteral/TemplateLiteralLine.php

@@ -0,0 +1,32 @@
+<?php
+
+/**
+ * PackageFactory.ComponentEngine - Universal View Components for PHP
+ *   Copyright (C) 2023 Contributors of PackageFactory.ComponentEngine
+ *
+ *   This program is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+namespace PackageFactory\ComponentEngine\Language\AST\Node\TemplateLiteral;
+
+final class TemplateLiteralLine
+{
+    public function __construct(
+        public readonly int $indentation,
+        public readonly TemplateLiteralSegments $segments
+    ) {
+    }
+}

src/Language/AST/Node/TemplateLiteral/TemplateLiteralLines.php

@@ -0,0 +1,36 @@
+<?php
+
+/**
+ * PackageFactory.ComponentEngine - Universal View Components for PHP
+ *   Copyright (C) 2023 Contributors of PackageFactory.ComponentEngine
+ *
+ *   This program is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+namespace PackageFactory\ComponentEngine\Language\AST\Node\TemplateLiteral;
+
+final class TemplateLiteralLines
+{
+    /**
+     * @var TemplateLiteralLine[]
+     */
+    public readonly array $items;
+
+    public function __construct(TemplateLiteralLine ...$items)
+    {
+        $this->items = $items;
+    }
+}

src/Language/AST/Node/TemplateLiteral/TemplateLiteralNode.php

@@ -29,7 +29,8 @@ final class TemplateLiteralNode extends Node
 {
     public function __construct(
         public readonly Range $rangeInSource,
-        public readonly TemplateLiteralSegments $segments
+        public readonly int $indentation,
+        public readonly TemplateLiteralLines $lines
     ) {
     }
 }

src/Language/Parser/BooleanLiteral/BooleanLiteralParser.php

@@ -24,29 +24,33 @@
 
 use PackageFactory\ComponentEngine\Framework\PHP\Singleton\Singleton;
 use PackageFactory\ComponentEngine\Language\AST\Node\BooleanLiteral\BooleanLiteralNode;
-use PackageFactory\ComponentEngine\Parser\Tokenizer\Scanner;
-use PackageFactory\ComponentEngine\Parser\Tokenizer\Token;
-use PackageFactory\ComponentEngine\Parser\Tokenizer\TokenType;
+use PackageFactory\ComponentEngine\Language\Lexer\Lexer;
+use PackageFactory\ComponentEngine\Language\Lexer\Token\TokenType;
+use PackageFactory\ComponentEngine\Language\Lexer\Token\TokenTypes;
 
 final class BooleanLiteralParser
 {
     use Singleton;
 
-    /**
-     * @param \Iterator<mixed,Token> $tokens
-     * @return BooleanLiteralNode
-     */
-    public function parse(\Iterator &$tokens): BooleanLiteralNode
+    private static TokenTypes $TOKEN_TYPES_BOOLEAN_KEYWORDS;
+
+    private function __construct()
+    {
+        self::$TOKEN_TYPES_BOOLEAN_KEYWORDS ??= TokenTypes::from(
+            TokenType::KEYWORD_TRUE,
+            TokenType::KEYWORD_FALSE
+        );
+    }
+
+    public function parse(Lexer $lexer): BooleanLiteralNode
     {
-        Scanner::assertType($tokens, TokenType::KEYWORD_TRUE, TokenType::KEYWORD_FALSE);
+        $lexer->readOneOf(self::$TOKEN_TYPES_BOOLEAN_KEYWORDS);
 
-        $token = $tokens->current();
+        $token = $lexer->getTokenUnderCursor();
         $value = $token->type === TokenType::KEYWORD_TRUE;
 
-        Scanner::skipOne($tokens);
-
         return new BooleanLiteralNode(
-            rangeInSource: $token->boundaries,
+            rangeInSource: $token->rangeInSource,
             value: $value
         );
     }

src/Language/Parser/ComponentDeclaration/ComponentDeclarationParser.php

@@ -28,150 +28,89 @@
 use PackageFactory\ComponentEngine\Language\AST\Node\ComponentDeclaration\ComponentNameNode;
 use PackageFactory\ComponentEngine\Language\AST\Node\Expression\ExpressionNode;
 use PackageFactory\ComponentEngine\Language\AST\Node\PropertyDeclaration\PropertyDeclarationNodes;
+use PackageFactory\ComponentEngine\Language\Lexer\Lexer;
+use PackageFactory\ComponentEngine\Language\Lexer\Token\TokenType;
+use PackageFactory\ComponentEngine\Language\Lexer\Token\TokenTypes;
 use PackageFactory\ComponentEngine\Language\Parser\Expression\ExpressionParser;
 use PackageFactory\ComponentEngine\Language\Parser\PropertyDeclaration\PropertyDeclarationParser;
 use PackageFactory\ComponentEngine\Parser\Source\Range;
-use PackageFactory\ComponentEngine\Parser\Tokenizer\Scanner;
-use PackageFactory\ComponentEngine\Parser\Tokenizer\Token;
-use PackageFactory\ComponentEngine\Parser\Tokenizer\TokenType;
 
 final class ComponentDeclarationParser
 {
     use Singleton;
 
+    private static TokenTypes $TOKEN_TYPES_SPACE;
+
     private ?PropertyDeclarationParser $propertyDeclarationParser = null;
     private ?ExpressionParser $returnParser = null;
 
-    /**
-     * @param \Iterator<mixed,Token> $tokens
-     * @return ComponentDeclarationNode
-     */
-    public function parse(\Iterator &$tokens): ComponentDeclarationNode
+    private function __construct()
     {
-        $componentKeywordToken = $this->extractComponentKeywordToken($tokens);
-        $name = $this->parseName($tokens);
-
-        $this->skipOpeningBracketToken($tokens);
+        self::$TOKEN_TYPES_SPACE ??= TokenTypes::from(
+            TokenType::SPACE,
+            TokenType::END_OF_LINE
+        );
+    }
 
-        $props = $this->parseProps($tokens);
+    public function parse(Lexer $lexer): ComponentDeclarationNode
+    {
+        $lexer->read(TokenType::KEYWORD_COMPONENT);
+        $start = $lexer->getStartPosition();
+        $lexer->skipSpace();
 
-        $this->skipReturnKeywordToken($tokens);
+        $name = $this->parseName($lexer);
+        $props = $this->parseProps($lexer);
+        $return = $this->parseReturn($lexer);
 
-        $return = $this->parseReturn($tokens);
-        $closingBracketToken = $this->extractClosingBracketToken($tokens);
+        $lexer->read(TokenType::BRACKET_CURLY_CLOSE);
+        $end = $lexer->getEndPosition();
 
         return new ComponentDeclarationNode(
-            rangeInSource: Range::from(
-                $componentKeywordToken->boundaries->start,
-                $closingBracketToken->boundaries->end
-            ),
+            rangeInSource: Range::from($start, $end),
             name: $name,
             props: $props,
             return: $return
         );
     }
 
-    /**
-     * @param \Iterator<mixed,Token> $tokens
-     * @return Token
-     */
-    private function extractComponentKeywordToken(\Iterator &$tokens): Token
+    private function parseName(Lexer $lexer): ComponentNameNode
     {
-        Scanner::assertType($tokens, TokenType::KEYWORD_COMPONENT);
-
-        $componentKeywordToken = $tokens->current();
+        $lexer->read(TokenType::WORD);
+        $componentNameToken = $lexer->getTokenUnderCursor();
 
-        Scanner::skipOne($tokens);
-        Scanner::skipSpace($tokens);
-
-        return $componentKeywordToken;
-    }
-
-    /**
-     * @param \Iterator<mixed,Token> $tokens
-     * @return ComponentNameNode
-     */
-    private function parseName(\Iterator &$tokens): ComponentNameNode
-    {
-        Scanner::assertType($tokens, TokenType::STRING);
-
-        $componentNameToken = $tokens->current();
-
-        Scanner::skipOne($tokens);
-        Scanner::skipSpace($tokens);
+        $lexer->skipSpace();
 
         return new ComponentNameNode(
-            rangeInSource: $componentNameToken->boundaries,
+            rangeInSource: $componentNameToken->rangeInSource,
             value: ComponentName::from($componentNameToken->value)
         );
     }
 
-    /**
-     * @param \Iterator<mixed,Token> $tokens
-     * @return void
-     */
-    private function skipOpeningBracketToken(\Iterator &$tokens): void
-    {
-        Scanner::assertType($tokens, TokenType::BRACKET_CURLY_OPEN);
-        Scanner::skipOne($tokens);
-        Scanner::skipSpaceAndComments($tokens);
-    }
-
-    /**
-     * @param \Iterator<mixed,Token> $tokens
-     * @return PropertyDeclarationNodes
-     */
-    private function parseProps(\Iterator &$tokens): PropertyDeclarationNodes
+    private function parseProps(Lexer $lexer): PropertyDeclarationNodes
     {
         $this->propertyDeclarationParser ??= PropertyDeclarationParser::singleton();
 
-        $items = [];
-        while (Scanner::type($tokens) !== TokenType::KEYWORD_RETURN) {
-            assert($this->propertyDeclarationParser !== null);
-            $items[] = $this->propertyDeclarationParser->parse($tokens);
+        $lexer->read(TokenType::BRACKET_CURLY_OPEN);
+        $lexer->skipSpaceAndComments();
 
-            Scanner::skipSpaceAndComments($tokens);
+        $items = [];
+        while (!$lexer->peek(TokenType::KEYWORD_RETURN)) {
+            $lexer->expect(TokenType::WORD);
+            $items[] = $this->propertyDeclarationParser->parse($lexer);
+            $lexer->skipSpaceAndComments();
         }
 
         return new PropertyDeclarationNodes(...$items);
     }
 
-    /**
-     * @param \Iterator<mixed,Token> $tokens
-     * @return void
-     */
-    private function skipReturnKeywordToken(\Iterator &$tokens): void
-    {
-        Scanner::assertType($tokens, TokenType::KEYWORD_RETURN);
-        Scanner::skipOne($tokens);
-        Scanner::skipSpaceAndComments($tokens);
-    }
-
-    /**
-     * @param \Iterator<mixed,Token> $tokens
-     * @return ExpressionNode
-     */
-    private function parseReturn(\Iterator &$tokens): ExpressionNode
+    private function parseReturn(Lexer $lexer): ExpressionNode
     {
-        $this->returnParser ??= new ExpressionParser(
-            stopAt: TokenType::BRACKET_CURLY_CLOSE
-        );
-
-        return $this->returnParser->parse($tokens);
-    }
-
-    /**
-     * @param \Iterator<mixed,Token> $tokens
-     * @return Token
-     */
-    private function extractClosingBracketToken(\Iterator &$tokens): Token
-    {
-        Scanner::assertType($tokens, TokenType::BRACKET_CURLY_CLOSE);
-
-        $closingBracketToken = $tokens->current();
+        $this->returnParser ??= new ExpressionParser();
 
-        Scanner::skipOne($tokens);
+        $lexer->read(TokenType::KEYWORD_RETURN);
+        $lexer->readOneOf(self::$TOKEN_TYPES_SPACE);
+        $lexer->skipSpaceAndComments();
 
-        return $closingBracketToken;
+        return $this->returnParser->parse($lexer);
     }
 }