Fix tokens starting on the same offset

tempestphp · Mar 19, 2024 · bf80244 · bf80244
1 parent 41a24b7
commit bf80244
Show file tree

Hide file tree

Showing 7 changed files with 59 additions and 36 deletions.
diff --git a/src/Highlighter.php b/src/Highlighter.php
@@ -9,6 +9,7 @@
 use Tempest\Highlight\Languages\Html\HtmlLanguage;
 use Tempest\Highlight\Languages\Php\PhpLanguage;
 use Tempest\Highlight\Themes\CssTheme;
+use Tempest\Highlight\Tokens\GroupTokens;
 use Tempest\Highlight\Tokens\ParseTokens;
 use Tempest\Highlight\Tokens\RenderTokens;
 
@@ -86,7 +87,9 @@ private function parseContent(string $content, Language $language): string
         // Patterns
         $tokens = (new ParseTokens())($content, $language);
 
-        $output = (new RenderTokens($this->theme))($content, $tokens);
+        $groupedTokens = (new GroupTokens())($tokens);
+
+        $output = (new RenderTokens($this->theme))($content, $groupedTokens);
 
         return $this->shouldEscape
             ? Escape::html($output)

diff --git a/src/Tokens/GroupTokens.php b/src/Tokens/GroupTokens.php
@@ -0,0 +1,44 @@
+<?php
+
+namespace Tempest\Highlight\Tokens;
+
+final readonly class GroupTokens
+{
+    public function __invoke(array $tokens): array
+    {
+        // Sort tokens in the right order
+        usort($tokens, function (Token $a, Token $b) {
+            if ($a->start === $b->start) {
+                return $b->end <=> $a->end;
+            }
+
+            return $a->start <=> $b->start;
+        });
+
+        // Group tokens by parent and child
+        /** @var \Tempest\Highlight\Tokens\Token[] $groupedTokens */
+        $groupedTokens = [];
+
+        while($token = current($tokens)) {
+            $token = $token->cloneWithoutParent();
+
+            foreach ($tokens as $compareKey => $compareToken) {
+                if ($token->contains($compareToken)) {
+                    if ($token->canContain($compareToken)) {
+                        $token->addChild($compareToken);
+                    }
+
+                    unset($tokens[$compareKey]);
+                }
+            }
+
+            if ($token->parent === null) {
+                $groupedTokens[] = $token;
+            }
+
+            next($tokens);
+        }
+
+        return $groupedTokens;
+    }
+}
diff --git a/src/Tokens/ParseTokens.php b/src/Tokens/ParseTokens.php
@@ -16,6 +16,7 @@ public function __invoke(string $content, Language $language): array
     {
         $tokens = [];
 
+        // Match tokens from patterns
         foreach ($language->getPatterns() as $key => $pattern) {
             if ($pattern instanceof TokenType) {
                 $pattern = new GenericPattern(

diff --git a/src/Tokens/RenderTokens.php b/src/Tokens/RenderTokens.php
@@ -25,33 +25,9 @@ public function __invoke(
         array $tokens,
         int $parsedOffset = 0
     ): string {
-        usort($tokens, fn (Token $a, Token $b) => $a->offset <=> $b->offset);
-
-        /** @var \Tempest\Highlight\Tokens\Token[] $groupedTokens */
-        $groupedTokens = [];
-
-        while($token = current($tokens)) {
-            $token = $token->cloneWithoutParent();
-
-            foreach ($tokens as $compareKey => $compareToken) {
-                if ($token->contains($compareToken)) {
-                    if ($token->canContain($compareToken)) {
-                        $token->addChild($compareToken);
-                    }
-                    unset($tokens[$compareKey]);
-                }
-            }
-
-            if ($token->parent === null) {
-                $groupedTokens[] = $token;
-            }
-
-            next($tokens);
-        }
-
         $output = $content;
 
-        foreach ($groupedTokens as $currentToken) {
+        foreach ($tokens as $currentToken) {
             $value = $currentToken->hasChildren()
                 ? ($this)(
                     content: $currentToken->value,

diff --git a/tests/Languages/Php/PhpLanguageTest.php b/tests/Languages/Php/PhpLanguageTest.php
@@ -28,6 +28,7 @@ public static function data(): array
             ["public const string|\Stringable MESSAGE = 'hi';", '<span class="hl-keyword">public</span> <span class="hl-keyword">const</span> <span class="hl-type">string|\Stringable</span> <span class="hl-property">MESSAGE</span> = \'<span class="hl-value">hi</span>\';'],
             ["public string|\Stringable \$message;", '<span class="hl-keyword">public</span> <span class="hl-type">string|\<span class="hl-type">Stringable</span></span> <span class="hl-property">$message</span>;'],
             ['for($x = 0; $x < 150; $x++) {', '<span class="hl-keyword">f<span class="hl-keyword">or</span></span>($x = 0; $x &lt; 150; $x++) {'],
+            ["'namespace ';", "'<span class=\"hl-value\">namespace </span>';"]
         ];
     }
 }
diff --git a/tests/RenderTokensTest.php b/tests/RenderTokensTest.php
@@ -8,6 +8,7 @@
 use PHPUnit\Framework\TestCase;
 use Tempest\Highlight\Escape;
 use Tempest\Highlight\Themes\CssTheme;
+use Tempest\Highlight\Tokens\GroupTokens;
 use Tempest\Highlight\Tokens\RenderTokens;
 use Tempest\Highlight\Tokens\Token;
 use Tempest\Highlight\Tokens\TokenType;
@@ -19,7 +20,7 @@ public function test_nested_tokens(): void
     {
         $content = '/** @var \Tempest\View\GenericView $this */';
 
-        $tokens = [
+        $tokens = (new GroupTokens())([
             new Token(
                 offset: 0,
                 value: '/** @var \Tempest\View\GenericView $this */',
@@ -30,7 +31,7 @@ public function test_nested_tokens(): void
                 value: 'GenericView',
                 type: TokenType::TYPE,
             ),
-        ];
+        ]);
 
         $parsed = Escape::html((new RenderTokens(new CssTheme()))($content, $tokens));
 
@@ -43,7 +44,7 @@ public function test_nested_tokens(): void
     #[Test]
     public function test_nested_tokens_b()
     {
-        $tokens = [
+        $tokens = (new GroupTokens())([
             new Token(
                 offset: 0,
                 value: "#[Get(hi: '/')]",
@@ -59,7 +60,7 @@ public function test_nested_tokens_b()
                 value: 'hi',
                 type: TokenType::PROPERTY,
             ),
-        ];
+        ]);
 
         $output = Escape::html((new RenderTokens(new CssTheme()))("#[Get(hi: '/')]", $tokens));
 
@@ -75,7 +76,7 @@ public function test_nested_tokens_c()
         $content = "    #[Get(hi: '/')]
     public";
 
-        $tokens = [
+        $tokens = (new GroupTokens())([
             new Token(
                 offset: 4,
                 value: "#[Get(hi: '/')]",
@@ -96,7 +97,7 @@ public function test_nested_tokens_c()
                 value: 'public',
                 type: TokenType::KEYWORD,
             ),
-        ];
+        ]);
 
         $output = Escape::html((new RenderTokens(new CssTheme()))($content, $tokens));
 

diff --git a/tests/test.md b/tests/test.md
@@ -1,6 +1,3 @@
 ```php
-public function __construct(
-    ?string $encoding = null,
-) {}
-
+'namespace ';
 ```