diff --git a/src/Highlighter.php b/src/Highlighter.php
index 15cc1ff..27fb64e 100644
--- a/src/Highlighter.php
+++ b/src/Highlighter.php
@@ -9,6 +9,7 @@
use Tempest\Highlight\Languages\Html\HtmlLanguage;
use Tempest\Highlight\Languages\Php\PhpLanguage;
use Tempest\Highlight\Themes\CssTheme;
+use Tempest\Highlight\Tokens\GroupTokens;
use Tempest\Highlight\Tokens\ParseTokens;
use Tempest\Highlight\Tokens\RenderTokens;
@@ -86,7 +87,9 @@ private function parseContent(string $content, Language $language): string
// Patterns
$tokens = (new ParseTokens())($content, $language);
- $output = (new RenderTokens($this->theme))($content, $tokens);
+ $groupedTokens = (new GroupTokens())($tokens);
+
+ $output = (new RenderTokens($this->theme))($content, $groupedTokens);
return $this->shouldEscape
? Escape::html($output)
diff --git a/src/Tokens/GroupTokens.php b/src/Tokens/GroupTokens.php
new file mode 100644
index 0000000..babaf66
--- /dev/null
+++ b/src/Tokens/GroupTokens.php
@@ -0,0 +1,44 @@
+start === $b->start) {
+ return $b->end <=> $a->end;
+ }
+
+ return $a->start <=> $b->start;
+ });
+
+ // Group tokens by parent and child
+ /** @var \Tempest\Highlight\Tokens\Token[] $groupedTokens */
+ $groupedTokens = [];
+
+ while($token = current($tokens)) {
+ $token = $token->cloneWithoutParent();
+
+ foreach ($tokens as $compareKey => $compareToken) {
+ if ($token->contains($compareToken)) {
+ if ($token->canContain($compareToken)) {
+ $token->addChild($compareToken);
+ }
+
+ unset($tokens[$compareKey]);
+ }
+ }
+
+ if ($token->parent === null) {
+ $groupedTokens[] = $token;
+ }
+
+ next($tokens);
+ }
+
+ return $groupedTokens;
+ }
+}
\ No newline at end of file
diff --git a/src/Tokens/ParseTokens.php b/src/Tokens/ParseTokens.php
index 0db2498..594b67d 100644
--- a/src/Tokens/ParseTokens.php
+++ b/src/Tokens/ParseTokens.php
@@ -16,6 +16,7 @@ public function __invoke(string $content, Language $language): array
{
$tokens = [];
+ // Match tokens from patterns
foreach ($language->getPatterns() as $key => $pattern) {
if ($pattern instanceof TokenType) {
$pattern = new GenericPattern(
diff --git a/src/Tokens/RenderTokens.php b/src/Tokens/RenderTokens.php
index 81f6eb3..12d1e07 100644
--- a/src/Tokens/RenderTokens.php
+++ b/src/Tokens/RenderTokens.php
@@ -25,33 +25,9 @@ public function __invoke(
array $tokens,
int $parsedOffset = 0
): string {
- usort($tokens, fn (Token $a, Token $b) => $a->offset <=> $b->offset);
-
- /** @var \Tempest\Highlight\Tokens\Token[] $groupedTokens */
- $groupedTokens = [];
-
- while($token = current($tokens)) {
- $token = $token->cloneWithoutParent();
-
- foreach ($tokens as $compareKey => $compareToken) {
- if ($token->contains($compareToken)) {
- if ($token->canContain($compareToken)) {
- $token->addChild($compareToken);
- }
- unset($tokens[$compareKey]);
- }
- }
-
- if ($token->parent === null) {
- $groupedTokens[] = $token;
- }
-
- next($tokens);
- }
-
$output = $content;
- foreach ($groupedTokens as $currentToken) {
+ foreach ($tokens as $currentToken) {
$value = $currentToken->hasChildren()
? ($this)(
content: $currentToken->value,
diff --git a/tests/Languages/Php/PhpLanguageTest.php b/tests/Languages/Php/PhpLanguageTest.php
index 52ade94..a988cdc 100644
--- a/tests/Languages/Php/PhpLanguageTest.php
+++ b/tests/Languages/Php/PhpLanguageTest.php
@@ -28,6 +28,7 @@ public static function data(): array
["public const string|\Stringable MESSAGE = 'hi';", 'public const string|\Stringable MESSAGE = \'hi\';'],
["public string|\Stringable \$message;", 'public string|\Stringable $message;'],
['for($x = 0; $x < 150; $x++) {', 'for($x = 0; $x < 150; $x++) {'],
+ ["'namespace ';", "'namespace ';"]
];
}
}
diff --git a/tests/RenderTokensTest.php b/tests/RenderTokensTest.php
index 8f9fea2..0d0e735 100644
--- a/tests/RenderTokensTest.php
+++ b/tests/RenderTokensTest.php
@@ -8,6 +8,7 @@
use PHPUnit\Framework\TestCase;
use Tempest\Highlight\Escape;
use Tempest\Highlight\Themes\CssTheme;
+use Tempest\Highlight\Tokens\GroupTokens;
use Tempest\Highlight\Tokens\RenderTokens;
use Tempest\Highlight\Tokens\Token;
use Tempest\Highlight\Tokens\TokenType;
@@ -19,7 +20,7 @@ public function test_nested_tokens(): void
{
$content = '/** @var \Tempest\View\GenericView $this */';
- $tokens = [
+ $tokens = (new GroupTokens())([
new Token(
offset: 0,
value: '/** @var \Tempest\View\GenericView $this */',
@@ -30,7 +31,7 @@ public function test_nested_tokens(): void
value: 'GenericView',
type: TokenType::TYPE,
),
- ];
+ ]);
$parsed = Escape::html((new RenderTokens(new CssTheme()))($content, $tokens));
@@ -43,7 +44,7 @@ public function test_nested_tokens(): void
#[Test]
public function test_nested_tokens_b()
{
- $tokens = [
+ $tokens = (new GroupTokens())([
new Token(
offset: 0,
value: "#[Get(hi: '/')]",
@@ -59,7 +60,7 @@ public function test_nested_tokens_b()
value: 'hi',
type: TokenType::PROPERTY,
),
- ];
+ ]);
$output = Escape::html((new RenderTokens(new CssTheme()))("#[Get(hi: '/')]", $tokens));
@@ -75,7 +76,7 @@ public function test_nested_tokens_c()
$content = " #[Get(hi: '/')]
public";
- $tokens = [
+ $tokens = (new GroupTokens())([
new Token(
offset: 4,
value: "#[Get(hi: '/')]",
@@ -96,7 +97,7 @@ public function test_nested_tokens_c()
value: 'public',
type: TokenType::KEYWORD,
),
- ];
+ ]);
$output = Escape::html((new RenderTokens(new CssTheme()))($content, $tokens));
diff --git a/tests/test.md b/tests/test.md
index b9ca3d4..24d2bba 100644
--- a/tests/test.md
+++ b/tests/test.md
@@ -1,6 +1,3 @@
```php
-public function __construct(
- ?string $encoding = null,
-) {}
-
+'namespace ';
```
\ No newline at end of file