Introduce knowledge subcontext

Classes/Command/KnowledgeCommandController.php

@@ -0,0 +1,49 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Sitegeist\Chatterbox\Command;
+
+use Neos\Flow\Annotations as Flow;
+use Neos\Flow\Cli\CommandController;
+use Neos\Flow\Utility\Environment;
+use OpenAI\Contracts\ClientContract as OpenAiClientContract;
+use Sitegeist\Chatterbox\Domain\Knowledge\KnowledgePool;
+use Sitegeist\Flow\OpenAiClientFactory\OpenAiClientFactory;
+
+#[Flow\Scope('singleton')]
+class KnowledgeCommandController extends CommandController
+{
+    private readonly OpenAiClientContract $client;
+
+    public function __construct(
+        private readonly KnowledgePool $knowledgePool,
+        private readonly Environment $environment,
+        OpenAiClientFactory $clientFactory
+    ) {
+        $this->client = $clientFactory->createClient();
+        parent::__construct();
+    }
+
+    public function updatePoolCommand(): void
+    {
+        $this->outputLine('Updating knowledge pool');
+        $sources = $this->knowledgePool->findAllSources();
+        $this->output->progressStart(count($sources));
+        foreach ($sources as $sourceOfKnowledge) {
+            $content = $sourceOfKnowledge->getContent();
+
+            $path = $this->environment->getPathToTemporaryDirectory() . '/' . $sourceOfKnowledge->getName() . '-' . time() . '.jsonl';
+            \file_put_contents($path, (string)$content);
+
+            $this->client->files()->upload([
+                'file' => fopen($path, 'r'),
+                'purpose' => 'assistants'
+            ]);
+            \unlink($path);
+            $this->output->progressAdvance();
+        }
+        $this->output->progressFinish();
+        $this->outputLine('');
+    }
+}

Classes/Controller/AssistantModuleController.php

@@ -15,6 +15,7 @@
 use Psr\Log\LoggerInterface;
 use Sitegeist\Chatterbox\Domain\AssistantDepartment;
 use Sitegeist\Chatterbox\Domain\AssistantRecord;
+use Sitegeist\Chatterbox\Domain\Knowledge\KnowledgePool;
 use Sitegeist\Chatterbox\Domain\MessageRecord;
 use Sitegeist\Chatterbox\Domain\Toolbox;
 use Sitegeist\Chatterbox\Tools\ToolContract;
@@ -24,9 +25,10 @@ class AssistantModuleController extends AbstractModuleController
     protected $defaultViewObjectName = FusionView::class;
 
     public function __construct(
-        private OpenAiClientContract $client,
-        private Toolbox $toolbox,
-        private AssistantDepartment $assistantDepartment,
+        private readonly OpenAiClientContract $client,
+        private readonly Toolbox $toolbox,
+        private readonly KnowledgePool $knowledgePool,
+        private readonly AssistantDepartment $assistantDepartment,
     ) {
     }
 
@@ -40,6 +42,7 @@ public function editAction(string $assistantId): void
     {
         $assistant = $this->assistantDepartment->findAssistantById($assistantId);
         $this->view->assign('availableTools', $this->toolbox->findAll());
+        $this->view->assign('availableSourcesOfKnowledge', $this->knowledgePool->findAllSources());
         $this->view->assign('assistant', $assistant);
     }
 

Classes/Domain/AssistantDepartment.php

@@ -57,7 +57,7 @@ public function updateAssistant(AssistantRecord $assistantRecord): void
      */
     private function createMetadataConfiguration(AssistantRecord $assistantRecord): array
     {
-        return ['selectedTools' => json_encode($assistantRecord->selectedTools), 'selectedFiles' => json_encode($assistantRecord->selectedFiles)];
+        return ['selectedTools' => json_encode($assistantRecord->selectedTools), 'selectedFiles' => json_encode($assistantRecord->fileIds)];
     }
 
     /**

Classes/Domain/AssistantRecord.php

@@ -17,7 +17,8 @@ final class AssistantRecord
      * @param mixed[] $tools
      * @param string[] $metadata
      * @param string[] $selectedTools
-     * @param string[] $selectedFiles
+     * @param string[] $selectedSourcesOfKnowledge
+     * @param string[] $fileIds
      */
     public function __construct(
         public readonly string $id,
@@ -28,14 +29,15 @@ public function __construct(
         public readonly ?array $tools = [],
         public readonly ?array $metadata = [],
         public readonly array $selectedTools = [],
-        public readonly array $selectedFiles = [],
+        public readonly array $selectedSourcesOfKnowledge = [],
+        public readonly array $fileIds = [],
     ) {
     }
 
     public static function fromAssistantResponse(AssistantResponse $response): self
     {
         $selectedTools = array_key_exists('selectedTools', $response->metadata) ? json_decode($response->metadata['selectedTools'], true) : [];
-        $selectedFiles = array_key_exists('selectedFiles', $response->metadata) ? json_decode($response->metadata['selectedFiles'], true) : [];
+        $selectedSourcesOfKnowledge = array_key_exists('selectedSourcesOfKnowledge', $response->metadata) ? json_decode($response->metadata['selectedSourcesOfKnowledge'], true) : [];
 
         return new self(
             $response->id,
@@ -46,7 +48,8 @@ public static function fromAssistantResponse(AssistantResponse $response): self
             array_map(fn(AssistantResponseToolCodeInterpreter|AssistantResponseToolRetrieval|AssistantResponseToolFunction $item) => $item->toArray(), $response->tools),
             $response->metadata,
             $selectedTools,
-            $selectedFiles,
+            $selectedSourcesOfKnowledge,
+            $response->fileIds,
         );
     }
 }

Classes/Domain/Knowledge/ContentRepositorySourceDesignator.php

@@ -0,0 +1,66 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Sitegeist\Chatterbox\Domain\Knowledge;
+
+use Neos\ContentRepository\Domain\ContentSubgraph\NodePath;
+use Neos\ContentRepository\Domain\Model\Node;
+use Neos\ContentRepository\Domain\NodeAggregate\NodeAggregateIdentifier;
+use Neos\ContentRepository\Domain\Service\ContextFactory;
+use Neos\Flow\Annotations as Flow;
+use Neos\Neos\Domain\Service\ContentDimensionPresetSourceInterface;
+
+#[Flow\Proxy(false)]
+final class ContentRepositorySourceDesignator
+{
+    private function __construct(
+        private readonly NodeAggregateIdentifier|NodePath $root,
+        private readonly array $dimensionValues,
+    ) {
+    }
+
+    public static function createFromConfiguration(array $values): self
+    {
+        $rootDesignator = $values['root'];
+        if (\str_starts_with($rootDesignator, '#')) {
+            return new self(
+                NodeAggregateIdentifier::fromString(\mb_substr($rootDesignator, 1)),
+                $values['dimensions']
+            );
+        } elseif (\str_starts_with($rootDesignator, '/')) {
+            return new self(
+                NodePath::fromString($rootDesignator),
+                $values['dimensions']
+            );
+        }
+
+        throw new \DomainException('ContentRepository source designators can only be instantiated from valid node aggregate ids or absolute node paths, "' . $rootDesignator . '" given.', 1705938983);
+    }
+
+    public function findRootNode(ContextFactory $contentContextFactory, ContentDimensionPresetSourceInterface $contentDimensionPresetSource): Node
+    {
+        $contextDimensions = [];
+        foreach ($contentDimensionPresetSource->getAllPresets() as $dimensionId => $presetConfig) {
+            $contextDimensions[$dimensionId] = $presetConfig['presets'][$this->dimensionValues[$dimensionId]]['values'];
+        }
+        $subgraph = $contentContextFactory->create([
+            'dimensions' => $contextDimensions,
+            'targetDimensions' => $this->dimensionValues,
+        ]);
+
+        $rootNode = $this->root instanceof NodeAggregateIdentifier
+            ? $subgraph->getNodeByIdentifier((string)$this->root)
+            : $subgraph->getNode((string)$this->root);
+
+        if (!$rootNode instanceof Node) {
+            $designatorType = $this->root instanceof NodeAggregateIdentifier
+                ? 'id'
+                : 'path';
+
+            throw new \DomainException('Could not find root node with ' . $designatorType . ' "' . $this->root . '".', 1705939289);
+        }
+
+        return $rootNode;
+    }
+}

Classes/Domain/Knowledge/ContentRepositorySourceOfKnowledge.php

@@ -0,0 +1,103 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Sitegeist\Chatterbox\Domain\Knowledge;
+
+use GuzzleHttp\Psr7\Uri;
+use Neos\ContentRepository\Domain\Model\NodeInterface;
+use Neos\Flow\Annotations as Flow;
+use Neos\Neos\Domain\Service\ContentContextFactory;
+use Neos\Neos\Domain\Service\ContentDimensionPresetSourceInterface;
+
+final class ContentRepositorySourceOfKnowledge implements SourceOfKnowledgeContract
+{
+    #[Flow\Inject]
+    protected ContentContextFactory $contentContextFactory;
+
+    #[Flow\Inject]
+    protected ContentDimensionPresetSourceInterface $contentDimensionPresetSource;
+
+    public function __construct(
+        private readonly string $name,
+        private readonly string $description,
+        private readonly ContentRepositorySourceDesignator $designator,
+    ) {
+    }
+
+    public static function createFromConfiguration(string $name, array $options): static
+    {
+        return new static(
+            $name,
+            $options['description'] ?? 'null',
+            ContentRepositorySourceDesignator::createFromConfiguration($options),
+        );
+    }
+
+    public function getName(): string
+    {
+        return $this->name;
+    }
+
+    public function getDescription(): string
+    {
+        return $this->description;
+    }
+
+    public function getContent(): JsonlRecordCollection
+    {
+        $rootNode = $this->designator->findRootNode($this->contentContextFactory, $this->contentDimensionPresetSource);
+
+        return new JsonlRecordCollection(...$this->traverseSubtree($rootNode));
+    }
+
+    /**
+     * @return JsonlRecord[]
+     */
+    private function traverseSubtree(NodeInterface $documentNode): array
+    {
+        $documents = [];
+        if (!$documentNode->getNodeType()->isOfType('Neos.Neos:Shortcut')) {
+            $documents[] = $this->transformDocument($documentNode);
+        }
+        foreach ($documentNode->getChildNodes('Neos.Neos:Document') as $childDocument) {
+            $documents = array_merge($documents, $this->traverseSubtree($childDocument));
+        }
+
+        return $documents;
+    }
+
+    private function transformDocument(NodeInterface $documentNode): JsonlRecord
+    {
+        $content = '';
+        foreach ($documentNode->getChildNodes('Neos.Neos:Content,Neos.Neos:ContentCollection') as $childNode) {
+            $content .= ' ' . $this->extractContent($childNode);
+        }
+
+        return new JsonlRecord(
+            $documentNode->getIdentifier(),
+            new Uri('node://' . $documentNode->getIdentifier()),
+            trim($content)
+        );
+    }
+
+    private function extractContent(NodeInterface $contentNode): string
+    {
+        $content = '';
+
+        if ($contentNode->getNodeType()->isOfType('Neos.Neos:ContentCollection')) {
+            foreach ($contentNode->getChildNodes('Neos.Neos:Content,Neos.Neos:ContentCollection') as $childNode) {
+                $content .= $this->extractContent($childNode);
+            }
+        }
+        if ($contentNode->getNodeType()->isOfType('Neos.Neos:Content')) {
+            foreach ($contentNode->getNodeType()->getProperties() as $propertyName => $propertyConfiguration) {
+                if (($propertyConfiguration['type'] ?? 'string') === 'string' && ($propertyConfiguration['ui']['inlineEditable'] ?? false) === true) {
+                    $content .= ' ' . $contentNode->getProperty($propertyName);
+                }
+            }
+        }
+
+        return trim($content);
+    }
+}

Classes/Domain/Knowledge/JsonlRecord.php

@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Sitegeist\Chatterbox\Domain\Knowledge;
+
+use Neos\Flow\Annotations as Flow;
+use Psr\Http\Message\UriInterface;
+
+#[Flow\Proxy(false)]
+final class JsonlRecord implements \JsonSerializable
+{
+    public function __construct(
+        public readonly string $id,
+        public readonly UriInterface $url,
+        public readonly string $content,
+    ) {
+    }
+
+    /**
+     * @return array<string,mixed>
+     */
+    public function jsonSerialize(): array
+    {
+        return [
+            'id' => $this->id,
+            'url' => (string)$this->url,
+            'content' => $this->content
+        ];
+    }
+}

Classes/Domain/Knowledge/JsonlRecordCollection.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Sitegeist\Chatterbox\Domain\Knowledge;
+
+final class JsonlRecordCollection implements \Stringable
+{
+    /**
+     * @var array<JsonlRecord>
+     */
+    private readonly array $records;
+
+    public function __construct(
+        JsonlRecord ...$records
+    ) {
+        $this->records = $records;
+    }
+
+    public function __toString(): string
+    {
+        return implode("\n", array_map(
+            fn (JsonlRecord $record): string => \str_replace(PHP_EOL, ' ', \json_encode($record, JSON_THROW_ON_ERROR)),
+            $this->records
+        ));
+    }
+}