From 69909da0350367dceef5c2558acd9606b1e89daa Mon Sep 17 00:00:00 2001 From: Julien Veyssier Date: Wed, 18 Dec 2024 18:11:58 +0100 Subject: [PATCH 1/6] adjust backend to use the agency feature, starting the frontend Signed-off-by: Julien Veyssier --- appinfo/info.xml | 2 +- lib/Controller/ChattyLLMController.php | 99 ++++++++++++++----- lib/Db/ChattyLLM/MessageMapper.php | 20 ++++ lib/Db/ChattyLLM/Session.php | 36 +++++-- lib/Db/ChattyLLM/SessionMapper.php | 4 +- lib/Listener/ChattyLLMTaskListener.php | 12 +++ .../Version020200Date20241218145833.php | 67 +++++++++++++ .../ChattyLLM/ChattyLLMInputForm.vue | 2 + 8 files changed, 206 insertions(+), 36 deletions(-) create mode 100644 lib/Migration/Version020200Date20241218145833.php diff --git a/appinfo/info.xml b/appinfo/info.xml index 37205223..ce07be3d 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -58,7 +58,7 @@ Known providers: More details on how to set this up in the [admin docs](https://docs.nextcloud.com/server/latest/admin_manual/ai/index.html) ]]> - 2.1.1 + 2.2.0 agpl Julien Veyssier Assistant diff --git a/lib/Controller/ChattyLLMController.php b/lib/Controller/ChattyLLMController.php index e7e87f79..390c79dd 100644 --- a/lib/Controller/ChattyLLMController.php +++ b/lib/Controller/ChattyLLMController.php @@ -292,15 +292,28 @@ public function generateForSession(int $sessionId): JSONResponse { return new JSONResponse(['error' => $this->l10n->t('Session not found')], Http::STATUS_NOT_FOUND); } - $stichedPrompt = - $this->getStichedMessages($sessionId) - . PHP_EOL - . 'assistant: '; - - try { - $taskId = $this->scheduleLLMTask($stichedPrompt, $sessionId); - } catch (\Exception $e) { - return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST); + if (class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentInteraction') + && isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID]) + ) { + $message = $this->messageMapper->getLastHumanMessage($sessionId); + $prompt = $message->getContent(); + $session = $this->sessionMapper->getUserSession($this->userId, $sessionId); + $lastConversationToken = $session->getAgencyConversationToken() ?? '{}'; + try { + $taskId = $this->scheduleAgencyTask($prompt, 0, $lastConversationToken, $sessionId); + } catch (\Exception $e) { + return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST); + } + } else { + $prompt = + $this->getStichedMessages($sessionId) + . PHP_EOL + . 'assistant: '; + try { + $taskId = $this->scheduleLLMTask($prompt, $sessionId); + } catch (\Exception $e) { + return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST); + } } return new JSONResponse(['taskId' => $taskId]); @@ -378,8 +391,11 @@ public function checkMessageGenerationTask(int $taskId, int $sessionId): JSONRes $message->setRole('assistant'); $message->setContent(trim($task->getOutput()['output'] ?? '')); $message->setTimestamp(time()); + $jsonMessage = $message->jsonSerialize(); + $session = $this->sessionMapper->getUserSession($this->userId, $sessionId); + $jsonMessage['session_agency_pending_actions'] = $session->getAgencyPendingActions(); // do not insert here, it is done by the listener - return new JSONResponse($message); + return new JSONResponse($jsonMessage); } catch (\OCP\DB\Exception $e) { $this->logger->warning('Failed to add a chat message into DB', ['exception' => $e]); return new JSONResponse(['error' => $this->l10n->t('Failed to add a chat message into DB')], Http::STATUS_INTERNAL_SERVER_ERROR); @@ -430,6 +446,7 @@ public function checkSession(int $sessionId): JSONResponse { 'messageTaskId' => null, 'titleTaskId' => null, 'sessionTitle' => $session->getTitle(), + 'session_agency_pending_actions' => $session->getAgencyPendingActions(), ]; if (!empty($messageTasks)) { $task = array_pop($messageTasks); @@ -578,12 +595,27 @@ private function getStichedMessages(int $sessionId): string { return $stichedPrompt; } + private function checkIfSessionIsThinking(string $customId): void { + try { + $tasks = $this->taskProcessingManager->getUserTasksByApp($this->userId, Application::APP_ID . ':chatty-llm', $customId); + } catch (\OCP\TaskProcessing\Exception\Exception $e) { + throw new \Exception('task_query_failed'); + } + $tasks = array_filter($tasks, static function (Task $task) { + return $task->getStatus() === Task::STATUS_RUNNING || $task->getStatus() === Task::STATUS_SCHEDULED; + }); + // prevent scheduling multiple llm tasks simultaneously for one session + if (!empty($tasks)) { + throw new \Exception('session_already_thinking'); + } + } + /** * Schedule the LLM task * * @param string $content * @param int $sessionId - * @param bool $isMessage + * @param bool $isMessage whether we want to generate a message or a session title * @return int|null * @throws Exception * @throws PreConditionNotMetException @@ -595,20 +627,41 @@ private function scheduleLLMTask(string $content, int $sessionId, bool $isMessag $customId = ($isMessage ? 'chatty-llm:' : 'chatty-title:') . $sessionId; - try { - $tasks = $this->taskProcessingManager->getUserTasksByApp($this->userId, Application::APP_ID . ':chatty-llm', $customId); - } catch (\OCP\TaskProcessing\Exception\Exception $e) { - throw new \Exception('task_query_failed'); - } - $tasks = array_filter($tasks, static function (Task $task) { - return $task->getStatus() === Task::STATUS_RUNNING || $task->getStatus() === Task::STATUS_SCHEDULED; - }); - // prevent scheduling multiple llm tasks simultaneously for one session - if (!empty($tasks)) { - throw new \Exception('session_already_thinking'); - } + $this->checkIfSessionIsThinking($customId); $task = new Task(TextToText::ID, ['input' => $content], Application::APP_ID . ':chatty-llm', $this->userId, $customId); $this->taskProcessingManager->scheduleTask($task); return $task->getId(); } + + /** + * Schedule an agency task + * + * @param string $content + * @param int $confirmation + * @param string $conversationToken + * @param int $sessionId + * @return int|null + * @throws Exception + * @throws PreConditionNotMetException + * @throws UnauthorizedException + * @throws ValidationException + */ + private function scheduleAgencyTask(string $content, int $confirmation, string $conversationToken, int $sessionId): ?int { + $customId = 'chatty-llm:' . $sessionId; + $this->checkIfSessionIsThinking($customId); + $taskInput = [ + 'input' => $content, + 'confirmation' => $confirmation, + 'conversation_token' => $conversationToken, + ]; + $task = new Task( + \OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID, + $taskInput, + Application::APP_ID . ':chatty-llm', + $this->userId, + $customId + ); + $this->taskProcessingManager->scheduleTask($task); + return $task->getId(); + } } diff --git a/lib/Db/ChattyLLM/MessageMapper.php b/lib/Db/ChattyLLM/MessageMapper.php index 17463c84..c4486478 100644 --- a/lib/Db/ChattyLLM/MessageMapper.php +++ b/lib/Db/ChattyLLM/MessageMapper.php @@ -58,6 +58,26 @@ public function getFirstNMessages(int $sessionId, int $n = 1): Message { return $this->findEntity($qb); } + /** + * @param integer $sessionId + * @return Message + * @throws \OCP\DB\Exception + * @throws \RuntimeException + * @throws \OCP\AppFramework\Db\DoesNotExistException + * @throws \OCP\AppFramework\Db\MultipleObjectsReturnedException + */ + public function getLastHumanMessage(int $sessionId): Message { + $qb = $this->db->getQueryBuilder(); + $qb->select(Message::$columns) + ->from($this->getTableName()) + ->where($qb->expr()->eq('session_id', $qb->createPositionalParameter($sessionId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('role', $qb->createPositionalParameter('human', IQueryBuilder::PARAM_STR))) + ->orderBy('timestamp', 'DESC') + ->setMaxResults(1); + + return $this->findEntity($qb); + } + /** * @param int $sessionId * @param int $cursor diff --git a/lib/Db/ChattyLLM/Session.php b/lib/Db/ChattyLLM/Session.php index 14a5e7c0..f469609e 100644 --- a/lib/Db/ChattyLLM/Session.php +++ b/lib/Db/ChattyLLM/Session.php @@ -29,12 +29,16 @@ use OCP\DB\Types; /** - * @method \string getUserId() - * @method \void setUserId(string $userId) - * @method \string|null getTitle() - * @method \void setTitle(?string $title) - * @method \int|null getTimestamp() - * @method \void setTimestamp(?int $timestamp) + * @method string getUserId() + * @method void setUserId(string $userId) + * @method \string|\null getTitle() + * @method \void setTitle(?\string $title) + * @method \int|\null getTimestamp() + * @method \void setTimestamp(?\int $timestamp) + * @method string|null getAgencyConversationToken() + * @method void setAgencyConversationToken(?string $agencyConversationToken) + * @method string|null getAgencyPendingActions() + * @method void setAgencyPendingActions(?string $agencyPendingActions) */ class Session extends Entity implements \JsonSerializable { /** @var string */ @@ -43,33 +47,45 @@ class Session extends Entity implements \JsonSerializable { protected $title; /** @var int */ protected $timestamp; + /** @var string */ + protected $agencyConversationToken; + /** @var string */ + protected $agencyPendingActions; public static $columns = [ 'id', 'user_id', 'title', 'timestamp', + 'agency_conversation_token', + 'agency_pending_actions', ]; public static $fields = [ 'id', 'userId', 'title', 'timestamp', + 'agencyConversationToken', + 'agencyPendingActions', ]; public function __construct() { $this->addType('user_id', Types::STRING); $this->addType('title', Types::STRING); $this->addType('timestamp', Types::INTEGER); + $this->addType('agency_conversation_token', Types::STRING); + $this->addType('agency_pending_actions', Types::STRING); } #[\ReturnTypeWillChange] public function jsonSerialize() { return [ - 'id' => $this->id, - 'user_id' => $this->userId, - 'title' => $this->title, - 'timestamp' => $this->timestamp, + 'id' => $this->getId(), + 'user_id' => $this->getUserId(), + 'title' => $this->getTitle(), + 'timestamp' => $this->getTimestamp(), + 'agency_conversation_token' => $this->getAgencyConversationToken(), + 'agency_pending_actions' => $this->getAgencyPendingActions(), ]; } } diff --git a/lib/Db/ChattyLLM/SessionMapper.php b/lib/Db/ChattyLLM/SessionMapper.php index 276fca67..bccc7e11 100644 --- a/lib/Db/ChattyLLM/SessionMapper.php +++ b/lib/Db/ChattyLLM/SessionMapper.php @@ -72,7 +72,7 @@ public function exists(string $userId, int $sessionId): bool { */ public function getUserSession(string $userId, int $sessionId): Session { $qb = $this->db->getQueryBuilder(); - $qb->select('id', 'title', 'timestamp') + $qb->select(Session::$columns) ->from($this->getTableName()) ->where($qb->expr()->eq('id', $qb->createPositionalParameter($sessionId, IQueryBuilder::PARAM_INT))) ->andWhere($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId, IQueryBuilder::PARAM_STR))); @@ -87,7 +87,7 @@ public function getUserSession(string $userId, int $sessionId): Session { */ public function getUserSessions(string $userId): array { $qb = $this->db->getQueryBuilder(); - $qb->select('id', 'title', 'timestamp') + $qb->select(Session::$columns) ->from($this->getTableName()) ->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId, IQueryBuilder::PARAM_STR))) ->orderBy('timestamp', 'DESC'); diff --git a/lib/Listener/ChattyLLMTaskListener.php b/lib/Listener/ChattyLLMTaskListener.php index b79deade..a13991e9 100644 --- a/lib/Listener/ChattyLLMTaskListener.php +++ b/lib/Listener/ChattyLLMTaskListener.php @@ -33,6 +33,7 @@ public function handle(Event $event): void { $task = $event->getTask(); $customId = $task->getCustomId(); $appId = $task->getAppId(); + $taskTypeId = $task->getTaskTypeId(); if ($customId === null || $appId !== (Application::APP_ID . ':chatty-llm')) { return; @@ -59,6 +60,17 @@ public function handle(Event $event): void { } catch (\OCP\DB\Exception $e) { $this->logger->error('Message insertion error in chattyllm task listener', ['exception' => $e]); } + + // store the conversation token and the actions if we are using the agency feature + if (class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentInteraction') + && $taskTypeId === \OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID) { + $session = $this->sessionMapper->getUserSession($task->getUserId(), $sessionId); + $conversationToken = ($task->getOutput()['conversation_token'] ?? null) ?: null; + $pendingActions = ($task->getOutput()['actions'] ?? null) ?: null; + $session->setAgencyConversationToken($conversationToken); + $session->setAgencyPendingActions($pendingActions); + $this->sessionMapper->update($session); + } } } } diff --git a/lib/Migration/Version020200Date20241218145833.php b/lib/Migration/Version020200Date20241218145833.php new file mode 100644 index 00000000..d8ee4d8a --- /dev/null +++ b/lib/Migration/Version020200Date20241218145833.php @@ -0,0 +1,67 @@ + + * + * @author Anupam Kumar + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +namespace OCA\Assistant\Migration; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +class Version020200Date20241218145833 extends SimpleMigrationStep { + + /** + * @param IOutput $output + * @param Closure(): ISchemaWrapper $schemaClosure + * @param array $options + * @return null|ISchemaWrapper + */ + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + $schemaChanged = false; + + if ($schema->hasTable('assistant_chat_sns')) { + $table = $schema->getTable('assistant_chat_sns'); + if (!$table->hasColumn('conversation_token')) { + $table->addColumn('agency_conversation_token', Types::TEXT, [ + 'notnull' => false, + 'default' => null, + ]); + $schemaChanged = true; + } + if (!$table->hasColumn('agency_pending_actions')) { + $table->addColumn('agency_pending_actions', Types::TEXT, [ + 'notnull' => false, + 'default' => null, + ]); + $schemaChanged = true; + } + } + + return $schemaChanged ? $schema : null; + } +} diff --git a/src/components/ChattyLLM/ChattyLLMInputForm.vue b/src/components/ChattyLLM/ChattyLLMInputForm.vue index ef193814..ac038cf9 100644 --- a/src/components/ChattyLLM/ChattyLLMInputForm.vue +++ b/src/components/ChattyLLM/ChattyLLMInputForm.vue @@ -653,6 +653,8 @@ export default { ).then(response => { clearInterval(this.pollMessageGenerationTimerId) if (sessionId === this.active.id) { + // TODO check that + this.session_agency_pending_actions = response.data.session_agency_pending_actions resolve(response.data) } else { console.debug('Ignoring received message for session ' + sessionId + ' that is not selected anymore') From c23fe1884771d3ddf4b403f21e4878fbdac71ebd Mon Sep 17 00:00:00 2001 From: Julien Veyssier Date: Thu, 19 Dec 2024 12:32:25 +0100 Subject: [PATCH 2/6] handle agency sensitive actions confirmation in the UI Signed-off-by: Julien Veyssier --- lib/Controller/ChattyLLMController.php | 26 +++++--- .../ChattyLLM/AgencyConfirmation.vue | 65 +++++++++++++++++++ .../ChattyLLM/ChattyLLMInputForm.vue | 62 +++++++++++++++--- src/components/ChattyLLM/ConversationBox.vue | 2 +- src/components/ChattyLLM/Message.vue | 4 +- 5 files changed, 138 insertions(+), 21 deletions(-) create mode 100644 src/components/ChattyLLM/AgencyConfirmation.vue diff --git a/lib/Controller/ChattyLLMController.php b/lib/Controller/ChattyLLMController.php index 390c79dd..dea7c6ee 100644 --- a/lib/Controller/ChattyLLMController.php +++ b/lib/Controller/ChattyLLMController.php @@ -181,7 +181,10 @@ public function newMessage(int $sessionId, string $role, string $content, int $t } $content = trim($content); - if (empty($content)) { + if (empty($content) + && (!class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentInteraction') + || !isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID])) + ) { return new JSONResponse(['error' => $this->l10n->t('Message content is empty')], Http::STATUS_BAD_REQUEST); } @@ -271,18 +274,14 @@ public function deleteMessage(int $messageId, int $sessionId): JSONResponse { * Schedule a task to generate a new message for the session * * @param integer $sessionId + * @param int $agencyConfirm * @return JSONResponse * @throws DoesNotExistException * @throws MultipleObjectsReturnedException - * @throws NotFoundException - * @throws PreConditionNotMetException - * @throws UnauthorizedException - * @throws ValidationException * @throws \OCP\DB\Exception - * @throws \OCP\TaskProcessing\Exception\Exception */ #[NoAdminRequired] - public function generateForSession(int $sessionId): JSONResponse { + public function generateForSession(int $sessionId, int $agencyConfirm = 0): JSONResponse { if ($this->userId === null) { return new JSONResponse(['error' => $this->l10n->t('User not logged in')], Http::STATUS_UNAUTHORIZED); } @@ -300,7 +299,7 @@ public function generateForSession(int $sessionId): JSONResponse { $session = $this->sessionMapper->getUserSession($this->userId, $sessionId); $lastConversationToken = $session->getAgencyConversationToken() ?? '{}'; try { - $taskId = $this->scheduleAgencyTask($prompt, 0, $lastConversationToken, $sessionId); + $taskId = $this->scheduleAgencyTask($prompt, $agencyConfirm, $lastConversationToken, $sessionId); } catch (\Exception $e) { return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST); } @@ -393,7 +392,10 @@ public function checkMessageGenerationTask(int $taskId, int $sessionId): JSONRes $message->setTimestamp(time()); $jsonMessage = $message->jsonSerialize(); $session = $this->sessionMapper->getUserSession($this->userId, $sessionId); - $jsonMessage['session_agency_pending_actions'] = $session->getAgencyPendingActions(); + $jsonMessage['sessionAgencyPendingActions'] = $session->getAgencyPendingActions(); + if ($jsonMessage['sessionAgencyPendingActions'] !== null) { + $jsonMessage['sessionAgencyPendingActions'] = json_decode($jsonMessage['sessionAgencyPendingActions']); + } // do not insert here, it is done by the listener return new JSONResponse($jsonMessage); } catch (\OCP\DB\Exception $e) { @@ -442,11 +444,15 @@ public function checkSession(int $sessionId): JSONResponse { return $task->getStatus() === Task::STATUS_RUNNING || $task->getStatus() === Task::STATUS_SCHEDULED; }); $session = $this->sessionMapper->getUserSession($this->userId, $sessionId); + $pendingActions = $session->getAgencyPendingActions(); + if ($pendingActions !== null) { + $pendingActions = json_decode($pendingActions); + } $responseData = [ 'messageTaskId' => null, 'titleTaskId' => null, 'sessionTitle' => $session->getTitle(), - 'session_agency_pending_actions' => $session->getAgencyPendingActions(), + 'sessionAgencyPendingActions' => $pendingActions, ]; if (!empty($messageTasks)) { $task = array_pop($messageTasks); diff --git a/src/components/ChattyLLM/AgencyConfirmation.vue b/src/components/ChattyLLM/AgencyConfirmation.vue new file mode 100644 index 00000000..8961e58b --- /dev/null +++ b/src/components/ChattyLLM/AgencyConfirmation.vue @@ -0,0 +1,65 @@ + + + + + diff --git a/src/components/ChattyLLM/ChattyLLMInputForm.vue b/src/components/ChattyLLM/ChattyLLMInputForm.vue index ac038cf9..edb46cd1 100644 --- a/src/components/ChattyLLM/ChattyLLMInputForm.vue +++ b/src/components/ChattyLLM/ChattyLLMInputForm.vue @@ -115,6 +115,11 @@ + session.id === sessionId) session.title = content } - await this.runGenerationTask(sessionId) + await this.runGenerationTask(sessionId, agencyConfirm) } catch (error) { this.loading.newHumanMessage = false console.error('newMessage error:', error) @@ -604,10 +622,16 @@ export default { } }, - async runGenerationTask(sessionId) { + async runGenerationTask(sessionId, agencyConfirm = null) { try { this.loading.llmGeneration = true - const response = await axios.get(getChatURL('/generate'), { params: { sessionId } }) + const params = { + sessionId, + } + if (agencyConfirm !== null) { + params.agencyConfirm = agencyConfirm ? 1 : 0 + } + const response = await axios.get(getChatURL('/generate'), { params }) console.debug('scheduleGenerationTask response:', response) const message = await this.pollGenerationTask(response.data.taskId, sessionId) console.debug('checkTaskPolling result:', message) @@ -653,8 +677,8 @@ export default { ).then(response => { clearInterval(this.pollMessageGenerationTimerId) if (sessionId === this.active.id) { - // TODO check that - this.session_agency_pending_actions = response.data.session_agency_pending_actions + this.active.sessionAgencyPendingActions = response.data.sessionAgencyPendingActions + this.active.agencyAnswered = false resolve(response.data) } else { console.debug('Ignoring received message for session ' + sessionId + ' that is not selected anymore') @@ -706,6 +730,22 @@ export default { }, 2000) }) }, + async onAgencyAnswer(confirm) { + this.active.agencyAnswered = true + // send accept/reject message + const role = Roles.HUMAN + const content = '' + const timestamp = +new Date() / 1000 | 0 + + if (this.active === null) { + await this.newSession() + } + + // this.messages.push({ role, content, timestamp }) + this.chatContent = '' + this.scrollToBottom() + await this.newMessage(role, content, timestamp, this.active.id, false, confirm) + }, }, } @@ -862,6 +902,10 @@ export default { padding-left: 1em; } + &__agency-confirmation { + margin-left: 1em; + } + &__input-area { position: sticky; bottom: 0; diff --git a/src/components/ChattyLLM/ConversationBox.vue b/src/components/ChattyLLM/ConversationBox.vue index 6c5bcc20..9d761e8a 100644 --- a/src/components/ChattyLLM/ConversationBox.vue +++ b/src/components/ChattyLLM/ConversationBox.vue @@ -1,6 +1,6 @@