From 7ab645a094976f03815c8d68cd589867c9a247e9 Mon Sep 17 00:00:00 2001 From: MB-Finski Date: Mon, 29 Jan 2024 09:32:27 +0000 Subject: [PATCH 01/31] Implement copywriter Signed-off-by: MB-Finski --- appinfo/info.xml | 1 - appinfo/routes.php | 4 +- composer.json | 12 +- lib/AppInfo/Application.php | 4 + lib/Controller/AssistantController.php | 88 ++++- lib/Controller/FreePromptController.php | 4 +- lib/Controller/SpeechToTextController.php | 47 +-- lib/Cron/CleanupTranscriptions.php | 45 --- lib/Db/SpeechToText/Transcript.php | 69 ---- lib/Db/SpeechToText/TranscriptMapper.php | 91 ----- lib/Db/Task.php | 108 +++++ lib/Db/TaskMapper.php | 213 ++++++++++ .../SpeechToTextResultListener.php | 60 ++- lib/Listener/TaskFailedListener.php | 17 +- lib/Listener/TaskSuccessfulListener.php | 17 +- .../Text2Image/Text2ImageResultListener.php | 14 +- .../Version010005Date20240115122933.php | 102 +++++ lib/Notification/Notifier.php | 32 +- lib/Service/AssistantService.php | 372 ++++++++++++++++-- lib/Service/FreePrompt/FreePromptService.php | 47 ++- .../SpeechToText/SpeechToTextService.php | 75 ++-- .../Text2Image/Text2ImageHelperService.php | 60 ++- src/assistant.js | 167 ++++---- src/components/AssistantFormInputs.vue | 214 ++++++++++ .../AssistantTextProcessingForm.vue | 60 ++- .../AssistantTextProcessingModal.vue | 16 +- src/speechToTextResultPage.js | 9 +- .../FreePromptCustomPickerElement.vue | 2 - src/views/PlainTextResultPage.vue | 17 +- src/views/TextProcessingTaskResultPage.vue | 138 +++++++ 30 files changed, 1575 insertions(+), 530 deletions(-) delete mode 100644 lib/Cron/CleanupTranscriptions.php delete mode 100644 lib/Db/SpeechToText/Transcript.php delete mode 100644 lib/Db/SpeechToText/TranscriptMapper.php create mode 100644 lib/Db/Task.php create mode 100644 lib/Db/TaskMapper.php create mode 100644 lib/Migration/Version010005Date20240115122933.php create mode 100644 src/components/AssistantFormInputs.vue create mode 100644 src/views/TextProcessingTaskResultPage.vue diff --git a/appinfo/info.xml b/appinfo/info.xml index 21e84814..f56b39d0 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -49,7 +49,6 @@ include text processing providers to: https://github.com/nextcloud/assistant/raw/main/img/screenshot3.jpg OCA\TpAssistant\Cron\CleanupImageGenerations - OCA\TpAssistant\Cron\CleanupTranscriptions OCA\TpAssistant\Command\CleanupImageGenerations diff --git a/appinfo/routes.php b/appinfo/routes.php index b84df620..fd7980d9 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -8,7 +8,10 @@ ['name' => 'assistant#getTextProcessingTaskResultPage', 'url' => '/t/{taskId}', 'verb' => 'GET'], ['name' => 'assistant#runTextProcessingTask', 'url' => '/run', 'verb' => 'POST'], + ['name' => 'assistant#scheduleTextProcessingTask', 'url' => '/schedule', 'verb' => 'POST'], ['name' => 'assistant#runOrScheduleTextProcessingTask', 'url' => '/run-or-schedule', 'verb' => 'POST'], + ['name' => 'assistant#getTextProcessingResult', 'url' => '/r/{taskId}', 'verb' => 'GET'], + ['name' => 'assistant#parseTextFromFile', 'url' => '/p', 'verb' => 'POST'], ['name' => 'Text2Image#processPrompt', 'url' => '/i/process_prompt', 'verb' => 'POST'], ['name' => 'Text2Image#getPromptHistory', 'url' => '/i/prompt_history', 'verb' => 'GET'], @@ -25,7 +28,6 @@ ['name' => 'FreePrompt#cancelGeneration', 'url' => '/f/cancel_generation', 'verb' => 'POST'], ['name' => 'SpeechToText#getResultPage', 'url' => '/stt/resultPage', 'verb' => 'GET'], - ['name' => 'SpeechToText#getTranscript', 'url' => '/stt/transcript', 'verb' => 'GET'], ['name' => 'SpeechToText#transcribeAudio', 'url' => '/stt/transcribeAudio', 'verb' => 'POST'], ['name' => 'SpeechToText#transcribeFile', 'url' => '/stt/transcribeFile', 'verb' => 'POST'], ], diff --git a/composer.json b/composer.json index d602b616..4d39d301 100644 --- a/composer.json +++ b/composer.json @@ -11,7 +11,9 @@ } ], "require": { - "php": "^8.0" + "php": "^8.0", + "erusev/parsedown": "^1.7", + "phpoffice/phpword": "^1.2" }, "scripts": { "lint": "find . -name \\*.php -not -path './vendor/*' -print0 | xargs -0 -n1 php -l", @@ -25,5 +27,13 @@ "psalm/phar": "^5.16", "nextcloud/ocp": "dev-master", "phpunit/phpunit": "^9.5" + }, + "config": { + "sort-packages": true, + "optimize-autoloader": true, + "platform": { + "php": "8.0" + }, + "autoloader-suffix": "TpAssistant" } } diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 636fedf0..d55db477 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -40,6 +40,10 @@ class Application extends App implements IBootstrap { public const IMAGE_FOLDER = 'generated_images'; public const SPEECH_TO_TEXT_REC_FOLDER = 'stt_recordings'; + public const STT_TASK_SCHEDULED = 0; + public const STT_TASK_SUCCESSFUL = 1; + public const STT_TASK_FAILED = -1; + public const TASK_TYPE_TEXT_GEN = 0; public const TASK_TYPE_TEXT_TO_IMAGE = 1; public const TASK_TYPE_SPEECH_TO_TEXT = 2; diff --git a/lib/Controller/AssistantController.php b/lib/Controller/AssistantController.php index 4162667d..baa5aed4 100644 --- a/lib/Controller/AssistantController.php +++ b/lib/Controller/AssistantController.php @@ -12,7 +12,9 @@ use OCP\AppFramework\Http\DataResponse; use OCP\AppFramework\Http\TemplateResponse; use OCP\AppFramework\Services\IInitialState; - +use OCA\TpAssistant\Db\TaskMapper; +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\AppFramework\Db\MultipleObjectsReturnedException; use OCP\IRequest; class AssistantController extends Controller { @@ -22,7 +24,8 @@ public function __construct( IRequest $request, private AssistantService $assistantService, private IInitialState $initialStateService, - private ?string $userId + private ?string $userId, + private TaskMapper $taskMapper, ) { parent::__construct($appName, $request); } @@ -33,9 +36,10 @@ public function __construct( */ #[NoAdminRequired] #[NoCSRFRequired] - #[BruteForceProtection(action: 'taskResultPage')] + #[BruteForceProtection(action: 'taskResults')] public function getTextProcessingTaskResultPage(int $taskId): TemplateResponse { $task = $this->assistantService->getTextProcessingTask($this->userId, $taskId); + if ($task === null) { $response = new TemplateResponse( '', @@ -47,45 +51,105 @@ public function getTextProcessingTaskResultPage(int $taskId): TemplateResponse { $response->throttle(['userId' => $this->userId, 'taskId' => $taskId]); return $response; } - $this->initialStateService->provideInitialState('task', $task->jsonSerialize()); + $this->initialStateService->provideInitialState('task', $task->jsonSerializeCc()); return new TemplateResponse(Application::APP_ID, 'taskResultPage'); } /** - * @param string $input + * @param int $taskId + * @return DataResponse + */ + #[NoAdminRequired] + #[NoCSRFRequired] + #[BruteForceProtection(action: 'taskResults')] + public function getTextProcessingResult(int $taskId): DataResponse { + $task = $this->assistantService->getTextProcessingTask($this->userId, $taskId); + + if ($task === null) { + $response = new DataResponse( + '', + Http::STATUS_NOT_FOUND + ); + $response->throttle(['userId' => $this->userId, 'taskId' => $taskId]); + return $response; + } + return new DataResponse([ + 'task' => $task->jsonSerializeCc(), + ]); + } + + /** + * @param array $inputs + * @param string $type + * @param string $appId + * @param string $identifier + * @return DataResponse + */ + #[NoAdminRequired] + public function runTextProcessingTask(string $type, array $inputs, string $appId, string $identifier): DataResponse { + try { + $task = $this->assistantService->runTextProcessingTask($type, $inputs, $appId, $this->userId, $identifier); + } catch (\Exception | \Throwable $e) { + return new DataResponse($e->getMessage(), Http::STATUS_BAD_REQUEST); + } + return new DataResponse([ + 'task' => $task->jsonSerializeCc(), + ]); + } + + /** + * @param array $inputs * @param string $type * @param string $appId * @param string $identifier * @return DataResponse */ #[NoAdminRequired] - public function runTextProcessingTask(string $type, string $input, string $appId, string $identifier): DataResponse { + public function scheduleTextProcessingTask(string $type, array $inputs, string $appId, string $identifier): DataResponse { try { - $task = $this->assistantService->runTextProcessingTask($type, $input, $appId, $this->userId, $identifier); + $task = $this->assistantService->scheduleTextProcessingTask($type, $inputs, $appId, $this->userId, $identifier); } catch (\Exception | \Throwable $e) { return new DataResponse($e->getMessage(), Http::STATUS_BAD_REQUEST); } return new DataResponse([ - 'task' => $task->jsonSerialize(), + 'task' => $task->jsonSerializeCc(), ]); } /** - * @param string $input + * @param array $inputs * @param string $type * @param string $appId * @param string $identifier * @return DataResponse */ #[NoAdminRequired] - public function runOrScheduleTextProcessingTask(string $type, string $input, string $appId, string $identifier): DataResponse { + public function runOrScheduleTextProcessingTask(string $type, array $inputs, string $appId, string $identifier): DataResponse { + try { + $task = $this->assistantService->runOrScheduleTextProcessingTask($type, $inputs, $appId, $this->userId, $identifier); + } catch (\Exception | \Throwable $e) { + return new DataResponse($e->getMessage(), Http::STATUS_BAD_REQUEST); + } + return new DataResponse([ + 'task' => $task->jsonSerializeCc(), + ]); + } + + /** + * Parse text from file (if parsing the file type is supported) + * + * @param string $filePath + * @return DataResponse + */ + #[NoAdminRequired] + public function parseTextFromFile(string $filePath): DataResponse { try { - $task = $this->assistantService->runOrScheduleTextProcessingTask($type, $input, $appId, $this->userId, $identifier); + $text = $this->assistantService->parseTextFromFile($filePath); } catch (\Exception | \Throwable $e) { return new DataResponse($e->getMessage(), Http::STATUS_BAD_REQUEST); } return new DataResponse([ - 'task' => $task->jsonSerialize(), + 'parsedText' => $text, ]); } } diff --git a/lib/Controller/FreePromptController.php b/lib/Controller/FreePromptController.php index ee0305e6..5dfda902 100644 --- a/lib/Controller/FreePromptController.php +++ b/lib/Controller/FreePromptController.php @@ -33,9 +33,9 @@ public function __construct( */ #[NoAdminRequired] #[NoCSRFRequired] - public function processPrompt(string $prompt, int $nResults = 1): DataResponse { + public function processPrompt(string $prompt): DataResponse { try { - $result = $this->freePromptService->processPrompt($prompt, $nResults); + $result = $this->freePromptService->processPrompt($prompt); } catch (Exception $e) { return new DataResponse(['error' => $e->getMessage()], (int)$e->getCode()); } diff --git a/lib/Controller/SpeechToTextController.php b/lib/Controller/SpeechToTextController.php index 3731089b..03e8b521 100644 --- a/lib/Controller/SpeechToTextController.php +++ b/lib/Controller/SpeechToTextController.php @@ -26,16 +26,15 @@ use Exception; use InvalidArgumentException; use OCA\TpAssistant\AppInfo\Application; -use OCA\TpAssistant\Db\SpeechToText\TranscriptMapper; use OCA\TpAssistant\Service\SpeechToText\SpeechToTextService; +use OCA\TpAssistant\Db\TaskMapper; +use OCA\TpAssistant\Db\Task; use OCP\AppFramework\Controller; use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\MultipleObjectsReturnedException; use OCP\AppFramework\Http; -use OCP\AppFramework\Http\Attribute\AnonRateLimit; use OCP\AppFramework\Http\Attribute\NoAdminRequired; use OCP\AppFramework\Http\Attribute\NoCSRFRequired; -use OCP\AppFramework\Http\Attribute\UserRateLimit; use OCP\AppFramework\Http\DataResponse; use OCP\AppFramework\Http\TemplateResponse; use OCP\AppFramework\Services\IInitialState; @@ -55,9 +54,9 @@ public function __construct( private SpeechToTextService $service, private LoggerInterface $logger, private IL10N $l10n, - private TranscriptMapper $transcriptMapper, private IInitialState $initialState, private ?string $userId, + private TaskMapper $taskMapper, ) { parent::__construct($appName, $request); } @@ -68,19 +67,16 @@ public function __construct( */ #[NoAdminRequired] #[NoCSRFRequired] - #[UserRateLimit(limit: 10, period: 60)] - #[AnonRateLimit(limit: 2, period: 60)] public function getResultPage(int $id): TemplateResponse { $response = new TemplateResponse(Application::APP_ID, 'speechToTextResultPage'); try { $initData = [ - 'status' => 'success', - 'result' => $this->internalGetTranscript($id), - 'taskType' => Application::TASK_TYPE_SPEECH_TO_TEXT, + 'task' => $this->internalGetTask($id), ]; } catch (Exception $e) { $initData = [ 'status' => 'failure', + 'task' => null, 'message' => $e->getMessage(), ]; $response->setStatus(intval($e->getCode())); @@ -96,39 +92,30 @@ public function getResultPage(int $id): TemplateResponse { #[NoAdminRequired] public function getTranscript(int $id): DataResponse { try { - return new DataResponse($this->internalGetTranscript($id)); + return new DataResponse($this->internalGetTask($id)->getOutput()); } catch (Exception $e) { return new DataResponse($e->getMessage(), intval($e->getCode())); } } /** - * Internal function to get transcript and throw a common exception + * Internal function to get transcription assistant tasks based on the assistant meta task id * * @param integer $id - * @return string + * @return Task */ - private function internalGetTranscript(int $id): string { + private function internalGetTask(int $id): Task { try { - $transcriptEntity = $this->transcriptMapper->find($id, $this->userId); - $transcript = $transcriptEntity->getTranscript(); + $task = $this->taskMapper->getTaskOfUser($id, $this->userId); + + if($task->getModality() !== Application::TASK_TYPE_SPEECH_TO_TEXT) { + throw new Exception('Task is not a speech to text task.', Http::STATUS_BAD_REQUEST); + } - $transcriptEntity->setLastAccessed(new DateTime()); - $this->transcriptMapper->update($transcriptEntity); - - return trim($transcript); - } catch (InvalidArgumentException $e) { - $this->logger->error( - 'Invalid argument in transcript access time update call: ' . $e->getMessage(), - ['app' => Application::APP_ID], - ); - throw new Exception( - $this->l10n->t('Error in transcript access time update call'), - Http::STATUS_INTERNAL_SERVER_ERROR, - ); + return $task; } catch (MultipleObjectsReturnedException $e) { - $this->logger->error('Multiple transcripts found: ' . $e->getMessage(), ['app' => Application::APP_ID]); - throw new Exception($this->l10n->t('Multiple transcripts found'), Http::STATUS_BAD_REQUEST); + $this->logger->error('Multiple tasks found for one id: ' . $e->getMessage(), ['app' => Application::APP_ID]); + throw new Exception($this->l10n->t('Multiple tasks found'), Http::STATUS_BAD_REQUEST); } catch (DoesNotExistException $e) { throw new Exception($this->l10n->t('Transcript not found'), Http::STATUS_NOT_FOUND); } catch (Exception $e) { diff --git a/lib/Cron/CleanupTranscriptions.php b/lib/Cron/CleanupTranscriptions.php deleted file mode 100644 index 5569d9fe..00000000 --- a/lib/Cron/CleanupTranscriptions.php +++ /dev/null @@ -1,45 +0,0 @@ - - * - * @author Anupam Kumar - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - */ - -namespace OCA\TpAssistant\Cron; - -use OCA\TpAssistant\Db\SpeechToText\TranscriptMapper; -use OCP\AppFramework\Utility\ITimeFactory; -use OCP\BackgroundJob\TimedJob; - -class CleanupTranscriptions extends TimedJob { - public function __construct( - ITimeFactory $time, - private TranscriptMapper $transcriptMapper, - ) { - parent::__construct($time); - $this->setInterval(60 * 60 * 24); // 24 hours - } - - protected function run($argument) { - $this->transcriptMapper->cleanupTranscriptions(); - } -} diff --git a/lib/Db/SpeechToText/Transcript.php b/lib/Db/SpeechToText/Transcript.php deleted file mode 100644 index 8d52a317..00000000 --- a/lib/Db/SpeechToText/Transcript.php +++ /dev/null @@ -1,69 +0,0 @@ - - * - * @author Anupam Kumar - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - */ - -namespace OCA\TpAssistant\Db\SpeechToText; - -use OCP\AppFramework\Db\Entity; -use OCP\DB\Types; - -/** - * Class Transcript - * - * @package OCA\Stt\Db - * @method ?string getUserId() - * @method void setUserId(?string $userId) - * @method string getTranscript() - * @method void setTranscript(string $transcript) - * @method \DateTime getLastAccessed() - * @method void setLastAccessed(\DateTime $lastAccessed) - */ -class Transcript extends Entity { - - protected $userId; - protected $transcript; - protected $lastAccessed; - - public static $columns = [ - 'id', - 'user_id', - 'transcript', - 'last_accessed', - ]; - public static $fields = [ - 'id', - 'userId', - 'transcript', - 'lastAccessed', - ]; - - public function __construct() { - $this->addType('id', Types::INTEGER); - $this->addType('userId', Types::STRING); - $this->addType('transcript', Types::STRING); - $this->addType('lastAccessed', Types::DATETIME); - } -} diff --git a/lib/Db/SpeechToText/TranscriptMapper.php b/lib/Db/SpeechToText/TranscriptMapper.php deleted file mode 100644 index 53691c55..00000000 --- a/lib/Db/SpeechToText/TranscriptMapper.php +++ /dev/null @@ -1,91 +0,0 @@ - - * - * @author Anupam Kumar - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - */ - -namespace OCA\TpAssistant\Db\SpeechToText; - -use DateTime; -use Exception; -use OCP\AppFramework\Db\DoesNotExistException; -use OCP\AppFramework\Db\MultipleObjectsReturnedException; -use OCP\AppFramework\Db\QBMapper; -use OCP\DB\QueryBuilder\IQueryBuilder; -use OCP\IDBConnection; -use Psr\Log\LoggerInterface; - -/** - * @template-extends QBMapper - */ -class TranscriptMapper extends QBMapper { - - public function __construct(IDBConnection $db, private LoggerInterface $logger) { - parent::__construct($db, 'assistant_stt_transcripts', Transcript::class); - $this->db = $db; - } - - /** - * @param integer $id - * @param string|null $userId - * @throws Exception - * @throws MultipleObjectsReturnedException if more than one item exist - * @throws DoesNotExistException if the item does not exist - * @return Transcript - */ - public function find(int $id, ?string $userId): Transcript { - $qb = $this->db->getQueryBuilder(); - - if (strlen($userId) > 0 && $userId !== 'admin') { - $qb - ->select(Transcript::$columns) - ->from($this->getTableName()) - ->where($qb->expr()->eq('id', $qb->createNamedParameter($id))) - ->andWhere($qb->expr()->eq('user_id', $qb->createNamedParameter($userId))) - ; - } else { - $qb - ->select(Transcript::$columns) - ->from($this->getTableName()) - ->where($qb->expr()->eq('id', $qb->createNamedParameter($id))) - ; - } - - return $this->findEntity($qb); - } - - public function cleanupTranscriptions(): void { - $qb = $this->db->getQueryBuilder(); - $qb - ->delete($this->getTableName()) - ->where($qb->expr()->lte( - 'last_accessed', - $qb->createNamedParameter(new DateTime('-2 weeks'), IQueryBuilder::PARAM_DATE) - )) - ; - - $deletedRows = $qb->executeStatement(); - $this->logger->debug('Cleared {count} old transcriptions', ['count' => $deletedRows]); - } -} diff --git a/lib/Db/Task.php b/lib/Db/Task.php new file mode 100644 index 00000000..044aa6ad --- /dev/null +++ b/lib/Db/Task.php @@ -0,0 +1,108 @@ + +// SPDX-License-Identifier: AGPL-3.0-or-later + +namespace OCA\TpAssistant\Db; + +use OCP\AppFramework\Db\Entity; + +/** + * @method string getUserId() + * @method void setUserId(string $userId) + * @method string getOutput() + * @method void setOutput(string $value) + * @method string getAppId() + * @method void setAppId(string $appId) + * @method int getOcpTaskId() + * @method void setOcpTaskId(int $value) + * @method int getTimestamp() + * @method void setTimestamp(int $timestamp) + * @method string getTaskType() + * @method void setTaskType(string $taskType) + * @method void setStatus(int $status) + * @method int getStatus() + * @method void setModality(int $modality) + * @method int getModality() + * @method string getInputs() + * @method void setInputs(string $inputs) + * @method string getIndentifer() + * @method void setIndentifer(string $indentifer) + */ +class Task extends Entity implements \JsonSerializable { + /** @var string */ + protected $userId; + /** @var string */ + protected $inputs; + /** @var string */ + protected $output; + /** @var string */ + protected $appId; + /** @var int */ + protected $ocpTaskId; + /** @var int */ + protected $timestamp; + /** @var string */ + protected $taskType; + /** @var int */ + protected $status; + /** @var int */ + protected $modality; + /** @var string */ + protected $indentifer; + + public function __construct() { + $this->addType('user_id', 'string'); + $this->addType('inputs', 'string'); + $this->addType('output', 'string'); + $this->addType('app_id', 'string'); + $this->addType('ocp_task_id', 'integer'); + $this->addType('timestamp', 'integer'); + $this->addType('task_type', 'string'); + $this->addType('status', 'integer'); + $this->addType('modality', 'integer'); + $this->addType('indentifer', 'string'); + } + + #[\ReturnTypeWillChange] + public function jsonSerialize() { + return [ + 'id' => $this->id, + 'user_id' => $this->userId, + 'inputs' => $this->getInputsAsArray(), + 'output' => $this->output, + 'app_id' => $this->appId, + 'ocp_task_id' => $this->ocpTaskId, + 'task_type' => $this->taskType, + 'timestamp' => $this->timestamp, + 'status' => $this->status, + 'modality' => $this->modality, + 'indentifer' => $this->indentifer, + ]; + } + + #[\ReturnTypeWillChange] + public function jsonSerializeCc() { + return [ + 'id' => $this->id, + 'userId' => $this->userId, + 'inputs' => $this->getInputsAsArray(), + 'output' => $this->output, + 'appId' => $this->appId, + 'ocpTaskId' => $this->ocpTaskId, + 'taskType' => $this->taskType, + 'timestamp' => $this->timestamp, + 'status' => $this->status, + 'modality' => $this->modality, + 'indentifer' => $this->indentifer, + ]; + } + + /** + * @return array + */ + public function getInputsAsArray(): array { + return json_decode($this->inputs, true) ?? []; + } +} diff --git a/lib/Db/TaskMapper.php b/lib/Db/TaskMapper.php new file mode 100644 index 00000000..33a483df --- /dev/null +++ b/lib/Db/TaskMapper.php @@ -0,0 +1,213 @@ + +// SPDX-License-Identifier: AGPL-3.0-or-later + +declare(strict_types=1); + +namespace OCA\TpAssistant\Db; + +use DateTime; +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\AppFramework\Db\MultipleObjectsReturnedException; +use OCP\AppFramework\Db\QBMapper; +use OCP\DB\Exception; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; +use OCA\TpAssistant\AppInfo\Application; + +/** + * @extends QBMapper + */ +class TaskMapper extends QBMapper { + public function __construct(IDBConnection $db) { + parent::__construct($db, 'assistant_text_tasks', Task::class); + } + + /** + * @param int $id + * @param int $taskType + * @return Task + * @throws DoesNotExistException + * @throws Exception + * @throws MultipleObjectsReturnedException + */ + public function getTask(int $id,): Task { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from($this->getTableName()) + ->where( + $qb->expr()->eq('id', $qb->createNamedParameter($id, IQueryBuilder::PARAM_INT)) + ); + + /** @var Task $retVal */ + $retVal = $this->findEntity($qb); + return $retVal; + } + + /** + * @param int $id + * @param int $modality + * @return Task + * @throws DoesNotExistException + * @throws Exception + * @throws MultipleObjectsReturnedException + */ + public function getTaskByOcpTaskIdAndModality(int $ocpTaskId, int $modality): Task { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from($this->getTableName()) + ->where( + $qb->expr()->eq('ocp_task_id', $qb->createNamedParameter($ocpTaskId, IQueryBuilder::PARAM_INT)) + ) + ->andWhere( + $qb->expr()->eq('modality', $qb->createNamedParameter($modality, IQueryBuilder::PARAM_INT)) + ); + + /** @var Task $retVal */ + $retVal = $this->findEntity($qb); + return $retVal; + } + + /** + * @param int $id + * @param int $modality + * @return array + * @throws Exception + */ + public function getTasksByOcpTaskIdAndModality(int $ocpTaskId, int $modality): array { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from($this->getTableName()) + ->where( + $qb->expr()->eq('ocp_task_id', $qb->createNamedParameter($ocpTaskId, IQueryBuilder::PARAM_INT)) + ) + ->andWhere( + $qb->expr()->eq('modality', $qb->createNamedParameter($modality, IQueryBuilder::PARAM_INT)) + ); + + /** @var array $retVal */ + $retVal = $this->findEntities($qb); + return $retVal; + } + + /** + * @param int $id + * @param string $userId + * @return Task + * @throws DoesNotExistException + * @throws Exception + * @throws MultipleObjectsReturnedException + */ + public function getTaskOfUser(int $id, string $userId): Task { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from($this->getTableName()) + ->where( + $qb->expr()->eq('id', $qb->createNamedParameter($id, IQueryBuilder::PARAM_INT)) + ) + ->andWhere( + $qb->expr()->eq('user_id', $qb->createNamedParameter($userId, IQueryBuilder::PARAM_STR)) + ); + /** @var Task $retVal */ + $retVal = $this->findEntity($qb); + return $retVal; + } + + /** + * @param string $userId + * @return array + * @throws Exception + */ + public function getTasksOfUser(string $userId): array { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from($this->getTableName()) + ->where( + $qb->expr()->eq('user_id', $qb->createNamedParameter($userId, IQueryBuilder::PARAM_STR)) + ); + + return $this->findEntities($qb); + } + + /** + * @param string $userId + * @param array $inputs + * @param string|null $output + * @param int|null $timestamp + * @param int|null $ocpTaskId + * @param string|null $taskType + * @param string|null $appId + * @param int $status + * @param int $modality + * @param string $identifier + * @return Task + * @throws Exception + */ + public function createTask( + string $userId, + array $inputs, + ?string $output, + ?int $timestamp = null, + ?int $ocpTaskId = null, + ?string $taskType = null, + ?string $appId = null, + int $status = 0, + int $modality= 0, + string $identifier = ''): Task { + if ($timestamp === null) { + $timestamp = (new DateTime())->getTimestamp(); + } + + $task = new Task(); + $task->setUserId($userId); + $task->setInputs(json_encode($inputs)); + $task->setTimestamp($timestamp); + $task->setOutput($output); + $task->setOcpTaskId($ocpTaskId); + $task->setTaskType($taskType); + $task->setAppId($appId); + $task->setStatus($status); + $task->setModality($modality); + $task->setIndentifer($identifier); + /** @var Task $insertedTask */ + $insertedTask = $this->insert($task); + + return $insertedTask; + } + + /** + * @param string $userId + * @return void + * @throws Exception + */ + public function deleteUserTasks(string $userId): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->getTableName()) + ->where( + $qb->expr()->eq('user_id', $qb->createNamedParameter($userId, IQueryBuilder::PARAM_STR)) + ); + $qb->executeStatement(); + $qb->resetQueryParts(); + } + + /** + * Clean up tasks older than 14 days + * @return int number of deleted rows + * @throws Exception + * @throws \RuntimeException + */ + public function cleanupOldTasks(): int { + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->getTableName()) + ->where( + $qb->expr()->lt('timestamp', $qb->createNamedParameter(time() - 14 * 24 * 60 * 60, IQueryBuilder::PARAM_INT)) + ); + return $qb->executeStatement(); + } +} diff --git a/lib/Listener/SpeechToText/SpeechToTextResultListener.php b/lib/Listener/SpeechToText/SpeechToTextResultListener.php index f38457d2..fd2edac6 100644 --- a/lib/Listener/SpeechToText/SpeechToTextResultListener.php +++ b/lib/Listener/SpeechToText/SpeechToTextResultListener.php @@ -24,12 +24,15 @@ use OCA\TpAssistant\AppInfo\Application; use OCA\TpAssistant\Service\SpeechToText\SpeechToTextService; +use OCA\TpAssistant\Service\AssistantService; use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventListener; use OCP\SpeechToText\Events\AbstractTranscriptionEvent; use OCP\SpeechToText\Events\TranscriptionFailedEvent; use OCP\SpeechToText\Events\TranscriptionSuccessfulEvent; +use OCA\TpAssistant\Db\TaskMapper; use Psr\Log\LoggerInterface; +use OCP\IURLGenerator; /** * @template-implements IEventListener @@ -38,6 +41,9 @@ class SpeechToTextResultListener implements IEventListener { public function __construct( private SpeechToTextService $sttService, private LoggerInterface $logger, + private TaskMapper $taskMapper, + private AssistantService $assistantService, + private IURLGenerator $urlGenerator, ) { } @@ -49,20 +55,68 @@ public function handle(Event $event): void { if ($event instanceof TranscriptionSuccessfulEvent) { $transcript = $event->getTranscript(); $userId = $event->getUserId(); + $file = $event->getFile(); + $tasks = $this->taskMapper->getTasksByOcpTaskIdAndModality($file->getId(), Application::TASK_TYPE_SPEECH_TO_TEXT); + + // Find a matching etag: + $etag = $file->getEtag(); + $assistantTask = null; + foreach ($tasks as $task) { + $taskEtag = $task->getInputsAsArray()['eTag']; + if ($taskEtag === $etag) { + $assistantTask = $task; + break; + } + } + + if ($assistantTask === null) { + $this->logger->error('No assistant task found for speech to text result out of ' . count($tasks) . ' tasks for file ' . $file->getId() . ' with etag ' . $etag); + return; + } + + // Update the meta task with the output and new status + $assistantTask->setOutput($transcript); + $assistantTask->setStatus(Application::STT_TASK_SUCCESSFUL); + $assistantTask = $this->taskMapper->update($assistantTask); + + // Generate the link to the result page: + $link = $this->urlGenerator->linkToRouteAbsolute(Application::APP_ID . '.SpeechToText.getResultPage', ['id' => $task->getId()]); + $this->logger->error('Generated link to result page: ' . $link); try { - $this->sttService->sendSpeechToTextNotification($userId, $transcript, true); + $this->assistantService->sendNotification($assistantTask, $link, null, $transcript); } catch (\InvalidArgumentException $e) { $this->logger->error('Failed to dispatch notification for successful transcription: ' . $e->getMessage()); } } if ($event instanceof TranscriptionFailedEvent) { - $userId = $event->getUserId(); $this->logger->error('Transcript generation failed: ' . $event->getErrorMessage()); + + $userId = $event->getUserId(); + $tasks = $this->taskMapper->getTasksByOcpTaskIdAndModality($file->getId(), Application::TASK_TYPE_SPEECH_TO_TEXT); + + // Find a matching etag: + $etag = $file->getEtag(); + $assistantTask = null; + foreach ($tasks as $task) { + if ($task->getEtag() === $etag) { + $assistantTask = $task; + break; + } + } + + if ($assistantTask === null) { + $this->logger->error('No assistant task found for speech to text result'); + return; + } + + // Update the meta task with the new status + $assistantTask->setStatus(Application::STT_TASK_FAILED); + $assistantTask = $this->taskMapper->update($assistantTask); try { - $this->sttService->sendSpeechToTextNotification($userId, '', false); + $this->assistantService->sendNotification($assistantTask); } catch (\InvalidArgumentException $e) { $this->logger->error('Failed to dispatch notification for failed transcription: ' . $e->getMessage()); } diff --git a/lib/Listener/TaskFailedListener.php b/lib/Listener/TaskFailedListener.php index 5eb10980..564807b5 100644 --- a/lib/Listener/TaskFailedListener.php +++ b/lib/Listener/TaskFailedListener.php @@ -9,6 +9,8 @@ use OCP\EventDispatcher\IEventDispatcher; use OCP\EventDispatcher\IEventListener; use OCP\TextProcessing\Events\TaskFailedEvent; +use OCP\AppFramework\Db\DoesNotExistException; +use OCA\TpAssistant\Db\TaskMapper; /** * @template-implements IEventListener @@ -18,6 +20,7 @@ class TaskFailedListener implements IEventListener { public function __construct( private AssistantService $assistantService, private IEventDispatcher $eventDispatcher, + private TaskMapper $taskMapper, ) { } @@ -47,6 +50,18 @@ public function handle(Event $event): void { $notificationActionLabel = $beforeAssistantNotificationEvent->getNotificationActionLabel(); } - $this->assistantService->sendNotification($task, $notificationTarget, $notificationActionLabel); + try { + $assistantTask = $this->taskMapper->getTaskByOcpTaskIdAndModality($task->getId(), Application::TASK_TYPE_TEXT_GEN); + } catch (DoesNotExistException $e) { + // Not an assistant task + return; + } + + // Update task status and output: + $assistantTask->setStatus($task->getStatus()); + $assistantTask->setOutput($task->getOutput()); + $assistantTask = $this->taskMapper->update($assistantTask); + + $this->assistantService->sendNotification($assistantTask, $notificationTarget, $notificationActionLabel); } } diff --git a/lib/Listener/TaskSuccessfulListener.php b/lib/Listener/TaskSuccessfulListener.php index ecbe1203..7eda571f 100644 --- a/lib/Listener/TaskSuccessfulListener.php +++ b/lib/Listener/TaskSuccessfulListener.php @@ -9,6 +9,8 @@ use OCP\EventDispatcher\IEventDispatcher; use OCP\EventDispatcher\IEventListener; use OCP\TextProcessing\Events\TaskSuccessfulEvent; +use OCA\TpAssistant\Db\TaskMapper; +use OCP\AppFramework\Db\DoesNotExistException; /** * @template-implements IEventListener @@ -18,6 +20,7 @@ class TaskSuccessfulListener implements IEventListener { public function __construct( private AssistantService $assistantService, private IEventDispatcher $eventDispatcher, + private TaskMapper $taskMapper, ) { } @@ -47,6 +50,18 @@ public function handle(Event $event): void { $notificationActionLabel = $beforeAssistantNotificationEvent->getNotificationActionLabel(); } - $this->assistantService->sendNotification($task, $notificationTarget, $notificationActionLabel); + try { + $assistantTask = $this->taskMapper->getTaskByOcpTaskIdAndModality($task->getId(), Application::TASK_TYPE_TEXT_GEN); + } catch (DoesNotExistException $e) { + // Not an assistant task + return; + } + + // Update task status and output: + $assistantTask->setStatus($task->getStatus()); + $assistantTask->setOutput($task->getOutput()); + $assistantTask = $this->taskMapper->update($assistantTask); + + $this->assistantService->sendNotification($assistantTask, $notificationTarget, $notificationActionLabel); } } diff --git a/lib/Listener/Text2Image/Text2ImageResultListener.php b/lib/Listener/Text2Image/Text2ImageResultListener.php index b831a433..323b50a1 100644 --- a/lib/Listener/Text2Image/Text2ImageResultListener.php +++ b/lib/Listener/Text2Image/Text2ImageResultListener.php @@ -16,6 +16,8 @@ use OCP\TextToImage\Events\AbstractTextToImageEvent; use OCP\TextToImage\Events\TaskFailedEvent; use OCP\TextToImage\Events\TaskSuccessfulEvent; +use OCP\TextToImage\Task; +use OCA\TpAssistant\Db\TaskMapper; use Psr\Log\LoggerInterface; /** @@ -29,6 +31,7 @@ public function __construct( private LoggerInterface $logger, private AssistantService $assistantService, private IURLGenerator $urlGenerator, + private TaskMapper $taskMapper, ) { } @@ -49,6 +52,7 @@ public function handle(Event $event): void { return; } + $assistantTask = $this->taskMapper->getTaskByOcpTaskIdAndModality($event->getTask()->getId(), Application::TASK_TYPE_TEXT_TO_IMAGE); $link = null; // A link to the image generation page (if the task succeeded) if ($event instanceof TaskSuccessfulEvent) { @@ -59,6 +63,8 @@ public function handle(Event $event): void { $this->text2ImageService->storeImages($images, $imageGenId); + $assistantTask->setStatus(Task::STATUS_SUCCESSFUL); + $assistantTask = $this->taskMapper->update($assistantTask); // Generate the link for the notification $link = $this->urlGenerator->linkToRouteAbsolute( Application::APP_ID . '.Text2Image.showGenerationPage', @@ -71,8 +77,12 @@ public function handle(Event $event): void { if ($event instanceof TaskFailedEvent) { $this->logger->warning('Image generation task failed: ' . $imageGenId); $this->imageGenerationMapper->setFailed($imageGenId, true); + + // Update the assistant meta task status: + $assistantTask->setStatus(Task::STATUS_FAILED); + $assistantTask = $this->taskMapper->update($assistantTask); - $this->assistantService->sendNotification($event->getTask()); + $this->assistantService->sendNotification($assistantTask); } // Only send the notification if the user enabled them for this task: @@ -80,7 +90,7 @@ public function handle(Event $event): void { /** @var ImageGeneration $imageGeneration */ $imageGeneration = $this->imageGenerationMapper->getImageGenerationOfImageGenId($imageGenId); if ($imageGeneration->getNotifyReady()) { - $this->assistantService->sendNotification($event->getTask(), $link); + $this->assistantService->sendNotification($assistantTask, $link); } } catch (\OCP\Db\Exception | DoesNotExistException | MultipleObjectsReturnedException $e) { $this->logger->warning('Could not notify user of a generation (id:' . $imageGenId . ') being ready: ' . $e->getMessage()); diff --git a/lib/Migration/Version010005Date20240115122933.php b/lib/Migration/Version010005Date20240115122933.php new file mode 100644 index 00000000..637bc820 --- /dev/null +++ b/lib/Migration/Version010005Date20240115122933.php @@ -0,0 +1,102 @@ + +// SPDX-License-Identifier: AGPL-3.0-or-later + +declare(strict_types=1); + +namespace OCA\TpAssistant\Migration; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +class Version010005Date20240115122933 extends SimpleMigrationStep { + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * @return void + */ + public function preSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void { + } + + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * @return null|ISchemaWrapper + */ + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options) { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + $schemaChanged = false; + + if ($schema->hasTable('assistant_stt_transcripts')) { + // Storing transcripts has been moved to the assistant meta task wrapper + $schemaChanged = true; + $table = $schema->getTable('assistant_stt_transcripts'); + $table->dropIndex('assistant_stt_transcript_user'); + $table->dropIndex('assistant_stt_transcript_la'); + $schema->dropTable('assistant_stt_transcripts'); + } + + if (!$schema->hasTable('assistant_text_tasks')) { + $schemaChanged = true; + $table = $schema->createTable('assistant_text_tasks'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + ]); + $table->addColumn('user_id', Types::STRING, [ + 'notnull' => true, + 'length' => 64, + ]); + $table->addColumn('app_id', Types::STRING, [ + 'notnull' => true, + ]); + $table->addColumn('inputs', Types::TEXT, [ + 'notnull' => false, + ]); + $table->addColumn('output', Types::TEXT, [ + 'notnull' => false, + ]); + $table->addColumn('ocp_task_id', Types::BIGINT, [ + 'notnull' => false, + ]); + $table->addColumn('timestamp', Types::BIGINT, [ + 'notnull' => true, + 'unsigned' => true, + ]); + $table->addColumn('task_type', Types::STRING, [ + 'notnull' => false, + ]); + $table->addColumn('status', Types::INTEGER, [ + 'notnull' => true, + 'default' => 0, // 0 = Unknown + ]); + $table->addColumn('modality', Types::INTEGER, [ + 'notnull' => false, + ]); + $table->addColumn('indentifer', Types::STRING, [ + 'notnull' => false, + ]); + $table->setPrimaryKey(['id']); + $table->addIndex(['user_id'], 'assistant_t_tasks_uid'); + $table->addIndex(['ocp_task_id','modality'], 'assistant_t_task_id_modality'); + } + + return $schemaChanged ? $schema : null; + } + + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * @return void + */ + public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void { + } +} \ No newline at end of file diff --git a/lib/Notification/Notifier.php b/lib/Notification/Notifier.php index 04006341..ec25648d 100644 --- a/lib/Notification/Notifier.php +++ b/lib/Notification/Notifier.php @@ -70,18 +70,25 @@ public function prepare(INotification $notification, string $languageCode): INot $schedulingAppName = $schedulingAppInfo['name']; $taskTypeName = null; - if ($params['taskType'] === Application::TASK_TYPE_TEXT_GEN && - isset($params['taskTypeClass']) && $params['taskTypeClass']) { - try { - /** @var ITaskType $taskType */ - $taskType = $this->container->get($params['taskTypeClass']); - $taskTypeName = $taskType->getName(); - } catch (\Exception | \Throwable $e) { - $this->logger->debug('Impossible to get task type ' . $params['taskTypeClass'], ['exception' => $e]); + $taskInput = $params['inputs']['prompt'] ?? null; + if ($params['taskModality'] === Application::TASK_TYPE_TEXT_GEN) { + + if ($params['taskTypeClass'] === 'copywriter') { + // Catch the custom copywriter task type built on top of the FreePrompt task type. + $taskTypeName = $l->t('Copywriting'); + $taskInput = $l->t('Writing style: %1$s; Source material: %2$s', [$params['inputs']['writingStyle'], $params['inputs']['sourceMaterial']]); + } else { + try { + /** @var ITaskType $taskType */ + $taskType = $this->container->get($params['taskTypeClass']); + $taskTypeName = $taskType->getName(); + } catch (\Exception | \Throwable $e) { + $this->logger->debug('Impossible to get task type ' . $params['taskTypeClass'], ['exception' => $e]); + } } - } elseif ($params['taskType'] === Application::TASK_TYPE_TEXT_TO_IMAGE) { + } elseif ($params['taskModality'] === Application::TASK_TYPE_TEXT_TO_IMAGE) { $taskTypeName = $l->t('Text to image'); - } elseif ($params['taskType'] === Application::TASK_TYPE_SPEECH_TO_TEXT) { + } elseif ($params['taskModality'] === Application::TASK_TYPE_SPEECH_TO_TEXT) { $taskTypeName = $l->t('Speech to text'); } @@ -92,8 +99,9 @@ public function prepare(INotification $notification, string $languageCode): INot : $l->t('"%1$s" task for "%2$s" has finished', [$taskTypeName, $schedulingAppName]); $content = ''; - if (isset($params['input'])) { - $content .= $l->t('Input: %1$s', [$params['input']]); + + if ($taskInput) { + $content .= $l->t('Input: %1$s', [$taskInput]); } if (isset($params['result'])) { diff --git a/lib/Service/AssistantService.php b/lib/Service/AssistantService.php index 9a6f5492..907c13dc 100644 --- a/lib/Service/AssistantService.php +++ b/lib/Service/AssistantService.php @@ -2,15 +2,29 @@ namespace OCA\TpAssistant\Service; +require_once __DIR__ . '/../../vendor/autoload.php'; + use DateTime; use OCA\TpAssistant\AppInfo\Application; use OCP\Common\Exception\NotFoundException; +use OCP\Files\GenericFileException; use OCP\IURLGenerator; use OCP\Notification\IManager as INotificationManager; use OCP\PreConditionNotMetException; use OCP\TextProcessing\IManager as ITextProcessingManager; use OCP\TextProcessing\Task as TextProcessingTask; use OCP\TextToImage\Task as TextToImageTask; +use OCA\TpAssistant\Db\TaskMapper; +use OCA\TpAssistant\Db\Task; +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\AppFramework\Db\MultipleObjectsReturnedException; +use OCP\TextProcessing\FreePromptTaskType; +use Psr\Log\LoggerInterface; +use OCP\Files\IRootFolder; +use OCP\Files\NotPermittedException; +use OCP\Lock\LockedException; +use PhpOffice\PhpWord\IOFactory; +use Parsedown; class AssistantService { @@ -18,45 +32,68 @@ public function __construct( private INotificationManager $notificationManager, private ITextProcessingManager $textProcessingManager, private IURLGenerator $url, + private TaskMapper $taskMapper, + private LoggerInterface $logger, + private IRootFolder $storage, + private ?string $userId, ) { } /** * Send a success or failure task result notification * - * @param TextProcessingTask|TextToImageTask $task + * @param Task $task * @param string|null $target optional notification link target * @param string|null $actionLabel optional label for the notification action button * @return void * @throws \InvalidArgumentException */ - public function sendNotification(TextProcessingTask|TextToImageTask $task, ?string $target = null, ?string $actionLabel = null): void { + public function sendNotification(Task $task, ?string $target = null, ?string $actionLabel = null, ?string $resultPreview = null): void { $manager = $this->notificationManager; $notification = $manager->createNotification(); $params = [ 'appId' => $task->getAppId(), 'id' => $task->getId(), - 'input' => $task->getInput(), + 'inputs' => $task->getInputsAsArray(), 'target' => $target, 'actionLabel' => $actionLabel, + 'result' => $resultPreview, ]; - if ($task instanceof TextToImageTask) { - $params['taskType'] = Application::TASK_TYPE_TEXT_TO_IMAGE; - $subject = $task->getStatus() === TextToImageTask::STATUS_SUCCESSFUL - ? 'success' - : 'failure'; - } else { - $params['taskType'] = Application::TASK_TYPE_TEXT_GEN; - $params['textTaskTypeClass'] = $task->getType(); - $subject = $task->getStatus() === TextProcessingTask::STATUS_SUCCESSFUL - ? 'success' - : 'failure'; + $params['taskTypeClass'] = $task->getTaskType(); + $params['taskModality'] = $task->getModality(); + + switch ($task->getModality()) { + case Application::TASK_TYPE_TEXT_TO_IMAGE: + { + $taskSuccessful = $task->getStatus() === TextToImageTask::STATUS_SUCCESSFUL; + break; + } + case Application::TASK_TYPE_TEXT_GEN: + { + $taskSuccessful = $task->getStatus() === TextProcessingTask::STATUS_SUCCESSFUL; + break; + } + case Application::TASK_TYPE_SPEECH_TO_TEXT: + { + $taskSuccessful = $task->getStatus() === Application::STT_TASK_SUCCESSFUL; + break; + } + default: + { + $taskSuccessful = false; + break; + } } + $subject = $taskSuccessful + ? 'success' + : 'failure'; + $objectType = $target === null ? 'task' : 'task-with-custom-target'; + $notification->setApp(Application::APP_ID) ->setUser($task->getUserId()) ->setDateTime(new DateTime()) @@ -66,50 +103,325 @@ public function sendNotification(TextProcessingTask|TextToImageTask $task, ?stri $manager->notify($notification); } + /** + * @param string $writingStyle + * @param string $sourceMaterial + * @return string + */ + private function formattedCopywriterPrompt(string $writingStyle, string $sourceMaterial): string { + return "You're a professional copywriter tasked with copying an instructed/demonstrated *WRITING STYLE* to write a text on the provided *SOURCE MATERIAL*. \n*WRITING STYLE*:\n$writingStyle\n\n*SOURCE MATERIAL*:\n\n$sourceMaterial\n\nNow write a text in the same style detailed under *WRITING STYLE* on the *SOURCE MATERIAL*. Also, follow any additional instructions detailed under *SOURCE MATERIAL*.\n\n*NOTE*: You can use the *SOURCE MATERIAL* as a reference, but you must not copy it verbatim."; + } + + /** + * Sanitize inputs for storage based on the input type + * @param string $type + * @param array $inputs + * @return array + * @throws \Exception + */ + private function sanitizeInputs(string $type, array $inputs): array { + switch ($type) { + case 'copywriter': + { + // Sanitize the input array based on the allowed keys and making sure all inputs are strings: + $inputs = array_filter($inputs, function($value, $key) { + return in_array($key, ['writingStyle', 'sourceMaterial']) && is_string($value); + }, ARRAY_FILTER_USE_BOTH); + + if (count($inputs) !== 2) { + throw new \Exception('Invalid input(s)'); + } + break; + } + default: + { + if (!is_string($inputs['prompt']) || count($inputs) !== 1) { + throw new \Exception('Invalid input(s)'); + } + break; + } + } + return $inputs; + } + /** * @param string|null $userId * @param int $taskId - * @return TextProcessingTask + * @return Task */ - public function getTextProcessingTask(?string $userId, int $taskId): ?TextProcessingTask { + public function getTextProcessingTask(?string $userId, int $taskId): ?Task { try { - $task = $this->textProcessingManager->getTask($taskId); - } catch (NotFoundException | \RuntimeException $e) { + $task = $this->taskMapper->getTask($taskId); + } catch (DoesNotExistException | MultipleObjectsReturnedException | \OCP\Db\Exception $e) { return null; } if ($task->getUserId() !== $userId) { return null; } + // Check if the task status is up-to-date (if not, update status and output) + try { + $ocpTask = $this->textProcessingManager->getTask($task->getOcpTaskId()); + + if($ocpTask->getStatus() !== $task->getStatus()) { + $task->setStatus($ocpTask->getStatus()); + $task->setOutput($ocpTask->getOutput()); + $task = $this->taskMapper->update($task); + } + } catch (NotFoundException $e) { + // Ocp task not found, so we can't update the status + } catch (\Exception | \Throwable $e) { + // Something else went wrong, so we can't update the status + } + return $task; } /** * @param string $type - * @param string $input + * @param array $input * @param string $appId * @param string|null $userId * @param string $identifier - * @return TextProcessingTask + * @return Task * @throws PreConditionNotMetException + * @throws \Exception */ - public function runTextProcessingTask(string $type, string $input, string $appId, ?string $userId, string $identifier): TextProcessingTask { - $task = new TextProcessingTask($type, $input, $appId, $userId, $identifier); - $this->textProcessingManager->runTask($task); - return $task; + public function runTextProcessingTask(string $type, array $inputs, string $appId, ?string $userId, string $identifier): Task { + $inputs = $this->sanitizeInputs($type, $inputs); + switch ($type) { + case 'copywriter': + { + // Format the input prompt + $input = $this->formattedCopywriterPrompt($inputs['writingStyle'], $inputs['sourceMaterial']); + $task = new TextProcessingTask(FreePromptTaskType::class, $input, $appId, $userId, $identifier); + $this->textProcessingManager->runTask($task); + break; + } + default: + { + $input = $inputs['prompt']; + $task = new TextProcessingTask($type, $input, $appId, $userId, $identifier); + $this->textProcessingManager->runTask($task); + break; + } + } + + $assistantTask = $this->taskMapper->createTask($userId, $inputs, $task->getOutput(), time(), $task->getId(),$type, $appId, $task->getStatus(), Application::TASK_TYPE_TEXT_GEN, $identifier); + + return $assistantTask; } /** * @param string $type - * @param string $input + * @param array $input * @param string $appId * @param string|null $userId * @param string $identifier - * @return TextProcessingTask + * @return Task * @throws PreConditionNotMetException + * @throws \Exception */ - public function runOrScheduleTextProcessingTask(string $type, string $input, string $appId, ?string $userId, string $identifier): TextProcessingTask { - $task = new TextProcessingTask($type, $input, $appId, $userId, $identifier); - $this->textProcessingManager->runOrScheduleTask($task); - return $task; + public function scheduleTextProcessingTask(string $type, array $inputs, string $appId, ?string $userId, string $identifier): Task { + $inputs = $this->sanitizeInputs($type, $inputs); + switch ($type) { + case 'copywriter': + { + // Format the input prompt + $input = $this->formattedCopywriterPrompt($inputs['writingStyle'], $inputs['sourceMaterial']); + $task = new TextProcessingTask(FreePromptTaskType::class, $input, $appId, $userId, $identifier); + $this->textProcessingManager->scheduleTask($task); + break; + } + default: + { + $input = $inputs['prompt']; + $task = new TextProcessingTask($type, $input, $appId, $userId, $identifier); + $this->textProcessingManager->scheduleTask($task); + break; + } + } + + $assistantTask = $this->taskMapper->createTask($userId, $inputs, $task->getOutput(), time(), $task->getId(),$type, $appId, $task->getStatus(), Application::TASK_TYPE_TEXT_GEN, $identifier); + + return $assistantTask; + } + + /** + * @param string $type + * @param array $inputs + * @param string $appId + * @param string|null $userId + * @param string $identifier + * @return Task + * @throws PreConditionNotMetException + * @throws \OCP\Db\Exception + * @throws \Exception + */ + public function runOrScheduleTextProcessingTask(string $type, array $inputs, string $appId, ?string $userId, string $identifier): Task { + $inputs = $this->sanitizeInputs($type, $inputs); + switch ($type) { + case 'copywriter': + { + // Format the input prompt + $input = $this->formattedCopywriterPrompt($inputs['writingStyle'], $inputs['sourceMaterial']); + $task = new TextProcessingTask(FreePromptTaskType::class, $input, $appId, $userId, $identifier); + $this->textProcessingManager->runOrScheduleTask($task); + break; + } + default: + { + $input = $inputs['prompt']; + $task = new TextProcessingTask($type, $input, $appId, $userId, $identifier); + $this->textProcessingManager->runOrScheduleTask($task); + break; + } + } + + $assistantTask = $this->taskMapper->createTask($userId, $inputs, $task->getOutput(), time(), $task->getId(),$type, $appId, $task->getStatus(), Application::TASK_TYPE_TEXT_GEN, $identifier); + + return $assistantTask; + } + + /** + * Parse text from file (if parsing the file type is supported) + * @param string $filePath + * @return string + * @throws \Exception + */ + public function parseTextFromFile(string $filePath): string { + try { + $userFolder = $this->storage->getUserFolder($this->userId); + } catch (\OC\User\NoUserException | NotPermittedException $e) { + throw new \Exception('Could not access user storage.'); + } + + + try { + $mimeType = $userFolder->get($filePath)->getMimeType(); + } catch (NotFoundException $e) { + throw new \Exception('File not found.'); + } + + try { + $contents = $userFolder->get($filePath)->getContent(); + } catch (NotFoundException | LockedException | GenericFileException | NotPermittedException $e) { + throw new \Exception('File not found or could not be accessed.'); + } + + switch ($mimeType) { + default: + case 'text/plain': + { + $text = $contents; + + break; + } + case 'text/markdown': + { + $parser = new Parsedown(); + $text = $parser->text($contents); + // Remove HTML tags: + $text = strip_tags($text); + break; + } + case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': + case 'application/msword': + case 'application/rtf': + case 'application/vnd.oasis.opendocument.text': + { + // Store the file in a temp dir and provide a path for the doc parser to use + $tempFilePath = sys_get_temp_dir() . '/assistant_app/' . uniqid() . '.tmp'; + // Make sure the temp dir exists + if (!file_exists(dirname($tempFilePath))) { + mkdir(dirname($tempFilePath), 0600, true); + } + file_put_contents($tempFilePath, $contents); + + $text = $this->parseDocument($tempFilePath, $mimeType); + + // Remove the hardlink to the file (delete it): + unlink($tempFilePath); + + break; + } + } + return $text; } + + /** + * Parse text from doc/docx/odt/rtf file + * @param string $filePath + * @return string + * @throws \Exception + */ + private function parseDocument(string $filePath, string $mimeType): string { + switch ($mimeType) { + case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': + { + $readerType = 'Word2007'; + break; + } + case 'application/msword': + { + $readerType = 'MsDoc'; + break; + } + case 'application/rtf': + { + $readerType = 'RTF'; + break; + } + case 'application/vnd.oasis.opendocument.text': + { + $readerType = 'ODText'; + break; + } + default: + { + throw new \Exception('Unsupported file mimetype'); + } + } + + + $phpWord = IOFactory::createReader($readerType); + $phpWord = $phpWord->load($filePath); + $sections = $phpWord->getSections(); + $outText = ''; + foreach ($sections as $section) { + $elements = $section->getElements(); + /** @var ElementTest $element */ + foreach ($elements as $element) { + $class = get_class($element); + if (method_exists($element, 'getText')) { + $outText .= $element->getText() . "\n"; + } + } + } + + return $outText; + } + + /** + * Parse text from docx file + * @param string $filePath + */ + private function parseDocx(string $filePath): string { + $phpWord = IOFactory::createReader('Word2007'); + $phpWord = $phpWord->load($filePath); + $sections = $phpWord->getSections(); + $outText = ''; + foreach ($sections as $section) { + $elements = $section->getElements(); + /** @var ElementTest $element */ + foreach ($elements as $element) { + $class = get_class($element); + if (method_exists($element, 'getText')) { + $outText .= $element->getText() . "\n"; + } + } + } + + return $outText; + } + } diff --git a/lib/Service/FreePrompt/FreePromptService.php b/lib/Service/FreePrompt/FreePromptService.php index 99d9e90d..65566982 100644 --- a/lib/Service/FreePrompt/FreePromptService.php +++ b/lib/Service/FreePrompt/FreePromptService.php @@ -18,6 +18,7 @@ use OCP\TextProcessing\IManager; use OCP\TextProcessing\Task; use Psr\Log\LoggerInterface; +use OCA\TpAssistant\Db\TaskMapper; class FreePromptService { public function __construct( @@ -26,18 +27,17 @@ public function __construct( private IManager $textProcessingManager, private ?string $userId, private PromptMapper $promptMapper, - private IL10N $l10n + private IL10N $l10n, + private TaskMapper $taskMapper, ) { } /* * @param string $prompt - * @param int $nResults - * @param bool $showPrompt * @return string * @throws Exception */ - public function processPrompt(string $prompt, int $nResults): string { + public function processPrompt(string $prompt): string { $taskTypes = $this->textProcessingManager->getAvailableTaskTypes(); if (!in_array(FreePromptTaskType::class, $taskTypes)) { $this->logger->warning('FreePromptTaskType not available'); @@ -60,24 +60,33 @@ public function processPrompt(string $prompt, int $nResults): string { } } - // Generate nResults prompts - for ($i = 0; $i < $nResults; $i++) { - - // Create a db entity for the generation - $promptTask = new Task(FreePromptTaskType::class, $prompt, Application::APP_ID, $this->userId, $genId); - - // Run or schedule the task: - try { - $this->textProcessingManager->runOrScheduleTask($promptTask); - } catch (DBException | PreConditionNotMetException | TaskFailureException $e) { - $this->logger->warning('Failed to run or schedule a task', ['exception' => $e]); - throw new Exception($this->l10n->t('Failed to run or schedule a task'), Http::STATUS_INTERNAL_SERVER_ERROR); - } + $promptTask = new Task(FreePromptTaskType::class, $prompt, Application::APP_ID, $this->userId, $genId); - // If the task was run immediately, we'll skip the notification.. - // Otherwise we would have to dispatch the notification here. + // Run or schedule the task: + try { + $this->textProcessingManager->runOrScheduleTask($promptTask); + } catch (DBException | PreConditionNotMetException | TaskFailureException $e) { + $this->logger->warning('Failed to run or schedule a task', ['exception' => $e]); + throw new Exception($this->l10n->t('Failed to run or schedule a task'), Http::STATUS_INTERNAL_SERVER_ERROR); } + // Create an assistant task for the free prompt task: + $this->taskMapper->createTask( + $this->userId, + ['prompt' => $prompt], + Application::APP_ID, + $promptTask->getId(), + time(), + FreePromptTaskType::class, + $promptTask->getStatus(), + Application::TASK_TYPE_TEXT_GEN, + $promptTask->getInput(), + $promptTask->getIdentifier() + ); + + // If the task was run immediately, we'll skip the notification.. + // Otherwise we would have to dispatch the notification here. + // Save prompt to database $this->promptMapper->createPrompt($this->userId, $prompt); diff --git a/lib/Service/SpeechToText/SpeechToTextService.php b/lib/Service/SpeechToText/SpeechToTextService.php index a810b6bb..28e72738 100644 --- a/lib/Service/SpeechToText/SpeechToTextService.php +++ b/lib/Service/SpeechToText/SpeechToTextService.php @@ -25,8 +25,6 @@ use DateTime; use InvalidArgumentException; use OCA\TpAssistant\AppInfo\Application; -use OCA\TpAssistant\Db\SpeechToText\Transcript; -use OCA\TpAssistant\Db\SpeechToText\TranscriptMapper; use OCP\Files\File; use OCP\Files\Folder; use OCP\Files\IRootFolder; @@ -38,6 +36,7 @@ use OCP\PreConditionNotMetException; use OCP\SpeechToText\ISpeechToTextManager; use Psr\Log\LoggerInterface; +use OCA\TpAssistant\Db\TaskMapper; use RuntimeException; class SpeechToTextService { @@ -49,7 +48,7 @@ public function __construct( private IURLGenerator $urlGenerator, private LoggerInterface $logger, private IConfig $config, - private TranscriptMapper $transcriptMapper, + private TaskMapper $taskMapper, ) { } @@ -72,6 +71,17 @@ public function transcribeFile(string $path, ?string $userId): void { $audioFile = $userFolder->get($path); $this->manager->scheduleFileTranscription($audioFile, $userId, Application::APP_ID); + + $this->taskMapper->createTask( + $userId, + ['fileId' => $audioFile->getId(), 'eTag' => $audioFile->getEtag()], + '', + time(), + $audioFile->getId(), + "Speech-to-text task", + Application::APP_ID, + Application::STT_TASK_SCHEDULED, + Application::TASK_TYPE_SPEECH_TO_TEXT); } /** @@ -88,7 +98,19 @@ public function transcribeAudio(string $tempFileLocation, ?string $userId): void } $audioFile = $this->getFileObject($userId, $tempFileLocation); + $this->manager->scheduleFileTranscription($audioFile, $userId, Application::APP_ID); + + $this->taskMapper->createTask( + $userId, + ['fileId' => $audioFile->getId(), 'eTag' => $audioFile->getEtag()], + '', + time(), + $audioFile->getId(), + "Speech-to-text task", + Application::APP_ID, + Application::STT_TASK_SCHEDULED, + Application::TASK_TYPE_SPEECH_TO_TEXT); } /** @@ -150,51 +172,4 @@ private function getUniqueNamedFolder(string $userId, int $try = 3): Folder { return $userFolder->newFolder($sttFolderPath); } - - /** - * Send transcription result notification - * @param string $userId - * @param string $result - * @param boolean $success - * @param int $taskType - * @return void - * @throws \InvalidArgumentException - */ - public function sendSpeechToTextNotification(string $userId, string $result, bool $success): void { - $manager = $this->notificationManager; - $notification = $manager->createNotification(); - - try { - $transcriptEntity = new Transcript(); - $transcriptEntity->setUserId($userId); - $transcriptEntity->setTranscript($result); - // never seen transcripts should also be deleted in the cleanup job - $transcriptEntity->setLastAccessed(new DateTime()); - $transcriptEntity = $this->transcriptMapper->insert($transcriptEntity); - - $id = $transcriptEntity->getId(); - } catch (\OCP\Db\Exception $e) { - $this->logger->error('Failed to save transcript in DB: ' . $e->getMessage()); - $success = false; - $id = 0; - } - - $params = [ - 'appId' => Application::APP_ID, - 'taskType' => Application::TASK_TYPE_SPEECH_TO_TEXT, - 'result' => $result, - 'target' => $this->urlGenerator->linkToRouteAbsolute(Application::APP_ID . '.SpeechToText.getResultPage', ['id' => $id]) - ]; - $subject = $success - ? 'success' - : 'failure'; - - $notification->setApp(Application::APP_ID) - ->setUser($userId) - ->setDateTime(new DateTime()) - ->setObject('speech-to-text-result', (string) $id) - ->setSubject($subject, $params); - - $manager->notify($notification); - } } diff --git a/lib/Service/Text2Image/Text2ImageHelperService.php b/lib/Service/Text2Image/Text2ImageHelperService.php index 17118227..752e2035 100644 --- a/lib/Service/Text2Image/Text2ImageHelperService.php +++ b/lib/Service/Text2Image/Text2ImageHelperService.php @@ -35,6 +35,7 @@ use OCP\TextToImage\Task; use Psr\Log\LoggerInterface; use Random\RandomException; +use OCA\TpAssistant\Db\TaskMapper; use RuntimeException; class Text2ImageHelperService { @@ -52,7 +53,8 @@ public function __construct( private IAppData $appData, private IURLGenerator $urlGenerator, private IL10N $l10n, - private AssistantService $assistantService + private AssistantService $assistantService, + private TaskMapper $taskMapper, ) { } @@ -102,6 +104,20 @@ public function processPrompt(string $prompt, int $nResults, bool $displayPrompt // Store the image id to the db: $this->imageGenerationMapper->createImageGeneration($imageGenId, $displayPrompt ? $prompt : '', $this->userId ?? '', $expCompletionTime->getTimestamp()); + // Create an assistant meta task for the image generation task: + $this->taskMapper->createTask( + $this->userId, + ['prompt' => $prompt], + $imageGenId, + time(), + $promptTask->getId(), + Task::class, + Application::APP_ID, + $promptTask->getStatus(), + Application::TASK_TYPE_TEXT_TO_IMAGE, + $promptTask->getIdentifier() + ); + if ($taskExecuted) { $this->storeImages($images, $imageGenId); } @@ -233,36 +249,6 @@ public function storeImages(?array $iImages, string $imageGenId): void { // For clarity we'll notify the user that the generation is ready in the event listener } - /** - * Notify user of generation being ready - * @param string $imageGenId - * @return void - */ - public function notifyUser(string $imageGenId): void { - // Get the task associated with the generation: - try { - $task = $this->textToImageManager->getUserTasksByApp(null, Application::APP_ID, $imageGenId); - if (count($task) === 0) { - throw new RuntimeException('empty task array'); - } - } catch (RuntimeException $e) { - $this->logger->debug('Task for the given generation id does not exist or could not be retrieved: ' . $e->getMessage(), ['app' => Application::APP_ID]); - return; - } - - // Generate the link: - $link = $this->urlGenerator->linkToRouteAbsolute( - Application::APP_ID . '.Text2Image.showGenerationPage', - [ - 'imageGenId' => $imageGenId, - ] - ); - - // Notify the user: - $this->assistantService->sendNotification($task[0], $link, $this->l10n->t('View')); - - } - /** * Get imageDataFolder * @return ISimpleFolder @@ -553,7 +539,17 @@ public function notifyWhenReady(string $imageGenId): void { // Just in case the image generation is already ready, notify the user immediately so that the result is not lost: if ($imageGeneration->getIsGenerated()) { - $this->notifyUser($imageGenId); + // Get the assistant task + try { + $assistantTask = $this->taskMapper->getTaskByOcpTaskIdAndModality($imageGeneration->getTaskId(), Application::TASK_TYPE_TEXT_TO_IMAGE); + } catch (Exception | DoesNotExistException | MultipleObjectsReturnedException $e) { + $this->logger->debug('Assistant meta task for the given generation id does not exist or could not be retrieved: ' . $e->getMessage(), ['app' => Application::APP_ID]); + return; + } + $assistantTask->setStatus($imageGeneration->getFailed() ? Task::STATUS_FAILED : Task::STATUS_SUCCESSFUL); + $assistantTask = $this->taskMapper->update($assistantTask); + + $this->assistantService->sendNotification($assistantTask); } } diff --git a/src/assistant.js b/src/assistant.js index d9e6ec81..338c1d90 100644 --- a/src/assistant.js +++ b/src/assistant.js @@ -1,6 +1,7 @@ import { STATUS, TASK_TYPES } from './constants.js' import { linkTo } from '@nextcloud/router' import { getRequestToken } from '@nextcloud/auth' + __webpack_nonce__ = btoa(getRequestToken()) // eslint-disable-line __webpack_public_path__ = linkTo('assistant', 'js/') // eslint-disable-line @@ -36,11 +37,12 @@ __webpack_public_path__ = linkTo('assistant', 'js/') // eslint-disable-line * @param {boolean} params.isInsideViewer Should be true if this function is called while the Viewer is displayed * @param {boolean} params.closeOnResult If true, the modal will be closed when getting a sync result * @param {Array} params.actionButtons List of extra buttons to show in the assistant result form (only if closeOnResult is false) + * @param {boolean} params.useMetaTasks If true, the promise will resolve with the meta task object instead of the ocp task * @return {Promise} */ export async function openAssistantTextProcessingForm({ appId, identifier = '', taskType = null, input = '', - isInsideViewer = undefined, closeOnResult = false, actionButtons = undefined, + isInsideViewer = undefined, closeOnResult = false, actionButtons = undefined, useMetaTasks = false, }) { const { default: Vue } = await import(/* webpackChunkName: "vue-lazy" */'vue') const { default: AssistantTextProcessingModal } = await import(/* webpackChunkName: "assistant-modal-lazy" */'./components/AssistantTextProcessingModal.vue') @@ -59,8 +61,7 @@ export async function openAssistantTextProcessingForm({ const view = new View({ propsData: { isInsideViewer, - input, - taskType: TASK_TYPES.text_generation, + inputs: { prompt: input }, textProcessingTaskTypeId, showScheduleConfirmation: false, showSyncTaskRunning: false, @@ -74,13 +75,13 @@ export async function openAssistantTextProcessingForm({ reject(new Error('User cancellation')) }) view.$on('submit', (data) => { - scheduleTask(appId, identifier, data.taskTypeId, data.input) + scheduleTask(appId, identifier, data.textProcessingTaskTypeId, data.inputs) .then((response) => { - view.input = data.input + view.inputs = data.inputs view.showScheduleConfirmation = true - const task = response.data?.ocs?.data?.task + const task = response.data?.task lastTask = task - resolve(task) + useMetaTasks ? resolve(task) : resolve(resolveMetaTaskToOcpTask(task)) }) .catch(error => { view.$destroy() @@ -91,14 +92,14 @@ export async function openAssistantTextProcessingForm({ view.$on('sync-submit', (data) => { view.loading = true view.showSyncTaskRunning = true - view.input = data.input - view.textProcessingTaskTypeId = data.taskTypeId - runOrScheduleTask(appId, identifier, data.taskTypeId, data.input) + view.inputs = data.inputs + view.textProcessingTaskTypeId = data.textProcessingTaskTypeId + runOrScheduleTask(appId, identifier, data.textProcessingTaskTypeId, data.inputs) .then((response) => { const task = response.data?.task lastTask = task - resolve(task) - view.input = task.input + useMetaTasks ? resolve(task) : resolve(resolveMetaTaskToOcpTask(task)) + view.inputs = task.inputs if (task.status === STATUS.successfull) { if (closeOnResult) { view.$destroy() @@ -125,13 +126,13 @@ export async function openAssistantTextProcessingForm({ }) view.$on('cancel-sync-n-schedule', () => { cancelCurrentSyncTask() - scheduleTask(appId, identifier, view.textProcessingTaskTypeId, view.input) + scheduleTask(appId, identifier, view.textProcessingTaskTypeId, view.inputs) .then((response) => { view.showSyncTaskRunning = false view.showScheduleConfirmation = true - const task = response.data?.ocs?.data?.task + const task = response.data?.task lastTask = task - resolve(task) + useMetaTasks ? resolve(task) : resolve(resolveMetaTaskToOcpTask(task)) }) .catch(error => { view.$destroy() @@ -149,18 +150,36 @@ export async function openAssistantTextProcessingForm({ }) } +async function resolveMetaTaskToOcpTask(metaTask) { + const { default: axios } = await import(/* webpackChunkName: "axios-lazy" */'@nextcloud/axios') + const { generateOcsUrl } = await import(/* webpackChunkName: "router-gen-lazy" */'@nextcloud/router') + if (metaTask.modality !== TASK_TYPES.text_generation) { + // For now we only resolve text generation tasks + return null + } + + const url = generateOcsUrl('textprocessing/tasks/{taskId}', { taskId: metaTask.ocpTaskId }) + axios.post(url).then(response => { + console.debug('resolved meta task', response.data?.ocs?.data?.task) + return response.data?.ocs?.data?.task + }).catch(error => { + console.error(error) + return null + }) +} + export async function cancelCurrentSyncTask() { window.assistantAbortController?.abort() } -export async function runTask(appId, identifier, taskType, input) { +export async function runTask(appId, identifier, taskType, inputs) { window.assistantAbortController = new AbortController() const { default: axios } = await import(/* webpackChunkName: "axios-lazy" */'@nextcloud/axios') const { generateUrl } = await import(/* webpackChunkName: "router-gen-lazy" */'@nextcloud/router') saveLastSelectedTaskType(taskType) const url = generateUrl('/apps/assistant/run') const params = { - input, + inputs, type: taskType, appId, identifier, @@ -168,14 +187,14 @@ export async function runTask(appId, identifier, taskType, input) { return axios.post(url, params, { signal: window.assistantAbortController.signal }) } -export async function runOrScheduleTask(appId, identifier, taskType, input) { +export async function runOrScheduleTask(appId, identifier, taskType, inputs) { window.assistantAbortController = new AbortController() const { default: axios } = await import(/* webpackChunkName: "axios-lazy" */'@nextcloud/axios') const { generateUrl } = await import(/* webpackChunkName: "router-gen-lazy" */'@nextcloud/router') saveLastSelectedTaskType(taskType) const url = generateUrl('/apps/assistant/run-or-schedule') const params = { - input, + inputs, type: taskType, appId, identifier, @@ -189,16 +208,16 @@ export async function runOrScheduleTask(appId, identifier, taskType, input) { * @param {string} appId the scheduling app id * @param {string} identifier the task identifier * @param {string} taskType the task type class - * @param {string} input the task input text + * @param {Array} inputs the task input texts as an array * @return {Promise<*>} */ -export async function scheduleTask(appId, identifier, taskType, input) { +export async function scheduleTask(appId, identifier, taskType, inputs) { const { default: axios } = await import(/* webpackChunkName: "axios-lazy" */'@nextcloud/axios') - const { generateOcsUrl } = await import(/* webpackChunkName: "router-genocs-lazy" */'@nextcloud/router') + const { generateUrl } = await import(/* webpackChunkName: "router-gen-lazy" */'@nextcloud/router') saveLastSelectedTaskType(taskType) - const url = generateOcsUrl('textprocessing/schedule', 2) + const url = generateUrl('/apps/assistant/schedule') const params = { - input, + inputs, type: taskType, appId, identifier, @@ -246,70 +265,36 @@ export function handleNotification(event) { // We use the object type to know if (event.notification.objectType === 'task') { event.cancelAction = true - showTextProcessingTaskResult(event.notification.objectId) - } else if (event.notification.objectType === 'speech-to-text-result') { - event.cancelAction = true - showSpeechToTextResult(event.notification) + showAssistantTaskResult(event.notification.objectId) } } /** - * Show the result of a task + * Show the result of a task based on the meta task id * - * @param {number} taskId the task id to show the result of + * @param {number} taskId the assistant meta task id to show the result of * @return {Promise} */ -async function showTextProcessingTaskResult(taskId) { +async function showAssistantTaskResult(taskId) { const { default: axios } = await import(/* webpackChunkName: "axios-lazy" */'@nextcloud/axios') - const { generateOcsUrl } = await import(/* webpackChunkName: "router-lazy" */'@nextcloud/router') + const { generateUrl } = await import(/* webpackChunkName: "router-lazy" */'@nextcloud/router') const { showError } = await import(/* webpackChunkName: "dialogs-lazy" */'@nextcloud/dialogs') - const url = generateOcsUrl('textprocessing/task/{taskId}', { taskId }) + const url = generateUrl('apps/assistant/r/{taskId}', { taskId }) axios.get(url).then(response => { - console.debug('showing results for task', response.data.ocs.data.task) - openAssistantTaskResult(response.data.ocs.data.task) + console.debug('showing results for task', response.data.task) + openAssistantTaskResult(response.data.task, true) }).catch(error => { console.error(error) showError(t('assistant', 'This task does not exist or has been cleaned up')) }) } -/** - * Show the result of a speech to text transcription - * @param {object} notification the notification object - * @return {Promise} - */ -async function showSpeechToTextResult(notification) { - const { default: Vue } = await import(/* webpackChunkName: "vue-lazy" */'vue') - const { showError } = await import(/* webpackChunkName: "dialogs-lazy" */'@nextcloud/dialogs') - Vue.mixin({ methods: { t, n } }) - - const { generateUrl } = await import(/* webpackChunkName: "router-lazy" */'@nextcloud/router') - const { default: axios } = await import(/* webpackChunkName: "axios-lazy" */'@nextcloud/axios') - - const params = { - params: { - id: notification.objectId, - }, - } - - const url = generateUrl('apps/assistant/stt/transcript') - - axios.get(url, params).then(response => { - console.debug('showing results for stt', response.data) - openAssistantPlainTextResult(response.data, TASK_TYPES.speech_to_text) - }).catch(error => { - console.error(error) - showError(t('assistant', 'This transcript does not exist or has been cleaned up')) - }) -} - /** * Open an assistant modal to show a plain text result - * @param {string} result the plain text result to show - * @param {number} taskType the task type + * @param {object} metaTask assistant meta task object * @return {Promise} */ -export async function openAssistantPlainTextResult(result, taskType) { +export async function openAssistantPlainTextResult(metaTask) { const { default: Vue } = await import(/* webpackChunkName: "vue-lazy" */'vue') const { default: AssistantPlainTextModal } = await import(/* webpackChunkName: "assistant-modal-lazy" */'./components/AssistantPlainTextModal.vue') Vue.mixin({ methods: { t, n } }) @@ -322,8 +307,8 @@ export async function openAssistantPlainTextResult(result, taskType) { const View = Vue.extend(AssistantPlainTextModal) const view = new View({ propsData: { - output: result, - taskType, + output: metaTask.output ?? '', + taskType: metaTask.modality, }, }).$mount(modalElement) @@ -332,17 +317,45 @@ export async function openAssistantPlainTextResult(result, taskType) { }) } +/** + * Open an assistant modal to show an image result + * @param {object} metaTask assistant meta task object + * @return {Promise} + */ +export async function openAssistantImageResult(metaTask) { + // For now just open the image generation result on a new page: + const { generateUrl } = await import(/* webpackChunkName: "router-lazy" */'@nextcloud/router') + const url = generateUrl('apps/assistant/i/{genId}', { genId: metaTask.output }) + window.open(url, '_blank') +} + /** * Open an assistant modal to show the result of a task * * @param {object} task the task we want to see the result of + * @param {boolean} useMetaTasks If false (default), treats the input task as an ocp task, otherwise as an assistant meta task * @return {Promise} */ -export async function openAssistantTaskResult(task) { - const { showError } = await import(/* webpackChunkName: "dialogs-lazy" */'@nextcloud/dialogs') +export async function openAssistantTaskResult(task, useMetaTasks = false) { + // Divert to the right modal/page if we have a meta task with a modality other than text generation: + if (useMetaTasks) { + switch (task.modality) { + case TASK_TYPES.speech_to_text: + openAssistantPlainTextResult(task) + return + case TASK_TYPES.image_generation: + openAssistantImageResult(task) + return + case TASK_TYPES.text_generation: + default: + break + } + } + const { default: Vue } = await import(/* webpackChunkName: "vue-lazy" */'vue') - const { default: AssistantTextProcessingModal } = await import(/* webpackChunkName: "assistant-modal-lazy" */'./components/AssistantTextProcessingModal.vue') Vue.mixin({ methods: { t, n } }) + const { showError } = await import(/* webpackChunkName: "dialogs-lazy" */'@nextcloud/dialogs') + const { default: AssistantTextProcessingModal } = await import(/* webpackChunkName: "assistant-modal-lazy" */'./components/AssistantTextProcessingModal.vue') const modalId = 'assistantTextProcessingModal' const modalElement = document.createElement('div') @@ -353,9 +366,9 @@ export async function openAssistantTaskResult(task) { const view = new View({ propsData: { // isInsideViewer, - input: task.input, + inputs: useMetaTasks ? task.inputs : [task.input], output: task.output ?? '', - textProcessingTaskTypeId: task.type, + textProcessingTaskTypeId: useMetaTasks ? task.taskType : task.type, showScheduleConfirmation: false, }, }).$mount(modalElement) @@ -367,7 +380,7 @@ export async function openAssistantTaskResult(task) { scheduleTask(task.appId, task.identifier, data.taskTypeId, data.input) .then((response) => { view.showScheduleConfirmation = true - console.debug('scheduled task', response.data?.ocs?.data?.task) + console.debug('scheduled task', response.data?.task) }) .catch(error => { view.$destroy() @@ -437,7 +450,7 @@ export async function addAssistantMenuEntry() { }).$mount(menuEntry) view.$on('click', () => { - openAssistantTextProcessingForm({ appId: 'assistant' }) + openAssistantTextProcessingForm({ appId: 'assistant', useMetaTasks: true }) .then(r => { console.debug('scheduled task', r) }) diff --git a/src/components/AssistantFormInputs.vue b/src/components/AssistantFormInputs.vue new file mode 100644 index 00000000..da6cc13b --- /dev/null +++ b/src/components/AssistantFormInputs.vue @@ -0,0 +1,214 @@ + + + + + diff --git a/src/components/AssistantTextProcessingForm.vue b/src/components/AssistantTextProcessingForm.vue index b072b395..0f2feb00 100644 --- a/src/components/AssistantTextProcessingForm.vue +++ b/src/components/AssistantTextProcessingForm.vue @@ -17,18 +17,10 @@ class="task-description"> {{ selectedTaskType.description }} - - +