[FR-22] feature: handle image and text attachments in chat modal (#2993)

resolves part of https://lablup.atlassian.net/browse/FR-22?atlOrigin=eyJpIjoiNjBiMTg0ZWVmN2M3NGNiNmEzYjZhNjE5NmM0NDA4NDAiLCJwIjoiaiJ9 Enable attaching image/text files to ChatSender. Test: 1. Login with admin 2. Visit `<dogbowl endpoint>/serving/9837a76e-d62e-4215-9f76-a10b4b9175fc`. (But, please note that this model only allows one image in one conversion.) 3. Test with Chat modal. 4. Attach image or text file. Features with images: - You can add a image file by clicking or drag and drop. ![image.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/2HueYSdFvL8pOB5mgrUQ/aadefd1c-794a-4b23-acb3-f88a249fe039.png) - Attach an image or text file. ![image.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/2HueYSdFvL8pOB5mgrUQ/379ef08e-c244-4adc-8cf3-32c0932a97b0.png) - Attach a file and close the attachments. You can see a Badge. ![image.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/2HueYSdFvL8pOB5mgrUQ/cc2f8c5e-b7f6-4e1a-9429-9d0317ee6734.png) - Conversation with LLM model. ![image.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/2HueYSdFvL8pOB5mgrUQ/db36028e-fda4-4a81-bf74-2f6616372d99.png) ![image.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/2HueYSdFvL8pOB5mgrUQ/605f09a2-7b60-4823-ae5f-544b2b484533.png) **Checklist:** (if applicable) - [x] Mention to the original issue - [ ] Documentation - [ ] Minium required manager version - [x] Specific setting for review (eg., KB link, endpoint or how to setup) - [x] Minimum requirements to check during review - [ ] Test case(s) to demonstrate the difference of before/after
lablup · Jan 2, 2025 · 34f9b24 · 34f9b24
1 parent 8071aa8
commit 34f9b24
Show file tree

Hide file tree

Showing 26 changed files with 486 additions and 235 deletions.
diff --git a/react/package.json b/react/package.json
@@ -3,8 +3,8 @@
   "version": "24.09.0-alpha.1",
   "private": true,
   "dependencies": {
-    "@ai-sdk/openai": "^1.0.10",
-    "@ai-sdk/react": "^0.0.70",
+    "@ai-sdk/openai": "^1.0.11",
+    "@ai-sdk/react": "^1.0.7",
     "@ant-design/cssinjs": "^1.22.0",
     "@ant-design/icons": "^5.5.1",
     "@ant-design/x": "^1.0.4",
@@ -23,7 +23,7 @@
     "@uiw/codemirror-extensions-langs": "^4.23.6",
     "@uiw/react-codemirror": "^4.23.6",
     "ahooks": "^3.8.1",
-    "ai": "^4.0.20",
+    "ai": "^4.0.22",
     "ansi_up": "^6.0.2",
     "antd": "^5.22.2",
     "antd-style": "^3.7.1",

diff --git a/react/pnpm-lock.yaml b/react/pnpm-lock.yaml
diff --git a/react/src/components/lablupTalkativotUI/ChatMessage.tsx b/react/src/components/lablupTalkativotUI/ChatMessage.tsx
@@ -2,11 +2,13 @@ import Flex from '../Flex';
 // ES 2015
 import ChatMessageContent from './ChatMessageContent';
 import { Message } from '@ai-sdk/react';
+import { Attachments } from '@ant-design/x';
 import { useThrottle } from 'ahooks';
-import { Avatar, theme } from 'antd';
+import { Avatar, theme, Image } from 'antd';
 import dayjs from 'dayjs';
 import localizedFormat from 'dayjs/plugin/localizedFormat';
 import relativeTime from 'dayjs/plugin/relativeTime';
+import _ from 'lodash';
 import React from 'react';
 import { useState } from 'react';
 
@@ -33,6 +35,7 @@ const ChatMessage: React.FC<{
   const [isHovered, setIsHovered] = useState(false);
 
   const throttledMessageContent = useThrottle(message.content, { wait: 50 });
+
   return (
     <Flex
       direction={placement === 'left' ? 'row' : 'row-reverse'}
@@ -60,8 +63,41 @@ const ChatMessage: React.FC<{
         align={placement === 'left' ? 'start' : 'end'}
         wrap="wrap"
         style={{ flex: 1 }}
-        gap={'xxs'}
+        gap={'xs'}
       >
+        {_.map(message.experimental_attachments, (attachment, index) =>
+          _.includes(attachment?.contentType, 'image/') ? (
+            <Flex
+              style={{
+                border: 'none',
+                textAlign: 'end',
+              }}
+              align="end"
+            >
+              <Image
+                key={`${message?.id}-${index}`}
+                src={attachment?.url}
+                alt={attachment?.name}
+                style={{
+                  maxWidth: '50vw',
+                  maxHeight: '12vh',
+                  borderRadius: token.borderRadius,
+                }}
+              />
+            </Flex>
+          ) : (
+            <Attachments.FileCard
+              key={index}
+              item={{
+                uid: `${message?.id}-${index}`,
+                name: attachment?.name || attachment?.url,
+                type: attachment?.contentType,
+                description: attachment?.name,
+                url: attachment?.url,
+              }}
+            />
+          ),
+        )}
         <Flex
           align="stretch"
           direction="column"

diff --git a/react/src/components/lablupTalkativotUI/LLMChatCard.tsx b/react/src/components/lablupTalkativotUI/LLMChatCard.tsx
@@ -8,11 +8,18 @@ import ModelSelect from './ModelSelect';
 import VirtualChatMessageList from './VirtualChatMessageList';
 import { createOpenAI } from '@ai-sdk/openai';
 import { useChat } from '@ai-sdk/react';
-import { DeleteOutlined, MoreOutlined } from '@ant-design/icons';
+import {
+  CloudUploadOutlined,
+  DeleteOutlined,
+  LinkOutlined,
+  MoreOutlined,
+} from '@ant-design/icons';
+import { Attachments, AttachmentsProps, Sender } from '@ant-design/x';
 import { useControllableValue } from 'ahooks';
 import { streamText } from 'ai';
 import {
   Alert,
+  Badge,
   Button,
   Card,
   CardProps,
@@ -25,7 +32,7 @@ import {
 } from 'antd';
 import _ from 'lodash';
 import { Scale } from 'lucide-react';
-import React, { useEffect, useRef } from 'react';
+import React, { useEffect, useRef, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 
 export type BAIModel = {
@@ -81,6 +88,7 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
   ...cardProps
 }) => {
   const webuiNavigate = useWebUINavigate();
+  const [isOpenAttachments, setIsOpenAttachments] = useState(false);
 
   const [modelId, setModelId] = useControllableValue(cardProps, {
     valuePropName: 'modelId',
@@ -89,16 +97,7 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
   });
 
   const customModelFormRef = useRef<FormInstance>(null);
-
-  // const [userInput, setUserInput] = useControllableValue(cardProps,{
-  //   valuePropName: "userInput",
-  //   trigger: "onChangeUserInput",
-  // });
-
-  // useControllableValue(cardProps, {
-  //   valuePropName: "agentId",
-  //   trigger: "onAgentChange",
-  // });
+  const cardRef = useRef<HTMLDivElement>(null);
 
   const {
     messages,
@@ -109,7 +108,7 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
     isLoading,
     append,
     setMessages,
-    // ...chatHelpers
+    // ...chatHelpers,
   } = useChat({
     api: baseURL,
     headers,
@@ -164,6 +163,8 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [submitKey]);
 
+  const [files, setFiles] = useState<AttachmentsProps['items']>([]);
+
   const items: MenuProps['items'] = filterEmptyItem([
     showCompareMenuItem && {
       key: 'compare',
@@ -191,6 +192,7 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
 
   return (
     <Card
+      ref={cardRef}
       bordered
       extra={
         [
@@ -260,6 +262,65 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
           autoFocus
           value={input}
           placeholder="Say something..."
+          header={
+            <Sender.Header
+              closable={false}
+              title={t('chatui.Attachments')}
+              open={!!isOpenAttachments && !_.isEmpty(files)}
+              onOpenChange={setIsOpenAttachments}
+              styles={{
+                content: {
+                  padding: 0,
+                },
+              }}
+            >
+              <Attachments
+                beforeUpload={() => false}
+                getDropContainer={() => cardRef.current}
+                accept="image/*,text/*"
+                items={files}
+                onChange={({ fileList }) => setFiles(fileList)}
+                placeholder={(type) =>
+                  type === 'drop'
+                    ? {
+                        title: t('chatui.DropFileHere'),
+                      }
+                    : {
+                        icon: <CloudUploadOutlined />,
+                        title: t('chatui.UploadFiles'),
+                        description: t('chatui.UploadFilesDescription'),
+                      }
+                }
+              />
+            </Sender.Header>
+          }
+          prefix={
+            <Attachments
+              beforeUpload={() => false}
+              getDropContainer={() => cardRef.current}
+              accept="image/*,text/*"
+              items={files}
+              onChange={({ fileList }) => {
+                setFiles(fileList);
+                setIsOpenAttachments(true);
+              }}
+              placeholder={(type) =>
+                type === 'drop'
+                  ? {
+                      title: t('chatui.DropFileHere'),
+                    }
+                  : {
+                      icon: <CloudUploadOutlined />,
+                      title: t('chatui.UploadFiles'),
+                      description: t('chatui.UploadFilesDescription'),
+                    }
+              }
+            >
+              <Badge dot={!_.isEmpty(files) && !isOpenAttachments}>
+                <Button type="text" icon={<LinkOutlined />} />
+              </Badge>
+            </Attachments>
+          }
           onChange={(v: string) => {
             setInput(v);
             if (onInputChange) {
@@ -271,17 +332,35 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
             stop();
           }}
           onSend={() => {
-            if (input) {
-              append({
-                role: 'user',
-                content: input,
-              });
+            if (input || !_.isEmpty(files)) {
+              const fileList = _.map(
+                files,
+                (item) => item.originFileObj as File,
+              );
+              // Filter after converting to `File`
+              const fileListArray = _.filter(fileList, Boolean);
+              const dataTransfer = new DataTransfer();
+              _.forEach(fileListArray, (file) => dataTransfer.items.add(file));
+
+              append(
+                {
+                  role: 'user',
+                  content: input,
+                },
+                {
+                  experimental_attachments: dataTransfer.files,
+                },
+              );
+
               setTimeout(() => {
                 setInput('');
+                setFiles([]);
+                setIsOpenAttachments(false);
               }, 0);
               onSubmitChange?.();
             }
           }}
+          style={{ flex: 1 }}
         />,
       ]}
     >
@@ -342,9 +421,9 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
         </Form>
       </Flex>
       {/* <ChatMessageList messages={messages}  /> */}
-      {!_.isEmpty((error as any)?.responseBody) ? (
+      {!_.isEmpty(error?.message) ? (
         <Alert
-          message={(error as any)?.responseBody}
+          message={error?.message}
           type="error"
           showIcon
           style={{

diff --git a/react/src/components/lablupTalkativotUI/VirtualChatMessageList.tsx b/react/src/components/lablupTalkativotUI/VirtualChatMessageList.tsx
@@ -5,6 +5,7 @@ import ScrollBottomHandlerButton from './ScrollBottomHandlerButton';
 import { Message } from '@ai-sdk/react';
 import { theme } from 'antd';
 import Compact from 'antd/es/space/Compact';
+import _ from 'lodash';
 import React, { useRef, useState } from 'react';
 import { Virtuoso, VirtuosoHandle } from 'react-virtuoso';
 
@@ -115,7 +116,7 @@ const VirtualChatMessageList: React.FC<VirtualizedListProps> = ({
               }
             }
           }}
-          lastMessageContent={messages[messages.length - 1]?.content}
+          lastMessageContent={_.get(_.last(messages), 'content')}
         />
       </div>
     </Flex>

diff --git a/resources/i18n/de.json b/resources/i18n/de.json
@@ -1743,7 +1743,11 @@
     "DeleteChattingSessionDescription": "Sie sind dabei, dieses Thema zu löschen. \nEinmal gelöscht, kann es nicht wiederhergestellt werden. \nBitte gehen Sie vorsichtig vor.",
     "SelectEndpoint": "Wählen Sie Endpunkt aus",
     "SyncInput": "Eingang synchronisieren",
-    "CompareWithOtherModels": "Vergleichen Sie mit anderen Modellen"
+    "CompareWithOtherModels": "Vergleichen Sie mit anderen Modellen",
+    "Attachments": "Anhänge",
+    "DropFileHere": "Datei hier ablegen",
+    "UploadFiles": "Dateien hochladen",
+    "UploadFilesDescription": "Klicken oder ziehen Sie Dateien zum Hochladen in diesen Bereich"
   },
   "time": {
     "ms": "ms",

diff --git a/resources/i18n/el.json b/resources/i18n/el.json
@@ -1743,7 +1743,11 @@
     "DeleteChattingSessionDescription": "Πρόκειται να διαγράψετε αυτό το θέμα. \nΑφού διαγραφεί, δεν μπορεί να ανακτηθεί. \nΠαρακαλούμε προχωρήστε με προσοχή.",
     "SelectEndpoint": "Επιλέξτε Τελικό σημείο",
     "SyncInput": "Συγχρονισμός εισόδου",
-    "CompareWithOtherModels": "Συγκρίνετε με άλλα μοντέλα"
+    "CompareWithOtherModels": "Συγκρίνετε με άλλα μοντέλα",
+    "Attachments": "Συνημμένα",
+    "DropFileHere": "Αποθέστε το αρχείο εδώ",
+    "UploadFiles": "Μεταφόρτωση αρχείων",
+    "UploadFilesDescription": "Κάντε κλικ ή σύρετε αρχεία σε αυτήν την περιοχή για μεταφόρτωση"
   },
   "time": {
     "ms": "ms",

diff --git a/resources/i18n/en.json b/resources/i18n/en.json
@@ -1747,7 +1747,11 @@
     "DeleteChattingSession": "Delete chatting session",
     "DeleteChattingSessionDescription": "You are about to delete this topic. Once deleted, it cannot be recovered. Please proceed with caution.",
     "SyncInput": "Sync input",
-    "CompareWithOtherModels": "Compare with other models"
+    "CompareWithOtherModels": "Compare with other models",
+    "Attachments": "Attachments",
+    "DropFileHere": "Drop file here",
+    "UploadFiles": "Upload files",
+    "UploadFilesDescription": "Click or drag files to this area to upload"
   },
   "time": {
     "ms": "ms",

diff --git a/resources/i18n/es.json b/resources/i18n/es.json
@@ -1745,7 +1745,11 @@
     "DeleteChattingSessionDescription": "Estás a punto de eliminar este tema. \nUna vez eliminado, no se puede recuperar. \nProceda con precaución.",
     "SelectEndpoint": "Seleccionar punto final",
     "SyncInput": "Entrada de sincronización",
-    "CompareWithOtherModels": "Comparar con otros modelos"
+    "CompareWithOtherModels": "Comparar con otros modelos",
+    "Attachments": "Adjuntos",
+    "DropFileHere": "Suelta el archivo aquí",
+    "UploadFiles": "Subir archivos",
+    "UploadFilesDescription": "Haga clic o arrastre archivos a esta área para cargarlos"
   },
   "time": {
     "ms": "ms",

diff --git a/resources/i18n/fi.json b/resources/i18n/fi.json
@@ -1742,7 +1742,11 @@
     "DeleteChatHistory": "Poista keskusteluhistoria",
     "SelectModel": "Valitse Malli",
     "SyncInput": "Synkronoi sisääntulo",
-    "CompareWithOtherModels": "Vertaa muihin malleihin"
+    "CompareWithOtherModels": "Vertaa muihin malleihin",
+    "Attachments": "Liitteet",
+    "DropFileHere": "Pudota tiedosto tähän",
+    "UploadFiles": "Lataa tiedostoja",
+    "UploadFilesDescription": "Napsauta tai vedä tiedostoja tälle alueelle ladataksesi"
   },
   "time": {
     "ms": "ms",

diff --git a/resources/i18n/fr.json b/resources/i18n/fr.json
@@ -1743,7 +1743,11 @@
     "DeleteChattingSessionDescription": "Vous êtes sur le point de supprimer ce sujet. \nUne fois supprimé, il ne peut pas être récupéré. \nVeuillez procéder avec prudence.",
     "SelectEndpoint": "Sélectionnez le point de terminaison",
     "SyncInput": "Entrée de synchronisation",
-    "CompareWithOtherModels": "Comparez avec d'autres modèles"
+    "CompareWithOtherModels": "Comparez avec d'autres modèles",
+    "Attachments": "Pièces jointes",
+    "DropFileHere": "Déposez le fichier ici",
+    "UploadFiles": "Télécharger des fichiers",
+    "UploadFilesDescription": "Cliquez ou faites glisser les fichiers vers cette zone pour les télécharger"
   },
   "time": {
     "ms": "ms",

diff --git a/resources/i18n/id.json b/resources/i18n/id.json
@@ -1743,7 +1743,11 @@
     "DeleteChattingSessionDescription": "Anda akan menghapus topik ini. \nSetelah dihapus, itu tidak dapat dipulihkan. \nSilakan lanjutkan dengan hati-hati.",
     "SelectEndpoint": "Pilih Titik Akhir",
     "SyncInput": "Sinkronkan masukan",
-    "CompareWithOtherModels": "Bandingkan dengan model lain"
+    "CompareWithOtherModels": "Bandingkan dengan model lain",
+    "Attachments": "Lampiran",
+    "DropFileHere": "Letakkan file di sini",
+    "UploadFiles": "Unggah file",
+    "UploadFilesDescription": "Klik atau seret file ke area ini untuk diunggah"
   },
   "time": {
     "ms": "ms",