From 0cb829247f82bb6dea0782414076f8f6e1f8f0c6 Mon Sep 17 00:00:00 2001 From: Kavin Valli Date: Thu, 9 Jan 2025 14:53:08 -0500 Subject: [PATCH 1/8] add option to run all vs run all unexecuted --- .../experiments/table/ExperimentTable.tsx | 73 ++++++++++++++----- .../table/cells/HypothesisCellRenderer.tsx | 8 ++ 2 files changed, 63 insertions(+), 18 deletions(-) diff --git a/web/components/templates/prompts/experiments/table/ExperimentTable.tsx b/web/components/templates/prompts/experiments/table/ExperimentTable.tsx index 8d54d59db..eac0bb59e 100644 --- a/web/components/templates/prompts/experiments/table/ExperimentTable.tsx +++ b/web/components/templates/prompts/experiments/table/ExperimentTable.tsx @@ -55,6 +55,12 @@ import useOnboardingContext, { ONBOARDING_STEPS, } from "@/components/layout/onboardingContext"; import { generateOpenAITemplate } from "@/components/shared/CreateNewEvaluator/evaluatorHelpers"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; type TableDataType = { index: number; @@ -118,27 +124,58 @@ export function ExperimentTable({ - + + + { await Promise.all( - rows.map(async (row) => { - const cellRef = cellRefs.current[`${row.id}-${pv.id}`]; - if (cellRef) { - await cellRef.runHypothesis(); - } + (promptVersionsData ?? []).map(async (pv) => { + const rows = table.getRowModel().rows; + await Promise.all( + rows.map(async (row) => { + const cellRef = + cellRefs.current[`${row.id}-${pv.id}`]; + if (cellRef) { + await cellRef.runHypothesis(); + } + }) + ); }) ); - }) - ); - }} - > - - + }} + > + Run all cells + + { + await Promise.all( + (promptVersionsData ?? []).map(async (pv) => { + const rows = table.getRowModel().rows; + await Promise.all( + rows.map(async (row) => { + const cellRef = + cellRefs.current[`${row.id}-${pv.id}`]; + if (cellRef) { + await cellRef.runHypothesisIfRequired(); + } + }) + ); + }) + ); + }} + > + Run unexecuted cells + + + ), columns: [ diff --git a/web/components/templates/prompts/experiments/table/cells/HypothesisCellRenderer.tsx b/web/components/templates/prompts/experiments/table/cells/HypothesisCellRenderer.tsx index 58d5bb9e2..c0c8bdb1c 100644 --- a/web/components/templates/prompts/experiments/table/cells/HypothesisCellRenderer.tsx +++ b/web/components/templates/prompts/experiments/table/cells/HypothesisCellRenderer.tsx @@ -203,8 +203,16 @@ export const HypothesisCellRenderer = forwardRef< setRunning(false); }; + const handleRunHypothesisIfRequired = async (e?: React.MouseEvent) => { + e?.stopPropagation(); + if (!content) { + await handleRunHypothesis(e); + } + }; + useImperativeHandle(ref, () => ({ runHypothesis: () => handleRunHypothesis(), + runHypothesisIfRequired: () => handleRunHypothesisIfRequired(), })); if (running) { From db47b1424d966bd907b931b1b878cb9afa2ecc4a Mon Sep 17 00:00:00 2001 From: Kavin Valli Date: Thu, 9 Jan 2025 15:16:15 -0500 Subject: [PATCH 2/8] change experiments page new experiment button --- .../experiments/table/ExperimentTable.tsx | 4 +- .../experiments/table/experimentsPage.tsx | 118 ++++++++---------- 2 files changed, 54 insertions(+), 68 deletions(-) diff --git a/web/components/templates/prompts/experiments/table/ExperimentTable.tsx b/web/components/templates/prompts/experiments/table/ExperimentTable.tsx index eac0bb59e..d2bc66509 100644 --- a/web/components/templates/prompts/experiments/table/ExperimentTable.tsx +++ b/web/components/templates/prompts/experiments/table/ExperimentTable.tsx @@ -135,7 +135,7 @@ export function ExperimentTable({ { + onSelect={async () => { await Promise.all( (promptVersionsData ?? []).map(async (pv) => { const rows = table.getRowModel().rows; @@ -155,7 +155,7 @@ export function ExperimentTable({ Run all cells { + onSelect={async () => { await Promise.all( (promptVersionsData ?? []).map(async (pv) => { const rows = table.getRowModel().rows; diff --git a/web/components/templates/prompts/experiments/table/experimentsPage.tsx b/web/components/templates/prompts/experiments/table/experimentsPage.tsx index 62983f008..692e5b347 100644 --- a/web/components/templates/prompts/experiments/table/experimentsPage.tsx +++ b/web/components/templates/prompts/experiments/table/experimentsPage.tsx @@ -2,16 +2,23 @@ import AuthHeader from "../../../../shared/authHeader"; import { useExperimentTables } from "../../../../../services/hooks/prompts/experiments"; import ThemedTable from "../../../../shared/themed/table/themedTable"; import { useRouter } from "next/router"; -import { PlusIcon, DocumentPlusIcon } from "@heroicons/react/24/outline"; +import { PlusIcon } from "@heroicons/react/24/outline"; import useNotification from "../../../../shared/notification/useNotification"; import { usePrompts } from "../../../../../services/hooks/prompts/prompts"; import { StartFromPromptDialog } from "./components/startFromPromptDialog"; -import { Dialog, DialogTrigger } from "../../../../ui/dialog"; +import { Dialog } from "../../../../ui/dialog"; import { useState } from "react"; import { useJawnClient } from "../../../../../lib/clients/jawnHook"; import { getExampleExperimentPrompt } from "./helpers/basePrompt"; import { useOrg } from "@/components/layout/org/organizationContext"; import { FeatureUpgradeCard } from "@/components/shared/helicone/FeatureUpgradeCard"; +import { + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { DropdownMenu } from "@/components/ui/dropdown-menu"; +import { Button } from "@/components/ui/button"; const ExperimentsPage = () => { const jawn = useJawnClient(); @@ -92,9 +99,51 @@ const ExperimentsPage = () => { const org = useOrg(); + const { setNotification } = useNotification(); + return ( <> - + + + + + + { + setNotification("Creating experiment...", "info"); + const res = await jawn.POST("/v2/experiment/create/empty"); + if (res.error) { + notification.setNotification( + "Failed to create experiment", + "error" + ); + } else { + router.push(`/experiments/${res.data?.data?.experimentId}`); + } + }} + > + Start from scratch + + setDialogOpen(true)}> + Start from prompt + + + + } + /> + + + setDialogOpen(false)} + /> + {org?.currentOrg?.tier === "free" ? (
@@ -108,69 +157,6 @@ const ExperimentsPage = () => {
) : ( <> -
-

- Create a new experiment -

-
-
- - - Start from scratch - -
-
- - - - - setDialogOpen(false)} - /> - - - Start from a prompt - -
- {templateOptions.map((template) => ( -
- - - {template.name} - -
- ))} -
-
- Date: Thu, 9 Jan 2025 15:17:28 -0500 Subject: [PATCH 3/8] remove unused code --- .../experiments/table/experimentsPage.tsx | 70 ------------------- 1 file changed, 70 deletions(-) diff --git a/web/components/templates/prompts/experiments/table/experimentsPage.tsx b/web/components/templates/prompts/experiments/table/experimentsPage.tsx index 692e5b347..8f33227da 100644 --- a/web/components/templates/prompts/experiments/table/experimentsPage.tsx +++ b/web/components/templates/prompts/experiments/table/experimentsPage.tsx @@ -9,7 +9,6 @@ import { StartFromPromptDialog } from "./components/startFromPromptDialog"; import { Dialog } from "../../../../ui/dialog"; import { useState } from "react"; import { useJawnClient } from "../../../../../lib/clients/jawnHook"; -import { getExampleExperimentPrompt } from "./helpers/basePrompt"; import { useOrg } from "@/components/layout/org/organizationContext"; import { FeatureUpgradeCard } from "@/components/shared/helicone/FeatureUpgradeCard"; import { @@ -28,75 +27,6 @@ const ExperimentsPage = () => { const router = useRouter(); const { experiments, isLoading } = useExperimentTables(); - const templateOptions = [ - { id: "text-classification", name: "Text classification" }, - { id: "knowledge-retrieval", name: "Knowledge retrieval" }, - { id: "step-by-step", name: "Step-by-step instructions" }, - ]; - - const handleStartFromScratch = async () => { - const exampleExperimentPrompt = getExampleExperimentPrompt(); - const res = await jawn.POST("/v1/prompt/create", { - body: { - userDefinedId: exampleExperimentPrompt.promptName, - prompt: exampleExperimentPrompt.basePrompt, - metadata: { - createdFromUi: true, - }, - }, - }); - if (res.error || !res.data) { - notification.setNotification("Failed to create prompt", "error"); - return; - } - - if (!res.data?.data?.id || !res.data?.data?.prompt_version_id) { - notification.setNotification("Failed to create prompt", "error"); - return; - } - - const dataset = await jawn.POST("/v1/helicone-dataset", { - body: { - datasetName: "Dataset for Experiment", - requestIds: [], - }, - }); - if (!dataset.data?.data?.datasetId) { - notification.setNotification("Failed to create dataset", "error"); - return; - } - const experimentTableResult = await jawn.POST("/v1/experiment/table/new", { - body: { - datasetId: dataset.data?.data?.datasetId!, - promptVersionId: res.data?.data?.prompt_version_id!, - newHeliconeTemplate: JSON.stringify(exampleExperimentPrompt.basePrompt), - isMajorVersion: false, - promptSubversionMetadata: { - experimentAssigned: true, - }, - experimentMetadata: { - prompt_id: res.data?.data?.id!, - prompt_version: res.data?.data?.prompt_version_id!, - experiment_name: `${exampleExperimentPrompt.promptName}_V1.0` || "", - }, - experimentTableMetadata: { - datasetId: dataset.data?.data?.datasetId!, - model: exampleExperimentPrompt.basePrompt.model, - prompt_id: res.data?.data?.id!, - prompt_version: res.data?.data?.prompt_version_id!, - }, - }, - }); - if (!experimentTableResult.data?.data?.experimentId) { - notification.setNotification("Failed to create experiment", "error"); - return; - } - - await router.push( - `/experiments/${experimentTableResult.data?.data?.tableId}` - ); - }; - const org = useOrg(); const { setNotification } = useNotification(); From a176bccb2909894e56ac92a51b7bc5a1a69695f7 Mon Sep 17 00:00:00 2001 From: Kavin Valli Date: Thu, 9 Jan 2025 15:30:09 -0500 Subject: [PATCH 4/8] update create new experiments button ui ft. lina --- .../prompts/experiments/table/experimentsPage.tsx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/web/components/templates/prompts/experiments/table/experimentsPage.tsx b/web/components/templates/prompts/experiments/table/experimentsPage.tsx index 8f33227da..bfbe9f343 100644 --- a/web/components/templates/prompts/experiments/table/experimentsPage.tsx +++ b/web/components/templates/prompts/experiments/table/experimentsPage.tsx @@ -18,6 +18,7 @@ import { } from "@/components/ui/dropdown-menu"; import { DropdownMenu } from "@/components/ui/dropdown-menu"; import { Button } from "@/components/ui/button"; +import { ChevronDownIcon } from "lucide-react"; const ExperimentsPage = () => { const jawn = useJawnClient(); @@ -38,12 +39,12 @@ const ExperimentsPage = () => { actions={ - - + { setNotification("Creating experiment...", "info"); From dd0568c9a743fac379ff6bb20aa32f1641fc0923 Mon Sep 17 00:00:00 2001 From: Kavin Valli Date: Fri, 10 Jan 2025 12:04:00 -0500 Subject: [PATCH 5/8] add column->add prompt --- .../templates/prompts/experiments/table/AddColumnHeader.tsx | 2 +- .../templates/prompts/experiments/table/ExperimentTable.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/components/templates/prompts/experiments/table/AddColumnHeader.tsx b/web/components/templates/prompts/experiments/table/AddColumnHeader.tsx index 91306dbb5..c0fce0472 100644 --- a/web/components/templates/prompts/experiments/table/AddColumnHeader.tsx +++ b/web/components/templates/prompts/experiments/table/AddColumnHeader.tsx @@ -63,7 +63,7 @@ const AddColumnHeader: React.FC = ({ > - Add Column + Add Prompt diff --git a/web/components/templates/prompts/experiments/table/ExperimentTable.tsx b/web/components/templates/prompts/experiments/table/ExperimentTable.tsx index d2bc66509..e358e16b0 100644 --- a/web/components/templates/prompts/experiments/table/ExperimentTable.tsx +++ b/web/components/templates/prompts/experiments/table/ExperimentTable.tsx @@ -638,7 +638,7 @@ export function ExperimentTable({ className="bg-white dark:bg-black rounded-sm inline-block min-w-0 w-max h-auto" // style={{ width: "fit-content" }} > - +
{table.getHeaderGroups().map((headerGroup, i) => ( Date: Fri, 10 Jan 2025 13:09:35 -0500 Subject: [PATCH 6/8] remove unused code --- .../components/tableElementsRenderer.tsx | 28 ------------------- .../experiments/table/experimentsPage.tsx | 1 - 2 files changed, 29 deletions(-) diff --git a/web/components/templates/prompts/experiments/table/components/tableElementsRenderer.tsx b/web/components/templates/prompts/experiments/table/components/tableElementsRenderer.tsx index adc79c0b8..bb1f35a74 100644 --- a/web/components/templates/prompts/experiments/table/components/tableElementsRenderer.tsx +++ b/web/components/templates/prompts/experiments/table/components/tableElementsRenderer.tsx @@ -110,34 +110,6 @@ const ExperimentTableHeader = (props: ExperimentHeaderProps) => { } ); - const { data: randomInputRecordsData } = useQuery( - ["randomInputRecords", originalPromptVersionId], - async () => { - console.log("fetching random input records"); - const res = await jawnClient.POST( - "/v1/prompt/version/{promptVersionId}/inputs/query", - { - params: { - path: { - promptVersionId: originalPromptVersionId ?? "", - }, - }, - body: { - limit: 1, - random: true, - }, - } - ); - return res.data?.data ?? []; - }, - { - enabled: showViewPrompt && originalPromptVersionId !== undefined, // Fetch only when the drawer is open - refetchOnWindowFocus: false, - refetchOnMount: false, - refetchOnReconnect: false, - } - ); - const queryClient = useQueryClient(); const promptVersionIdScore = useQuery<{ diff --git a/web/components/templates/prompts/experiments/table/experimentsPage.tsx b/web/components/templates/prompts/experiments/table/experimentsPage.tsx index bfbe9f343..90c697e66 100644 --- a/web/components/templates/prompts/experiments/table/experimentsPage.tsx +++ b/web/components/templates/prompts/experiments/table/experimentsPage.tsx @@ -2,7 +2,6 @@ import AuthHeader from "../../../../shared/authHeader"; import { useExperimentTables } from "../../../../../services/hooks/prompts/experiments"; import ThemedTable from "../../../../shared/themed/table/themedTable"; import { useRouter } from "next/router"; -import { PlusIcon } from "@heroicons/react/24/outline"; import useNotification from "../../../../shared/notification/useNotification"; import { usePrompts } from "../../../../../services/hooks/prompts/prompts"; import { StartFromPromptDialog } from "./components/startFromPromptDialog"; From 4b6a478b2a39a3f7b33258e7a9262412c7af7d0a Mon Sep 17 00:00:00 2001 From: Kavin Valli Date: Fri, 10 Jan 2025 14:40:47 -0500 Subject: [PATCH 7/8] fix claude extra assistant message issue --- .../templates/prompts/id/promptPlayground.tsx | 8 ++++++++ .../requests/chatComponent/messageUtils.ts | 20 +++++++++---------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/web/components/templates/prompts/id/promptPlayground.tsx b/web/components/templates/prompts/id/promptPlayground.tsx index d8f0f5708..ede47abb6 100644 --- a/web/components/templates/prompts/id/promptPlayground.tsx +++ b/web/components/templates/prompts/id/promptPlayground.tsx @@ -113,6 +113,7 @@ const PromptPlayground: React.FC = ({ ); const { requestMessages, responseMessage, messages } = useMemo(() => { + console.log("idhar hai prompt", prompt); const requestMessages = getRequestMessages(undefined, prompt); const responseMessage = getResponseMessage( undefined, @@ -120,6 +121,11 @@ const PromptPlayground: React.FC = ({ (prompt as PromptObject).model ); const messages = getMessages(requestMessages, responseMessage, 200); + console.log({ + requestMessages, + responseMessage, + messages, + }); return { requestMessages, responseMessage, messages }; }, [prompt, selectedInput]); const [promptVariables, setPromptVariables] = useState< @@ -222,6 +228,8 @@ const PromptPlayground: React.FC = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [currentChat, selectedModel]); + console.log("messages", messages); + if ( playgroundMode === "experiment-compact" || playgroundMode === "experiment" diff --git a/web/components/templates/requests/chatComponent/messageUtils.ts b/web/components/templates/requests/chatComponent/messageUtils.ts index 77b152c76..53231b74c 100644 --- a/web/components/templates/requests/chatComponent/messageUtils.ts +++ b/web/components/templates/requests/chatComponent/messageUtils.ts @@ -53,16 +53,16 @@ export function getResponseMessage( model: string ): Message | null { if (/^claude/.test(model)) { - // Handle Anthropic (Claude) response - if (Array.isArray(responseBody?.content)) { - return { - id: responseBody.id || crypto.randomUUID(), - role: "assistant", - content: responseBody.content, - model: responseBody.model, - }; - } else { - // Fallback for unexpected structure + if (responseBody?.content) { + // Handle Anthropic (Claude) response + if (Array.isArray(responseBody?.content)) { + return { + id: responseBody.id || crypto.randomUUID(), + role: "assistant", + content: responseBody.content, + model: responseBody.model, + }; + } return { id: responseBody.id || crypto.randomUUID(), role: "assistant", From 6d863933459609f6ccf72887491c5cc8ffcab987 Mon Sep 17 00:00:00 2001 From: Kavin Valli Date: Fri, 10 Jan 2025 14:44:00 -0500 Subject: [PATCH 8/8] Remove console logs --- web/components/templates/prompts/id/promptPlayground.tsx | 8 -------- 1 file changed, 8 deletions(-) diff --git a/web/components/templates/prompts/id/promptPlayground.tsx b/web/components/templates/prompts/id/promptPlayground.tsx index ede47abb6..d8f0f5708 100644 --- a/web/components/templates/prompts/id/promptPlayground.tsx +++ b/web/components/templates/prompts/id/promptPlayground.tsx @@ -113,7 +113,6 @@ const PromptPlayground: React.FC = ({ ); const { requestMessages, responseMessage, messages } = useMemo(() => { - console.log("idhar hai prompt", prompt); const requestMessages = getRequestMessages(undefined, prompt); const responseMessage = getResponseMessage( undefined, @@ -121,11 +120,6 @@ const PromptPlayground: React.FC = ({ (prompt as PromptObject).model ); const messages = getMessages(requestMessages, responseMessage, 200); - console.log({ - requestMessages, - responseMessage, - messages, - }); return { requestMessages, responseMessage, messages }; }, [prompt, selectedInput]); const [promptVariables, setPromptVariables] = useState< @@ -228,8 +222,6 @@ const PromptPlayground: React.FC = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [currentChat, selectedModel]); - console.log("messages", messages); - if ( playgroundMode === "experiment-compact" || playgroundMode === "experiment"