-
-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Feature] AnythingLLM use locally hosted Llama.cpp and GGUF files for…
… inferencing (#413) * Implement use of native embedder (all-Mini-L6-v2) stop showing prisma queries during dev * Add native embedder as an available embedder selection * wrap model loader in try/catch * print progress on download * add built-in LLM support (expiermental) * Update to progress output for embedder * move embedder selection options to component * saftey checks for modelfile * update ref * Hide selection when on hosted subdomain * update documentation hide localLlama when on hosted * saftey checks for storage of models * update dockerfile to pre-build Llama.cpp bindings * update lockfile * add langchain doc comment * remove extraneous --no-metal option * Show data handling for private LLM * persist model in memory for N+1 chats * update import update dev comment on token model size * update primary README * chore: more readme updates and remove screenshots - too much to maintain, just use the app! * remove screeshot link
- Loading branch information
1 parent
fecfb0f
commit 655ebd9
Showing
22 changed files
with
1,304 additions
and
99 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
84 changes: 84 additions & 0 deletions
84
frontend/src/components/LLMSelection/NativeLLMOptions/index.jsx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| import { useEffect, useState } from "react"; | ||
| import { Flask } from "@phosphor-icons/react"; | ||
| import System from "@/models/system"; | ||
|
|
||
| export default function NativeLLMOptions({ settings }) { | ||
| return ( | ||
| <div className="w-full flex flex-col gap-y-4"> | ||
| <div className="flex flex-col md:flex-row md:items-center gap-x-2 text-white mb-4 bg-orange-800/30 w-fit rounded-lg px-4 py-2"> | ||
| <div className="gap-x-2 flex items-center"> | ||
| <Flask size={18} /> | ||
| <p className="text-sm md:text-base"> | ||
| Using a locally hosted LLM is experimental. Use with caution. | ||
| </p> | ||
| </div> | ||
| </div> | ||
| <div className="w-full flex items-center gap-4"> | ||
| <NativeModelSelection settings={settings} /> | ||
| </div> | ||
| </div> | ||
| ); | ||
| } | ||
|
|
||
| function NativeModelSelection({ settings }) { | ||
| const [customModels, setCustomModels] = useState([]); | ||
| const [loading, setLoading] = useState(true); | ||
|
|
||
| useEffect(() => { | ||
| async function findCustomModels() { | ||
| setLoading(true); | ||
| const { models } = await System.customModels("native-llm", null, null); | ||
| setCustomModels(models || []); | ||
| setLoading(false); | ||
| } | ||
| findCustomModels(); | ||
| }, []); | ||
|
|
||
| if (loading || customModels.length == 0) { | ||
| return ( | ||
| <div className="flex flex-col w-60"> | ||
| <label className="text-white text-sm font-semibold block mb-4"> | ||
| Model Selection | ||
| </label> | ||
| <select | ||
| name="NativeLLMModelPref" | ||
| disabled={true} | ||
| className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5" | ||
| > | ||
| <option disabled={true} selected={true}> | ||
| -- waiting for models -- | ||
| </option> | ||
| </select> | ||
| </div> | ||
| ); | ||
| } | ||
|
|
||
| return ( | ||
| <div className="flex flex-col w-60"> | ||
| <label className="text-white text-sm font-semibold block mb-4"> | ||
| Model Selection | ||
| </label> | ||
| <select | ||
| name="NativeLLMModelPref" | ||
| required={true} | ||
| className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5" | ||
| > | ||
| {customModels.length > 0 && ( | ||
| <optgroup label="Your loaded models"> | ||
| {customModels.map((model) => { | ||
| return ( | ||
| <option | ||
| key={model.id} | ||
| value={model.id} | ||
| selected={settings.NativeLLMModelPref === model.id} | ||
| > | ||
| {model.id} | ||
| </option> | ||
| ); | ||
| })} | ||
| </optgroup> | ||
| )} | ||
| </select> | ||
| </div> | ||
| ); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.