From 7f7450a035709b382b9f42dd0f2fc9d17b8b05d0 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Mon, 20 May 2024 10:23:44 -0300 Subject: [PATCH 01/24] Implement basic fill-in-the-middle support --- mikupad.html | 226 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 164 insertions(+), 62 deletions(-) diff --git a/mikupad.html b/mikupad.html index f160eca..e159b7d 100644 --- a/mikupad.html +++ b/mikupad.html @@ -3873,8 +3873,7 @@ const promptText = useMemo(() => joinPrompt(promptChunks), [promptChunks]); - // compute separately as I imagine this can get expensive - const assembledWorldInfo = useMemo(() => { + const assembleWorldInfo = (promptText) => { // assemble non-empty wi const validWorldInfo = !Array.isArray(worldInfo.entries) ? [] : worldInfo.entries.filter(entry => entry.keys.length > 0 && !(entry.keys.length == 1 && entry.keys[0] == "") && entry.text !== ""); @@ -3909,16 +3908,19 @@ }); }); - const assembledWorldInfo = activeWorldInfo.length > 0 + return activeWorldInfo.length > 0 ? activeWorldInfo.map(entry => entry.text).join("\n") : ""; + }; - return assembledWorldInfo + // compute separately as I imagine this can get expensive + const assembledWorldInfo = useMemo(() => { + return assembleWorldInfo(promptText); }, [worldInfo]); - const additionalContextPrompt = useMemo(() => { - // add world info to memory for easier assembly - memoryTokens["worldInfo"] = assembledWorldInfo; + const assembleAdditionalContext = (assembledWorldInfo, promptText) => { + if ("worldInfo" in memoryTokens) + delete memoryTokens["worldInfo"]; const order = ["prefix","text","suffix"] const assembledAuthorNote = authorNoteTokens.text && authorNoteTokens.text !== "" @@ -3927,19 +3929,19 @@ // replacements for the contextOrder string const contextReplacements = { - "{wiPrefix}": memoryTokens.worldInfo && memoryTokens.worldInfo !== "" + "{wiPrefix}": assembledWorldInfo && assembledWorldInfo !== "" ? worldInfo.prefix : "", // wi prefix and suffix will be added whenever wi isn't empty - "{wiText}": memoryTokens.worldInfo, - "{wiSuffix}": memoryTokens.worldInfo && memoryTokens.worldInfo !== "" + "{wiText}": assembledWorldInfo, + "{wiSuffix}": assembledWorldInfo && assembledWorldInfo !== "" ? worldInfo.suffix : "", - "{memPrefix}": memoryTokens.text && memoryTokens.text !== "" || memoryTokens.worldInfo !== "" + "{memPrefix}": memoryTokens.text && memoryTokens.text !== "" || assembledWorldInfo !== "" ? memoryTokens.prefix : "", // memory prefix and suffix will be added whenever memory or wi aren't empty "{memText}": memoryTokens.text, - "{memSuffix}": memoryTokens.text && memoryTokens.text !== "" || memoryTokens.worldInfo !== "" + "{memSuffix}": memoryTokens.text && memoryTokens.text !== "" || assembledWorldInfo !== "" ? memoryTokens.suffix : "", } @@ -3984,13 +3986,79 @@ }).join("\n").replace(/\\n/g, '\n'); return permContextPrompt; + }; + + const additionalContextPrompt = useMemo(() => { + return assembleAdditionalContext(assembledWorldInfo, promptText); }, [contextLength, promptText, memoryTokens, authorNoteTokens, authorNoteDepth, assembledWorldInfo, worldInfo.prefix, worldInfo.suffix]); const modifiedPrompt = useMemo(() => { - return replacePlaceholders(additionalContextPrompt,templateReplacements); + return replacePlaceholders(additionalContextPrompt, templateReplacements); }, [additionalContextPrompt, templates, selectedTemplate]); - async function predict(prompt = modifiedPrompt, chunkCount = promptChunks.length) { + // predict all {fill} placeholders + async function fillsPredict() { + const fillPlaceholder = "{fill}"; + + let leftPromptChunks = []; + let rightPromptChunks = []; + let fillIdx = undefined; + + for (let i = 0; i < promptChunks.length; i++) { + const chunk = promptChunks[i]; + if (chunk.content.includes(fillPlaceholder)) { + // split the chunk in 2 + const left = { content: chunk.content.substring(0, chunk.content.indexOf(fillPlaceholder)), type: "user" }; + const right = { content: chunk.content.substring(chunk.content.indexOf(fillPlaceholder) + fillPlaceholder.length), type: "user" }; + fillIdx = i + 1; + leftPromptChunks = [ + ...promptChunks.slice(0, Math.max(0, i - 1)), + ...[left] + ]; + rightPromptChunks = [ + ...[right], + ...promptChunks.slice(i + 1, promptChunks.length - 1), + ]; + break; + } + } + + if (!fillIdx) + return; + + const promptText = joinPrompt(leftPromptChunks); + const assembledWorldInfo = assembleWorldInfo(promptText); + const additionalContextPrompt = assembleAdditionalContext(assembledWorldInfo, promptText); + const finalPrompt = assembleFinalPrompt(additionalContextPrompt, templateReplacements); + + predict(finalPrompt, leftPromptChunks.length, (chunk) => { + console.log(chunk); + if (rightPromptChunks[0]) { + if (chunk.content.trim().startsWith(rightPromptChunks[0].content[0])) { + if (chunk.content[0] == ' ' && rightPromptChunks[0].content[0] != ' ') { + rightPromptChunks[0].content = ' ' + rightPromptChunks[0].content; + setPromptChunks(p => [ + ...leftPromptChunks, + ...rightPromptChunks + ]); + } + return false; + } + } + leftPromptChunks = [ + ...leftPromptChunks, + chunk + ]; + setPromptChunks(p => [ + ...leftPromptChunks, + ...rightPromptChunks + ]); + setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); + return true; + }); + } + + async function predict(prompt = modifiedPrompt, chunkCount = promptChunks.length, callback = undefined) { if (cancel) { cancel?.(); @@ -4000,7 +4068,7 @@ setCancel(() => () => cancelled = true); await new Promise(resolve => setTimeout(resolve, 500)); if (cancelled) - return; + return false; } const ac = new AbortController(); @@ -4020,34 +4088,36 @@ // so let's set the predictStartTokens beforehand. setPredictStartTokens(tokens); - const tokenCount = await getTokenCount({ - endpoint, - endpointAPI, - ...(endpointAPI == 3 || endpointAPI == 0 ? { endpointAPIKey } : {}), - content: prompt, - signal: ac.signal, - ...(isMikupadEndpoint ? { proxyEndpoint: sessionStorage.proxyEndpoint } : {}) - }); - setTokens(tokenCount); - setPredictStartTokens(tokenCount); - - // Chat Mode - if (chatMode && !restartedPredict && templates[selectedTemplate]) { - // add user EOT template (instruct suffix) if not switch completion - const { instSuf, instPre } = replaceNewlines(templates[selectedTemplate]); - const instSufIndex = instSuf ? prompt.lastIndexOf(instSuf) : -1; - const instPreIndex = instPre ? prompt.lastIndexOf(instPre) : -1; - if (instSufIndex <= instPreIndex) { - setPromptChunks(p => [...p, { type: 'user', content: instSuf }]) - prompt += instSuf; + if (!callback) { + const tokenCount = await getTokenCount({ + endpoint, + endpointAPI, + ...(endpointAPI == 3 || endpointAPI == 0 ? { endpointAPIKey } : {}), + content: prompt, + signal: ac.signal, + ...(isMikupadEndpoint ? { proxyEndpoint: sessionStorage.proxyEndpoint } : {}) + }); + setTokens(tokenCount); + setPredictStartTokens(tokenCount); + + // Chat Mode + if (chatMode && !restartedPredict && templates[selectedTemplate]) { + // add user EOT template (instruct suffix) if not switch completion + const { instSuf, instPre } = replaceNewlines(templates[selectedTemplate]); + const instSufIndex = instSuf ? prompt.lastIndexOf(instSuf) : -1; + const instPreIndex = instPre ? prompt.lastIndexOf(instPre) : -1; + if (instSufIndex <= instPreIndex) { + setPromptChunks(p => [...p, { type: 'user', content: instSuf }]) + prompt += instSuf; + } } - } - setRestartedPredict(false) + setRestartedPredict(false) - while (undoStack.current.at(-1) >= chunkCount) - undoStack.current.pop(); - undoStack.current.push(chunkCount); - redoStack.current = []; + while (undoStack.current.at(-1) >= chunkCount) + undoStack.current.pop(); + undoStack.current.push(chunkCount); + redoStack.current = []; + } setUndoHovered(false); setRejectedAPIKey(false); promptArea.current.scrollTarget = undefined; @@ -4100,8 +4170,13 @@ chunk.content = chunk.stopping_word; if (!chunk.content) continue; - setPromptChunks(p => [...p, chunk]); - setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); + if (callback) { + if (!callback(chunk)) + break; + } else { + setPromptChunks(p => [...p, chunk]); + setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); + } chunkCount += 1; } } catch (e) { @@ -4120,16 +4195,21 @@ return false; } finally { setCancel(c => c === cancelThis ? null : c); - if (undoStack.current.at(-1) === chunkCount) - undoStack.current.pop(); + if (!callback) { + if (undoStack.current.at(-1) === chunkCount) + undoStack.current.pop(); + } } + // Chat Mode - if (chatMode) { + if (!callback && chatMode) { // add bot EOT template (instruct prefix) const eotBot = templates[selectedTemplate]?.instPre.replace(/\\n/g, '\n') setPromptChunks(p => [...p, { type: 'user', content: eotBot }]) prompt += `${eotBot}` } + + return true; } function undo() { @@ -4357,7 +4437,7 @@ switch (`${altKey}:${ctrlKey}:${shiftKey}:${key}`) { case 'false:false:true:Enter': case 'false:true:false:Enter': - predict(); + fillsPredict();//predict(); break; case 'false:false:false:Escape': cancel(); @@ -4504,28 +4584,50 @@ newValue = newValue.slice(0, -chunk.content.length); } + // Merge chunks if they're from the user + let mergeUserChunks = (chunks, newContent) => { + let lastChunk = chunks[chunks.length - 1]; + while (lastChunk && lastChunk.type === 'user') { + lastChunk.content += newContent; + if (chunks[chunks.length - 2] && chunks[chunks.length - 2].type === 'user') { + newContent = lastChunk.content; + lastChunk = chunks[chunks.length - 2]; + chunks.splice(chunks.length - 1, 1); + } else { + return chunks; + } + } + return [...chunks, { type: 'user', content: newContent }]; + }; + + let newPrompt = [...start]; + if (newValue) { + newPrompt = mergeUserChunks(newPrompt, newValue); + } + if (end.length && end[0].type === 'user') { + newPrompt = mergeUserChunks(newPrompt, end.shift().content); + } + newPrompt.push(...end); + // Remove all undo positions within the modified range. - undoStack.current = undoStack.current.filter(pos => start.length < pos); + undoStack.current = undoStack.current.filter(pos => pos > start.length && pos < newPrompt.length); if (!undoStack.current.length) setUndoHovered(false); - // Update all undo positions. - if (start.length + end.length + (+!!newValue) !== oldPromptLength) { - // Reset redo stack if a new chunk is added/removed at the end. - if (!end.length) - redoStack.current = []; + // Adjust undo/redo stacks. + const chunkDifference = oldPromptLength - newPrompt.length; + undoStack.current = undoStack.current.map(pos => { + if (pos >= start.length) { + return pos - chunkDifference; + } + return pos; + }); - if (!oldPrompt.length) - undoStack.current = undoStack.current.map(pos => pos + 1); - else - undoStack.current = undoStack.current.map(pos => pos - oldPrompt.length); + // Reset redo stack if a new chunk is added/removed at the end. + if (chunkDifference < 0 && !end.length) { + redoStack.current = []; } - const newPrompt = [ - ...start, - ...(newValue ? [{ type: 'user', content: newValue }] : []), - ...end, - ]; return newPrompt; }); } From 4d81c69b5665609b8ae8166d7699723b6bf015d6 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Mon, 20 May 2024 23:33:23 -0300 Subject: [PATCH 02/24] Some fixes --- mikupad.html | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mikupad.html b/mikupad.html index e159b7d..09a5211 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4010,15 +4010,15 @@ // split the chunk in 2 const left = { content: chunk.content.substring(0, chunk.content.indexOf(fillPlaceholder)), type: "user" }; const right = { content: chunk.content.substring(chunk.content.indexOf(fillPlaceholder) + fillPlaceholder.length), type: "user" }; - fillIdx = i + 1; leftPromptChunks = [ - ...promptChunks.slice(0, Math.max(0, i - 1)), - ...[left] + ...promptChunks.slice(0, i), + left ]; rightPromptChunks = [ - ...[right], + right, ...promptChunks.slice(i + 1, promptChunks.length - 1), ]; + fillIdx = i + 1; break; } } @@ -4032,9 +4032,9 @@ const finalPrompt = assembleFinalPrompt(additionalContextPrompt, templateReplacements); predict(finalPrompt, leftPromptChunks.length, (chunk) => { - console.log(chunk); if (rightPromptChunks[0]) { - if (chunk.content.trim().startsWith(rightPromptChunks[0].content[0])) { + const trimmedChunk = chunk.content.replace(/^ +| +$/gm, "") + if (trimmedChunk.startsWith(rightPromptChunks[0].content[0])) { if (chunk.content[0] == ' ' && rightPromptChunks[0].content[0] != ' ') { rightPromptChunks[0].content = ' ' + rightPromptChunks[0].content; setPromptChunks(p => [ @@ -4116,8 +4116,10 @@ while (undoStack.current.at(-1) >= chunkCount) undoStack.current.pop(); undoStack.current.push(chunkCount); - redoStack.current = []; + } else { + undoStack.current = []; } + redoStack.current = []; setUndoHovered(false); setRejectedAPIKey(false); promptArea.current.scrollTarget = undefined; From e3d952f1e600d09ea33aeba1217e18f96fd100a6 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Thu, 23 May 2024 22:43:02 -0300 Subject: [PATCH 03/24] Update --- mikupad.html | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/mikupad.html b/mikupad.html index 09a5211..714fb3c 100644 --- a/mikupad.html +++ b/mikupad.html @@ -3996,41 +3996,49 @@ return replacePlaceholders(additionalContextPrompt, templateReplacements); }, [additionalContextPrompt, templates, selectedTemplate]); - // predict all {fill} placeholders - async function fillsPredict() { + // predicts one {fill} placeholder + async function fillPredict() { const fillPlaceholder = "{fill}"; let leftPromptChunks = []; let rightPromptChunks = []; - let fillIdx = undefined; + let foundFillPlaceholder = false; for (let i = 0; i < promptChunks.length; i++) { const chunk = promptChunks[i]; + if (chunk.type !== 'user') + continue; if (chunk.content.includes(fillPlaceholder)) { // split the chunk in 2 - const left = { content: chunk.content.substring(0, chunk.content.indexOf(fillPlaceholder)), type: "user" }; - const right = { content: chunk.content.substring(chunk.content.indexOf(fillPlaceholder) + fillPlaceholder.length), type: "user" }; + foundFillPlaceholder = true; + + let left = chunk.content.substring(0, chunk.content.indexOf(fillPlaceholder)); + if ((left.at(-2) != ' ' || left.at(-2) != '\t') && left.at(-1) == ' ') { + // This is most likely an unintentional mistake by the user. + left = left.substring(0, left.length - 1); + } leftPromptChunks = [ ...promptChunks.slice(0, i), - left + ...(left ? [{ type: 'user', content: left }] : []) ]; + + let right = chunk.content.substring(chunk.content.indexOf(fillPlaceholder) + fillPlaceholder.length); rightPromptChunks = [ - right, + ...(right ? [{ type: 'user', content: right }] : []), ...promptChunks.slice(i + 1, promptChunks.length - 1), ]; - fillIdx = i + 1; break; } } - if (!fillIdx) - return; + if (!foundFillPlaceholder) + return false; const promptText = joinPrompt(leftPromptChunks); const assembledWorldInfo = assembleWorldInfo(promptText); const additionalContextPrompt = assembleAdditionalContext(assembledWorldInfo, promptText); const finalPrompt = assembleFinalPrompt(additionalContextPrompt, templateReplacements); - + predict(finalPrompt, leftPromptChunks.length, (chunk) => { if (rightPromptChunks[0]) { const trimmedChunk = chunk.content.replace(/^ +| +$/gm, "") @@ -4056,6 +4064,7 @@ setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); return true; }); + return true; } async function predict(prompt = modifiedPrompt, chunkCount = promptChunks.length, callback = undefined) { @@ -4071,6 +4080,10 @@ return false; } + // predict the fill placeholder if it exists. + if (!callback && fillPredict()) + return true; + const ac = new AbortController(); const cancelThis = () => { abortCompletion({ @@ -4439,7 +4452,7 @@ switch (`${altKey}:${ctrlKey}:${shiftKey}:${key}`) { case 'false:false:true:Enter': case 'false:true:false:Enter': - fillsPredict();//predict(); + predict(); break; case 'false:false:false:Escape': cancel(); From 51a9bc82896d6fbf7e68c251d9cbb48dbfd0892f Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Thu, 23 May 2024 22:48:24 -0300 Subject: [PATCH 04/24] await fillPredict --- mikupad.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mikupad.html b/mikupad.html index 714fb3c..01a90f1 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4081,7 +4081,7 @@ } // predict the fill placeholder if it exists. - if (!callback && fillPredict()) + if (!callback && await fillPredict()) return true; const ac = new AbortController(); From 6005fa850f8ab1b61b89cffacacc423ec1085852 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Thu, 23 May 2024 23:43:23 -0300 Subject: [PATCH 05/24] Make infilling replace parts of existing prompt chunks --- mikupad.html | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/mikupad.html b/mikupad.html index 01a90f1..868a07d 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4040,29 +4040,36 @@ const finalPrompt = assembleFinalPrompt(additionalContextPrompt, templateReplacements); predict(finalPrompt, leftPromptChunks.length, (chunk) => { + let stop = false; if (rightPromptChunks[0]) { const trimmedChunk = chunk.content.replace(/^ +| +$/gm, "") if (trimmedChunk.startsWith(rightPromptChunks[0].content[0])) { if (chunk.content[0] == ' ' && rightPromptChunks[0].content[0] != ' ') { rightPromptChunks[0].content = ' ' + rightPromptChunks[0].content; - setPromptChunks(p => [ - ...leftPromptChunks, - ...rightPromptChunks - ]); + chunk = null; + } else if (rightPromptChunks[0].type === 'user') { + if (rightPromptChunks[0].content === chunk.content) { + rightPromptChunks[0] = chunk; + chunk = null; + } else if (rightPromptChunks[0].content.startsWith(chunk.content)) { + rightPromptChunks[0].content = rightPromptChunks[0].content.substring(chunk.content.length); + } + } else { + chunk = null; } - return false; + stop = true; } } leftPromptChunks = [ ...leftPromptChunks, - chunk + ...(chunk ? [chunk] : []) ]; setPromptChunks(p => [ ...leftPromptChunks, ...rightPromptChunks ]); setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); - return true; + return !stop; }); return true; } From f19cdb23cca63e76c7cbf60abb5ca2f21020dd89 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Fri, 24 May 2024 00:02:57 -0300 Subject: [PATCH 06/24] Little code improvement --- mikupad.html | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/mikupad.html b/mikupad.html index 868a07d..a537356 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4041,36 +4041,38 @@ predict(finalPrompt, leftPromptChunks.length, (chunk) => { let stop = false; + let omitChunk = false; + if (rightPromptChunks[0]) { const trimmedChunk = chunk.content.replace(/^ +| +$/gm, "") - if (trimmedChunk.startsWith(rightPromptChunks[0].content[0])) { + if (trimmedChunk[0] == rightPromptChunks[0].content[0]) { + omitChunk = true; if (chunk.content[0] == ' ' && rightPromptChunks[0].content[0] != ' ') { rightPromptChunks[0].content = ' ' + rightPromptChunks[0].content; - chunk = null; } else if (rightPromptChunks[0].type === 'user') { if (rightPromptChunks[0].content === chunk.content) { rightPromptChunks[0] = chunk; - chunk = null; } else if (rightPromptChunks[0].content.startsWith(chunk.content)) { rightPromptChunks[0].content = rightPromptChunks[0].content.substring(chunk.content.length); + omitChunk = false; } - } else { - chunk = null; } stop = true; } } - leftPromptChunks = [ - ...leftPromptChunks, - ...(chunk ? [chunk] : []) - ]; + + if (!omitChunk) { + setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); + leftPromptChunks.push(chunk); + } setPromptChunks(p => [ ...leftPromptChunks, ...rightPromptChunks ]); - setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); + return !stop; }); + return true; } From e61f5e2fcfd3fa9d679619e16283c47a9c632850 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Fri, 24 May 2024 17:58:46 -0300 Subject: [PATCH 07/24] Keep replacing until mismatch --- mikupad.html | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/mikupad.html b/mikupad.html index a537356..c6b7540 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4039,28 +4039,46 @@ const additionalContextPrompt = assembleAdditionalContext(assembledWorldInfo, promptText); const finalPrompt = assembleFinalPrompt(additionalContextPrompt, templateReplacements); + let isReplacing = false; predict(finalPrompt, leftPromptChunks.length, (chunk) => { let stop = false; let omitChunk = false; + let wasReplacing = isReplacing; + isReplacing = false; + if (rightPromptChunks[0]) { - const trimmedChunk = chunk.content.replace(/^ +| +$/gm, "") - if (trimmedChunk[0] == rightPromptChunks[0].content[0]) { + const trimmedChunk = chunk.content.replace(/^ /, ""); + if (rightPromptChunks[0].content[0] === chunk.content[0] || + rightPromptChunks[0].content[0] === trimmedChunk[0] + ) { omitChunk = true; if (chunk.content[0] == ' ' && rightPromptChunks[0].content[0] != ' ') { + if (rightPromptChunks[0].type !== 'user') + console.warn("Predicted token changed, this shouldn't happen."); rightPromptChunks[0].content = ' ' + rightPromptChunks[0].content; - } else if (rightPromptChunks[0].type === 'user') { - if (rightPromptChunks[0].content === chunk.content) { - rightPromptChunks[0] = chunk; - } else if (rightPromptChunks[0].content.startsWith(chunk.content)) { + } + if (rightPromptChunks[0].type === 'user') { + if (rightPromptChunks[0].content.startsWith(chunk.content)) { rightPromptChunks[0].content = rightPromptChunks[0].content.substring(chunk.content.length); + if (!rightPromptChunks[0].content) + rightPromptChunks.shift(); omitChunk = false; + isReplacing = true; } } stop = true; } } - + + // When replacing, we continue until any mismatch. + if (wasReplacing && !isReplacing) { + if (rightPromptChunks.length !== 0) + return false; + // This means that the mismatch was caused by the end of the chunks to be replaced. + isReplacing = false; + } + if (!omitChunk) { setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); leftPromptChunks.push(chunk); @@ -4070,7 +4088,7 @@ ...rightPromptChunks ]); - return !stop; + return !stop || isReplacing; }); return true; From 53fd82b137483dc7976589fea52f3f692fa8e6d1 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Sat, 1 Jun 2024 12:50:17 -0300 Subject: [PATCH 08/24] Fix rebase --- mikupad.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mikupad.html b/mikupad.html index c6b7540..cc5ac87 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4037,7 +4037,7 @@ const promptText = joinPrompt(leftPromptChunks); const assembledWorldInfo = assembleWorldInfo(promptText); const additionalContextPrompt = assembleAdditionalContext(assembledWorldInfo, promptText); - const finalPrompt = assembleFinalPrompt(additionalContextPrompt, templateReplacements); + const finalPrompt = replacePlaceholders(additionalContextPrompt, templateReplacements); let isReplacing = false; predict(finalPrompt, leftPromptChunks.length, (chunk) => { From 9825c56c9a7c28f6fa085aa6ff125980101e8533 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Mon, 20 May 2024 10:23:44 -0300 Subject: [PATCH 09/24] Implement basic fill-in-the-middle support --- mikupad.html | 226 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 164 insertions(+), 62 deletions(-) diff --git a/mikupad.html b/mikupad.html index a674e17..69c6af2 100644 --- a/mikupad.html +++ b/mikupad.html @@ -3882,8 +3882,7 @@ const promptText = useMemo(() => joinPrompt(promptChunks), [promptChunks]); - // compute separately as I imagine this can get expensive - const assembledWorldInfo = useMemo(() => { + const assembleWorldInfo = (promptText) => { // assemble non-empty wi const validWorldInfo = !Array.isArray(worldInfo.entries) ? [] : worldInfo.entries.filter(entry => entry.keys.length > 0 && !(entry.keys.length == 1 && entry.keys[0] == "") && entry.text !== ""); @@ -3918,16 +3917,19 @@ }); }); - const assembledWorldInfo = activeWorldInfo.length > 0 + return activeWorldInfo.length > 0 ? activeWorldInfo.map(entry => entry.text).join("\n") : ""; + }; - return assembledWorldInfo + // compute separately as I imagine this can get expensive + const assembledWorldInfo = useMemo(() => { + return assembleWorldInfo(promptText); }, [worldInfo]); - const additionalContextPrompt = useMemo(() => { - // add world info to memory for easier assembly - memoryTokens["worldInfo"] = assembledWorldInfo; + const assembleAdditionalContext = (assembledWorldInfo, promptText) => { + if ("worldInfo" in memoryTokens) + delete memoryTokens["worldInfo"]; const order = ["prefix","text","suffix"] const assembledAuthorNote = authorNoteTokens.text && authorNoteTokens.text !== "" @@ -3936,19 +3938,19 @@ // replacements for the contextOrder string const contextReplacements = { - "{wiPrefix}": memoryTokens.worldInfo && memoryTokens.worldInfo !== "" + "{wiPrefix}": assembledWorldInfo && assembledWorldInfo !== "" ? worldInfo.prefix : "", // wi prefix and suffix will be added whenever wi isn't empty - "{wiText}": memoryTokens.worldInfo, - "{wiSuffix}": memoryTokens.worldInfo && memoryTokens.worldInfo !== "" + "{wiText}": assembledWorldInfo, + "{wiSuffix}": assembledWorldInfo && assembledWorldInfo !== "" ? worldInfo.suffix : "", - "{memPrefix}": memoryTokens.text && memoryTokens.text !== "" || memoryTokens.worldInfo !== "" + "{memPrefix}": memoryTokens.text && memoryTokens.text !== "" || assembledWorldInfo !== "" ? memoryTokens.prefix : "", // memory prefix and suffix will be added whenever memory or wi aren't empty "{memText}": memoryTokens.text, - "{memSuffix}": memoryTokens.text && memoryTokens.text !== "" || memoryTokens.worldInfo !== "" + "{memSuffix}": memoryTokens.text && memoryTokens.text !== "" || assembledWorldInfo !== "" ? memoryTokens.suffix : "", } @@ -3993,13 +3995,79 @@ }).join("\n").replace(/\\n/g, '\n'); return permContextPrompt; + }; + + const additionalContextPrompt = useMemo(() => { + return assembleAdditionalContext(assembledWorldInfo, promptText); }, [contextLength, promptText, memoryTokens, authorNoteTokens, authorNoteDepth, assembledWorldInfo, worldInfo.prefix, worldInfo.suffix]); const modifiedPrompt = useMemo(() => { - return replacePlaceholders(additionalContextPrompt,templateReplacements); + return replacePlaceholders(additionalContextPrompt, templateReplacements); }, [additionalContextPrompt, templates, selectedTemplate]); - async function predict(prompt = modifiedPrompt, chunkCount = promptChunks.length) { + // predict all {fill} placeholders + async function fillsPredict() { + const fillPlaceholder = "{fill}"; + + let leftPromptChunks = []; + let rightPromptChunks = []; + let fillIdx = undefined; + + for (let i = 0; i < promptChunks.length; i++) { + const chunk = promptChunks[i]; + if (chunk.content.includes(fillPlaceholder)) { + // split the chunk in 2 + const left = { content: chunk.content.substring(0, chunk.content.indexOf(fillPlaceholder)), type: "user" }; + const right = { content: chunk.content.substring(chunk.content.indexOf(fillPlaceholder) + fillPlaceholder.length), type: "user" }; + fillIdx = i + 1; + leftPromptChunks = [ + ...promptChunks.slice(0, Math.max(0, i - 1)), + ...[left] + ]; + rightPromptChunks = [ + ...[right], + ...promptChunks.slice(i + 1, promptChunks.length - 1), + ]; + break; + } + } + + if (!fillIdx) + return; + + const promptText = joinPrompt(leftPromptChunks); + const assembledWorldInfo = assembleWorldInfo(promptText); + const additionalContextPrompt = assembleAdditionalContext(assembledWorldInfo, promptText); + const finalPrompt = assembleFinalPrompt(additionalContextPrompt, templateReplacements); + + predict(finalPrompt, leftPromptChunks.length, (chunk) => { + console.log(chunk); + if (rightPromptChunks[0]) { + if (chunk.content.trim().startsWith(rightPromptChunks[0].content[0])) { + if (chunk.content[0] == ' ' && rightPromptChunks[0].content[0] != ' ') { + rightPromptChunks[0].content = ' ' + rightPromptChunks[0].content; + setPromptChunks(p => [ + ...leftPromptChunks, + ...rightPromptChunks + ]); + } + return false; + } + } + leftPromptChunks = [ + ...leftPromptChunks, + chunk + ]; + setPromptChunks(p => [ + ...leftPromptChunks, + ...rightPromptChunks + ]); + setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); + return true; + }); + } + + async function predict(prompt = modifiedPrompt, chunkCount = promptChunks.length, callback = undefined) { if (cancel) { cancel?.(); @@ -4009,7 +4077,7 @@ setCancel(() => () => cancelled = true); await new Promise(resolve => setTimeout(resolve, 500)); if (cancelled) - return; + return false; } const ac = new AbortController(); @@ -4029,34 +4097,36 @@ // so let's set the predictStartTokens beforehand. setPredictStartTokens(tokens); - const tokenCount = await getTokenCount({ - endpoint, - endpointAPI, - ...(endpointAPI == 3 || endpointAPI == 0 ? { endpointAPIKey } : {}), - content: prompt, - signal: ac.signal, - ...(isMikupadEndpoint ? { proxyEndpoint: sessionStorage.proxyEndpoint } : {}) - }); - setTokens(tokenCount); - setPredictStartTokens(tokenCount); - - // Chat Mode - if (chatMode && !restartedPredict && templates[selectedTemplate]) { - // add user EOT template (instruct suffix) if not switch completion - const { instSuf, instPre } = replaceNewlines(templates[selectedTemplate]); - const instSufIndex = instSuf ? prompt.lastIndexOf(instSuf) : -1; - const instPreIndex = instPre ? prompt.lastIndexOf(instPre) : -1; - if (instSufIndex <= instPreIndex) { - setPromptChunks(p => [...p, { type: 'user', content: instSuf }]) - prompt += instSuf; + if (!callback) { + const tokenCount = await getTokenCount({ + endpoint, + endpointAPI, + ...(endpointAPI == 3 || endpointAPI == 0 ? { endpointAPIKey } : {}), + content: prompt, + signal: ac.signal, + ...(isMikupadEndpoint ? { proxyEndpoint: sessionStorage.proxyEndpoint } : {}) + }); + setTokens(tokenCount); + setPredictStartTokens(tokenCount); + + // Chat Mode + if (chatMode && !restartedPredict && templates[selectedTemplate]) { + // add user EOT template (instruct suffix) if not switch completion + const { instSuf, instPre } = replaceNewlines(templates[selectedTemplate]); + const instSufIndex = instSuf ? prompt.lastIndexOf(instSuf) : -1; + const instPreIndex = instPre ? prompt.lastIndexOf(instPre) : -1; + if (instSufIndex <= instPreIndex) { + setPromptChunks(p => [...p, { type: 'user', content: instSuf }]) + prompt += instSuf; + } } - } - setRestartedPredict(false) + setRestartedPredict(false) - while (undoStack.current.at(-1) >= chunkCount) - undoStack.current.pop(); - undoStack.current.push(chunkCount); - redoStack.current = []; + while (undoStack.current.at(-1) >= chunkCount) + undoStack.current.pop(); + undoStack.current.push(chunkCount); + redoStack.current = []; + } setUndoHovered(false); setRejectedAPIKey(false); promptArea.current.scrollTarget = undefined; @@ -4109,8 +4179,13 @@ chunk.content = chunk.stopping_word; if (!chunk.content) continue; - setPromptChunks(p => [...p, chunk]); - setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); + if (callback) { + if (!callback(chunk)) + break; + } else { + setPromptChunks(p => [...p, chunk]); + setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); + } chunkCount += 1; } } catch (e) { @@ -4129,16 +4204,21 @@ return false; } finally { setCancel(c => c === cancelThis ? null : c); - if (undoStack.current.at(-1) === chunkCount) - undoStack.current.pop(); + if (!callback) { + if (undoStack.current.at(-1) === chunkCount) + undoStack.current.pop(); + } } + // Chat Mode - if (chatMode) { + if (!callback && chatMode) { // add bot EOT template (instruct prefix) const eotBot = templates[selectedTemplate]?.instPre.replace(/\\n/g, '\n') setPromptChunks(p => [...p, { type: 'user', content: eotBot }]) prompt += `${eotBot}` } + + return true; } function undo() { @@ -4366,7 +4446,7 @@ switch (`${altKey}:${ctrlKey}:${shiftKey}:${key}`) { case 'false:false:true:Enter': case 'false:true:false:Enter': - predict(); + fillsPredict();//predict(); break; case 'false:false:false:Escape': cancel(); @@ -4513,28 +4593,50 @@ newValue = newValue.slice(0, -chunk.content.length); } + // Merge chunks if they're from the user + let mergeUserChunks = (chunks, newContent) => { + let lastChunk = chunks[chunks.length - 1]; + while (lastChunk && lastChunk.type === 'user') { + lastChunk.content += newContent; + if (chunks[chunks.length - 2] && chunks[chunks.length - 2].type === 'user') { + newContent = lastChunk.content; + lastChunk = chunks[chunks.length - 2]; + chunks.splice(chunks.length - 1, 1); + } else { + return chunks; + } + } + return [...chunks, { type: 'user', content: newContent }]; + }; + + let newPrompt = [...start]; + if (newValue) { + newPrompt = mergeUserChunks(newPrompt, newValue); + } + if (end.length && end[0].type === 'user') { + newPrompt = mergeUserChunks(newPrompt, end.shift().content); + } + newPrompt.push(...end); + // Remove all undo positions within the modified range. - undoStack.current = undoStack.current.filter(pos => start.length < pos); + undoStack.current = undoStack.current.filter(pos => pos > start.length && pos < newPrompt.length); if (!undoStack.current.length) setUndoHovered(false); - // Update all undo positions. - if (start.length + end.length + (+!!newValue) !== oldPromptLength) { - // Reset redo stack if a new chunk is added/removed at the end. - if (!end.length) - redoStack.current = []; + // Adjust undo/redo stacks. + const chunkDifference = oldPromptLength - newPrompt.length; + undoStack.current = undoStack.current.map(pos => { + if (pos >= start.length) { + return pos - chunkDifference; + } + return pos; + }); - if (!oldPrompt.length) - undoStack.current = undoStack.current.map(pos => pos + 1); - else - undoStack.current = undoStack.current.map(pos => pos - oldPrompt.length); + // Reset redo stack if a new chunk is added/removed at the end. + if (chunkDifference < 0 && !end.length) { + redoStack.current = []; } - const newPrompt = [ - ...start, - ...(newValue ? [{ type: 'user', content: newValue }] : []), - ...end, - ]; return newPrompt; }); } From 6ce6d8ef96588dc8d1627b20b56d61cf7d795bce Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Mon, 20 May 2024 23:33:23 -0300 Subject: [PATCH 10/24] Some fixes --- mikupad.html | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mikupad.html b/mikupad.html index 69c6af2..4a9f4b7 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4019,15 +4019,15 @@ // split the chunk in 2 const left = { content: chunk.content.substring(0, chunk.content.indexOf(fillPlaceholder)), type: "user" }; const right = { content: chunk.content.substring(chunk.content.indexOf(fillPlaceholder) + fillPlaceholder.length), type: "user" }; - fillIdx = i + 1; leftPromptChunks = [ - ...promptChunks.slice(0, Math.max(0, i - 1)), - ...[left] + ...promptChunks.slice(0, i), + left ]; rightPromptChunks = [ - ...[right], + right, ...promptChunks.slice(i + 1, promptChunks.length - 1), ]; + fillIdx = i + 1; break; } } @@ -4041,9 +4041,9 @@ const finalPrompt = assembleFinalPrompt(additionalContextPrompt, templateReplacements); predict(finalPrompt, leftPromptChunks.length, (chunk) => { - console.log(chunk); if (rightPromptChunks[0]) { - if (chunk.content.trim().startsWith(rightPromptChunks[0].content[0])) { + const trimmedChunk = chunk.content.replace(/^ +| +$/gm, "") + if (trimmedChunk.startsWith(rightPromptChunks[0].content[0])) { if (chunk.content[0] == ' ' && rightPromptChunks[0].content[0] != ' ') { rightPromptChunks[0].content = ' ' + rightPromptChunks[0].content; setPromptChunks(p => [ @@ -4125,8 +4125,10 @@ while (undoStack.current.at(-1) >= chunkCount) undoStack.current.pop(); undoStack.current.push(chunkCount); - redoStack.current = []; + } else { + undoStack.current = []; } + redoStack.current = []; setUndoHovered(false); setRejectedAPIKey(false); promptArea.current.scrollTarget = undefined; From 9d7cbaa9883f066a91bdfddc801e9f03b3883b0b Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Thu, 23 May 2024 22:43:02 -0300 Subject: [PATCH 11/24] Update --- mikupad.html | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/mikupad.html b/mikupad.html index 4a9f4b7..de4a8c5 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4005,41 +4005,49 @@ return replacePlaceholders(additionalContextPrompt, templateReplacements); }, [additionalContextPrompt, templates, selectedTemplate]); - // predict all {fill} placeholders - async function fillsPredict() { + // predicts one {fill} placeholder + async function fillPredict() { const fillPlaceholder = "{fill}"; let leftPromptChunks = []; let rightPromptChunks = []; - let fillIdx = undefined; + let foundFillPlaceholder = false; for (let i = 0; i < promptChunks.length; i++) { const chunk = promptChunks[i]; + if (chunk.type !== 'user') + continue; if (chunk.content.includes(fillPlaceholder)) { // split the chunk in 2 - const left = { content: chunk.content.substring(0, chunk.content.indexOf(fillPlaceholder)), type: "user" }; - const right = { content: chunk.content.substring(chunk.content.indexOf(fillPlaceholder) + fillPlaceholder.length), type: "user" }; + foundFillPlaceholder = true; + + let left = chunk.content.substring(0, chunk.content.indexOf(fillPlaceholder)); + if ((left.at(-2) != ' ' || left.at(-2) != '\t') && left.at(-1) == ' ') { + // This is most likely an unintentional mistake by the user. + left = left.substring(0, left.length - 1); + } leftPromptChunks = [ ...promptChunks.slice(0, i), - left + ...(left ? [{ type: 'user', content: left }] : []) ]; + + let right = chunk.content.substring(chunk.content.indexOf(fillPlaceholder) + fillPlaceholder.length); rightPromptChunks = [ - right, + ...(right ? [{ type: 'user', content: right }] : []), ...promptChunks.slice(i + 1, promptChunks.length - 1), ]; - fillIdx = i + 1; break; } } - if (!fillIdx) - return; + if (!foundFillPlaceholder) + return false; const promptText = joinPrompt(leftPromptChunks); const assembledWorldInfo = assembleWorldInfo(promptText); const additionalContextPrompt = assembleAdditionalContext(assembledWorldInfo, promptText); const finalPrompt = assembleFinalPrompt(additionalContextPrompt, templateReplacements); - + predict(finalPrompt, leftPromptChunks.length, (chunk) => { if (rightPromptChunks[0]) { const trimmedChunk = chunk.content.replace(/^ +| +$/gm, "") @@ -4065,6 +4073,7 @@ setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); return true; }); + return true; } async function predict(prompt = modifiedPrompt, chunkCount = promptChunks.length, callback = undefined) { @@ -4080,6 +4089,10 @@ return false; } + // predict the fill placeholder if it exists. + if (!callback && fillPredict()) + return true; + const ac = new AbortController(); const cancelThis = () => { abortCompletion({ @@ -4448,7 +4461,7 @@ switch (`${altKey}:${ctrlKey}:${shiftKey}:${key}`) { case 'false:false:true:Enter': case 'false:true:false:Enter': - fillsPredict();//predict(); + predict(); break; case 'false:false:false:Escape': cancel(); From 67bb0d3c5c6573d2b136e15b5d9b5931b8b91358 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Thu, 23 May 2024 22:48:24 -0300 Subject: [PATCH 12/24] await fillPredict --- mikupad.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mikupad.html b/mikupad.html index de4a8c5..bfe1e5a 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4090,7 +4090,7 @@ } // predict the fill placeholder if it exists. - if (!callback && fillPredict()) + if (!callback && await fillPredict()) return true; const ac = new AbortController(); From ca979ab087cf1de359a5caa15f14219ed5053484 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Thu, 23 May 2024 23:43:23 -0300 Subject: [PATCH 13/24] Make infilling replace parts of existing prompt chunks --- mikupad.html | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/mikupad.html b/mikupad.html index bfe1e5a..4e5bbdc 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4049,29 +4049,36 @@ const finalPrompt = assembleFinalPrompt(additionalContextPrompt, templateReplacements); predict(finalPrompt, leftPromptChunks.length, (chunk) => { + let stop = false; if (rightPromptChunks[0]) { const trimmedChunk = chunk.content.replace(/^ +| +$/gm, "") if (trimmedChunk.startsWith(rightPromptChunks[0].content[0])) { if (chunk.content[0] == ' ' && rightPromptChunks[0].content[0] != ' ') { rightPromptChunks[0].content = ' ' + rightPromptChunks[0].content; - setPromptChunks(p => [ - ...leftPromptChunks, - ...rightPromptChunks - ]); + chunk = null; + } else if (rightPromptChunks[0].type === 'user') { + if (rightPromptChunks[0].content === chunk.content) { + rightPromptChunks[0] = chunk; + chunk = null; + } else if (rightPromptChunks[0].content.startsWith(chunk.content)) { + rightPromptChunks[0].content = rightPromptChunks[0].content.substring(chunk.content.length); + } + } else { + chunk = null; } - return false; + stop = true; } } leftPromptChunks = [ ...leftPromptChunks, - chunk + ...(chunk ? [chunk] : []) ]; setPromptChunks(p => [ ...leftPromptChunks, ...rightPromptChunks ]); setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); - return true; + return !stop; }); return true; } From 6185e9009f18f9417ffe167cf0695d0ad04c7932 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Fri, 24 May 2024 00:02:57 -0300 Subject: [PATCH 14/24] Little code improvement --- mikupad.html | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/mikupad.html b/mikupad.html index 4e5bbdc..78204a1 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4050,36 +4050,38 @@ predict(finalPrompt, leftPromptChunks.length, (chunk) => { let stop = false; + let omitChunk = false; + if (rightPromptChunks[0]) { const trimmedChunk = chunk.content.replace(/^ +| +$/gm, "") - if (trimmedChunk.startsWith(rightPromptChunks[0].content[0])) { + if (trimmedChunk[0] == rightPromptChunks[0].content[0]) { + omitChunk = true; if (chunk.content[0] == ' ' && rightPromptChunks[0].content[0] != ' ') { rightPromptChunks[0].content = ' ' + rightPromptChunks[0].content; - chunk = null; } else if (rightPromptChunks[0].type === 'user') { if (rightPromptChunks[0].content === chunk.content) { rightPromptChunks[0] = chunk; - chunk = null; } else if (rightPromptChunks[0].content.startsWith(chunk.content)) { rightPromptChunks[0].content = rightPromptChunks[0].content.substring(chunk.content.length); + omitChunk = false; } - } else { - chunk = null; } stop = true; } } - leftPromptChunks = [ - ...leftPromptChunks, - ...(chunk ? [chunk] : []) - ]; + + if (!omitChunk) { + setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); + leftPromptChunks.push(chunk); + } setPromptChunks(p => [ ...leftPromptChunks, ...rightPromptChunks ]); - setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); + return !stop; }); + return true; } From d34bc7be879b5b0baba85ecfd0ceb2ae48bcde9e Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Fri, 24 May 2024 17:58:46 -0300 Subject: [PATCH 15/24] Keep replacing until mismatch --- mikupad.html | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/mikupad.html b/mikupad.html index 78204a1..7c2c34c 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4048,28 +4048,46 @@ const additionalContextPrompt = assembleAdditionalContext(assembledWorldInfo, promptText); const finalPrompt = assembleFinalPrompt(additionalContextPrompt, templateReplacements); + let isReplacing = false; predict(finalPrompt, leftPromptChunks.length, (chunk) => { let stop = false; let omitChunk = false; + let wasReplacing = isReplacing; + isReplacing = false; + if (rightPromptChunks[0]) { - const trimmedChunk = chunk.content.replace(/^ +| +$/gm, "") - if (trimmedChunk[0] == rightPromptChunks[0].content[0]) { + const trimmedChunk = chunk.content.replace(/^ /, ""); + if (rightPromptChunks[0].content[0] === chunk.content[0] || + rightPromptChunks[0].content[0] === trimmedChunk[0] + ) { omitChunk = true; if (chunk.content[0] == ' ' && rightPromptChunks[0].content[0] != ' ') { + if (rightPromptChunks[0].type !== 'user') + console.warn("Predicted token changed, this shouldn't happen."); rightPromptChunks[0].content = ' ' + rightPromptChunks[0].content; - } else if (rightPromptChunks[0].type === 'user') { - if (rightPromptChunks[0].content === chunk.content) { - rightPromptChunks[0] = chunk; - } else if (rightPromptChunks[0].content.startsWith(chunk.content)) { + } + if (rightPromptChunks[0].type === 'user') { + if (rightPromptChunks[0].content.startsWith(chunk.content)) { rightPromptChunks[0].content = rightPromptChunks[0].content.substring(chunk.content.length); + if (!rightPromptChunks[0].content) + rightPromptChunks.shift(); omitChunk = false; + isReplacing = true; } } stop = true; } } - + + // When replacing, we continue until any mismatch. + if (wasReplacing && !isReplacing) { + if (rightPromptChunks.length !== 0) + return false; + // This means that the mismatch was caused by the end of the chunks to be replaced. + isReplacing = false; + } + if (!omitChunk) { setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); leftPromptChunks.push(chunk); @@ -4079,7 +4097,7 @@ ...rightPromptChunks ]); - return !stop; + return !stop || isReplacing; }); return true; From 3526dfeedd9d9e907f73ddd5860c670a2d37f0bc Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Sat, 1 Jun 2024 12:50:17 -0300 Subject: [PATCH 16/24] Fix rebase --- mikupad.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mikupad.html b/mikupad.html index 7c2c34c..438ce9c 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4046,7 +4046,7 @@ const promptText = joinPrompt(leftPromptChunks); const assembledWorldInfo = assembleWorldInfo(promptText); const additionalContextPrompt = assembleAdditionalContext(assembledWorldInfo, promptText); - const finalPrompt = assembleFinalPrompt(additionalContextPrompt, templateReplacements); + const finalPrompt = replacePlaceholders(additionalContextPrompt, templateReplacements); let isReplacing = false; predict(finalPrompt, leftPromptChunks.length, (chunk) => { From ad174ba5bb61681db012b29194b3ea0767734356 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Sat, 1 Jun 2024 19:19:20 -0300 Subject: [PATCH 17/24] Add fim template to instruct modal --- mikupad.html | 164 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 125 insertions(+), 39 deletions(-) diff --git a/mikupad.html b/mikupad.html index 438ce9c..6e94274 100644 --- a/mikupad.html +++ b/mikupad.html @@ -595,6 +595,9 @@ .instructmodal-edits .hbox { margin-top:8px; } +.instructmodal-edits .vbox { + margin-top:8px; +} @@ -2669,6 +2672,7 @@ "sysSuf": "", "instPre": "", "instSuf": "", + "fimTemplate": undefined, } return { ...newState } }) @@ -2748,6 +2752,7 @@ "sysSuf": template.affixes.sysSuf, "instPre": template.affixes.instPre, "instSuf": template.affixes.instSuf, + "fimTemplate": template.affixes.fimTemplate, } } return { ...newState } @@ -2888,7 +2893,7 @@ onInput=${e => handleInstructTemplateChange(selectedTemplate,"instPre",e.target.value)} onValueChange=${() => {}}/> - <${InputBox} label="Instruct Suffix {/inst}" + <${InputBox} label="Instruct Suffix {/inst}" placeholder="[/INST]" className="" tooltip="" @@ -2908,7 +2913,7 @@ onInput=${e => handleInstructTemplateChange(selectedTemplate,"sysPre",e.target.value)} onValueChange=${() => {}}/> - <${InputBox} label="System Prompt Suffix {/sys}" + <${InputBox} label="System Prompt Suffix {/sys}" placeholder="<>\n\n" className="" tooltip="" @@ -2917,6 +2922,32 @@ onInput=${e => handleInstructTemplateChange(selectedTemplate,"sysSuf",e.target.value)} onValueChange=${() => {}}/> + +
+
+ <${Checkbox} label="Supports Fill-In-The-Middle" + value=${getArrObjByName(templateList,selectedTemplate)?.affixes.fimTemplate !== undefined} + onValueChange=${(value) => handleInstructTemplateChange(selectedTemplate,"fimTemplate", value ? '' : undefined)}/> + ${getArrObjByName(templateList,selectedTemplate)?.affixes.fimTemplate !== undefined && html` + <${InputBox} label="Fill-In-The-Middle Template" + placeholder="[SUFFIX]{suffix}[PREFIX]{prefix}" + className="" + tooltip="" + readOnly=${!!cancel} + value=${getArrObjByName(templateList,selectedTemplate)?.affixes.fimTemplate || ""} + onInput=${e => handleInstructTemplateChange(selectedTemplate,"fimTemplate",e.target.value)} + onValueChange=${() => {}}/>`} +
+
+ ${getArrObjByName(templateList,selectedTemplate)?.affixes.fimTemplate !== undefined + ? html` +
Use the {fill} placeholder to use the Fill-In-The-Middle template and start the prediction from that point.
+
{prefix} represents the text before the placeholder, and {suffix} represents the text after it.
` + : html` +
This template doesn't have a Fill-In-The-Middle template.
+
You can use the {predict} placeholder to start the prediction from that point, but the model won't be aware of the text after the placeholder.
`} +
+
@@ -3532,6 +3563,38 @@ ); } +/** + * Split a string with a RegExp separator an optionally limited number of times. + * (https://stackoverflow.com/a/64296576) + * @param {string} input + * @param {RegExp} separator + * @param {number} [limit] - If not included, splits the maximum times + * @returns {[string[], string[]]} + */ +function split(input, separator, limit) { + separator = new RegExp(separator, 'g'); + limit = limit ?? -1; + + const output = []; + const separators = []; + let finalIndex = 0; + + while (limit--) { + const lastIndex = separator.lastIndex; + const search = separator.exec(input); + if (search === null) { + break; + } + finalIndex = separator.lastIndex; + output.push(input.slice(lastIndex, search.index)); + separators.push(search[0]); + } + + output.push(input.slice(finalIndex)); + + return [output, separators]; +} + function useSessionState(sessionStorage, name, initialState) { const savedState = useMemo(() => { try { @@ -4008,20 +4071,30 @@ // predicts one {fill} placeholder async function fillPredict() { const fillPlaceholder = "{fill}"; + const predictPlaceholder = "{predict}"; - let leftPromptChunks = []; - let rightPromptChunks = []; + let placeholderRegex = predictPlaceholder; + if (templates[selectedTemplate]?.fimTemplate !== undefined) { + placeholderRegex += `|${fillPlaceholder}`; + } + + let leftPromptChunks = undefined; + let rightPromptChunks = undefined; let foundFillPlaceholder = false; + let foundPredictPlaceholder = false; for (let i = 0; i < promptChunks.length; i++) { const chunk = promptChunks[i]; if (chunk.type !== 'user') continue; - if (chunk.content.includes(fillPlaceholder)) { + + if (chunk.content.includes(fillPlaceholder) || chunk.content.includes(predictPlaceholder)) { // split the chunk in 2 - foundFillPlaceholder = true; + let [sides, separators] = split(chunk.content, placeholderRegex, 1); + foundFillPlaceholder = separators[0] == fillPlaceholder; + foundPredictPlaceholder = separators[0] == predictPlaceholder; - let left = chunk.content.substring(0, chunk.content.indexOf(fillPlaceholder)); + let left = sides[0]; if ((left.at(-2) != ' ' || left.at(-2) != '\t') && left.at(-1) == ' ') { // This is most likely an unintentional mistake by the user. left = left.substring(0, left.length - 1); @@ -4031,7 +4104,7 @@ ...(left ? [{ type: 'user', content: left }] : []) ]; - let right = chunk.content.substring(chunk.content.indexOf(fillPlaceholder) + fillPlaceholder.length); + let right = sides[1]; rightPromptChunks = [ ...(right ? [{ type: 'user', content: right }] : []), ...promptChunks.slice(i + 1, promptChunks.length - 1), @@ -4040,10 +4113,21 @@ } } - if (!foundFillPlaceholder) + if (!foundFillPlaceholder && !foundPredictPlaceholder) return false; - const promptText = joinPrompt(leftPromptChunks); + let promptText; + if (foundFillPlaceholder) { + const prefix = joinPrompt(leftPromptChunks); + const suffix = joinPrompt(rightPromptChunks); + + promptText = replacePlaceholders(templates[selectedTemplate].fimTemplate, { + '{prefix}': prefix, + '{suffix}': suffix + }); + } else { + promptText = joinPrompt(leftPromptChunks); + } const assembledWorldInfo = assembleWorldInfo(promptText); const additionalContextPrompt = assembleAdditionalContext(assembledWorldInfo, promptText); const finalPrompt = replacePlaceholders(additionalContextPrompt, templateReplacements); @@ -4053,39 +4137,41 @@ let stop = false; let omitChunk = false; - let wasReplacing = isReplacing; - isReplacing = false; - - if (rightPromptChunks[0]) { - const trimmedChunk = chunk.content.replace(/^ /, ""); - if (rightPromptChunks[0].content[0] === chunk.content[0] || - rightPromptChunks[0].content[0] === trimmedChunk[0] - ) { - omitChunk = true; - if (chunk.content[0] == ' ' && rightPromptChunks[0].content[0] != ' ') { - if (rightPromptChunks[0].type !== 'user') - console.warn("Predicted token changed, this shouldn't happen."); - rightPromptChunks[0].content = ' ' + rightPromptChunks[0].content; - } - if (rightPromptChunks[0].type === 'user') { - if (rightPromptChunks[0].content.startsWith(chunk.content)) { - rightPromptChunks[0].content = rightPromptChunks[0].content.substring(chunk.content.length); - if (!rightPromptChunks[0].content) - rightPromptChunks.shift(); - omitChunk = false; - isReplacing = true; + if (!foundFillPlaceholder) { + let wasReplacing = isReplacing; + isReplacing = false; + + if (rightPromptChunks[0]) { + const trimmedChunk = chunk.content.replace(/^ /, ""); + if (rightPromptChunks[0].content[0] === chunk.content[0] || + rightPromptChunks[0].content[0] === trimmedChunk[0] + ) { + omitChunk = true; + if (chunk.content[0] == ' ' && rightPromptChunks[0].content[0] != ' ') { + if (rightPromptChunks[0].type !== 'user') + console.warn("Predicted token changed, this shouldn't happen."); + rightPromptChunks[0].content = ' ' + rightPromptChunks[0].content; } + if (rightPromptChunks[0].type === 'user') { + if (rightPromptChunks[0].content.startsWith(chunk.content)) { + rightPromptChunks[0].content = rightPromptChunks[0].content.substring(chunk.content.length); + if (!rightPromptChunks[0].content) + rightPromptChunks.shift(); + omitChunk = false; + isReplacing = true; + } + } + stop = true; } - stop = true; } - } - // When replacing, we continue until any mismatch. - if (wasReplacing && !isReplacing) { - if (rightPromptChunks.length !== 0) - return false; - // This means that the mismatch was caused by the end of the chunks to be replaced. - isReplacing = false; + // When replacing, we continue until any mismatch. + if (wasReplacing && !isReplacing) { + if (rightPromptChunks.length !== 0) + return false; + // This means that the mismatch was caused by the end of the chunks to be replaced. + isReplacing = false; + } } if (!omitChunk) { From 9468dedad4438664afae5f562a423ee01488adf5 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Sat, 1 Jun 2024 19:50:22 -0300 Subject: [PATCH 18/24] Remove automatic stopping --- mikupad.html | 53 +++++----------------------------------------------- 1 file changed, 5 insertions(+), 48 deletions(-) diff --git a/mikupad.html b/mikupad.html index 5a60073..a995401 100644 --- a/mikupad.html +++ b/mikupad.html @@ -4114,7 +4114,7 @@ let right = sides[1]; rightPromptChunks = [ ...(right ? [{ type: 'user', content: right }] : []), - ...promptChunks.slice(i + 1, promptChunks.length - 1), + ...promptChunks.slice(i + 1, promptChunks.length), ]; break; } @@ -4135,62 +4135,19 @@ } else { promptText = joinPrompt(leftPromptChunks); } + const assembledWorldInfo = assembleWorldInfo(promptText); const additionalContextPrompt = assembleAdditionalContext(assembledWorldInfo, promptText); const finalPrompt = replacePlaceholders(additionalContextPrompt, templateReplacements); - let isReplacing = false; predict(finalPrompt, leftPromptChunks.length, (chunk) => { - let stop = false; - let omitChunk = false; - - if (!foundFillPlaceholder) { - let wasReplacing = isReplacing; - isReplacing = false; - - if (rightPromptChunks[0]) { - const trimmedChunk = chunk.content.replace(/^ /, ""); - if (rightPromptChunks[0].content[0] === chunk.content[0] || - rightPromptChunks[0].content[0] === trimmedChunk[0] - ) { - omitChunk = true; - if (chunk.content[0] == ' ' && rightPromptChunks[0].content[0] != ' ') { - if (rightPromptChunks[0].type !== 'user') - console.warn("Predicted token changed, this shouldn't happen."); - rightPromptChunks[0].content = ' ' + rightPromptChunks[0].content; - } - if (rightPromptChunks[0].type === 'user') { - if (rightPromptChunks[0].content.startsWith(chunk.content)) { - rightPromptChunks[0].content = rightPromptChunks[0].content.substring(chunk.content.length); - if (!rightPromptChunks[0].content) - rightPromptChunks.shift(); - omitChunk = false; - isReplacing = true; - } - } - stop = true; - } - } - - // When replacing, we continue until any mismatch. - if (wasReplacing && !isReplacing) { - if (rightPromptChunks.length !== 0) - return false; - // This means that the mismatch was caused by the end of the chunks to be replaced. - isReplacing = false; - } - } - - if (!omitChunk) { - setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); - leftPromptChunks.push(chunk); - } + leftPromptChunks.push(chunk); setPromptChunks(p => [ ...leftPromptChunks, ...rightPromptChunks ]); - - return !stop || isReplacing; + setTokens(t => t + (chunk?.completion_probabilities?.length ?? 1)); + return true; }); return true; From c67c9ce938bf033c582cccad7c33a00727254b74 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Sat, 1 Jun 2024 20:19:31 -0300 Subject: [PATCH 19/24] Use better regex split --- mikupad.html | 52 ++++++++++++++++++++-------------------------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/mikupad.html b/mikupad.html index a995401..f08c921 100644 --- a/mikupad.html +++ b/mikupad.html @@ -3570,36 +3570,24 @@ ); } -/** - * Split a string with a RegExp separator an optionally limited number of times. - * (https://stackoverflow.com/a/64296576) - * @param {string} input - * @param {RegExp} separator - * @param {number} [limit] - If not included, splits the maximum times - * @returns {[string[], string[]]} - */ -function split(input, separator, limit) { - separator = new RegExp(separator, 'g'); - limit = limit ?? -1; - - const output = []; - const separators = []; - let finalIndex = 0; - - while (limit--) { - const lastIndex = separator.lastIndex; - const search = separator.exec(input); - if (search === null) { - break; - } - finalIndex = separator.lastIndex; - output.push(input.slice(lastIndex, search.index)); - separators.push(search[0]); - } - - output.push(input.slice(finalIndex)); - - return [output, separators]; +function regexSplitString(str, separator, limit) { + const result = []; + const separators = []; + let lastIndex = 0; + let match; + const regex = new RegExp(separator, 'g'); + + while ((match = regex.exec(str)) !== null) { + if (limit !== undefined && result.length >= limit) break; + + result.push(str.slice(lastIndex, match.index)); + separators.push(match[0]); + lastIndex = match.index + match[0].length; + } + + result.push(str.slice(lastIndex)); // Add the remainder of the string + + return [result, separators]; } function useSessionState(sessionStorage, name, initialState) { @@ -4097,7 +4085,7 @@ if (chunk.content.includes(fillPlaceholder) || chunk.content.includes(predictPlaceholder)) { // split the chunk in 2 - let [sides, separators] = split(chunk.content, placeholderRegex, 1); + let [sides, separators] = regexSplitString(chunk.content, placeholderRegex, 1); foundFillPlaceholder = separators[0] == fillPlaceholder; foundPredictPlaceholder = separators[0] == predictPlaceholder; @@ -4135,7 +4123,7 @@ } else { promptText = joinPrompt(leftPromptChunks); } - + const assembledWorldInfo = assembleWorldInfo(promptText); const additionalContextPrompt = assembleAdditionalContext(assembledWorldInfo, promptText); const finalPrompt = replacePlaceholders(additionalContextPrompt, templateReplacements); From 7e5e2952a2a19938691daf05610c63fa2f88e6dc Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Sat, 1 Jun 2024 22:36:40 -0300 Subject: [PATCH 20/24] Cache FIM prompt and show it in the Context Modal --- mikupad.html | 52 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/mikupad.html b/mikupad.html index f08c921..7ca84df 100644 --- a/mikupad.html +++ b/mikupad.html @@ -2076,7 +2076,7 @@ `; } -function ContextModal({ isOpen, closeModal, tokens, memoryTokens, authorNoteTokens, handleMemoryTokensChange, modifiedPrompt, defaultPresets, cancel }) { +function ContextModal({ isOpen, closeModal, tokens, memoryTokens, authorNoteTokens, handleMemoryTokensChange, modifiedPrompt, fimPromptInfo, defaultPresets, cancel }) { return html` <${Modal} isOpen=${isOpen} onClose=${closeModal} title="Context" @@ -2155,7 +2155,7 @@