diff --git a/core/.gitignore b/core/.gitignore index ad1c543342..7d8f1fa533 100644 --- a/core/.gitignore +++ b/core/.gitignore @@ -4,4 +4,5 @@ target npm-debug.log* .env .continue-test -testDir \ No newline at end of file +testDir +coverage \ No newline at end of file diff --git a/core/autocomplete/completionProvider.ts b/core/autocomplete/completionProvider.ts index c67ce1f0af..39d2d1e736 100644 --- a/core/autocomplete/completionProvider.ts +++ b/core/autocomplete/completionProvider.ts @@ -37,7 +37,7 @@ import { import { isOnlyPunctuationAndWhitespace } from "./filter.js"; import { AutocompleteLanguageInfo } from "./languages.js"; import { - avoidPathLine, + avoidPathLineAndEmptyComments, noTopLevelKeywordsMidline, skipPrefixes, stopAtLines, @@ -478,9 +478,9 @@ export class CompletionProvider { const lines = fullPrefix.split("\n"); fullPrefix = `${lines.slice(0, -1).join("\n")}\n${ lang.singleLineComment - } ${input.injectDetails.split("\n").join(`\n${lang.singleLineComment} `)}\n${ - lines[lines.length - 1] - }`; + } ${input.injectDetails + .split("\n") + .join(`\n${lang.singleLineComment} `)}\n${lines[lines.length - 1]}`; } const fullSuffix = getRangeInString(fileContents, { @@ -581,7 +581,10 @@ export class CompletionProvider { prefix = `${formattedSnippets}\n\n${prefix}`; } else if (prefix.trim().length === 0 && suffix.trim().length === 0) { // If it's an empty file, include the file name as a comment - prefix = `${lang.singleLineComment} ${getLastNPathParts(filepath, 2)}\n${prefix}`; + prefix = `${lang.singleLineComment} ${getLastNPathParts( + filepath, + 2, + )}\n${prefix}`; } prompt = compiledTemplate({ @@ -689,7 +692,10 @@ export class CompletionProvider { let lineGenerator = streamLines(charGenerator); lineGenerator = stopAtLines(lineGenerator, fullStop); lineGenerator = stopAtRepeatingLines(lineGenerator, fullStop); - lineGenerator = avoidPathLine(lineGenerator, lang.singleLineComment); + lineGenerator = avoidPathLineAndEmptyComments( + lineGenerator, + lang.singleLineComment, + ); lineGenerator = skipPrefixes(lineGenerator); lineGenerator = noTopLevelKeywordsMidline( lineGenerator, diff --git a/core/autocomplete/lineStream.test.ts b/core/autocomplete/lineStream.test.ts new file mode 100644 index 0000000000..df8cd1263f --- /dev/null +++ b/core/autocomplete/lineStream.test.ts @@ -0,0 +1,421 @@ +import { jest } from "@jest/globals"; +import * as lineStream from "./lineStream"; + +describe("lineStream", () => { + let mockFullStop: jest.Mock; + + async function getLineGenerator(lines: any) { + return (async function* () { + for (const line of lines) { + yield line; + } + })(); + } + + async function getFilteredLines(results: any) { + const output = []; + + for await (const line of results) { + output.push(line); + } + + return output; + } + + beforeEach(() => { + mockFullStop = jest.fn(); + }); + + describe("noTopLevelKeywordsMidline", () => { + it.todo("Need some sample inputs to properly test this"); + }); + + describe("avoidPathLine", () => { + it("should filter out path lines and empty comments", async () => { + const linesGenerator = await getLineGenerator([ + "// Path: src/index.ts", + "const x = 5;", + "//", + "console.log(x);", + ]); + + const result = lineStream.avoidPathLineAndEmptyComments( + linesGenerator, + "//", + ); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["const x = 5;", "console.log(x);"]); + }); + }); + + describe("streamWithNewLines", () => { + it("should add newlines between lines", async () => { + const linesGenerator = await getLineGenerator([ + "line1", + "line2", + "line3", + ]); + + const result = lineStream.streamWithNewLines(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["line1", "\n", "line2", "\n", "line3"]); + }); + }); + + describe("lineIsRepeated", () => { + it("should return true for similar lines", () => { + expect(lineStream.lineIsRepeated("const x = 5;", "const x = 6;")).toBe( + true, + ); + }); + + it("should return false for different lines", () => { + expect(lineStream.lineIsRepeated("const x = 5;", "let y = 10;")).toBe( + false, + ); + }); + + it("should return false for short lines", () => { + expect(lineStream.lineIsRepeated("x=5", "x=6")).toBe(false); + }); + }); + + describe("stopAtSimilarLine", () => { + it("should stop at the exact same line", async () => { + const lineToTest = "const x = 6;"; + const linesGenerator = await getLineGenerator([ + "console.log();", + "const y = () => {};", + lineToTest, + ]); + + const result = lineStream.stopAtSimilarLine( + linesGenerator, + lineToTest, + mockFullStop, + ); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["console.log();", "const y = () => {};"]); + expect(mockFullStop).toHaveBeenCalledTimes(1); + }); + + it("should stop at a similar line", async () => { + const lineToTest = "const x = 6;"; + const linesGenerator = await getLineGenerator([ + "console.log();", + "const y = () => {};", + lineToTest, + ]); + + const result = lineStream.stopAtSimilarLine( + linesGenerator, + "a" + lineToTest, + mockFullStop, + ); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["console.log();", "const y = () => {};"]); + expect(mockFullStop).toHaveBeenCalledTimes(1); + }); + + it("should continue on bracket ending lines", async () => { + const linesGenerator = await getLineGenerator([ + " if (x > 0) {", + " console.log(x);", + " }", + ]); + + const result = lineStream.stopAtSimilarLine( + linesGenerator, + "}", + mockFullStop, + ); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual([ + " if (x > 0) {", + " console.log(x);", + " }", + ]); + expect(mockFullStop).toHaveBeenCalledTimes(0); + }); + }); + + describe("stopAtLines", () => { + it("should stop at specified lines", async () => { + const linesGenerator = await getLineGenerator([ + "const x = 5;", + "let y = 10;", + lineStream.LINES_TO_STOP_AT[0], + "const z = 15;", + ]); + + const result = lineStream.stopAtLines(linesGenerator, mockFullStop); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["const x = 5;", "let y = 10;"]); + expect(mockFullStop).toHaveBeenCalledTimes(1); + }); + }); + + describe("skipPrefixes", () => { + it("should skip specified prefixes", async () => { + const linesGenerator = await getLineGenerator([ + `${lineStream.PREFIXES_TO_SKIP[0]}const x = 5;`, + "let y = 10;", + ]); + + const result = lineStream.skipPrefixes(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["const x = 5;", "let y = 10;"]); + }); + }); + + describe("skipLines", () => { + it("should skip specified lines", async () => { + const linesGenerator = await getLineGenerator([ + `${lineStream.LINES_TO_SKIP[0]}const x = 5;`, + "let y = 10;", + ]); + + const result = lineStream.skipLines(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["let y = 10;"]); + }); + }); + + describe("filterCodeBlockLines", () => { + it("should remove lines before the first valid line", async () => { + const linesGenerator = await getLineGenerator(["```ts", "const x = 5;"]); + + const result = lineStream.filterCodeBlockLines(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["const x = 5;"]); + }); + + it.todo("Need some sample inputs to properly test this"); + }); + + describe("filterEnglishLinesAtStart", () => { + it("should skip initial empty line", async () => { + const linesGenerator = await getLineGenerator([ + "", + "const x = 5;", + "let y = 10;", + ]); + + const result = lineStream.filterEnglishLinesAtStart(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["const x = 5;", "let y = 10;"]); + }); + + it("should filter out English first line", async () => { + const linesGenerator = await getLineGenerator([ + lineStream.ENGLISH_START_PHRASES[0], + "let y = 10;", + ]); + + const result = lineStream.filterEnglishLinesAtStart(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["let y = 10;"]); + }); + + it("should not filter out non-English first line", async () => { + const linesGenerator = await getLineGenerator([ + "const x = 5;", + "let y = 10;", + ]); + + const result = lineStream.filterEnglishLinesAtStart(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["const x = 5;", "let y = 10;"]); + }); + + it("should filter out empty newline after first line english word", async () => { + const linesGenerator = await getLineGenerator([ + lineStream.ENGLISH_START_PHRASES[0], + "", + "const x = 5;", + ]); + + const result = lineStream.filterEnglishLinesAtStart(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["const x = 5;"]); + }); + + it("should filter out sentences ending in a semi-colon that are not code keywords", async () => { + const linesGenerator = await getLineGenerator([ + "a" + lineStream.CODE_KEYWORDS_ENDING_IN_SEMICOLON[0] + ":", + "const x = 5;", + ]); + + const result = lineStream.filterEnglishLinesAtStart(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["const x = 5;"]); + }); + + it("should not filter out sentences ending in a semi-colon that are code keywords", async () => { + const keyword = lineStream.CODE_KEYWORDS_ENDING_IN_SEMICOLON[0] + ":"; + + const linesGenerator = await getLineGenerator([keyword, "const x = 5;"]); + + const result = lineStream.filterEnglishLinesAtStart(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual([keyword, "const x = 5;"]); + }); + }); + + describe("filterEnglishLinesAtEnd", () => { + it("should stop at English explanation after code block", async () => { + const linesGenerator = await getLineGenerator([ + "const x = 5;", + "```", + lineStream.ENGLISH_POST_PHRASES[0], + ]); + + const result = lineStream.filterEnglishLinesAtEnd(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["const x = 5;", "```"]); + }); + }); + + describe("fixCodeLlamaFirstLineIndentation", () => { + it("should fix indentation of the first line", async () => { + const linesGenerator = await getLineGenerator([ + " const x = 5;", + "let y = 10;", + ]); + + const result = + lineStream.fixCodeLlamaFirstLineIndentation(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["const x = 5;", "let y = 10;"]); + }); + }); + + describe("filterLeadingAndTrailingNewLineInsertion", () => { + it("should ignore 'useless' new first lines", async () => { + const linesGenerator = await getLineGenerator([ + { type: "new", line: lineStream.USELESS_LINES[0] }, + { type: "new", line: "const x = 5;" }, + ]); + + const result = + lineStream.filterLeadingAndTrailingNewLineInsertion(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual([{ type: "new", line: "const x = 5;" }]); + }); + + it("should handle preserve newlines chars in old lines", async () => { + const linesGenerator = await getLineGenerator([ + { type: "new", line: "const x = 5;" }, + { type: "old", line: "let y = 10;" }, + { type: "old", line: "" }, + { type: "new", line: "const z = 15;" }, + ]); + + const result = + lineStream.filterLeadingAndTrailingNewLineInsertion(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual([ + { type: "new", line: "const x = 5;" }, + { type: "old", line: "let y = 10;" }, + { type: "old", line: "" }, + { type: "new", line: "const z = 15;" }, + ]); + }); + + it("should filter leading and trailing new line insertions", async () => { + const linesGenerator = await getLineGenerator([ + { type: "new", line: "" }, + { type: "new", line: "const x = 5;" }, + { type: "new", line: "let y = 10;" }, + { type: "new", line: "" }, + ]); + + const result = + lineStream.filterLeadingAndTrailingNewLineInsertion(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual([ + { type: "new", line: "const x = 5;" }, + { type: "new", line: "let y = 10;" }, + ]); + }); + + it("should buffer newline chars and then render them if we encounter another non-empty line", async () => { + const linesGenerator = await getLineGenerator([ + { type: "new", line: "const x = 5;" }, + { type: "new", line: "" }, + { type: "new", line: "let y = 10;" }, + ]); + + const result = + lineStream.filterLeadingAndTrailingNewLineInsertion(linesGenerator); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual([ + { type: "new", line: "const x = 5;" }, + { type: "new", line: "" }, + { type: "new", line: "let y = 10;" }, + ]); + }); + }); + + describe("stopAtRepeatingLines", () => { + it("should handle non-repeating lines correctly", async () => { + const linesGenerator = await getLineGenerator([ + "const x = 5;", + "let y = 10;", + "const z = 15;", + ]); + + const result = lineStream.stopAtRepeatingLines( + linesGenerator, + mockFullStop, + ); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual([ + "const x = 5;", + "let y = 10;", + "const z = 15;", + ]); + }); + + it("should stop at repeating lines", async () => { + const linesGenerator = await getLineGenerator([ + "const x = 5;", + "let y = 10;", + "let y = 10;", + "let y = 10;", + "const z = 15;", + ]); + + const result = lineStream.stopAtRepeatingLines( + linesGenerator, + mockFullStop, + ); + const filteredLines = await getFilteredLines(result); + + expect(filteredLines).toEqual(["const x = 5;", "let y = 10;"]); + expect(mockFullStop).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/core/autocomplete/lineStream.ts b/core/autocomplete/lineStream.ts index 674f0d46e2..6c8b9fd3a7 100644 --- a/core/autocomplete/lineStream.ts +++ b/core/autocomplete/lineStream.ts @@ -1,12 +1,106 @@ import { distance } from "fastest-levenshtein"; -import { LineStream } from "../diff/util.js"; -import { DiffLine } from "../index.js"; +import { LineStream } from "../diff/util"; +import { DiffLine } from "../"; export type LineFilter = (args: { lines: LineStream; fullStop: () => void; }) => LineStream; +function isBracketEnding(line: string): boolean { + return line + .trim() + .split("") + .some((char) => BRACKET_ENDING_CHARS.includes(char)); +} + +function commonPrefixLength(a: string, b: string): number { + let i = 0; + while (i < a.length && i < b.length && a[i] === b[i]) { + i++; + } + return i; +} + +function isEnglishFirstLine(line: string) { + line = line.trim().toLowerCase(); + + if ( + line.endsWith(":") && + !CODE_KEYWORDS_ENDING_IN_SEMICOLON.some((keyword) => + line.startsWith(keyword), + ) + ) { + return true; + } + + return ENGLISH_START_PHRASES.some((phrase) => line.startsWith(phrase)); +} + +function isEnglishPostExplanation(line: string): boolean { + const lower = line.toLowerCase(); + return ENGLISH_POST_PHRASES.some((phrase) => lower.startsWith(phrase)); +} + +function shouldRemoveLineBeforeStart(line: string): boolean { + return ( + line.trimStart().startsWith("```") || + LINES_TO_REMOVE_BEFORE_START.some((l) => line.trim() === l) + ); +} + +function shouldChangeLineAndStop(line: string): string | undefined { + if (line.trimStart() === "```") { + return line; + } + + if (line.includes(CODE_START_BLOCK)) { + return line.split(CODE_START_BLOCK)[0].trimEnd(); + } + + return undefined; +} + +function isUselessLine(line: string): boolean { + const trimmed = line.trim().toLowerCase(); + const hasUselessLine = USELESS_LINES.some( + (uselessLine) => trimmed === uselessLine, + ); + + return hasUselessLine || trimmed.startsWith("// end"); +} + +export const USELESS_LINES = ["", "```"]; +export const CODE_KEYWORDS_ENDING_IN_SEMICOLON = ["def"]; +export const CODE_START_BLOCK = "[/CODE]"; +export const BRACKET_ENDING_CHARS = [")", "]", "}", ";"]; +export const PREFIXES_TO_SKIP = [""]; +export const LINES_TO_STOP_AT = ["# End of file.", ""]; +export const LINES_TO_REMOVE_BEFORE_START = [ + "", + "[CODE]", + "", +]; +export const ENGLISH_START_PHRASES = [ + "here is", + "here's", + "sure, here", + "sure thing", + "sure!", + "to fill", + "certainly", + "of course", + "the code should", +]; + +export const ENGLISH_POST_PHRASES = [ + "explanation:", + "here is", + "here's how", + "the above", +]; + export async function* noTopLevelKeywordsMidline( lines: LineStream, topLevelKeywords: string[], @@ -15,6 +109,7 @@ export async function* noTopLevelKeywordsMidline( for await (const line of lines) { for (const keyword of topLevelKeywords) { const indexOf = line.indexOf(`${keyword} `); + // TODO: What is this second clause for? if (indexOf >= 0 && line.slice(indexOf - 1, indexOf).trim() !== "") { yield line.slice(0, indexOf); fullStop(); @@ -25,7 +120,14 @@ export async function* noTopLevelKeywordsMidline( } } -export async function* avoidPathLine( +/** + * Filters out unwanted lines from a LineStream, specifically those starting with '// Path: ' or empty comments. + * + * @param {LineStream} stream - The input stream of lines to filter. + * @param {string} comment - The comment syntax to filter (e.g., '//' for JavaScript-style comments). + * @yields {string} The filtered lines, excluding unwanted path lines and empty comments. + */ +export async function* avoidPathLineAndEmptyComments( stream: LineStream, comment: string, ): LineStream { @@ -40,6 +142,12 @@ export async function* avoidPathLine( } } +/** + * Transforms a LineStream by adding newline characters between lines. + * + * @param {LineStream} stream - The input stream of lines. + * @yields {string} The lines from the input stream with newline characters added between them. + */ export async function* streamWithNewLines(stream: LineStream): LineStream { let firstLine = true; for await (const nextLine of stream) { @@ -51,22 +159,19 @@ export async function* streamWithNewLines(stream: LineStream): LineStream { } } -const bracketEnding = [")", "]", "}", ";"]; -function isBracketEnding(line: string): boolean { - return line - .trim() - .split("") - .some((char) => bracketEnding.includes(char)); -} - -function commonPrefixLength(a: string, b: string): number { - let i = 0; - while (i < a.length && i < b.length && a[i] === b[i]) { - i++; - } - return i; -} - +/** + * Determines if two lines of text are considered repeated or very similar. + * + * @param {string} a - The first line of text to compare. + * @param {string} b - The second line of text to compare. + * @returns {boolean} True if the lines are considered repeated, false otherwise. + * + * @description + * This function checks if two lines are repeated or very similar based on two criteria: + * 1. They have a common prefix longer than 12 characters. + * 2. The Levenshtein distance between them is less than 10% of the length of the second line. + * Lines shorter than 5 characters are never considered repeated. + */ export function lineIsRepeated(a: string, b: string): boolean { if (a.length <= 4 || b.length <= 4) { return false; @@ -80,6 +185,21 @@ export function lineIsRepeated(a: string, b: string): boolean { ); } +/** + * Filters a LineStream, stopping when a line similar to the provided one is encountered. + * + * @param {LineStream} stream - The input stream of lines to filter. + * @param {string} line - The line to compare against for similarity. + * @param {() => void} fullStop - Function to call when stopping the stream. + * @yields {string} Filtered lines until a similar line is encountered. + * + * @description + * This generator function processes the input stream, yielding lines until it encounters: + * 1. An exact match to the provided line. + * 2. A line that is considered repeated or very similar to the provided line. + * 3. For lines ending with brackets, it allows exact matches of trimmed content. + * When any of these conditions are met, it calls the fullStop function and stops yielding. + */ export async function* stopAtSimilarLine( stream: LineStream, line: string, @@ -87,6 +207,7 @@ export async function* stopAtSimilarLine( ): AsyncGenerator { const trimmedLine = line.trim(); const lineIsBracketEnding = isBracketEnding(trimmedLine); + for await (const nextLine of stream) { if (nextLine === line) { fullStop(); @@ -102,12 +223,17 @@ export async function* stopAtSimilarLine( fullStop(); break; } + yield nextLine; } } -const LINES_TO_STOP_AT = ["# End of file.", " void} fullStop - Function to call when stopping. + * @yields {string} Filtered lines until a stop phrase is encountered. + */ export async function* stopAtLines( stream: LineStream, fullStop: () => void, @@ -121,7 +247,11 @@ export async function* stopAtLines( } } -const PREFIXES_TO_SKIP = [""]; +/** + * Filters a LineStream, skipping specified prefixes on the first line. + * @param {LineStream} lines - The input stream of lines. + * @yields {string} Filtered lines with prefixes removed from the first line if applicable. + */ export async function* skipPrefixes(lines: LineStream): LineStream { let isFirstLine = true; for await (const line of lines) { @@ -137,8 +267,11 @@ export async function* skipPrefixes(lines: LineStream): LineStream { } } -const LINES_TO_SKIP = [""]; - +/** + * Filters out lines starting with specified prefixes from a LineStream. + * @param {LineStream} stream - The input stream of lines. + * @yields {string} Filtered lines that don't start with any of the LINES_TO_SKIP prefixes. + */ export async function* skipLines(stream: LineStream): LineStream { for await (const line of stream) { if (!LINES_TO_SKIP.some((skipAt) => line.startsWith(skipAt))) { @@ -147,34 +280,21 @@ export async function* skipLines(stream: LineStream): LineStream { } } -const LINES_TO_REMOVE_BEFORE_START = [ - "", - "[CODE]", - "", -]; - -function shouldRemoveLineBeforeStart(line: string): boolean { - return ( - line.trimStart().startsWith("```") || - LINES_TO_REMOVE_BEFORE_START.some((l) => line.trim() === l) - ); -} - -function shouldChangeLineAndStop(line: string): string | undefined { - if (line.trimStart() === "```") { - return line; - } - - if (line.includes("[/CODE]")) { - return line.split("[/CODE]")[0].trimEnd(); - } - - return undefined; -} - +/** + * Filters and processes lines from a code block, removing unnecessary markers and handling edge cases. + * + * @param {LineStream} rawLines - The input stream of lines to filter. + * @yields {string} Filtered and processed lines from the code block. + * + * @description + * This generator function performs the following tasks: + * 1. Removes initial lines that should be removed before the actual code starts. + * 2. Filters out ending code block markers (```) unless they are the last line. + * 3. Handles special cases where lines should be changed and the stream should stop. + * 4. Yields processed lines that are part of the actual code block content. + */ export async function* filterCodeBlockLines(rawLines: LineStream): LineStream { let seenValidLine = false; - let waitingToSeeIfLineIsLast = undefined; for await (const line of rawLines) { @@ -206,28 +326,19 @@ export async function* filterCodeBlockLines(rawLines: LineStream): LineStream { } } -function isEnglishFirstLine(line: string) { - line = line.trim().toLowerCase(); - if (line.endsWith(":") && !line.trimStart().startsWith("def")) { - return true; - } - if ( - line.startsWith("here is") || - line.startsWith("here's") || - line.startsWith("sure, here") || - line.startsWith("sure thing") || - line.startsWith("sure!") || - line.startsWith("to fill") || - line.startsWith("certainly") || - line.startsWith("of course") || - line.startsWith("the code should") - ) { - return true; - } - - return false; -} - +/** + * Filters out English explanations at the start of a code block. + * + * @param {LineStream} lines - The input stream of lines. + * @yields {string} Filtered lines with English explanations removed from the start. + * + * @description + * This generator function performs the following tasks: + * 1. Skips initial blank lines. + * 2. Removes the first line if it's identified as an English explanation. + * 3. Removes a subsequent blank line if the first line was an English explanation. + * 4. Yields all remaining lines. + */ export async function* filterEnglishLinesAtStart(lines: LineStream) { let i = 0; let wasEnglishFirstLine = false; @@ -235,6 +346,7 @@ export async function* filterEnglishLinesAtStart(lines: LineStream) { if (i === 0 && line.trim() === "") { continue; } + if (i === 0) { if (isEnglishFirstLine(line)) { wasEnglishFirstLine = true; @@ -250,18 +362,14 @@ export async function* filterEnglishLinesAtStart(lines: LineStream) { } } -function isEnglishPostExplanation(line: string): boolean { - const lower = line.toLowerCase(); - return ( - lower.startsWith("explanation:") || - lower.startsWith("here is") || - lower.startsWith("here's how") || - lower.startsWith("the above") - ); -} - +/** + * Filters out English explanations at the end of a code block. + * @param {LineStream} lines - The input stream of lines. + * @yields {string} Lines up to the end of the code block or start of English explanation. + */ export async function* filterEnglishLinesAtEnd(lines: LineStream) { let finishedCodeBlock = false; + for await (const line of lines) { if (line.trim() === "```") { finishedCodeBlock = true; @@ -273,8 +381,14 @@ export async function* filterEnglishLinesAtEnd(lines: LineStream) { } } +/** + * Removes leading indentation from the first line of a CodeLlama output. + * @param {LineStream} lines - The input stream of lines. + * @yields {string} Lines with the first line's indentation fixed if necessary. + */ export async function* fixCodeLlamaFirstLineIndentation(lines: LineStream) { let isFirstLine = true; + for await (const line of lines) { if (isFirstLine && line.startsWith(" ")) { yield line.slice(2); @@ -285,23 +399,36 @@ export async function* fixCodeLlamaFirstLineIndentation(lines: LineStream) { } } -function isUselessLine(line: string): boolean { - const trimmed = line.trim().toLowerCase(); - return trimmed === "" || trimmed === "```" || trimmed.startsWith("// end"); -} - +/** + * Filters leading and trailing blank line insertions from a stream of diff lines. + * + * @param {AsyncGenerator} diffLines - An async generator that yields DiffLine objects. + * @yields {DiffLine} Filtered DiffLine objects, with leading and trailing blank line insertions removed. + * + * @description + * This generator function processes a stream of diff lines, removing leading and trailing + * blank line insertions. It performs the following tasks: + * 1. Skips the first blank line insertion if it occurs at the beginning. + * 2. Buffers subsequent blank line insertions. + * 3. Yields buffered blank lines when a non-blank insertion is encountered. + * 4. Clears the buffer when an old line is encountered. + * 5. Yields all non-blank insertions and old lines. + */ export async function* filterLeadingAndTrailingNewLineInsertion( diffLines: AsyncGenerator, ): AsyncGenerator { let isFirst = true; let buffer: DiffLine[] = []; + for await (const diffLine of diffLines) { const isBlankLineInsertion = diffLine.type === "new" && isUselessLine(diffLine.line); + if (isFirst && isBlankLineInsertion) { isFirst = false; continue; } + isFirst = false; if (isBlankLineInsertion) { @@ -319,31 +446,38 @@ export async function* filterLeadingAndTrailingNewLineInsertion( } } +/** + * Filters a LineStream, stopping when a line repeats more than a specified number of times. + * + * @param {LineStream} lines - The input stream of lines to filter. + * @param {() => void} fullStop - Function to call when stopping the stream. + * @yields {string} Filtered lines until excessive repetition is detected. + * + * @description + * This function yields lines from the input stream until a line is repeated + * for a maximum of 3 consecutive times. When this limit is reached, it calls + * the fullStop function and stops yielding. Only the first of the repeating + * lines is yieled. + */ export async function* stopAtRepeatingLines( lines: LineStream, fullStop: () => void, ): LineStream { - const repeatedLines: string[] = []; + let previousLine: string | undefined; + let repeatCount = 0; + const MAX_REPEATS = 3; + for await (const line of lines) { - if (repeatedLines.length === 0) { - repeatedLines.push(line); - } else if (repeatedLines.length < 3) { - if (repeatedLines[repeatedLines.length - 1] === line) { - repeatedLines.push(line); - } else { - while (repeatedLines.length > 0) { - yield repeatedLines.shift()!; - } - yield line; + if (line === previousLine) { + repeatCount++; + if (repeatCount === MAX_REPEATS) { + fullStop(); + return; } } else { - yield repeatedLines[0]; - fullStop(); - return; + yield line; + repeatCount = 1; } - } - - while (repeatedLines.length > 0) { - yield repeatedLines.shift()!; + previousLine = line; } }