exclude empty sentences from sentenceit

jparkerweb · Dec 13, 2024 · 71a1ca2 · 71a1ca2
1 parent bc1a4a1
commit 71a1ca2
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/chunkit.js b/chunkit.js
@@ -333,7 +333,9 @@ export async function sentenceit(
         // Split the text into sentences
         const chunks = [];
         for (const { segment } of splitBySentence(doc.document_text)) {
-            chunks.push(segment.trim());
+            if (segment.trim().length > 0) {
+                chunks.push(segment.trim());
+            }
         }
 
         if (logging) {