From 9c9bd5ac4a2e3e1017b3e0c9e6ddd64d46c758a5 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Wed, 7 Oct 2026 19:04:38 -0400 Subject: [PATCH] fix(engineering): enforce RAG chunk bounds for nonspaced text Signed-off-by: Rudy Celekli --- engineering/engineering-rag-pipeline-engineer.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engineering/engineering-rag-pipeline-engineer.md b/engineering/engineering-rag-pipeline-engineer.md index 17c64c56..6f468dc0 100644 --- a/engineering/engineering-rag-pipeline-engineer.md +++ b/engineering/engineering-rag-pipeline-engineer.md @@ -90,7 +90,7 @@ def chunk_document(text: str, doc_type: str) -> list[dict]: char_splitter = RecursiveCharacterTextSplitter( chunk_size=800, chunk_overlap=100, - separators=["\n\n", "\n", ". ", " "] + separators=["\n\n", "\n", ". ", " ", ""] ) chunks = [] for doc in header_chunks: @@ -103,7 +103,7 @@ def chunk_document(text: str, doc_type: str) -> list[dict]: splitter = RecursiveCharacterTextSplitter( chunk_size=600, chunk_overlap=80, - separators=["\n\n", "\n", ". ", "! ", "? ", " "] + separators=["\n\n", "\n", ". ", "! ", "? ", " ", ""] ) return [ {"content": doc.page_content, "metadata": doc.metadata}