• Fixed-size: split every $n$ characters/tokens (simple but may cut mid-sentence)
• Sentence-based or paragraph-based: split on sentence or paragraph boundaries (preserves meaning better; leverages author intuitions about conceptual units)
• Semantic: use topic shifts to determine chunk boundaries (best quality but most complex)
• Recursive: split on paragraphs first, then sentences if chunks are still too long

1from sentence_transformers import SentenceTransformer, util
2
3embedder = SentenceTransformer("all-MiniLM-L6-v2")
4
5# Knowledge base (in practice, loaded from files)
6documents = [
7    "The transformer was introduced by Vaswani et al. in 2017.",
8    "BERT uses bidirectional attention for language understanding.",
9    "GPT models use autoregressive (left-to-right) generation.",
10    "Attention mechanisms allow models to focus on relevant context.",
11    "Fine-tuning adapts a pre-trained model to a specific task.",
12]
13doc_embeddings = embedder.encode(documents, convert_to_tensor=True)

1from transformers import pipeline
2generator = pipeline("text2text-generation", model="google/flan-t5-base")
3
4def rag_answer(query, documents, doc_embeddings, top_k=3):
5    """Answer a question using RAG."""
6    query_emb = embedder.encode(query, convert_to_tensor=True)
7    scores = util.cos_sim(query_emb, doc_embeddings)[0]
8    top_idx = scores.argsort(descending=True)[:top_k]
9    context = "\n".join([documents[i] for i in top_idx])
10    prompt = f"""Answer based on context. Say "I don't know" if unsure.
11Context: {context}
12Question: {query}
13Answer:"""
14    return generator(prompt, max_new_tokens=128)[0]["generated_text"]
15
16print(rag_answer("What is BERT?", documents, doc_embeddings))

1import chromadb
2from transformers import pipeline
3
4# Set up ChromaDB (handles embedding automatically)
5client = chromadb.Client()
6collection = client.create_collection("course_notes")
7
8# Add documents
9collection.add(
10    documents=[
11        "The transformer uses self-attention to process sequences in parallel.",
12        "Cross-entropy loss measures prediction error for classification.",
13        "RAG combines retrieval with generation for knowledge-grounded answers.",
14    ],
15    ids=["doc1", "doc2", "doc3"]
16)