add chunk and embed

2026-04-21 15:06:04 +02:00
parent 35992e6029
commit 746b951f0b
4 changed files with 296 additions and 0 deletions
@@ -8,4 +8,7 @@ dependencies = [
    "chromadb>=1.5.8",
    "openai>=2.32.0",
    "python-dotenv>=1.2.2",
+    "semchunk>=4.0.0",
+    "tiktoken>=0.12.0",
+    "transformers>=5.5.4",
 ]