Chromy/chromy/chunking/service.py

from __future__ import annotations

from pathlib import Path
from typing import cast

from semchunk import semchunk


def chunk_text(text: str, chunk_size: int = 800) -> list[str]:
    chunker = semchunk.chunkerify("gpt-4", chunk_size)
    chunks = chunker(text)

    return cast("list[str]", chunks)


def chunk_file(filename: str, chunk_size: int = 800) -> list[str]:
    contents = Path(filename).read_text()

    return chunk_text(contents, chunk_size)