Why we chunk, and the main chunking strategies

import re

def recursive_split(text: str, max_tokens: int, count, seps=("\n\n", "\n", ". ", " ")) -> list[str]:
    """Split at the largest natural break that makes every piece fit."""
    if count(text) <= max_tokens:
        return [text]
    sep, *rest = seps
    parts, out, buf = text.split(sep), [], ""
    for p in parts:
        joined = f"{buf}{sep}{p}" if buf else p
        if count(joined) <= max_tokens:
            buf = joined
        else:
            if buf:
                out.append(buf)
            buf = p if count(p) <= max_tokens else ""
            if not buf:  # this piece alone is too big: split it at the next smaller break
                out.extend(recursive_split(p, max_tokens, count, tuple(rest)) if rest else [p])
    return out + ([buf] if buf else [])

def semantic_split(sentences: list[str], embed, pct: float = 95) -> list[str]:
    """Cut where the next sentence is far from this one in meaning (needs one embedding per sentence)."""
    import numpy as np
    v = np.array(embed(sentences))                      # normalised vectors
    dist = 1 - (v[:-1] * v[1:]).sum(axis=1)             # distance to the next sentence
    cut = np.percentile(dist, pct)
    chunks, buf = [], [sentences[0]]
    for s, d in zip(sentences[1:], dist):
        if d > cut:
            chunks.append(" ".join(buf)); buf = []
        buf.append(s)
    return chunks + [" ".join(buf)]

def split_by_headings(md: str) -> list[str]:
    return [s for s in re.split(r"(?m)^(?=#{1,6} )", md) if s.strip()]