Docling for RAG — chunks that respect the document

from docling.chunking import HybridChunker
from docling.document_converter import DocumentConverter

doc = DocumentConverter().convert("https://arxiv.org/pdf/2408.09869").document

chunker = HybridChunker(max_tokens=256)          # match your embedding model
for chunk in chunker.chunk(doc):
    text = chunker.contextualize(chunk)          # headings + text: embed this
    pages = {p.page_no for item in chunk.meta.doc_items for p in item.prov}
    print(len(text), chunk.meta.headings, sorted(pages))

from docling.chunking import HybridChunker
from docling_core.transforms.chunker.tokenizer.huggingface import HuggingFaceTokenizer
from transformers import AutoTokenizer

EMBED_MODEL = "BAAI/bge-small-en-v1.5"           # whatever you embed with
tokenizer = HuggingFaceTokenizer(
    tokenizer=AutoTokenizer.from_pretrained(EMBED_MODEL),
    max_tokens=512,
)
chunker = HybridChunker(tokenizer=tokenizer)

docling convert report.pdf --to chunks --chunks-max-tokens 512 --output chunks/
docling convert report.pdf --to chunks --chunks-type hierarchical
docling convert report.pdf --to chunks --chunks-tokenizer BAAI/bge-small-en-v1.5

pip install "langchain-docling[local]"

from langchain_docling import DoclingLoader
from langchain_docling.loader import ExportType

loader = DoclingLoader(
    file_path=["https://arxiv.org/pdf/2408.09869"],
    export_type=ExportType.DOC_CHUNKS,           # default; or ExportType.MARKDOWN
)
docs = loader.load()                             # one LangChain Document per chunk
print(docs[0].metadata.keys())

pip install llama-index-readers-docling llama-index-node-parser-docling

from llama_index.node_parser.docling import DoclingNodeParser
from llama_index.readers.docling import DoclingReader

reader = DoclingReader(export_type=DoclingReader.ExportType.JSON)
docs = reader.load_data(file_path="https://arxiv.org/pdf/2408.09869")
nodes = DoclingNodeParser().get_nodes_from_documents(documents=docs)