Docling 用于 RAG——尊重文档结构的分块

from docling.chunking import HybridChunker
from docling.document_converter import DocumentConverter

doc = DocumentConverter().convert("https://arxiv.org/pdf/2408.09869").document

chunker = HybridChunker(max_tokens=256)          # 与嵌入模型一致
for chunk in chunker.chunk(doc):
    text = chunker.contextualize(chunk)          # 标题 + 正文：嵌入这个
    pages = {p.page_no for item in chunk.meta.doc_items for p in item.prov}
    print(len(text), chunk.meta.headings, sorted(pages))

from docling.chunking import HybridChunker
from docling_core.transforms.chunker.tokenizer.huggingface import HuggingFaceTokenizer
from transformers import AutoTokenizer

EMBED_MODEL = "BAAI/bge-small-zh-v1.5"           # 换成你用来嵌入的模型
tokenizer = HuggingFaceTokenizer(
    tokenizer=AutoTokenizer.from_pretrained(EMBED_MODEL),
    max_tokens=512,
)
chunker = HybridChunker(tokenizer=tokenizer)

docling convert report.pdf --to chunks --chunks-max-tokens 512 --output chunks/
docling convert report.pdf --to chunks --chunks-type hierarchical
docling convert report.pdf --to chunks --chunks-tokenizer BAAI/bge-small-zh-v1.5

pip install "langchain-docling[local]"

from langchain_docling import DoclingLoader
from langchain_docling.loader import ExportType

loader = DoclingLoader(
    file_path=["https://arxiv.org/pdf/2408.09869"],
    export_type=ExportType.DOC_CHUNKS,           # 默认；或 ExportType.MARKDOWN
)
docs = loader.load()                             # 每个分块一个 LangChain Document
print(docs[0].metadata.keys())

pip install llama-index-readers-docling llama-index-node-parser-docling

from llama_index.node_parser.docling import DoclingNodeParser
from llama_index.readers.docling import DoclingReader

reader = DoclingReader(export_type=DoclingReader.ExportType.JSON)
docs = reader.load_data(file_path="https://arxiv.org/pdf/2408.09869")
nodes = DoclingNodeParser().get_nodes_from_documents(documents=docs)