RAG のための Docling——文書の構造を尊重するチャンク

from docling.chunking import HybridChunker
from docling.document_converter import DocumentConverter

doc = DocumentConverter().convert("https://arxiv.org/pdf/2408.09869").document

chunker = HybridChunker(max_tokens=256)          # 埋め込みモデルに合わせる
for chunk in chunker.chunk(doc):
    text = chunker.contextualize(chunk)          # 見出し + 本文：これを埋め込む
    pages = {p.page_no for item in chunk.meta.doc_items for p in item.prov}
    print(len(text), chunk.meta.headings, sorted(pages))

from docling.chunking import HybridChunker
from docling_core.transforms.chunker.tokenizer.huggingface import HuggingFaceTokenizer
from transformers import AutoTokenizer

EMBED_MODEL = "intfloat/multilingual-e5-small"   # 実際に埋め込みに使うモデル
tokenizer = HuggingFaceTokenizer(
    tokenizer=AutoTokenizer.from_pretrained(EMBED_MODEL),
    max_tokens=512,
)
chunker = HybridChunker(tokenizer=tokenizer)

docling convert report.pdf --to chunks --chunks-max-tokens 512 --output chunks/
docling convert report.pdf --to chunks --chunks-type hierarchical
docling convert report.pdf --to chunks --chunks-tokenizer intfloat/multilingual-e5-small

pip install "langchain-docling[local]"

from langchain_docling import DoclingLoader
from langchain_docling.loader import ExportType

loader = DoclingLoader(
    file_path=["https://arxiv.org/pdf/2408.09869"],
    export_type=ExportType.DOC_CHUNKS,           # 既定。または ExportType.MARKDOWN
)
docs = loader.load()                             # チャンクごとに LangChain の Document
print(docs[0].metadata.keys())

pip install llama-index-readers-docling llama-index-node-parser-docling

from llama_index.node_parser.docling import DoclingNodeParser
from llama_index.readers.docling import DoclingReader

reader = DoclingReader(export_type=DoclingReader.ExportType.JSON)
docs = reader.load_data(file_path="https://arxiv.org/pdf/2408.09869")
nodes = DoclingNodeParser().get_nodes_from_documents(documents=docs)