# /// script # requires-python = ">=3.10" # dependencies = ["chromadb-client==1.5.9", "openai>=1.60"] # /// """Load exe.dev's documentation into a Chroma collection called exe-docs. Fetches the docs listed in https://exe.dev/llms.txt, splits each page at its headings, and stores the pieces in Chroma with embeddings from text-embedding-3-small through the exe.dev LLM integration (no API key). Run it again any time: records are upserted by URL and heading. uv run index-docs.py [--host http://localhost:8000] """ import argparse, os, re, urllib.request from concurrent.futures import ThreadPoolExecutor import chromadb from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction p = argparse.ArgumentParser() p.add_argument("--host", default="http://localhost:8000") p.add_argument("--collection", default="exe-docs") args = p.parse_args() # The LLM integration needs no key, but the OpenAI library wants one. os.environ.setdefault("CHROMA_OPENAI_API_KEY", "exe-gateway") embed = OpenAIEmbeddingFunction( model_name="text-embedding-3-small", api_base="https://llm.int.exe.xyz/v1", ) def fetch(url): with urllib.request.urlopen(url, timeout=30) as r: return r.read().decode() def chunks(text, limit=2000): """Split markdown at ## and ### headings, then into paragraphs up to `limit` chars.""" for section in re.split(r"\n(?=#{2,3} )", text): heading = section.split("\n", 1)[0].lstrip("# ").strip() piece = "" for para in section.split("\n\n"): if piece and len(piece) + len(para) > limit: yield heading, piece.strip() piece = "" piece += para + "\n\n" if piece.strip(): yield heading, piece.strip() urls = sorted(set(re.findall(r"\((https://exe\.dev/docs/[^)\s]+\.md)\)", fetch("https://exe.dev/llms.txt")))) client = chromadb.HttpClient(host=args.host) col = client.get_or_create_collection( args.collection, embedding_function=embed, configuration={"hnsw": {"space": "cosine"}}, ) def load(url): try: return url, fetch(url) except Exception as e: print(f"skip {url}: {e}") return url, "" ids, docs, metas = [], [], [] with ThreadPoolExecutor(8) as pool: for url, page in pool.map(load, urls): title = next((l[2:].strip() for l in page.splitlines() if l.startswith("# ")), url) for i, (heading, text) in enumerate(chunks(page)): ids.append(f"{url}#{i}") docs.append(text if heading == title else f"{title}: {heading}\n\n{text}") metas.append({"url": url.removesuffix(".md"), "title": title, "heading": heading}) for i in range(0, len(ids), 100): # Chroma embeds each batch with one API call col.upsert(ids=ids[i:i+100], documents=docs[i:i+100], metadatas=metas[i:i+100]) print(f"{len(urls)} pages, {len(ids)} chunks; {col.count()} records in {args.collection}")