#!/usr/bin/env python3 """Copy Chroma collections from Chroma Cloud (or any Chroma server) to another Chroma server, then check the copy. Uses Chroma's HTTP API and only the Python standard library. Records are read in pages of --batch with their embeddings, documents, metadata and URIs, and upserted by ID, so a run that stops halfway can simply be run again. Afterwards each collection's count is compared, and a few records are read back from both sides and looked up by their own embedding. From Chroma Cloud (reads CHROMA_API_KEY, and CHROMA_TENANT and CHROMA_DATABASE if set; otherwise they come from the key): python3 chroma-migrate.py --from-cloud --all From another Chroma server, to a server other than localhost:8000: python3 chroma-migrate.py --from http://old-host:8000 --to http://localhost:8001 docs notes --reembed text-embedding-3-small embeds the documents again through the exe.dev LLM integration instead of copying the vectors, for collections whose embedding function won't exist after the move (Chroma Cloud's hosted Qwen embeddings). https://exe.dev/deploy/chroma explains it all. """ import argparse, json, os, random, sys, urllib.error, urllib.parse, urllib.request GATEWAY = "https://llm.int.exe.xyz/v1" class Chroma: def __init__(self, url, headers, tenant=None, database=None): self.url, self.headers = url.rstrip("/"), headers if not tenant or not database: ident = self.call("GET", "/api/v2/auth/identity") tenant = tenant or ident["tenant"] dbs = ident.get("databases") or [] if not database: if len(dbs) != 1: sys.exit(f"{url}: say which database with CHROMA_DATABASE (the key sees {dbs})") database = dbs[0] self.tenant, self.database = tenant, database self.base = f"/api/v2/tenants/{q(tenant)}/databases/{q(database)}" def call(self, method, path, body=None): data = None if body is None else json.dumps(body).encode() req = urllib.request.Request(self.url + path, data=data, method=method, headers={"Content-Type": "application/json", **self.headers}) try: with urllib.request.urlopen(req, timeout=300) as r: return json.loads(r.read() or b"null") except urllib.error.HTTPError as e: raise SystemExit(f"{method} {self.url}{path}: HTTP {e.code}: {e.read().decode()[:500]}") def collections(self): out, offset = [], 0 while True: page = self.call("GET", f"{self.base}/collections?limit=100&offset={offset}") out += page if len(page) < 100: return out offset += 100 def coll(self, cid, op, body=None): return self.call("POST" if body is not None else "GET", f"{self.base}/collections/{cid}/{op}", body) def q(s): return urllib.parse.quote(s, safe="") def header_list(values): out = {} for h in values or []: name, _, value = h.partition(":") out[name.strip()] = value.strip() return out def embed(model, texts): req = urllib.request.Request(f"{GATEWAY}/embeddings", method="POST", data=json.dumps({"model": model, "input": texts}).encode(), headers={"Content-Type": "application/json", "Authorization": "Bearer exe-gateway"}) with urllib.request.urlopen(req, timeout=300) as r: return [d["embedding"] for d in sorted(json.load(r)["data"], key=lambda d: d["index"])] def main(): p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) src = p.add_mutually_exclusive_group(required=True) src.add_argument("--from-cloud", action="store_true", help="read from Chroma Cloud (CHROMA_API_KEY)") src.add_argument("--from", dest="from_url", metavar="URL", help="read from this Chroma server") p.add_argument("--from-header", action="append", metavar="'NAME: VALUE'", help="extra header for the source") p.add_argument("--to", default="http://localhost:8000", metavar="URL", help="write here (default %(default)s)") p.add_argument("--to-header", action="append", metavar="'NAME: VALUE'", help="extra header for the target") p.add_argument("--to-database", default="default_database", help="target database, created if missing") p.add_argument("--all", action="store_true", help="copy every collection") p.add_argument("--batch", type=int, default=300, help="records per request (Chroma Cloud allows 300)") p.add_argument("--reembed", metavar="MODEL", help="embed documents again with this model via the LLM integration") p.add_argument("names", nargs="*", help="collections to copy") a = p.parse_args() if bool(a.names) == a.all: p.error("name the collections to copy, or say --all") if a.from_cloud: key = os.environ.get("CHROMA_API_KEY") or sys.exit("Set CHROMA_API_KEY to a Chroma Cloud API key.") host = os.environ.get("CHROMA_HOST", "api.trychroma.com") source = Chroma(f"https://{host}", {"x-chroma-token": key, **header_list(a.from_header)}, os.environ.get("CHROMA_TENANT"), os.environ.get("CHROMA_DATABASE")) else: source = Chroma(a.from_url, header_list(a.from_header)) target = Chroma(a.to, header_list(a.to_header), "default_tenant", "default_database") if a.to_database != "default_database": if a.to_database not in [d["name"] for d in target.call("GET", "/api/v2/tenants/default_tenant/databases")]: target.call("POST", "/api/v2/tenants/default_tenant/databases", {"name": a.to_database}) target = Chroma(a.to, target.headers, "default_tenant", a.to_database) found = {c["name"]: c for c in source.collections()} names = sorted(found) if a.all else a.names missing = [n for n in names if n not in found] if missing: sys.exit(f"Not in the source: {', '.join(missing)}. It has: {', '.join(sorted(found))}") print(f"Copying {len(names)} collection(s) from {source.url} ({source.tenant}/{source.database}) " f"to {target.url} ({target.database})") ok = True for name in names: ok &= copy(source, target, found[name], a) print("All collections copied and checked." if ok else "Some checks FAILED; see above.") sys.exit(0 if ok else 1) def copy(source, target, sc, a): name, conf = sc["name"], sc.get("configuration_json") or {} index = conf.get("hnsw") or conf.get("spann") or {} space = index.get("space") or "l2" ef = conf.get("embedding_function") if a.reembed: ef = {"type": "known", "name": "openai", "config": { "api_base": GATEWAY, "api_key_env_var": "CHROMA_OPENAI_API_KEY", "model_name": a.reembed}} elif ef and str(ef.get("name", "")).startswith("chroma-cloud"): print(f" ! {name}: embedding function {ef['name']} is a Chroma Cloud service. The vectors copy fine,\n" f" but nothing will embed new queries the same way after Cloud shuts down; consider --reembed.") body = {"name": name, "get_or_create": True, "configuration": {"hnsw": {"space": space}}} if sc.get("metadata"): body["metadata"] = sc["metadata"] if ef and ef.get("type") == "known": body["configuration"]["embedding_function"] = ef tc = target.call("POST", f"{target.base}/collections", body) total = source.coll(sc["id"], "count") print(f"- {name}: {total} records, space {space}, embeddings " f"{'from ' + a.reembed if a.reembed else (ef or {}).get('name', 'as stored')}") samples, done, dropped = [], 0, set() for offset in range(0, total, a.batch): page = source.coll(sc["id"], "get", {"limit": a.batch, "offset": offset, "include": ["embeddings", "documents", "metadatas", "uris"]}) if not page["ids"]: break rec = {"ids": page["ids"]} if a.reembed: docs = page.get("documents") or [] if any(d is None for d in docs) or len(docs) != len(page["ids"]): sys.exit(f" {name}: --reembed needs a document on every record") rec["embeddings"] = embed(a.reembed, docs) else: rec["embeddings"] = page["embeddings"] for k in ("documents", "metadatas", "uris"): if page.get(k) and any(v is not None for v in page[k]): rec[k] = page[k] for m in rec.get("metadatas") or []: # Sparse vectors (Chroma Cloud hybrid search) live in metadata as # objects; single-node Chroma can't index them, so leave them out. for key in [k for k, v in (m or {}).items() if isinstance(v, dict)]: if key not in dropped: print(f" ! {name}: leaving out metadata key {key!r} (a sparse vector; Cloud only)") dropped.add(key) del m[key] target.coll(tc["id"], "upsert", rec) for i in random.sample(range(len(page["ids"])), min(3, len(page["ids"]))): samples.append((page["ids"][i], rec["embeddings"][i])) done += len(page["ids"]) print(f" {done}/{total}", end="\r", flush=True) # Check: same count; a few records come back the same on both sides, and # find themselves first when queried by their own embedding. (Further # neighbors can differ a little: the index is approximate.) copied = target.coll(tc["id"], "count") good = copied == total print(f" count: source {total}, copy {copied} {'ok' if good else 'MISMATCH'}") picked = random.sample(samples, min(3, len(samples))) if not picked: return good inc = {"ids": [r[0] for r in picked], "include": ["documents", "metadatas", "uris"]} sg, tg = source.coll(sc["id"], "get", inc), target.coll(tc["id"], "get", inc) def norm(g): n = len(g["ids"]) metas = [{k: v for k, v in (m or {}).items() if not isinstance(v, dict)} or None for m in (g.get("metadatas") or [None] * n)] return sorted(zip(g["ids"], g.get("documents") or [None] * n, metas, g.get("uris") or [None] * n), key=str) same = norm(sg) == norm(tg) good &= same print(f" records {', '.join(inc['ids'])}: documents, metadata and URIs {'match' if same else 'DIFFER'}") for rid, temb in picked: t = target.coll(tc["id"], "query", {"query_embeddings": [temb], "n_results": 3, "include": ["distances"]}) found = t["ids"][0][:1] == [rid] good &= found print(f" query by {rid}'s embedding: {t['ids'][0]} {'ok' if found else 'NOT FOUND FIRST'}") return good if __name__ == "__main__": main()