#!/usr/bin/env python3 """Replit Database (the key-value store at REPLIT_DB_URL), on your VM. replitdb serve [PORT] answer Replit Database's HTTP API on 127.0.0.1:PORT (default 8099), so the app's `replit` or `@replit/database` client works with REPLIT_DB_URL=http://127.0.0.1:8099 replitdb import copy every key from the URL on stdin (your Repl's REPLIT_DB_URL) into the local store replitdb export print the local store as JSON The store is one SQLite file, ~/replitdb.sqlite (or $REPLITDB_FILE). From https://exe.dev/deploy/migrate-replit """ import email.parser, json, os, sqlite3, sys, threading, urllib.parse, urllib.request from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer FILE = os.environ.get("REPLITDB_FILE", os.path.expanduser("~/replitdb.sqlite")) def connect(): db = sqlite3.connect(FILE, check_same_thread=False) db.execute("PRAGMA journal_mode=WAL") db.execute("PRAGMA busy_timeout=5000") db.execute("CREATE TABLE IF NOT EXISTS kv (key TEXT PRIMARY KEY, value TEXT NOT NULL)") return db def form(handler): """The key/value pairs in a POST or DELETE body, urlencoded or multipart.""" n = int(handler.headers.get("Content-Length") or 0) body = handler.rfile.read(n) if n else b"" ctype = handler.headers.get("Content-Type", "") if ctype.startswith("multipart/form-data"): msg = email.parser.BytesParser().parsebytes( b"Content-Type: " + ctype.encode() + b"\r\n\r\n" + body) return [(part.get_param("name", header="content-disposition"), part.get_payload(decode=True).decode()) for part in msg.get_payload()] return urllib.parse.parse_qsl(body.decode(), keep_blank_values=True) class Handler(BaseHTTPRequestHandler): db = None lock = threading.Lock() # one connection, shared by the request threads def reply(self, code, text=""): data = text.encode() self.send_response(code) self.send_header("Content-Type", "text/plain; charset=utf-8") self.send_header("Content-Length", str(len(data))) self.end_headers() self.wfile.write(data) def key(self): path = urllib.parse.urlsplit(self.path).path return urllib.parse.unquote(path[1:]) if len(path) > 1 else None def do_GET(self): key = self.key() if key is None: # list keys q = urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query) prefix = q.get("prefix", [""])[0] with self.lock: rows = self.db.execute( "SELECT key FROM kv WHERE substr(key, 1, ?) = ? ORDER BY key", (len(prefix), prefix)).fetchall() keys = [r[0] for r in rows] if q.get("encode", [""])[0] == "true": keys = [urllib.parse.quote(k, safe="") for k in keys] return self.reply(200, "\n".join(keys)) with self.lock: row = self.db.execute("SELECT value FROM kv WHERE key = ?", (key,)).fetchone() self.reply(200, row[0]) if row else self.reply(404) def do_POST(self): pairs = form(self) with self.lock, self.db: self.db.executemany("INSERT OR REPLACE INTO kv VALUES (?, ?)", pairs) self.reply(200) def do_DELETE(self): keys = [self.key()] if self.key() is not None else [v for k, v in form(self) if k == "key"] with self.lock, self.db: n = sum(self.db.execute("DELETE FROM kv WHERE key = ?", (k,)).rowcount for k in keys) self.reply(204 if n else 404) def log_message(self, *args): pass def serve(port): Handler.db = connect() srv = ThreadingHTTPServer(("127.0.0.1", port), Handler) print(f"replitdb: serving {FILE} at http://127.0.0.1:{port}", flush=True) srv.serve_forever() def get(url): with urllib.request.urlopen(url, timeout=60) as r: return r.read().decode() def do_import(): url = sys.stdin.readline().strip().rstrip("/") if not url.startswith("http"): sys.exit("replitdb import: paste the URL that echo $REPLIT_DB_URL prints in the Repl's Shell") listing = get(url + "?encode=true&prefix=") keys = [urllib.parse.unquote(k) for k in listing.split("\n") if k] db = connect() with db: for i, k in enumerate(keys, 1): db.execute("INSERT OR REPLACE INTO kv VALUES (?, ?)", (k, get(url + "/" + urllib.parse.quote(k, safe="")))) if i % 500 == 0: print(f"replitdb: {i} of {len(keys)} keys", flush=True) total = db.execute("SELECT count(*) FROM kv").fetchone()[0] print(f"replitdb: copied {len(keys)} keys; the store has {total}") def main(): cmd = sys.argv[1] if len(sys.argv) > 1 else "" if cmd == "serve": serve(int(sys.argv[2]) if len(sys.argv) > 2 else 8099) elif cmd == "import": do_import() elif cmd == "export": rows = connect().execute("SELECT key, value FROM kv ORDER BY key").fetchall() json.dump(dict(rows), sys.stdout, indent=1, ensure_ascii=False) print() else: sys.exit(__doc__) if __name__ == "__main__": main()