# /// script # requires-python = ">=3.10" # dependencies = ["scrapling[fetchers]==0.5.1"] # /// """Crawl books.toscrape.com and add today's prices to ~/scrapling/data/books.db.""" import logging import sqlite3 from datetime import datetime, timezone from pathlib import Path from scrapling.spiders import Response, Spider DB = Path.home() / "scrapling" / "data" / "books.db" RATINGS = {"One": 1, "Two": 2, "Three": 3, "Four": 4, "Five": 5} class BooksSpider(Spider): name = "books" start_urls = ["https://books.toscrape.com/"] allowed_domains = {"books.toscrape.com"} robots_txt_obey = True concurrent_requests_per_domain = 2 download_delay = 0.5 logging_level = logging.WARNING # development_mode = True # while editing: cache pages on disk and replay them async def parse(self, response: Response): for book in response.css("article.product_pod"): yield { "url": response.urljoin(book.css("h3 a::attr(href)").get("")), "title": book.css("h3 a::attr(title)").get(""), "price": float(book.css("p.price_color::text").get("0").lstrip("£")), "rating": RATINGS.get(book.css("p.star-rating::attr(class)").get("").split()[-1], 0), "in_stock": "In stock" in " ".join(book.css("p.availability::text").getall()), } next_page = response.css("li.next a::attr(href)").get() if next_page: yield response.follow(next_page, callback=self.parse) result = BooksSpider().start() now = datetime.now(timezone.utc).isoformat(timespec="seconds") DB.parent.mkdir(parents=True, exist_ok=True) with sqlite3.connect(DB) as db: db.execute("""CREATE TABLE IF NOT EXISTS prices ( crawled_at TEXT, url TEXT, title TEXT, price REAL, rating INTEGER, in_stock INTEGER)""") db.executemany( "INSERT INTO prices VALUES (?, ?, ?, ?, ?, ?)", [(now, i["url"], i["title"], i["price"], i["rating"], i["in_stock"]) for i in result.items], ) print(f"{len(result.items)} books saved to {DB} ({result.stats.requests_count} requests, {result.stats.elapsed_seconds:.0f} s)")