Detect changes over time: hashes and diffs

changes/page_vs_record_hash.py

"""Five days of fifty pages: changes counted by a hash of the raw page and by a hash of the normalised record."""
import hashlib

from extract import record, record_hash
from pages import SKUS, fetch

previous = {}
print(f"{'day':>3} {'raw page changed':>17} {'record changed':>15}")
totals = [0, 0]
for day in range(1, 6):
    raw = rec = 0
    for sku in SKUS:
        html = fetch(sku, day)
        now = (hashlib.sha256(html.encode()).hexdigest(), record_hash(record(html)))
        if sku in previous:
            raw += now[0] != previous[sku][0]
            rec += now[1] != previous[sku][1]
        previous[sku] = now
    if day > 1:
        totals = [totals[0] + raw, totals[1] + rec]
        print(f"{day:3} {raw:17} {rec:15}")
print(f"{'all':>3} {totals[0]:17} {totals[1]:15}")
print("\nThe record of NB-0003 on day 2:", record(fetch("NB-0003", 2)))

day  raw page changed  record changed
  2                50               3
  3                50               1
  4                50               3
  5                50               5
all               200              12

The record of NB-0003 on day 2: {'name': 'Jaggery 2 kg', 'stock': 'In stock', 'description': "Jaggery from the practice shop's own suppliers. Packed in small batches.", 'price_paise': 19600}

extract.py (record and hash) · changes/extract.py

"""The fields that matter on a product page, normalised, and a hash of them."""
import hashlib
import json
import re
import unicodedata
from decimal import Decimal

FIELDS = {"name": r"<h1 class='name'>(.*?)</h1>", "price": r"<p class='price'>(.*?)</p>",
          "stock": r"<p class='stock'>(.*?)</p>", "description": r"<div class='description'>(.*?)</div>"}


def tidy(text):
    """NFC, one plain space between words, no spaces at the ends."""
    return " ".join(unicodedata.normalize("NFC", text).split())


def record(html):
    """The page's fields as a record; a field the page does not have is None."""
    out = {}
    for field, pattern in FIELDS.items():
        m = re.search(pattern, html)
        out[field] = tidy(m[1]) if m else None
    m = re.fullmatch(r"₹([0-9,]+)\.([0-9]{2})", out.pop("price") or "")
    out["price_paise"] = int(Decimal(m[1].replace(",", "") + "." + m[2]) * 100) if m else None
    return out


def record_hash(rec):
    """SHA-256 of the record as canonical JSON: keys sorted, no spaces, so equal records hash alike."""
    text = json.dumps(rec, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
    return hashlib.sha256(text.encode("utf-8")).hexdigest()

pages.py (five days) · changes/pages.py

"""Fifty product pages of the practice shop over five days, as its server sends them. Every product is invented.

Each fetch differs from the last in ways that are not data: the time the page was generated, a form token, which of
two offer banners is shown and a "recently viewed" box. The data changes too, on purpose: two prices and a
description on day 2, a product out of stock on day 3, three pages of a new template without a price on day 4 only,
and on day 5 a price shown wrongly on the first fetch and a real price cut.
"""
import hashlib

KINDS = ["Basmati rice", "Toor dal", "Green tea", "Jaggery", "Turmeric", "Mustard oil", "Poha", "Rock salt",
         "Cashews", "Cardamom"]
SKUS = [f"NB-{i:04d}" for i in range(1, 51)]
NO_PRICE_DAY_4 = {"NB-0011", "NB-0025", "NB-0042"}


def product(sku, day, attempt):
    """The data behind one page on one day; attempt 2 is a second fetch a few minutes after the first."""
    i = int(sku[3:])
    name = f"{KINDS[i % 10]} {[250, 500, 1000, 2000, 5000][i % 5]} g".replace("1000 g", "1 kg").replace("2000 g", "2 kg").replace("5000 g", "5 kg")
    paise = (60 + (i * 37) % 900) * 100 + 1000
    data = {"name": name, "paise": paise, "stock": "In stock",
            "description": f"{KINDS[i % 10]} from the practice shop's own suppliers. Packed in small batches."}
    if day >= 2 and sku in ("NB-0003", "NB-0017"):
        data["paise"] += 1500
    if day >= 2 and sku == "NB-0021":
        data["description"] = data["description"].replace("small batches.", "small batches and sealed the same day.")
    if day >= 3 and sku == "NB-0009":
        data["stock"] = "Out of stock"
    if day == 4 and sku in NO_PRICE_DAY_4:
        data["paise"] = None  # a new template, tried on three pages for one day, has no price element
    if day == 5 and sku == "NB-0030" and attempt == 1:
        data["paise"] //= 100  # the first fetch shows the price without its last two digits
    if day == 5 and sku == "NB-0040":
        data["paise"] = data["paise"] * 4 // 10  # a real cut of 60 %
    return data


def fetch(sku, day, attempt=1):
    """The HTML the server sends for a product page."""
    i, d = int(sku[3:]), product(sku, day, attempt)
    token = hashlib.sha256(f"{sku}/{day}/{attempt}".encode()).hexdigest()[:20]
    banner = ["Free delivery above 499", "Festive offers this week"][(i + day + attempt) % 2]
    recent = ", ".join(KINDS[(i + day + k) % 10] for k in range(1, 4))
    price = "" if d["paise"] is None else f"<p class='price'>₹{d['paise'] // 100:,}.{d['paise'] % 100:02d}</p>"
    return (f"<!doctype html><html lang='en'><head><title>{d['name']} · Nimbu Bazaar</title></head><body>"
            f"<div class='banner'>{banner}</div><main><h1 class='name'>{d['name']}</h1>{price}"
            f"<p class='stock'>{d['stock']}</p><div class='description'>{d['description']}</div></main>"
            f"<aside class='recent'>Recently viewed: {recent}</aside>"
            f"<form><input type='hidden' name='token' value='{token}'></form>"
            f"<p class='generated'>Page generated at 09:{(i * 7 + day * 3) % 60:02d}:{(i * 13 + day * 11 + attempt) % 60:02d}</p>"
            "</body></html>")