Caching and conditional requests: ETag, 304

cache/conditional_get.py

"""Visits to the price board with validators: the first downloads it, the next ask "has it changed?" and get 304
with no body until the board changes at 12 hours. Then a day of visits every 10 minutes, with and without them."""
import nimbu_lab as lab


def visit(when, headers):
    lab.clock.seconds = when
    status, answer, body = lab.get("/board", headers)
    shown = ", ".join(f"{k}: {v}" for k, v in headers.items()) or "no validators"
    print(f"{when / 3600:>5.1f} h  {shown:<52}  {status}  {len(body):>5} bytes  ETag {answer['ETag']}")
    return answer


lab.reset()
first = visit(0, {})
visit(600, {"If-None-Match": first["ETag"]})
newer = visit(43_800, {"If-None-Match": first["ETag"]})
visit(44_400, {"If-None-Match": newer["ETag"]})
print("Last-Modified of the first answer:", first["Last-Modified"])
visit(600, {"If-Modified-Since": first["Last-Modified"]})
print()

for label, use_validators in [("without validators", False), ("with If-None-Match", True)]:
    lab.reset()
    etag, downloaded = None, 0
    for visit_number in range(144):  # every 10 minutes for a day
        lab.clock.seconds = visit_number * 600
        status, answer, body = lab.get("/board", {"If-None-Match": etag} if use_validators and etag else {})
        etag = answer["ETag"]
        downloaded += len(body)
    statuses = [entry["status"] for entry in lab.LOG]
    print(f"A day, {label}: {len(statuses)} requests, {statuses.count(200)} answered 200 and "
          f"{statuses.count(304)} answered 304, {downloaded:,} bytes of pages")

  0.0 h  no validators                                         200   2111 bytes  ETag "ed6f053bf0293c66"
  0.2 h  If-None-Match: "ed6f053bf0293c66"                     304      0 bytes  ETag "ed6f053bf0293c66"
 12.2 h  If-None-Match: "ed6f053bf0293c66"                     200   2112 bytes  ETag "b1a526728549bf32"
 12.3 h  If-None-Match: "b1a526728549bf32"                     304      0 bytes  ETag "b1a526728549bf32"
Last-Modified of the first answer: Fri, 15 Jan 2027 08:00:00 GMT
  0.2 h  If-Modified-Since: Fri, 15 Jan 2027 08:00:00 GMT      304      0 bytes  ETag "ed6f053bf0293c66"

A day, without validators: 144 requests, 144 answered 200 and 0 answered 304, 304,056 bytes of pages
A day, with If-None-Match: 144 requests, 2 answered 200 and 142 answered 304, 4,223 bytes of pages

nimbu_lab.py (the shop) · cache/nimbu_lab.py

"""Nimbu Bazaar, the practice shop of this track, with the pages of the caching lesson. Every product, price and
person in it is invented.

The shop is a WSGI application (PEP 3333): app() answers one request. get() sends a request through Python's own
HTTP client, urllib.request, whose last step hands it to app() instead of opening a socket, so nothing leaves your
program. Time is the shop's clock, which you move on with clock.sleep(), and LOG keeps what the shop saw.

The pages:
  /board                the wholesale price board: ETag, Last-Modified and Cache-Control: max-age=300; it changes
                        at 12 hours on the shop's clock
  /products/NB-0001/ …  240 product pages, each with its ETag and Last-Modified; new_night(n) changes 12 of them
  /broken/rotating      an ETag that changes on every answer, although the page does not
  /broken/ignores       validators sent, but If-None-Match ignored: always 200 with the whole page
  /broken/second        Last-Modified only, to the second, on a page that changes twice within one second

Conditional requests are answered as RFC 9110 section 13.2.2 orders them: If-None-Match first, compared with the
weak comparison; If-Modified-Since only when there is no If-None-Match.
"""
import email.utils
import hashlib
import http.client
import io
import random
import urllib.error
import urllib.request
import urllib.response
from urllib.parse import urlsplit

HOST = "bazaar.nimbu.test"
EPOCH = 1_800_000_000  # the clock reads 0 at this moment, in seconds since 1970
KINDS = ["Basmati rice", "Toor dal", "Green tea", "Jaggery", "Turmeric", "Mustard oil", "Poha", "Rock salt",
         "Cashews", "Cardamom", "Ragi flour", "Ghee", "Honey", "Peanuts", "Coffee", "Raisins", "Sesame seeds",
         "Tamarind", "Cumin", "Black pepper", "Rolled oats", "Cloves", "Fennel", "Moong dal", "Chana dal",
         "Wheat flour", "Semolina", "Coriander", "Red chilli", "Almonds", "Dates", "Coconut oil", "Brown rice",
         "Millet", "Flax seeds", "Cinnamon", "Ginger powder", "Vermicelli", "Masoor dal", "Sago"]
SIZES = [250, 500, 1000, 2000, 5000, 10000]  # grams


class Clock:
    """The shop's clock, in seconds."""

    def __init__(self):
        self.seconds = 0.0

    def now(self):
        return self.seconds

    def sleep(self, seconds):
        self.seconds += max(0.0, seconds)


clock = Clock()
LOG = []  # what the shop saw: one entry per request
STATE = {}  # each product's version and when it last changed; counters of the broken pages


def products():
    """The 240 products: 40 kinds in 6 pack sizes."""
    out = []
    for k, kind in enumerate(KINDS):
        per_kg = 60 + (k * 37) % 900
        for grams in SIZES:
            size = f"{grams // 1000} kg" if grams >= 1000 else f"{grams} g"
            out.append({"sku": f"NB-{len(out) + 1:04d}", "name": f"{kind} {size}", "paise": per_kg * grams // 10 + 1000})
    return out


PRODUCTS = {p["sku"]: p for p in products()}


def reset():
    """A fresh shop: the clock at 0, an empty log, every product at version 1."""
    clock.seconds = 0.0
    LOG.clear()
    STATE.clear()
    STATE.update({sku: (1, 0.0) for sku in PRODUCTS}, rotating=0, second=[])


def new_night(n):
    """Move the clock to the start of night n (n days after 0) and change 12 products' prices, chosen by n."""
    clock.seconds = n * 86_400.0
    for sku in random.Random(n).sample(sorted(PRODUCTS), 12):
        version, _ = STATE[sku]
        STATE[sku] = (version + 1, clock.seconds)


def http_date(seconds):
    return email.utils.formatdate(EPOCH + seconds, usegmt=True)


def etag_of(body):
    """A strong validator: a hash of the bytes, which changes whenever the page does."""
    return '"' + hashlib.sha256(body).hexdigest()[:16] + '"'


def tags_in(field):
    """The opaque tags of an If-None-Match field, with any W/ prefix dropped: the weak comparison."""
    return {t.strip().removeprefix("W/") for t in field.split(",") if t.strip()}


def conditional(headers, etag, modified, body, extra=()):
    """A 304 with no body when the client's copy is current, else a 200 with the page (RFC 9110, 13.2.2)."""
    common = [("Date", http_date(clock.now())), *extra]
    if etag:
        common.append(("ETag", etag))
    if modified is not None:
        common.append(("Last-Modified", http_date(modified)))
    inm, ims = headers.get("If-None-Match"), headers.get("If-Modified-Since")
    if inm is not None:
        unchanged = "*" in tags_in(inm) or (etag is not None and etag.removeprefix("W/") in tags_in(inm))
    elif ims is not None and modified is not None:
        try:
            unchanged = int(modified) <= email.utils.parsedate_to_datetime(ims).timestamp() - EPOCH
        except (TypeError, ValueError):
            unchanged = False
    else:
        unchanged = False
    if unchanged:
        return "304 Not Modified", common, b""
    return "200 OK", [("Content-Type", "text/html; charset=utf-8"), ("Content-Length", str(len(body))), *common], body


def board_page():
    """The price board: 40 rows; its prices change at 12 hours."""
    turn = 1 if clock.now() >= 43_200 else 0
    rows = "".join(f"<tr><td>{p['name']}</td><td>₹{(p['paise'] + 150 * turn) // 100:,}.00</td></tr>\n"
                   for p in list(PRODUCTS.values())[::6])
    body = f"<!doctype html>\n<title>Price board · Nimbu Bazaar</title>\n<table>\n{rows}</table>\n".encode()
    return body, 43_200.0 if turn else 0.0


def product_page(sku):
    version, changed = STATE[sku]
    p = PRODUCTS[sku]
    price = p["paise"] + 25 * (version - 1)
    filler = "".join(f"<li>Pack note {i}: store in a cool, dry place.</li>\n" for i in range(24))
    body = (f"<!doctype html>\n<title>{p['name']} · Nimbu Bazaar</title>\n<h1>{p['name']}</h1>\n"
            f"<p class=\"price\">₹{price // 100:,}.{price % 100:02d}</p>\n<ul>\n{filler}</ul>\n").encode()
    return body, changed


def app(environ, start_response):
    """The shop: answers one request, and writes it into LOG first."""
    path = environ["PATH_INFO"]
    headers = {k[5:].replace("_", "-").title(): v for k, v in environ.items() if k.startswith("HTTP_")}
    if environ.get("HTTP_HOST") != HOST:
        status, out, body = "421 Misdirected Request", [("Content-Type", "text/plain")], b"not a Nimbu Bazaar address\n"
    elif path == "/board":
        page, modified = board_page()
        status, out, body = conditional(headers, etag_of(page), modified, page, [("Cache-Control", "max-age=300")])
    elif path.startswith("/products/") and path.split("/")[2] in PRODUCTS:
        page, modified = product_page(path.split("/")[2])
        status, out, body = conditional(headers, etag_of(page), modified, page)
    elif path == "/broken/rotating":  # the same page, a new ETag every time (say, a counter in the tag)
        STATE["rotating"] += 1
        page = b"<!doctype html>\n<title>Delivery areas</title>\n<p>We deliver to 18 areas.</p>\n"
        status, out, body = conditional(headers, f'"r{STATE["rotating"]}"', 0.0, page)
    elif path == "/broken/ignores":  # sends validators, never reads them
        page = b"<!doctype html>\n<title>Opening hours</title>\n<p>9 am to 9 pm, every day.</p>\n"
        status, out, body = conditional({}, etag_of(page), 0.0, page)
    elif path == "/broken/second":  # two changes at 100.2 s and 100.7 s, one Last-Modified of 100 s
        edits = [t for t in (100.2, 100.7) if clock.now() >= t]
        page = f"<!doctype html>\n<title>Stock note</title>\n<p>Edits so far: {len(edits)}</p>\n".encode()
        status, out, body = conditional(headers, None, int(edits[-1]) if edits else 0.0, page)
    else:
        status, out, body = "404 Not Found", [("Content-Type", "text/plain")], b"no such page\n"
    LOG.append({"t": clock.now(), "path": path, "status": int(status[:3]), "bytes": len(body),
                "if_none_match": headers.get("If-None-Match"), "if_modified_since": headers.get("If-Modified-Since")})
    start_response(status, out)
    return [body]


class ShopTransport(urllib.request.AbstractHTTPHandler):
    """urllib's last step, with app() where the socket would be."""

    handler_order = 100
    http_request = https_request = urllib.request.AbstractHTTPHandler.do_request_

    def https_open(self, req):
        headers = dict(req.unredirected_hdrs)
        headers.update({k: v for k, v in req.headers.items() if k not in headers})
        parts = urlsplit(req.full_url)
        environ = {"REQUEST_METHOD": req.get_method(), "PATH_INFO": parts.path or "/", "QUERY_STRING": parts.query,
                   "wsgi.input": io.BytesIO(b"")}
        environ.update({"HTTP_" + k.upper().replace("-", "_"): v for k, v in headers.items()})
        got = {}
        body = b"".join(app(environ, lambda status, hdrs, exc_info=None: got.update(status=status, headers=hdrs)))
        raw = "".join(f"{k}: {v}\r\n" for k, v in got["headers"]) + "\r\n"
        message = http.client.parse_headers(io.BytesIO(raw.encode("iso-8859-1")))
        response = urllib.response.addinfourl(io.BytesIO(body), message, req.full_url, int(got["status"][:3]))
        response.reason = response.msg = got["status"][4:]
        return response

    http_open = https_open


OPENER = urllib.request.OpenerDirector()
for _handler in [ShopTransport(), urllib.request.HTTPDefaultErrorHandler(), urllib.request.HTTPErrorProcessor(),
                 urllib.request.UnknownHandler()]:
    OPENER.add_handler(_handler)


def get(path, headers=None):
    """GET path through urllib: (status, headers, body). urllib raises HTTPError for 304 and errors; this returns
    them, so a 304 comes back as (304, its headers, b"")."""
    request = urllib.request.Request(f"https://{HOST}{path}", headers={
        "User-Agent": "NimbuLearner/1.0 (+https://example.org/my-bot; you@example.org)", **(headers or {})})
    try:
        with OPENER.open(request, timeout=10) as response:
            return response.status, response.headers, response.read()
    except urllib.error.HTTPError as error:
        return error.code, error.headers, error.read()


reset()