Canonical URLs and duplicate content

[
  {
    "address": "https://punebakes.example/cakes/mango-mousse",
    "status": 200,
    "html": "<head><title>Mango mousse cake | Pune Bakes</title><link rel=\"canonical\" href=\"https://punebakes.example/cakes/mango-mousse\"></head><main><h1>Mango mousse cake</h1><p>Eggless, made with Alphonso mango pulp, half a kilo or one kilo.</p></main>"
  },
  {
    "address": "http://punebakes.example/cakes/mango-mousse",
    "status": 301,
    "location": "https://punebakes.example/cakes/mango-mousse"
  },
  {
    "address": "https://www.punebakes.example/cakes/mango-mousse",
    "status": 301,
    "location": "https://punebakes.example/cakes/mango-mousse"
  },
  {
    "address": "http://www.punebakes.example/cakes/mango-mousse",
    "status": 301,
    "location": "https://punebakes.example/cakes/mango-mousse"
  },
  {
    "address": "https://punebakes.example/cakes/mango-mousse/",
    "status": 200,
    "html": "<head><title>Mango mousse cake | Pune Bakes</title><link rel=\"canonical\" href=\"https://punebakes.example/cakes/mango-mousse\"><link rel=\"canonical\" href=\"https://punebakes.example/cakes/mango-mousse/\"></head><main><h1>Mango mousse cake</h1><p>Eggless, made with Alphonso mango pulp, half a kilo or one kilo.</p></main>"
  },
  {
    "address": "https://punebakes.example/cakes/Mango-Mousse",
    "status": 200,
    "html": "<head><title>Mango mousse cake | Pune Bakes</title></head><main><h1>Mango mousse cake</h1><p>Eggless, made with Alphonso mango pulp, half a kilo or one kilo.</p></main>"
  },
  {
    "address": "https://punebakes.example/cakes/mango-mousse?utm_source=instagram&utm_medium=social",
    "status": 200,
    "html": "<head><title>Mango mousse cake | Pune Bakes</title><link rel=\"canonical\" href=\"https://punebakes.example/cakes/mango-mousse\"></head><main><h1>Mango mousse cake</h1><p>Eggless, made with Alphonso mango pulp, half a kilo or one kilo.</p></main>"
  },
  {
    "address": "https://punebakes.example/cakes/mango-mousse?fbclid=IwAR2x",
    "status": 200,
    "html": "<head><title>Mango mousse cake | Pune Bakes</title><link rel=\"canonical\" href=\"https://punebakes.example/cakes/mango-mousse\"></head><main><h1>Mango mousse cake</h1><p>Eggless, made with Alphonso mango pulp, half a kilo or one kilo.</p></main>"
  },
  {
    "address": "https://punebakes.example/cakes/mango-mousse?ref=menu",
    "status": 200,
    "html": "<head><title>Mango mousse cake | Pune Bakes</title><link rel=\"canonical\" href=\"https://punebakes.example/cakes/mango-mousse?ref=menu\"></head><main><h1>Mango mousse cake</h1><p>Eggless, made with Alphonso mango pulp, half a kilo or one kilo.</p></main>"
  },
  {
    "address": "https://punebakes.example/print/cakes/mango-mousse",
    "status": 200,
    "html": "<head><title>Mango mousse cake (print) | Pune Bakes</title><link rel=\"canonical\" href=\"https://punebakes.example/cakes/mango-mousse\"></head><main><h1>Mango mousse cake</h1><p>Eggless, made with Alphonso mango pulp, half a kilo or one kilo.</p></main>"
  },
  {
    "address": "https://punebakes.example/product.php?id=482",
    "status": 200,
    "html": "<head><title>Mango mousse cake | Pune Bakes</title></head><main><h1>Mango mousse cake</h1><p>Eggless, made with Alphonso mango pulp, half a kilo or one kilo.</p></main>"
  },
  {
    "address": "https://punebakes.example/cakes/mango-mousse?lang=mr",
    "status": 200,
    "html": "<head><title>मँगो मूस केक | Pune Bakes</title><link rel=\"canonical\" href=\"https://punebakes.example/cakes/mango-mousse?lang=mr\"></head><main><h1>मँगो मूस केक</h1><p>अंड्याशिवाय, हापूस आंब्याच्या गरापासून, अर्धा किलो किंवा एक किलो.</p></main>"
  }
]

"""Twelve addresses that answer with Pune Bakes' mango mousse page, and what each one needs.

responses.json holds what the bakery's server sends for each address: a status, a Location for a redirect, and the
HTML of a page. The program groups the pages by their main text, as Google groups duplicates, works out the one
address the bakery prefers for each, and says whether an address needs a redirect, a canonical tag, or nothing.
"""

import json
from html.parser import HTMLParser
from urllib.parse import parse_qsl, urlencode, urlsplit

SITE = "https://punebakes.example"
TRACKING = ("utm_", "fbclid", "gclid", "msclkid", "ref")  # parameters that never change what the page shows
OLD_IDS = {"482": "/cakes/mango-mousse"}  # the shop's old product.php addresses


class Page(HTMLParser):
    """Reads the canonical link of a page and the text of its main element."""

    def __init__(self, html):
        super().__init__()
        self.canonical, self.main, self.inside = None, "", False
        self.feed(html)

    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        if tag == "link" and a.get("rel") == "canonical":
            self.canonical = a.get("href")
        self.inside = self.inside or tag == "main"

    def handle_data(self, data):
        self.main += data if self.inside else ""


def preferred(address):
    """https, no www, lower case, no slash after a product, no tracking parameters, no print folder."""
    u = urlsplit(address)
    path = u.path.lower().removeprefix("/print")
    query = [(k, v) for k, v in parse_qsl(u.query) if not k.startswith(TRACKING)]
    if path == "/product.php":
        path, query = OLD_IDS[dict(query)["id"]], []
    if path.count("/") > 2:
        path = path.rstrip("/")
    return SITE + path + ("?" + urlencode(sorted(query)) if query else "")


def short(address):
    return address.removeprefix(SITE) or "/"


def needs(r, want):
    if r["status"] == 301:
        return "nothing: one 301 to it" if r["location"] == want else f"301 to {short(want)}, in one hop"
    page = Page(r["html"])
    if r["address"] == want:
        return "nothing: it names itself" if page.canonical == want else "a canonical tag to itself"
    u = urlsplit(r["address"])
    reachable = u.path.startswith("/print/") or any(k.startswith(TRACKING) for k, _ in parse_qsl(u.query))
    if not reachable:
        return f"301 to {short(want)}"
    return "nothing: its canonical names it" if page.canonical == want else f"canonical tag to {short(want)}"


responses = json.load(open("responses.json", encoding="utf-8"))
by_address = {r["address"]: r for r in responses}
groups = {}  # main text -> addresses that show it (a redirect shows its target's page)
for r in responses:
    shown = by_address[r["location"]] if r["status"] == 301 else r
    groups.setdefault(Page(shown["html"]).main, []).append(r)
sizes = ", ".join(f"one at {len(g)}" for g in groups.values())
print(f"{len(responses)} addresses show {len(groups)} different pages: {sizes}")
fixes = []
for group in groups.values():
    want = preferred(group[0]["address"])
    print(f"\nThe page whose preferred address is {short(want)}:")
    for r in group:
        need = needs(r, want)
        fixes += [need] if not need.startswith("nothing") else []
        shown = short(r["address"]) if r["address"].startswith(SITE + "/") else r["address"]
        print(f"  {r['status']}  {shown}\n       needs {need}")
redirects = sum(f.startswith("301") for f in fixes)
print(f"\nTo fix: {redirects} addresses need a 301, {len(fixes) - redirects} needs a canonical tag")

12 addresses show 2 different pages: one at 11, one at 1

The page whose preferred address is /cakes/mango-mousse:
  200  /cakes/mango-mousse
       needs nothing: it names itself
  301  http://punebakes.example/cakes/mango-mousse
       needs nothing: one 301 to it
  301  https://www.punebakes.example/cakes/mango-mousse
       needs nothing: one 301 to it
  301  http://www.punebakes.example/cakes/mango-mousse
       needs nothing: one 301 to it
  200  /cakes/mango-mousse/
       needs 301 to /cakes/mango-mousse
  200  /cakes/Mango-Mousse
       needs 301 to /cakes/mango-mousse
  200  /cakes/mango-mousse?utm_source=instagram&utm_medium=social
       needs nothing: its canonical names it
  200  /cakes/mango-mousse?fbclid=IwAR2x
       needs nothing: its canonical names it
  200  /cakes/mango-mousse?ref=menu
       needs canonical tag to /cakes/mango-mousse
  200  /print/cakes/mango-mousse
       needs nothing: its canonical names it
  200  /product.php?id=482
       needs 301 to /cakes/mango-mousse

The page whose preferred address is /cakes/mango-mousse?lang=mr:
  200  /cakes/mango-mousse?lang=mr
       needs nothing: it names itself

To fix: 3 addresses need a 301, 1 needs a canonical tag

"""Check the canonical declarations of eight Pune Bakes addresses against Google's rules.

Each page lists the HTML its server sends (only the start of it) and any Link header. ANSWERS says how each target
answers. A canonical counts only in a valid head: Google stops reading the head at the first element that does not
belong there, such as an img. The target should be an absolute address that answers 200 and may be indexed.
"""

from html.parser import HTMLParser

SITE = "https://punebakes.example"
HEAD_ELEMENTS = {"title", "meta", "link", "script", "style", "base", "noscript", "template"}
ANSWERS = {  # address -> (status, carries noindex)
    "/cakes/rasmalai-cake": (200, False),
    "/cakes/eggless-truffle": (200, False),
    "/cakes/blueberry-cake": (301, False),
    "/offers/": (200, True),
    "/cakes/coffee-walnut-cake": (404, False),
    "/cakes/mango-mousse": (200, False),
    "/menu/": (200, False),
}
C = '<link rel="canonical" href="https://punebakes.example'  # the start of an absolute canonical link
PAGES = [  # address, the start of the HTML its server sends, its Link header (or None)
    ("/cakes/rasmalai-cake", f'<head><title>Rasmalai cake</title>{C}/cakes/rasmalai-cake"></head>', None),
    ("/cakes/eggless-truffle", '<head><link rel="canonical" href="/cakes/eggless-truffle"></head>', None),
    ("/cakes/red-velvet-heart",
     f'<head><title>Red velvet</title><img src="/badge.png">{C}/cakes/red-velvet-heart"></head>', None),
    ("/cakes/mango-mousse/", f'<head>{C}/cakes/mango-mousse">{C}/cakes/mango-mousse/"></head>', None),
    ("/cakes/blueberry-cheesecake", f'<head>{C}/cakes/blueberry-cake"></head>', None),
    ("/offers/diwali-hampers", f'<head>{C}/offers/"></head>', None),
    ("/cakes/coffee-walnut", f'<head>{C}/cakes/coffee-walnut-cake"></head>', None),
    ("/menu.pdf", "", '<https://punebakes.example/menu/>; rel="canonical"'),
]


class Head(HTMLParser):
    """The canonical links Google reads: those before the head ends, or before an element that ends it early."""

    def __init__(self, html):
        super().__init__()
        self.read, self.ignored, self.open, self.ended_by = [], [], True, None
        self.feed(html)

    def handle_starttag(self, tag, attrs):
        if self.open and tag not in HEAD_ELEMENTS | {"head"}:
            self.open, self.ended_by = False, tag  # Google assumes the head ended here
        a = dict(attrs)
        if tag == "link" and a.get("rel") == "canonical":
            (self.read if self.open else self.ignored).append(a.get("href", ""))

    def handle_endtag(self, tag):
        self.open = self.open and tag != "head"


def problems(address, html, header):
    head = Head(html)
    named = head.read + ([header.split(">")[0].lstrip("<")] if header else [])
    found = [f"canonical after <{head.ended_by}> is ignored: the head ended there" for _ in head.ignored]
    if len(set(named)) > 1:
        found.append(f"{len(named)} canonicals that disagree")
    for href in dict.fromkeys(named):  # each address once, in the order the page names them
        target = href.removeprefix(SITE)
        if not href.startswith("https://"):
            found.append(f"{href} is relative: write the absolute address")
        status, noindex = ANSWERS.get(target, (None, False))
        if status not in (200, None):
            found.append(f"the target {target} answers {status}")
        if noindex:
            found.append(f"the target {target} carries noindex")
    if not named and not head.ignored:
        return ["no canonical"]
    return found or ["ok"]


for address, html, header in PAGES:
    print(address)
    for problem in problems(address, html, header):
        print(f"  {problem}")

/cakes/rasmalai-cake
  ok
/cakes/eggless-truffle
  /cakes/eggless-truffle is relative: write the absolute address
/cakes/red-velvet-heart
  canonical after <img> is ignored: the head ended there
/cakes/mango-mousse/
  2 canonicals that disagree
/cakes/blueberry-cheesecake
  the target /cakes/blueberry-cake answers 301
/offers/diwali-hampers
  the target /offers/ carries noindex
/cakes/coffee-walnut
  the target /cakes/coffee-walnut-cake answers 404
/menu.pdf
  ok

"""Which address does each signal of Pune Bakes' cakes listing name, and what makes them agree?

The listing answers 200 at three addresses. A crawl of the site counted the internal links to each one; the page's
canonical tag, the sitemap and the server's redirects are as they are today. The program lists every signal, says
whether they agree, and then makes the changes that point them all at one preferred address.
"""

SITE = {
    "redirects": {},  # address -> where it redirects (none yet)
    "canonical tag": "/cakes/?ref=menu",  # what <link rel="canonical"> on the listing says
    "sitemap": ["/cakes/"],
    "internal links": {"/cakes": 46, "/cakes/": 3, "/cakes/?ref=menu": 1},  # links found on the site's pages
}
PREFERRED = "/cakes/"  # the bakery's rule: a folder keeps its slash, and no tracking parameters


def votes(site):
    """(signal, the address it names) for every signal the listing sends."""
    out = [("redirect", target) for target in site["redirects"].values()]
    out.append(("canonical tag", site["canonical tag"]))
    out += [("sitemap", address) for address in site["sitemap"]]
    for address, count in site["internal links"].items():
        out.append((f"{count} internal link{'s' * (count > 1)}", address))
    return out


def report(title, site):
    named = votes(site)
    print(title)
    for signal, address in named:
        print(f"  {signal:<18} {address}")
    addresses = sorted({address for _, address in named})
    if len(addresses) == 1:
        print(f"  Every signal names {addresses[0]}.")
    else:
        print(f"  The signals name {len(addresses)} addresses: {', '.join(addresses)}. Google must choose.")


def agree(site, preferred):
    """The changes that make every signal name the preferred address."""
    others = [a for a in site["internal links"] if a != preferred]
    moved = sum(site["internal links"][a] for a in others)
    changes = [f"301 from {a} to {preferred}" for a in others if "?" not in a]
    changes.append(f"canonical tag: {preferred} instead of {site['canonical tag']}")
    changes.append(f"{moved} internal links: {preferred} instead of {' or '.join(others)}")
    fixed = {
        "redirects": {a: preferred for a in others if "?" not in a},
        "canonical tag": preferred,
        "sitemap": [preferred],
        "internal links": {preferred: sum(site["internal links"].values())},
    }
    return changes, fixed


report("Today:", SITE)
changes, fixed = agree(SITE, PREFERRED)
print("\nChanges:")
for n, change in enumerate(changes, 1):
    print(f"  {n}. {change}")
print()
report("After the changes:", fixed)

Today:
  canonical tag      /cakes/?ref=menu
  sitemap            /cakes/
  46 internal links  /cakes
  3 internal links   /cakes/
  1 internal link    /cakes/?ref=menu
  The signals name 3 addresses: /cakes, /cakes/, /cakes/?ref=menu. Google must choose.

Changes:
  1. 301 from /cakes to /cakes/
  2. canonical tag: /cakes/ instead of /cakes/?ref=menu
  3. 47 internal links: /cakes/ instead of /cakes or /cakes/?ref=menu

After the changes:
  redirect           /cakes/
  canonical tag      /cakes/
  sitemap            /cakes/
  50 internal links  /cakes/
  Every signal names /cakes/.