Archival PDF/A and final submission checks

from session import files, run

# pdfa/thesis.tex exported as PDF/A-2u and, in a copy, as PDF/A-2b; pdfa/thesis.typ compiled with each standard; then
# vera.py validates every file, first against the 2u profile and then the two 2b files against 2b.
files("vera.py", "pdfa/thesis.tex=thesis.tex", "pdfa/thesis.typ=thesis.typ")
run(r"""
latexmk -pdf -interaction=nonstopmode -silent thesis.tex > /dev/null 2>&1
sed 's/pdfstandard=a-2u/pdfstandard=a-2b/' thesis.tex > thesis-2b.tex
latexmk -pdf -interaction=nonstopmode -silent thesis-2b.tex > /dev/null 2>&1
typst compile --pdf-standard a-2u thesis.typ typst-2u.pdf
typst compile --pdf-standard a-2b thesis.typ typst-2b.pdf
pdfinfo -meta thesis.pdf | grep -E "pdfaid:(part|conformance)"
python3 vera.py 2u thesis.pdf typst-2u.pdf thesis-2b.pdf typst-2b.pdf
python3 vera.py 2b thesis-2b.pdf typst-2b.pdf
""")

$ latexmk -pdf -interaction=nonstopmode -silent thesis.tex > /dev/null 2>&1
$ sed 's/pdfstandard=a-2u/pdfstandard=a-2b/' thesis.tex > thesis-2b.tex
$ latexmk -pdf -interaction=nonstopmode -silent thesis-2b.tex > /dev/null 2>&1
$ typst compile --pdf-standard a-2u thesis.typ typst-2u.pdf
$ typst compile --pdf-standard a-2b thesis.typ typst-2b.pdf
$ pdfinfo -meta thesis.pdf | grep -E "pdfaid:(part|conformance)"
   <pdfaid:part>2</pdfaid:part>
   <pdfaid:conformance>U</pdfaid:conformance>
$ python3 vera.py 2u thesis.pdf typst-2u.pdf thesis-2b.pdf typst-2b.pdf
thesis.pdf: PDF/A-2u validation profile: compliant, 146 rules passed, 0 failed
typst-2u.pdf: PDF/A-2u validation profile: compliant, 146 rules passed, 0 failed
thesis-2b.pdf: PDF/A-2u validation profile: NOT compliant, 145 rules passed, 1 failed
  6.6.4-3 (1 check): The "conformance" property of the PDF/A Identification Schema is B instead of "U" for PDF/A-2u conforming file
typst-2b.pdf: PDF/A-2u validation profile: NOT compliant, 145 rules passed, 1 failed
  6.6.4-3 (1 check): The "conformance" property of the PDF/A Identification Schema is B instead of "U" for PDF/A-2u conforming file
$ python3 vera.py 2b thesis-2b.pdf typst-2b.pdf
thesis-2b.pdf: PDF/A-2b validation profile: compliant, 144 rules passed, 0 failed
typst-2b.pdf: PDF/A-2b validation profile: compliant, 144 rules passed, 0 failed

"""Validates PDFs with veraPDF against one PDF/A or PDF/UA profile and prints a short report of each: the profile, the
verdict, how many of the profile's rules passed and failed, and every failed rule with its clause, its test number,
the number of failed checks and veraPDF's message.

    python3 vera.py 2u thesis.pdf other.pdf

The flavour is veraPDF's name for the profile (2b, 2u, 2a, ua1 …). veraPDF runs once for all the files; its JSON
report is read here because the report also holds times and build dates, which change on every run and are left out,
and the failed rules are printed in the order of the standard's clauses.
"""

import json
import subprocess
import sys


def main(flavour, pdfs):
    done = subprocess.run(["verapdf", "--format", "json", "--flavour", flavour, *pdfs], capture_output=True, text=True)
    report = json.loads(done.stdout)["report"]
    versions = {item["id"]: item["version"] for item in report["buildInformation"]["releaseDetails"]}
    if not versions.get("core", "").startswith("1.30."):
        sys.exit(f"this report needs veraPDF 1.30; found {versions}")
    for job in report["jobs"]:
        name = job["itemDetails"]["name"].rsplit("/", 1)[-1]
        results = job["validationResult"]
        result = results[0] if isinstance(results, list) else results
        details = result["details"]
        verdict = "compliant" if result["compliant"] else "NOT compliant"
        print(f"{name}: {result['profileName']}: {verdict}, "
              f"{details['passedRules']} rules passed, {details['failedRules']} failed")
        # veraPDF lists the failed rules in no fixed order, so they are sorted by clause and test number.
        for rule in sorted(details["ruleSummaries"], key=lambda r: ([int(n) for n in r["clause"].split(".")], r["testNumber"])):
            message = rule["checks"][0].get("errorMessage") if rule["checks"] else None
            checks = f"{rule['failedChecks']} check" + ("s" if rule["failedChecks"] != 1 else "")
            print(f"  {rule['clause']}-{rule['testNumber']} ({checks}): {message or rule['description']}")


if __name__ == "__main__":
    main(sys.argv[1], sys.argv[2:])

from session import files, read, run, shell, write

# The thesis with one more figure: chart.pdf, made by make_chart.py, which uses Helvetica without embedding it.
# pdflatex builds it as PDF/A-2u, and veraPDF judges the result.
files("vera.py", "make_chart.py", "pdfa/thesis.tex=thesis.tex")
shell("python3 make_chart.py chart.pdf")
write("with-chart.tex", read("thesis.tex").replace("% FIGURE", r"\includegraphics{chart.pdf}"))
run(r"""
pdffonts chart.pdf
latexmk -pdf -interaction=nonstopmode -silent with-chart.tex > /dev/null 2>&1
python3 vera.py 2u with-chart.pdf
""")

$ pdffonts chart.pdf
name                                 type              encoding         emb sub uni object ID
------------------------------------ ----------------- ---------------- --- --- --- ---------
Helvetica                            Type 1            WinAnsi          no  no  no       5  0
$ latexmk -pdf -interaction=nonstopmode -silent with-chart.tex > /dev/null 2>&1
$ python3 vera.py 2u with-chart.pdf
with-chart.pdf: PDF/A-2u validation profile: NOT compliant, 145 rules passed, 1 failed
  6.2.11.4.1-1 (1 check): The font program is not embedded

from session import files, read, run, shell, write

# The same figure in the Typst thesis, exported as PDF/A-2u, and the thesis without a date, exported as PDF/A-2u and
# as PDF/A-2b: Typst checks these rules while it exports, and stops.
files("make_chart.py", "pdfa/thesis.typ=thesis.typ")
shell("python3 make_chart.py chart.pdf")
write("with-chart.typ", read("thesis.typ").replace("// FIGURE", '#image("chart.pdf", alt: "Rainfall in mm")'))
write("nodate.typ", "#set document(date: none)\n" + read("thesis.typ"))
run(r"""
!typst compile --pdf-standard a-2u with-chart.typ
!typst compile --pdf-standard a-2u nodate.typ
!typst compile --pdf-standard a-2b nodate.typ
""")

$ typst compile --pdf-standard a-2u with-chart.typ
error: embedding PDFs is currently not supported in this export mode
   ┌─ with-chart.typ:15:1
   │
15 │ #image("chart.pdf", alt: "Rainfall in mm")
   │  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   │
   = hint: try converting the PDF to an SVG before embedding it

[exit status 1]
$ typst compile --pdf-standard a-2u nodate.typ
error: PDF/A-2u error: missing document date
 = hint: set the date of the document

[exit status 1]
$ typst compile --pdf-standard a-2b nodate.typ
error: PDF/A-2b error: missing document date
 = hint: set the date of the document

[exit status 1]

from session import files, read, run, shell, write

# The same figure fixed before it goes into the thesis: Ghostscript writes it again with its font embedded, for LaTeX,
# and pdftocairo turns it into SVG, for Typst. Both theses are then exported as PDF/A-2u and validated.
files("vera.py", "make_chart.py", "pdfa/thesis.tex=thesis.tex", "pdfa/thesis.typ=thesis.typ")
shell("python3 make_chart.py chart.pdf")
write("fixed-latex.tex", read("thesis.tex").replace("% FIGURE", r"\includegraphics{chart-embedded.pdf}"))
write("fixed-typst.typ", read("thesis.typ").replace("// FIGURE", '#image("chart.svg", alt: "Rainfall in mm")'))
run(r"""
gs -q -sDEVICE=pdfwrite -dEmbedAllFonts=true -o chart-embedded.pdf -c "<</NeverEmbed []>> setdistillerparams" -f chart.pdf
pdffonts chart-embedded.pdf
latexmk -pdf -interaction=nonstopmode -silent fixed-latex.tex > /dev/null 2>&1
pdftocairo -svg chart.pdf chart.svg
typst compile --pdf-standard a-2u fixed-typst.typ
python3 vera.py 2u fixed-latex.pdf fixed-typst.pdf
""")

$ gs -q -sDEVICE=pdfwrite -dEmbedAllFonts=true -o chart-embedded.pdf -c "<</NeverEmbed []>> setdistillerparams" -f chart.pdf
$ pdffonts chart-embedded.pdf
name                                 type              encoding         emb sub uni object ID
------------------------------------ ----------------- ---------------- --- --- --- ---------
XBCPUG+Helvetica                     Type 1C           WinAnsi          yes yes no       7  0
$ latexmk -pdf -interaction=nonstopmode -silent fixed-latex.tex > /dev/null 2>&1
$ pdftocairo -svg chart.pdf chart.svg
$ typst compile --pdf-standard a-2u fixed-typst.typ
$ python3 vera.py 2u fixed-latex.pdf fixed-typst.pdf
fixed-latex.pdf: PDF/A-2u validation profile: compliant, 146 rules passed, 0 failed
fixed-typst.pdf: PDF/A-2u validation profile: compliant, 146 rules passed, 0 failed

"""Checks a thesis PDF the way a submission portal might before it accepts the file, and prints one line per check:
what it found, and PASS or FAIL.

    python3 submit_check.py thesis.pdf --max-mb 10 --standard 2u

The checks are the page size (A4, 210 by 297 mm, within 1 mm, on every page), that every font is embedded (pdffonts'
emb column), that the PDF's metadata gives a title and an author (pdfinfo), that the file is no larger than the limit,
that its XMP metadata claims the PDF/A part and level asked for (pdfinfo -meta), and that veraPDF finds it compliant
with that profile. The limit and the standard are the portal's; the ones above are an example.
"""

import json
import os
import re
import subprocess
import sys

MM = 25.4 / 72


def output(*command):
    return subprocess.run(command, capture_output=True, text=True).stdout


def checks(pdf, max_mb, standard):
    info = output("pdfinfo", pdf)
    fields = dict(re.findall(r"^([A-Za-z ]+?):[ \t]*(.*)$", info, re.M))
    pages = int(fields["Pages"])
    sizes = re.findall(r"size:\s+([\d.]+) x ([\d.]+) pts", output("pdfinfo", "-f", "1", "-l", str(pages), pdf))
    mm = {(round(float(w) * MM, 1), round(float(h) * MM, 1)) for w, h in sizes}
    a4 = bool(mm) and all(abs(w - 210) <= 1 and abs(h - 297) <= 1 for w, h in mm)
    yield "page size", ", ".join(f"{w} × {h} mm" for w, h in sorted(mm)) + f" on {pages} pages", a4

    rows = output("pdffonts", pdf).splitlines()[2:]
    embedded = sum(1 for row in rows if row.split()[-5] == "yes")
    yield "fonts embedded", f"{embedded} of {len(rows)}", embedded == len(rows)

    for field in ("Title", "Author"):
        value = fields.get(field, "").strip()
        shown = value if len(value) <= 36 else value[:35] + "…"
        yield field.lower(), shown or "(none)", bool(value)

    size_mb = os.path.getsize(pdf) / 1_000_000
    yield "file size", f"{size_mb:.2f} MB, limit {max_mb:g} MB", size_mb <= max_mb

    meta = output("pdfinfo", "-meta", pdf)
    part = re.search(r"<pdfaid:part>(\d)</pdfaid:part>", meta)
    level = re.search(r"<pdfaid:conformance>([ABU])</pdfaid:conformance>", meta)
    claim = f"PDF/A-{part[1]}{level[1].lower()}" if part and level else "none"
    yield "PDF/A claim", claim, claim == f"PDF/A-{standard}"

    report = json.loads(output("verapdf", "--format", "json", "--flavour", standard, pdf))["report"]
    core = next(item["version"] for item in report["buildInformation"]["releaseDetails"] if item["id"] == "core")
    if not core.startswith("1.30."):
        sys.exit(f"these checks need veraPDF 1.30; found {core}")
    result = report["jobs"][0]["validationResult"]
    result = result[0] if isinstance(result, list) else result
    failed = result["details"]["failedRules"]
    yield "veraPDF", f"PDF/A-{standard}: " + ("compliant" if result["compliant"] else f"{failed} rules failed"), \
        result["compliant"]


def main(pdf, max_mb, standard):
    print(f"{pdf}:")
    print(f"  {'check':<16}{'found':<40}result")
    passed = total = 0
    for name, found, ok in checks(pdf, max_mb, standard):
        total += 1
        passed += ok
        print(f"  {name:<16}{found:<40}{'PASS' if ok else 'FAIL'}")
    print(f"  {passed} of {total} checks pass")


if __name__ == "__main__":
    arguments = sys.argv[1:]
    main(arguments[0], float(arguments[arguments.index("--max-mb") + 1]), arguments[arguments.index("--standard") + 1])

from session import files, read, run, shell, write

# Two PDFs through the same submission checks: a first try, the thesis on the report class's own settings with the
# chart as it came, and the final thesis, PDF/A-2u with the chart whose font Ghostscript embedded.
files("vera.py", "make_chart.py", "submit_check.py", "pdfa/thesis.tex=thesis.tex")
shell("python3 make_chart.py chart.pdf")
shell('gs -q -sDEVICE=pdfwrite -dEmbedAllFonts=true -o chart-embedded.pdf -c "<</NeverEmbed []>> setdistillerparams" '
      '-f chart.pdf')
final = read("thesis.tex").replace("% FIGURE", r"\includegraphics{chart-embedded.pdf}")
first_try = (final.replace("\\DocumentMetadata{lang=en-IN, pdfstandard=a-2u}\n", "")
             .replace("[a4paper,12pt]{report}", "{report}")
             .replace("\\hypersetup{pdftitle={Rooftop Rainwater Harvesting in Example District}, pdfauthor={Nisha Varghese}}\n", "")
             .replace("chart-embedded.pdf", "chart.pdf"))
write("first-try.tex", first_try)
write("final.tex", final)
run(r"""
latexmk -pdf -interaction=nonstopmode -silent first-try.tex > /dev/null 2>&1
latexmk -pdf -interaction=nonstopmode -silent final.tex > /dev/null 2>&1
python3 submit_check.py first-try.pdf --max-mb 10 --standard 2u
python3 submit_check.py final.pdf --max-mb 10 --standard 2u
""")

$ latexmk -pdf -interaction=nonstopmode -silent first-try.tex > /dev/null 2>&1
$ latexmk -pdf -interaction=nonstopmode -silent final.tex > /dev/null 2>&1
$ python3 submit_check.py first-try.pdf --max-mb 10 --standard 2u
first-try.pdf:
  check           found                                   result
  page size       215.9 × 279.4 mm on 2 pages             FAIL
  fonts embedded  2 of 3                                  FAIL
  title           (none)                                  FAIL
  author          (none)                                  FAIL
  file size       0.03 MB, limit 10 MB                    PASS
  PDF/A claim     none                                    FAIL
  veraPDF         PDF/A-2u: 5 rules failed                FAIL
  1 of 7 checks pass
$ python3 submit_check.py final.pdf --max-mb 10 --standard 2u
final.pdf:
  check           found                                   result
  page size       210.0 × 297.0 mm on 2 pages             PASS
  fonts embedded  3 of 3                                  PASS
  title           Rooftop Rainwater Harvesting in Exa…    PASS
  author          Nisha Varghese                          PASS
  file size       0.05 MB, limit 10 MB                    PASS
  PDF/A claim     PDF/A-2u                                PASS
  veraPDF         PDF/A-2u: compliant                     PASS
  7 of 7 checks pass

from session import files, run

# pdfa/thesis.tex exported as PDF/A-2u, then passed through four programs that rewrite a PDF after export, as
# compressors, optimisers and cleaners do: Ghostscript with its ebook settings, qpdf linearising without and with its
# --newline-before-endstream option, and MuPDF's mutool clean. vera.py validates every result; wc -c gives the sizes.
files("vera.py", "pdfa/thesis.tex=thesis.tex")
run(r"""
latexmk -pdf -interaction=nonstopmode -silent thesis.tex > /dev/null 2>&1
gs -q -sDEVICE=pdfwrite -dPDFSETTINGS=/ebook -o ebook.pdf thesis.pdf
qpdf --linearize thesis.pdf linearized.pdf
qpdf --linearize --newline-before-endstream thesis.pdf linearized-eol.pdf
mutool clean -gggg thesis.pdf cleaned.pdf
wc -c thesis.pdf ebook.pdf linearized.pdf linearized-eol.pdf cleaned.pdf
python3 vera.py 2u thesis.pdf ebook.pdf linearized.pdf linearized-eol.pdf cleaned.pdf
""")

$ latexmk -pdf -interaction=nonstopmode -silent thesis.tex > /dev/null 2>&1
$ gs -q -sDEVICE=pdfwrite -dPDFSETTINGS=/ebook -o ebook.pdf thesis.pdf
$ qpdf --linearize thesis.pdf linearized.pdf
$ qpdf --linearize --newline-before-endstream thesis.pdf linearized-eol.pdf
$ mutool clean -gggg thesis.pdf cleaned.pdf
$ wc -c thesis.pdf ebook.pdf linearized.pdf linearized-eol.pdf cleaned.pdf
   48302 thesis.pdf
   11750 ebook.pdf
   49402 linearized.pdf
   49411 linearized-eol.pdf
   50763 cleaned.pdf
  209628 total
$ python3 vera.py 2u thesis.pdf ebook.pdf linearized.pdf linearized-eol.pdf cleaned.pdf
thesis.pdf: PDF/A-2u validation profile: compliant, 146 rules passed, 0 failed
ebook.pdf: PDF/A-2u validation profile: NOT compliant, 143 rules passed, 3 failed
  6.1.7.1-2 (2 checks): Extra spacings or missing EOL characters around keywords 'stream' and 'endstream'
  6.2.4.3-4 (12 checks): DeviceGray colour space is used without output intent profile
  6.6.4-1 (1 check): The document metadata stream doesn't contain PDF/A Identification Schema
linearized.pdf: PDF/A-2u validation profile: NOT compliant, 145 rules passed, 1 failed
  6.1.7.1-2 (10 checks): Extra spacings or missing EOL characters around keywords 'stream' and 'endstream'
linearized-eol.pdf: PDF/A-2u validation profile: compliant, 146 rules passed, 0 failed
cleaned.pdf: PDF/A-2u validation profile: compliant, 146 rules passed, 0 failed