Request handling: Host, headers and smuggling

// Two ways to build the link in a password-reset email. The weak one copies the request's Host header, which any client
// can set to any text; the fixed one takes the link from configuration and answers 400 to requests for hosts the shop
// does not serve. The token is a placeholder, and every host name is reserved for examples.

const PUBLIC_ORIGIN = 'https://shop.example'; // configuration: one value per environment
const ALLOWED_HOSTS = new Set(['shop.example', 'www.shop.example']);

function weakResetLink(headers, token) {
  return `https://${headers.host ?? ''}/reset?token=${token}`;
}

/** The host name of a Host value, in lower case without the port; null when it is not a plain host[:port]. */
function hostName(value) {
  const m = /^([a-z0-9.-]+)(?::\d{1,5})?$/i.exec(value ?? '');
  return m ? m[1].toLowerCase() : null;
}

function fixedResetLink(headers, token) {
  const host = hostName(headers.host);
  if (host === null || !ALLOWED_HOSTS.has(host)) return '400 Bad Request: not a host this site serves';
  return `${PUBLIC_ORIGIN}/reset?token=${token}`;
}

const HOSTS = ['shop.example', 'SHOP.EXAMPLE:443', 'other.example', 'shop.example.other.example', 'shop.example/other', ''];

for (const value of HOSTS) {
  const headers = { host: value };
  console.log(`Host: ${value === '' ? '(empty)' : value}`);
  console.log(`  weak   ${weakResetLink(headers, 'T1')}`);
  console.log(`  fixed  ${fixedResetLink(headers, 'T1')}`);
}

Host: shop.example
  weak   https://shop.example/reset?token=T1
  fixed  https://shop.example/reset?token=T1
Host: SHOP.EXAMPLE:443
  weak   https://SHOP.EXAMPLE:443/reset?token=T1
  fixed  https://shop.example/reset?token=T1
Host: other.example
  weak   https://other.example/reset?token=T1
  fixed  400 Bad Request: not a host this site serves
Host: shop.example.other.example
  weak   https://shop.example.other.example/reset?token=T1
  fixed  400 Bad Request: not a host this site serves
Host: shop.example/other
  weak   https://shop.example/other/reset?token=T1
  fixed  400 Bad Request: not a host this site serves
Host: (empty)
  weak   https:///reset?token=T1
  fixed  400 Bad Request: not a host this site serves

// Which address is the client's? The app sees the address of the connection's other end and an X-Forwarded-For list
// that each proxy appends to, and that a client may start with any text. The shop's path: client -> CDN edge
// (203.0.113.0/24) -> load balancer (10.0.1.5) -> app. Every address is a documentation or a private one.

/** IPv4 address as a number, and an a.b.c.d/n range test. */
const ip4 = (a) => a.split('.').reduce((n, part) => n * 256 + Number(part), 0);
const inRange = (addr, cidr) => {
  const [base, bits] = cidr.split('/');
  const size = 2 ** (32 - Number(bits));
  return Math.floor(ip4(addr) / size) === Math.floor(ip4(base) / size);
};

/** Every X-Forwarded-For line as one list, left to right (several lines are one list, in order). */
const forwardedList = (lines) => lines.flatMap((line) => line.split(',').map((s) => s.trim()).filter(Boolean));

// Three ways to choose. Each walks from the right: the connection first, then the list from its last entry.
const leftmost = (peer, lines) => forwardedList(lines)[0] ?? peer;
function byHops(peer, lines, hops) {
  const chain = [peer, ...forwardedList(lines).reverse()];
  return chain[Math.min(hops, chain.length - 1)];
}
function byTrustedRanges(peer, lines, ranges) {
  const chain = [peer, ...forwardedList(lines).reverse()];
  return chain.find((addr) => !ranges.some((r) => inRange(addr, r))) ?? chain[chain.length - 1];
}

const TRUSTED = ['10.0.1.5/32', '203.0.113.0/24']; // the load balancer and the CDN's edge addresses
const CASES = [
  ['an ordinary visit', '198.51.100.7', '10.0.1.5', ['198.51.100.7, 203.0.113.10']],
  ['the client wrote its own entry', '198.51.100.7', '10.0.1.5', ['192.0.2.66, 198.51.100.7, 203.0.113.10']],
  ['two lines, read as one list', '198.51.100.7', '10.0.1.5', ['192.0.2.66', '198.51.100.7, 203.0.113.10']],
  ['the CDN bypassed, LB reached', '198.51.100.9', '10.0.1.5', ['192.0.2.66, 198.51.100.9']],
];

console.log(`${'Request'.padEnd(32)} ${'client'.padEnd(14)} ${'leftmost'.padEnd(16)} ${'2 hops'.padEnd(16)} trusted ranges`);
for (const [label, client, peer, lines] of CASES) {
  const mark = (addr) => `${addr} ${addr === client ? 'ok' : 'NO'}`;
  const row = [mark(leftmost(peer, lines)).padEnd(16), mark(byHops(peer, lines, 2)).padEnd(16), mark(byTrustedRanges(peer, lines, TRUSTED))];
  console.log(`${label.padEnd(32)} ${client.padEnd(14)} ${row.join(' ')}`);
}

Request                          client         leftmost         2 hops           trusted ranges
an ordinary visit                198.51.100.7   198.51.100.7 ok  198.51.100.7 ok  198.51.100.7 ok
the client wrote its own entry   198.51.100.7   192.0.2.66 NO    198.51.100.7 ok  198.51.100.7 ok
two lines, read as one list      198.51.100.7   192.0.2.66 NO    198.51.100.7 ok  198.51.100.7 ok
the CDN bypassed, LB reached     198.51.100.9   192.0.2.66 NO    192.0.2.66 NO    198.51.100.9 ok

// One access rule at the edge, one router in the app, seven request targets, then a query with a repeated key. The
// edge refuses a raw path that starts with /admin. The weak app parses the target with Node's URL class (the WHATWG URL
// Standard, as browsers parse) and routes without regard to case, as Express's router does by default. The fixed app
// accepts only targets that are already in canonical form and checks access itself, on the same path it routes on.

const BASE = 'https://shop.example';

const edgeAllows = (target) => !target.startsWith('/admin');

function weakApp(target) {
  const url = new URL(target, BASE); // a relative reference: "//x/y" names another host
  const route = url.pathname.toLowerCase().startsWith('/admin') ? 'admin handler' : `page ${url.pathname}`;
  return url.host === 'shop.example' ? route : `${route} on host ${url.host}`;
}

/** The canonical path, or null when the target is not origin-form or the parser would change it. */
function canonical(target) {
  if (!target.startsWith('/') || target.startsWith('//')) return null;
  if (/%2e|%2f|%5c|\\/i.test(target)) return null; // encoded dots and slashes, and backslashes
  const path = new URL(`${BASE}${target}`).pathname; // the base and the target joined, as Node's docs show
  return path === target.split('?')[0] ? path : null;
}

function fixedApp(target, user) {
  const path = canonical(target);
  if (path === null) return '400 not a canonical path';
  if (path.toLowerCase().startsWith('/admin') && user.role !== 'admin') return '403 not an admin';
  return path.toLowerCase().startsWith('/admin') ? 'admin handler' : `page ${path}`;
}

const TARGETS = ['/admin/users', '/Admin/users', '/public/../admin/users', '/public/%2e%2e/admin/users', '/public\\..\\admin/users', '//admin/users', '/products/42'];
const visitor = { role: 'customer' };

console.log(`${'Target'.padEnd(28)} ${'Edge'.padEnd(8)} ${'Weak app'.padEnd(30)} Fixed app`);
for (const target of TARGETS) {
  const edge = edgeAllows(target) ? 'passes' : 'blocks';
  const weak = edge === 'blocks' ? '(never reached)' : weakApp(target);
  console.log(`${target.padEnd(28)} ${edge.padEnd(8)} ${weak.padEnd(30)} ${fixedApp(target, visitor)}`);
}

// Repeated query parameters: two readers of the same query, one value each.
const query = new URLSearchParams('role=customer&role=admin');
console.log(`\n?role=customer&role=admin: get() ${query.get('role')}, Object.fromEntries() ${Object.fromEntries(query).role}, getAll() ${query.getAll('role').join(' and ')}`);

Target                       Edge     Weak app                       Fixed app
/admin/users                 blocks   (never reached)                403 not an admin
/Admin/users                 passes   admin handler                  403 not an admin
/public/../admin/users       passes   admin handler                  400 not a canonical path
/public/%2e%2e/admin/users   passes   admin handler                  400 not a canonical path
/public\..\admin/users       passes   admin handler                  400 not a canonical path
//admin/users                passes   page /users on host admin      400 not a canonical path
/products/42                 passes   page /products/42              page /products/42

?role=customer&role=admin: get() customer, Object.fromEntries() admin, getAll() customer and admin

# Two hops read the same bytes. The front proxy trusts Content-Length when a message has both length headers; the back
# server follows Transfer-Encoding, as RFC 9112 says Transfer-Encoding overrides Content-Length. A model of message
# framing only: nothing here opens a socket. Then both hops use one strict parser that refuses the ambiguous message.

BODY = b"5\r\nhello\r\n0\r\n\r\n"  # "hello" in chunked coding: one chunk of 5 bytes, then the last chunk
EXTRA = b"LEFTOVER-BYTES"  # stands for anything that follows in the same stream
RAW = (
    b"POST /upload HTTP/1.1\r\n"
    b"Host: shop.example\r\n"
    + f"Content-Length: {len(BODY + EXTRA)}\r\n".encode()
    + b"Transfer-Encoding: chunked\r\n"
    b"\r\n"
    + BODY
    + EXTRA
)


def split_head(raw):
    head, _, rest = raw.partition(b"\r\n\r\n")
    fields = [line.split(b":", 1) for line in head.split(b"\r\n")[1:]]
    return {name.strip().lower(): value.strip() for name, value in fields}, rest


def front_reads(raw):
    """A lenient proxy: the body is Content-Length bytes long, whatever else the message says."""
    fields, rest = split_head(raw)
    length = int(fields[b"content-length"])
    return rest[:length], rest[length:]


def back_reads(rest):
    """A server that follows Transfer-Encoding: chunked; the body ends at the chunk of size 0."""
    body, pos = b"", 0
    while True:
        line_end = rest.index(b"\r\n", pos)
        size = int(rest[pos:line_end], 16)
        pos = line_end + 2
        if size == 0:
            return body, rest[pos + 2 :]
        body += rest[pos : pos + size]
        pos += size + 2


def strict(raw):
    """One parser for every hop: a message with both length headers is refused (RFC 9112 sections 6.1 and 6.3)."""
    fields, _ = split_head(raw)
    if b"transfer-encoding" in fields and b"content-length" in fields:
        return "400 Bad Request, connection closed: both Content-Length and Transfer-Encoding"
    return "accepted"


forwarded, after_front = front_reads(RAW)
body, after_back = back_reads(forwarded)
print(f"One message of {len(RAW)} bytes, with Content-Length: {len(BODY + EXTRA)} and Transfer-Encoding: chunked")
print(f"front proxy: one request with a {len(forwarded)}-byte body; {len(after_front)} bytes left over")
print(f"back server: one request with the body {body!r}; {len(after_back)} bytes left over: {after_back!r}")
print("  the back server now reads those bytes as the start of the next request on this connection")
print()
print("With one strict parser on both hops")
print(f"front proxy: {strict(RAW)}")
print("back server: never receives the message")

One message of 122 bytes, with Content-Length: 29 and Transfer-Encoding: chunked
front proxy: one request with a 29-byte body; 0 bytes left over
back server: one request with the body b'hello'; 14 bytes left over: b'LEFTOVER-BYTES'
  the back server now reads those bytes as the start of the next request on this connection

With one strict parser on both hops
front proxy: 400 Bad Request, connection closed: both Content-Length and Transfer-Encoding
back server: never receives the message

// Fifteen raw requests for Node's own HTTP server, with its default options. Each one is handed to the server through
// an in-memory socket (the 'connection' event accepts any Duplex stream), so the bytes reach Node's HTTP parser exactly
// as written, and no network is used. The handler reads the whole body, then answers with what it saw. The same bytes
// then go to a second server whose handler starts with a guard: the checks the parser leaves to the application.
import http from 'node:http';
import { Duplex } from 'node:stream';

const ALLOWED_HOSTS = new Set(['shop.example']);
const CRLF = '\r\n';
const request = (...lines) => lines.join(CRLF);

const REQUESTS = [
  ['an ordinary GET', request('GET /orders HTTP/1.1', 'Host: shop.example', '', '')],
  ['no Host header', request('GET /orders HTTP/1.1', '', '')],
  ['Content-Length and Transfer-Encoding', request('POST /orders HTTP/1.1', 'Host: shop.example', 'Content-Length: 5', 'Transfer-Encoding: chunked', '', '0', '', '')],
  ['two different Content-Length values', request('POST /orders HTTP/1.1', 'Host: shop.example', 'Content-Length: 3', 'Content-Length: 5', '', 'abcde')],
  ['Transfer-Encoding without chunked', request('POST /orders HTTP/1.1', 'Host: shop.example', 'Transfer-Encoding: gzip', '', 'abc')],
  ['a space before the colon', request('GET /orders HTTP/1.1', 'Host : shop.example', '', '')],
  ['obsolete line folding', request('GET /orders HTTP/1.1', 'Host: shop.example', 'X-Note: one', ' two', '', '')],
  ['bare LF line endings', 'GET /orders HTTP/1.1\nHost: shop.example\n\n'],
  ['a NUL byte in a value', request('GET /orders HTTP/1.1', 'Host: shop.example', 'X-Note: a\0b', '', '')],
  ['a 20 KiB header', request('GET /orders HTTP/1.1', 'Host: shop.example', `X-Note: ${'a'.repeat(20 * 1024)}`, '', '')],
  ['two Host lines', request('GET /orders HTTP/1.1', 'Host: shop.example', 'Host: other.example', '', '')],
  ['absolute-form, another host', request('GET http://other.example/orders HTTP/1.1', 'Host: shop.example', '', '')],
  ['a path inside Host', request('GET /orders HTTP/1.1', 'Host: shop.example/other', '', '')],
  ['two X-Forwarded-For lines', request('GET /orders HTTP/1.1', 'Host: shop.example', 'X-Forwarded-For: 192.0.2.66', 'X-Forwarded-For: 198.51.100.7', '', '')],
  ['an underscore in a header name', request('GET /orders HTTP/1.1', 'Host: shop.example', 'X_Forwarded_For: 192.0.2.66', '', '')],
];

/** What a handler saw, in a few words. */
function seen(req) {
  const parts = [`url ${req.url}`, `host ${req.headers.host ?? '(none)'}`];
  if (req.headers['x-forwarded-for']) parts.push(`x-forwarded-for "${req.headers['x-forwarded-for']}"`);
  if (req.headers.x_forwarded_for) parts.push(`x_forwarded_for "${req.headers.x_forwarded_for}"`);
  return parts.join(', ');
}

/** The checks the parser leaves to the application: one Host, a host this site serves, no names with underscores. */
function guard(req) {
  const names = req.rawHeaders.filter((_, i) => i % 2 === 0).map((n) => n.toLowerCase());
  if (names.filter((n) => n === 'host').length !== 1) return 'exactly one Host header';
  const authority = /^https?:\/\/([^/?#]*)/i.exec(req.url)?.[1] ?? req.headers.host; // absolute-form: its own host
  if (!ALLOWED_HOSTS.has(authority.toLowerCase())) return `not a host this site serves: ${authority}`;
  if (names.some((n) => n.includes('_'))) return 'a header name with an underscore';
  return null;
}

function server(withGuard) {
  return http.createServer((req, res) => {
    req.resume(); // read the whole body before answering, as a handler that needs it would
    req.on('end', () => {
      const refused = withGuard ? guard(req) : null;
      const body = refused ? `refused: ${refused}` : seen(req);
      res.writeHead(refused ? 400 : 200, { connection: 'close', 'content-length': Buffer.byteLength(body) });
      res.end(body);
    });
  });
}

/** Sends raw bytes to a server through an in-memory socket and returns its first status line and body. */
function exchange(srv, raw) {
  return new Promise((resolve) => {
    let out = '';
    let sent = false;
    const socket = new Duplex({
      read() {
        if (!sent) this.push(Buffer.from(raw, 'latin1'));
        sent = true;
      },
      write(chunk, _encoding, done) {
        out += chunk.toString('latin1');
        done();
      },
    });
    const finish = () => resolve({ status: out.split(CRLF)[0].replace('HTTP/1.1 ', ''), body: out.split(CRLF + CRLF)[1] ?? '' });
    socket.on('finish', finish);
    socket.on('close', finish);
    srv.emit('connection', socket);
  });
}

const plain = server(false);
const guarded = server(true);
for (const [label, raw] of REQUESTS) {
  const a = await exchange(plain, raw);
  const b = await exchange(guarded, raw);
  console.log(`${label}`);
  console.log(`  Node: ${a.status}${a.body && a.status.startsWith('200') ? `; the handler saw ${a.body}` : ''}`);
  if (a.status.startsWith('200')) console.log(`  with the guard: ${b.status}${b.body.startsWith('refused') ? `, ${b.body}` : ''}`);
}
plain.close();
guarded.close();

an ordinary GET
  Node: 200 OK; the handler saw url /orders, host shop.example
  with the guard: 200 OK
no Host header
  Node: 400 Bad Request
Content-Length and Transfer-Encoding
  Node: 400 Bad Request
two different Content-Length values
  Node: 400 Bad Request
Transfer-Encoding without chunked
  Node: 400 Bad Request
a space before the colon
  Node: 400 Bad Request
obsolete line folding
  Node: 400 Bad Request
bare LF line endings
  Node: 400 Bad Request
a NUL byte in a value
  Node: 400 Bad Request
a 20 KiB header
  Node: 431 Request Header Fields Too Large
two Host lines
  Node: 200 OK; the handler saw url /orders, host shop.example
  with the guard: 400 Bad Request, refused: exactly one Host header
absolute-form, another host
  Node: 200 OK; the handler saw url http://other.example/orders, host shop.example
  with the guard: 400 Bad Request, refused: not a host this site serves: other.example
a path inside Host
  Node: 200 OK; the handler saw url /orders, host shop.example/other
  with the guard: 400 Bad Request, refused: not a host this site serves: shop.example/other
two X-Forwarded-For lines
  Node: 200 OK; the handler saw url /orders, host shop.example, x-forwarded-for "192.0.2.66, 198.51.100.7"
  with the guard: 200 OK
an underscore in a header name
  Node: 200 OK; the handler saw url /orders, host shop.example, x_forwarded_for "192.0.2.66"
  with the guard: 400 Bad Request, refused: a header name with an underscore