JSON Lines, JSONC, JSON5 and other variants

Python · lines/write_jsonl.py

# Write JSON Lines: one json.dumps per record, without indent, and a line ending after each. A line break inside a
# string is written as \n, so no record can spill onto a second line, and a new record is simply appended.
import json


def line_of(record):
    return json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n"


records = [
    {"id": "BK-10046", "note": "gift wrap\nno invoice"},
    {"id": "BK-10047", "note": "पुणे शाखा"},
]
text = "".join(line_of(r) for r in records)
print(f"{len(records)} records, {text.count(chr(10))} lines:")
print(text, end="")

text += line_of({"id": "BK-10048", "note": None})  # appending rewrites nothing
print(f"after appending BK-10048: {text.count(chr(10))} lines")

# The mistake: indent= spreads every record over several lines, and no line parses on its own.
spread = "".join(json.dumps(r, ensure_ascii=False, indent=2) + "\n" for r in records)
lines = spread.splitlines()
parsed = 0
for line in lines:
    try:
        json.loads(line)
        parsed += 1
    except json.JSONDecodeError:
        pass
print(f"with indent=2: {len(lines)} lines, {parsed} of them parse on their own")

2 records, 2 lines:
{"id":"BK-10046","note":"gift wrap\nno invoice"}
{"id":"BK-10047","note":"पुणे शाखा"}
after appending BK-10048: 3 lines
with indent=2: 8 lines, 0 of them parse on their own

JavaScript · lines/write_jsonl.mjs

// Write JSON Lines: one JSON.stringify per record, without a space argument, and a line ending after each. A line break
// inside a string is written as \n, so no record can spill onto a second line, and a new record is simply appended.
const lineOf = (record) => `${JSON.stringify(record)}\n`;

const records = [
  { id: 'BK-10046', note: 'gift wrap\nno invoice' },
  { id: 'BK-10047', note: 'पुणे शाखा' },
];
let text = records.map(lineOf).join('');
const count = (s) => s.split('\n').length - 1;
console.log(`${records.length} records, ${count(text)} lines:`);
console.log(text.slice(0, -1));

text += lineOf({ id: 'BK-10048', note: null }); // appending rewrites nothing
console.log(`after appending BK-10048: ${count(text)} lines`);

// The mistake: a space argument spreads every record over several lines, and no line parses on its own.
const spread = records.map((r) => `${JSON.stringify(r, null, 2)}\n`).join('');
const lines = spread.split('\n').slice(0, -1);
let parsed = 0;
for (const line of lines) {
  try {
    JSON.parse(line);
    parsed += 1;
  } catch {
    // not a whole JSON text
  }
}
console.log(`with a space argument of 2: ${lines.length} lines, ${parsed} of them parse on their own`);

2 records, 2 lines:
{"id":"BK-10046","note":"gift wrap\nno invoice"}
{"id":"BK-10047","note":"पुणे शाखा"}
after appending BK-10048: 3 lines
with a space argument of 2: 8 lines, 0 of them parse on their own

Python · lines/read_jsonl.py

# Read a JSON Lines export one line at a time: keep every good record and report each bad line by its number.
import json

good = []
bad = []
with open("orders.jsonl", encoding="utf-8") as export:
    for number, line in enumerate(export, start=1):
        text = line.rstrip("\n")
        if not text.strip():
            bad.append((number, "blank line: JSON Lines has no empty records"))
            continue
        try:
            good.append(json.loads(text))
        except json.JSONDecodeError as e:
            problem = f"{e.msg} (column {e.colno})"
            if not line.endswith("\n"):
                problem += "; the file ends here without a line ending, so it may be cut off"
            bad.append((number, problem))

for number, problem in bad:
    print(f"line {number}: {problem}")
print(f"kept {len(good)} records, reported {len(bad)} lines")
print("kept:", ", ".join(record["id"] for record in good))
print("paise in the kept records:", sum(record["amount_paise"] for record in good))

line 3: blank line: JSON Lines has no empty records
line 4: Illegal trailing comma before end of object (column 61)
line 6: Expecting ',' delimiter (column 56); the file ends here without a line ending, so it may be cut off
kept 3 records, reported 3 lines
kept: BK-10041, BK-10042, BK-10044
paise in the kept records: 177800

JavaScript · lines/read_jsonl.mjs

// The same export as orders.jsonl, as one string: a page gets it from fetch() or a file input. Each line is parsed
// on its own, so one bad line costs one record, never the file.
const text = [
  '{"id": "BK-10041", "branch": "Pune", "amount_paise": 59900}',
  '{"id": "BK-10042", "branch": "Pune", "amount_paise": 37900}',
  '',
  '{"id": "BK-10043", "branch": "Nagpur", "amount_paise": 12500,}',
  '{"id": "BK-10044", "branch": "Nashik", "amount_paise": 80000}',
  '{"id": "BK-10045", "branch": "Pune", "amount_paise": 24',
].join('\n');

const lines = text.split('\n');
// A file that ends with a line ending, as JSON Lines recommends, splits into one empty string after the last record.
if (lines.at(-1) === '') lines.pop();

const good = [];
const bad = [];
lines.forEach((line, i) => {
  const number = i + 1;
  if (line.trim() === '') {
    bad.push([number, 'blank line: JSON Lines has no empty records']);
    return;
  }
  try {
    good.push(JSON.parse(line));
  } catch (e) {
    bad.push([number, `${e.name}: ${e.message}`]);
  }
});

for (const [number, problem] of bad) console.log(`line ${number}: ${problem}`);
console.log(`kept ${good.length} records, reported ${bad.length} lines`);
console.log('kept:', good.map((record) => record.id).join(', '));
console.log('paise in the kept records:', good.reduce((sum, record) => sum + record.amount_paise, 0));

line 3: blank line: JSON Lines has no empty records
line 4: SyntaxError: Expected double-quoted property name in JSON at position 61 (line 1 column 62)
line 6: SyntaxError: Expected ',' or '}' after property value in JSON at position 55 (line 1 column 56)
kept 3 records, reported 3 lines
kept: BK-10041, BK-10042, BK-10044
paise in the kept records: 177800

Python · seq/json_seq.py

# RFC 7464 JSON text sequences: RS (U+001E) before every JSON text and LF after it. A reader splits at RS, so a
# damaged record costs only itself, and the LF shows whether a number at the end was cut off.
import json

RS, LF = "\x1e", "\n"


def show(text):
    """The stream with its control characters made visible."""
    return text.replace(RS, "<RS>").replace(LF, "<LF>")


def write_seq(values):
    return "".join(RS + json.dumps(v, separators=(",", ":")) + LF for v in values)


def read_seq(text):
    kept, warnings = [], []
    # Text before the first RS belongs to no record; RS twice in a row leaves an empty piece, which is no record.
    pieces = [p for p in text.split(RS)[1:] if p]
    for number, piece in enumerate(pieces, start=1):
        try:
            value = json.loads(piece)
        except json.JSONDecodeError:
            warnings.append(f"record {number}: {show(piece)} is not a whole JSON text, skipped")
            continue
        # Numbers, true, false and null do not show where they end, so whitespace must follow them (RFC 7464, 2.4).
        if not isinstance(value, (dict, list, str)) and not piece[-1].isspace():
            warnings.append(f"record {number}: {show(piece)} has nothing after it and may be cut off, dropped")
            continue
        kept.append(value)
    return kept, warnings


stream = write_seq([{"id": "BK-10041", "qty": 2}, {"id": "BK-10042", "qty": 1}])
print("written:", show(stream))
print("read back:", json.dumps(read_seq(stream)[0], separators=(",", ":")))

# A writer stopped in the middle of record 2, the next writer appended record 3, and the stream was cut after "12".
damaged = RS + '{"id":"BK-10041","qty":2}' + LF + RS + '{"id":"BK-100' + RS + '{"id":"BK-10043","qty":4}' + LF + RS + "12"
kept, warnings = read_seq(damaged)
for w in warnings:
    print(w)
print("kept:", ", ".join(r["id"] for r in kept))

# The same damage in JSON Lines: the cut-off record and the next one share a line, and both are lost.
as_lines = '{"id":"BK-10041","qty":2}\n{"id":"BK-100{"id":"BK-10043","qty":4}\n12'
for number, line in enumerate(as_lines.split("\n"), start=1):
    try:
        json.loads(line)
        print(f"JSON Lines line {number}: parsed")
    except json.JSONDecodeError:
        print(f"JSON Lines line {number}: not JSON")

written: <RS>{"id":"BK-10041","qty":2}<LF><RS>{"id":"BK-10042","qty":1}<LF>
read back: [{"id":"BK-10041","qty":2},{"id":"BK-10042","qty":1}]
record 2: {"id":"BK-100 is not a whole JSON text, skipped
record 4: 12 has nothing after it and may be cut off, dropped
kept: BK-10041, BK-10043
JSON Lines line 1: parsed
JSON Lines line 2: not JSON
JSON Lines line 3: parsed

JavaScript · seq/json_seq.mjs

// RFC 7464 JSON text sequences: RS (U+001E) before every JSON text and LF after it. A reader splits at RS, so a
// damaged record costs only itself, and the LF shows whether a number at the end was cut off.
const RS = '\u001e';
const LF = '\n';

/** The stream with its control characters made visible. */
const show = (text) => text.replaceAll(RS, '<RS>').replaceAll(LF, '<LF>');

const writeSeq = (values) => values.map((v) => RS + JSON.stringify(v) + LF).join('');

function readSeq(text) {
  const kept = [];
  const warnings = [];
  // Text before the first RS belongs to no record; RS twice in a row leaves an empty piece, which is no record.
  const pieces = text.split(RS).slice(1).filter((p) => p !== '');
  pieces.forEach((piece, i) => {
    const number = i + 1;
    let value;
    try {
      value = JSON.parse(piece);
    } catch {
      warnings.push(`record ${number}: ${show(piece)} is not a whole JSON text, skipped`);
      return;
    }
    // Numbers, true, false and null do not show where they end, so whitespace must follow them (RFC 7464, 2.4).
    const delimitsItself = typeof value === 'string' || (typeof value === 'object' && value !== null);
    if (!delimitsItself && !/\s$/.test(piece)) {
      warnings.push(`record ${number}: ${show(piece)} has nothing after it and may be cut off, dropped`);
      return;
    }
    kept.push(value);
  });
  return { kept, warnings };
}

const stream = writeSeq([{ id: 'BK-10041', qty: 2 }, { id: 'BK-10042', qty: 1 }]);
console.log('written:', show(stream));
console.log('read back:', JSON.stringify(readSeq(stream).kept));

// A writer stopped in the middle of record 2, the next writer appended record 3, and the stream was cut after "12".
const damaged = RS + '{"id":"BK-10041","qty":2}' + LF + RS + '{"id":"BK-100' + RS + '{"id":"BK-10043","qty":4}' + LF + RS + '12';
const { kept, warnings } = readSeq(damaged);
for (const w of warnings) console.log(w);
console.log('kept:', kept.map((r) => r.id).join(', '));

// The same damage in JSON Lines: the cut-off record and the next one share a line, and both are lost.
const asLines = '{"id":"BK-10041","qty":2}\n{"id":"BK-100{"id":"BK-10043","qty":4}\n12';
asLines.split('\n').forEach((line, i) => {
  try {
    JSON.parse(line);
    console.log(`JSON Lines line ${i + 1}: parsed`);
  } catch {
    console.log(`JSON Lines line ${i + 1}: not JSON`);
  }
});

written: <RS>{"id":"BK-10041","qty":2}<LF><RS>{"id":"BK-10042","qty":1}<LF>
read back: [{"id":"BK-10041","qty":2},{"id":"BK-10042","qty":1}]
record 2: {"id":"BK-100 is not a whole JSON text, skipped
record 4: 12 has nothing after it and may be cut off, dropped
kept: BK-10041, BK-10043
JSON Lines line 1: parsed
JSON Lines line 2: not JSON
JSON Lines line 3: parsed

JavaScript · jsonc/jsonc.mjs

// A small JSONC reader. Comments outside strings become spaces (line endings stay, so positions still point at the
// right line and column), trailing commas are refused unless allowed, and what is left goes to JSON.parse, which
// accepts only strict JSON. Strings are skipped whole, escapes included, so "//" or "/*" inside a string stays text.

const JSON_SPACE = new Set([' ', '\t', '\n', '\r']);

/** "line 6, column 32" for position i of text. */
export function where(text, i) {
  const before = text.slice(0, i);
  return `line ${before.split('\n').length}, column ${i - before.lastIndexOf('\n')}`;
}

/** Where the string that starts at i (an opening quote) ends: the position after its closing quote. */
function endOfString(text, i) {
  let j = i + 1;
  while (j < text.length && text[j] !== '"') j += text[j] === '\\' ? 2 : 1;
  return j + 1;
}

/** text with every comment outside strings replaced by spaces, line endings kept. */
export function blankComments(text) {
  let out = '';
  let i = 0;
  while (i < text.length) {
    if (text[i] === '"') {
      const end = endOfString(text, i);
      out += text.slice(i, end);
      i = end;
    } else if (text.startsWith('//', i)) {
      let end = text.indexOf('\n', i);
      if (end === -1) end = text.length;
      out += ' '.repeat(end - i);
      i = end;
    } else if (text.startsWith('/*', i)) {
      const close = text.indexOf('*/', i + 2);
      if (close === -1) throw new SyntaxError(`comment not closed (${where(text, i)})`);
      out += text.slice(i, close + 2).replace(/[^\n]/g, ' ');
      i = close + 2;
    } else {
      out += text[i];
      i += 1;
    }
  }
  return out;
}

/** Positions of the commas, outside strings, that only white space separates from a closing } or ]. */
export function trailingCommas(text) {
  const found = [];
  let i = 0;
  while (i < text.length) {
    if (text[i] === '"') {
      i = endOfString(text, i);
      continue;
    }
    if (text[i] === ',') {
      let j = i + 1;
      while (JSON_SPACE.has(text[j])) j += 1;
      if (text[j] === '}' || text[j] === ']') found.push(i);
    }
    i += 1;
  }
  return found;
}

/** The value of a JSONC text; trailing commas only with allowTrailingCommas, as jsonc.org leaves them optional. */
export function parseJsonc(text, { allowTrailingCommas = false } = {}) {
  let strict = blankComments(text);
  const commas = trailingCommas(strict);
  if (commas.length && !allowTrailingCommas) throw new SyntaxError(`trailing comma (${where(text, commas[0])})`);
  for (const i of commas) strict = `${strict.slice(0, i)} ${strict.slice(i + 1)}`;
  return JSON.parse(strict);
}

Python · jsonc/jsonc.py

# A small JSONC reader. Comments outside strings become spaces (line endings stay, so positions still point at the
# right line and column), trailing commas are refused unless allowed, and what is left goes to json.loads. Strings are
# skipped whole, escapes included, so "//" or "/*" inside a string stays text.
import json
import re

JSON_SPACE = " \t\n\r"


class JsoncError(ValueError):
    """A JSONC text that this reader refuses."""


def refuse_constant(name):
    """json.loads accepts NaN and Infinity unless told otherwise; strict JSON has neither."""
    raise JsoncError(f"{name} is not JSON")


def where(text, i):
    """'line 6, column 32' for position i of text."""
    before = text[:i]
    return f"line {before.count(chr(10)) + 1}, column {i - before.rfind(chr(10))}"


def end_of_string(text, i):
    """Where the string that starts at i (an opening quote) ends: the position after its closing quote."""
    j = i + 1
    while j < len(text) and text[j] != '"':
        j += 2 if text[j] == "\\" else 1
    return j + 1


def blank_comments(text):
    """text with every comment outside strings replaced by spaces, line endings kept."""
    out = []
    i = 0
    while i < len(text):
        if text[i] == '"':
            end = end_of_string(text, i)
            out.append(text[i:end])
            i = end
        elif text.startswith("//", i):
            end = text.find("\n", i)
            if end == -1:
                end = len(text)
            out.append(" " * (end - i))
            i = end
        elif text.startswith("/*", i):
            close = text.find("*/", i + 2)
            if close == -1:
                raise JsoncError(f"comment not closed ({where(text, i)})")
            out.append(re.sub(r"[^\n]", " ", text[i : close + 2]))
            i = close + 2
        else:
            out.append(text[i])
            i += 1
    return "".join(out)


def trailing_commas(text):
    """Positions of the commas, outside strings, that only white space separates from a closing } or ]."""
    found = []
    i = 0
    while i < len(text):
        if text[i] == '"':
            i = end_of_string(text, i)
            continue
        if text[i] == ",":
            j = i + 1
            while j < len(text) and text[j] in JSON_SPACE:
                j += 1
            if j < len(text) and text[j] in "}]":
                found.append(i)
        i += 1
    return found


def parse_jsonc(text, allow_trailing_commas=False):
    """The value of a JSONC text; trailing commas only when allowed, as jsonc.org leaves them optional."""
    strict = blank_comments(text)
    commas = trailing_commas(strict)
    if commas and not allow_trailing_commas:
        raise JsoncError(f"trailing comma ({where(text, commas[0])})")
    for i in commas:
        strict = strict[:i] + " " + strict[i + 1 :]
    return json.loads(strict, parse_constant=refuse_constant)

JavaScript · jsonc/strip_comments.mjs

// A settings file written in JSONC, read three ways: a regular expression that deletes comments, and the reader of
// jsonc.mjs with trailing commas refused, then allowed.
import { parseJsonc } from './jsonc.mjs';

const settings = `{
  // Where the nightly export goes
  "exportUrl": "https://example.com/orders",
  /* Branches that send orders;
     Nashik joins later. */
  "branches": ["Pune", "Nagpur",],
  "retry": { "times": 3, "pauseMs": 500 } // milliseconds
}`;

// 1. Delete everything from // to the end of a line, and every /* ... */.
const naive = settings.replace(/\/\/.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '');
console.log('regular expression, line 3 is now:', JSON.stringify(naive.split('\n')[2]));
try {
  JSON.parse(naive);
  console.log('  JSON.parse accepted it');
} catch {
  console.log('  JSON.parse refused it: the URL lost its // and the string its closing quote');
}

// 2. The reader that knows where strings are, with trailing commas refused (the default).
try {
  parseJsonc(settings);
} catch (e) {
  console.log('reader, trailing commas refused:', e.message);
}

// 3. The same reader with trailing commas allowed: strict JSON comes out, ready for any parser.
const value = parseJsonc(settings, { allowTrailingCommas: true });
console.log('reader, trailing commas allowed:', JSON.stringify(value));

regular expression, line 3 is now: "  \"exportUrl\": \"https:"
  JSON.parse refused it: the URL lost its // and the string its closing quote
reader, trailing commas refused: trailing comma (line 6, column 32)
reader, trailing commas allowed: {"exportUrl":"https://example.com/orders","branches":["Pune","Nagpur"],"retry":{"times":3,"pauseMs":500}}

Python · jsonc/strip_comments.py

# A settings file written in JSONC, read three ways: a regular expression that deletes comments, and the reader of
# jsonc.py with trailing commas refused, then allowed.
import json
import re

from jsonc import JsoncError, parse_jsonc

settings = """{
  // Where the nightly export goes
  "exportUrl": "https://example.com/orders",
  /* Branches that send orders;
     Nashik joins later. */
  "branches": ["Pune", "Nagpur",],
  "retry": { "times": 3, "pauseMs": 500 } // milliseconds
}"""

# 1. Delete everything from // to the end of a line, and every /* ... */.
naive = re.sub(r"/\*.*?\*/", "", re.sub(r"//.*$", "", settings, flags=re.M), flags=re.S)
print("regular expression, line 3 is now:", json.dumps(naive.split("\n")[2]))
try:
    json.loads(naive)
    print("  json.loads accepted it")
except json.JSONDecodeError:
    print("  json.loads refused it: the URL lost its // and the string its closing quote")

# 2. The reader that knows where strings are, with trailing commas refused (the default).
try:
    parse_jsonc(settings)
except JsoncError as e:
    print("reader, trailing commas refused:", e)

# 3. The same reader with trailing commas allowed: strict JSON comes out, ready for any parser.
value = parse_jsonc(settings, allow_trailing_commas=True)
print("reader, trailing commas allowed:", json.dumps(value, separators=(",", ":")))

regular expression, line 3 is now: "  \"exportUrl\": \"https:"
  json.loads refused it: the URL lost its // and the string its closing quote
reader, trailing commas refused: trailing comma (line 6, column 32)
reader, trailing commas allowed: {"exportUrl":"https://example.com/orders","branches":["Pune","Nagpur"],"retry":{"times":3,"pauseMs":500}}

JavaScript · jsonc/variant_matrix.mjs

// Ten small documents, from strict JSON to features only JSON5 has, through JSON.parse and through the JSONC reader of
// jsonc.mjs (trailing commas allowed). "yes" means the reader returned a value.
import { parseJsonc } from './jsonc.mjs';

const inputs = [
  '{"qty": 2}',
  '{"qty": 2 /* boxes */}',
  '{"qty": 2} // per order',
  '{"qty": 2,}',
  '{qty: 2}',
  "{'sku': 'BK-101'}",
  '{"mask": 0xFF}',
  '{"ratio": .5}',
  '{"limit": Infinity}',
  '{"delta": +1}',
];

const accepts = (read, text) => {
  try {
    read(text);
    return 'yes';
  } catch {
    return 'no';
  }
};

console.log(`${'input'.padEnd(26)}${'JSON.parse'.padEnd(12)}JSONC reader`);
for (const text of inputs) {
  const strict = accepts(JSON.parse, text);
  const jsonc = accepts((t) => parseJsonc(t, { allowTrailingCommas: true }), text);
  console.log(`${text.padEnd(26)}${strict.padEnd(12)}${jsonc}`);
}

input                     JSON.parse  JSONC reader
{"qty": 2}                yes         yes
{"qty": 2 /* boxes */}    no          yes
{"qty": 2} // per order   no          yes
{"qty": 2,}               no          yes
{qty: 2}                  no          no
{'sku': 'BK-101'}         no          no
{"mask": 0xFF}            no          no
{"ratio": .5}             no          no
{"limit": Infinity}       no          no
{"delta": +1}             no          no

Python · jsonc/variant_matrix.py

# Ten small documents, from strict JSON to features only JSON5 has, through json.loads and through the JSONC reader of
# jsonc.py (trailing commas allowed). "yes" means the reader returned a value.
import json

from jsonc import parse_jsonc

inputs = [
    '{"qty": 2}',
    '{"qty": 2 /* boxes */}',
    '{"qty": 2} // per order',
    '{"qty": 2,}',
    "{qty: 2}",
    "{'sku': 'BK-101'}",
    '{"mask": 0xFF}',
    '{"ratio": .5}',
    '{"limit": Infinity}',
    '{"delta": +1}',
]


def accepts(read, text):
    try:
        read(text)
        return "yes"
    except ValueError:  # json.JSONDecodeError and JsoncError are both ValueErrors
        return "no"


print(f"{'input':<26}{'json.loads':<12}JSONC reader")
for text in inputs:
    strict = accepts(json.loads, text)
    jsonc = accepts(lambda t: parse_jsonc(t, allow_trailing_commas=True), text)
    print(f"{text:<26}{strict:<12}{jsonc}")

input                     json.loads  JSONC reader
{"qty": 2}                yes         yes
{"qty": 2 /* boxes */}    no          yes
{"qty": 2} // per order   no          yes
{"qty": 2,}               no          yes
{qty: 2}                  no          no
{'sku': 'BK-101'}         no          no
{"mask": 0xFF}            no          no
{"ratio": .5}             no          no
{"limit": Infinity}       yes         no
{"delta": +1}             no          no