JSON APIs: decoding limits, validation and errors

// decode.go: POST /orders reads its body through three gates, the media type, a 64 KB limit and one strict decode,
// and eight requests show the status each way of refusing a body gets.
package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"mime"
	"net/http"
	"net/http/httptest"
	"strings"
)

type Order struct {
	Quantity int    `json:"quantity"`
	Pincode  string `json:"pincode"`
	Payment  string `json:"payment"`
	Name     string `json:"name"`
}

const maxBody = 64 << 10 // 64 KB, far more than any order needs

// decodeOrder reads exactly one Order from the request, or returns the status and the message that refuse it.
func decodeOrder(w http.ResponseWriter, r *http.Request) (Order, int, string) {
	mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
	if err != nil || mediaType != "application/json" {
		return Order{}, http.StatusUnsupportedMediaType, "send the order as application/json"
	}
	r.Body = http.MaxBytesReader(w, r.Body, maxBody)
	dec := json.NewDecoder(r.Body)
	dec.DisallowUnknownFields()
	var o Order
	if err := dec.Decode(&o); err != nil {
		if tooBig, ok := errors.AsType[*http.MaxBytesError](err); ok {
			return Order{}, http.StatusRequestEntityTooLarge, fmt.Sprintf("the body is over %d bytes", tooBig.Limit)
		}
		return Order{}, http.StatusBadRequest, "the body is not one JSON order"
	}
	if err := dec.Decode(&struct{}{}); err != io.EOF {
		return Order{}, http.StatusBadRequest, "the body holds more than one JSON value"
	}
	return o, 0, ""
}

func createOrder(w http.ResponseWriter, r *http.Request) {
	o, status, msg := decodeOrder(w, r)
	if status != 0 {
		http.Error(w, msg, status)
		return
	}
	w.WriteHeader(http.StatusCreated)
	fmt.Fprintf(w, "order for %d copies to %s, paid by %s, for %s\n", o.Quantity, o.Pincode, o.Payment, o.Name)
}

func main() {
	good := `{"quantity":2,"pincode":"560001","payment":"card","name":"Asha"}`
	requests := []struct{ what, contentType, body string }{
		{"a valid order", "application/json", good},
		{"the same, with a charset", "application/json; charset=utf-8", good},
		{"sent as a form", "application/x-www-form-urlencoded", "quantity=2"},
		{"70,000 bytes", "application/json", `{"name":"` + strings.Repeat("a", 70000) + `"}`},
		{"a misspelt field", "application/json", `{"quantitty":2,"pincode":"560001","payment":"card","name":"Asha"}`},
		{"a string for a number", "application/json", `{"quantity":"two"}`},
		{"two orders in one body", "application/json", good + good},
		{"an empty body", "application/json", ""},
	}
	mux := http.NewServeMux()
	mux.HandleFunc("POST /orders", createOrder)
	for i, rq := range requests {
		req := httptest.NewRequest("POST", "/orders", strings.NewReader(rq.body))
		req.Header.Set("Content-Type", rq.contentType)
		rec := httptest.NewRecorder()
		mux.ServeHTTP(rec, req)
		fmt.Printf("%d. %s: %d %s\n", i+1, rq.what, rec.Code, strings.TrimSpace(rec.Body.String()))
	}
}

1. a valid order: 201 order for 2 copies to 560001, paid by card, for Asha
2. the same, with a charset: 201 order for 2 copies to 560001, paid by card, for Asha
3. sent as a form: 415 send the order as application/json
4. 70,000 bytes: 413 the body is over 65536 bytes
5. a misspelt field: 400 the body is not one JSON order
6. a string for a number: 400 the body is not one JSON order
7. two orders in one body: 400 the body holds more than one JSON value
8. an empty body: 400 the body is not one JSON order

// validate.go: the rules an order must meet once it has decoded, written as an allowlist, a range, a format and a
// length, with every broken rule collected so that one reply can name them all.
package main

import (
	"fmt"
	"strings"
	"unicode/utf8"
)

type Order struct {
	Quantity int    `json:"quantity"`
	Pincode  string `json:"pincode"`
	Payment  string `json:"payment"`
	Name     string `json:"name"`
}

// FieldError names one rule an order broke: the field, as a JSON Pointer into the body, and what it must be.
type FieldError struct {
	Pointer string `json:"pointer"`
	Detail  string `json:"detail"`
}

// payments is an allowlist: a mode the shop accepts is listed, and everything else is refused.
var payments = map[string]bool{"card": true, "cod": true, "netbanking": true, "upi": true}

func validate(o Order) []FieldError {
	var errs []FieldError
	if o.Quantity < 1 || o.Quantity > 20 {
		errs = append(errs, FieldError{"/quantity", "must be from 1 to 20"})
	}
	if !isPincode(o.Pincode) {
		errs = append(errs, FieldError{"/pincode", "must be six digits, the first one not 0"})
	}
	if !payments[o.Payment] {
		errs = append(errs, FieldError{"/payment", "must be one of card, cod, netbanking, upi"})
	}
	if n := utf8.RuneCountInString(o.Name); n < 1 || n > 80 { // the name as it will be stored, counted in characters
		errs = append(errs, FieldError{"/name", "must have 1 to 80 characters"})
	}
	return errs
}

// isPincode accepts exactly six ASCII digits whose first digit is not 0.
func isPincode(s string) bool {
	if len(s) != 6 || s[0] == '0' {
		return false
	}
	for i := 0; i < len(s); i++ {
		if s[i] < '0' || s[i] > '9' {
			return false
		}
	}
	return true
}

func main() {
	orders := []Order{
		{2, "560001", "card", "Asha"},
		{2, "560001", "card", "अनन्या शर्मा"},
		{0, "560001", "card", "Asha"},
		{25, "056001", "wallet", ""},
		{3, "5600011", "Card", strings.Repeat("x", 81)},
	}
	for i, o := range orders {
		errs := validate(o)
		if len(errs) == 0 {
			fmt.Printf("%d. valid: %d for %q (%d runes, %d bytes)\n", i+1, o.Quantity, o.Name,
				utf8.RuneCountInString(o.Name), len(o.Name))
			continue
		}
		fmt.Printf("%d. refused, rules broken: %d\n", i+1, len(errs))
		for _, e := range errs {
			fmt.Printf("     %s %s\n", e.Pointer, e.Detail)
		}
	}
}

1. valid: 2 for "Asha" (4 runes, 4 bytes)
2. valid: 2 for "अनन्या शर्मा" (12 runes, 34 bytes)
3. refused, rules broken: 1
     /quantity must be from 1 to 20
4. refused, rules broken: 4
     /quantity must be from 1 to 20
     /pincode must be six digits, the first one not 0
     /payment must be one of card, cod, netbanking, upi
     /name must have 1 to 80 characters
5. refused, rules broken: 3
     /pincode must be six digits, the first one not 0
     /payment must be one of card, cod, netbanking, upi
     /name must have 1 to 80 characters

// problem.go: every error of POST /orders written as RFC 9457 problem details, application/problem+json with type,
// title, status and detail, and an errors member that points at each broken field. A 500 says nothing about the cause.
package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
)

type FieldError struct {
	Pointer string `json:"pointer"`
	Detail  string `json:"detail"`
}

// Problem is the body of every error reply. Type is a URI that names the kind of problem, the same for every
// occurrence; Detail is about this occurrence.
type Problem struct {
	Type   string       `json:"type"`
	Title  string       `json:"title"`
	Status int          `json:"status"`
	Detail string       `json:"detail,omitempty"`
	Errors []FieldError `json:"errors,omitempty"`
}

func writeProblem(w http.ResponseWriter, p Problem) {
	w.Header().Set("Content-Type", "application/problem+json")
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(p.Status) // the same status as in the body, as RFC 9457 requires
	enc := json.NewEncoder(w)
	enc.SetIndent("", "  ")
	enc.Encode(p)
}

var errStore = errors.New(`pq: relation "orders_v2" does not exist`) // a failure the client must never see

var logger = log.New(os.Stdout, "  log: ", 0)

func createOrder(w http.ResponseWriter, r *http.Request) {
	switch r.URL.Query().Get("case") {
	case "invalid":
		writeProblem(w, Problem{
			Type:   "https://library.test/problems/invalid-order",
			Title:  "The order is not valid",
			Status: http.StatusUnprocessableEntity,
			Detail: "2 fields break the rules for an order.",
			Errors: []FieldError{
				{"/quantity", "must be from 1 to 20"},
				{"/payment", "must be one of card, cod, netbanking, upi"},
			},
		})
	case "form":
		writeProblem(w, Problem{
			Type:   "https://library.test/problems/media-type",
			Title:  "The body must be JSON",
			Status: http.StatusUnsupportedMediaType,
			Detail: "Send the order with Content-Type: application/json.",
		})
	default:
		// The cause goes to the log, with what a person needs to find it; the reply gets a generic problem.
		logger.Printf("POST /orders: saving the order: %v", errStore)
		writeProblem(w, Problem{
			Type:   "about:blank",
			Title:  "Internal Server Error",
			Status: http.StatusInternalServerError,
		})
	}
}

func main() {
	for _, c := range []string{"invalid", "form", "store-down"} {
		rec := httptest.NewRecorder()
		createOrder(rec, httptest.NewRequest("POST", "/orders?case="+c, nil))
		fmt.Printf("%s: %d, Content-Type %s\n", c, rec.Code, rec.Header().Get("Content-Type"))
		fmt.Print(rec.Body.String())
		if strings.Contains(rec.Body.String(), "orders_v2") {
			fmt.Println("  check, the reply names no table: FAIL")
		} else {
			fmt.Println("  check, the reply names no table: pass")
		}
	}
}

invalid: 422, Content-Type application/problem+json
{
  "type": "https://library.test/problems/invalid-order",
  "title": "The order is not valid",
  "status": 422,
  "detail": "2 fields break the rules for an order.",
  "errors": [
    {
      "pointer": "/quantity",
      "detail": "must be from 1 to 20"
    },
    {
      "pointer": "/payment",
      "detail": "must be one of card, cod, netbanking, upi"
    }
  ]
}
  check, the reply names no table: pass
form: 415, Content-Type application/problem+json
{
  "type": "https://library.test/problems/media-type",
  "title": "The body must be JSON",
  "status": 415,
  "detail": "Send the order with Content-Type: application/json."
}
  check, the reply names no table: pass
  log: POST /orders: saving the order: pq: relation "orders_v2" does not exist
store-down: 500, Content-Type application/problem+json
{
  "type": "about:blank",
  "title": "Internal Server Error",
  "status": 500
}
  check, the reply names no table: pass

from session import run

# The starter CreateOrder against its tests: it decodes leniently, ignores the error and answers 201 to everything.
run(
    """
cd practice && go test .
""",
    expect_failure=("cd practice && go test .",),
)

$ cd practice && go test .
--- FAIL: TestCreateOrder (0.00s)
    orders_test.go:36: a form: status 201, want 415
    orders_test.go:36: 70,000 bytes: status 201, want 413
    orders_test.go:36: a misspelt field: status 201, want 400
    orders_test.go:36: two orders: status 201, want 400
    orders_test.go:36: zero copies: status 201, want 422
--- FAIL: TestInvalidOrderNamesEveryField (0.00s)
    orders_test.go:60: status 201 with errors at [], want 422 with errors at /quantity and /payment
FAIL
FAIL	example.com/practice	1.141s
FAIL
[exit status 1]