Batch processing and lakehouse architectures

Python 3.14 · main.py

# Word count in three phases: map, shuffle and reduce.
SPLITS = ["the cat sat", "the dog sat", "the cat ran"]  # one split per map task
REDUCERS = 2


def crc32(word: str) -> int:
    # CRC-32, bit by bit, as zlib computes it
    crc = 0xFFFFFFFF
    for byte in word.encode():
        crc ^= byte
        for _ in range(8):
            crc = (crc >> 1) ^ (0xEDB88320 if crc & 1 else 0)
    return crc ^ 0xFFFFFFFF


print("map: each task emits (word, 1) pairs")
pairs = []
for n, split in enumerate(SPLITS):
    emitted = [(word, 1) for word in split.split()]
    pairs += emitted
    print(f"  map {n}: " + " ".join(f"({word}, {one})" for word, one in emitted))

print("shuffle by crc32(word) mod 2, then reduce: each reducer sums its words")
groups: list[dict[str, list[int]]] = [{} for _ in range(REDUCERS)]
for word, one in pairs:
    groups[crc32(word) % REDUCERS].setdefault(word, []).append(one)
for r, group in enumerate(groups):
    words = sorted(group.items())
    received = sum(len(values) for _, values in words)
    print(
        f"  reducer {r} receives {received} pairs: "
        + ", ".join(f"{w} {v}" for w, v in words)
    )
    print(f"  reducer {r} outputs: " + ", ".join(f"{w} {sum(v)}" for w, v in words))

In Python: zlib.crc32(word.encode()) gives the same value in one call; the loop is written out so that every language computes it the same way.

Recorded with CPython 3.14.8 on macOS 26 arm64. It runs in your browser with Pyodide 314.0.7 (CPython 3.14.2). To run it yourself: python3 main.py

JavaScript Node 24 · main.mjs

// Word count in three phases: map, shuffle and reduce.
const SPLITS = ["the cat sat", "the dog sat", "the cat ran"]; // one split per map task
const REDUCERS = 2;

function crc32(word) {
  // CRC-32, bit by bit, as zlib computes it
  let crc = 0xffffffff;
  for (const ch of word) {
    crc ^= ch.charCodeAt(0);
    for (let i = 0; i < 8; i++) {
      crc = (crc >>> 1) ^ (crc & 1 ? 0xedb88320 : 0);
    }
  }
  return (crc ^ 0xffffffff) >>> 0;
}

console.log("map: each task emits (word, 1) pairs");
const pairs = [];
SPLITS.forEach((split, n) => {
  const emitted = split.split(" ").map((word) => [word, 1]);
  pairs.push(...emitted);
  console.log(`  map ${n}: ${emitted.map(([word, one]) => `(${word}, ${one})`).join(" ")}`);
});

console.log("shuffle by crc32(word) mod 2, then reduce: each reducer sums its words");
const groups = Array.from({ length: REDUCERS }, () => new Map());
for (const [word, one] of pairs) {
  const group = groups[crc32(word) % REDUCERS];
  group.set(word, [...(group.get(word) ?? []), one]);
}
groups.forEach((group, r) => {
  const words = [...group.entries()].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
  const received = words.reduce((n, [, values]) => n + values.length, 0);
  const shown = words.map(([word, values]) => `${word} [${values.join(", ")}]`);
  console.log(`  reducer ${r} receives ${received} pairs: ${shown.join(", ")}`);
  const sums = words.map(([word, values]) => `${word} ${values.reduce((a, b) => a + b)}`);
  console.log(`  reducer ${r} outputs: ${sums.join(", ")}`);
});

In JavaScript: The CRC needs >>> for an unsigned shift, and the last >>> 0 turns the result back into a number from 0 to 2³² − 1.

Recorded with Node.js 24.21.0 LTS (V8 13.6) on macOS 26 arm64. It runs in your browser with QuickJS (quickjs-emscripten 0.32.0). To run it yourself: node main.mjs

Java 25 LTS · Main.java

import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.TreeMap;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class Main {
  // Word count in three phases: map, shuffle and reduce.
  static final List<String> SPLITS = List.of("the cat sat", "the dog sat", "the cat ran");
  static final int REDUCERS = 2;

  record Pair(String word, int one) {}

  static long crc32(String word) {
    // CRC-32, bit by bit, as zlib computes it
    int crc = 0xFFFFFFFF;
    for (byte b : word.getBytes(StandardCharsets.UTF_8)) {
      crc ^= b & 0xFF;
      for (int i = 0; i < 8; i++) {
        crc = (crc >>> 1) ^ ((crc & 1) != 0 ? 0xEDB88320 : 0);
      }
    }
    return Integer.toUnsignedLong(~crc);
  }

  public static void main(String[] args) {
    System.out.println("map: each task emits (word, 1) pairs");
    List<Pair> pairs = new ArrayList<>();
    for (int n = 0; n < SPLITS.size(); n++) {
      List<Pair> emitted = Stream.of(SPLITS.get(n).split(" ")).map(w -> new Pair(w, 1)).toList();
      pairs.addAll(emitted);
      String shown =
          emitted.stream()
              .map(p -> "(" + p.word() + ", " + p.one() + ")")
              .collect(Collectors.joining(" "));
      System.out.println("  map " + n + ": " + shown);
    }

    System.out.println("shuffle by crc32(word) mod 2, then reduce: each reducer sums its words");
    List<TreeMap<String, List<Integer>>> groups = new ArrayList<>();
    for (int r = 0; r < REDUCERS; r++) {
      groups.add(new TreeMap<>());
    }
    for (Pair p : pairs) {
      var group = groups.get((int) (crc32(p.word()) % REDUCERS));
      group.computeIfAbsent(p.word(), k -> new ArrayList<>()).add(p.one());
    }
    for (int r = 0; r < REDUCERS; r++) {
      var words = groups.get(r).entrySet();
      int received = words.stream().mapToInt(e -> e.getValue().size()).sum();
      String shown =
          words.stream()
              .map(e -> e.getKey() + " " + e.getValue())
              .collect(Collectors.joining(", "));
      System.out.println("  reducer " + r + " receives " + received + " pairs: " + shown);
      String sums =
          words.stream()
              .map(e -> e.getKey() + " " + e.getValue().stream().mapToInt(Integer::intValue).sum())
              .collect(Collectors.joining(", "));
      System.out.println("  reducer " + r + " outputs: " + sums);
    }
  }
}

In Java: java.util.zip.CRC32 computes the same value. TreeMap keeps each reducer's words sorted, and a List prints itself as [1, 1].

Recorded with OpenJDK 25.0.4.1 LTS (Eclipse Temurin) on macOS 26 arm64. It runs in your browser with a TeaVM 0.16.0 build in QuickJS (quickjs-emscripten 0.32.0). To run it yourself: java Main.java

C++23 · main.cpp

#include <cstdint>
#include <format>
#include <map>
#include <numeric>
#include <print>
#include <sstream>
#include <string>
#include <utility>
#include <vector>

// Word count in three phases: map, shuffle and reduce.
const std::vector<std::string> splits = {"the cat sat", "the dog sat",
                                         "the cat ran"};
constexpr int reducers = 2;

std::uint32_t crc32(const std::string &word) {
  // CRC-32, bit by bit, as zlib computes it
  std::uint32_t crc = 0xFFFFFFFF;
  for (const unsigned char byte : word) {
    crc ^= byte;
    for (int i = 0; i < 8; i++) {
      crc = (crc >> 1) ^ ((crc & 1) != 0 ? 0xEDB88320u : 0u);
    }
  }
  return ~crc;
}

int main() {
  std::println("map: each task emits (word, 1) pairs");
  std::vector<std::pair<std::string, int>> pairs;
  for (std::size_t n = 0; n < splits.size(); n++) {
    std::istringstream words(splits[n]);
    std::string shown;
    for (std::string word; words >> word;) {
      pairs.emplace_back(word, 1);
      shown += std::format(" ({}, 1)", word);
    }
    std::println("  map {}:{}", n, shown);
  }

  std::println(
      "shuffle by crc32(word) mod 2, then reduce: each reducer sums its words");
  std::vector<std::map<std::string, std::vector<int>>> groups(reducers);
  for (const auto &[word, one] : pairs) {
    groups[crc32(word) % reducers][word].push_back(one);
  }
  for (int r = 0; r < reducers; r++) {
    std::size_t received = 0;
    std::string shown, sums;
    for (const auto &[word, values] : groups[r]) {
      received += values.size();
      std::string list;
      for (const int v : values) {
        list += (list.empty() ? "" : ", ") + std::to_string(v);
      }
      shown += std::format("{}{} [{}]", shown.empty() ? "" : ", ", word, list);
      sums += std::format("{}{} {}", sums.empty() ? "" : ", ", word,
                          std::accumulate(values.begin(), values.end(), 0));
    }
    std::println("  reducer {} receives {} pairs: {}", r, received, shown);
    std::println("  reducer {} outputs: {}", r, sums);
  }
}

In C++: std::map keeps each reducer's words in order, and groups[r][word] creates a word's list the first time it is used.

Recorded with Apple clang 21.0.0 (C++23) and GCC 16.2.0 on macOS 26 arm64. It runs in your browser with Clang in WebAssembly (YoWASP) 22.0.0-git20542-10. To run it yourself: clang++ -std=c++23 -O2 -Wall -Wextra -Werror -pedantic -fsanitize=address,undefined main.cpp -o main && ./main

C# 14 · .NET 10 · app.cs

using System.Text;

// Word count in three phases: map, shuffle and reduce.
string[] splits = ["the cat sat", "the dog sat", "the cat ran"]; // one split per map task
const int Reducers = 2;

Console.WriteLine("map: each task emits (word, 1) pairs");
var pairs = new List<(string Word, int One)>();
for (int n = 0; n < splits.Length; n++)
{
    var emitted = splits[n].Split(' ').Select(word => (Word: word, One: 1)).ToList();
    pairs.AddRange(emitted);
    var shown = emitted.Select(p => $"({p.Word}, {p.One})");
    Console.WriteLine($"  map {n}: {string.Join(" ", shown)}");
}

Console.WriteLine("shuffle by crc32(word) mod 2, then reduce: each reducer sums its words");
var groups = Enumerable.Range(0, Reducers)
    .Select(_ => new SortedDictionary<string, List<int>>(StringComparer.Ordinal))
    .ToArray();
foreach (var (word, one) in pairs)
{
    var group = groups[(int)(Crc32(word) % Reducers)];
    if (!group.TryGetValue(word, out var values))
    {
        group[word] = values = [];
    }
    values.Add(one);
}
for (int r = 0; r < Reducers; r++)
{
    int received = groups[r].Values.Sum(values => values.Count);
    var shown = groups[r].Select(e => $"{e.Key} [{string.Join(", ", e.Value)}]");
    Console.WriteLine($"  reducer {r} receives {received} pairs: {string.Join(", ", shown)}");
    var sums = groups[r].Select(e => $"{e.Key} {e.Value.Sum()}");
    Console.WriteLine($"  reducer {r} outputs: {string.Join(", ", sums)}");
}

static uint Crc32(string word)
{
    // CRC-32, bit by bit, as zlib computes it
    uint crc = 0xFFFFFFFF;
    foreach (byte b in Encoding.UTF8.GetBytes(word))
    {
        crc ^= b;
        for (int i = 0; i < 8; i++)
        {
            crc = (crc >> 1) ^ ((crc & 1) != 0 ? 0xEDB88320 : 0);
        }
    }
    return ~crc;
}

In C#: SortedDictionary with StringComparer.Ordinal sorts the words by code point, the order every other language uses.

Recorded with C# 14 on .NET 10.0.12 (SDK 10.0.401) on macOS 26 arm64. It runs in your browser with the .NET runtime for WebAssembly 10.0.12. To run it yourself: dotnet run app.cs

C23 · main.c

#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

// Word count in three phases: map, shuffle and reduce.
static const char *SPLITS[][3] = {
    {"the", "cat", "sat"}, {"the", "dog", "sat"}, {"the", "cat", "ran"}};
enum { TASKS = 3, WORDS = 3, REDUCERS = 2 };

typedef struct {
  const char *word;
  int ones; // the values of the word's pairs, all 1
} Group;

static uint32_t crc32(const char *word) {
  // CRC-32, bit by bit, as zlib computes it
  uint32_t crc = 0xFFFFFFFF;
  for (const unsigned char *p = (const unsigned char *)word; *p; p++) {
    crc ^= *p;
    for (int i = 0; i < 8; i++) {
      crc = (crc >> 1) ^ ((crc & 1) ? 0xEDB88320u : 0);
    }
  }
  return ~crc;
}

static int by_word(const void *a, const void *b) {
  return strcmp(((const Group *)a)->word, ((const Group *)b)->word);
}

int main(void) {
  Group groups[REDUCERS][TASKS * WORDS];
  int count[REDUCERS] = {0}, received[REDUCERS] = {0};
  puts("map: each task emits (word, 1) pairs");
  for (int n = 0; n < TASKS; n++) {
    printf("  map %d:", n);
    for (int w = 0; w < WORDS; w++) {
      printf(" (%s, 1)", SPLITS[n][w]);
    }
    puts("");
  }
  puts(
      "shuffle by crc32(word) mod 2, then reduce: each reducer sums its words");
  for (int n = 0; n < TASKS; n++) {
    for (int w = 0; w < WORDS; w++) {
      const char *word = SPLITS[n][w];
      const int r = (int)(crc32(word) % REDUCERS);
      int k = 0;
      while (k < count[r] && strcmp(groups[r][k].word, word) != 0) {
        k++;
      }
      if (k == count[r]) {
        groups[r][count[r]++] = (Group){word, 0};
      }
      groups[r][k].ones++;
      received[r]++;
    }
  }
  for (int r = 0; r < REDUCERS; r++) {
    qsort(groups[r], (size_t)count[r], sizeof(Group), by_word);
    printf("  reducer %d receives %d pairs:", r, received[r]);
    for (int k = 0; k < count[r]; k++) {
      printf("%s %s [1", k > 0 ? "," : "", groups[r][k].word);
      for (int v = 1; v < groups[r][k].ones; v++) {
        printf(", 1");
      }
      printf("]");
    }
    printf("\n  reducer %d outputs:", r);
    for (int k = 0; k < count[r]; k++) {
      printf("%s %s %d", k > 0 ? "," : "", groups[r][k].word,
             groups[r][k].ones);
    }
    puts("");
  }
  return 0;
}

In C: strtok, the tokenizer of C, writes into the string it splits, so the splits are written here already cut into words. Every value is 1, so each group counts its values instead of keeping a list, and qsort with strcmp sorts the words.

Recorded with Apple clang 21.0.0 (C23) and GCC 16.2.0 on macOS 26 arm64. It runs in your browser with Clang in WebAssembly (YoWASP) 22.0.0-git20542-10. To run it yourself: clang -std=c23 -O2 -Wall -Wextra -Werror -pedantic -fsanitize=address,undefined main.c -o main && ./main

PHP 8.5 · main.php

<?php

declare(strict_types=1);

// Word count in three phases: map, shuffle and reduce.
const SPLITS = ['the cat sat', 'the dog sat', 'the cat ran']; // one split per map task
const REDUCERS = 2;

function crc32Of(string $word): int
{
    // CRC-32, bit by bit, as zlib computes it
    $crc = -1; // 0xFFFFFFFF as a 32-bit pattern
    foreach (str_split($word) as $char) {
        $crc ^= ord($char);
        for ($i = 0; $i < 8; $i++) {
            $crc = (($crc >> 1) & 0x7FFFFFFF) ^ (($crc & 1) !== 0 ? -306674912 : 0);
        }
    }
    return ~$crc;
}

echo "map: each task emits (word, 1) pairs\n";
$pairs = [];
foreach (SPLITS as $n => $split) {
    $emitted = array_map(fn(string $word): array => [$word, 1], explode(' ', $split));
    $pairs = [...$pairs, ...$emitted];
    $shown = implode(' ', array_map(fn(array $p): string => "({$p[0]}, {$p[1]})", $emitted));
    echo "  map {$n}: {$shown}\n";
}

echo "shuffle by crc32(word) mod 2, then reduce: each reducer sums its words\n";
$groups = array_fill(0, REDUCERS, []);
foreach ($pairs as [$word, $one]) {
    $groups[crc32Of($word) & 1][$word][] = $one;
}
foreach ($groups as $r => $group) {
    ksort($group, SORT_STRING);
    $received = array_sum(array_map('count', $group));
    $shown = $sums = [];
    foreach ($group as $word => $values) {
        $shown[] = "{$word} [" . implode(', ', $values) . ']';
        $sums[] = "{$word} " . array_sum($values);
    }
    echo "  reducer {$r} receives {$received} pairs: " . implode(', ', $shown) . "\n";
    echo "  reducer {$r} outputs: " . implode(', ', $sums) . "\n";
}

In PHP: The polynomial and the start value are written as signed 32-bit numbers, so the CRC also fits a 32-bit int, and & 1 takes its remainder by 2.

Recorded with PHP 8.5.11 (CLI) on macOS 26 arm64. It runs in your browser with php-wasm 0.2.0 (PHP 8.5.2). To run it yourself: php main.php

Rust 1.99 · main.rs

use std::collections::BTreeMap;

// Word count in three phases: map, shuffle and reduce.
const SPLITS: [&str; 3] = ["the cat sat", "the dog sat", "the cat ran"]; // one split per map task
const REDUCERS: usize = 2;

fn crc32(word: &str) -> u32 {
    // CRC-32, bit by bit, as zlib computes it
    let mut crc = 0xFFFF_FFFF_u32;
    for byte in word.bytes() {
        crc ^= u32::from(byte);
        for _ in 0..8 {
            crc = (crc >> 1) ^ if crc & 1 == 1 { 0xEDB8_8320 } else { 0 };
        }
    }
    !crc
}

fn main() {
    println!("map: each task emits (word, 1) pairs");
    let mut pairs = Vec::new();
    for (n, split) in SPLITS.iter().enumerate() {
        let emitted: Vec<(&str, u32)> = split.split(' ').map(|word| (word, 1)).collect();
        let shown: Vec<String> = emitted
            .iter()
            .map(|(w, one)| format!("({w}, {one})"))
            .collect();
        println!("  map {n}: {}", shown.join(" "));
        pairs.extend(emitted);
    }

    println!("shuffle by crc32(word) mod 2, then reduce: each reducer sums its words");
    let mut groups: Vec<BTreeMap<&str, Vec<u32>>> = vec![BTreeMap::new(); REDUCERS];
    for (word, one) in pairs {
        groups[crc32(word) as usize % REDUCERS]
            .entry(word)
            .or_default()
            .push(one);
    }
    for (r, group) in groups.iter().enumerate() {
        let received: usize = group.values().map(Vec::len).sum();
        let shown: Vec<String> = group.iter().map(|(w, v)| format!("{w} {v:?}")).collect();
        println!(
            "  reducer {r} receives {received} pairs: {}",
            shown.join(", ")
        );
        let sums: Vec<String> = group
            .iter()
            .map(|(w, v)| format!("{w} {}", v.iter().sum::<u32>()))
            .collect();
        println!("  reducer {r} outputs: {}", sums.join(", "));
    }
}

In Rust: BTreeMap keeps each reducer's words sorted, entry().or_default() starts a word's list, and {:?} prints a Vec as [1, 1].

Recorded with rustc 1.99.0 (edition 2024) on macOS 26 arm64. It runs in your browser with a WebAssembly build on the WASI runner (browser_wasi_shim 0.4.2). To run it yourself: rustc --edition 2024 -O main.rs && ./main

TypeScript 6.0 · main.ts

// Word count in three phases: map, shuffle and reduce.
const SPLITS = ["the cat sat", "the dog sat", "the cat ran"]; // one split per map task
const REDUCERS = 2;

function crc32(word: string): number {
  // CRC-32, bit by bit, as zlib computes it
  let crc = 0xffffffff;
  for (const ch of word) {
    crc ^= ch.charCodeAt(0);
    for (let i = 0; i < 8; i++) {
      crc = (crc >>> 1) ^ (crc & 1 ? 0xedb88320 : 0);
    }
  }
  return (crc ^ 0xffffffff) >>> 0;
}

console.log("map: each task emits (word, 1) pairs");
const pairs: [string, number][] = [];
SPLITS.forEach((split, n) => {
  const emitted = split.split(" ").map((word): [string, number] => [word, 1]);
  pairs.push(...emitted);
  console.log(`  map ${n}: ${emitted.map(([word, one]) => `(${word}, ${one})`).join(" ")}`);
});

console.log("shuffle by crc32(word) mod 2, then reduce: each reducer sums its words");
const groups: Map<string, number[]>[] = Array.from({ length: REDUCERS }, () => new Map());
for (const [word, one] of pairs) {
  const group = groups[crc32(word) % REDUCERS];
  group.set(word, [...(group.get(word) ?? []), one]);
}
groups.forEach((group, r) => {
  const words = [...group.entries()].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
  const received = words.reduce((n, [, values]) => n + values.length, 0);
  const shown = words.map(([word, values]) => `${word} [${values.join(", ")}]`);
  console.log(`  reducer ${r} receives ${received} pairs: ${shown.join(", ")}`);
  const sums = words.map(([word, values]) => `${word} ${values.reduce((a, b) => a + b)}`);
  console.log(`  reducer ${r} outputs: ${sums.join(", ")}`);
});

In TypeScript: The pairs are typed as tuples of a string and a number; the rest is the JavaScript program.

Recorded with TypeScript 6.0.3 (strict), run by Node.js 24.21.0 on macOS 26 arm64. It runs in your browser with QuickJS (quickjs-emscripten 0.32.0) after Sucrase 3.35.1. To run it yourself: npx -p typescript@6.0.3 tsc --noEmit --strict main.ts && node main.ts

Go 1.27 · main.go

package main

import (
	"fmt"
	"maps"
	"slices"
	"strings"
)

// Word count in three phases: map, shuffle and reduce.
var splits = []string{"the cat sat", "the dog sat", "the cat ran"} // one split per map task

const reducers = 2

type pair struct {
	word string
	one  int
}

func crc32(word string) uint32 {
	// CRC-32, bit by bit, as zlib computes it
	crc := uint32(0xFFFFFFFF)
	for _, b := range []byte(word) {
		crc ^= uint32(b)
		for range 8 {
			if crc&1 == 1 {
				crc = crc>>1 ^ 0xEDB88320
			} else {
				crc >>= 1
			}
		}
	}
	return ^crc
}

func main() {
	fmt.Println("map: each task emits (word, 1) pairs")
	var pairs []pair
	for n, split := range splits {
		var shown []string
		for _, word := range strings.Fields(split) {
			pairs = append(pairs, pair{word, 1})
			shown = append(shown, fmt.Sprintf("(%s, 1)", word))
		}
		fmt.Printf("  map %d: %s\n", n, strings.Join(shown, " "))
	}

	fmt.Println("shuffle by crc32(word) mod 2, then reduce: each reducer sums its words")
	groups := make([]map[string][]int, reducers)
	for r := range groups {
		groups[r] = map[string][]int{}
	}
	for _, p := range pairs {
		r := crc32(p.word) % reducers
		groups[r][p.word] = append(groups[r][p.word], p.one)
	}
	for r, group := range groups {
		received := 0
		var shown, sums []string
		for _, word := range slices.Sorted(maps.Keys(group)) {
			values, total := []string{}, 0
			for _, v := range group[word] {
				values = append(values, fmt.Sprint(v))
				total += v
			}
			received += len(values)
			shown = append(shown, fmt.Sprintf("%s [%s]", word, strings.Join(values, ", ")))
			sums = append(sums, fmt.Sprintf("%s %d", word, total))
		}
		fmt.Printf("  reducer %d receives %d pairs: %s\n", r, received, strings.Join(shown, ", "))
		fmt.Printf("  reducer %d outputs: %s\n", r, strings.Join(sums, ", "))
	}
}

In Go: Go's hash/crc32 computes the same value. Map order is random in Go, so slices.Sorted(maps.Keys(group)) sorts the words first.

Recorded with Go 1.27.1 on macOS 26 arm64. It runs in your browser with the Go compiler in WebAssembly 1.27.1. To run it yourself: go run main.go

Swift 6.4 · main.swift

// Word count in three phases: map, shuffle and reduce.
let splits = ["the cat sat", "the dog sat", "the cat ran"]  // one split per map task
let reducers = 2

func crc32(_ word: String) -> UInt32 {
  // CRC-32, bit by bit, as zlib computes it
  var crc: UInt32 = 0xFFFF_FFFF
  for byte in word.utf8 {
    crc ^= UInt32(byte)
    for _ in 0..<8 {
      crc = (crc >> 1) ^ (crc & 1 == 1 ? 0xEDB8_8320 : 0)
    }
  }
  return ~crc
}

print("map: each task emits (word, 1) pairs")
var pairs: [(word: String, one: Int)] = []
for (n, split) in splits.enumerated() {
  let emitted = split.split(separator: " ").map { (word: String($0), one: 1) }
  pairs += emitted
  print("  map \(n): \(emitted.map { "(\($0.word), \($0.one))" }.joined(separator: " "))")
}

print("shuffle by crc32(word) mod 2, then reduce: each reducer sums its words")
var groups = [[String: [Int]]](repeating: [:], count: reducers)
for (word, one) in pairs {
  groups[Int(crc32(word) % UInt32(reducers))][word, default: []].append(one)
}
for (r, group) in groups.enumerated() {
  let words = group.sorted { $0.key < $1.key }
  let received = words.map { $0.value.count }.reduce(0, +)
  let shown = words.map { "\($0.key) [\($0.value.map(String.init).joined(separator: ", "))]" }
  print("  reducer \(r) receives \(received) pairs: \(shown.joined(separator: ", "))")
  let sums = words.map { "\($0.key) \($0.value.reduce(0, +))" }
  print("  reducer \(r) outputs: \(sums.joined(separator: ", "))")
}

In Swift: A dictionary has no fixed order in Swift, so sorted orders each reducer's words before they are printed.

Recorded with Apple Swift 6.4 on macOS 26 arm64. It runs in your browser with a WebAssembly build on the WASI runner (browser_wasi_shim 0.4.2). To run it yourself: swiftc -O main.swift -o main && ./main

Ruby 4.0 · main.rb

# frozen_string_literal: true

# Word count in three phases: map, shuffle and reduce.
SPLITS = ['the cat sat', 'the dog sat', 'the cat ran'].freeze # one split per map task
REDUCERS = 2

def crc32(word)
  # CRC-32, bit by bit, as zlib computes it
  crc = 0xFFFFFFFF
  word.each_byte do |byte|
    crc ^= byte
    8.times { crc = (crc >> 1) ^ (crc.odd? ? 0xEDB88320 : 0) }
  end
  crc ^ 0xFFFFFFFF
end

puts 'map: each task emits (word, 1) pairs'
pairs = []
SPLITS.each_with_index do |split, n|
  emitted = split.split.map { |word| [word, 1] }
  pairs.concat(emitted)
  puts "  map #{n}: #{emitted.map { |word, one| "(#{word}, #{one})" }.join(' ')}"
end

puts 'shuffle by crc32(word) mod 2, then reduce: each reducer sums its words'
groups = Array.new(REDUCERS) { Hash.new { |hash, word| hash[word] = [] } }
pairs.each { |word, one| groups[crc32(word) % REDUCERS][word] << one }
groups.each_with_index do |group, r|
  words = group.sort
  shown = words.map { |word, values| "#{word} [#{values.join(', ')}]" }
  puts "  reducer #{r} receives #{group.values.sum(&:size)} pairs: #{shown.join(', ')}"
  puts "  reducer #{r} outputs: #{words.map { |word, values| "#{word} #{values.sum}" }.join(', ')}"
end

In Ruby: Zlib.crc32 computes the same value. Hash.new with a block starts each word's list, and sort orders the words.

Recorded with CRuby 4.0.7 on macOS 26 arm64. It runs in your browser with ruby.wasm 2.10.1 (Ruby 4.0.0). To run it yourself: ruby main.rb

Kotlin 2.4 · Main.kt

// Word count in three phases: map, shuffle and reduce.
val splits = listOf("the cat sat", "the dog sat", "the cat ran") // one split per map task
const val REDUCERS = 2

fun crc32(word: String): UInt {
    // CRC-32, bit by bit, as zlib computes it
    var crc = 0xFFFFFFFFu
    for (byte in word.encodeToByteArray()) {
        crc = crc xor byte.toUByte().toUInt()
        repeat(8) {
            crc = (crc shr 1) xor (if (crc and 1u != 0u) 0xEDB88320u else 0u)
        }
    }
    return crc.inv()
}

fun main() {
    println("map: each task emits (word, 1) pairs")
    val pairs = mutableListOf<Pair<String, Int>>()
    splits.forEachIndexed { n, split ->
        val emitted = split.split(" ").map { it to 1 }
        pairs += emitted
        println("  map $n: ${emitted.joinToString(" ") { (word, one) -> "($word, $one)" }}")
    }

    println("shuffle by crc32(word) mod 2, then reduce: each reducer sums its words")
    val groups = List(REDUCERS) { mutableMapOf<String, MutableList<Int>>() }
    for ((word, one) in pairs) {
        groups[(crc32(word) % REDUCERS.toUInt()).toInt()].getOrPut(word) { mutableListOf() } += one
    }
    groups.forEachIndexed { r, group ->
        val words = group.keys.sorted()
        val received = group.values.sumOf { it.size }
        val shown = words.joinToString(", ") { "$it ${group[it]}" }
        println("  reducer $r receives $received pairs: $shown")
        val sums = words.joinToString(", ") { "$it ${group.getValue(it).sum()}" }
        println("  reducer $r outputs: $sums")
    }
}

In Kotlin: UInt holds the CRC as an unsigned 32-bit number, so shr shifts in zeros, and getOrPut starts a word's list.

Recorded with Kotlin 2.4.20 on Java 25.0.4.1 on macOS 26 arm64. It runs in your browser with a WebAssembly build on the WASI runner (browser_wasi_shim 0.4.2). To run it yourself: kotlinc Main.kt -include-runtime -d main.jar && java -jar main.jar

Dart 3.13 · main.dart

import 'dart:convert';

/// Word count in three phases: map, shuffle and reduce.
const splits = [
  'the cat sat',
  'the dog sat',
  'the cat ran',
]; // one per map task
const reducers = 2;

int crc32(String word) {
  // CRC-32, bit by bit, as zlib computes it
  var crc = 0xFFFFFFFF;
  for (final byte in utf8.encode(word)) {
    crc ^= byte;
    for (var i = 0; i < 8; i++) {
      crc = (crc >> 1) ^ (crc & 1 == 1 ? 0xEDB88320 : 0);
    }
  }
  return crc ^ 0xFFFFFFFF;
}

void main() {
  print('map: each task emits (word, 1) pairs');
  final pairs = <(String, int)>[];
  for (final (n, split) in splits.indexed) {
    final emitted = [for (final word in split.split(' ')) (word, 1)];
    pairs.addAll(emitted);
    print('  map $n: ${emitted.map((p) => '(${p.$1}, ${p.$2})').join(' ')}');
  }

  print(
    'shuffle by crc32(word) mod 2, then reduce: each reducer sums its words',
  );
  final groups = List.generate(reducers, (_) => <String, List<int>>{});
  for (final (word, one) in pairs) {
    groups[crc32(word) % reducers].putIfAbsent(word, () => []).add(one);
  }
  for (final (r, group) in groups.indexed) {
    final words = group.keys.toList()..sort();
    final received = group.values.fold(0, (n, values) => n + values.length);
    print(
      '  reducer $r receives $received pairs: ${words.map((w) => '$w ${group[w]}').join(', ')}',
    );
    final sums = words.map((w) => '$w ${group[w]!.fold(0, (a, b) => a + b)}');
    print('  reducer $r outputs: ${sums.join(', ')}');
  }
}

In Dart: Dart's int has 64 bits, so the CRC stays positive below 2³², and putIfAbsent starts each word's list.

Recorded with Dart 3.13.5 on macOS 26 arm64. It runs in your browser with a WebAssembly build by dart2wasm 3.13.5. To run it yourself: dart run main.dart