Strings in five languages

Python · py/lengths.py

"""What len() counts in Python (code points), next to the size of the same text in UTF-16 and in UTF-8."""

import unicodedata

SAMPLES = [
    ("Asha", "plain ASCII"),
    ("Jos\u00e9", "é as one code point, U+00E9"),
    ("Jose\u0301", "e followed by U+0301, a combining acute accent"),
    ("\u0928\u0947\u0939\u093e", "Neha in Devanagari: two letters, each with a vowel sign"),
    ("\U0001F600", "one emoji, U+1F600"),
    ("\U0001F1EE\U0001F1F3", "a flag: the regional indicator letters I and N"),
]

print("len  UTF-16  UTF-8  text")
for text, note in SAMPLES:
    utf16_units = len(text.encode("utf-16-le")) // 2  # two bytes per UTF-16 code unit
    utf8_bytes = len(text.encode("utf-8"))
    print(f"{len(text):3}  {utf16_units:6}  {utf8_bytes:5}  {text} ({note})")

composed, decomposed = SAMPLES[1][0], SAMPLES[2][0]
print("an index gives one code point:", ascii(composed[3]), ascii(SAMPLES[4][0][0]))
print("the two Josés are equal:", composed == decomposed)
nfc = unicodedata.normalize
print("equal after NFC normalization:", nfc("NFC", composed) == nfc("NFC", decomposed))

len  UTF-16  UTF-8  text
  4       4      4  Asha (plain ASCII)
  4       4      5  José (é as one code point, U+00E9)
  5       5      6  José (e followed by U+0301, a combining acute accent)
  4       4     12  नेहा (Neha in Devanagari: two letters, each with a vowel sign)
  1       2      4  😀 (one emoji, U+1F600)
  2       4      8  🇮🇳 (a flag: the regional indicator letters I and N)
an index gives one code point: '\xe9' '\U0001f600'
the two Josés are equal: False
equal after NFC normalization: True

JavaScript · js/lengths.mjs

// What .length counts in JavaScript (UTF-16 code units), next to the code points that the string iterator walks over.
const samples = [
  ['Asha', 'plain ASCII'],
  ['Jos\u00e9', 'é as one code point, U+00E9'],
  ['Jose\u0301', 'e followed by U+0301, a combining acute accent'],
  ['\u0928\u0947\u0939\u093e', 'Neha in Devanagari: two letters, each with a vowel sign'],
  ['\u{1F600}', 'one emoji, U+1F600'],
  ['\u{1F1EE}\u{1F1F3}', 'a flag: the regional indicator letters I and N'],
];

console.log('length  code points  text');
for (const [text, note] of samples) {
  console.log(`${String(text.length).padStart(6)}  ${String([...text].length).padStart(11)}  ${text} (${note})`);
}

const smile = samples[4][0];
console.log('an index gives one code unit:', JSON.stringify(smile[0]), 'codePointAt(0):', smile.codePointAt(0).toString(16));
const composed = samples[1][0];
const decomposed = samples[2][0];
console.log('the two Josés are equal:', composed === decomposed);
console.log('equal after NFC normalization:', composed.normalize('NFC') === decomposed.normalize('NFC'));

length  code points  text
     4            4  Asha (plain ASCII)
     4            4  José (é as one code point, U+00E9)
     5            5  José (e followed by U+0301, a combining acute accent)
     4            4  नेहा (Neha in Devanagari: two letters, each with a vowel sign)
     2            1  😀 (one emoji, U+1F600)
     4            2  🇮🇳 (a flag: the regional indicator letters I and N)
an index gives one code unit: "\ud83d" codePointAt(0): 1f600
the two Josés are equal: false
equal after NFC normalization: true

Java · compiled/Lengths.java

// What length() counts in Java (UTF-16 code units), next to codePointCount and the size of the text in UTF-8.
// A Unicode escape in Java names one UTF-16 code unit, so the emoji and the flag are written as surrogate pairs.
import java.nio.charset.StandardCharsets;
import java.text.Normalizer;

public class Lengths {
    public static void main(String[] args) {
        String[][] samples = {
            {"Asha", "plain ASCII"},
            {"Jos\u00e9", "é as one code point, U+00E9"},
            {"Jose\u0301", "e followed by U+0301, a combining acute accent"},
            {"\u0928\u0947\u0939\u093e", "Neha in Devanagari: two letters, each with a vowel sign"},
            {"\uD83D\uDE00", "one emoji, U+1F600"},
            {"\uD83C\uDDEE\uD83C\uDDF3", "a flag: the regional indicator letters I and N"},
        };
        System.out.println("length  code points  UTF-8  text");
        for (String[] sample : samples) {
            String text = sample[0];
            System.out.printf("%6d  %11d  %5d  %s (%s)%n", text.length(), text.codePointCount(0, text.length()),
                    text.getBytes(StandardCharsets.UTF_8).length, text, sample[1]);
        }
        String smile = samples[4][0];
        System.out.println("charAt(0) gives one code unit: " + Integer.toHexString(smile.charAt(0))
                + ", codePointAt(0): " + Integer.toHexString(smile.codePointAt(0)));
        String composed = samples[1][0];
        String decomposed = samples[2][0];
        System.out.println("the two Josés are equal: " + composed.equals(decomposed));
        System.out.println("equal after NFC normalization: " + Normalizer.normalize(composed, Normalizer.Form.NFC)
                .equals(Normalizer.normalize(decomposed, Normalizer.Form.NFC)));
    }
}

C++ · compiled/lengths.cpp

// What size() counts in C++: the bytes of the string. clang stores ordinary string literals as UTF-8, so a code
// point takes 1 to 4 bytes; they are counted by hand, since every byte except a continuation byte (10xxxxxx) starts one.
#include <cstddef>
#include <cstdio>
#include <string>
#include <string_view>

std::size_t code_points(std::string_view s) {
    std::size_t count = 0;
    for (unsigned char byte : s) count += (byte & 0xC0) != 0x80;
    return count;
}

int main() {
    struct Sample {
        std::string text;
        const char* note;
    };
    const Sample samples[] = {
        {"Asha", "plain ASCII"},
        {"Jos\u00e9", "é as one code point, U+00E9"},
        {"Jose\u0301", "e followed by U+0301, a combining acute accent"},
        {"\u0928\u0947\u0939\u093e", "Neha in Devanagari: two letters, each with a vowel sign"},
        {"\U0001F600", "one emoji, U+1F600"},
        {"\U0001F1EE\U0001F1F3", "a flag: the regional indicator letters I and N"},
    };
    std::printf("size  code points  text\n");
    for (const auto& [text, note] : samples) {
        std::printf("%4zu  %11zu  %s (%s)\n", text.size(), code_points(text), text.c_str(), note);
    }
    const std::string& jose = samples[1].text;
    std::printf("an index gives one byte: %d as char, %d as unsigned char\n", jose[3], static_cast<unsigned char>(jose[3]));
}

Go · compiled/lengths.go

// What len counts in Go: the bytes of the string, which Go keeps as UTF-8. unicode/utf8 counts the code points
// (runes), and unicode/utf16 encodes them, to compare with Java and JavaScript.
package main

import (
	"fmt"
	"unicode/utf16"
	"unicode/utf8"
)

func main() {
	samples := []struct{ text, note string }{
		{"Asha", "plain ASCII"},
		{"Jos\u00e9", "é as one code point, U+00E9"},
		{"Jose\u0301", "e followed by U+0301, a combining acute accent"},
		{"\u0928\u0947\u0939\u093e", "Neha in Devanagari: two letters, each with a vowel sign"},
		{"\U0001F600", "one emoji, U+1F600"},
		{"\U0001F1EE\U0001F1F3", "a flag: the regional indicator letters I and N"},
	}
	fmt.Println("len  runes  UTF-16  text")
	for _, s := range samples {
		units := len(utf16.Encode([]rune(s.text)))
		fmt.Printf("%3d  %5d  %6d  %s (%s)\n", len(s.text), utf8.RuneCountInString(s.text), units, s.text, s.note)
	}
	word := "Jos\u00e9\U0001F600"
	fmt.Println("an index gives one byte:", word[3])
	fmt.Print("range gives each rune with the byte offset where it starts:")
	for offset, r := range word {
		fmt.Printf(" %d:%c", offset, r)
	}
	fmt.Println()
}

Build and run all three · compiled/lengths_session.py

from session import run, versions

# Lengths.java runs as a single source file, lengths.cpp is built with the sanitizers on, and go run builds lengths.go.
versions("java -version", "clang++ --version", "go version")
run("""
java Lengths.java
clang++ -std=c++23 -Wall -Wextra -fsanitize=address,undefined lengths.cpp -o lengths && ./lengths
go run lengths.go
""")

# openjdk version "25.0.4.1" LTS
# Apple clang version 21.0.0 (clang-2100.3.34.2)
# go version go1.27.1 darwin/arm64
$ java Lengths.java
length  code points  UTF-8  text
     4            4      4  Asha (plain ASCII)
     4            4      5  José (é as one code point, U+00E9)
     5            5      6  José (e followed by U+0301, a combining acute accent)
     4            4     12  नेहा (Neha in Devanagari: two letters, each with a vowel sign)
     2            1      4  😀 (one emoji, U+1F600)
     4            2      8  🇮🇳 (a flag: the regional indicator letters I and N)
charAt(0) gives one code unit: d83d, codePointAt(0): 1f600
the two Josés are equal: false
equal after NFC normalization: true
$ clang++ -std=c++23 -Wall -Wextra -fsanitize=address,undefined lengths.cpp -o lengths && ./lengths
size  code points  text
   4            4  Asha (plain ASCII)
   5            4  José (é as one code point, U+00E9)
   6            5  José (e followed by U+0301, a combining acute accent)
  12            4  नेहा (Neha in Devanagari: two letters, each with a vowel sign)
   4            1  😀 (one emoji, U+1F600)
   8            2  🇮🇳 (a flag: the regional indicator letters I and N)
an index gives one byte: -61 as char, 195 as unsigned char
$ go run lengths.go
len  runes  UTF-16  text
  4      4       4  Asha (plain ASCII)
  5      4       4  José (é as one code point, U+00E9)
  6      5       5  José (e followed by U+0301, a combining acute accent)
 12      4       4  नेहा (Neha in Devanagari: two letters, each with a vowel sign)
  4      1       2  😀 (one emoji, U+1F600)
  8      2       4  🇮🇳 (a flag: the regional indicator letters I and N)
an index gives one byte: 195
range gives each rune with the byte offset where it starts: 0:J 1:o 2:s 3:é 5:😀

Java · compiled/Build.java

// Building a string of 100,000 characters in Java. += makes a new String, a copy of everything so far, on every
// pass; a StringBuilder appends to one buffer. The times change from run to run; the comparison at the end does not.
public class Build {
    static final int N = 100_000;

    public static void main(String[] args) {
        long start = System.nanoTime();
        String s = "";
        for (int i = 0; i < N; i++) {
            s += "x";
        }
        long plus = System.nanoTime() - start;

        start = System.nanoTime();
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < N; i++) {
            sb.append('x');
        }
        String built = sb.toString();
        long builder = System.nanoTime() - start;

        System.out.println("+= in a loop:  " + plus / 1_000_000 + " ms");
        System.out.println("StringBuilder: " + builder / 1_000_000 + " ms");
        System.out.println("same text: " + s.equals(built));
        System.out.println("+= took over 10 times as long: " + (plus > 10 * builder));
    }
}

C++ · compiled/build.cpp

// Building a string of 100,000 characters in C++. A std::string can change in place, so s += 'x' appends to it;
// s = s + 'x' builds a new string, a copy of everything so far, on every pass. The times change from run to run;
// the comparison at the end does not.
#include <chrono>
#include <cstdio>
#include <string>

int main() {
    constexpr int n = 100'000;
    using clock = std::chrono::steady_clock;

    auto start = clock::now();
    std::string copied;
    for (int i = 0; i < n; ++i) copied = copied + 'x';
    const auto copy_time = clock::now() - start;

    start = clock::now();
    std::string appended;
    for (int i = 0; i < n; ++i) appended += 'x';
    const auto append_time = clock::now() - start;

    auto ms = [](auto d) { return static_cast<long long>(std::chrono::duration_cast<std::chrono::milliseconds>(d).count()); };
    std::printf("s = s + 'x': %lld ms\n", ms(copy_time));
    std::printf("s += 'x':    %lld ms\n", ms(append_time));
    std::printf("same text: %s\n", copied == appended ? "true" : "false");
    std::printf("s = s + 'x' took over 10 times as long: %s\n", copy_time > 10 * append_time ? "true" : "false");
}

Go · compiled/build.go

// Building a string of 100,000 bytes in Go. += makes a new string, a copy of everything so far, on every pass; a
// strings.Builder appends to one buffer. The times change from run to run; the comparison at the end does not.
package main

import (
	"fmt"
	"strings"
	"time"
)

const n = 100_000

func main() {
	start := time.Now()
	s := ""
	for range n {
		s += "x"
	}
	plus := time.Since(start)

	start = time.Now()
	var b strings.Builder
	for range n {
		b.WriteByte('x')
	}
	built := b.String()
	builder := time.Since(start)

	fmt.Println("+= in a loop:   ", plus.Milliseconds(), "ms")
	fmt.Println("strings.Builder:", builder.Milliseconds(), "ms")
	fmt.Println("same text:", s == built)
	fmt.Println("+= took over 10 times as long:", plus > 10*builder)
}

Build and run all three · compiled/build_session.py

from session import run, versions

# Build.java runs as a single source file, build.cpp is built with the sanitizers on, and go run builds build.go.
versions("java -version", "clang++ --version", "go version")
run("""
java Build.java
clang++ -std=c++23 -Wall -Wextra -fsanitize=address,undefined build.cpp -o build && ./build
go run build.go
""")

# openjdk version "25.0.4.1" LTS
# Apple clang version 21.0.0 (clang-2100.3.34.2)
# go version go1.27.1 darwin/arm64
$ java Build.java
+= in a loop:  1595 ms
StringBuilder: 5 ms
same text: true
+= took over 10 times as long: true
$ clang++ -std=c++23 -Wall -Wextra -fsanitize=address,undefined build.cpp -o build && ./build
s = s + 'x': 2531 ms
s += 'x':    0 ms
same text: true
s = s + 'x' took over 10 times as long: true
$ go run build.go
+= in a loop:    868 ms
strings.Builder: 0 ms
same text: true
+= took over 10 times as long: true

Java · compiled/Reverse.java

// Reversing names in Java three ways: by UTF-16 code unit (swapping the chars of an array), by code point
// (StringBuilder.reverse keeps each surrogate pair together) and by grapheme cluster (the regular expression \X).
import java.util.List;
import java.util.function.UnaryOperator;
import java.util.regex.MatchResult;
import java.util.regex.Pattern;

public class Reverse {
    static final String[][] NAMES = {
        {"Asha", "ahsA"},
        {"Jos\u00e9", "\u00e9soJ"},
        {"Jose\u0301", "e\u0301soJ"},
        {"\u0928\u0947\u0939\u093e", "\u0939\u093e\u0928\u0947"},
        {"Kabir \uD83D\uDC4B\uD83C\uDFFD", "\uD83D\uDC4B\uD83C\uDFFD ribaK"},
        {"Priya \uD83C\uDDEE\uD83C\uDDF3", "\uD83C\uDDEE\uD83C\uDDF3 ayirP"},
    };

    static String byCodeUnit(String s) {
        char[] a = s.toCharArray();
        for (int i = 0, j = a.length - 1; i < j; i++, j--) {
            char t = a[i];
            a[i] = a[j];
            a[j] = t;
        }
        return new String(a);
    }

    static String byCodePoint(String s) {
        return new StringBuilder(s).reverse().toString();
    }

    static final Pattern GRAPHEME = Pattern.compile("\\X");

    static String byGrapheme(String s) {
        List<String> clusters = GRAPHEME.matcher(s).results().map(MatchResult::group).toList();
        return String.join("", clusters.reversed());
    }

    static void report(String label, UnaryOperator<String> reverse) {
        int right = 0;
        for (String[] pair : NAMES) {
            String result = reverse.apply(pair[0]);
            boolean ok = result.equals(pair[1]);
            if (ok) right++;
            System.out.println(label + ": " + pair[0] + " -> " + result + "  " + (ok ? "right" : "wrong"));
        }
        System.out.println(label + " reversed " + right + " of " + NAMES.length + " names as a reader expects");
    }

    public static void main(String[] args) {
        report("char[] swap", Reverse::byCodeUnit);
        report("StringBuilder.reverse", Reverse::byCodePoint);
        report("regex \\X", Reverse::byGrapheme);
    }
}

C++ · compiled/reverse.cpp

// Reversing names in C++ two ways: std::ranges::reverse on the bytes, and by code point, keeping each UTF-8
// sequence (a lead byte and its continuation bytes, 10xxxxxx) together. The standard library has no grapheme clusters.
#include <algorithm>
#include <cstdio>
#include <string>
#include <string_view>
#include <utility>
#include <vector>

std::string by_byte(std::string_view s) {
    std::string out(s);
    std::ranges::reverse(out);
    return out;
}

std::string by_code_point(std::string_view s) {
    std::vector<std::string_view> points;
    for (std::size_t i = 0; i < s.size();) {
        std::size_t j = i + 1;
        while (j < s.size() && (static_cast<unsigned char>(s[j]) & 0xC0) == 0x80) ++j;
        points.push_back(s.substr(i, j - i));
        i = j;
    }
    std::string out;
    out.reserve(s.size());
    for (auto it = points.rbegin(); it != points.rend(); ++it) out += *it;
    return out;
}

const std::pair<std::string, std::string> names[] = {
    {"Asha", "ahsA"},
    {"Jos\u00e9", "\u00e9soJ"},
    {"Jose\u0301", "e\u0301soJ"},
    {"\u0928\u0947\u0939\u093e", "\u0939\u093e\u0928\u0947"},
    {"Kabir \U0001F44B\U0001F3FD", "\U0001F44B\U0001F3FD ribaK"},
    {"Priya \U0001F1EE\U0001F1F3", "\U0001F1EE\U0001F1F3 ayirP"},
};

void report(const char* label, std::string (*reverse)(std::string_view)) {
    int right = 0;
    for (const auto& [name, expected] : names) {
        const std::string result = reverse(name);
        const bool ok = result == expected;
        right += ok;
        std::printf("%s: %s -> %s  %s\n", label, name.c_str(), result.c_str(), ok ? "right" : "wrong");
    }
    std::printf("%s reversed %d of %zu names as a reader expects\n", label, right, std::size(names));
}

int main() {
    report("std::ranges::reverse", by_byte);
    report("by code point", by_code_point);
}

Go · compiled/reverse.go

// Reversing names in Go two ways: the bytes of the string, and its runes (code points). The standard library has
// no grapheme clusters.
package main

import (
	"fmt"
	"slices"
)

func byByte(s string) string {
	b := []byte(s)
	slices.Reverse(b)
	return string(b)
}

func byRune(s string) string {
	r := []rune(s)
	slices.Reverse(r)
	return string(r)
}

func main() {
	names := [][2]string{
		{"Asha", "ahsA"},
		{"Jos\u00e9", "\u00e9soJ"},
		{"Jose\u0301", "e\u0301soJ"},
		{"\u0928\u0947\u0939\u093e", "\u0939\u093e\u0928\u0947"},
		{"Kabir \U0001F44B\U0001F3FD", "\U0001F44B\U0001F3FD ribaK"},
		{"Priya \U0001F1EE\U0001F1F3", "\U0001F1EE\U0001F1F3 ayirP"},
	}
	ways := []struct {
		label   string
		reverse func(string) string
	}{{"[]byte", byByte}, {"[]rune", byRune}}
	for _, way := range ways {
		right := 0
		for _, pair := range names {
			result := way.reverse(pair[0])
			verdict := "wrong"
			if result == pair[1] {
				verdict = "right"
				right++
			}
			fmt.Printf("%s: %s -> %s  %s\n", way.label, pair[0], result, verdict)
		}
		fmt.Printf("%s reversed %d of %d names as a reader expects\n", way.label, right, len(names))
	}
}

Build and run all three · compiled/reverse_session.py

from session import run, versions

# Reverse.java runs as a single source file, reverse.cpp is built with the sanitizers on, and go run builds reverse.go.
versions("java -version", "clang++ --version", "go version")
run("""
java Reverse.java
clang++ -std=c++23 -Wall -Wextra -fsanitize=address,undefined reverse.cpp -o reverse && ./reverse
go run reverse.go
""")

# openjdk version "25.0.4.1" LTS
# Apple clang version 21.0.0 (clang-2100.3.34.2)
# go version go1.27.1 darwin/arm64
$ java Reverse.java
char[] swap: Asha -> ahsA  right
char[] swap: José -> ésoJ  right
char[] swap: José -> ́esoJ  wrong
char[] swap: नेहा -> ाहेन  wrong
char[] swap: Kabir 👋🏽 -> ?🁋? ribaK  wrong
char[] swap: Priya 🇮🇳 -> ?🇮? ayirP  wrong
char[] swap reversed 2 of 6 names as a reader expects
StringBuilder.reverse: Asha -> ahsA  right
StringBuilder.reverse: José -> ésoJ  right
StringBuilder.reverse: José -> ́esoJ  wrong
StringBuilder.reverse: नेहा -> ाहेन  wrong
StringBuilder.reverse: Kabir 👋🏽 -> 🏽👋 ribaK  wrong
StringBuilder.reverse: Priya 🇮🇳 -> 🇳🇮 ayirP  wrong
StringBuilder.reverse reversed 2 of 6 names as a reader expects
regex \X: Asha -> ahsA  right
regex \X: José -> ésoJ  right
regex \X: José -> ésoJ  right
regex \X: नेहा -> हाने  right
regex \X: Kabir 👋🏽 -> 👋🏽 ribaK  right
regex \X: Priya 🇮🇳 -> 🇮🇳 ayirP  right
regex \X reversed 6 of 6 names as a reader expects
$ clang++ -std=c++23 -Wall -Wextra -fsanitize=address,undefined reverse.cpp -o reverse && ./reverse
std::ranges::reverse: Asha -> ahsA  right
std::ranges::reverse: José -> ��soJ  wrong
std::ranges::reverse: José -> ��esoJ  wrong
std::ranges::reverse: नेहा -> ��๤���ਤ�  wrong
std::ranges::reverse: Kabir 👋🏽 -> �������� ribaK  wrong
std::ranges::reverse: Priya 🇮🇳 -> ���𮇟� ayirP  wrong
std::ranges::reverse reversed 1 of 6 names as a reader expects
by code point: Asha -> ahsA  right
by code point: José -> ésoJ  right
by code point: José -> ́esoJ  wrong
by code point: नेहा -> ाहेन  wrong
by code point: Kabir 👋🏽 -> 🏽👋 ribaK  wrong
by code point: Priya 🇮🇳 -> 🇳🇮 ayirP  wrong
by code point reversed 2 of 6 names as a reader expects
$ go run reverse.go
[]byte: Asha -> ahsA  right
[]byte: José -> ��soJ  wrong
[]byte: José -> ��esoJ  wrong
[]byte: नेहा -> ��๤���ਤ�  wrong
[]byte: Kabir 👋🏽 -> �������� ribaK  wrong
[]byte: Priya 🇮🇳 -> ���𮇟� ayirP  wrong
[]byte reversed 1 of 6 names as a reader expects
[]rune: Asha -> ahsA  right
[]rune: José -> ésoJ  right
[]rune: José -> ́esoJ  wrong
[]rune: नेहा -> ाहेन  wrong
[]rune: Kabir 👋🏽 -> 🏽👋 ribaK  wrong
[]rune: Priya 🇮🇳 -> 🇳🇮 ayirP  wrong
[]rune reversed 2 of 6 names as a reader expects