Memory-mapped files in C++: mmap and binary formats

C++ header · mapped_file.hpp

// mapped_file.hpp: a read-only memory mapping of a whole file, with one owner that unmaps it.
#pragma once
#include "fd.hpp"

#include <cstddef>
#include <expected>
#include <span>
#include <utility>

#include <sys/mman.h>
#include <sys/stat.h>

class MappedFile {
public:
    static std::expected<MappedFile, std::error_code> open(const char* path) {
        auto fd = open_file(path, O_RDONLY);
        if (!fd) return std::unexpected(fd.error());
        struct stat st {};
        if (::fstat(fd->get(), &st) != 0) return std::unexpected(last_error());
        MappedFile m;
        m.size_ = static_cast<std::size_t>(st.st_size);
        if (m.size_ == 0) return m; // mmap of 0 bytes fails, and an empty span needs no mapping
        void* p = ::mmap(nullptr, m.size_, PROT_READ, MAP_PRIVATE, fd->get(), 0);
        if (p == MAP_FAILED) return std::unexpected(last_error());
        m.data_ = static_cast<const std::byte*>(p);
        return m; // the descriptor closes here; the mapping stays valid until munmap
    }

    MappedFile(MappedFile&& other) noexcept
        : data_(std::exchange(other.data_, nullptr)), size_(std::exchange(other.size_, 0)) {}
    MappedFile& operator=(MappedFile&& other) noexcept {
        if (this != &other) {
            unmap();
            data_ = std::exchange(other.data_, nullptr);
            size_ = std::exchange(other.size_, 0);
        }
        return *this;
    }
    ~MappedFile() { unmap(); }

    std::span<const std::byte> bytes() const { return {data_, size_}; }

private:
    MappedFile() = default;
    void unmap() {
        if (data_) ::munmap(const_cast<std::byte*>(data_), size_);
        data_ = nullptr;
    }
    const std::byte* data_ = nullptr;
    std::size_t size_ = 0;
};

C++ · count_lines.cpp

// count_lines.cpp: counts the lines of a 100 MB file three ways, and times each (the fastest of three runs).
#include "mapped_file.hpp"

#include <algorithm>
#include <chrono>
#include <fstream>
#include <print>
#include <string>
#include <vector>

// Makes the compiler finish computing value here, and not assume memory unchanged across this point (an empty asm
// statement, a GCC and Clang extension), so that no work moves out of the timed span.
template <class T>
void keep(const T& value) {
    asm volatile("" : : "r,m"(value) : "memory");
}

long count_mapped() {
    auto file = MappedFile::open("big.txt");
    return std::ranges::count(file->bytes(), std::byte{'\n'});
}

long count_getline() {
    std::ifstream in("big.txt");
    std::string line;
    long n = 0;
    while (std::getline(in, line)) ++n;
    return n;
}

long count_chunks() {
    std::ifstream in("big.txt", std::ios::binary);
    std::vector<char> buffer(1 << 20);
    long n = 0;
    while (in.read(buffer.data(), static_cast<std::streamsize>(buffer.size())) || in.gcount() > 0)
        n += std::count(buffer.data(), buffer.data() + in.gcount(), '\n');
    return n;
}

int main() {
    {
        // 1,250,000 lines of 80 bytes: 100,000,000 bytes.
        std::string block;
        for (int i = 0; i < 1000; ++i) block += std::string(79, 'x') + '\n';
        std::ofstream out("big.txt", std::ios::binary);
        for (int i = 0; i < 1250; ++i) out.write(block.data(), static_cast<std::streamsize>(block.size()));
    }
    struct Way {
        const char* name;
        long (*count)();
        long lines = 0;
        double best_ms = 1e12;
    } ways[] = {{"mmap and std::ranges::count:  ", count_mapped},
                {"std::getline line by line:    ", count_getline},
                {"ifstream::read, 1 MiB at once:", count_chunks}};
    for (int run = 0; run < 3; ++run) {
        for (Way& w : ways) {
            auto t0 = std::chrono::steady_clock::now();
            keep(t0);
            w.lines = w.count();
            keep(w.lines);
            std::chrono::duration<double, std::milli> took = std::chrono::steady_clock::now() - t0;
            w.best_ms = std::min(w.best_ms, took.count());
        }
    }
    std::println("lines counted: mmap {}, getline {}, 1 MiB reads {}", ways[0].lines, ways[1].lines, ways[2].lines);
    for (const Way& w : ways) std::println("{} {:7.1f} ms", w.name, w.best_ms);
}

C++ header · byte_reader.hpp

// byte_reader.hpp: reads fixed-size little-endian fields from bytes, and refuses to read past the end.
#pragma once
#include <cstddef>
#include <cstdint>
#include <optional>
#include <span>
#include <string_view>

class ByteReader {
public:
    explicit ByteReader(std::span<const std::byte> bytes) : bytes_(bytes) {}
    std::size_t size() const { return bytes_.size(); }

    // True when n bytes starting at offset at lie inside the data. Written with a subtraction, so a huge at or n
    // cannot wrap around and pass.
    bool has(std::size_t at, std::size_t n) const { return at <= bytes_.size() && n <= bytes_.size() - at; }

    // An unsigned little-endian integer of N bytes at offset at, built with shifts: the same on every processor.
    template <std::size_t N>
    std::optional<std::uint64_t> le(std::size_t at) const {
        if (!has(at, N)) return std::nullopt;
        std::uint64_t value = 0;
        for (std::size_t i = N; i-- > 0;) value = value << 8 | std::to_integer<std::uint64_t>(bytes_[at + i]);
        return value;
    }
    std::optional<std::uint16_t> u16(std::size_t at) const { return le<2>(at).transform([](auto v) { return static_cast<std::uint16_t>(v); }); }
    std::optional<std::uint32_t> u32(std::size_t at) const { return le<4>(at).transform([](auto v) { return static_cast<std::uint32_t>(v); }); }

    // The 4 characters of a chunk name such as "RIFF".
    std::optional<std::string_view> tag(std::size_t at) const {
        if (!has(at, 4)) return std::nullopt;
        return std::string_view(reinterpret_cast<const char*>(bytes_.data() + at), 4);
    }

private:
    std::span<const std::byte> bytes_;
};

C++ · wav_header.cpp

// wav_header.cpp: writes a short WAV file, maps it, and reads its header field by field with ByteReader.
#include "byte_reader.hpp"
#include "mapped_file.hpp"

#include <cstdint>
#include <fstream>
#include <print>
#include <string>

// Appends n bytes of value, least significant byte first, as WAV files store every number.
void put(std::string& out, std::uint32_t value, int n) {
    for (int i = 0; i < n; ++i) out.push_back(static_cast<char>(value >> (8 * i) & 0xff));
}

void write_wav(const char* path, std::uint32_t rate, std::uint16_t channels, std::uint32_t frames) {
    std::string samples;
    for (std::uint32_t i = 0; i < frames * channels; ++i) put(samples, (i / 40) % 2 ? 0x2000 : 0xe000, 2); // a square wave
    std::string wav = "RIFF";
    put(wav, static_cast<std::uint32_t>(36 + samples.size()), 4);
    wav += "WAVEfmt ";
    put(wav, 16, 4);                       // the size of the fmt chunk
    put(wav, 1, 2);                        // format 1: PCM
    put(wav, channels, 2);
    put(wav, rate, 4);
    put(wav, rate * channels * 2, 4);      // bytes per second
    put(wav, channels * 2, 2);             // bytes per frame
    put(wav, 16, 2);                       // bits per sample
    wav += "data";
    put(wav, static_cast<std::uint32_t>(samples.size()), 4);
    std::ofstream(path, std::ios::binary) << wav << samples;
}

int main() {
    write_wav("tone.wav", 48000, 2, 480); // 10 ms of stereo sound
    auto file = MappedFile::open("tone.wav");
    ByteReader r(file->bytes());
    std::println("{} bytes; starts with {} and {}", r.size(), r.tag(0).value_or("?"), r.tag(8).value_or("?"));
    // Chunks follow the 12-byte RIFF header: a 4-byte name, a 4-byte size, then the body.
    for (std::size_t at = 12; r.has(at, 8);) {
        std::string_view name = *r.tag(at);
        std::uint32_t size = *r.u32(at + 4);
        std::println("chunk {:?} at offset {}, {} bytes", name, at, size);
        if (name == "fmt " && size >= 16) {
            std::println("  format {}, {} channels, {} Hz, {} bits per sample, {} bytes per second", *r.u16(at + 8),
                         *r.u16(at + 10), *r.u32(at + 12), *r.u16(at + 22), *r.u32(at + 16));
        }
        if (!r.has(at + 8, size)) {
            std::println("  the chunk claims more bytes than the file has: stop");
            break;
        }
        at += 8 + size + (size & 1); // chunks are padded to an even length
    }
}