Streams, compression and System.IO.Pipelines

// Three streams stacked: a StreamWriter turns text into UTF-8 bytes, a GZipStream compresses them and a FileStream
// writes the result to a file. A counting stream between each pair shows how many bytes pass. Then the same CSV is
// compressed into memory twice: once with the GZipStream disposed too late, once in time.
using System.IO.Compression;
using System.Text;

Directory.CreateDirectory("out");
string rows = MakeCsv(10_000);
Console.WriteLine($"CSV text: {rows.Length:N0} characters");

var toFile = new CountingStream(File.Create(Path.Join("out", "fares.csv.gz")));
var toGzip = new CountingStream(new GZipStream(toFile, CompressionLevel.Optimal));
using (var writer = new StreamWriter(toGzip, new UTF8Encoding(false)))
    writer.Write(rows);
Console.WriteLine($"into the GZipStream: {toGzip.Written:N0} bytes; into the FileStream: {toFile.Written:N0} bytes");
Console.WriteLine($"read back through FileStream, GZipStream and StreamReader: {CountLines(File.ReadAllBytes(Path.Join("out", "fares.csv.gz")))} lines");

byte[] data = Encoding.UTF8.GetBytes(rows);
byte[] late = CompressTooLate(data);
byte[] onTime = Compress(data);
Console.WriteLine($"disposed too late: {late.Length:N0} bytes, which decompress to {CountLines(late)} lines, and nothing threw");
Console.WriteLine($"disposed in time:  {onTime.Length:N0} bytes, which decompress to {CountLines(onTime)} lines");
Console.WriteLine($"can seek? FileStream {File.OpenRead(Path.Join("out", "fares.csv.gz")).CanSeek}, GZipStream {new GZipStream(Stream.Null, CompressionMode.Decompress).CanSeek}");

// The bug: a using declaration disposes at the end of the method, after ToArray has copied the buffer.
static byte[] CompressTooLate(byte[] data)
{
    var output = new MemoryStream();
    using var gzip = new GZipStream(output, CompressionLevel.Optimal);
    gzip.Write(data);
    return output.ToArray();
}

// The fix: the using statement ends, and the GZipStream writes its last block and its footer, before ToArray.
static byte[] Compress(byte[] data)
{
    var output = new MemoryStream();
    using (var gzip = new GZipStream(output, CompressionLevel.Optimal, leaveOpen: true))
        gzip.Write(data);
    return output.ToArray();
}

static int CountLines(byte[] gzip)
{
    using var reader = new StreamReader(new GZipStream(new MemoryStream(gzip), CompressionMode.Decompress));
    int count = 0;
    while (reader.ReadLine() is not null) count++;
    return count;
}

static string MakeCsv(int count)
{
    var text = new StringBuilder("from,to,fare\n");
    string[] stations = ["PUNE", "NDLS", "MAS", "HWH", "SBC", "ADI"];
    for (int i = 0; i < count; i++)
        text.Append(stations[i % 6]).Append(',').Append(stations[(i * 5 + 1) % 6]).Append(',').Append(200 + (i * 37) % 1800).Append('\n');
    return text.ToString();
}

// A stream that passes every write on to the stream under it and counts the bytes.
sealed class CountingStream(Stream inner) : Stream
{
    public long Written { get; private set; }
    public override void Write(byte[] buffer, int offset, int count) { Written += count; inner.Write(buffer, offset, count); }
    public override void Write(ReadOnlySpan<byte> buffer) { Written += buffer.Length; inner.Write(buffer); }
    public override void Flush() => inner.Flush();
    protected override void Dispose(bool disposing) { if (disposing) inner.Dispose(); base.Dispose(disposing); }
    public override bool CanRead => false;
    public override bool CanSeek => false;
    public override bool CanWrite => true;
    public override long Length => throw new NotSupportedException();
    public override long Position { get => Written; set => throw new NotSupportedException(); }
    public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
    public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
    public override void SetLength(long value) => throw new NotSupportedException();
}

CSV text: 132,227 characters
into the GZipStream: 132,227 bytes; into the FileStream: 13,002 bytes
read back through FileStream, GZipStream and StreamReader: 10001 lines
disposed too late: 10 bytes, which decompress to 0 lines, and nothing threw
disposed in time:  11,489 bytes, which decompress to 10001 lines
can seek? FileStream True, GZipStream False

// The same CSV compressed with GZip, Brotli, ZLib and Deflate at three levels, and each result decompressed again
// to check that nothing changed.
using System.IO.Compression;
using System.Text;

byte[] csv = Encoding.UTF8.GetBytes(MakeCsv(10_000));
Console.WriteLine($"CSV: {csv.Length:N0} bytes");
Console.WriteLine($"{"level",-13}{"GZip",9}{"Brotli",9}{"ZLib",9}{"Deflate",9}");
foreach (CompressionLevel level in new[] { CompressionLevel.Fastest, CompressionLevel.Optimal, CompressionLevel.SmallestSize })
{
    int gzip = Size(s => new GZipStream(s, level));
    int brotli = Size(s => new BrotliStream(s, level));
    int zlib = Size(s => new ZLibStream(s, level));
    int deflate = Size(s => new DeflateStream(s, level));
    Console.WriteLine($"{level,-13}{gzip,9:N0}{brotli,9:N0}{zlib,9:N0}{deflate,9:N0}");
}
byte[] packed = Compress(s => new GZipStream(s, CompressionLevel.Optimal));
Console.WriteLine($"GZip Optimal is {(double)csv.Length / packed.Length:F1} times smaller; it starts with {packed[0]:X2} {packed[1]:X2} and decompresses to {Decompress(packed).Length:N0} bytes, same bytes: {Decompress(packed).AsSpan().SequenceEqual(csv)}");

// The same bytes handed to the compressor 1,024 at a time, as a StreamWriter with its default buffer hands them over.
var pieces = new MemoryStream();
using (var z = new GZipStream(pieces, CompressionLevel.Optimal, leaveOpen: true))
    for (int at = 0; at < csv.Length; at += 1024) z.Write(csv, at, Math.Min(1024, csv.Length - at));
Console.WriteLine($"GZip Optimal, the CSV written in pieces of 1,024 bytes: {pieces.Length:N0} bytes");

int Size(Func<Stream, Stream> compressor) => Compress(compressor).Length;

byte[] Compress(Func<Stream, Stream> compressor)
{
    var output = new MemoryStream();
    using (Stream z = compressor(output)) z.Write(csv);
    return output.ToArray();
}

static byte[] Decompress(byte[] gzip)
{
    var output = new MemoryStream();
    using (var z = new GZipStream(new MemoryStream(gzip), CompressionMode.Decompress)) z.CopyTo(output);
    return output.ToArray();
}

static string MakeCsv(int count)
{
    var text = new StringBuilder("from,to,fare\n");
    string[] stations = ["PUNE", "NDLS", "MAS", "HWH", "SBC", "ADI"];
    for (int i = 0; i < count; i++)
        text.Append(stations[i % 6]).Append(',').Append(stations[(i * 5 + 1) % 6]).Append(',').Append(200 + (i * 37) % 1800).Append('\n');
    return text.ToString();
}

CSV: 132,227 bytes
level             GZip   Brotli     ZLib  Deflate
Fastest         25,248    4,635   25,236   25,230
Optimal         11,489    2,596   11,477   11,471
SmallestSize     5,778    2,342    5,766    5,760
GZip Optimal is 11.5 times smaller; it starts with 1F 8B and decompresses to 132,227 bytes, same bytes: True
GZip Optimal, the CSV written in pieces of 1,024 bytes: 12,996 bytes