Files, paths and safe file access

// The File class reads and writes a whole file in one call; File.ReadLines and FileStream work through it piece by
// piece. Every file goes into the folder "data", next to this program.
using System.Text;

Directory.CreateDirectory("data");
string stations = Path.Join("data", "stations.csv");

// 1. A whole file in one call each way.
File.WriteAllText(stations, "code,name\nPUNE,Pune Junction\nNDLS,New Delhi\nMAS,Chennai Central\n");
string[] rows = File.ReadAllLines(stations);
Console.WriteLine($"WriteAllText wrote {new FileInfo(stations).Length} bytes; ReadAllLines gave {rows.Length} lines, the last \"{rows[^1]}\"");

// 2. The text encoding: UTF-8 without a byte order mark unless you ask for one.
File.WriteAllText(Path.Join("data", "plain.txt"), "₹");
File.WriteAllText(Path.Join("data", "marked.txt"), "₹", Encoding.UTF8);
Console.WriteLine($"\"₹\" as written by default: {Hex("plain.txt")}; with Encoding.UTF8: {Hex("marked.txt")}");

// 3. A log of 100,000 lines, searched two ways for the line of one train.
string log = Path.Join("data", "trains.log");
File.WriteAllLines(log, Enumerable.Range(1, 100_000).Select(i => $"{i:D6} train {12000 + i} on time"));
Measure("ReadAllLines", () => File.ReadAllLines(log).First(line => line.Contains("12042")));
Measure("ReadLines", () => File.ReadLines(log).First(line => line.Contains("12042")));

// 4. A FileStream opened for asynchronous I/O, and a read at an offset with RandomAccess.
await using (var stream = new FileStream(Path.Join("data", "counts.bin"), FileMode.Create, FileAccess.Write,
                 FileShare.None, bufferSize: 4096, FileOptions.Asynchronous))
{
    await stream.WriteAsync(new byte[] { 10, 20, 30, 40, 50 });
    Console.WriteLine($"FileStream.IsAsync: {stream.IsAsync}, wrote {stream.Position} bytes");
}
using (var handle = File.OpenHandle(Path.Join("data", "counts.bin")))
{
    byte[] two = new byte[2];
    int read = RandomAccess.Read(handle, two, fileOffset: 3);
    Console.WriteLine($"RandomAccess read {read} bytes at offset 3: {two[0]} {two[1]} (file length {RandomAccess.GetLength(handle)})");
}

static string Hex(string name) => string.Join(" ", File.ReadAllBytes(Path.Join("data", name)).Select(b => b.ToString("X2")));

static void Measure(string name, Func<string> find)
{
    find(); // once first, so that only the search itself is measured
    long before = GC.GetAllocatedBytesForCurrentThread();
    string found = find();
    double mb = (GC.GetAllocatedBytesForCurrentThread() - before) / 1_000_000.0;
    Console.WriteLine($"{name,-12} found \"{found}\" and allocated {mb:F1} MB");
}

WriteAllText wrote 64 bytes; ReadAllLines gave 4 lines, the last "MAS,Chennai Central"
"₹" as written by default: E2 82 B9; with Encoding.UTF8: EF BB BF E2 82 B9
ReadAllLines found "000042 train 12042 on time" and allocated 10.9 MB
ReadLines    found "000042 train 12042 on time" and allocated 0.0 MB
FileStream.IsAsync: True, wrote 5 bytes
RandomAccess read 2 bytes at offset 3: 40 50 (file length 5)

// Building paths: what Path.Combine, Path.Join and Path.GetFullPath return for the same parts. Paths inside this
// program's folder are printed relative to it.
string here = Path.GetFullPath(".");
string Show(string path) => path.StartsWith(here + "/", StringComparison.Ordinal) ? path[(here.Length + 1)..] : path;

Console.WriteLine($"separator '{Path.DirectorySeparatorChar}', alternative '{Path.AltDirectorySeparatorChar}', in PATH '{Path.PathSeparator}'");

(string Folder, string Name)[] cases = [("uploads", "report.pdf"), ("uploads", "/etc/hosts"), ("uploads", "../app.config"), ("uploads/", "/docs/a.txt")];
foreach (var (folder, name) in cases)
{
    Console.WriteLine($"\"{folder}\" + \"{name}\"");
    Console.WriteLine($"  Combine:     {Path.Combine(folder, name)}");
    Console.WriteLine($"  Join:        {Path.Join(folder, name)}");
    Console.WriteLine($"  GetFullPath: {Show(Path.GetFullPath(Path.Join(folder, name)))}");
}

// GetFullPath with a base folder of your own: the result never depends on the current directory.
string root = Path.GetFullPath("uploads");
Console.WriteLine($"GetFullPath(\"a/./b/../c.txt\", root) = {Show(Path.GetFullPath("a/./b/../c.txt", root))}");
Console.WriteLine($"GetRelativePath(root, its parent's app.config) = {Path.GetRelativePath(root, Path.GetFullPath("app.config"))}");

// A backslash is a separator on Windows only; here it is part of a file name.
Console.WriteLine($"GetFileName(\"..\\\\notes.txt\") = {Path.GetFileName("..\\notes.txt")}");
Console.WriteLine($"Characters a file name may not contain here: {Path.GetInvalidFileNameChars().Length} (codes {string.Join(", ", Path.GetInvalidFileNameChars().Select(c => (int)c))})");

// The file system decides whether case matters: one file, asked for in another case.
Directory.CreateDirectory("uploads");
File.WriteAllText(Path.Join("uploads", "report.pdf"), "demo");
Console.WriteLine($"File.Exists(\"uploads/REPORT.PDF\") on this Mac: {File.Exists(Path.Join("uploads", "REPORT.PDF"))}");

separator '/', alternative '/', in PATH ':'
"uploads" + "report.pdf"
  Combine:     uploads/report.pdf
  Join:        uploads/report.pdf
  GetFullPath: uploads/report.pdf
"uploads" + "/etc/hosts"
  Combine:     /etc/hosts
  Join:        uploads/etc/hosts
  GetFullPath: uploads/etc/hosts
"uploads" + "../app.config"
  Combine:     uploads/../app.config
  Join:        uploads/../app.config
  GetFullPath: app.config
"uploads/" + "/docs/a.txt"
  Combine:     /docs/a.txt
  Join:        uploads//docs/a.txt
  GetFullPath: uploads/docs/a.txt
GetFullPath("a/./b/../c.txt", root) = uploads/a/c.txt
GetRelativePath(root, its parent's app.config) = ../app.config
GetFileName("..\\notes.txt") = ..\notes.txt
Characters a file name may not contain here: 2 (codes 0, 47)
File.Exists("uploads/REPORT.PDF") on this Mac: True

// Listing a folder: Directory.EnumerateFiles with and without EnumerationOptions, on a tree this program makes.
// The file system returns names in no promised order, so every list is sorted before it is printed.
string root = "records";
foreach (string file in new[] { "a.csv", "B.CSV", ".cache", "notes.txt", "archive/q1.csv", "archive/q2.csv", "archive/old/q4.csv" })
{
    string path = Path.Join(root, file);
    Directory.CreateDirectory(Path.GetDirectoryName(path)!);
    File.WriteAllText(path, "x");
}

string List(IEnumerable<string> paths) => string.Join(", ", paths.Select(p => Path.GetRelativePath(root, p)).Order(StringComparer.Ordinal));

Console.WriteLine($"attributes of .cache: {File.GetAttributes(Path.Join(root, ".cache"))}; of notes.txt: {File.GetAttributes(Path.Join(root, "notes.txt"))}");
var defaults = new EnumerationOptions();
Console.WriteLine($"new EnumerationOptions(): recurse {defaults.RecurseSubdirectories}, skip {defaults.AttributesToSkip}, ignore inaccessible {defaults.IgnoreInaccessible}, casing {defaults.MatchCasing}");
Console.WriteLine($"EnumerateFiles(root)                 {List(Directory.EnumerateFiles(root))}");
Console.WriteLine($"EnumerateFiles(root, \"*\", options)   {List(Directory.EnumerateFiles(root, "*", defaults))}");
Console.WriteLine($"\"*.csv\", default options             {List(Directory.EnumerateFiles(root, "*.csv", defaults))}");
Console.WriteLine($"\"*.csv\", CaseSensitive               {List(Directory.EnumerateFiles(root, "*.csv", new EnumerationOptions { MatchCasing = MatchCasing.CaseSensitive }))}");
Console.WriteLine($"\"*.csv\", RecurseSubdirectories       {List(Directory.EnumerateFiles(root, "*.csv", new EnumerationOptions { RecurseSubdirectories = true }))}");
Console.WriteLine($"\"*.csv\", recurse, MaxRecursionDepth 1 {List(Directory.EnumerateFiles(root, "*.csv", new EnumerationOptions { RecurseSubdirectories = true, MaxRecursionDepth = 1 }))}");

// EnumerateFiles hands names over one at a time; GetFiles builds the whole array first.
using IEnumerator<string> names = Directory.EnumerateFiles(root, "*", new EnumerationOptions { RecurseSubdirectories = true }).GetEnumerator();
names.MoveNext();
Console.WriteLine($"EnumerateFiles gave its first name before reading the rest; GetFiles returned {Directory.GetFiles(root, "*", new EnumerationOptions { RecurseSubdirectories = true }).Length} at once");

attributes of .cache: Hidden; of notes.txt: Normal
new EnumerationOptions(): recurse False, skip Hidden, System, ignore inaccessible True, casing PlatformDefault
EnumerateFiles(root)                 .cache, B.CSV, a.csv, notes.txt
EnumerateFiles(root, "*", options)   B.CSV, a.csv, notes.txt
"*.csv", default options             B.CSV, a.csv
"*.csv", CaseSensitive               a.csv
"*.csv", RecurseSubdirectories       B.CSV, a.csv, archive/old/q4.csv, archive/q1.csv, archive/q2.csv
"*.csv", recurse, MaxRecursionDepth 1 B.CSV, a.csv, archive/q1.csv, archive/q2.csv
EnumerateFiles gave its first name before reading the rest; GetFiles returned 6 at once

// A download handler gets a file name from a user. Path.Combine alone lets the name leave the folder; the guard
// resolves the name first and refuses anything that does not stay inside. The program makes "uploads", one report
// in it and a file next to it, secrets.txt, that no user may read.
Directory.CreateDirectory("uploads");
File.WriteAllText(Path.Join("uploads", "report.pdf"), "%PDF demo");
File.WriteAllText("secrets.txt", "keys of the payment gateway");
string uploads = Path.GetFullPath("uploads");
string here = Path.GetFullPath(".");
string Show(string path) => path.StartsWith(here + "/", StringComparison.Ordinal) ? path[(here.Length + 1)..] : path;

foreach (string name in new[] { "report.pdf", "../secrets.txt", "sub/../../secrets.txt", "/etc/hosts" })
{
    string naive = Path.GetFullPath(Path.Combine(uploads, name));
    string read = naive.StartsWith(here + "/", StringComparison.Ordinal) && File.Exists(naive)
        ? $"and reads \"{File.ReadAllText(naive)}\""
        : "(outside this folder: not read here)";
    string verdict = TryResolve(uploads, name, out string? safe) ? $"allowed: {Show(safe!)}" : "refused";
    Console.WriteLine($"{name,-22} Combine opens {Show(naive)} {read}");
    Console.WriteLine($"{"",-22} guard: {verdict}");
}

// The guard compares names only. A symbolic link inside the folder can still point outside it.
File.CreateSymbolicLink(Path.Join("uploads", "notes.txt"), Path.Join("..", "secrets.txt"));
Console.WriteLine($"notes.txt (a link)     guard: {(TryResolve(uploads, "notes.txt", out string? link) ? "allowed" : "refused")}, but reading it gives \"{File.ReadAllText(link!)}\"");
Console.WriteLine($"with the link check    guard: {(TryResolveNoLinks(uploads, "notes.txt", out _) ? "allowed" : "refused")}");

// Resolves name inside baseDir. True only when the full path is baseDir plus a separator plus more.
static bool TryResolve(string baseDir, string name, out string? fullPath)
{
    fullPath = null;
    if (string.IsNullOrEmpty(name) || name.Contains('\0')) return false;
    string root = Path.TrimEndingDirectorySeparator(Path.GetFullPath(baseDir)) + Path.DirectorySeparatorChar;
    string candidate = Path.GetFullPath(name, root);
    if (!candidate.StartsWith(root, StringComparison.Ordinal)) return false;
    fullPath = candidate;
    return true;
}

// The same, and the file must not be a symbolic link.
static bool TryResolveNoLinks(string baseDir, string name, out string? fullPath) =>
    TryResolve(baseDir, name, out fullPath) && new FileInfo(fullPath!).LinkTarget is null;

report.pdf             Combine opens uploads/report.pdf and reads "%PDF demo"
                       guard: allowed: uploads/report.pdf
../secrets.txt         Combine opens secrets.txt and reads "keys of the payment gateway"
                       guard: refused
sub/../../secrets.txt  Combine opens secrets.txt and reads "keys of the payment gateway"
                       guard: refused
/etc/hosts             Combine opens /etc/hosts (outside this folder: not read here)
                       guard: refused
notes.txt (a link)     guard: allowed, but reading it gives "keys of the payment gateway"
with the link check    guard: refused

// Saving a settings file. The save is stopped half way, as a crash or a power cut would stop it, first while
// overwriting the file in place, then while writing a temporary file that is moved over the old one at the end.
using System.Text;
using System.Text.Json;

Directory.CreateDirectory("config");
string path = Path.Join("config", "settings.json");
string old = """{"theme":"dark","fontSize":14}""";
string next = """{"theme":"light","fontSize":16,"language":"hi-IN"}""";

File.WriteAllText(path, old);
Try("In place", () => SaveInPlace(path, next, stopAfter: 20));
Report(path);

File.WriteAllText(path, old);
Try("Temporary file", () => SaveAtomically(path, next, stopAfter: 20));
Report(path);

// The same save, not interrupted, and File.Replace, which also keeps the old version as a backup.
SaveAtomically(path, next);
Report(path);
File.WriteAllText(path + ".tmp", """{"theme":"light","fontSize":18}""");
File.Replace(path + ".tmp", path, path + ".bak");
Console.WriteLine($"After File.Replace: settings.json {File.ReadAllText(path)}, settings.json.bak {File.ReadAllText(path + ".bak")}");

// Overwrites the file where it is: File.Create empties it first, so a stop in between leaves part of the text.
static void SaveInPlace(string path, string text, int stopAfter = -1)
{
    using FileStream stream = File.Create(path);
    Write(stream, text, stopAfter);
}

// Writes the text to path.tmp in the same folder, forces it to the disk, then renames it over path.
static void SaveAtomically(string path, string text, int stopAfter = -1)
{
    string temp = path + ".tmp";
    using (FileStream stream = new(temp, FileMode.Create, FileAccess.Write, FileShare.None))
    {
        Write(stream, text, stopAfter);
        stream.Flush(flushToDisk: true);
    }
    File.Move(temp, path, overwrite: true);
}

static void Write(FileStream stream, string text, int stopAfter)
{
    byte[] bytes = Encoding.UTF8.GetBytes(text);
    if (stopAfter < 0) { stream.Write(bytes); return; }
    stream.Write(bytes, 0, stopAfter);
    throw new IOException($"stopped after {stopAfter} of {bytes.Length} bytes");
}

static void Try(string name, Action save)
{
    try { save(); }
    catch (IOException e) { Console.WriteLine($"{name}: the save was {e.Message}"); }
}

static void Report(string path)
{
    string text = File.ReadAllText(path);
    bool valid;
    try { using var _ = JsonDocument.Parse(text); valid = true; } catch (JsonException) { valid = false; }
    string others = string.Join(", ", Directory.EnumerateFiles(Path.GetDirectoryName(path)!).Select(Path.GetFileName).Where(n => n != "settings.json").Order(StringComparer.Ordinal));
    Console.WriteLine($"  settings.json: {text} (valid JSON: {valid}); other files: {(others.Length == 0 ? "none" : others)}");
}

In place: the save was stopped after 20 of 50 bytes
  settings.json: {"theme":"light","fo (valid JSON: False); other files: none
Temporary file: the save was stopped after 20 of 50 bytes
  settings.json: {"theme":"dark","fontSize":14} (valid JSON: True); other files: settings.json.tmp
  settings.json: {"theme":"light","fontSize":16,"language":"hi-IN"} (valid JSON: True); other files: none
After File.Replace: settings.json {"theme":"light","fontSize":18}, settings.json.bak {"theme":"light","fontSize":16,"language":"hi-IN"}

// A lab: SafeFileStore keeps every file a user saves inside one folder. Save and Load start out unwritten, so the
// checks fail. Write them until all nine pass (each check names the rule it tests).
var store = new SafeFileStore("store");
int passed = 0, total = 0;

Check("saves and loads report.pdf", () => { store.Save("report.pdf", Bytes(100)); return store.Load("report.pdf").Length == 100; });
Check("accepts an extension in capitals, SCAN.PNG", () => { store.Save("SCAN.PNG", Bytes(10)); return store.Load("SCAN.PNG").Length == 10; });
Check("refuses ../escape.txt", () => Refused(() => store.Save("../escape.txt", Bytes(1))));
Check("refuses an absolute path, /etc/hosts", () => Refused(() => store.Load("/etc/hosts")));
Check("refuses a name with a backslash", () => Refused(() => store.Save("..\\escape.txt", Bytes(1))));
Check("refuses run.exe: .pdf, .png and .txt only", () => Refused(() => store.Save("run.exe", Bytes(1))));
Check("refuses invoice.pdf.exe", () => Refused(() => store.Save("invoice.pdf.exe", Bytes(1))));
Check("refuses a file over 2 MB", () => Refused(() => store.Save("big.pdf", Bytes(2 * 1024 * 1024 + 1))));
Check("tells the user nothing about the folder", () => Message(() => store.Load("../store.cs")) == SafeFileStore.Refusal);

Console.WriteLine($"{passed} of {total} checks pass");
return passed == total ? 0 : 1;

void Check(string rule, Func<bool> test)
{
    total++;
    bool ok;
    try { ok = test(); } catch (Exception) { ok = false; }
    if (ok) passed++;
    Console.WriteLine($"{(ok ? "pass" : "FAIL")}  {rule}");
}

static byte[] Bytes(int count) => new byte[count];

// Refused means the store threw ArgumentException; any other exception is a bug.
static bool Refused(Action action)
{
    try { action(); return false; }
    catch (ArgumentException) { return true; }
}

static string? Message(Action action)
{
    try { action(); return null; }
    catch (ArgumentException e) { return e.Message; }
}

sealed class SafeFileStore(string folder)
{
    // The one message a user sees when a name or a size is refused.
    public const string Refusal = "That file is not allowed.";

    public void Save(string name, byte[] content) => throw new NotImplementedException();

    public byte[] Load(string name) => throw new NotImplementedException();
}

FAIL  saves and loads report.pdf
FAIL  accepts an extension in capitals, SCAN.PNG
FAIL  refuses ../escape.txt
FAIL  refuses an absolute path, /etc/hosts
FAIL  refuses a name with a backslash
FAIL  refuses run.exe: .pdf, .png and .txt only
FAIL  refuses invoice.pdf.exe
FAIL  refuses a file over 2 MB
FAIL  tells the user nothing about the folder
0 of 9 checks pass