Add duplicate-immutable detector — the project's "gold" (Plan.md §2 cat. 11)

audit/runtime/DuplicateDetector/DuplicateDetector.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <!--
+    Own.NET Audit — duplicate-immutable detector (Plan.md §2 cat. 11, the project's
+    "gold"). Build-required, WINDOWS-ONLY: reads a full process dump with ClrMD. NOT
+    part of the Linux CI gate — the Python ingest bridge (audit/runtime/ingest.py)
+    is what CI tests. net472 + x64 to match and read the legacy target's heap.
+  -->
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net472</TargetFramework>
+    <LangVersion>latest</LangVersion>
+    <Nullable>enable</Nullable>
+    <AssemblyName>DuplicateDetector</AssemblyName>
+    <RootNamespace>OwnNet.Audit.Runtime</RootNamespace>
+    <Platforms>x64</Platforms>
+    <PlatformTarget>x64</PlatformTarget>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Diagnostics.Runtime" Version="3.1.512801" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+</Project>

audit/runtime/DuplicateDetector/Program.cs

@@ -0,0 +1,249 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Diagnostics.Runtime;
+using Newtonsoft.Json;
+
+namespace OwnNet.Audit.Runtime
+{
+    /// <summary>
+    /// Duplicate-immutable-data detector (Plan.md §2 cat. 11 — the project's "gold").
+    /// Walks the TARGET process heap from a full dump (ClrMD) and groups identical
+    /// immutable values; a value held by many separate instances is wasted memory that
+    /// interning / a flyweight / a reference-by-id would collapse (units, countries,
+    /// currencies, repeated reference-table strings).
+    ///
+    /// This first cut covers STRINGS — the highest-value, most common case. Arbitrary
+    /// immutable reference types (field-by-field content equality) are a later
+    /// refinement. Emits a JSON result that audit/runtime/ingest.py converts to SARIF
+    /// (rule RUNTIME-DUP-IMMUTABLE -> category 11) for the unified report.
+    ///
+    /// Windows / build-required: needs a full dump (procdump) + ClrMD. Not part of the
+    /// Linux CI gate — the ingest bridge is what CI tests.
+    /// </summary>
+    internal static class Program
+    {
+        private static int Main(string[] args)
+        {
+            var opts = DupOptions.Parse(args);
+            if (opts == null)
+            {
+                return 2;
+            }
+            try
+            {
+                return Run(opts);
+            }
+            catch (Exception ex)
+            {
+                Console.Error.WriteLine($"duplicate-detector: failed — {ex.Message}");
+                return 2;
+            }
+        }
+
+        private static int Run(DupOptions opts)
+        {
+            // Analyze an existing dump, or take one of a live pid (then delete it).
+            var ownsDump = opts.DumpPath == null;
+            var dump = opts.DumpPath ?? CreateDump(opts);
+            var findings = new List<Dictionary<string, object>>();
+            try
+            {
+                using var dataTarget = DataTarget.LoadDump(dump);
+                var clr = dataTarget.ClrVersions.FirstOrDefault()
+                    ?? throw new InvalidOperationException(
+                        "dump contains no CLR — is the target a managed process?");
+                using var runtime = clr.CreateRuntime();
+
+                // Group live strings by value: instance count + total bytes per value.
+                // Group by the FULL string — the display cap (--max-string-length) must
+                // NOT be the grouping key, or distinct long strings that share the first
+                // N chars (JSON/XML blobs with a common prefix) collapse into one bogus
+                // group with inflated count/wastedBytes (Codex review on #103). Read the
+                // whole string for the key, hashing anything longer than the cap so the
+                // dictionary never retains large blobs, and keep a short readable sample
+                // only for display.
+                var groups = new Dictionary<string, (long Count, ulong Bytes, string Sample)>();
+                foreach (var obj in runtime.Heap.EnumerateObjects())
+                {
+                    if (obj.Type == null || !obj.Type.IsString)
+                    {
+                        continue;
+                    }
+                    var full = obj.AsString(int.MaxValue);   // actual length, not capped
+                    if (string.IsNullOrEmpty(full))
+                    {
+                        continue;
+                    }
+                    var key = full!.Length <= opts.MaxStringLength ? full : LongKey(full);
+                    if (groups.TryGetValue(key, out var cur))
+                    {
+                        groups[key] = (cur.Count + 1, cur.Bytes + obj.Size, cur.Sample);
+                    }
+                    else
+                    {
+                        groups[key] = (1, obj.Size, Truncate(full, opts.MaxStringLength));
+                    }
+                }
+
+                foreach (var kv in groups)
+                {
+                    var count = kv.Value.Count;
+                    if (count < 2)
+                    {
+                        continue;   // a unique value is not duplication
+                    }
+                    var bytesPer = (long)(kv.Value.Bytes / (ulong)count);
+                    var wasted = (count - 1) * bytesPer;   // duplicates beyond one canonical instance
+                    var report = wasted >= opts.MinWastedBytes;
+                    if (!report && !opts.IncludeBelowThreshold)
+                    {
+                        continue;
+                    }
+                    var sample = kv.Value.Sample;
+                    findings.Add(new Dictionary<string, object>
+                    {
+                        ["rule"] = "RUNTIME-DUP-IMMUTABLE",
+                        ["type"] = "System.String",
+                        ["value"] = Truncate(sample, 80),
+                        ["count"] = count,
+                        ["bytesPerInstance"] = bytesPer,
+                        ["wastedBytes"] = wasted,
+                        ["report"] = report,
+                        ["message"] = $"{count} duplicate \"{Truncate(sample, 40)}\" strings "
+                                      + $"(~{wasted / 1024} KB wasted; intern or use a flyweight)",
+                    });
+                }
+            }
+            finally
+            {
+                if (ownsDump)
+                {
+                    File.Delete(dump);   // dumps are large; the findings are the artifact
+                }
+            }
+
+            findings = findings.OrderByDescending(f => (long)f["wastedBytes"]).ToList();
+            var result = new Dictionary<string, object>
+            {
+                ["tool"] = "duplicate-detector",
+                ["target"] = opts.Target,
+                ["commit"] = opts.Commit,
+                ["minWastedBytes"] = opts.MinWastedBytes,
+                ["findings"] = findings,
+            };
+            Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(opts.OutPath))!);
+            File.WriteAllText(opts.OutPath, JsonConvert.SerializeObject(result, Formatting.Indented));
+
+            var reported = findings.Count(f => (bool)f["report"]);
+            Console.WriteLine(
+                $"duplicate-detector: {reported} duplicate group(s) over threshold -> {opts.OutPath}");
+            return reported > 0 ? 1 : 0;
+        }
+
+        private static string CreateDump(DupOptions opts)
+        {
+            if (opts.Pid == 0)
+            {
+                throw new ArgumentException("either --dump or --pid is required");
+            }
+            Directory.CreateDirectory(opts.ScratchDir);
+            var dump = Path.Combine(opts.ScratchDir, $"dup-{opts.Pid}.dmp");
+            var psi = new ProcessStartInfo(opts.ProcdumpPath, $"-accepteula -ma {opts.Pid} \"{dump}\"")
+            {
+                UseShellExecute = false,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                CreateNoWindow = true,
+            };
+            using var p = Process.Start(psi)!;
+            // Drain pipes before waiting (avoid the full-buffer deadlock); bound the wait.
+            var stdout = p.StandardOutput.ReadToEndAsync();
+            var stderr = p.StandardError.ReadToEndAsync();
+            if (!p.WaitForExit(120_000))
+            {
+                try { p.Kill(); } catch { /* best effort */ }
+                throw new IOException($"procdump timed out (>120s) for pid {opts.Pid}");
+            }
+            Task.WaitAll(stdout, stderr);
+            if (p.ExitCode != 0 || !File.Exists(dump))
+            {
+                throw new IOException(
+                    $"procdump failed for pid {opts.Pid} (exit {p.ExitCode}): {stderr.Result}");
+            }
+            return dump;
+        }
+
+        private static string Truncate(string s, int n) => s.Length <= n ? s : s.Substring(0, n) + "…";
+
+        // For strings longer than the display cap, key on length + a hash of the FULL
+        // content so distinct long strings never merge, without retaining the blob.
+        private static string LongKey(string s)
+        {
+            using var sha = SHA256.Create();
+            var hash = sha.ComputeHash(Encoding.Unicode.GetBytes(s));
+            return "len=" + s.Length + ":" + BitConverter.ToString(hash).Replace("-", "");
+        }
+    }
+
+    /// <summary>Command-line options for the duplicate detector.</summary>
+    internal sealed class DupOptions
+    {
+        public string? DumpPath { get; private set; }
+        public int Pid { get; private set; }
+        public string ProcdumpPath { get; private set; } = "procdump.exe";
+        public string ScratchDir { get; private set; } = "artifacts/own-audit/dumps";
+        public long MinWastedBytes { get; private set; } = 65536;   // 64 KB
+        public int MaxStringLength { get; private set; } = 512;
+        public bool IncludeBelowThreshold { get; private set; }
+        public string OutPath { get; private set; } = "artifacts/own-audit/duplicate-detector.json";
+        public string Target { get; private set; } = "";
+        public string Commit { get; private set; } = "";
+
+        public static DupOptions? Parse(string[] args)
+        {
+            var o = new DupOptions();
+            for (var i = 0; i < args.Length; i++)
+            {
+                switch (args[i])
+                {
+                    case "--dump": o.DumpPath = Next(args, ref i); break;
+                    case "--pid": o.Pid = int.Parse(Next(args, ref i)); break;
+                    case "--procdump": o.ProcdumpPath = Next(args, ref i); break;
+                    case "--scratch": o.ScratchDir = Next(args, ref i); break;
+                    case "--min-wasted-bytes": o.MinWastedBytes = long.Parse(Next(args, ref i)); break;
+                    case "--max-string-length": o.MaxStringLength = int.Parse(Next(args, ref i)); break;
+                    case "--include-below-threshold": o.IncludeBelowThreshold = true; break;
+                    case "--out": o.OutPath = Next(args, ref i); break;
+                    case "--target": o.Target = Next(args, ref i); break;
+                    case "--commit": o.Commit = Next(args, ref i); break;
+                    default:
+                        Console.Error.WriteLine($"duplicate-detector: unknown arg {args[i]}");
+                        return null;
+                }
+            }
+            if (o.DumpPath == null && o.Pid == 0)
+            {
+                Console.Error.WriteLine(
+                    "Usage: DuplicateDetector (--dump <file> | --pid <n> --procdump <path>) "
+                    + "[--min-wasted-bytes N] [--out json] [--target owner/repo] [--commit sha]");
+                return null;
+            }
+            return o;
+        }
+
+        private static string Next(string[] args, ref int i)
+        {
+            if (i + 1 >= args.Length)
+            {
+                throw new ArgumentException($"{args[i]} requires a value");
+            }
+            return args[++i];
+        }
+    }
+}

audit/runtime/README.md

@@ -29,14 +29,17 @@ the CoreCLR-only `dotnet-*` tools:
 
 ```text
 audit/runtime/
-  ingest.py            # leak-harness JSON -> SARIF -> the unified pipeline (PURE PYTHON, CI-gated)
+  ingest.py            # runtime JSON -> SARIF -> the unified pipeline (PURE PYTHON, CI-gated)
   scenarios/
     open-close-declaration.yml   # one deterministic leak-harness scenario (+ schema docs)
-  LeakHarness/         # C# harness — Windows/build-required, NOT CI-gated
+  LeakHarness/         # C# leak-harness — Windows/build-required, NOT CI-gated
     LeakHarness.csproj # net472; FlaUI.UIA3 + Microsoft.Diagnostics.Runtime + YamlDotNet
     Program.cs         # GC+snapshot loop, growth assertion, JSON result
     Scenario.cs        # YAML model
     HeapCounter.cs     # procdump + ClrMD: count live instances of suspect types
+  DuplicateDetector/   # C# duplicate-immutable detector — Windows/build-required, NOT CI-gated
+    DuplicateDetector.csproj  # net472; Microsoft.Diagnostics.Runtime
+    Program.cs         # ClrMD over a full dump: group identical strings, wasted-bytes findings
 ```
 
 ## How the leak-harness works (Plan.md §4.1)
@@ -64,6 +67,27 @@ python audit/runtime/ingest.py --leak-harness artifacts/own-audit/leak-harness.j
 #    folds it in and a confirmed leak clusters with its static OWN014/OWN001.
 ```
 
+## Duplicate-immutable detector (Plan.md §2 cat. 11 — the "gold")
+
+A heap full of identical immutable values (the same `"Country"` / unit / currency
+string held by thousands of separate instances) is wasted memory that interning, a
+flyweight, or a reference-by-id would collapse. The detector walks a full dump with
+ClrMD, groups strings by value, and reports each group whose duplicates waste more
+than `--min-wasted-bytes`. (Strings first — the highest-value case; arbitrary
+immutable types are a later refinement.) It needs no UI scenario — it's a one-shot
+heap analysis.
+
+```bash
+# on Windows, against a dump (or a live --pid with --procdump):
+DuplicateDetector.exe --dump target.dmp --min-wasted-bytes 65536 \
+    --out artifacts/own-audit/duplicate-detector.json --target acme/LegacyApp --commit "$COMMIT"
+
+# then, anywhere (CI exercises this conversion):
+python audit/runtime/ingest.py --duplicate-detector artifacts/own-audit/duplicate-detector.json \
+    --out artifacts/own-audit/duplicate-detector.sarif
+# -> run_static folds duplicate-detector.sarif in as a category-11 (P2) finding set.
+```
+
 ## Selftest
 
 `ingest.py` carries embedded-fixture selftests (no harness, no Windows needed) and
@@ -76,9 +100,10 @@ python audit/runtime/ingest.py --selftest
 
 ## Status
 
-- **Done:** the runtime→pipeline bridge (`ingest.py`, CI-gated), the leak-harness
-  scenario schema + one scenario, runtime rule mappings in the taxonomy (categories
-  2/3/4/11), and the C# leak-harness skeleton.
-- **Deferred:** the ClrMD duplicate-immutable detector and PropertyChanged-storm
-  profiler (more C# over the same dump/ETW), PerfView/SematixTrace wiring, and a
-  scenario corpus for the top-N screens.
+- **Done:** the runtime→pipeline bridge (`ingest.py`, CI-gated, for both the
+  leak-harness and the duplicate detector), the leak-harness scenario schema + one
+  scenario, runtime rule mappings in the taxonomy (categories 2/3/4/11), the C#
+  leak-harness skeleton, and the C# duplicate-immutable detector (strings).
+- **Deferred:** duplicate detection for arbitrary immutable types (field-by-field
+  content equality), the PropertyChanged-storm profiler (C# over ETW),
+  PerfView/SematixTrace wiring, and a scenario corpus for the top-N screens.