How to remove duplicates from a list in C#

List<int> numbers = [3, 1, 3, 2, 1, 5, 2];

// Distinct() is lazy; ToList() builds the new list
List<int> unique = numbers.Distinct().ToList();
Console.WriteLine(string.Join(", ", unique));

// The original list is untouched: reassign to replace it
Console.WriteLine(numbers.Count);
numbers = numbers.Distinct().ToList();
Console.WriteLine(numbers.Count);

// Unique and sorted (Order() is .NET 7+; use OrderBy(n => n) before that)
List<int> scores = [70, 95, 70, 88, 95];
Console.WriteLine(string.Join(", ", scores.Distinct().Order()));

// Strings compare case-sensitively unless you pass a comparer
List<string> tags = ["csharp", "dotnet", "CSharp", "linq", "DotNet"];
Console.WriteLine(string.Join(", ", tags.Distinct()));
Console.WriteLine(string.Join(", ", tags.Distinct(StringComparer.OrdinalIgnoreCase)));

// Works on arrays and any other IEnumerable<T>
string[] words = "the cat and the hat and the bat".Split(' ');
string[] uniqueWords = words.Distinct().ToArray();
Console.WriteLine(string.Join(" ", uniqueWords));

List<int> numbers = [3, 1, 3, 2, 1, 5, 2];

// A HashSet<T> holds each value once
HashSet<int> set = numbers.ToHashSet();
Console.WriteLine(set.Count);
Console.WriteLine(set.Contains(5));

// Add returns false when the value is already there
Console.WriteLine(set.Add(5));
Console.WriteLine(set.Add(8));

// Remove duplicates in place, keeping the first of each
var seen = new HashSet<int>();
int removed = numbers.RemoveAll(n => !seen.Add(n));
Console.WriteLine($"removed {removed}: {string.Join(", ", numbers)}");

// The same thing as a plain loop over a copy
List<string> names = ["Ana", "Ben", "Ana", "Cleo", "Ben"];
var kept = new List<string>();
var seenNames = new HashSet<string>();
foreach (string name in names)
{
    if (seenNames.Add(name))
        kept.Add(name);
}
Console.WriteLine(string.Join(", ", kept));

// Need it sorted too? SortedSet<T> dedupes and keeps order by value
var sorted = new SortedSet<int>([9, 4, 9, 1, 4]);
Console.WriteLine(string.Join(", ", sorted));

// Does a list contain duplicates at all?
List<string> ids = ["a1", "b2", "a1"];
Console.WriteLine(ids.Count != ids.Distinct().Count());
Console.WriteLine(ids.Count != ids.ToHashSet().Count);

List<Person> people =
[
    new("Ana", "Lopez", "[email protected]"),
    new("Ben", "Okafor", "[email protected]"),
    new("Ana", "Lopez", "[email protected]"),
    new("Cleo", "Okafor", "[email protected]"),
];

// One person per first name; the first one wins (.NET 6+)
foreach (var p in people.DistinctBy(p => p.First))
    Console.WriteLine(p);

// Several properties: use a tuple as the key
Console.WriteLine(people.DistinctBy(p => (p.First, p.Last)).Count());

// Pass a comparer for the key, e.g. case-insensitive emails
Console.WriteLine(people.DistinctBy(p => p.Email, StringComparer.OrdinalIgnoreCase).Count());

// Before .NET 6: group by the key and take the first of each group
var firstPerLastName = people.GroupBy(p => p.Last).Select(g => g.First()).ToList();
Console.WriteLine(string.Join(", ", firstPerLastName.Select(p => p.First)));

record Person(string First, string Last, string Email);

// A class compares by reference: equal data is still two different objects
List<PointClass> classes = [new(1, 2), new(1, 2), new(3, 4)];
Console.WriteLine(classes.Distinct().Count());

// The same object added twice is a real duplicate
var shared = new PointClass(5, 6);
List<PointClass> twice = [shared, shared];
Console.WriteLine(twice.Distinct().Count());

// A record compares by value, so Distinct() just works
List<PointRecord> records = [new(1, 2), new(1, 2), new(3, 4)];
Console.WriteLine(records.Distinct().Count());

// So do value tuples
List<(int, int)> tuples = [(1, 2), (1, 2), (3, 4)];
Console.WriteLine(tuples.Distinct().Count());

// Or give a class value equality: IEquatable<T> plus GetHashCode
List<Sku> skus = [new("A-1"), new("a-1"), new("B-2")];
Console.WriteLine(string.Join(", ", skus.Distinct().Select(s => s.Code)));

class PointClass(int x, int y)
{
    public int X { get; } = x;
    public int Y { get; } = y;
}

record PointRecord(int X, int Y);

sealed class Sku(string code) : IEquatable<Sku>
{
    public string Code { get; } = code;

    // Codes match ignoring case, and equal objects must have equal hash codes
    public bool Equals(Sku? other) =>
        other is not null && string.Equals(Code, other.Code, StringComparison.OrdinalIgnoreCase);
    public override bool Equals(object? obj) => Equals(obj as Sku);
    public override int GetHashCode() => StringComparer.OrdinalIgnoreCase.GetHashCode(Code);
}

List<Product> products =
[
    new("Mug", 8.50m, "Kitchen"),
    new("mug", 9.00m, "Kitchen"),
    new("Lamp", 24.00m, "Home"),
    new("Pan", 31.00m, "Kitchen"),
];

// .NET 8+: build a comparer from two lambdas, equality and hash code
var sameName = EqualityComparer<Product>.Create(
    (a, b) => string.Equals(a?.Name, b?.Name, StringComparison.OrdinalIgnoreCase),
    p => StringComparer.OrdinalIgnoreCase.GetHashCode(p.Name));
Console.WriteLine(string.Join(", ", products.Distinct(sameName).Select(p => p.Name)));

// A comparer class works on every version and can be reused
var sameCategory = new SameCategory();
Console.WriteLine(string.Join(", ", products.Distinct(sameCategory).Select(p => p.Name)));

// The same comparer plugs into HashSet, Dictionary, Union, Except, Contains...
var oneOfEach = new HashSet<Product>(products, sameCategory);
Console.WriteLine(oneOfEach.Count);
Console.WriteLine(products.Contains(new Product("Rug", 50m, "Home"), sameCategory));

// Leave out the hash code and Distinct throws, because it hashes every item
var noHash = EqualityComparer<Product>.Create((a, b) => a?.Category == b?.Category);
try
{
    products.Distinct(noHash).ToList();
}
catch (NotSupportedException e)
{
    Console.WriteLine($"{e.GetType().Name}: {e.Message}");
}

record Product(string Name, decimal Price, string Category);

sealed class SameCategory : IEqualityComparer<Product>
{
    public bool Equals(Product? a, Product? b) => a?.Category == b?.Category;
    public int GetHashCode(Product p) => p.Category.GetHashCode();
}

List<string> emails = ["[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]"];

// Which values are duplicated, and how often?
var dupes = emails.GroupBy(e => e)
    .Where(g => g.Count() > 1)
    .Select(g => $"{g.Key} x{g.Count()}");
Console.WriteLine(string.Join(", ", dupes));

// .NET 9+: CountBy counts per key in one call
foreach (var (email, count) in emails.CountBy(e => e).Where(kv => kv.Value > 1).OrderBy(kv => kv.Key))
    Console.WriteLine($"{email}: {count}");

// Keep only the values that appear exactly once
var once = emails.GroupBy(e => e).Where(g => g.Count() == 1).Select(g => g.Key);
Console.WriteLine(string.Join(", ", once));

// Keep the LAST occurrence of each key instead of the first
List<Order> orders = [new(1, "pending"), new(2, "pending"), new(1, "paid"), new(1, "shipped")];

foreach (var o in orders.GroupBy(o => o.Id).Select(g => g.Last()))
    Console.WriteLine(o);

// Reverse, keep the first, reverse back: ordered by each key's last position
var latest = orders.AsEnumerable().Reverse().DistinctBy(o => o.Id).Reverse();
Console.WriteLine(string.Join(" | ", latest));

record Order(int Id, string Status);