Large Data CRUD in .NET: Batching Techniques to Avoid Out of Memory

BatchHelper

public static class BatchHelper
{
    public static IEnumerable<IEnumerable<T>> Batch<T>(this IEnumerable<T> source, int size)
    {
        using var e = source.GetEnumerator();
        while (e.MoveNext())
            yield return GetBatch(e, size);
    }


    private static IEnumerable<T> GetBatch<T>(IEnumerator<T> e, int size)
    {
        do yield return e.Current;
        while (--size > 0 && e.MoveNext());
    }
}

Batched Create (Insert)

public async Task BulkInsertAsync(IEnumerable<MyModel> data, MyDbContext dbContext)
{
    foreach (var batch in data.Batch(1000))
    {
        dbContext.MyModels.AddRange(batch);
        await dbContext.SaveChangesAsync();
    }
}

Batched Update

public async Task BulkUpdateAsync(IEnumerable<MyModel> updatedData, MyDbContext dbContext)
{
    foreach (var batch in updatedData.Batch(1000))
    {
        dbContext.MyModels.UpdateRange(batch);
        await dbContext.SaveChangesAsync();
    }
}

⚠️ If you're using detached objects or want to attach only modified values, you'll need to Attach() them and set their state to Modified.

// Alternative: attach manually
foreach (var item in batch)
{
    dbContext.MyModels.Attach(item);
    dbContext.Entry(item).State = EntityState.Modified;
}

Batched Delete

public async Task BulkDeleteAsync(IEnumerable<int> idsToDelete, MyDbContext dbContext)
{
    foreach (var batch in idsToDelete.Batch(1000))
    {
        var records = await dbContext.MyModels
            .Where(m => batch.Contains(m.Id))
            .ToListAsync();


        dbContext.MyModels.RemoveRange(records);
        await dbContext.SaveChangesAsync();
    }
}