Laravel Large Data — chunk, cursor, lazy

<?php
// You have 500,000 users in your database
// This will CRASH with memory error!

$users = User::all(); // Loads ALL 500,000 users into RAM

foreach ($users as $user) {
    $user->sendMonthlyReport();
}

// Error: Allowed memory size of 134217728 bytes exhausted
?>

<?php
// Process 1000 users at a time
User::chunk(1000, function ($users) {
    foreach ($users as $user) {
        $user->sendMonthlyReport();
    }
    // After this batch, memory is freed automatically
});

// With query constraints
User::where('active', true)
    ->orderBy('id')
    ->chunk(500, function ($users) {
        foreach ($users as $user) {
            Mail::to($user)->send(new NewsletterMail());
        }
    });

// Stop processing early by returning false
User::chunk(1000, function ($users) {
    foreach ($users as $user) {
        if ($user->hasError()) {
            return false; // Stop chunking
        }
        $user->process();
    }
});
?>

<?php
// PROBLEM with chunk() when updating:
// chunk() uses OFFSET which can skip records when rows are deleted/updated

// SOLUTION — chunkById() uses WHERE id > last_id instead of OFFSET
User::chunkById(1000, function ($users) {
    foreach ($users as $user) {
        $user->update(['points' => $user->points + 10]);
        // Safe! chunkById won't skip records even when updating
    }
});

// With custom column
Order::chunkById(500, function ($orders) {
    foreach ($orders as $order) {
        $order->recalculateTotal();
    }
}, column: 'id'); // specify the column to chunk by
?>

<?php
// cursor() uses a PHP Generator under the hood
// Only 1 record in memory at any time!

foreach (User::cursor() as $user) {
    $user->generateReport();
}

// With constraints
foreach (User::where('active', true)->cursor() as $user) {
    $user->sendEmail();
}

// Memory comparison with 100,000 users:
// User::all()    → ~200MB RAM
// chunk(1000)    → ~10MB RAM (1000 at a time)
// cursor()       → ~2MB RAM  (1 at a time!)
?>

<?php
// lazy() returns a LazyCollection — use Collection methods on huge datasets!
User::lazy()->each(function ($user) {
    $user->sendEmail();
});

// Use Collection methods — filter, map, etc. without loading all into memory
User::lazy()
    ->filter(fn($user) => $user->isActive())
    ->each(fn($user) => $user->sendReport());

// lazyById() — safer version like chunkById
User::lazyById()->each(fn($user) => $user->process());

// Real example — generate CSV for millions of orders
Order::where('status', 'completed')
    ->lazy()
    ->each(function ($order) {
        exportOrderToCSV($order);
    });
?>

<?php
public function exportOrders()
{
    return response()->streamDownload(function () {

        $handle = fopen('php://output', 'w');

        // Write CSV header
        fputcsv($handle, ['ID', 'Customer', 'Total', 'Status', 'Date']);

        // Stream 500,000 orders without memory issues!
        Order::with('customer')
            ->lazy()
            ->each(function ($order) use ($handle) {
                fputcsv($handle, [
                    $order->id,
                    $order->customer->name,
                    $order->total,
                    $order->status,
                    $order->created_at->format('Y-m-d'),
                ]);
            });

        fclose($handle);

    }, 'orders-' . now()->format('Y-m-d') . '.csv');
}
?>

<?php
// Best practice: chunk + queue jobs
// Don't send emails directly — dispatch to queue!

User::where('newsletter', true)
    ->chunk(500, function ($users) {
        foreach ($users as $user) {
            // Dispatch to queue — don't send directly!
            SendNewsletterJob::dispatch($user)
                ->delay(now()->addSeconds(rand(1, 60))); // Spread over time
        }
    });

// This way:
// ✅ No memory issues (chunk)
// ✅ No timeout issues (queue)
// ✅ Retries on failure (queue)
// ✅ Can monitor progress (Horizon)
?>

<?php
// NEVER do this in a web controller:
$orders = Order::all(); // Could be millions of records!

// ALWAYS paginate:
$orders = Order::latest()->paginate(20);      // 20 per page
$orders = Order::latest()->simplePaginate(20); // Faster (no total count)
$orders = Order::latest()->cursorPaginate(20); // Most efficient for large tables

// In Blade:
// {{ $orders->links() }} — renders pagination links automatically
?>

✅ Web page showing data       → paginate()
✅ Background job, bulk email  → chunk() + queue
✅ Updating/deleting records   → chunkById()
✅ Read-only export, report    → cursor() or lazy()
✅ Need Collection methods     → lazy()
❌ Never                       → all() on large tables
❌ Never                       → memory_limit = -1