Big SQL Select Query in Laravel

Question

I have table lids with under 430k entries. How can I do SELECT query faster, optimized because now this query working under 5-7 minutes. Heard something about Eloquent chunk in Laravel, but need clear PHP solution or with DB object. Maybe with 'for' construction to process 100-1000 entries at a time, smthng like this.

Tried to do 'for' construction, but don't know how to do this optimized.

Be gentle with me pls:)

Want to know your opinion.

How can i upgrade it?

UPD: Did something like this

$count = $db->doQuery("SELECT count(*) FROM `lids`"); // db - my custom object with Database connection
$count = $count[0]['count(*)'];
$hundreds = $count / 100; // get 'for' counts

$start = 43; // id index starts from 43 in the table
$end = 142;

for ($index = 1; $index < $hundreds; $index++) {

    $leads = [];
    $leads = $db->doQuery("SELECT * FROM `lids` WHERE `id` BETWEEN " . $start . " AND " . $end . "");

    // var_dump($leads);
    // die();

    if (empty($leads)) {
        $start += 100;
        $end += 100;
        continue;
    }

    $uniques = [];

    foreach ($leads as $lead) {
        if (empty($lead['json_vars'])) continue;

        $vars = preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $lead['json_vars']); // hot fix for charset, from DB comes string in utf8mb4
        $json = json_decode($vars, true);

        if (!isset($json['ser']['HTTP_COOKIE']) || !str_contains($json['ser']['HTTP_COOKIE'], '_fbc')) continue;

        $lead['json_vars'] = $json['ser']['HTTP_COOKIE'];

        // check for unique values
        if (!isset($uniques[$lead['id']]) || $uniques[$lead['id']]['phone'] != $lead['phone']) {
            $uniques[$lead['id']] = $lead;
        }

    }

    foreach ($uniques as $unique) {
        $lid = Lid::create($unique);
    }

    $start += 100;
    $end += 100;

    // here i get residue of entries
    if ($hundreds - $index < 1) {
        $leads = $db->doQuery("SELECT * FROM `lids` WHERE `id` IN(" . ($count - $hundreds * 100) . ", " . $count . ") AND WHERE ");
 
        foreach ($leads as $lead) {
            $json = json_decode(preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $lead['json_vars']), true);
            if (!str_contains($json['ser']['HTTP_COOKIE'], '_fbc')) continue;
            $lead['json_vars'] = $json['ser']['HTTP_COOKIE'];
            $lid = Lid::create($lead);
        }
    }

}

Upd2:

$db = new Database();
$counter = 0;

$scriptStart = date('d.m.Y H:i:s', strtotime('now'));

$lastRemote = $db->lastId('lids');

$lastInner = Lid::all(['id'])->last();
$lastInner = $lastInner->id;

$count = $lastRemote - $lastInner;

if ($count < 0) {

    echo 'no new objects, canceled';
    return false;
}

$start = $lastInner + 1;
$end = $lastRemote;
$fewEntries = false;

if ($count < 500) {

    $fewEntries = true;
    $index = $lastInner;
    $hundreds = $lastRemote;
} else {

    $index = 1;
    $hundreds = $count / 100;
    $end = $start + 99;
}

if ($fewEntries) {

    $leads = $db->itemsBetween('lids', 'id', [$start, $end]);

    $uniques = [];

    foreach ($leads as $lead) {

        if (empty($lead['json_vars'])) continue;

        $vars = preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $lead['json_vars']);
        $json = json_decode($vars, true);

        if (!isset($json['ser']['HTTP_COOKIE']) || !str_contains($json['ser']['HTTP_COOKIE'], '_fbc')) continue;

        $lead['json_vars'] = $json['ser']['HTTP_COOKIE'];

        if (
            !isset($uniques[$lead['id']]) ||
            $uniques[$lead['id']]['phone'] != $lead['phone'] &&
            $uniques[$lead['id']]['ip'] != $lead['ip'] &&
            $uniques[$lead['id']]['request_link'] != $lead['request_link']
        ) {
            $uniques[$lead['id']] = $lead;
        }
    }

    foreach ($uniques as $unique) {
        $lid = Lid::create($unique);
        $counter++;
    }
} else {

    for ($index; $index < $hundreds; $index++) {

        $leads = [];
        $leads = $db->itemsBetween('lids', 'id', [$start, $end]);

        // var_dump($leads);
        // die();

        $uniques = [];

        foreach ($leads as $lead) {

            if (empty($lead['json_vars'])) continue;

            $vars = preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $lead['json_vars']);
            $json = json_decode($vars, true);

            if (!isset($json['ser']['HTTP_COOKIE']) || !str_contains($json['ser']['HTTP_COOKIE'], '_fbc')) continue;

            $lead['json_vars'] = $json['ser']['HTTP_COOKIE'];

            if (
                !isset($uniques[$lead['id']]) ||
                $uniques[$lead['id']]['phone'] != $lead['phone'] &&
                $uniques[$lead['id']]['ip'] != $lead['ip'] &&
                $uniques[$lead['id']]['request_link'] != $lead['request_link']
            ) {
                $uniques[$lead['id']] = $lead;
            }
        }

        foreach ($uniques as $unique) {
            $lid = Lid::create($unique);
            $counter++;
        }

        $start += 100;
        $end += 100;
    }
}

$scriptEnd = date('d.m.Y H:i:s', strtotime('now'));

echo 'added in table: ' . $counter . PHP_EOL;
echo 'started at: ' . $scriptStart . PHP_EOL;
echo 'ended at: ' . $scriptEnd . PHP_EOL;

Answer 1

Resolved my problem with optimization of my trash code XD

$db = new Database();
$counter = 0;

$scriptStart = date('d.m.Y H:i:s', strtotime('now'));

$lastRemote = $db->lastId('lids');

$lastInner = Lid::all(['id'])->last();
$lastInner = $lastInner->id;

$count = $lastRemote - $lastInner;

if ($count < 0) {

    echo 'no new objects, canceled';
    return false;
}

$start = $lastInner + 1;
$end = $lastRemote;
$fewEntries = false;

if ($count < 500) {

    $fewEntries = true;
    $index = $lastInner;
    $hundreds = $lastRemote;
} else {

    $index = 1;
    $hundreds = $count / 100;
    $end = $start + 99;
}

if ($fewEntries) {

    $leads = $db->itemsBetween('lids', 'id', [$start, $end]);

    $uniques = [];

    foreach ($leads as $lead) {

        if (empty($lead['json_vars'])) continue;

        $vars = preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $lead['json_vars']);
        $json = json_decode($vars, true);

        if (!isset($json['ser']['HTTP_COOKIE']) || !str_contains($json['ser']['HTTP_COOKIE'], '_fbc')) continue;

        $lead['json_vars'] = $json['ser']['HTTP_COOKIE'];

        if (
            !isset($uniques[$lead['id']]) ||
            $uniques[$lead['id']]['phone'] != $lead['phone'] &&
            $uniques[$lead['id']]['ip'] != $lead['ip'] &&
            $uniques[$lead['id']]['request_link'] != $lead['request_link']
        ) {
            $uniques[$lead['id']] = $lead;
        }
    }

    foreach ($uniques as $unique) {
        $lid = Lid::create($unique);
        $counter++;
    }
} else {

    for ($index; $index < $hundreds; $index++) {

        $leads = [];
        $leads = $db->itemsBetween('lids', 'id', [$start, $end]);

        // var_dump($leads);
        // die();

        $uniques = [];

        foreach ($leads as $lead) {

            if (empty($lead['json_vars'])) continue;

            $vars = preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $lead['json_vars']);
            $json = json_decode($vars, true);

            if (!isset($json['ser']['HTTP_COOKIE']) || !str_contains($json['ser']['HTTP_COOKIE'], '_fbc')) continue;

            $lead['json_vars'] = $json['ser']['HTTP_COOKIE'];

            if (
                !isset($uniques[$lead['id']]) ||
                $uniques[$lead['id']]['phone'] != $lead['phone'] &&
                $uniques[$lead['id']]['ip'] != $lead['ip'] &&
                $uniques[$lead['id']]['request_link'] != $lead['request_link']
            ) {
                $uniques[$lead['id']] = $lead;
            }
        }

        foreach ($uniques as $unique) {
            $lid = Lid::create($unique);
            $counter++;
        }

        $start += 100;
        $end += 100;
    }
}

$scriptEnd = date('d.m.Y H:i:s', strtotime('now'));

echo 'added in table: ' . $counter . PHP_EOL;
echo 'started at: ' . $scriptStart . PHP_EOL;
echo 'ended at: ' . $scriptEnd . PHP_EOL;