MySQL - query fetching very slow
Question
CREATE TABLE `offers` (
`id` mediumint(8) unsigned NOT NULL auto_increment,
`cap_id` varchar(255),
`deposit_value` varchar(255) default NULL,
`term` varchar(255) default NULL,
`annual_mileage` varchar(255) default NULL,
`finance_type` varchar(255) default NULL,
`monthly_payment` mediumint default NULL,
`stock` varchar(255) default NULL,
PRIMARY KEY (`id`)
) AUTO_INCREMENT=1;
INSERT INTO `offers` (`cap_id`,`deposit_value`,`term`,`annual_mileage`,`finance_type`,`monthly_payment`,`stock`) VALUES ("93897","6","24","12000","B",363,"0"),("90626","1","24","12000","P",810,"0"),("93607","6","36","25000","B",172,"1"),("35877","1","48","10000","B",678,"1"),("77134","3","24","10000","P",454,"0"),("47080","6","36","10000","B",737,"0"),("46392","3","24","30000","P",261,"0"),("21418","3","36","20000","B",390,"1"),("80175","9","48","10000","B",713,"1"),("91487","6","48","12000","P",778,"1");
INSERT INTO `offers` (`cap_id`,`deposit_value`,`term`,`annual_mileage`,`finance_type`,`monthly_payment`,`stock`) VALUES ("18311","9","24","20000","B",384,"0"),("41740","9","24","12000","P",674,"1"),("69030","9","24","10000","P",518,"0"),("53342","3","36","8000","B",746,"1"),("65566","6","36","30000","P",145,"1"),("63172","6","48","5000","P",698,"1"),("79712","1","48","30000","B",330,"1"),("90505","1","36","8000","B",458,"0"),("42393","1","36","20000","B",363,"1"),("70454","9","24","5000","B",673,"1");
INSERT INTO `offers` (`cap_id`,`deposit_value`,`term`,`annual_mileage`,`finance_type`,`monthly_payment`,`stock`) VALUES ("81215","1","48","8000","B",472,"0"),("76538","3","24","15000","B",226,"0"),("05094","9","36","12000","P",721,"1"),("57363","9","48","5000","B",777,"1"),("23233","1","48","12000","B",381,"0"),("40542","3","48","12000","P",610,"0"),("63824","3","24","12000","B",761,"1"),("17686","3","24","5000","P",893,"1"),("57669","9","48","30000","P",805,"0"),("21864","1","24","25000","P",530,"0");
INSERT INTO `offers` (`cap_id`,`deposit_value`,`term`,`annual_mileage`,`finance_type`,`monthly_payment`,`stock`) VALUES ("48360","3","48","12000","P",159,"0"),("88614","3","48","20000","B",730,"0"),("17693","1","24","10000","B",298,"0"),("34049","6","48","20000","B",728,"0"),("15038","9","24","10000","P",720,"1"),("31809","1","36","20000","P",237,"0"),("49277","9","48","25000","P",235,"1"),("54607","1","24","12000","P",661,"1"),("65098","1","48","20000","P",548,"1"),("76440","9","48","10000","P",495,"1");
CREATE TABLE `offers_lowest` (
`id` mediumint(8) unsigned NOT NULL auto_increment,
`date` DATE,
`cap_id` varchar(255),
`deposit` varchar(255) default NULL,
`term` varchar(255) default NULL,
`mileage` varchar(255) default NULL,
`finance_type` varchar(255) default NULL,
`lowest_price` mediumint default NULL,
`stock` varchar(255) default NULL,
PRIMARY KEY (`id`)
) AUTO_INCREMENT=1;
INSERT INTO `offers_lowest` (`date`,`cap_id`,`deposit`,`term`,`mileage`,`finance_type`,`lowest_price`,`stock`) VALUES ("2021-04-09","93897","6","24","12000","B",363,"0"),("2021-04-010","90626","1","24","12000","P",810,"0"),("2021-04-010","93607","6","36","25000","B",172,"1"),("2021-04-10","35877","1","48","10000","B",678,"1"),("2021-04-11","77134","3","24","10000","P",454,"0"),("2021-04-11","47080","6","36","10000","B",737,"0"),("2021-04-11","46392","3","24","30000","P",261,"0"),("2021-04-11","21418","3","36","20000","B",390,"1"),("2021-04-12","80175","9","48","10000","B",713,"1"),("2021-04-12","91487","6","48","12000","P",778,"1");
INSERT INTO `offers_lowest` (`date`,`cap_id`,`deposit`,`term`,`mileage`,`finance_type`,`lowest_price`,`stock`) VALUES ("2021-04-09","18311","9","24","20000","B",384,"0"),("2021-04-010","41740","9","24","12000","P",674,"1"),("2021-04-010","69030","9","24","10000","P",518,"0"),("2021-04-10","53342","3","36","8000","B",746,"1"),("2021-04-11","65566","6","36","30000","P",145,"1"),("2021-04-11","63172","6","48","5000","P",698,"1"),("2021-04-11","79712","1","48","30000","B",330,"1"),("2021-04-11","90505","1","36","8000","B",458,"0"),("2021-04-12","42393","1","36","20000","B",363,"1"),("2021-04-12","70454","9","24","5000","B",673,"1");
INSERT INTO `offers_lowest` (`date`,`cap_id`,`deposit`,`term`,`mileage`,`finance_type`,`lowest_price`,`stock`) VALUES ("2021-04-09","81215","1","48","8000","B",472,"0"),("2021-04-09","76538","3","24","15000","B",226,"0"),("2021-04-010","05094","9","36","12000","P",721,"1"),("2021-04-10","57363","9","48","5000","B",777,"1"),("2021-04-11","23233","1","48","12000","B",381,"0"),("2021-04-11","40542","3","48","12000","P",610,"0"),("2021-04-11","63824","3","24","12000","B",761,"1"),("2021-04-11","17686","3","24","5000","P",893,"1"),("2021-04-12","57669","9","48","30000","P",805,"0"),("2021-04-12","21864","1","24","25000","P",530,"0");
INSERT INTO `offers_lowest` (`date`,`cap_id`,`deposit`,`term`,`mileage`,`finance_type`,`lowest_price`,`stock`) VALUES ("2021-04-09","48360","3","48","12000","P",159,"0"),("2021-04-09","88614","3","48","20000","B",730,"0"),("2021-04-010","17693","1","24","10000","B",298,"0"),("2021-04-10","34049","6","48","20000","B",728,"0"),("2021-04-11","15038","9","24","10000","P",720,"1"),("2021-04-11","31809","1","36","20000","P",237,"0"),("2021-04-11","49277","9","48","25000","P",235,"1"),("2021-04-11","54607","1","24","12000","P",661,"1"),("2021-04-12","65098","1","48","20000","P",548,"1"),("2021-04-12","76440","9","48","10000","P",495,"1");
CREATE INDEX idx_profile_grouping ON offers (cap_id, deposit_value, term, annual_mileage);
CREATE INDEX idx_specials_query ON offers_lowest (cap_id, deposit, term, mileage);
So, I'm trying to join two tables - one a list of active offers on our platform (we're a car leasing comparison site) and a custom table which records, daily, the best price for every vehicle (cap_id), and every possible finance profile ie, 9 month deposit value, 24 month contract and 8,000 miles per year, etc. etc.
This is with the end goal of identifying what a 'good' price is, with the output containing both MIN(offers.monthly_payment) and MIN(offers_lowest.lowest_price) , and a calculated percentage difference, with the historical price data taken from a 7-day date range not inclusive of the current day.
offers_lowest is aggregated from offers daily, finding the minimum monthly_payment grouped by: cap_id, deposit_value, term, annual_mileage, finance_type
This is what I've come up with so far:
SELECT
s.cap_id,
s.deposit,
s.term,
s.mileage,
s.best_price,
f.previous_best,
(( s.best_price - f.previous_best ) / f.previous_best ) * 100 AS difference,
s.stock
FROM
(SELECT
o.cap_id,
o.deposit_value as deposit,
o.term,
o.annual_mileage as mileage,
o.finance_type,
MIN(o.monthly_payment) AS best_price,
o.stock,
o.brand_id
FROM
offers o USE INDEX(idx_profile_grouping)
WHERE o.finance_type = 'P'
GROUP BY o.cap_id, o.deposit_value, o.term, o.annual_mileage
) s
INNER JOIN
(SELECT
ol.cap_id,
ol.deposit,
ol.term,
ol.mileage,
MIN(lowest_price) as previous_best
FROM
offers_lowest ol USE INDEX(idx_specials_query)
WHERE finance_type = 'P'
AND ol.date > CURDATE() - INTERVAL 7 DAY
AND ol.date <= CURDATE() - INTERVAL 1 DAY
GROUP BY ol.cap_id, ol.deposit, ol.term, ol.mileage
) f ON s.cap_id = f.cap_id AND s.deposit = f.deposit AND s.term = f.term AND s.mileage = f.mileage
GROUP BY s.cap_id, s.deposit, s.term, s.mileage
The issue is with speed. When I run either one of the subqueries individually, it takes <0.1s to run, then around 75 seconds to fetch. When I run the whole thing, it runs until I stop it (>10 minutes).
The indexes are ones I created over the cap_id, deposit, term and mileage fields (both tables) in an attempt to speed things up.
EXPLAIN returns this:
| # id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
|---|---|---|---|---|---|---|---|---|---|---|---|
| 1 | PRIMARY | ALL | 965741 | 100.00 | Using where; Using temporary; Using filesort | ||||||
| 1 | PRIMARY | ref | <auto_key0> | <auto_key0> | 10 | f.cap_id | f.deposit | f.term | f.mileage | ||
| 3 | DERIVED | ol | index | idx_specials_query | idx_specials_query | 13 | 17386818 | 5.55 | Using where | ||
| 2 | DERIVED | o | index | idx_profile_grouping | idx_profile_grouping | 10 | 4800964 | 50.00 | Using where |
And SHOW PROFILE returns this:
| Status | Duration |
|---|---|
| starting | 0.000017 |
| checking query cache for query | 0.000171 |
| checking permissions | 0.000006 |
| checking permissions | 0.000005 |
| Opening tables | 0.000020 |
| init | 0.000109 |
| System lock | 0.000009 |
| optimizing | 0.000005 |
| optimizing | 0.000011 |
| statistics | 0.000033 |
| preparing | 0.000037 |
| Sorting result | 0.000007 |
| optimizing | 0.000015 |
| statistics | 0.000017 |
| preparing | 0.000017 |
| Sorting result | 0.000012 |
| statistics | 0.000046 |
| preparing | 0.000021 |
| Creating tmp table | 0.000020 |
| Sorting result | 0.000007 |
| executing | 0.000013 |
| Sending data | 0.000012 |
| executing | 0.000004 |
| Sending data | 68.205032 |
| converting HEAP to ondisk | 0.542756 |
| Sending data | 3.132642 |
| executing | 0.000019 |
| Sending data | 7.968966 |
| converting HEAP to ondisk | 0.746200 |
| Sending data | 22.506716 |
| converting HEAP to ondisk | 0.567816 |
| Sending data | 4.886870 |
| Creating sort index | 2.151519 |
| end | 0.000019 |
| query end | 0.000010 |
| removing tmp table | 0.002051 |
| query end | 0.000019 |
| closing tables | 0.000005 |
| removing tmp table | 0.001385 |
| closing tables | 0.000009 |
| removing tmp table | 0.002780 |
| closing tables | 0.000017 |
| freeing items | 0.000069 |
| cleaned up | 0.000007 |
| cleaning up | 0.000021 |
It seems from this that I've not written the most efficient query..
We use an AWS RDS instance, running MySQL 5.7.12.
If there's anything additional I can provide to add background, please let me know.
1 answers
solution1
1 2021-04-12 16:40:09
FROM ( SELECT... ) JOIN ( SELECT... )is inefficient. Try to move at least one of them into the outerSELECT. (This may eliminate "Creating sort index" if you add a suitable index for theJOIN. It may also eliminate most of the cryptic "sending data".)- That date range includes only 6 DATEs.
- What if one row has MIN(offers.monthly_payment) and a different row has MIN(offers_lowest.lowest_price)?
- Don't use VARCHAR(255) for numeric values. Eg, MEDIUMINT for mileage. (This may eliminate "converting HEAP to ondisk".)
- Note how useless PROFILE is; it likes to say "sending data" for most of the time (95% of total). I could identify 4% and recommend fixes, see above. The other 1% is unavoidable noise.
- The first
GROUP BYis inappropriate since it does not say which value to give for stock or brand_id. If necessary, grab them with an extraJOINafter finding the rest of the info. -
offers_lowestneedsINDEX(finance_type, date) - Get rid of "index hints" (
USE INDEX...)
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.