Skip to content
/ server Public

WIP: MDEV-38728 Improve join size estimation for ref access #4608

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Olernov wants to merge 1 commit into
base: 11.4
Choose a base branch
from
Open

WIP: MDEV-38728 Improve join size estimation for ref access #4608

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions mysql-test/main/delete_use_source_engines.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -731,15 +731,15 @@ create table tmp as select * from t1 where c1 in (select a.c2 from t1 a where a.
explain select * from t1 where c1 in (select a.c2 from t1 a where a.c2 = t1.c3) limit 1;
id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY t1 ALL NULL NULL NULL NULL 32 Using where
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 5 Using index; FirstMatch(t1)
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 4 Using index; FirstMatch(t1)
explain delete from t1 where c1 in (select a.c2 from t1 a where a.c2 = t1.c3) limit 1;
id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY t1 ALL NULL NULL NULL NULL 32 Using where
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 5 Using index; FirstMatch(t1)
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 4 Using index; FirstMatch(t1)
analyze delete from t1 where c1 in (select a.c2 from t1 a where a.c2 = t1.c3) limit 1;
id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra
1 PRIMARY t1 ALL NULL NULL NULL NULL 32 32.00 100.00 12.50 Using where
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 5 0.25 18.75 100.00 Using index; FirstMatch(t1)
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 4 0.25 25.00 100.00 Using index; FirstMatch(t1)
select * from t1;
c1 c2 c3
1 2 2
Expand Down Expand Up @@ -2709,15 +2709,15 @@ create table tmp as select * from t1 where c1 in (select a.c2 from t1 a where a.
explain select * from t1 where c1 in (select a.c2 from t1 a where a.c2 = t1.c3) limit 1;
id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY t1 ALL NULL NULL NULL NULL 32 Using where
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 5 Using index; FirstMatch(t1)
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 4 Using index; FirstMatch(t1)
explain delete from t1 where c1 in (select a.c2 from t1 a where a.c2 = t1.c3) limit 1;
id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY t1 ALL NULL NULL NULL NULL 32 Using where
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 5 Using index; FirstMatch(t1)
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 4 Using index; FirstMatch(t1)
analyze delete from t1 where c1 in (select a.c2 from t1 a where a.c2 = t1.c3) limit 1;
id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra
1 PRIMARY t1 ALL NULL NULL NULL NULL 32 27.00 100.00 14.81 Using where
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 5 0.25 18.75 100.00 Using index; FirstMatch(t1)
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 4 0.25 25.00 100.00 Using index; FirstMatch(t1)
select * from t1;
c1 c2 c3
1 2 2
Expand Down Expand Up @@ -2762,7 +2762,7 @@ explain select * from t1 where c1 in (select a.c2 from t1 a where a.c3 = t1.c3)
order by c3 desc limit 1;
id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY t1 ALL NULL NULL NULL NULL 32 Using where; Using filesort
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 5 Using where; FirstMatch(t1)
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 4 Using where; FirstMatch(t1)
explain delete from t1 where c1 in (select a.c2 from t1 a where a.c3 = t1.c3)
order by c3 desc limit 1;
id select_type table type possible_keys key key_len ref rows Extra
Expand Down Expand Up @@ -5114,15 +5114,15 @@ create table tmp as select * from t1 where c1 in (select a.c2 from t1 a where a.
explain select * from t1 where c1 in (select a.c2 from t1 a where a.c2 = t1.c3) limit 1;
id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY t1 ALL NULL NULL NULL NULL 32 Using where
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 5 Using index; FirstMatch(t1)
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 4 Using index; FirstMatch(t1)
explain delete from t1 where c1 in (select a.c2 from t1 a where a.c2 = t1.c3) limit 1;
id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY t1 ALL NULL NULL NULL NULL 32 Using where
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 5 Using index; FirstMatch(t1)
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 4 Using index; FirstMatch(t1)
analyze delete from t1 where c1 in (select a.c2 from t1 a where a.c2 = t1.c3) limit 1;
id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra
1 PRIMARY t1 ALL NULL NULL NULL NULL 32 29.00 100.00 13.79 Using where
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 5 0.25 18.75 100.00 Using index; FirstMatch(t1)
1 PRIMARY a ref t1_c2 t1_c2 5 test.t1.c1 4 0.25 25.00 100.00 Using index; FirstMatch(t1)
select * from t1;
c1 c2 c3
1 2 2
Expand Down
12 changes: 6 additions & 6 deletions mysql-test/main/derived_cond_pushdown.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20147,7 +20147,7 @@ from t2, t3, (select c, max(b) max, min(b) min from t4 group by c) t
where t2.b < 40 and t2.a=t3.a and t3.c=t.c;
id select_type table type possible_keys key key_len ref rows filtered Extra
1 PRIMARY t2 ALL NULL NULL NULL NULL 90 60.00 Using where
1 PRIMARY t3 ref idx_a idx_a 5 test.t2.a 1 100.00 Using where
1 PRIMARY t3 ref idx_a idx_a 5 test.t2.a 0 100.00 Using where
1 PRIMARY <derived2> ref key0 key0 128 test.t3.c 5 100.00
2 DERIVED t4 ALL idx_c NULL NULL NULL 160 100.00 Using temporary; Using filesort
Warnings:
Expand Down Expand Up @@ -20182,7 +20182,7 @@ EXPLAIN
"used_key_parts": ["a"],
"ref": ["test.t2.a"],
"loops": 53.99999991,
"rows": 1,
"rows": 0,
"cost": "COST_REPLACED",
"filtered": 100,
"attached_condition": "t3.c is not null"
Expand All @@ -20197,7 +20197,7 @@ EXPLAIN
"key_length": "128",
"used_key_parts": ["c"],
"ref": ["test.t3.c"],
"loops": 80.99999987,
"loops": 52.89695991,
"rows": 5,
"cost": "COST_REPLACED",
"filtered": 100,
Expand Down Expand Up @@ -20690,7 +20690,7 @@ from t2, t3, (select c, b, sum(b) over (partition by c) from t4 ) t
where t2.b < 40 and t2.a=t3.a and t3.c=t.c;
id select_type table type possible_keys key key_len ref rows filtered Extra
1 PRIMARY t2 ALL NULL NULL NULL NULL 90 60.00 Using where
1 PRIMARY t3 ref idx_a idx_a 5 test.t2.a 1 100.00 Using where
1 PRIMARY t3 ref idx_a idx_a 5 test.t2.a 0 100.00 Using where
1 PRIMARY <derived2> ref key0 key0 128 test.t3.c 10 100.00
2 DERIVED t4 ALL idx_c NULL NULL NULL 160 100.00 Using temporary
Warnings:
Expand Down Expand Up @@ -20725,7 +20725,7 @@ EXPLAIN
"used_key_parts": ["a"],
"ref": ["test.t2.a"],
"loops": 53.99999991,
"rows": 1,
"rows": 0,
"cost": "COST_REPLACED",
"filtered": 100,
"attached_condition": "t3.c is not null"
Expand All @@ -20740,7 +20740,7 @@ EXPLAIN
"key_length": "128",
"used_key_parts": ["c"],
"ref": ["test.t3.c"],
"loops": 80.99999987,
"loops": 52.89695991,
"rows": 10,
"cost": "COST_REPLACED",
"filtered": 100,
Expand Down
4 changes: 2 additions & 2 deletions mysql-test/main/join_cache.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6365,7 +6365,7 @@ LEFT JOIN t3 ON t2.d = t3.c
LEFT JOIN t4 ON t3.c=1;
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE t1 index NULL PRIMARY 4 NULL 10000 Using index
1 SIMPLE t2 ref b b 4 test.t1.b 1
1 SIMPLE t2 ref b b 4 test.t1.b 0
1 SIMPLE t3 eq_ref PRIMARY PRIMARY 4 test.t2.d 1 Using where; Using index
1 SIMPLE t4 index NULL PRIMARY 4 NULL 3000 Using where; Using index
SELECT COUNT(*)
Expand All @@ -6383,7 +6383,7 @@ LEFT JOIN t3 ON t2.d = t3.c
LEFT JOIN t4 ON t3.c=1;
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE t1 index NULL PRIMARY 4 NULL 10000 Using index
1 SIMPLE t2 ref b b 4 test.t1.b 1
1 SIMPLE t2 ref b b 4 test.t1.b 0
1 SIMPLE t3 eq_ref PRIMARY PRIMARY 4 test.t2.d 1 Using where; Using index
1 SIMPLE t4 index NULL PRIMARY 4 NULL 3000 Using where; Using index; Using join buffer (flat, BNL join)
SELECT COUNT(*)
Expand Down
148 changes: 148 additions & 0 deletions mysql-test/main/match_probability.result
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
# Setup: Two tables with skewed foreign key distribution
CREATE TABLE t_ndv100 (a INT);
CREATE TABLE t_ndv3 (a INT, b VARCHAR(50), KEY idx_a (a));
# All values in t_ndv100.a are different (100 distinct values)
INSERT INTO t_ndv100 (a) SELECT seq FROM seq_1_to_100;
# There are only 3 distinct values in t_ndv3.a
INSERT INTO t_ndv3 (a, b) SELECT seq/100, 'def' FROM seq_1_to_300;
set optimizer_trace=1;
# Inefficient plan until EITS is collected (full scan of t_ndv3)
EXPLAIN SELECT t_ndv100.a, t_ndv3.b
FROM t_ndv100
STRAIGHT_JOIN t_ndv3 ON t_ndv100.a = t_ndv3.a;
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE t_ndv100 ALL NULL NULL NULL NULL 100
1 SIMPLE t_ndv3 ALL idx_a NULL NULL NULL 300 Using where; Using join buffer (flat, BNL join)
# Collect statistics only for the driving table (t_ndv100)
ANALYZE TABLE t_ndv100 PERSISTENT FOR ALL;
Table Op Msg_type Msg_text
test.t_ndv100 analyze status Engine-independent statistics collected
test.t_ndv100 analyze status OK
# It is not enough until there are statistics for the inner table (t_ndv3)
EXPLAIN SELECT t_ndv100.a, t_ndv3.b
FROM t_ndv100
STRAIGHT_JOIN t_ndv3 ON t_ndv100.a = t_ndv3.a;
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE t_ndv100 ALL NULL NULL NULL NULL 100
1 SIMPLE t_ndv3 ALL idx_a NULL NULL NULL 300 Using where; Using join buffer (flat, BNL join)
ANALYZE TABLE t_ndv3 PERSISTENT FOR ALL;
Table Op Msg_type Msg_text
test.t_ndv3 analyze status Engine-independent statistics collected
test.t_ndv3 analyze status Table is already up to date
# After EITS is collected for both tables, the plan is efficient
# (index scan of t_ndv3). "rows" in the output are close to actual "r_rows"
ANALYZE SELECT t_ndv100.a, t_ndv3.b
FROM t_ndv100
STRAIGHT_JOIN t_ndv3 ON t_ndv100.a = t_ndv3.a;
id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra
1 SIMPLE t_ndv100 ALL NULL NULL NULL NULL 100 100.00 100.00 100.00 Using where
1 SIMPLE t_ndv3 ref idx_a idx_a 5 test.t_ndv100.a 3 2.51 100.00 100.00
# Optimization must be reflected in the trace
SELECT json_detailed(json_extract(trace, '$**.match_probability',
'$**.rows_before_adjustment'))
FROM information_schema.optimizer_trace;
json_detailed(json_extract(trace, '$**.match_probability',
'$**.rows_before_adjustment'))
[
0.04,
75
]
# Match probability is not applicable because the left part of the
# join condition is not a field but an expression (t_ndv100.a + 10). See the
# difference between "rows" and "r_rows"
ANALYZE SELECT t_ndv100.a, t_ndv3.b
FROM t_ndv100
STRAIGHT_JOIN t_ndv3 ON t_ndv100.a + 10 = t_ndv3.a;
id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra
1 SIMPLE t_ndv100 ALL NULL NULL NULL NULL 100 100.00 100.00 100.00
1 SIMPLE t_ndv3 ALL idx_a NULL NULL NULL 300 300.00 25.00 0.00 Using where; Using join buffer (flat, BNL join)
# Must be no records in the trace
SELECT json_detailed(json_extract(trace, '$**.match_probability',
'$**.rows_before_adjustment'))
FROM information_schema.optimizer_trace;
json_detailed(json_extract(trace, '$**.match_probability',
'$**.rows_before_adjustment'))
NULL
# If all values are NULL, match probability is not applicable
CREATE TABLE t_nulls (a INT, b VARCHAR(50), KEY idx_a (a));
INSERT INTO t_nulls (a, b) SELECT NULL, 'def' FROM seq_1_to_300;
ANALYZE TABLE t_nulls PERSISTENT FOR ALL;
Table Op Msg_type Msg_text
test.t_nulls analyze status Engine-independent statistics collected
test.t_nulls analyze status Table is already up to date
EXPLAIN SELECT t_ndv100.a, t_nulls.b
FROM t_ndv100
STRAIGHT_JOIN t_nulls ON t_ndv100.a = t_nulls.a;
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE t_ndv100 ALL NULL NULL NULL NULL 100 Using where
1 SIMPLE t_nulls ref idx_a idx_a 5 test.t_ndv100.a 1
SELECT json_detailed(json_extract(trace, '$**.match_probability',
'$**.rows_before_adjustment'))
FROM information_schema.optimizer_trace;
json_detailed(json_extract(trace, '$**.match_probability',
'$**.rows_before_adjustment'))
NULL
#------------------------------
# Test multi-part index
#------------------------------
CREATE TABLE t_ndv30 (a INT, b INT, c VARCHAR(50), KEY idx_ab (a, b));
INSERT INTO t_ndv30 (a, b, c) SELECT seq/10, seq/10, 'def' FROM seq_1_to_300;
ANALYZE TABLE t_ndv30 PERSISTENT FOR ALL;
Table Op Msg_type Msg_text
test.t_ndv30 analyze status Engine-independent statistics collected
test.t_ndv30 analyze status Table is already up to date
# If more than one key part is used, match probability is not applicable.
# See the mismatch between "rows" and "r_rows" in ANALYZE output
ANALYZE SELECT t_ndv100.a, t_ndv30.b
FROM t_ndv100
STRAIGHT_JOIN t_ndv30 ON t_ndv100.a = t_ndv30.a
AND t_ndv100.a = t_ndv30.b;
id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra
1 SIMPLE t_ndv100 ALL NULL NULL NULL NULL 100 100.00 100.00 100.00 Using where
1 SIMPLE t_ndv30 ref idx_ab idx_ab 10 test.t_ndv100.a,test.t_ndv100.a 9 2.96 100.00 100.00 Using index
# Must be no records in the trace
SELECT json_detailed(json_extract(trace, '$**.match_probability',
'$**.rows_before_adjustment'))
FROM information_schema.optimizer_trace;
json_detailed(json_extract(trace, '$**.match_probability',
'$**.rows_before_adjustment'))
NULL
# Only first key part of `idx_ab` is used, match probability is applicable.
# "rows" is now much closer to actual "r_rows".
ANALYZE SELECT t_ndv100.a, t_ndv30.b
FROM t_ndv100
STRAIGHT_JOIN t_ndv30 ON t_ndv100.a = t_ndv30.a;
id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra
1 SIMPLE t_ndv100 ALL NULL NULL NULL NULL 100 100.00 100.00 100.00 Using where
1 SIMPLE t_ndv30 ref idx_ab idx_ab 5 test.t_ndv100.a 3 2.96 100.00 100.00 Using index
# Must be reflected in the trace
SELECT json_detailed(json_extract(trace, '$**.match_probability',
'$**.rows_before_adjustment'))
FROM information_schema.optimizer_trace;
json_detailed(json_extract(trace, '$**.match_probability',
'$**.rows_before_adjustment'))
[
0.31000062,
9.6774
]
# Three tables with equality propagation.
# min_driving_ndv = min(NDV(t_ndv30.a), NDV(t_ndv100.a)) = min(30, 100) = 30
ANALYZE SELECT t_ndv100.a, t_ndv3.b
FROM t_ndv30
STRAIGHT_JOIN t_ndv100 ON t_ndv30.a = t_ndv100.a
STRAIGHT_JOIN t_ndv3 ON t_ndv100.a = t_ndv3.a;
id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra
1 SIMPLE t_ndv30 index idx_ab idx_ab 10 NULL 300 300.00 100.00 100.00 Using where; Using index
1 SIMPLE t_ndv100 ALL NULL NULL NULL NULL 100 100.00 100.00 0.99 Using where; Using join buffer (flat, BNL join)
1 SIMPLE t_ndv3 ref idx_a idx_a 5 test.t_ndv30.a 9 8.48 100.00 100.00
# Must be reflected in the trace (match_prob ~= 0.1)
SELECT json_detailed(json_extract(trace, '$**.match_probability',
'$**.rows_before_adjustment'))
FROM information_schema.optimizer_trace;
json_detailed(json_extract(trace, '$**.match_probability',
'$**.rows_before_adjustment'))
[
0.129032,
75
]
DROP TABLE t_ndv100, t_ndv3, t_ndv30, t_nulls;
Loading
Loading