MySQL 字符集/collate 不同导致无法使用索引过滤

Posted 2022-02-13 渔夫数据库笔记

对于因为字符集不同导致两表关联时无法使用索引的问题我们可能经常遇到，但是你知道吗，就算字符集相同，你的编码规则(collate)不同也可能导致两表关联时无法使用索引的

一：实验1 字符集不同可能会导致两表关联时无法使用索引
结论：两个表字符集不同，关联查询时就一定会导致无法使用字符类型字段上的索引无法使用吗？ 不一定，看下面的例子
两个测试表一个字符集为utf8，一个为utf8mb4，如果使用utf8字符集的表做驱动表，通过他的字符列去关联utf8mb4字符集的表(utf8mb4 表上的字符类型关联字段上有索引)，是可以正常使用utf8mb4上字符类型列上的索引进行数据过滤的。反之则不行(其实这里应该说关联字段上的字符集不同更准确)

先建两个测试表并插入部分测试数据
drop table if exists test_character_1;
create table test_character_1(
id int not null auto_increment primary key,
name_1 varchar(20)) default character set utf8mb4 collate utf8mb4_general_ci;

insert into test_character_1(name_1) values('Add'),('add'),('Bdd'),('bdd'),('Cdd'),('cdd'),('Edd'),('edd'),('Fdd'),('fdd'),('Gdd'),('gdd'),('Hdd'),('hdd'),('Idd'),('idd'),('Jdd'),('jdd'),('Kdd'),('kdd'),('Ldd'),('ldd');
             

drop table if exists test_character_2;
create table test_character_2(
id int not null auto_increment primary key,
name_2 varchar(20)) default character set utf8 collate utf8_general_ci;
insert into test_character_2(name_2) values('Add'),('add'),('Bdd'),('bdd'),('Cdd'),('cdd'),('Edd'),('edd'),('Fdd'),('fdd'),('Gdd'),('gdd'),('Hdd'),('hdd'),('Idd'),('idd'),('Jdd'),('jdd'),('Kdd'),('kdd'),('Ldd'),('ldd');


alter table test_character_1 add index idx_name_1(name_1);
alter table test_character_2 add index idx_name_2(name_2);


explain select a.*,b.* from test_character_1 a,test_character_2 b where a.name_1=b.name_2;
+----+-------------+-------+------------+-------+---------------+------------+---------+------+------+----------+--------------------------+
| id | select_type | table | partitions | type  | possible_keys | key        | key_len | ref  | rows | filtered | Extra                    |
+----+-------------+-------+------------+-------+---------------+------------+---------+------+------+----------+--------------------------+
|  1 | SIMPLE      | b     | NULL       | index | NULL          | idx_name_2 | 63      | NULL |   22 |   100.00 | Using index              |
|  1 | SIMPLE      | a     | NULL       | ref   | idx_name_1    | idx_name_1 | 83      | func |    2 |   100.00 | Using where; Using index |
+----+-------------+-------+------------+-------+---------------+------------+---------+------+------+----------+--------------------------+
2 rows in set, 1 warning (0.00 sec)

[root@127.0.0.1][test_shao]> show warnings;
+-------+------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Level | Code | Message                                                                                                                                                                                                                                                                                                                      |
+-------+------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Note  | 1003 | /* select#1 */ select `test_shao`.`a`.`id` AS `id`,`test_shao`.`a`.`name_1` AS `name_1`,`test_shao`.`b`.`id` AS `id`,`test_shao`.`b`.`name_2` AS `name_2` from `test_shao`.`test_character_1` `a` join `test_shao`.`test_character_2` `b` where (`test_shao`.`a`.`name_1` = convert(`test_shao`.`b`.`name_2` using utf8mb4)) |
+-------+------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (0.00 sec)

[root@127.0.0.1][test_shao]> explain select a.*,b.* from test_character_1 a straight_join test_character_2 b on a.name_1=b.name_2;
+----+-------------+-------+------------+-------+---------------+------------+---------+------+------+----------+-----------------------------------------------------------------+
| id | select_type | table | partitions | type  | possible_keys | key        | key_len | ref  | rows | filtered | Extra                                                           |
+----+-------------+-------+------------+-------+---------------+------------+---------+------+------+----------+-----------------------------------------------------------------+
|  1 | SIMPLE      | a     | NULL       | index | idx_name_1    | idx_name_1 | 83      | NULL |   22 |   100.00 | Using index                                                     |
|  1 | SIMPLE      | b     | NULL       | index | NULL          | idx_name_2 | 63      | NULL |   22 |   100.00 | Using where; Using index; Using join buffer (Block Nested Loop) |
+----+-------------+-------+------------+-------+---------------+------------+---------+------+------+----------+-----------------------------------------------------------------+
2 rows in set, 1 warning (0.00 sec)

[root@127.0.0.1][test_shao]> show warnings;
+-------+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Level | Code | Message                                                                                                                                                                                                                                                                                                                               |
+-------+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Note  | 1003 | /* select#1 */ select `test_shao`.`a`.`id` AS `id`,`test_shao`.`a`.`name_1` AS `name_1`,`test_shao`.`b`.`id` AS `id`,`test_shao`.`b`.`name_2` AS `name_2` from `test_shao`.`test_character_1` `a` straight_join `test_shao`.`test_character_2` `b` where (`test_shao`.`a`.`name_1` = convert(`test_shao`.`b`.`name_2` using utf8mb4)) |
+-------+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (0.00 sec)

二：实验2  字符集相同，collate不同的两个表通过字符类型列进行关系可能导致无法使用索引
结论：字符集相同时，两表关联，如果驱动表的关联字段(字符类型)使用的utf8_bin 比较规则，而被驱动表上的关联列使用 utf8_general_ci 比较规则，这时无法使用被驱动表上该列的索引进行数据过滤。相反的话则可以使用索引就行过滤，下面是测试场景    

drop table if exists test_collate_1;
create table test_collate_1(
id int not null auto_increment primary key,
name_1 varchar(20)) default character set utf8 collate utf8_bin;

insert into test_collate_1(name_1) values('Add'),('add'),('Bdd'),('bdd'),('Cdd'),('cdd'),('Edd'),('edd'),('Fdd'),('fdd'),('Gdd'),('gdd'),('Hdd'),('hdd'),('Idd'),('idd'),('Jdd'),('jdd'),('Kdd'),('kdd'),('Ldd'),('ldd');
             

drop table if exists test_collate_2;
create table test_collate_2(
id int not null auto_increment primary key,
name_2 varchar(20)) default character set utf8 collate utf8_general_ci;
insert into test_collate_2(name_2) values('Add'),('add'),('Bdd'),('bdd'),('Cdd'),('cdd'),('Edd'),('edd'),('Fdd'),('fdd'),('Gdd'),('gdd'),('Hdd'),('hdd'),('Idd'),('idd'),('Jdd'),('jdd'),('Kdd'),('kdd'),('Ldd'),('ldd');


alter table test_collate_1 add index idx_name_1(name_1);
alter table test_collate_2 add index idx_name_2(name_2);

explain select a.*,b.* from test_collate_1 a,test_collate_2 b where a.name_1=b.name_2;
+----+-------------+-------+------------+-------+---------------+------------+---------+--------------------+------+----------+--------------------------+
| id | select_type | table | partitions | type  | possible_keys | key        | key_len | ref                | rows | filtered | Extra                    |
+----+-------------+-------+------------+-------+---------------+------------+---------+--------------------+------+----------+--------------------------+
|  1 | SIMPLE      | b     | NULL       | index | idx_name_2    | idx_name_2 | 63      | NULL               |   22 |   100.00 | Using where; Using index |
|  1 | SIMPLE      | a     | NULL       | ref   | idx_name_1    | idx_name_1 | 63      | test_shao.b.name_2 |    1 |   100.00 | Using where; Using index |
+----+-------------+-------+------------+-------+---------------+------------+---------+--------------------+------+----------+--------------------------+
2 rows in set, 2 warnings (0.00 sec)

[root@127.0.0.1][test_shao]> show warnings;
+---------+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Level   | Code | Message                                                                                                                                                                                                                                                                                           |
+---------+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Warning | 1739 | Cannot use ref access on index 'idx_name_2' due to type or collation conversion on field 'name_2'                                                                                                                                                                                                 |
| Note    | 1003 | /* select#1 */ select `test_shao`.`a`.`id` AS `id`,`test_shao`.`a`.`name_1` AS `name_1`,`test_shao`.`b`.`id` AS `id`,`test_shao`.`b`.`name_2` AS `name_2` from `test_shao`.`test_collate_1` `a` join `test_shao`.`test_collate_2` `b` where (`test_shao`.`a`.`name_1` = `test_shao`.`b`.`name_2`) |
+---------+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
2 rows in set (0.00 sec)


explain select a.*,b.* from test_collate_1 a straight_join test_collate_2 b on  a.name_1=b.name_2;
+----+-------------+-------+------------+-------+---------------+------------+---------+------+------+----------+------------------------------------------------+
| id | select_type | table | partitions | type  | possible_keys | key        | key_len | ref  | rows | filtered | Extra                                          |
+----+-------------+-------+------------+-------+---------------+------------+---------+------+------+----------+------------------------------------------------+
|  1 | SIMPLE      | a     | NULL       | index | idx_name_1    | idx_name_1 | 63      | NULL |   22 |   100.00 | Using index                                    |
|  1 | SIMPLE      | b     | NULL       | ALL   | idx_name_2    | NULL       | NULL    | NULL |   22 |    10.00 | Range checked for each record (index map: 0x2) |
+----+-------------+-------+------------+-------+---------------+------------+---------+------+------+----------+------------------------------------------------+
2 rows in set, 2 warnings (0.00 sec)

[root@127.0.0.1][test_shao]> show warnings;
+---------+------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Level   | Code | Message                                                                                                                                                                                                                                                                                                    |
+---------+------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Warning | 1739 | Cannot use ref access on index 'idx_name_2' due to type or collation conversion on field 'name_2'                                                                                                                                                                                                          |
| Note    | 1003 | /* select#1 */ select `test_shao`.`a`.`id` AS `id`,`test_shao`.`a`.`name_1` AS `name_1`,`test_shao`.`b`.`id` AS `id`,`test_shao`.`b`.`name_2` AS `name_2` from `test_shao`.`test_collate_1` `a` straight_join `test_shao`.`test_collate_2` `b` where (`test_shao`.`a`.`name_1` = `test_shao`.`b`.`name_2`) |
+---------+------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+