mycat 实践扩容

Posted 2021-01-21 yhq1314

实践扩容， travelrecord表定义为10个分片，尝试将10个分片中的2个分片转移到第二台mysql上

1--定义10个分片
<table name="travelrecord_t" dataNode="dn1-10" rule="auto-sharding-long_t" />
<dataNode name="dn4" dataHost="localhost0" database="db4" />
并建立db1-10 database
CREATE database db1-10;

<tableRule name="auto-sharding-long_t">
<rule>
<columns>id</columns>
<algorithm>rang-long_t</algorithm>
</rule>
</tableRule>

<function name="rang-long_t"
class="io.mycat.route.function.AutoPartitionByLong">
<property name="mapFile">autopartition-long_t.txt</property>
</function>

vim autopartition-long_t.txt

Caused by: io.mycat.config.util.ConfigException: Illegal table conf : table [ TRAVELRECORD ] rule function
[ rang-long ] partition size : 10 > table datanode size : 3, please make sure table datanode size = function partition size

warn: bad line int autopartition-long.txt :

INFO | jvm 1 | 2016/12/14 02:14:33 | WrapperSimpleApp: Encountered an error running main: java.lang.ExceptionInInitializerError
INFO | jvm 1 | 2016/12/14 02:14:33 | java.lang.ExceptionInInitializerError
INFO | jvm 1 | 2016/12/14 02:14:33 | at io.mycat.MycatStartup.main(MycatStartup.java:53)
INFO | jvm 1 | 2016/12/14 02:14:33 | at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
INFO | jvm 1 | 2016/12/14 02:14:33 | at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
INFO | jvm 1 | 2016/12/14 02:14:33 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
INFO | jvm 1 | 2016/12/14 02:14:33 | at java.lang.reflect.Method.invoke(Method.java:498)
INFO | jvm 1 | 2016/12/14 02:14:33 | at org.tanukisoftware.wrapper.WrapperSimpleApp.run(WrapperSimpleApp.java:240)
INFO | jvm 1 | 2016/12/14 02:14:33 | at java.lang.Thread.run(Thread.java:745)
INFO | jvm 1 | 2016/12/14 02:14:33 | Caused by: io.mycat.config.util.ConfigException: java.lang.NullPointerException
INFO | jvm 1 | 2016/12/14 02:14:33 | at io.mycat.config.loader.xml.XMLSchemaLoader.load(XMLSchemaLoader.java:126)
INFO | jvm 1 | 2016/12/14 02:14:33 | at io.mycat.config.loader.xml.XMLSchemaLoader.<init>(XMLSchemaLoader.java:83)
INFO | jvm 1 | 2016/12/14 02:14:33 | at io.mycat.config.loader.xml.XMLSchemaLoader.<init>(XMLSchemaLoader.java:87)
INFO | jvm 1 | 2016/12/14 02:14:33 | at io.mycat.config.ConfigInitializer.<init>(ConfigInitializer.java:74)
INFO | jvm 1 | 2016/12/14 02:14:33 | at io.mycat.config.MycatConfig.<init>(MycatConfig.java:72)
INFO | jvm 1 | 2016/12/14 02:14:33 | at io.mycat.MycatServer.<init>(MycatServer.java:140)
INFO | jvm 1 | 2016/12/14 02:14:33 | at io.mycat.MycatServer.<clinit>(MycatServer.java:92)
INFO | jvm 1 | 2016/12/14 02:14:33 | ... 7 more
INFO | jvm 1 | 2016/12/14 02:14:33 | Caused by: java.lang.NullPointerException
INFO | jvm 1 | 2016/12/14 02:14:33 | at io.mycat.config.loader.xml.XMLSchemaLoader.getDbType(XMLSchemaLoader.java:458)
INFO | jvm 1 | 2016/12/14 02:14:33 | at io.mycat.config.loader.xml.XMLSchemaLoader.loadTables(XMLSchemaLoader.java:387)
INFO | jvm 1 | 2016/12/14 02:14:33 | at io.mycat.config.loader.xml.XMLSchemaLoader.loadSchemas(XMLSchemaLoader.java:173)
INFO | jvm 1 | 2016/12/14 02:14:33 | at io.mycat.config.loader.xml.XMLSchemaLoader.load(XMLSchemaLoader.java:122)
INFO | jvm 1 | 2016/12/14 02:14:33 | ... 13 more
STATUS | wrapper | 2016/12/14 02:14:35 | <-- Wrapper Stopped
dataNode="dn1-10"的问题
2--测试10个分片的数据
CREATE TABLE `travelrecord_t` (
`id` bigint(20) NOT NULL,
`user_id` varchar(100) DEFAULT NULL,
`traveldate` date DEFAULT NULL,
`fee` decimal(10,0) DEFAULT NULL,
`days` int(11) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
mysql> explain select * from travelrecord_t;
+-----------+----------------------------------------+
| DATA_NODE | SQL |
+-----------+----------------------------------------+
| dn1 | SELECT * FROM travelrecord_t LIMIT 100 |
| dn10 | SELECT * FROM travelrecord_t LIMIT 100 |
| dn2 | SELECT * FROM travelrecord_t LIMIT 100 |
| dn3 | SELECT * FROM travelrecord_t LIMIT 100 |
| dn4 | SELECT * FROM travelrecord_t LIMIT 100 |
| dn5 | SELECT * FROM travelrecord_t LIMIT 100 |
| dn6 | SELECT * FROM travelrecord_t LIMIT 100 |
| dn7 | SELECT * FROM travelrecord_t LIMIT 100 |
| dn8 | SELECT * FROM travelrecord_t LIMIT 100 |
| dn9 | SELECT * FROM travelrecord_t LIMIT 100 |

explain insert into travelrecord_t (id,user_id,traveldate,fee,days) values(1,‘wang‘,now(),510.5,3);
explain insert into travelrecord_t (id,user_id,traveldate,fee,days) values(6000001,‘wang‘,now(),510.5,3);
explain insert into travelrecord_t (id,user_id,traveldate,fee,days) values(11000001,‘wang‘,now(),510.5,3);
explain insert into travelrecord_t (id,user_id,traveldate,fee,days) values(16000001,‘yang‘,now(),510.5,3);
explain insert into travelrecord_t (id,user_id,traveldate,fee,days) values(21000001,‘yang‘,now(),510.5,3);
explain insert into travelrecord_t (id,user_id,traveldate,fee,days) values(26000001,‘yang‘,now(),510.5,3);
explain insert into travelrecord_t (id,user_id,traveldate,fee,days) values(31000001,‘zhang‘,now(),510.5,3);
explain insert into travelrecord_t (id,user_id,traveldate,fee,days) values(36000001,‘yang‘,now(),510.5,3);

explain insert into travelrecord_t (id,user_id,traveldate,fee,days) values(41000001,‘ce‘,now(),510.5,3);
explain insert into travelrecord_t (id,user_id,traveldate,fee,days) values(46000001,‘ce1‘,now(),510.5,3);
explain insert into travelrecord_t (id,user_id,traveldate,fee,days) values(51000001,‘ce2‘,now(),510.5,3);
explain insert into travelrecord_t (id,user_id,traveldate,fee,days) values(56000001,‘vre‘,now(),510.5,3);
mysql> insert into travelrecord_t (id,user_id,traveldate,fee,days) values(51000001,‘ce2‘,now(),510.5,3);
ERROR 1064 (HY000): can‘t find any valid datanode :TRAVELRECORD_T -> ID -> 51000001
mysql> explain select * from travelrecord_t where id=36000002;
+-----------+------------------------------------------------------------+
| DATA_NODE | SQL |
+-----------+------------------------------------------------------------+
| dn8 | SELECT * FROM travelrecord_t WHERE id = 36000002 LIMIT 100 |
+-----------+------------------------------------------------------------+
1 row in set (0.05 sec)
数据迁移阶段
mysqldump,xtrabackup
[[email protected] ~]$ scp db* [email protected]:/home/mysql/.
[[email protected] ~]$ mysql -usystem -pmysql -h127.0.0.1 -P3307 db9 <db9.sql
Warning: Using a password on the command line interface can be insecure.
[[email protected] ~]$ mysql -usystem -pmysql -h127.0.0.1 -P3307 db10 <db10.sql
Warning: Using a password on the command line interface can be insecure.
修改mycat的配置
<dataNode name="dn9" dataHost="135_host" database="db9" />
<dataNode name="dn10" dataHost="135_host" database="db10" />

<dataHost name="135_host" maxCon="1000" minCon="10" balance="0"
writeType="0" dbType="mysql" dbDriver="native" switchType="1" slaveThreshold="100">
<heartbeat>select user() </heartbeat>
<writeHost host="hostM2" url="10.0.1.135:3307" user="system"
password="mysql">
</writeHost>
</dataHost>

mysql> reload @@config_all;
Query OK, 1 row affected (0.78 sec)
Reload config success
mysql> show @@heartbeat;
+--------+-------+---------------+------+---------+-------+--------+---------+--------------+---------------------+-------+
| NAME | TYPE | HOST | PORT | RS_CODE | RETRY | STATUS | TIMEOUT | EXECUTE_TIME | LAST_ACTIVE_TIME | STOP |
+--------+-------+---------------+------+---------+-------+--------+---------+--------------+---------------------+-------+
| hostM1 | mysql | 10.0.1.134 | 3306 | 1 | 0 | idle | 0 | 5,5,5 | 2016-12-15 01:01:52 | false |
| hostS1 | mysql | 192.168.3.110 | 3306 | 1 | 0 | idle | 0 | 4,4,4 | 2016-12-15 01:01:52 | false |
| hostM2 | mysql | 10.0.1.135 | 3307 | 1 | 0 | idle | 0 | 3,3,4 | 2016-12-15 01:01:52 | false |
+--------+-------+---------------+------+---------+-------+--------+---------+--------------+---------------------+-------+
3 rows in set (0.03 sec)
mysql> show @@datasource;
+----------+--------+-------+---------------+------+------+--------+------+------+---------+-----------+------------+
| DATANODE | NAME | TYPE | HOST | PORT | W/R | ACTIVE | IDLE | SIZE | EXECUTE | READ_LOAD | WRITE_LOAD |
+----------+--------+-------+---------------+------+------+--------+------+------+---------+-----------+------------+
| dn1 | hostM1 | mysql | 10.0.1.134 | 3306 | W | 0 | 6 | 1000 | 97 | 0 | 0 |
| dn1 | hostS1 | mysql | 192.168.3.110 | 3306 | R | 0 | 6 | 1000 | 87 | 0 | 0 |
| dn10 | hostM2 | mysql | 10.0.1.135 | 3307 | W | 0 | 11 | 1000 | 93 | 0 | 0 |
| dn3 | hostM1 | mysql | 10.0.1.134 | 3306 | W | 0 | 6 | 1000 | 97 | 0 | 0 |
| dn3 | hostS1 | mysql | 192.168.3.110 | 3306 | R | 0 | 6 | 1000 | 87 | 0 | 0 |
| dn2 | hostM1 | mysql | 10.0.1.134 | 3306 | W | 0 | 6 | 1000 | 97 | 0 | 0 |
| dn2 | hostS1 | mysql | 192.168.3.110 | 3306 | R | 0 | 6 | 1000 | 87 | 0 | 0 |
| dn5 | hostM1 | mysql | 10.0.1.134 | 3306 | W | 0 | 6 | 1000 | 97 | 0 | 0 |
| dn5 | hostS1 | mysql | 192.168.3.110 | 3306 | R | 0 | 6 | 1000 | 87 | 0 | 0 |
| dn4 | hostM1 | mysql | 10.0.1.134 | 3306 | W | 0 | 6 | 1000 | 97 | 0 | 0 |
| dn4 | hostS1 | mysql | 192.168.3.110 | 3306 | R | 0 | 6 | 1000 | 87 | 0 | 0 |
| dn7 | hostM1 | mysql | 10.0.1.134 | 3306 | W | 0 | 6 | 1000 | 97 | 0 | 0 |
| dn7 | hostS1 | mysql | 192.168.3.110 | 3306 | R | 0 | 6 | 1000 | 87 | 0 | 0 |
| dn6 | hostM1 | mysql | 10.0.1.134 | 3306 | W | 0 | 6 | 1000 | 97 | 0 | 0 |
| dn6 | hostS1 | mysql | 192.168.3.110 | 3306 | R | 0 | 6 | 1000 | 87 | 0 | 0 |
| dn9 | hostM2 | mysql | 10.0.1.135 | 3307 | W | 0 | 11 | 1000 | 93 | 0 | 0 |
| dn8 | hostM1 | mysql | 10.0.1.134 | 3306 | W | 0 | 6 | 1000 | 97 | 0 | 0 |
| dn8 | hostS1 | mysql | 192.168.3.110 | 3306 | R | 0 | 6 | 1000 | 87 | 0 | 0 |
+----------+--------+-------+---------------+------+------+--------+------+------+---------+-----------+------------+
18 rows in set (0.01 sec)
mysql> insert into travelrecord_t (id,user_id,traveldate,fee,days) values(46000003,‘ce1‘,now(),510.5,3);
Query OK, 1 row affected, 2 warnings (0.06 sec)

mysql> insert into travelrecord_t (id,user_id,traveldate,fee,days) values(46000004,‘ce1‘,now(),510.5,3);
Query OK, 1 row affected, 2 warnings (0.01 sec)

2016-12-15 01:05:16.283 DEBUG [$_NIOREACTOR-0-RW] (io.mycat.server.NonBlockingSession.execute(NonBlockingSession.java:119)) - ServerConnection [id=3, schema=TESTDB, host=10.0.1.134, user=system,txIsolation=3, autocommit=true, schema=TESTDB]insert into travelrecord_t (id,user_id,traveldate,fee,days) values(46000003,‘ce1‘,now(),510.5,3), route={
1 -> dn10{insert into travelrecord_t (id,user_id,traveldate,fee,days) values(46000003,‘ce1‘,now(),510.5,3)}
} rrs
2016-12-15 01:05:16.284 DEBUG [$_NIOREACTOR-0-RW] (io.mycat.backend.mysql.nio.handler.SingleNodeHandler.execute(SingleNodeHandler.java:173)) - rrs.getRunOnSlave() null
2016-12-15 01:05:16.284 DEBUG [$_NIOREACTOR-0-RW] (io.mycat.backend.mysql.nio.handler.SingleNodeHandler.execute(SingleNodeHandler.java:175)) - node.getRunOnSlave() null
2016-12-15 01:05:16.284 DEBUG [$_NIOREACTOR-0-RW] (io.mycat.backend.mysql.nio.handler.SingleNodeHandler.execute(SingleNodeHandler.java:184)) - node.getRunOnSlave() null
2016-12-15 01:05:16.284 DEBUG [$_NIOREACTOR-0-RW] (io.mycat.backend.mysql.nio.handler.SingleNodeHandler.execute(SingleNodeHandler.java:186)) - node.getRunOnSlave() null
2016-12-15 01:05:16.284 DEBUG [$_NIOREACTOR-0-RW] (io.mycat.backend.datasource.PhysicalDBNode.getConnection(PhysicalDBNode.java:96)) - rrs.getRunOnSlave() null
2016-12-15 01:05:16.285 DEBUG [$_NIOREACTOR-0-RW] (io.mycat.backend.datasource.PhysicalDBNode.getConnection(PhysicalDBNode.java:127)) - rrs.getRunOnSlave() null
2016-12-15 01:05:16.335 DEBUG [$_NIOREACTOR-0-RW] (io.mycat.server.NonBlockingSession.releaseConnection(NonBlockingSession.java:354)) - release connection MySQLConnection [id=129, lastTime=1481792716283, user=system, schema=db10, old shema=db10, borrowed=true, fromSlaveDB=false, threadId=17, charset=utf8, txIsolation=3, autocommit=true, attachment=dn10{insert into travelrecord_t (id,user_id,traveldate,fee,days) values(46000003,‘ce1‘,now(),510.5,3)}, respHandler=SingleNodeHandler [node=dn10{insert into travelrecord_t (id,user_id,traveldate,fee,days) values(46000003,‘ce1‘,now(),510.5,3)}, packetId=1], host=10.0.1.135, port=3307, statusSync=null, writeQueue=0, modifiedSQLExecuted=true]
2016-12-15 01:05:16.335 DEBUG [$_NIOREACTOR-0-RW] (io.mycat.backend.datasource.PhysicalDatasource.releaseChannel(PhysicalDatasource.java:442)) - release channel MySQLConnection [id=129, lastTime=1481792716283, user=system, schema=db10, old shema=db10, borrowed=true, fromSlaveDB=false, threadId=17, charset=utf8, txIsolation=3, autocommit=true, attachment=null, respHandler=null, host=10.0.1.135, port=3307, statusSync=null, writeQueue=0, modifiedSQLExecuted=false]
mysql> explain select * from travelrecord_t where id=46000003;
+-----------+------------------------------------------------------------+
| DATA_NODE | SQL |
+-----------+------------------------------------------------------------+
| dn10 | SELECT * FROM travelrecord_t WHERE id = 46000003 LIMIT 100 |
+-----------+------------------------------------------------------------+
1 row in set (0.01 sec)

2016-12-15 01:06:37.130 DEBUG [$_NIOREACTOR-0-RW] (io.mycat.cache.impl.EnchachePool.get(EnchachePool.java:77)) - SQLRouteCache miss cache ,key:TESTDBselect * from travelrecord_t where id=46000003
2016-12-15 01:06:37.138 DEBUG [$_NIOREACTOR-0-RW] (io.mycat.cache.impl.EnchachePool.putIfAbsent(EnchachePool.java:60)) - SQLRouteCache add cache ,key:TESTDBselect * from travelrecord_t where id=46000003 value:select * from travelrecord_t where id=46000003, route={
1 -> dn10{SELECT *
FROM travelrecord_t
WHERE id = 46000003
LIMIT 100}
}
2016-12-15 01:06:42.006 DEBUG [Timer0] (io.mycat.sqlengine.SQLJob.connectionAcquired(SQLJob.java:88)) - con query sql:show slave status to con:MySQLConnection [id=133, lastTime=1481792802005, user=system, schema=db1, old shema=db1, borrowed=true, fromSlaveDB=false, threadId=1770714, charset=utf8, txIsolation=3, autocommit=true, attachment=null, respHandler=null, host=10.0.1.134, port=3306, statusSync=null, writeQueue=0, modifiedSQLExecuted=false]
2016-12-15 01:06:42.006 DEBUG [Timer0] (io.mycat.sqlengine.SQLJob.connectionAcquired(SQLJob.java:88)) - con query sql:show slave status to con:MySQLConnection [id=137, lastTime=1481792802006, user=system, schema=db1, old shema=db1, borrowed=true, fromSlaveDB=true, threadId=731, charset=utf8, txIsolation=3, autocommit=true, attachment=null, respHandler=null, host=192.168.3.110, port=3306, statusSync=null, writeQueue=0, modifiedSQLExecuted=false]
2016-12-15 01:06:42.007 DEBUG [Timer0] (io.mycat.sqlengine.SQLJob.connectionAcquired(SQLJob.java:88)) - con query sql:select user() to con:MySQLConnection [id=127, lastTime=1481792802006, user=system, schema=db10, old shema=db10, borrowed=true, fromSlaveDB=false, threadId=15, charset=utf8, txIsolation=3, autocommit=true, attachment=null, respHandler=null, host=10.0.1.135, port=3307, statusSync=null, writeQueue=0, modifiedSQLExecuted=false]
登录查看
[[email protected] ~]$ mysql -usystem -pmysql -h127.0.0.1 -P3307
mysql> select * from db10.travelrecord_t;
+----------+---------+------------+------+------+
| id | user_id | traveldate | fee | days |
+----------+---------+------------+------+------+
| 46000001 | ce1 | 2016-12-14 | 511 | 3 |
| 46000002 | ce1 | 2016-12-14 | 511 | 3 |
| 46000003 | ce1 | 2016-12-15 | 511 | 3 |
| 46000004 | ce1 | 2016-12-15 | 511 | 3 |
+----------+---------+------------+------+------+
4 rows in set (0.00 sec)