Mybatis批量操作与JDBC批量操作效率测试
Posted
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了Mybatis批量操作与JDBC批量操作效率测试相关的知识，希望对你有一定的参考价值。
Mybatis-spring配置文件
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:tx="http://www.springframework.org/schema/tx" xmlns:aop="http://www.springframework.org/schema/aop"
xmlns:context="http://www.springframework.org/schema/context" xmlns="http://www.springframework.org/schema/beans"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/tx
http://www.springframework.org/schema/tx/spring-tx.xsd
http://www.springframework.org/schema/aop
http://www.springframework.org/schema/aop/spring-aop.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context.xsd">
<!-- datasource -->
<bean id="dataSource" class="com.alibaba.druid.pool.DruidDataSource"
init-method="init" destroy-method="close">
<property name="url" value="${admin.datasource.url}" />
<property name="username" value="${admin.datasource.username}" />
<property name="password" value="${datasource.password}" />
<property name="maxActive" value="${admin.datasource.maxActive}" />
<property name="maxWait" value="${admin.datasource.maxWait}" />
<property name="initialSize" value="1" />
<property name="minIdle" value="1" />
<property name="timeBetweenEvictionRunsMillis" value="3000" />
<property name="minEvictableIdleTimeMillis" value="300000" />
<property name="validationQuery" value="SELECT ‘x‘" />
<property name="testWhileIdle" value="true" />
<property name="testOnBorrow" value="false" />
<property name="testOnReturn" value="false" />
<!-- <property name="poolPreparedStatements" value="false" /> -->
<property name="maxPoolPreparedStatementPerConnectionSize"
value="500" />
</bean>
<!-- sqlSessionFactory,mybatis -->
<bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
<property name="dataSource" ref="dataSource" />
<!-- 生成的所有Mapper -->
<property name="configLocation" value="classpath:application-entity.xml" />
<!-- 自定义Mapper.xml -->
<property name="mapperLocations">
<list>
<value>classpath:batchtest/*.xml</value>
</list>
</property>
<!-- 扫描model包 -->
<property name="typeAliasesPackage"
value="com.momix.dbao.dao.entity,com.momix.heda.model.entity" />
<property name="plugins"><!--分页插件 -->
<array>
<bean class="com.github.pagehelper.PageHelper">
<property name="properties">
<value>
dialect=mysql
reasonable=true
</value>
</property>
</bean>
</array>
</property>
</bean>
<bean id="sqlSessionTemplate" class="org.mybatis.spring.SqlSessionTemplate">
<constructor-arg index="0" ref="sqlSessionFactory" />
</bean>
<!-- scann mapper -->
<bean class="tk.mybatis.spring.mapper.MapperScannerConfigurer">
<property name="basePackage" value="com.momix.dbao.dao.mapper,com.momix.heda.dao" />
<property name="properties">
<value>
mappers=com.momix.dbao.dao.core.MyBatisBaseMapper
</value>
</property>
</bean>
<!-- transaction -->
<bean id="transactionManager"
class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
<property name="dataSource" ref="dataSource" />
</bean>
<tx:advice id="txAdvice" transaction-manager="transactionManager">
<tx:attributes>
<tx:method name="get*" propagation="SUPPORTS" />
<tx:method name="find*" propagation="SUPPORTS" />
<tx:method name="create*" propagation="REQUIRED" read-only="false"
rollback-for="java.lang.Exception" />
<tx:method name="del*" propagation="REQUIRED" read-only="false"
rollback-for="java.lang.Exception" />
<tx:method name="update*" propagation="REQUIRED" read-only="false"
rollback-for="java.lang.Exception" />
<tx:method name="*" propagation="REQUIRED" read-only="false"
rollback-for="java.lang.Exception" />
</tx:attributes>
</tx:advice>
<aop:config proxy-target-class="true" expose-proxy="true">
<aop:pointcut id="dbao"
expression="execution(* com.momix.dbao.service..*(..))" />
<aop:advisor pointcut-ref="dbao" advice-ref="txAdvice" />
</aop:config>
<!-- scan service -->
<context:component-scan base-package="com.momix.dbao.service,com.momix.heda.service,com.momix.basedao,com.momix.dbao.dao.sqlxml">
<context:exclude-filter type="annotation"
expression="org.springframework.stereotype.Controller" />
</context:component-scan>
</beans>Mybatis3批量插入
配置文件Sql语句:
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.momix.dbao.dao.mapper.sqlxml.BatchTestMapper">
<!-- 批量新增Sql -->
<insert id="batchInsert" parameterType="java.util.List">
insert into batch_test (name,sex,birthday,mobile) values
<foreach collection="list" item="item" index="index" separator="," >
(#{item.name},#{item.sex},#{item.birthday},#{item.mobile})
</foreach>
</insert>
<!-- 批量修改Sql -->
<update id="batchUpdate" parameterType="java.util.List">
<foreach collection="list" item="item" index="index" open="" close="" separator=";">
update batch_test set
name = #{item.name},sex = #{item.sex},birthday = #{item.birthday},mobile = #{item.mobile}
where id = #{item.id}
</foreach>
</update>
<select id="getCount" resultType="Integer">
select count(*) from batch_test
</select>
<delete id="deleteData">
delete from batch_test
</delete>
</mapper>Mapper方法:
package com.momix.dbao.dao.mapper.sqlxml;
import java.util.List;
import com.momix.dbao.dao.core.MyBatisBaseMapper;
import com.momix.dbao.dao.entity.sqlxml.BatchTest;
public interface BatchTestMapper extends MyBatisBaseMapper<BatchTest> {
/**
* 批量插入方法
*
* @param params
* 插入集合
*/
void batchInsert(List<BatchTest> params);
/**
* 批量修改方法
*
* @param params
* 修改集合
*/
void batchUpdate(List<BatchTest> params);
/**
* 删除数据
*
* @return 删除行数
*/
int deleteData();
/**
* 获取数据库所有行数
*/
int getCount();
}测试代码:
采用TestNG进行测试
package com.momix.test.batch;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.momix.dbao.dao.dto.sqlxml.BatchTestDTO;
import com.momix.dbao.dao.entity.sqlxml.BatchTest;
import com.momix.dbao.dao.mapper.sqlxml.BatchTestMapper;
import com.momix.dbao.service.test.TestOneService;
@ContextConfiguration(locations = { "classpath:applicationContext.xml" })
public class BatchTestMain extends AbstractTestNGSpringContextTests {
@Autowired
private TestOneService testOneService;
@Autowired
private BatchTestMapper batchTestMapper;
private List<BatchTest> paramsMapper;
// 每次测试插入行数
int size = 0;
// 所有测试案例
List<Integer> sizes = new ArrayList<Integer>();
@BeforeTest
public void initData() {
sizes.add(100);
sizes.add(300);
sizes.add(500);
sizes.add(800);
sizes.add(1000);
sizes.add(1500);
sizes.add(2000);
sizes.add(3000);
sizes.add(5000);
sizes.add(8000);
sizes.add(10000);
sizes.add(30000);
sizes.add(50000);
sizes.add(100000);
}
public void printData(List<Map<String, Long>> infos) {
System.out
.println("***********************************************************************");
for (int i = 0; i < infos.size(); i++) {
Map<String, Long> info = infos.get(i);
System.out.println("测试插入行数 : " + info.get("a") + " 实际插入行数: "
+ info.get("b") + " 删除行数 : " + info.get("d") + " 耗时 : "
+ info.get("c"));
}
System.out
.println("***********************************************************************");
}
@Test(enabled = true)
public void testBatchInsertMapper() {
List<Map<String, Long>> infos = new ArrayList<Map<String, Long>>();
try {
long start = 0l;
long end = 0l;
int b = 0;
int d = 0;
Map<String, Long> info = null;
for (int i = 0; i < sizes.size(); i++) {
info = new HashMap<String, Long>();
// 当前插入行数
size = sizes.get(i);
// 当前测试插入数据
paramsMapper = this.getDataMapper();
System.out.println(paramsMapper.size());
// 开始时间
start = System.currentTimeMillis();
// 批量插入开始
testOneService.batchTestInsertMapper(paramsMapper);
// 结束时间
end = System.currentTimeMillis();
b = batchTestMapper.getCount();
d = batchTestMapper.deleteData();
info.put("a", (long) size); // 应当插入行数
info.put("b", (long) b);// 实际插入行数
info.put("c", (end - start)); // 耗时
info.put("d", (long) d); // 删除行数
infos.add(info);
}
} catch (Exception e) {
e.printStackTrace();
}
printData(infos);
}
private List<BatchTest> getDataMapper() {
List<BatchTest> tests = new ArrayList<BatchTest>();
BatchTest test = null;
for (int i = 0; i < size; i++) {
test = new BatchTest();
test.setName(String.valueOf(i));
test.setSex(0);
tests.add(test);
}
return tests;
}
}通过org.mybatis.spring.SqlSessionTemplate进行批量操作，分步提交
Sql配置文件
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.momix.dbao.dao.dto.sqlxml.BathcTest">
<!-- 批量新增Sql -->
<insert id="batchInsertBatchTest" useGeneratedKeys="true" keyProperty="id" parameterType="com.momix.dbao.dao.dto.sqlxml.BatchTestDTO">
insert into batch_test (name,sex,birthday,mobile)
values (#{name},#{sex},#{birthday},#{mobile})
</insert>
<!-- 批量修改Sql -->
<insert id="batchUpdateBatchTest" parameterType="com.momix.dbao.dao.dto.sqlxml.BatchTestDTO">
update batch_test set
name = #{name},sex = #{sex},birthday = #{birthday},mobile = #{mobile}
where id = #{id}
</insert>
</mapper>底层代码:
BaseDao代码
package com.momix.basedao;
import java.util.List;
import javax.annotation.Resource;
import org.apache.ibatis.session.SqlSession;
import org.mybatis.spring.SqlSessionTemplate;
public abstract class BaseDao<O> {
@Resource
private SqlSessionTemplate sqlSessionTemplate;
/**
* 获取当前xml的namespace
* @return String
*/
protected abstract String getNameSpace();
/**
* 批量修改 分步提交
*
* @param statementName
* ID
* @param data
* 数据集合
*/
protected void batchUpdate(String statementName, List<O> data) {
// 当前方法没有用Spring事务
SqlSession session = sqlSessionTemplate.getSqlSessionFactory()
.openSession();
try {
int size = data.size();
for (int i = 0; i < size; i++) {
O param = data.get(i);
session.update(getNameSpace() + "." + statementName, param);
if (size % 100 == 0) {
// 每200条提交
session.commit();
// 清楚缓存,防止溢出
session.clearCache();
}
}
} catch (Exception e) {
// 出现异常,回滚还没有提交的数据
session.rollback();
e.printStackTrace();
} finally {
session.close();
}
}
/**
* 批量插入 分步提交
*
* @param statementName
* ID
* @param data
* 数据集合
*/
protected void batchInsert(String statementName, List<O> data) {
// 当前方法没有用Spring事务
SqlSession session = sqlSessionTemplate.getSqlSessionFactory()
.openSession();
try {
int size = data.size();
for (int i = 0; i < size; i++) {
O param = data.get(i);
session.update(getNameSpace() + "." + statementName, param);
if (size % 100 == 0) {
// 每200条提交
session.commit();
// 清楚缓存,防止溢出
session.clearCache();
}
}
} catch (Exception e) {
// 出现异常,回滚还没有提交的数据
session.rollback();
e.printStackTrace();
} finally {
session.close();
}
}
}Dao代码:
package com.momix.dbao.dao.sqlxml;
import java.util.List;
import org.springframework.stereotype.Repository;
import com.momix.basedao.BaseDao;
import com.momix.dbao.dao.dto.sqlxml.BatchTestDTO;
@Repository
public class BatchTestDao extends BaseDao<BatchTestDTO> {
@Override
protected String getNameSpace() {
return "com.momix.dbao.dao.dto.sqlxml.BathcTest";
}
public void batchInsert(List<BatchTestDTO> params) {
super.batchInsert("batchInsertBatchTest", params);
}
public void batchUpdate(List<BatchTestDTO> params) {
super.batchUpdate("batchUpdateBatchTest", params);
}
}测试代码:
package com.momix.test.batch;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.momix.dbao.dao.dto.sqlxml.BatchTestDTO;
import com.momix.dbao.dao.entity.sqlxml.BatchTest;
import com.momix.dbao.dao.mapper.sqlxml.BatchTestMapper;
import com.momix.dbao.service.test.TestOneService;
@ContextConfiguration(locations = { "classpath:applicationContext.xml" })
public class BatchTestMain extends AbstractTestNGSpringContextTests {
@Autowired
private TestOneService testOneService;
@Autowired
private BatchTestMapper batchTestMapper;
private List<BatchTestDTO> paramsDao;
// 每次测试插入行数
int size = 0;
// 所有测试案例
List<Integer> sizes = new ArrayList<Integer>();
@BeforeTest
public void initData() {
sizes.add(100);
sizes.add(300);
sizes.add(500);
sizes.add(800);
sizes.add(1000);
sizes.add(1500);
sizes.add(2000);
sizes.add(3000);
sizes.add(5000);
sizes.add(8000);
sizes.add(10000);
sizes.add(30000);
sizes.add(50000);
sizes.add(100000);
}
@Test(enabled = false)
public void testBatchInsertDao() {
long start = 0l;
long end = 0l;
int b = 0;
int d = 0;
List<Map<String, Long>> infos = new ArrayList<Map<String, Long>>();
Map<String, Long> info = null;
for (int i = 0; i < sizes.size(); i++) {
info = new HashMap<String, Long>();
// 当前插入行数
size = sizes.get(i);
// 当前测试插入数据
paramsDao = this.getDataDao();
// 开始时间
start = System.currentTimeMillis();
// 批量插入开始
testOneService.batchTestInsertDao(paramsDao);
// 结束时间
end = System.currentTimeMillis();
b = batchTestMapper.getCount();
d = batchTestMapper.deleteData();
info.put("a", (long) size); // 应当插入行数
info.put("b", (long) b);// 实际插入行数
info.put("c", (end - start)); // 耗时
info.put("d", (long) d); // 删除行数
infos.add(info);
}
printData(infos);
}
public void printData(List<Map<String, Long>> infos) {
System.out
.println("***********************************************************************");
for (int i = 0; i < infos.size(); i++) {
Map<String, Long> info = infos.get(i);
System.out.println("测试插入行数 : " + info.get("a") + " 实际插入行数: "
+ info.get("b") + " 删除行数 : " + info.get("d") + " 耗时 : "
+ info.get("c"));
}
System.out
.println("***********************************************************************");
}
private List<BatchTestDTO> getDataDao() {
List<BatchTestDTO> tests = new ArrayList<BatchTestDTO>();
BatchTestDTO test = null;
for (int i = 0; i < size; i++) {
test = new BatchTestDTO();
test.setName(String.valueOf(i));
test.setSex(0);
tests.add(test);
}
return tests;
}
}测试结果:
总结：
通过jdbc循环分步提交，耗时较长，但比较稳定。
通过Mybatis用foreach进行批量操作，耗时较短，但数据量过大会出现异常
选择哪种方式，只能视业务场景而定。
以上是关于Mybatis批量操作与JDBC批量操作效率测试的主要内容，如果未能解决你的问题，请参考以下文章
MyBatis # 使用mybatis批量操作数据，提升效率
MyBatis-Plus的saveBatch批量插入为何效率很低耗时长详解及解决方案