开发背景
多个大表数据均值3-5亿，故使用mysql 分库分表策略 水平拆分成小表
工程引入依赖
<!--shardingsphere 分库分表-->
<dependency>
<groupId>org.apache.shardingsphere</groupId>
<artifactId>sharding-jdbc-spring-boot-starter</artifactId>
<version>${sharding-sphere.version}</version>
</dependency>
<dependency>
<groupId>org.apache.shardingsphere</groupId>
<artifactId>sharding-jdbc-spring-namespace</artifactId>
<version>${sharding-sphere.version}</version>
</dependency>
引入nacos配置
记一次不知原因的问题: 分库分表的配置 tables 配置三个可用，两个可用，四个不可用，五个可用。 即为了可用性，配置一张虚拟表到五张表配置!!!!!!!!!!
# 数据源 spring: datasource: type: com.alibaba.druid.pool.DruidDataSource druid: driver-class-name: com.mysql.cj.jdbc.Driver username: ${MYSQL-USER:ms_wk} password: ENC(cYIdeHxqoBk7BXyBhLliz5YqQLlD5kPs) url: jdbc:mysql://${MYSQL-HOST:msdp-mysql}:${MYSQL-PORT:9306}/${MYSQL-DB:ms_wk}?characterEncoding=utf8&allowPublicKeyRetrieval=true&allowPublicKeyRetrieval=true&zeroDateTimeBehavior=convertToNull&useSSL=false&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT%2B8&allowMultiQueries=true stat-view-servlet: enabled: true url-pattern: /druid/* #login-username: admin #login-password: admin filter: stat: enabled: true log-slow-sql: true slow-sql-millis: 10000 merge-sql: false wall: config: multi-statement-allow: true #sharding-jdbc shardingsphere: datasource: names: history,master,slave1,slave2,mswkhis # 主库 master: driver-class-name: com.mysql.cj.jdbc.Driver type: com.alibaba.druid.pool.DruidDataSource url: jdbc:mysql://${MYSQL-HOST:msdp-mysql}:${MYSQL-PORT:9306}/${MYSQL-DB:ms_wk}?characterEncoding=utf8&allowPublicKeyRetrieval=true&allowPublicKeyRetrieval=true&zeroDateTimeBehavior=convertToNull&useSSL=false&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT%2B8&allowMultiQueries=true username: ${MYSQL-USER:ms_wk} password: ENC(cYIdeHxqoBk7BXyBhLliz5YqQLlD5kPs) # 从库1 slave1: driver-class-name: com.mysql.cj.jdbc.Driver type: com.alibaba.druid.pool.DruidDataSource url: jdbc:mysql://${MYSQL-HOST:msdp-mysql}:${MYSQL-PORT:9306}/${MYSQL-DB:ms_wk}?characterEncoding=utf8&allowPublicKeyRetrieval=true&allowPublicKeyRetrieval=true&zeroDateTimeBehavior=convertToNull&useSSL=false&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT%2B8&allowMultiQueries=true username: ${MYSQL-USER:ms_wk} password: ENC(cYIdeHxqoBk7BXyBhLliz5YqQLlD5kPs) # 从库2 slave2: driver-class-name: com.mysql.cj.jdbc.Driver type: com.alibaba.druid.pool.DruidDataSource url: jdbc:mysql://${MYSQL-HOST:msdp-mysql}:${MYSQL-PORT:9306}/${MYSQL-DB:ms_wk}?characterEncoding=utf8&allowPublicKeyRetrieval=true&allowPublicKeyRetrieval=true&zeroDateTimeBehavior=convertToNull&useSSL=false&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT%2B8&allowMultiQueries=true username: ${MYSQL-USER:ms_wk} password: ENC(cYIdeHxqoBk7BXyBhLliz5YqQLlD5kPs) # 历史数据库 history: driver-class-name: com.mysql.cj.jdbc.Driver type: com.alibaba.druid.pool.DruidDataSource url: jdbc:mysql://${MYSQL-HOST:msdp-mysql}:${MYSQL-PORT:9306}/ms_history?serverTimezone=UTC&useSSL=false&useUnicode=true&characterEncoding=UTF-8 username: ${MYSQL-USER:ms_wk} password: ENC(cYIdeHxqoBk7BXyBhLliz5YqQLlD5kPs) # 分库分表数据库 mswkhis: driver-class-name: com.mysql.cj.jdbc.Driver type: com.alibaba.druid.pool.DruidDataSource url: jdbc:mysql://${MYSQL-HOST:msdp-mysql}:${MYSQL-PORT:9306}/ms_wk_his?serverTimezone=UTC&useSSL=false&useUnicode=true&characterEncoding=UTF-8 username: ${MYSQL-USER:ms_wk} password: ENC(cYIdeHxqoBk7BXyBhLliz5YqQLlD5kPs) props: sql: show: true # 分片配置 sharding: # 分片读写分离配置 master-slave-rules: # 默认主从 ds_master_slave: master-data-source-name: master slave-data-source-names: slave1,slave2 # 分片数据源读写分离配置（历史数据源） ds_history: master-data-source-name: history slave-data-source-names: history # 分片数据源读写分离配置（分库分表数据源） ds_ms_wk_his: master-data-source-name: mswkhis slave-data-source-names: mswkhis # 未配置分片规则的表将通过默认数据源定位 default-data-source-name: ds_master_slave binding-tables: wk_sync_log broadcast-tables: t_address tables: wk_sync_log: actual-data-nodes: ds_history.wk_sync_log_$->{2020..2021}$->{['01','02','03','04','05','06','07','08','09','10','11','12']}$->{['01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19','20','21','22','23','24','25','26','27','28','29','30','31']} key-generator: column: jid type: UUID table-strategy: standard: precise-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayShardingAlgorithm range-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayRangeShardingAlgorithm sharding-column: create_time wk_atdmac_record: actual-data-nodes: ds_history.wk_atdmac_record_$->{2020..2021}$->{['01','02','03','04','05','06','07','08','09','10','11','12']}$->{['01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19','20','21','22','23','24','25','26','27','28','29','30','31']} key-generator: column: jid type: UUID table-strategy: standard: precise-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayShardingAlgorithm range-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayRangeShardingAlgorithm sharding-column: show_time wk_worker_attendance: actual-data-nodes: ds_ms_wk_his.wk_worker_attendance_$->{2020..2021}$->{['01','02','03','04','05','06','07','08','09','10','11','12']}$->{['01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19','20','21','22','23','24','25','26','27','28','29','30','31']} key-generator: column: jid type: UUID table-strategy: standard: precise-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayShardingAlgorithm range-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayRangeShardingAlgorithm sharding-column: attend_time wk_worker_attend_month: actual-data-nodes: ds_ms_wk_his.wk_worker_attend_month_$->{2020..2021}$->{['01','02','03','04','05','06','07','08','09','10','11','12']}$->{['01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19','20','21','22','23','24','25','26','27','28','29','30','31']} key-generator: column: jid type: UUID table-strategy: standard: precise-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayShardingAlgorithm range-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayRangeShardingAlgorithm sharding-column: attend_date t_sharding_bak: actual-data-nodes: ds_ms_wk_his.t_sharding_bak_20210101 key-generator: column: jid type: UUID table-strategy: standard: precise-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayShardingAlgorithm range-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayRangeShardingAlgorithm sharding-column: attend_date
设置分片规则
package com.iyysoft.cloud.msdp.wk.config.shardingsphere; import com.google.common.collect.Range; import org.apache.shardingsphere.api.sharding.standard.RangeShardingAlgorithm; import org.apache.shardingsphere.api.sharding.standard.RangeShardingValue; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Collection; import java.util.Date; import java.util.LinkedHashSet; /** * 日期范围分片 * * @author lvlinguang * @date 2020-08-10 19:09 */ public class DayRangeShardingAlgorithm implements RangeShardingAlgorithm<String> { /** * 设置分片 * * @param collection * @param rangeShardingValue * @return */ @Override public Collection<String> doSharding(Collection<String> collection, RangeShardingValue<String> rangeShardingValue) { Collection<String> result = new LinkedHashSet<>(); DateFormat sdf = new SimpleDateFormat("yyyyMMdd"); //日期 Range<String> ranges = rangeShardingValue.getValueRange(); Date startTime = dateFormat(ranges.lowerEndpoint()); Date endTime = dateFormat(ranges.upperEndpoint()); Calendar cal = Calendar.getInstance(); while (startTime.getTime() <= endTime.getTime()) { String value = sdf.format(startTime); for (String each : collection) { if (each.endsWith(value)) { result.add(each); break; } } cal.setTime(startTime); cal.add(Calendar.DAY_OF_YEAR, 1); startTime = cal.getTime(); } if (result.size() == 0) { result = collection; } return result; } /** * 日期转换 * * @param date * @return */ public Date dateFormat(String date) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); try { return sdf.parse(date); } catch (ParseException e) { return null; } } }
package com.iyysoft.cloud.msdp.wk.config.shardingsphere; import lombok.SneakyThrows; import org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm; import org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue; import java.text.ParseException; import java.util.Collection; /** * 日期精确分片 * * @author lvlinguang * @date 2020-08-10 19:11 */ public class DayShardingAlgorithm implements PreciseShardingAlgorithm<String> { /** * 设置分片 * * @param tableNames 数据表 * @param shardingValue 分片列信息 * @return */ @SneakyThrows @Override public String doSharding(Collection<String> tableNames, PreciseShardingValue<String> shardingValue) { String tableName = shardingValue.getLogicTableName(); String key = getDate(shardingValue.getValue(), 8); return tableName.concat("_").concat(key); } /** * 得到日期数字 * * @param date 字符串日期 * @param len 长度 * @return 202008 * @throws ParseException */ public String getDate(String date, int len) throws ParseException { String number = date.replaceAll("\D", ""); return number.substring(0, len); } }
package com.iyysoft.cloud.msdp.wk.config.shardingsphere; import com.google.common.collect.Range; import org.apache.shardingsphere.api.sharding.standard.RangeShardingAlgorithm; import org.apache.shardingsphere.api.sharding.standard.RangeShardingValue; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Collection; import java.util.Date; import java.util.LinkedHashSet; /** * 月范围分片 * * @author lvlinguang * @date 2020-08-10 19:09 */ public class MonthRangeShardingAlgorithm implements RangeShardingAlgorithm<String> { /** * 设置分片 * * @param collection * @param rangeShardingValue * @return */ @Override public Collection<String> doSharding(Collection<String> collection, RangeShardingValue<String> rangeShardingValue) { Collection<String> result = new LinkedHashSet<>(); DateFormat sdf = new SimpleDateFormat("yyyyMM"); //日期 Range<String> ranges = rangeShardingValue.getValueRange(); Date startTime = dateFormat(ranges.lowerEndpoint()); Date endTime = dateFormat(ranges.upperEndpoint()); Calendar cal = Calendar.getInstance(); while (startTime.getTime() <= endTime.getTime()) { String value = sdf.format(startTime); for (String each : collection) { if (each.endsWith(value)) { result.add(each); break; } } cal.setTime(startTime); cal.add(Calendar.MONTH, 1); startTime = cal.getTime(); } if (result.size() == 0) { result = collection; } return result; } /** * 日期转换 * * @param date * @return */ public Date dateFormat(String date) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); try { return sdf.parse(date); } catch (ParseException e) { return null; } } }
package com.iyysoft.cloud.msdp.wk.config.shardingsphere; import lombok.SneakyThrows; import org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm; import org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue; import java.text.ParseException; import java.util.Collection; /** * 月精确分片 * * @author lvlinguang * @date 2020-08-10 19:11 */ public class MonthShardingAlgorithm implements PreciseShardingAlgorithm<String> { /** * 设置分片 * * @param tableNames 数据表 * @param shardingValue 分片列信息 * @return */ @SneakyThrows @Override public String doSharding(Collection<String> tableNames, PreciseShardingValue<String> shardingValue) { String tableName = shardingValue.getLogicTableName(); String key = getDate(shardingValue.getValue(), 6); return tableName.concat("_").concat(key); } /** * 得到日期数字 * * @param date 字符串日期 * @param len 长度 * @return 202008 * @throws ParseException */ public String getDate(String date, int len) throws ParseException { String number = date.replaceAll("\D", ""); return number.substring(0, len); } }
大表数据归档
-- 创建定时任务事件
create event his_wk_worker_attendance_joint_event
ON SCHEDULE EVERY 1 DAY STARTS DATE_ADD(DATE_ADD(CURDATE(), INTERVAL 1 DAY), INTERVAL 1 HOUR)
on completion preserve enable
do call his_wk_worker_attendance_joint();
-- 查看定时任务
SELECT event_name,event_definition,interval_value,interval_field,status FROM information_schema.EVENTS;
-- 启停已经创建好的event(事件)
alter event his_wk_worker_attendance_joint_event on completion preserve enable; -- 开启定时任务
alter event his_wk_worker_attendance_joint_event on completion preserve disable;-- 关闭定时任务
-- 开启事件调度器
SET GLOBAL event_scheduler = ON;
-- 关闭事件调度器
SET GLOBAL event_scheduler = OFF;
-- 查看事件调度器状态
SHOW VARIABLES LIKE 'event_scheduler';
CREATE DEFINER=`croot`@`%` PROCEDURE `his_wk_worker_attendance_joint`() BEGIN insert into ms_wk.wk_worker_attendance_joint_his select * from wk_worker_attendance_joint where create_date < date_sub(now(),interval 5 day); commit; delete from ms_wk.wk_worker_attendance_joint where create_date < date_sub(now(),interval 5 day); commit; END
大表数据迁移
CREATE DEFINER=`croot`@`%` PROCEDURE `sub_wk_worker_attendance`() BEGIN declare x int; declare y int; declare z int; set x=2020; set y=1; set z=1; while x<=2022 do while y<=12 do while z<=31 do set @sql_create_table_gpstrail = concat( 'CREATE TABLE IF NOT EXISTS wk_worker_attendance_',lPAD(x,4,'0'),lPAD(y,2,'0'),lPAD(z,2,'0'), "(`jid` char(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '在jtg平台的数据id ', `tenant_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '所属租户', `create_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '创建人', `create_date` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `update_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '更新人', `update_date` datetime NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', `has_use` char(1) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '是否启用', `has_del` char(1) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT '0' COMMENT '是否删除', `project_jid` char(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '项目id', `team_jid` char(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '班组id', `worker_jid` char(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '工人id', `worker_name` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '工人姓名', `attend_time` datetime NOT NULL COMMENT '考勤时间', `direction` char(2) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '考勤方向 考勤方向字典表: 1入场 0出场 ', `attend_type` char(3) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '001' COMMENT '通行方式 参考通行方式字典表 : 001 人脸识别 002 虹膜识别 003 指纹识别 004 掌纹识别 005 身份证识别 006 实名卡 007 异常清退（适用于人员没有通过闸机系统出工地而导致人员状态不一致的情况） 008 一键开闸(需要与闸机交互) 009 应急通道（不需要与闸机交互） 010 二维码识别 011 其他方式 012系统自动签出', `img_url` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '考勤照片 期望是绝对路径', `lng` decimal(18, 15) NULL DEFAULT NULL COMMENT '经度 WGS84经度', `lat` decimal(18, 15) NULL DEFAULT NULL COMMENT '纬度 WGS84纬度', `channel` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '通道', `attend_source_type` int(3) UNSIGNED NOT NULL DEFAULT 0 COMMENT '考勤数据来源: 0未定义 , 10个人app考勤 , 11班组长app考勤 , 12管工端考勤机模式考勤 , 20 考勤机考勤 定义: 1X APP端考勤 , 2X 硬件设备考勤', `atdmac_jid` char(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '考勤设备jid', PRIMARY KEY (`jid`) USING BTREE, UNIQUE INDEX `attend_time`(`attend_time`, `project_jid`, `worker_jid`, `has_del`) USING BTREE, INDEX `team_jid`(`team_jid`) USING BTREE, INDEX `project_jid`(`project_jid`, `worker_jid`) USING BTREE ) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = '工人考勤数据原始记录' ROW_FORMAT = Dynamic"); PREPARE sql_create_table_gpstrail FROM @sql_create_table_gpstrail; EXECUTE sql_create_table_gpstrail; set z=z+1; end while; set y=y+1; set z=1; end while; set x= x +1; set y=1; set z=1; end while; END
CREATE DEFINER=`croot`@`%` PROCEDURE `insert_wk_worker_attendance`() BEGIN declare x int; declare y int; declare z int; set x=2020; set y=1; set z=1; while x<=2022 do while y<=12 do while z<=31 do set @sql_create_table_gpstrail = concat('insert into ms_wk_his.wk_worker_attendance_',lPAD(x,4,'0'),lPAD(y,2,'0'),lPAD(z,2,'0'), " select * from ms_wk.wk_worker_attendance_bak where attend_time BETWEEN '",lPAD(x,4,'0'),"-",lPAD(y,2,'0'),"-",lPAD(z,2,'0')," 00:00:00' AND '",lPAD(x,4,'0'),"-",lPAD(y,2,'0'),"-",lPAD(z,2,'0')," 23:59:59'"); PREPARE sql_create_table_gpstrail FROM @sql_create_table_gpstrail; EXECUTE sql_create_table_gpstrail; commit; set z=z+1; end while; set y=y+1; set z=1; end while; set x= x +1; set y=1; set z=1; end while; END
CREATE DEFINER=`croot`@`%` PROCEDURE `subDBtest`() BEGIN declare x int; declare y int; declare z int; set x=2020; set y=1; set z=1; while x<=2022 do while y<=12 do while z<=31 do select concat(lPAD(x,4,'0'),lPAD(y,2,'0'),lPAD(z,2,'0')); set z=z+1; end while; set y=y+1; set z=1; end while; set x= x +1; set y=1; set z=1; end while; END
数据迁移问题 走索引 + 存储过程脚本 会快一点
分库分表引发的问题
分库分表不支持子查询
分库分表不支持mybatis-plus分页查询(自带子查询)
分库分表不带上分表字段，即会查询所有分表
// 弃用mybatis-plus自带分页，手动分页参考文献
https://blog.csdn.net/lq2418c/article/details/120039274?utm_medium=distribute.pc_relevant.none-task-blog-2%7Edefault%7ECTRLIST%7Edefault-3.no_search_link&depth_1-utm_source=distribute.pc_relevant.none-task-blog-2%7Edefault%7ECTRLIST%7Edefault-3.no_search_link
https://blog.csdn.net/qq_44086060/article/details/116273587
package com.iyysoft.msdp.basic.bean.util; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.List; /** * @author cjq * @date 2021-09-27 * mybatis-plus 分页帮助类 */ @Component public class PageHelperUtils { /** * 分页方法 * * @param currentPage 页数 * @param pageSize 分页大小 * @param list 分页对象 * @return */ public static Page getPages(Integer currentPage, Integer pageSize, List list) { Page page = new Page(); int size = list.size(); if(size == 0){ return page.setCurrent(currentPage).setSize(pageSize).setTotal(list.size()).setRecords(list); } if (pageSize > size) { pageSize = size; } //求出最大页数，防止currentPage越界 int maxPage = size % pageSize == 0 ? size / pageSize : size / pageSize + 1; if (currentPage > maxPage) { currentPage = maxPage; } //当前页第一条数据下标 int curIds = currentPage > 1 ? (currentPage - 1) * pageSize : 0; List pageList = new ArrayList<>(); //将当前页的数据放进pageList for (int i = 0; i < pageSize && curIds + i < size; i++) { pageList.add(list.get(curIds + i)); } page.setCurrent(currentPage).setSize(pageSize).setTotal(list.size()).setRecords(pageList); return page; } }