SpringBoot+Mybatis-plus+shardingsphere實現(xiàn)分庫分表

介紹

實現(xiàn)億級數(shù)據(jù)量分庫分表的項目是一個挑戰(zhàn)性很高的任務，下面是一個基于Spring Boot的簡單實現(xiàn)方案：

1. 數(shù)據(jù)庫選擇：使用MySQL數(shù)據(jù)庫，因為MySQL在分庫分表方面有較成熟的解決方案。

2. 分庫分表策略：可以采用水平分庫分表的策略，根據(jù)一定的規(guī)則將數(shù)據(jù)分散存儲在不同的數(shù)據(jù)庫和表中，例如可以根據(jù)用戶ID、訂單ID等進行分片。

3. 數(shù)據(jù)分片策略：可以采用基于雪花算法的分布式ID生成器來生成全局唯一的ID，確保數(shù)據(jù)在不同數(shù)據(jù)庫和表中的唯一性。

4. 數(shù)據(jù)同步：考慮到數(shù)據(jù)分散存儲在不同的數(shù)據(jù)庫和表中，需要實現(xiàn)數(shù)據(jù)同步機制來保證數(shù)據(jù)的一致性，可以使用Canal等開源工具來實現(xiàn)MySQL數(shù)據(jù)的實時同步。

5. 連接池優(yōu)化：在處理大量數(shù)據(jù)時，連接池的配置尤為重要，可以使用Druid等高性能的連接池來提升數(shù)據(jù)庫連接的效率。

6. 緩存機制：考慮使用Redis等緩存工具來緩存熱點數(shù)據(jù)，減輕數(shù)據(jù)庫的壓力，提升系統(tǒng)性能。

7. 分布式事務：在分庫分表的場景下，涉及到跨庫事務，可以考慮使用分布式事務框架，如Seata等來保證事務的一致性。

8. 監(jiān)控與調優(yōu)：實時監(jiān)控數(shù)據(jù)庫的性能指標，及時調整分片策略和數(shù)據(jù)庫配置，保證系統(tǒng)的穩(wěn)定性和性能。

引入依賴

<dependencies><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-web</artifactId></dependency><dependency><groupId>mysql</groupId><artifactId>mysql-connector-java</artifactId><version>8.0.28</version></dependency><dependency><groupId>com.baomidou</groupId><artifactId>mybatis-plus-boot-starter</artifactId><version>3.4.3</version></dependency><!--分庫分表--><!-- Sharding-JDBC --><dependency><groupId>org.apache.shardingsphere</groupId><artifactId>shardingsphere-jdbc-core-spring-boot-starter</artifactId><version>5.2.0</version></dependency><dependency><groupId>org.projectlombok</groupId><artifactId>lombok</artifactId><optional>true</optional></dependency><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-test</artifactId><scope>test</scope></dependency></dependencies>

yaml配置

server:port: 10086spring:shardingsphere:# 數(shù)據(jù)源配置datasource:# 數(shù)據(jù)源名稱，多數(shù)據(jù)源以逗號分隔names: db0,db1db0:type: com.zaxxer.hikari.HikariDataSourcedriver-class-name: com.mysql.cj.jdbc.Driverjdbc-url: jdbc:mysql://127.0.0.1:3306/ds0?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=trueusername: rootpassword: rootdb1:type: com.zaxxer.hikari.HikariDataSourcedriver-class-name: com.mysql.cj.jdbc.Driverjdbc-url: jdbc:mysql://127.0.0.1:3306/ds1?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=trueusername: rootpassword: root# 分片規(guī)則配置rules:sharding:# 分片算法配置sharding-algorithms:database-inline:# 分片算法類型type: INLINEprops:# 分片算法的行表達式（算法自行定義，此處為方便演示效果）algorithm-expression: db$->{age % 2}table-inline:# 分片算法類型type: INLINEprops:# 分片算法的行表達式algorithm-expression: user_$->{age % 3}tables:# 邏輯表名稱user:# 行表達式標識符可以使用 ${...} 或 $->{...}，但前者與 Spring 本身的屬性文件占位符沖突，因此在 Spring 環(huán)境中使用行表達式標識符建議使用 $->{...}actual-data-nodes: db${0..1}.user_${0..2}# 分庫策略database-strategy:standard:# 分片列名稱sharding-column: age# 分片算法名稱sharding-algorithm-name: database-inline# 分表策略table-strategy:standard:# 分片列名稱sharding-column: age# 分片算法名稱sharding-algorithm-name: table-inline# 屬性配置props:# 展示修改以后的sql語句sql-show: true

DDL準備

數(shù)據(jù)庫ds0

-- ds0.user_0 definitionCREATE TABLE `user_0` (`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主鍵',`name` varchar(32) NOT NULL COMMENT '姓名',`age` int NOT NULL COMMENT '年齡',PRIMARY KEY (`id`)
) ENGINE=InnoDB COMMENT='用戶表';

-- ds0.user_1 definitionCREATE TABLE `user_1` (`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主鍵',`name` varchar(32) NOT NULL COMMENT '姓名',`age` int NOT NULL COMMENT '年齡',PRIMARY KEY (`id`)
) ENGINE=InnoDB COMMENT='用戶表';

-- ds0.user_2 definitionCREATE TABLE `user_2` (`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主鍵',`name` varchar(32) NOT NULL COMMENT '姓名',`age` int NOT NULL COMMENT '年齡',PRIMARY KEY (`id`)
) ENGINE=InnoDB COMMENT='用戶表';

數(shù)據(jù)庫ds1

-- ds1.user_0 definitionCREATE TABLE `user_0` (`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主鍵',`name` varchar(32) NOT NULL COMMENT '姓名',`age` int NOT NULL COMMENT '年齡',PRIMARY KEY (`id`)
) ENGINE=InnoDB COMMENT='用戶表';

-- ds1.user_1 definitionCREATE TABLE `user_1` (`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主鍵',`name` varchar(32) NOT NULL COMMENT '姓名',`age` int NOT NULL COMMENT '年齡',PRIMARY KEY (`id`)
) ENGINE=InnoDB COMMENT='用戶表';

-- ds1.user_2 definitionCREATE TABLE `user_2` (`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主鍵',`name` varchar(32) NOT NULL COMMENT '姓名',`age` int NOT NULL COMMENT '年齡',PRIMARY KEY (`id`)
) ENGINE=InnoDB COMMENT='用戶表';

entity

package com.kang.sharding.entity;import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;@Data
@TableName("user")  
public class User {@TableId(value = "id",type = IdType.AUTO)private Long id;private String name;private Integer age;// getter, setter, toString...  
}

cotroller

package com.kang.sharding.controller;import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.kang.sharding.entity.User;
import com.kang.sharding.service.UserService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;import java.util.List;@RestController  
@RequestMapping("/user")  
public class UserController {  @Autowired  private UserService userService;@PostMapping ("add")public boolean createUser(@RequestBody User user) {return userService.save(user);  }@PostMapping ("update")public boolean updateByAge(@RequestBody User user){LambdaUpdateWrapper<User> updateWrapper = new LambdaUpdateWrapper<>();// 分片數(shù)據(jù)不允許更新,否則會報錯updateWrapper.eq(User::getAge,user.getAge()).set(User::getName,user.getName());return userService.update(updateWrapper);}@GetMapping ("delete")public boolean deleteUserByAge(@RequestParam("age") Integer age) {LambdaQueryWrapper<User> queryWrapper = new LambdaQueryWrapper<>();queryWrapper.eq(User::getAge,age);return userService.remove(queryWrapper);}@GetMapping("/{age}")public List<User> getUserByAge(@PathVariable Integer age) {LambdaQueryWrapper<User> queryWrapper = new LambdaQueryWrapper<>();queryWrapper.eq(User::getAge,age);return userService.list(queryWrapper);}  // 其他方法...  
}

service

package com.kang.sharding.service;import com.kang.sharding.entity.User;
import com.kang.sharding.mapper.UserMapper;
import org.springframework.stereotype.Service;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;  @Service  
public class UserService extends ServiceImpl<UserMapper, User> {
}

Mapper

package com.kang.sharding.mapper;import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.kang.sharding.entity.User;
import org.apache.ibatis.annotations.Mapper;@Mapper
public interface UserMapper extends BaseMapper<User> {
}

啟動類

package com.kang.sharding;import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;@SpringBootApplication
public class ShardingJdbcProjectApplication {public static void main(String[] args) {SpringApplication.run(ShardingJdbcProjectApplication.class, args);}}

測試

添加

修改

查詢

刪除

總結

這只是個簡單的入門示例,后續(xù)深入研究