chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

adapter 分批同步

Browse Source
This commit is contained in:
mcy
2018-11-21 13:46:30 +08:00
parent 5d71d50d4e
commit 56ec66f90e
17 changed files with 478 additions and 641 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java
View File

@@ -27,7 +27,7 @@ public interface OuterAdapter {
/**
* 往适配器中同步数据
*
* @param dml 数据包
* @param dmls 数据包
*/
void sync(List<Dml> dmls);

28
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java
View File

@@ -13,23 +13,25 @@ import java.util.Map;
*/
public class CanalClientConfig {
private String canalServerHost; // 单机模式下canal server的 ip:port
private String canalServerHost; // 单机模式下canal server的 ip:port
private String zookeeperHosts; // 集群模式下的zk地址, 如果配置了单机地址则以单机为准!!
private String zookeeperHosts; // 集群模式下的zk地址, 如果配置了单机地址则以单机为准!!
private String mqServers; // kafka or rocket mq 地址
private String mqServers; // kafka or rocket mq 地址
private Boolean flatMessage = true; // 是否已flatMessage模式传输, 只适用于mq模式
private Boolean flatMessage = true; // 是否已flatMessage模式传输, 只适用于mq模式
private Integer batchSize; // 批大小
private Integer batchSize; // 批大小
private Integer retries; // 重试次数
private Integer syncBatchSize = 1000; // 同步分批提交大小
private Long timeout; // 消费超时时间
private Integer retries; // 重试次数
private List<MQTopic> mqTopics; // mq topic 列表
private Long timeout; // 消费超时时间
private List<CanalInstance> canalInstances; // tcp 模式下 canal 实例列表, 与mq模式不能共存!!
private List<MQTopic> mqTopics; // mq topic 列表
private List<CanalInstance> canalInstances; // tcp 模式下 canal 实例列表, 与mq模式不能共存!!
public String getCanalServerHost() {
return canalServerHost;
@@ -83,6 +85,14 @@ public class CanalClientConfig {
return retries;
}
public Integer getSyncBatchSize() {
return syncBatchSize;
}
public void setSyncBatchSize(Integer syncBatchSize) {
this.syncBatchSize = syncBatchSize;
}
public void setRetries(Integer retries) {
this.retries = retries;
}

20
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java
View File

@@ -19,9 +19,9 @@ import com.alibaba.otter.canal.protocol.Message;
*/
public class MessageUtil {
public static void parse4Dml(String destination, Message message, Consumer<List<Dml>> consumer) {
public static List<Dml> parse4Dml(String destination, Message message) {
if (message == null) {
return;
return null;
}
List<CanalEntry.Entry> entries = message.getEntries();
List<Dml> dmls = new ArrayList<Dml>(entries.size());
@@ -89,10 +89,11 @@ public class MessageUtil {
Map<String, Object> rowOld = new LinkedHashMap<>();
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
if (updateSet.contains(column.getName())) {
rowOld.put(column.getName(), JdbcTypeUtil.typeConvert(column.getName(),
column.getValue(),
column.getSqlType(),
column.getMysqlType()));
rowOld.put(column.getName(),
JdbcTypeUtil.typeConvert(column.getName(),
column.getValue(),
column.getSqlType(),
column.getMysqlType()));
}
}
// update操作将记录修改前的值
@@ -110,7 +111,7 @@ public class MessageUtil {
}
}
consumer.accept(dmls);
return dmls;
}
public static List<Dml> flatMessage2Dml(String destination, List<FlatMessage> flatMessages) {
@@ -177,9 +178,4 @@ public class MessageUtil {
}
return result;
}
public interface Consumer<T> {
void accept(T t);
}
}