chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #30 from alibaba/master

Browse Source 
merge
This commit is contained in:
rewerma
2019-01-11 12:59:01 +08:00
committed by GitHub
parent 690222bde8 f1753e79f9
commit 92666b41e8
16 changed files with 206 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java
View File

@@ -3,11 +3,8 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List; import java.util.List;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.OuterAdapter;
@@ -44,8 +41,12 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
@Override @Override
protected void process() { protected void process() {
while (!running) while (!running) {
; try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries(); int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒

23
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java
View File

@@ -3,11 +3,7 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List; import java.util.List;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.OuterAdapter;
@@ -33,18 +29,27 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
this.topic = topic; this.topic = topic;
this.flatMessage = flatMessage; this.flatMessage = flatMessage;
this.canalDestination = topic; this.canalDestination = topic;
this.connector = new RocketMQCanalConnector(nameServers, topic, groupId, accessKey, secretKey, flatMessage); this.connector = new RocketMQCanalConnector(nameServers,
topic,
groupId,
accessKey,
secretKey,
canalClientConfig.getBatchSize(),
flatMessage);
logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId); logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
} }
@Override @Override
protected void process() { protected void process() {
while (!running) while (!running) {
; try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
int retry = canalClientConfig.getRetries() == null int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) { while (running) {

16
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java
View File

@@ -2,7 +2,8 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
import java.net.SocketAddress; import java.net.SocketAddress;
import java.util.List; import java.util.List;
import java.util.concurrent.*; import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.alibaba.otter.canal.client.CanalConnector; import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors; import com.alibaba.otter.canal.client.CanalConnectors;
@@ -58,11 +59,16 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
@Override @Override
protected void process() { protected void process() {
while (!running) while (!running) { // waiting until running == true
; // waiting until running == true while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
}
int retry = canalClientConfig.getRetries() == null int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
if (retry == -1) { if (retry == -1) {
// 重试次数-1代表异常时一直阻塞重试 // 重试次数-1代表异常时一直阻塞重试
retry = Integer.MAX_VALUE; retry = Integer.MAX_VALUE;

10
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java
View File

@@ -12,6 +12,7 @@ import java.util.concurrent.Future;
import javax.sql.DataSource; import javax.sql.DataSource;
import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -130,12 +131,17 @@ public class RdbAdapter implements OuterAdapter {
String threads = properties.get("threads"); String threads = properties.get("threads");
// String commitSize = properties.get("commitSize"); // String commitSize = properties.get("commitSize");
rdbSyncService = new RdbSyncService(dataSource, threads != null ? Integer.valueOf(threads) : null); boolean skipDupException = BooleanUtils.toBoolean(configuration.getProperties()
.getOrDefault("skipDupException", "true"));
rdbSyncService = new RdbSyncService(dataSource,
threads != null ? Integer.valueOf(threads) : null,
skipDupException);
rdbMirrorDbSyncService = new RdbMirrorDbSyncService(mirrorDbConfigCache, rdbMirrorDbSyncService = new RdbMirrorDbSyncService(mirrorDbConfigCache,
dataSource, dataSource,
threads != null ? Integer.valueOf(threads) : null, threads != null ? Integer.valueOf(threads) : null,
rdbSyncService.getColumnsTypeCache()); rdbSyncService.getColumnsTypeCache(),
skipDupException);
rdbConfigMonitor = new RdbConfigMonitor(); rdbConfigMonitor = new RdbConfigMonitor();
rdbConfigMonitor.init(configuration.getKey(), this); rdbConfigMonitor.init(configuration.getKey(), this);

5
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java
View File

@@ -35,10 +35,11 @@ public class RdbMirrorDbSyncService {
private RdbSyncService rdbSyncService; // rdbSyncService代理 private RdbSyncService rdbSyncService; // rdbSyncService代理
public RdbMirrorDbSyncService(Map<String, MirrorDbConfig> mirrorDbConfigCache, DataSource dataSource, public RdbMirrorDbSyncService(Map<String, MirrorDbConfig> mirrorDbConfigCache, DataSource dataSource,
Integer threads, Map<String, Map<String, Integer>> columnsTypeCache){ Integer threads, Map<String, Map<String, Integer>> columnsTypeCache,
boolean skipDupException){
this.mirrorDbConfigCache = mirrorDbConfigCache; this.mirrorDbConfigCache = mirrorDbConfigCache;
this.dataSource = dataSource; this.dataSource = dataSource;
this.rdbSyncService = new RdbSyncService(dataSource, threads, columnsTypeCache); this.rdbSyncService = new RdbSyncService(dataSource, threads, columnsTypeCache, skipDupException);
} }
/** /**

34
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -44,6 +44,7 @@ public class RdbSyncService {
private Map<String, Map<String, Integer>> columnsTypeCache; private Map<String, Map<String, Integer>> columnsTypeCache;
private int threads = 3; private int threads = 3;
private boolean skipDupException;
private List<SyncItem>[] dmlsPartition; private List<SyncItem>[] dmlsPartition;
private BatchExecutor[] batchExecutors; private BatchExecutor[] batchExecutors;
@@ -58,13 +59,15 @@ public class RdbSyncService {
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public RdbSyncService(DataSource dataSource, Integer threads){ public RdbSyncService(DataSource dataSource, Integer threads, boolean skipDupException){
this(dataSource, threads, new ConcurrentHashMap<>()); this(dataSource, threads, new ConcurrentHashMap<>(), skipDupException);
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public RdbSyncService(DataSource dataSource, Integer threads, Map<String, Map<String, Integer>> columnsTypeCache){ public RdbSyncService(DataSource dataSource, Integer threads, Map<String, Map<String, Integer>> columnsTypeCache,
boolean skipDupException){
this.columnsTypeCache = columnsTypeCache; this.columnsTypeCache = columnsTypeCache;
this.skipDupException = skipDupException;
try { try {
if (threads != null) { if (threads != null) {
this.threads = threads; this.threads = threads;
@@ -182,6 +185,7 @@ public class RdbSyncService {
*/ */
public void sync(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) { public void sync(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) {
if (config != null) { if (config != null) {
try {
String type = dml.getType(); String type = dml.getType();
if (type != null && type.equalsIgnoreCase("INSERT")) { if (type != null && type.equalsIgnoreCase("INSERT")) {
insert(batchExecutor, config, dml); insert(batchExecutor, config, dml);
@@ -193,6 +197,9 @@ public class RdbSyncService {
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue)); logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
} }
} catch (SQLException e) {
throw new RuntimeException(e);
}
} }
} }
@@ -202,7 +209,7 @@ public class RdbSyncService {
* @param config 配置项 * @param config 配置项
* @param dml DML数据 * @param dml DML数据
*/ */
private void insert(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) { private void insert(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
Map<String, Object> data = dml.getData(); Map<String, Object> data = dml.getData();
if (data == null || data.isEmpty()) { if (data == null || data.isEmpty()) {
return; return;
@@ -240,11 +247,20 @@ public class RdbSyncService {
throw new RuntimeException("Target column: " + targetColumnName + " not matched"); throw new RuntimeException("Target column: " + targetColumnName + " not matched");
} }
Object value = data.get(srcColumnName); Object value = data.get(srcColumnName);
BatchExecutor.setValue(values, type, value); BatchExecutor.setValue(values, type, value);
} }
try {
batchExecutor.execute(insertSql.toString(), values); batchExecutor.execute(insertSql.toString(), values);
} catch (SQLException e) {
if (skipDupException
&& (e.getMessage().contains("Duplicate entry") || e.getMessage().startsWith("ORA-00001: 违反唯一约束条件"))) {
// ignore
// TODO 增加更多关系数据库的主键冲突的错误码
} else {
throw e;
}
}
if (logger.isTraceEnabled()) { if (logger.isTraceEnabled()) {
logger.trace("Insert into target table, sql: {}", insertSql); logger.trace("Insert into target table, sql: {}", insertSql);
} }
@@ -257,7 +273,7 @@ public class RdbSyncService {
* @param config 配置项 * @param config 配置项
* @param dml DML数据 * @param dml DML数据
*/ */
private void update(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) { private void update(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
Map<String, Object> data = dml.getData(); Map<String, Object> data = dml.getData();
if (data == null || data.isEmpty()) { if (data == null || data.isEmpty()) {
return; return;
@@ -301,9 +317,7 @@ public class RdbSyncService {
// 拼接主键 // 拼接主键
appendCondition(dbMapping, updateSql, ctype, values, data, old); appendCondition(dbMapping, updateSql, ctype, values, data, old);
batchExecutor.execute(updateSql.toString(), values); batchExecutor.execute(updateSql.toString(), values);
if (logger.isTraceEnabled()) { if (logger.isTraceEnabled()) {
logger.trace("Update target table, sql: {}", updateSql); logger.trace("Update target table, sql: {}", updateSql);
} }
@@ -315,7 +329,7 @@ public class RdbSyncService {
* @param config * @param config
* @param dml * @param dml
*/ */
private void delete(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) { private void delete(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
Map<String, Object> data = dml.getData(); Map<String, Object> data = dml.getData();
if (data == null || data.isEmpty()) { if (data == null || data.isEmpty()) {
return; return;
@@ -331,9 +345,7 @@ public class RdbSyncService {
List<Map<String, ?>> values = new ArrayList<>(); List<Map<String, ?>> values = new ArrayList<>();
// 拼接主键 // 拼接主键
appendCondition(dbMapping, sql, ctype, values, data); appendCondition(dbMapping, sql, ctype, values, data);
batchExecutor.execute(sql.toString(), values); batchExecutor.execute(sql.toString(), values);
if (logger.isTraceEnabled()) { if (logger.isTraceEnabled()) {
logger.trace("Delete from target table, sql: {}", sql); logger.trace("Delete from target table, sql: {}", sql);
} }

31
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/BatchExecutor.java
View File

@@ -19,15 +19,10 @@ public class BatchExecutor implements Closeable {
private Connection conn; private Connection conn;
private AtomicInteger idx = new AtomicInteger(0); private AtomicInteger idx = new AtomicInteger(0);
public BatchExecutor(Connection conn){ public BatchExecutor(Connection conn) throws SQLException{
this.conn = conn; this.conn = conn;
try {
this.conn.setAutoCommit(false); this.conn.setAutoCommit(false);
} catch (SQLException e) {
logger.error(e.getMessage(), e);
} }
}
public Connection getConn() { public Connection getConn() {
return conn; return conn;
@@ -40,8 +35,8 @@ public class BatchExecutor implements Closeable {
values.add(valueItem); values.add(valueItem);
} }
public void execute(String sql, List<Map<String, ?>> values) { public void execute(String sql, List<Map<String, ?>> values) throws SQLException {
try (PreparedStatement pstmt = conn.prepareStatement(sql)) { PreparedStatement pstmt = conn.prepareStatement(sql);
int len = values.size(); int len = values.size();
for (int i = 0; i < len; i++) { for (int i = 0; i < len; i++) {
int type = (Integer) values.get(i).get("type"); int type = (Integer) values.get(i).get("type");
@@ -51,42 +46,32 @@ public class BatchExecutor implements Closeable {
pstmt.execute(); pstmt.execute();
idx.incrementAndGet(); idx.incrementAndGet();
} catch (SQLException e) {
logger.error(e.getMessage(), e);
}
} }
public void commit() { public void commit() throws SQLException {
try {
conn.commit(); conn.commit();
if (logger.isTraceEnabled()) { if (logger.isTraceEnabled()) {
logger.trace("Batch executor commit " + idx.get() + " rows"); logger.trace("Batch executor commit " + idx.get() + " rows");
} }
idx.set(0); idx.set(0);
} catch (SQLException e) {
logger.error(e.getMessage(), e);
}
} }
public void rollback() { public void rollback() throws SQLException {
try {
conn.rollback(); conn.rollback();
if (logger.isTraceEnabled()) { if (logger.isTraceEnabled()) {
logger.trace("Batch executor rollback " + idx.get() + " rows"); logger.trace("Batch executor rollback " + idx.get() + " rows");
} }
idx.set(0); idx.set(0);
} catch (SQLException e) {
logger.error(e.getMessage(), e);
}
} }
@Override @Override
public void close() { public void close() {
if (conn != null) { if (conn != null) {
try { try {
if (conn != null) {
conn.close(); conn.close();
} catch (SQLException e) { }
logger.error(e.getMessage(), e); } catch (SQLException ioe) {
} }
} }
} }

22
client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnector.java
View File

@@ -1,7 +1,5 @@
package com.alibaba.otter.canal.client.rocketmq; package com.alibaba.otter.canal.client.rocketmq;
import com.aliyun.openservices.apache.api.impl.authority.SessionCredentials;
import com.aliyun.openservices.apache.api.impl.rocketmq.ClientRPCHook;
import java.util.List; import java.util.List;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
@@ -26,6 +24,8 @@ import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.exception.CanalClientException; import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.aliyun.openservices.apache.api.impl.authority.SessionCredentials;
import com.aliyun.openservices.apache.api.impl.rocketmq.ClientRPCHook;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
/** /**
@@ -48,24 +48,26 @@ public class RocketMQCanalConnector implements CanalMQConnector {
private volatile boolean connected = false; private volatile boolean connected = false;
private DefaultMQPushConsumer rocketMQConsumer; private DefaultMQPushConsumer rocketMQConsumer;
private BlockingQueue<ConsumerBatchMessage> messageBlockingQueue; private BlockingQueue<ConsumerBatchMessage> messageBlockingQueue;
private int batchSize = -1;
private long batchProcessTimeout = 60 * 1000; private long batchProcessTimeout = 60 * 1000;
private boolean flatMessage; private boolean flatMessage;
private volatile ConsumerBatchMessage lastGetBatchMessage = null; private volatile ConsumerBatchMessage lastGetBatchMessage = null;
private String accessKey; private String accessKey;
private String secretKey; private String secretKey;
public RocketMQCanalConnector(String nameServer, String topic, String groupName, Integer batchSize,
public RocketMQCanalConnector(String nameServer, String topic, String groupName, boolean flatMessage){ boolean flatMessage){
this.nameServer = nameServer; this.nameServer = nameServer;
this.topic = topic; this.topic = topic;
this.groupName = groupName; this.groupName = groupName;
this.flatMessage = flatMessage; this.flatMessage = flatMessage;
this.messageBlockingQueue = new LinkedBlockingQueue<>(1024); this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
this.batchSize = batchSize;
} }
public RocketMQCanalConnector(String nameServer, String topic, String groupName, public RocketMQCanalConnector(String nameServer, String topic, String groupName, String accessKey,
String accessKey, String secretKey, boolean flatMessage){ String secretKey, Integer batchSize, boolean flatMessage){
this(nameServer, topic, groupName, flatMessage); this(nameServer, topic, groupName, batchSize, flatMessage);
this.accessKey = accessKey; this.accessKey = accessKey;
this.secretKey = secretKey; this.secretKey = secretKey;
} }
@@ -73,8 +75,7 @@ public class RocketMQCanalConnector implements CanalMQConnector {
public void connect() throws CanalClientException { public void connect() throws CanalClientException {
RPCHook rpcHook = null; RPCHook rpcHook = null;
if(null != accessKey && accessKey.length() > 0 if (null != accessKey && accessKey.length() > 0 && null != secretKey && secretKey.length() > 0) {
&& null != secretKey && secretKey.length() > 0){
SessionCredentials sessionCredentials = new SessionCredentials(); SessionCredentials sessionCredentials = new SessionCredentials();
sessionCredentials.setAccessKey(accessKey); sessionCredentials.setAccessKey(accessKey);
sessionCredentials.setSecretKey(secretKey); sessionCredentials.setSecretKey(secretKey);
@@ -85,6 +86,9 @@ public class RocketMQCanalConnector implements CanalMQConnector {
if (!StringUtils.isBlank(nameServer)) { if (!StringUtils.isBlank(nameServer)) {
rocketMQConsumer.setNamesrvAddr(nameServer); rocketMQConsumer.setNamesrvAddr(nameServer);
} }
if (batchSize != -1) {
rocketMQConsumer.setConsumeMessageBatchMaxSize(batchSize);
}
} }
public void disconnect() throws CanalClientException { public void disconnect() throws CanalClientException {

9
client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java
View File

@@ -99,8 +99,13 @@ public class CanalKafkaClientExample {
} }
private void process() { private void process() {
while (!running) while (!running) {
; try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
while (running) { while (running) {
try { try {
connector.connect(); connector.connect();

18
client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaOffsetClientExample.java
View File

@@ -1,14 +1,15 @@
package com.alibaba.otter.canal.client.running.kafka; package com.alibaba.otter.canal.client.running.kafka;
import com.alibaba.otter.canal.client.kafka.KafkaOffsetCanalConnector; import java.util.List;
import com.alibaba.otter.canal.client.kafka.protocol.KafkaMessage; import java.util.concurrent.TimeUnit;
import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.errors.WakeupException;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.util.Assert; import org.springframework.util.Assert;
import java.util.List; import com.alibaba.otter.canal.client.kafka.KafkaOffsetCanalConnector;
import java.util.concurrent.TimeUnit; import com.alibaba.otter.canal.client.kafka.protocol.KafkaMessage;
/** /**
* KafkaOffsetCanalConnector 使用示例 * KafkaOffsetCanalConnector 使用示例
@@ -99,8 +100,13 @@ public class CanalKafkaOffsetClientExample {
} }
private void process() { private void process() {
while (!running) while (!running) {
; try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
while (running) { while (running) {
try { try {
// 修改 AutoOffsetReset 的值，默认（earliest） // 修改 AutoOffsetReset 的值，默认（earliest）

11
client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/CanalRocketMQClientExample.java
View File

@@ -36,7 +36,7 @@ public class CanalRocketMQClientExample extends AbstractRocektMQTest {
}; };
public CanalRocketMQClientExample(String nameServers, String topic, String groupId){ public CanalRocketMQClientExample(String nameServers, String topic, String groupId){
connector = new RocketMQCanalConnector(nameServers, topic, groupId, false); connector = new RocketMQCanalConnector(nameServers, topic, groupId, 500, false);
} }
public static void main(String[] args) { public static void main(String[] args) {
@@ -97,8 +97,13 @@ public class CanalRocketMQClientExample extends AbstractRocektMQTest {
} }
private void process() { private void process() {
while (!running) while (!running) {
; try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
while (running) { while (running) {
try { try {
connector.connect(); connector.connect();

6
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java
View File

@@ -32,7 +32,7 @@ public final class CharsetConversion {
} }
// Character set data used in lookups. The array will be sparse. // Character set data used in lookups. The array will be sparse.
static final Entry[] entries = new Entry[0xff]; static final Entry[] entries = new Entry[2048];
static Entry getEntry(final int id) { static Entry getEntry(final int id) {
if (id >= 0 && id < entries.length) { if (id >= 0 && id < entries.length) {
@@ -95,7 +95,7 @@ public final class CharsetConversion {
putEntry(34, "cp1250", "cp1250_czech_cs", "Cp1250"); putEntry(34, "cp1250", "cp1250_czech_cs", "Cp1250");
putEntry(35, "ucs2", "ucs2_general_ci", "UnicodeBig"); putEntry(35, "ucs2", "ucs2_general_ci", "UnicodeBig");
putEntry(36, "cp866", "cp866_general_ci", "Cp866"); putEntry(36, "cp866", "cp866_general_ci", "Cp866");
putEntry(37, "keybcs2", "keybcs2_general_ci", "Cp895"); putEntry(37, "keybcs2", "keybcs2_general_ci", "Cp852");
putEntry(38, "macce", "macce_general_ci", "MacCentralEurope"); putEntry(38, "macce", "macce_general_ci", "MacCentralEurope");
putEntry(39, "macroman", "macroman_general_ci", "MacRoman"); putEntry(39, "macroman", "macroman_general_ci", "MacRoman");
putEntry(40, "cp852", "cp852_general_ci", "Cp852"); putEntry(40, "cp852", "cp852_general_ci", "Cp852");
@@ -129,7 +129,7 @@ public final class CharsetConversion {
putEntry(70, "greek", "greek_bin", "ISO8859_7"); putEntry(70, "greek", "greek_bin", "ISO8859_7");
putEntry(71, "hebrew", "hebrew_bin", "ISO8859_8"); putEntry(71, "hebrew", "hebrew_bin", "ISO8859_8");
putEntry(72, "hp8", "hp8_bin", "US-ASCII"); putEntry(72, "hp8", "hp8_bin", "US-ASCII");
putEntry(73, "keybcs2", "keybcs2_bin", "Cp895"); putEntry(73, "keybcs2", "keybcs2_bin", "Cp852");
putEntry(74, "koi8r", "koi8r_bin", "KOI8_R"); putEntry(74, "koi8r", "koi8r_bin", "KOI8_R");
putEntry(75, "koi8u", "koi8u_bin", "KOI8_U"); putEntry(75, "koi8u", "koi8u_bin", "KOI8_U");
putEntry(77, "latin2", "latin2_bin", "ISO8859_2"); putEntry(77, "latin2", "latin2_bin", "ISO8859_2");

23
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/ManagerRemoteConfigMonitor.java
View File

@@ -4,8 +4,16 @@ import java.io.ByteArrayInputStream;
import java.io.File; import java.io.File;
import java.io.FileWriter; import java.io.FileWriter;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.sql.*; import java.sql.Connection;
import java.util.*; import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@@ -75,8 +83,8 @@ public class ManagerRemoteConfigMonitor {
overrideLocalCanalConfig(configItem.getContent()); overrideLocalCanalConfig(configItem.getContent());
properties = new Properties(); properties = new Properties();
properties.load(new ByteArrayInputStream(configItem.getContent().getBytes(StandardCharsets.UTF_8))); properties.load(new ByteArrayInputStream(configItem.getContent().getBytes(StandardCharsets.UTF_8)));
scanIntervalInSecond = Integer scanIntervalInSecond = Integer.valueOf(properties.getProperty(CanalConstants.CANAL_AUTO_SCAN_INTERVAL,
.valueOf(properties.getProperty(CanalConstants.CANAL_AUTO_SCAN_INTERVAL, "5")); "5"));
} }
} }
} catch (Exception e) { } catch (Exception e) {
@@ -223,8 +231,7 @@ public class ManagerRemoteConfigMonitor {
Map<String, ConfigItem> changedInstanceConfigs = modifiedInstanceConfigs[0]; Map<String, ConfigItem> changedInstanceConfigs = modifiedInstanceConfigs[0];
if (changedInstanceConfigs != null) { if (changedInstanceConfigs != null) {
for (ConfigItem configItem : changedInstanceConfigs.values()) { for (ConfigItem configItem : changedInstanceConfigs.values()) {
try (FileWriter writer = new FileWriter( try (FileWriter writer = new FileWriter(getConfPath() + configItem.getName() + "/instance.properties")) {
getConfPath() + configItem.getName() + "/instance.properties")) {
writer.write(configItem.getContent()); writer.write(configItem.getContent());
writer.flush(); writer.flush();
} catch (Exception e) { } catch (Exception e) {
@@ -351,8 +358,8 @@ public class ManagerRemoteConfigMonitor {
} }
} }
public interface Listener<Properties> { public interface Listener<T> {
void onChange(Properties properties); void onChange(T properties);
} }
} }

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java
View File

@@ -160,7 +160,7 @@ public abstract class AbstractRequest<T> {
} }
public final String formatUTCTZ(Date date) { public final String formatUTCTZ(Date date) {
SimpleDateFormat sdf = new SimpleDateFormat("YYYY-MM-dd'T'HH:mm:ss'Z'"); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("UTC")); sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
return sdf.format(date); return sdf.format(date);
} }

82
server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java
View File

@@ -1,6 +1,12 @@
package com.alibaba.otter.canal.common; package com.alibaba.otter.canal.common;
import java.util.*; import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
@@ -24,46 +30,36 @@ import com.google.protobuf.InvalidProtocolBufferException;
public class MQMessageUtils { public class MQMessageUtils {
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private static Map<String, List<PartitionData>> partitionDatas = MigrateMap private static Map<String, List<PartitionData>> partitionDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
.makeComputingMap(new MapMaker().softValues(), new Function<String, List<PartitionData>>() { new Function<String, List<PartitionData>>() {
public List<PartitionData> apply(String pkHashConfigs) { public List<PartitionData> apply(String pkHashConfigs) {
List<PartitionData> datas = Lists List<PartitionData> datas = Lists.newArrayList();
.newArrayList(); String[] pkHashConfigArray = StringUtils.split(pkHashConfigs,
String[] pkHashConfigArray = StringUtils ",");
.split(pkHashConfigs, ",");
// schema.table:id^name // schema.table:id^name
for (String pkHashConfig : pkHashConfigArray) { for (String pkHashConfig : pkHashConfigArray) {
PartitionData data = new PartitionData(); PartitionData data = new PartitionData();
int i = pkHashConfig int i = pkHashConfig.lastIndexOf(":");
.lastIndexOf(":");
if (i > 0) { if (i > 0) {
String pkStr = pkHashConfig String pkStr = pkHashConfig.substring(i + 1);
.substring(i + 1); if (pkStr.equalsIgnoreCase("$pk$")) {
if (pkStr.equalsIgnoreCase(
"$pk$")) {
data.hashMode.autoPkHash = true; data.hashMode.autoPkHash = true;
} else { } else {
data.hashMode.pkNames = Lists data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,
.newArrayList(
StringUtils
.split(
pkStr,
'^')); '^'));
} }
pkHashConfig = pkHashConfig pkHashConfig = pkHashConfig.substring(0,
.substring(0, i); i);
} else { } else {
data.hashMode.tableHash = true; data.hashMode.tableHash = true;
} }
if (!isWildCard( if (!isWildCard(pkHashConfig)) {
pkHashConfig)) {
data.simpleName = pkHashConfig; data.simpleName = pkHashConfig;
} else { } else {
data.regexFilter = new AviaterRegexFilter( data.regexFilter = new AviaterRegexFilter(pkHashConfig);
pkHashConfig);
} }
datas.add(data); datas.add(data);
} }
@@ -73,29 +69,24 @@ public class MQMessageUtils {
}); });
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
.makeComputingMap(new MapMaker().softValues(), new Function<String, List<DynamicTopicData>>() { new Function<String, List<DynamicTopicData>>() {
public List<DynamicTopicData> apply(String pkHashConfigs) { public List<DynamicTopicData> apply(String pkHashConfigs) {
List<DynamicTopicData> datas = Lists List<DynamicTopicData> datas = Lists.newArrayList();
.newArrayList(); String[] dynamicTopicArray = StringUtils.split(pkHashConfigs,
String[] dynamicTopicArray = StringUtils ",");
.split(pkHashConfigs, ",");
// schema.table // schema.table
for (String dynamicTopic : dynamicTopicArray) { for (String dynamicTopic : dynamicTopicArray) {
DynamicTopicData data = new DynamicTopicData(); DynamicTopicData data = new DynamicTopicData();
if (!isWildCard( if (!isWildCard(dynamicTopic)) {
dynamicTopic)) {
data.simpleName = dynamicTopic; data.simpleName = dynamicTopic;
} else { } else {
if (dynamicTopic if (dynamicTopic.contains("\\.")) {
.contains("\\.")) { data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
data.tableRegexFilter = new AviaterRegexFilter(
dynamicTopic);
} else { } else {
data.schemaRegexFilter = new AviaterRegexFilter( data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
dynamicTopic);
} }
} }
datas.add(data); datas.add(data);
@@ -232,6 +223,9 @@ public class MQMessageUtils {
partitionEntries[pkHash].add(entry); partitionEntries[pkHash].add(entry);
} }
} }
} else {
// 针对stmt/mixed binlog格式的query事件
partitionEntries[0].add(entry);
} }
} }
} }
@@ -281,9 +275,8 @@ public class MQMessageUtils {
try { try {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException( throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:"
"ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), + entry.toString(), e);
e);
} }
CanalEntry.EventType eventType = rowChange.getEventType(); CanalEntry.EventType eventType = rowChange.getEventType();
@@ -455,6 +448,9 @@ public class MQMessageUtils {
idx++; idx++;
} }
} }
} else {
// 针对stmt/mixed binlog格式的query事件
partitionMessages[0] = flatMessage;
} }
} }
return partitionMessages; return partitionMessages;
@@ -524,8 +520,8 @@ public class MQMessageUtils {
private static boolean isWildCard(String value) { private static boolean isWildCard(String value) {
// not contaiins '.' ? // not contaiins '.' ?
return StringUtils.containsAny(value, return StringUtils.containsAny(value, new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$',
new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$', '^' }); '^' });
} }
private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName, private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName,

7
sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java
View File

@@ -100,11 +100,12 @@ public class EntryEventSink extends AbstractCanalEventSink<List<CanalEntry.Entry
&& (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND)) { && (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND)) {
long currentTimestamp = entry.getHeader().getExecuteTime(); long currentTimestamp = entry.getHeader().getExecuteTime();
// 基于一定的策略控制，放过空的事务头和尾，便于及时更新数据库位点，表明工作正常 // 基于一定的策略控制，放过空的事务头和尾，便于及时更新数据库位点，表明工作正常
if (Math.abs(currentTimestamp - lastTransactionTimestamp) > emptyTransactionInterval if (lastTransactionCount.incrementAndGet() <= emptyTransctionThresold
|| lastTransactionCount.incrementAndGet() > emptyTransctionThresold) { && Math.abs(currentTimestamp - lastTransactionTimestamp) <= emptyTransactionInterval) {
continue;
} else {
lastTransactionCount.set(0L); lastTransactionCount.set(0L);
lastTransactionTimestamp = currentTimestamp; lastTransactionTimestamp = currentTimestamp;
continue;
} }
} }