fixed issue #1374 , support queryLogEvent partition

增加adapter远程配置

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java

@@ -75,8 +75,8 @@ public class ESAdapter implements OuterAdapter {
             // 过滤不匹配的key的配置
             esSyncConfigTmp.forEach((key, config) -> {
                 if ((config.getOuterAdapterKey() == null && configuration.getKey() == null)
-                    || (config.getOuterAdapterKey() != null && config.getOuterAdapterKey()
+                    || (config.getOuterAdapterKey() != null
-                        .equalsIgnoreCase(configuration.getKey()))) {
+                        && config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
                     esSyncConfig.put(key, config);
                 }
             });
@@ -98,13 +98,9 @@ public class ESAdapter implements OuterAdapter {
                 }
                 String schema = matcher.group(2);
 
-                schemaItem.getAliasTableItems()
+                schemaItem.getAliasTableItems().values().forEach(tableItem -> {
-                    .values()
+                    Map<String, ESSyncConfig> esSyncConfigMap = dbTableEsSyncConfig
-                    .forEach(tableItem -> {
+                        .computeIfAbsent(schema + "-" + tableItem.getTableName(), k -> new HashMap<>());
-                        Map<String, ESSyncConfig> esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(schema
-                                                                                                        + "-"
-                                                                                                        + tableItem.getTableName(),
-                            k -> new HashMap<>());
                     esSyncConfigMap.put(configName, config);
                 });
             }
@@ -140,8 +136,10 @@ public class ESAdapter implements OuterAdapter {
         String database = dml.getDatabase();
         String table = dml.getTable();
         Map<String, ESSyncConfig> configMap = dbTableEsSyncConfig.get(database + "-" + table);
+        if (configMap != null) {
             esSyncService.sync(configMap.values(), dml);
         }
+    }
 
     @Override
     public EtlResult etl(String task, List<String> params) {

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/monitor/ESConfigMonitor.java

@@ -84,6 +84,10 @@ public class ESConfigMonitor {
                     // 加载配置文件
                     String configContent = MappingConfigsLoader
                         .loadConfig(adapterName + File.separator + file.getName());
+                    if (configContent == null) {
+                        onFileDelete(file);
+                        return;
+                    }
                     ESSyncConfig config = new Yaml().loadAs(configContent, ESSyncConfig.class);
                     config.validate();
                     if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java

@@ -100,9 +100,9 @@ public class ESSyncService {
                     dml.getDestination(),
                     config.getEsMapping().get_index());
             }
-        } catch (Exception e) {
+        } catch (Throwable e) {
             logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml);
-            logger.error(e.getMessage(), e);
+            throw new RuntimeException(e);
         }
     }
 

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java

@@ -13,8 +13,6 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
@@ -30,11 +28,7 @@ import com.alibaba.otter.canal.client.adapter.hbase.monitor.HbaseConfigMonitor;
 import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseEtlService;
 import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseSyncService;
 import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.*;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
-import com.alibaba.otter.canal.client.adapter.support.SPI;
 
 /**
  * HBase外部适配器
@@ -50,7 +44,6 @@ public class HbaseAdapter implements OuterAdapter {
     private Map<String, MappingConfig>              hbaseMapping       = new ConcurrentHashMap<>();                  // 文件名对应配置
     private Map<String, Map<String, MappingConfig>> mappingConfigCache = new ConcurrentHashMap<>();                  // 库名-表名对应配置
 
-    private Connection                              conn;
     private HbaseSyncService                        hbaseSyncService;
     private HbaseTemplate                           hbaseTemplate;
 
@@ -90,8 +83,7 @@ public class HbaseAdapter implements OuterAdapter {
 
             Configuration hbaseConfig = HBaseConfiguration.create();
             properties.forEach(hbaseConfig::set);
-            conn = ConnectionFactory.createConnection(hbaseConfig);
+            hbaseTemplate = new HbaseTemplate(hbaseConfig);
-            hbaseTemplate = new HbaseTemplate(conn);
             hbaseSyncService = new HbaseSyncService(hbaseTemplate);
 
             configMonitor = new HbaseConfigMonitor();
@@ -115,8 +107,10 @@ public class HbaseAdapter implements OuterAdapter {
         String database = dml.getDatabase();
         String table = dml.getTable();
         Map<String, MappingConfig> configMap = mappingConfigCache.get(destination + "." + database + "." + table);
+        if (configMap != null) {
             configMap.values().forEach(config -> hbaseSyncService.sync(config, dml));
         }
+    }
 
     @Override
     public EtlResult etl(String task, List<String> params) {
@@ -172,7 +166,7 @@ public class HbaseAdapter implements OuterAdapter {
         String hbaseTable = config.getHbaseMapping().getHbaseTable();
         long rowCount = 0L;
         try {
-            HTable table = (HTable) conn.getTable(TableName.valueOf(hbaseTable));
+            HTable table = (HTable) hbaseTemplate.getConnection().getTable(TableName.valueOf(hbaseTable));
             Scan scan = new Scan();
             scan.setFilter(new FirstKeyOnlyFilter());
             ResultScanner resultScanner = table.getScanner(scan);
@@ -193,14 +187,12 @@ public class HbaseAdapter implements OuterAdapter {
         if (configMonitor != null) {
             configMonitor.destroy();
         }
-        if (conn != null) {
         try {
-                conn.close();
+            hbaseTemplate.close();
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
     }
-    }
 
     @Override
     public String getDestination(String task) {

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/monitor/HbaseConfigMonitor.java

@@ -79,6 +79,10 @@ public class HbaseConfigMonitor {
                     // 加载配置文件
                     String configContent = MappingConfigsLoader
                         .loadConfig(adapterName + File.separator + file.getName());
+                    if (configContent == null) {
+                        onFileDelete(file);
+                        return;
+                    }
                     MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class);
                     config.validate();
                     if (hbaseAdapter.getHbaseMapping().containsKey(file.getName())) {

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java

@@ -29,7 +29,6 @@ public class HbaseSyncService {
     }
 
     public void sync(MappingConfig config, Dml dml) {
-        try {
         if (config != null) {
             String type = dml.getType();
             if (type != null && type.equalsIgnoreCase("INSERT")) {
@@ -43,9 +42,6 @@ public class HbaseSyncService {
                 logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
             }
         }
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
     }
 
     /**

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/HbaseTemplate.java

@@ -5,10 +5,12 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
@@ -27,14 +29,31 @@ public class HbaseTemplate {
 
     private Logger        logger = LoggerFactory.getLogger(this.getClass());
 
-    private Connection conn;
+    private Configuration hbaseConfig;                                      // hbase配置对象
+    private Connection    conn;                                             // hbase连接
 
-    public HbaseTemplate(Connection conn){
+    public HbaseTemplate(Configuration hbaseConfig){
-        this.conn = conn;
+        this.hbaseConfig = hbaseConfig;
+        initConn();
+    }
+
+    private void initConn() {
+        try {
+            this.conn = ConnectionFactory.createConnection(hbaseConfig);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public Connection getConnection() {
+        if (conn == null || conn.isAborted() || conn.isClosed()) {
+            initConn();
+        }
+        return conn;
     }
 
     public boolean tableExists(String tableName) {
-        try (HBaseAdmin admin = (HBaseAdmin) conn.getAdmin()) {
+        try (HBaseAdmin admin = (HBaseAdmin) getConnection().getAdmin()) {
 
             return admin.tableExists(TableName.valueOf(tableName));
         } catch (IOException e) {
@@ -43,7 +62,7 @@ public class HbaseTemplate {
     }
 
     public void createTable(String tableName, String... familyNames) {
-        try (HBaseAdmin admin = (HBaseAdmin) conn.getAdmin()) {
+        try (HBaseAdmin admin = (HBaseAdmin) getConnection().getAdmin()) {
 
             HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
             // 添加列簇
@@ -60,7 +79,7 @@ public class HbaseTemplate {
     }
 
     public void disableTable(String tableName) {
-        try (HBaseAdmin admin = (HBaseAdmin) conn.getAdmin()) {
+        try (HBaseAdmin admin = (HBaseAdmin) getConnection().getAdmin()) {
             admin.disableTable(tableName);
         } catch (IOException e) {
             logger.error(e.getMessage(), e);
@@ -69,7 +88,7 @@ public class HbaseTemplate {
     }
 
     public void deleteTable(String tableName) {
-        try (HBaseAdmin admin = (HBaseAdmin) conn.getAdmin()) {
+        try (HBaseAdmin admin = (HBaseAdmin) getConnection().getAdmin()) {
             if (admin.isTableEnabled(tableName)) {
                 disableTable(tableName);
             }
@@ -90,7 +109,7 @@ public class HbaseTemplate {
     public Boolean put(String tableName, HRow hRow) {
         boolean flag = false;
         try {
-            HTable table = (HTable) conn.getTable(TableName.valueOf(tableName));
+            HTable table = (HTable) getConnection().getTable(TableName.valueOf(tableName));
             Put put = new Put(hRow.getRowKey());
             for (HRow.HCell hCell : hRow.getCells()) {
                 put.addColumn(Bytes.toBytes(hCell.getFamily()), Bytes.toBytes(hCell.getQualifier()), hCell.getValue());
@@ -99,6 +118,7 @@ public class HbaseTemplate {
             flag = true;
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
+            throw new RuntimeException(e);
         }
         return flag;
 
@@ -114,7 +134,7 @@ public class HbaseTemplate {
     public Boolean puts(String tableName, List<HRow> rows) {
         boolean flag = false;
         try {
-            HTable table = (HTable) conn.getTable(TableName.valueOf(tableName));
+            HTable table = (HTable) getConnection().getTable(TableName.valueOf(tableName));
             List<Put> puts = new ArrayList<>();
             for (HRow hRow : rows) {
                 Put put = new Put(hRow.getRowKey());
@@ -131,6 +151,7 @@ public class HbaseTemplate {
             flag = true;
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
+            throw new RuntimeException(e);
         }
         return flag;
     }
@@ -145,7 +166,7 @@ public class HbaseTemplate {
     public Boolean deletes(String tableName, Set<byte[]> rowKeys) {
         boolean flag = false;
         try {
-            HTable table = (HTable) conn.getTable(TableName.valueOf(tableName));
+            HTable table = (HTable) getConnection().getTable(TableName.valueOf(tableName));
             List<Delete> deletes = new ArrayList<>();
             for (byte[] rowKey : rowKeys) {
                 Delete delete = new Delete(rowKey);
@@ -157,7 +178,14 @@ public class HbaseTemplate {
             flag = true;
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
+            throw new RuntimeException(e);
         }
         return flag;
     }
+
+    public void close() throws IOException {
+        if (conn != null) {
+            conn.close();
+        }
+    }
 }

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/BootstrapConfiguration.java

@@ -0,0 +1,44 @@
+package com.alibaba.otter.canal.adapter.launcher.config;
+
+import javax.annotation.PostConstruct;
+
+import com.alibaba.otter.canal.adapter.launcher.monitor.AdapterRemoteConfigMonitor;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+
+/**
+ * Bootstrap级别配置加载
+ *
+ * @author rewerma @ 2019-01-05
+ * @version 1.0.0
+ */
+public class BootstrapConfiguration {
+
+    private static final Logger logger = LoggerFactory.getLogger(BootstrapConfiguration.class);
+
+    @Autowired
+    private Environment         env;
+
+    @PostConstruct
+    public void loadRemoteConfig() {
+        try {
+            // 加载远程配置
+            String jdbcUrl = env.getProperty("canal.manager.jdbc.url");
+            if (StringUtils.isNotEmpty(jdbcUrl)) {
+                String jdbcUsername = env.getProperty("canal.manager.jdbc.username");
+                String jdbcPassword = env.getProperty("canal.manager.jdbc.password");
+                AdapterRemoteConfigMonitor configMonitor = new AdapterRemoteConfigMonitor(jdbcUrl,
+                    jdbcUsername,
+                    jdbcPassword);
+                configMonitor.loadRemoteConfig();
+                configMonitor.loadRemoteAdapterConfigs();
+                configMonitor.start(); // 启动监听
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java

@@ -2,7 +2,6 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
@@ -77,19 +76,23 @@ public abstract class AbstractCanalAdapterWorker {
 
             // 等待所有适配器写入完成
             // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
-            futures.forEach(future -> {
+            RuntimeException exception = null;
+            for (Future<Boolean> future : futures) {
                 try {
                     if (!future.get()) {
-                        logger.error("Outer adapter write failed");
+                        exception = new RuntimeException("Outer adapter sync failed! ");
                     }
-                } catch (InterruptedException | ExecutionException e) {
+                } catch (Exception e) {
-                    // ignore
+                    exception = new RuntimeException(e);
+                }
+            }
+            if (exception != null) {
+                throw exception;
             }
-            });
         });
     }
 
-    private void writeOut(final List<FlatMessage> flatMessages) {
+    protected void writeOut(final List<FlatMessage> flatMessages) {
         List<Future<Boolean>> futures = new ArrayList<>();
         // 组间适配器并行运行
         canalOuterAdapters.forEach(outerAdapters -> {
@@ -116,22 +119,25 @@ public abstract class AbstractCanalAdapterWorker {
 
             // 等待所有适配器写入完成
             // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
-            futures.forEach(future -> {
+            RuntimeException exception = null;
+            for (Future<Boolean> future : futures) {
                 try {
                     if (!future.get()) {
-                        logger.error("Outer adapter write failed");
+                        exception = new RuntimeException("Outer adapter sync failed! ");
                     }
-                } catch (InterruptedException | ExecutionException e) {
+                } catch (Exception e) {
-                    // ignore
+                    exception = new RuntimeException(e);
+                }
+            }
+            if (exception != null) {
+                throw exception;
             }
-            });
         });
     }
 
     @SuppressWarnings("unchecked")
-    protected void mqWriteOutData(int retry, long timeout, final boolean flatMessage, CanalMQConnector connector,
+    protected boolean mqWriteOutData(int retry, long timeout, int i, final boolean flatMessage,
-                                  ExecutorService workerExecutor) {
+                                     CanalMQConnector connector, ExecutorService workerExecutor) {
-        for (int i = 0; i < retry; i++) {
         try {
             List<?> messages;
             if (!flatMessage) {
@@ -139,7 +145,7 @@ public abstract class AbstractCanalAdapterWorker {
             } else {
                 messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS);
             }
-                if (messages != null) {
+            if (messages != null && !messages.isEmpty()) {
                 Future<Boolean> future = workerExecutor.submit(() -> {
                     if (flatMessage) {
                         // batch write
@@ -158,24 +164,25 @@ public abstract class AbstractCanalAdapterWorker {
                     future.cancel(true);
                     throw e;
                 }
-                }
                 connector.ack();
-                break;
+            }
+            return true;
         } catch (Throwable e) {
             if (i == retry - 1) {
                 connector.ack();
+                logger.error(e.getMessage() + " Error sync but ACK!");
+                return true;
             } else {
                 connector.rollback();
+                logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
             }
-
-                logger.error(e.getMessage(), e);
             try {
-                    TimeUnit.SECONDS.sleep(1L);
+                Thread.sleep(500);
             } catch (InterruptedException e1) {
                 // ignore
             }
         }
-        }
+        return false;
     }
 
     /**

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java

@@ -3,8 +3,11 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.protocol.Message;
 import org.apache.kafka.common.errors.WakeupException;
 
 import com.alibaba.otter.canal.client.adapter.OuterAdapter;
@@ -61,7 +64,17 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
                         connector.disconnect();
                         break;
                     }
-                    mqWriteOutData(retry, timeout, flatMessage, connector, workerExecutor);
+                    if (retry == -1) {
+                        retry = Integer.MAX_VALUE;
+                    }
+                    for (int i = 0; i < retry; i++) {
+                        if (!running) {
+                            break;
+                        }
+                        if (mqWriteOutData(retry, timeout, i, flatMessage, connector, workerExecutor)) {
+                            break;
+                        }
+                    }
                 }
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java

@@ -3,7 +3,11 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.protocol.Message;
 import org.apache.kafka.common.errors.WakeupException;
 
 import com.alibaba.otter.canal.client.adapter.OuterAdapter;
@@ -39,7 +43,8 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
             ;
 
         ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
-        int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
+        int retry = canalClientConfig.getRetries() == null
+                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
         long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
 
         while (running) {
@@ -56,7 +61,17 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
                         connector.disconnect();
                         break;
                     }
-                    mqWriteOutData(retry, timeout, flatMessage, connector, workerExecutor);
+                    if (retry == -1) {
+                        retry = Integer.MAX_VALUE;
+                    }
+                    for (int i = 0; i < retry; i++) {
+                        if (!running) {
+                            break;
+                        }
+                        if (mqWriteOutData(retry, timeout, i, flatMessage, connector, workerExecutor)) {
+                            break;
+                        }
+                    }
                 }
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java

@@ -4,9 +4,13 @@ import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import javax.annotation.Resource;
 
+import com.alibaba.otter.canal.adapter.launcher.monitor.AdapterRemoteConfigMonitor;
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.cloud.context.config.annotation.RefreshScope;
+import org.springframework.cloud.context.refresh.ContextRefresher;
+import org.springframework.core.env.Environment;
 import org.springframework.stereotype.Component;
 
 import com.alibaba.druid.pool.DruidDataSource;
@@ -29,8 +33,13 @@ public class CanalAdapterService {
 
     private CanalAdapterLoader         adapterLoader;
 
+    @Resource
+    private ContextRefresher           contextRefresher;
+
     @Resource
     private AdapterCanalConfig         adapterCanalConfig;
+    @Resource
+    private Environment                env;
 
     // 注入bean保证优先注册
     @Resource
@@ -40,6 +49,8 @@ public class CanalAdapterService {
 
     private volatile boolean           running       = false;
 
+    private AdapterRemoteConfigMonitor configMonitor = null;
+
     @PostConstruct
     public synchronized void init() {
         if (running) {
@@ -64,6 +75,10 @@ public class CanalAdapterService {
         try {
             running = false;
             logger.info("## stop the canal client adapters");
+            if (configMonitor != null) {
+                configMonitor.destroy();
+            }
+
             if (adapterLoader != null) {
                 adapterLoader.destroy();
                 adapterLoader = null;

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java

@@ -61,10 +61,14 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
         while (!running)
             ; // waiting until running == true
 
-        ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
         int retry = canalClientConfig.getRetries() == null
                     || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
-        long timeout = canalClientConfig.getTimeout() == null ? 300000 : canalClientConfig.getTimeout(); // 默认超时5分钟
+        if (retry == -1) {
+            // 重试次数-1代表异常时一直阻塞重试
+            retry = Integer.MAX_VALUE;
+        }
+        // long timeout = canalClientConfig.getTimeout() == null ? 300000 :
+        // canalClientConfig.getTimeout(); // 默认超时5分钟
         Integer batchSize = canalClientConfig.getBatchSize();
         if (batchSize == null) {
             batchSize = BATCH_SIZE;
@@ -90,6 +94,9 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
                     }
 
                     for (int i = 0; i < retry; i++) {
+                        if (!running) {
+                            break;
+                        }
                         Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
                         long batchId = message.getId();
                         try {
@@ -97,7 +104,6 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
                             if (batchId == -1 || size == 0) {
                                 Thread.sleep(500);
                             } else {
-                                Future<Boolean> future = workerExecutor.submit(() -> {
                                 if (logger.isDebugEnabled()) {
                                     logger.debug("destination: {} batchId: {} batchSize: {} ",
                                         canalDestination,
@@ -112,25 +118,17 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
                                         batchId,
                                         System.currentTimeMillis() - begin);
                                 }
-                                    return true;
-                                });
-
-                                try {
-                                    future.get(timeout, TimeUnit.MILLISECONDS);
-                                } catch (Exception e) {
-                                    future.cancel(true);
-                                    throw e;
-                                }
                             }
                             connector.ack(batchId); // 提交确认
                             break;
                         } catch (Exception e) {
                             if (i != retry - 1) {
                                 connector.rollback(batchId); // 处理失败, 回滚数据
+                                logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
                             } else {
                                 connector.ack(batchId);
+                                logger.error(e.getMessage() + " Error sync but ACK!");
                             }
-                            logger.error("sync error!", e);
                             Thread.sleep(500);
                         }
                     }
@@ -151,8 +149,6 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
                 }
             }
         }
-
-        workerExecutor.shutdown();
     }
 
     @Override

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/AdapterRemoteConfigMonitor.java

@@ -0,0 +1,342 @@
+package com.alibaba.otter.canal.adapter.launcher.monitor;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.net.URL;
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+
+import com.alibaba.otter.canal.client.adapter.support.Constant;
+import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
+import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
+import com.google.common.base.Joiner;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 远程配置装载、监控类
+ *
+ * @author rewerma @ 2019-01-05
+ * @version 1.0.0
+ */
+public class AdapterRemoteConfigMonitor {
+
+    private static final Logger      logger                 = LoggerFactory.getLogger(AdapterRemoteConfigMonitor.class);
+
+    private Connection               conn;
+    private String                   jdbcUrl;
+    private String                   jdbcUsername;
+    private String                   jdbcPassword;
+
+    private long                     currentConfigTimestamp = 0;
+    private Map<String, ConfigItem>  remoteAdapterConfigs   = new MapMaker().makeMap();
+
+    private ScheduledExecutorService executor               = Executors.newScheduledThreadPool(2,
+        new NamedThreadFactory("remote-adapter-config-scan"));
+
+    public AdapterRemoteConfigMonitor(String jdbcUrl, String jdbcUsername, String jdbcPassword){
+        this.jdbcUrl = jdbcUrl;
+        this.jdbcUsername = jdbcUsername;
+        this.jdbcPassword = jdbcPassword;
+    }
+
+    private Connection getConn() throws Exception {
+        if (conn == null || conn.isClosed()) {
+            Class.forName("com.mysql.jdbc.Driver");
+            conn = DriverManager.getConnection(jdbcUrl, jdbcUsername, jdbcPassword);
+        }
+        return conn;
+    }
+
+    /**
+     * 加载远程application.yml配置到本地
+     */
+    public void loadRemoteConfig() {
+        try {
+            // 加载远程adapter配置
+            ConfigItem configItem = getRemoteAdapterConfig();
+            if (configItem != null) {
+                if (configItem.getModifiedTime() != currentConfigTimestamp) {
+                    currentConfigTimestamp = configItem.getModifiedTime();
+                    overrideLocalCanalConfig(configItem.getContent());
+                }
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 获取远程application.yml配置
+     * 
+     * @return 配置对象
+     */
+    private ConfigItem getRemoteAdapterConfig() {
+        String sql = "select name, content, modified_time from canal_config where id=2";
+        try (Statement stmt = getConn().createStatement(); ResultSet rs = stmt.executeQuery(sql)) {
+            if (rs.next()) {
+                ConfigItem configItem = new ConfigItem();
+                configItem.setId(2L);
+                configItem.setName(rs.getString("name"));
+                configItem.setContent(rs.getString("content"));
+                configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
+                return configItem;
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+        return null;
+    }
+
+    /**
+     * 覆盖本地application.yml文件
+     * 
+     * @param content 文件内容
+     */
+    private void overrideLocalCanalConfig(String content) {
+        try (FileWriter writer = new FileWriter(getConfPath() + "application.yml")) {
+            writer.write(content);
+            writer.flush();
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 启动监听数据库变化
+     */
+    public void start() {
+        // 监听application.yml变化
+        executor.scheduleWithFixedDelay(() -> {
+            try {
+                loadRemoteConfig();
+            } catch (Throwable e) {
+                logger.error("scan remote application.yml failed", e);
+            }
+        }, 10, 3, TimeUnit.SECONDS);
+
+        // 监听adapter变化
+        executor.scheduleWithFixedDelay(() -> {
+            try {
+                loadRemoteAdapterConfigs();
+            } catch (Throwable e) {
+                logger.error("scan remote adapter configs failed", e);
+            }
+        }, 10, 3, TimeUnit.SECONDS);
+    }
+
+    /**
+     * 加载adapter配置到本地
+     */
+    public void loadRemoteAdapterConfigs() {
+        try {
+            // 加载远程instance配置
+            Map<String, ConfigItem>[] modifiedConfigs = getModifiedAdapterConfigs();
+            if (modifiedConfigs != null) {
+                overrideLocalAdapterConfigs(modifiedConfigs);
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 获取有变动的adapter配置
+     * 
+     * @return Map[0]: 新增修改的配置, Map[1]: 删除的配置
+     */
+    @SuppressWarnings("unchecked")
+    private Map<String, ConfigItem>[] getModifiedAdapterConfigs() {
+        Map<String, ConfigItem>[] res = new Map[2];
+        Map<String, ConfigItem> remoteConfigStatus = new HashMap<>();
+        String sql = "select id, category, name, modified_time from canal_adapter_config";
+        try (Statement stmt = getConn().createStatement(); ResultSet rs = stmt.executeQuery(sql)) {
+            while (rs.next()) {
+                ConfigItem configItem = new ConfigItem();
+                configItem.setId(rs.getLong("id"));
+                configItem.setCategory(rs.getString("category"));
+                configItem.setName(rs.getString("name"));
+                configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
+                remoteConfigStatus.put(configItem.getCategory() + "/" + configItem.getName(), configItem);
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            return null;
+        }
+
+        if (!remoteConfigStatus.isEmpty()) {
+            List<Long> changedIds = new ArrayList<>();
+
+            for (ConfigItem remoteConfigStat : remoteConfigStatus.values()) {
+                ConfigItem currentConfig = remoteAdapterConfigs
+                    .get(remoteConfigStat.getCategory() + "/" + remoteConfigStat.getName());
+                if (currentConfig == null) {
+                    // 新增
+                    changedIds.add(remoteConfigStat.getId());
+                } else {
+                    // 修改
+                    if (currentConfig.getModifiedTime() != remoteConfigStat.getModifiedTime()) {
+                        changedIds.add(remoteConfigStat.getId());
+                    }
+                }
+            }
+            if (!changedIds.isEmpty()) {
+                Map<String, ConfigItem> changedInstanceConfig = new HashMap<>();
+                String contentsSql = "select id, category, name, content, modified_time from canal_adapter_config  where id in ("
+                                     + Joiner.on(",").join(changedIds) + ")";
+                try (Statement stmt = getConn().createStatement(); ResultSet rs = stmt.executeQuery(contentsSql)) {
+                    while (rs.next()) {
+                        ConfigItem configItemNew = new ConfigItem();
+                        configItemNew.setId(rs.getLong("id"));
+                        configItemNew.setCategory(rs.getString("category"));
+                        configItemNew.setName(rs.getString("name"));
+                        configItemNew.setContent(rs.getString("content"));
+                        configItemNew.setModifiedTime(rs.getTimestamp("modified_time").getTime());
+
+                        remoteAdapterConfigs.put(configItemNew.getCategory() + "/" + configItemNew.getName(),
+                            configItemNew);
+                        changedInstanceConfig.put(configItemNew.getCategory() + "/" + configItemNew.getName(),
+                            configItemNew);
+                    }
+
+                    res[0] = changedInstanceConfig.isEmpty() ? null : changedInstanceConfig;
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+        }
+
+        Map<String, ConfigItem> removedInstanceConfig = new HashMap<>();
+        for (ConfigItem configItem : remoteAdapterConfigs.values()) {
+            if (!remoteConfigStatus.containsKey(configItem.getCategory() + "/" + configItem.getName())) {
+                // 删除
+                remoteAdapterConfigs.remove(configItem.getCategory() + "/" + configItem.getName());
+                removedInstanceConfig.put(configItem.getCategory() + "/" + configItem.getName(), null);
+            }
+        }
+        res[1] = removedInstanceConfig.isEmpty() ? null : removedInstanceConfig;
+
+        if (res[0] == null && res[1] == null) {
+            return null;
+        } else {
+            return res;
+        }
+    }
+
+    /**
+     * 覆盖adapter配置到本地
+     * 
+     * @param modifiedInstanceConfigs 变动的配置集合
+     */
+    private void overrideLocalAdapterConfigs(Map<String, ConfigItem>[] modifiedInstanceConfigs) {
+        Map<String, ConfigItem> changedInstanceConfigs = modifiedInstanceConfigs[0];
+        if (changedInstanceConfigs != null) {
+            for (ConfigItem configItem : changedInstanceConfigs.values()) {
+                try (FileWriter writer = new FileWriter(
+                    getConfPath() + configItem.getCategory() + "/" + configItem.getName())) {
+                    writer.write(configItem.getContent());
+                    writer.flush();
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+        }
+        Map<String, ConfigItem> removedInstanceConfigs = modifiedInstanceConfigs[1];
+        if (removedInstanceConfigs != null) {
+            for (String name : removedInstanceConfigs.keySet()) {
+                File file = new File(getConfPath() + name);
+                if (file.exists()) {
+                    file.delete();
+                }
+            }
+        }
+    }
+
+    /**
+     * 获取conf文件夹所在路径
+     *
+     * @return 路径地址
+     */
+    private String getConfPath() {
+        String classpath = this.getClass().getResource("/").getPath();
+        String confPath = classpath + "../conf/";
+        if (new File(confPath).exists()) {
+            return confPath;
+        } else {
+            return classpath;
+        }
+    }
+
+    public void destroy() {
+        executor.shutdownNow();
+        if (conn != null) {
+            try {
+                conn.close();
+            } catch (SQLException e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+    }
+
+    /**
+     * 配置对应对象
+     */
+    public static class ConfigItem {
+
+        private Long   id;
+        private String category;
+        private String name;
+        private String content;
+        private long   modifiedTime;
+
+        public Long getId() {
+            return id;
+        }
+
+        public void setId(Long id) {
+            this.id = id;
+        }
+
+        public String getCategory() {
+            return category;
+        }
+
+        public void setCategory(String category) {
+            this.category = category;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        public String getContent() {
+            return content;
+        }
+
+        public void setContent(String content) {
+            this.content = content;
+        }
+
+        public long getModifiedTime() {
+            return modifiedTime;
+        }
+
+        public void setModifiedTime(long modifiedTime) {
+            this.modifiedTime = modifiedTime;
+        }
+    }
+}

client-adapter/launcher/src/main/resources/META-INF/spring.factories

@@ -0,0 +1,3 @@
+# Bootstrap Configuration
+org.springframework.cloud.bootstrap.BootstrapConfiguration=\
+  com.alibaba.otter.canal.adapter.launcher.config.BootstrapConfiguration

client-adapter/launcher/src/main/resources/application.yml

@@ -39,7 +39,7 @@ canal.conf:
 #        key: mysql1
 #        properties:
 #          jdbc.driverClassName: com.mysql.jdbc.Driver
-#          jdbc.url: jdbc:mysql://192.168.0.36/mytest?useUnicode=true
+#          jdbc.url: jdbc:mysql://127.0.0.1:3306/mytest2?useUnicode=true
 #          jdbc.username: root
 #          jdbc.password: 121212
 #      - name: rdb

client-adapter/launcher/src/main/resources/bootstrap.yml

@@ -0,0 +1,6 @@
+#canal:
+#  manager:
+#    jdbc:
+#      url: jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8
+#      username: root
+#      password: 121212

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java

@@ -5,10 +5,14 @@ import java.sql.SQLException;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
 import javax.sql.DataSource;
 
+import org.apache.commons.lang.BooleanUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -23,7 +27,12 @@ import com.alibaba.otter.canal.client.adapter.rdb.service.RdbEtlService;
 import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService;
 import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService;
 import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil;
-import com.alibaba.otter.canal.client.adapter.support.*;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+import com.alibaba.otter.canal.client.adapter.support.SPI;
+import com.alibaba.otter.canal.client.adapter.support.Util;
 
 /**
  * RDB适配器实现类
@@ -70,8 +79,8 @@ public class RdbAdapter implements OuterAdapter {
         // 过滤不匹配的key的配置
         rdbMappingTmp.forEach((key, mappingConfig) -> {
             if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
-                || (mappingConfig.getOuterAdapterKey() != null
+                || (mappingConfig.getOuterAdapterKey() != null && mappingConfig.getOuterAdapterKey()
-                    && mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
+                    .equalsIgnoreCase(configuration.getKey()))) {
                 rdbMapping.put(key, mappingConfig);
             }
         });
@@ -84,17 +93,17 @@ public class RdbAdapter implements OuterAdapter {
             String configName = entry.getKey();
             MappingConfig mappingConfig = entry.getValue();
             if (!mappingConfig.getDbMapping().getMirrorDb()) {
-                Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(
+                String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
-                    StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + mappingConfig.getDbMapping()
+                             + mappingConfig.getDbMapping().getDatabase() + "."
-                        .getDatabase() + "." + mappingConfig.getDbMapping().getTable(),
+                             + mappingConfig.getDbMapping().getTable();
+                Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key,
                     k1 -> new ConcurrentHashMap<>());
                 configMap.put(configName, mappingConfig);
             } else {
                 // mirrorDB
-
+                String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
-                mirrorDbConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
+                             + mappingConfig.getDbMapping().getDatabase();
-                                        + mappingConfig.getDbMapping().getDatabase(),
+                mirrorDbConfigCache.put(key, MirrorDbConfig.create(configName, mappingConfig));
-                    MirrorDbConfig.create(configName, mappingConfig));
             }
         }
 
@@ -107,10 +116,11 @@ public class RdbAdapter implements OuterAdapter {
         dataSource.setPassword(properties.get("jdbc.password"));
         dataSource.setInitialSize(1);
         dataSource.setMinIdle(1);
-        dataSource.setMaxActive(10);
+        dataSource.setMaxActive(30);
         dataSource.setMaxWait(60000);
         dataSource.setTimeBetweenEvictionRunsMillis(60000);
         dataSource.setMinEvictableIdleTimeMillis(300000);
+        dataSource.setUseUnfairLock(true);
 
         try {
             dataSource.init();
@@ -121,12 +131,17 @@ public class RdbAdapter implements OuterAdapter {
         String threads = properties.get("threads");
         // String commitSize = properties.get("commitSize");
 
-        rdbSyncService = new RdbSyncService(dataSource, threads != null ? Integer.valueOf(threads) : null);
+        boolean skipDupException = BooleanUtils.toBoolean(configuration.getProperties()
+            .getOrDefault("skipDupException", "true"));
+        rdbSyncService = new RdbSyncService(dataSource,
+            threads != null ? Integer.valueOf(threads) : null,
+            skipDupException);
 
         rdbMirrorDbSyncService = new RdbMirrorDbSyncService(mirrorDbConfigCache,
             dataSource,
             threads != null ? Integer.valueOf(threads) : null,
-            rdbSyncService.getColumnsTypeCache());
+            rdbSyncService.getColumnsTypeCache(),
+            skipDupException);
 
         rdbConfigMonitor = new RdbConfigMonitor();
         rdbConfigMonitor.init(configuration.getKey(), this);
@@ -152,8 +167,8 @@ public class RdbAdapter implements OuterAdapter {
         try {
             future1.get();
             future2.get();
-        } catch (ExecutionException | InterruptedException e) {
+        } catch (Exception e) {
-            // ignore
+            throw new RuntimeException(e);
         }
     }
 

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java

@@ -1,5 +1,7 @@
 package com.alibaba.otter.canal.client.adapter.rdb.config;
 
+import org.apache.commons.lang.StringUtils;
+
 import java.util.LinkedHashMap;
 import java.util.Map;
 
@@ -92,7 +94,7 @@ public class MappingConfig {
         private Map<String, String> allMapColumns;
 
         public Boolean getMirrorDb() {
-            return mirrorDb;
+            return mirrorDb == null ? false : mirrorDb;
         }
 
         public void setMirrorDb(Boolean mirrorDb) {
@@ -148,6 +150,13 @@ public class MappingConfig {
         }
 
         public Map<String, String> getTargetColumns() {
+            if (targetColumns != null) {
+                targetColumns.forEach((key, value) -> {
+                    if (StringUtils.isEmpty(value)) {
+                        targetColumns.put(key, key);
+                    }
+                });
+            }
             return targetColumns;
         }
 

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java

@@ -16,6 +16,7 @@ import org.yaml.snakeyaml.Yaml;
 
 import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter;
 import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.rdb.config.MirrorDbConfig;
 import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
 import com.alibaba.otter.canal.client.adapter.support.Util;
 
@@ -86,6 +87,10 @@ public class RdbConfigMonitor {
                     // 加载配置文件
                     String configContent = MappingConfigsLoader
                         .loadConfig(adapterName + File.separator + file.getName());
+                    if (configContent == null) {
+                        onFileDelete(file);
+                        return;
+                    }
                     MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class);
                     config.validate();
                     if ((key == null && config.getOuterAdapterKey() == null)
@@ -121,34 +126,44 @@ public class RdbConfigMonitor {
         }
 
         private void addConfigToCache(File file, MappingConfig mappingConfig) {
+            if (mappingConfig == null || mappingConfig.getDbMapping() == null) {
+                return;
+            }
             rdbAdapter.getRdbMapping().put(file.getName(), mappingConfig);
+            if (!mappingConfig.getDbMapping().getMirrorDb()) {
                 Map<String, MappingConfig> configMap = rdbAdapter.getMappingConfigCache()
                     .computeIfAbsent(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
                                      + mappingConfig.getDbMapping().getDatabase() + "."
                                      + mappingConfig.getDbMapping().getTable(),
                         k1 -> new HashMap<>());
                 configMap.put(file.getName(), mappingConfig);
-
+            } else {
                 Map<String, MirrorDbConfig> mirrorDbConfigCache = rdbAdapter.getMirrorDbConfigCache();
                 mirrorDbConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
                                         + mappingConfig.getDbMapping().getDatabase(),
                     MirrorDbConfig.create(file.getName(), mappingConfig));
             }
+        }
 
         private void deleteConfigFromCache(File file) {
+            MappingConfig mappingConfig = rdbAdapter.getRdbMapping().remove(file.getName());
 
-            rdbAdapter.getRdbMapping().remove(file.getName());
+            if (mappingConfig == null || mappingConfig.getDbMapping() == null) {
+                return;
+            }
+            if (!mappingConfig.getDbMapping().getMirrorDb()) {
                 for (Map<String, MappingConfig> configMap : rdbAdapter.getMappingConfigCache().values()) {
                     if (configMap != null) {
                         configMap.remove(file.getName());
                     }
                 }
-
+            } else {
                 rdbAdapter.getMirrorDbConfigCache().forEach((key, mirrorDbConfig) -> {
                     if (mirrorDbConfig.getFileName().equals(file.getName())) {
                         rdbAdapter.getMirrorDbConfigCache().remove(key);
                     }
                 });
+            }
 
         }
     }

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java

@@ -36,8 +36,7 @@ public class RdbEtlService {
     /**
      * 导入数据
      */
-    public static EtlResult importData(DataSource srcDS, DataSource targetDS, MappingConfig config,
+    public static EtlResult importData(DataSource srcDS, DataSource targetDS, MappingConfig config, List<String> params) {
-                                       List<String> params) {
         EtlResult etlResult = new EtlResult();
         AtomicLong successCount = new AtomicLong();
         List<String> errMsg = new ArrayList<>();
@@ -54,8 +53,8 @@ public class RdbEtlService {
             long start = System.currentTimeMillis();
 
             // 拼接sql
-            StringBuilder sql = new StringBuilder(
+            StringBuilder sql = new StringBuilder("SELECT * FROM " + dbMapping.getDatabase() + "."
-                "SELECT * FROM " + dbMapping.getDatabase() + "." + dbMapping.getTable());
+                                                  + dbMapping.getTable());
 
             // 拼接条件
             appendCondition(params, dbMapping, srcDS, sql);
@@ -92,8 +91,12 @@ public class RdbEtlService {
                     } else {
                         sqlFinal = sql + " LIMIT " + offset + "," + cnt;
                     }
-                    Future<Boolean> future = executor
+                    Future<Boolean> future = executor.submit(() -> executeSqlImport(srcDS,
-                        .submit(() -> executeSqlImport(srcDS, targetDS, sqlFinal, dbMapping, successCount, errMsg));
+                        targetDS,
+                        sqlFinal,
+                        dbMapping,
+                        successCount,
+                        errMsg));
                     futures.add(future);
                 }
 
@@ -106,10 +109,11 @@ public class RdbEtlService {
                 executeSqlImport(srcDS, targetDS, sql.toString(), dbMapping, successCount, errMsg);
             }
 
-            logger.info(
+            logger.info(dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000
-                dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!");
+                        + "s!");
 
-            etlResult.setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get() + " 条");
+            etlResult.setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get()
+                                       + " 条");
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
             errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
@@ -123,8 +127,8 @@ public class RdbEtlService {
         return etlResult;
     }
 
-    private static void appendCondition(List<String> params, DbMapping dbMapping, DataSource ds,
+    private static void appendCondition(List<String> params, DbMapping dbMapping, DataSource ds, StringBuilder sql)
-                                        StringBuilder sql) throws SQLException {
+                                                                                                                   throws SQLException {
         if (params != null && params.size() == 1 && dbMapping.getEtlCondition() == null) {
             AtomicBoolean stExists = new AtomicBoolean(false);
             // 验证是否有SYS_TIME字段
@@ -188,8 +192,7 @@ public class RdbEtlService {
 
                 StringBuilder insertSql = new StringBuilder();
                 insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
-                    columnsMap
+                columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(","));
-                        .forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(","));
 
                 int len = insertSql.length();
                 insertSql.delete(len - 1, len).append(") VALUES (");
@@ -208,8 +211,8 @@ public class RdbEtlService {
 
                         // 删除数据
                         Map<String, Object> values = new LinkedHashMap<>();
-                            StringBuilder deleteSql = new StringBuilder(
+                        StringBuilder deleteSql = new StringBuilder("DELETE FROM " + SyncUtil.getDbTableName(dbMapping)
-                                "DELETE FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE ");
+                                                                    + " WHERE ");
                         appendCondition(dbMapping, deleteSql, values, rs);
                         try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) {
                             int k = 1;
@@ -275,8 +278,8 @@ public class RdbEtlService {
     /**
      * 拼接目标表主键where条件
      */
-    private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values,
+    private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values, ResultSet rs)
-                                        ResultSet rs) throws SQLException {
+                                                                                                                         throws SQLException {
         // 拼接主键
         for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
             String targetColumnName = entry.getKey();

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java

@@ -35,10 +35,11 @@ public class RdbMirrorDbSyncService {
     private RdbSyncService              rdbSyncService;                                                // rdbSyncService代理
 
     public RdbMirrorDbSyncService(Map<String, MirrorDbConfig> mirrorDbConfigCache, DataSource dataSource,
-                                  Integer threads, Map<String, Map<String, Integer>> columnsTypeCache){
+                                  Integer threads, Map<String, Map<String, Integer>> columnsTypeCache,
+                                  boolean skipDupException){
         this.mirrorDbConfigCache = mirrorDbConfigCache;
         this.dataSource = dataSource;
-        this.rdbSyncService = new RdbSyncService(dataSource, threads, columnsTypeCache);
+        this.rdbSyncService = new RdbSyncService(dataSource, threads, columnsTypeCache, skipDupException);
     }
 
     /**
@@ -47,7 +48,6 @@ public class RdbMirrorDbSyncService {
      * @param dmls 批量 DML
      */
     public void sync(List<Dml> dmls) {
-        try {
         List<Dml> dmlList = new ArrayList<>();
         for (Dml dml : dmls) {
             String destination = StringUtils.trimToEmpty(dml.getDestination());
@@ -71,11 +71,13 @@ public class RdbMirrorDbSyncService {
             }
         }
         if (!dmlList.isEmpty()) {
-                rdbSyncService.sync(dmlList, dml -> {
+            rdbSyncService.sync(dmlList,
-                    MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache
+                dml -> {
-                        .get(dml.getDestination() + "." + dml.getDatabase());
+                    MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "."
-                    String destination = StringUtils.trimToEmpty(dml.getDestination());
+                                                                            + dml.getDatabase());
-                    String database = dml.getDatabase();
+                    if (mirrorDbConfig == null) {
+                        return false;
+                    }
                     String table = dml.getTable();
                     MappingConfig config = mirrorDbConfig.getTableConfig().get(table);
 
@@ -101,9 +103,6 @@ public class RdbMirrorDbSyncService {
                     return true;
                 });
         }
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
     }
 
     /**
@@ -152,7 +151,7 @@ public class RdbMirrorDbSyncService {
                 logger.trace("Execute DDL sql: {} for database: {}", ddl.getSql(), ddl.getDatabase());
             }
         } catch (Exception e) {
-            logger.error(e.getMessage(), e);
+            throw new RuntimeException(e);
         }
     }
 }

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java

@@ -8,6 +8,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
@@ -43,6 +44,7 @@ public class RdbSyncService {
     private Map<String, Map<String, Integer>> columnsTypeCache;
 
     private int                               threads = 3;
+    private boolean                           skipDupException;
 
     private List<SyncItem>[]                  dmlsPartition;
     private BatchExecutor[]                   batchExecutors;
@@ -57,13 +59,15 @@ public class RdbSyncService {
     }
 
     @SuppressWarnings("unchecked")
-    public RdbSyncService(DataSource dataSource, Integer threads){
+    public RdbSyncService(DataSource dataSource, Integer threads, boolean skipDupException){
-        this(dataSource, threads, new ConcurrentHashMap<>());
+        this(dataSource, threads, new ConcurrentHashMap<>(), skipDupException);
     }
 
     @SuppressWarnings("unchecked")
-    public RdbSyncService(DataSource dataSource, Integer threads, Map<String, Map<String, Integer>> columnsTypeCache){
+    public RdbSyncService(DataSource dataSource, Integer threads, Map<String, Map<String, Integer>> columnsTypeCache,
+                          boolean skipDupException){
         this.columnsTypeCache = columnsTypeCache;
+        this.skipDupException = skipDupException;
         try {
             if (threads != null) {
                 this.threads = threads;
@@ -77,7 +81,7 @@ public class RdbSyncService {
                 executorThreads[i] = Executors.newSingleThreadExecutor();
             }
         } catch (SQLException e) {
-            logger.error(e.getMessage(), e);
+            throw new RuntimeException(e);
         }
     }
 
@@ -88,7 +92,6 @@ public class RdbSyncService {
      * @param function 回调方法
      */
     public void sync(List<Dml> dmls, Function<Dml, Boolean> function) {
-        try {
         boolean toExecute = false;
         for (Dml dml : dmls) {
             if (!toExecute) {
@@ -102,27 +105,27 @@ public class RdbSyncService {
             for (int i = 0; i < threads; i++) {
                 int j = i;
                 futures.add(executorThreads[i].submit(() -> {
-                        dmlsPartition[j]
+                    try {
-                            .forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
+                        dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j],
+                            syncItem.config,
+                            syncItem.singleDml));
+                        dmlsPartition[j].clear();
                         batchExecutors[j].commit();
                         return true;
+                    } catch (Throwable e) {
+                        batchExecutors[j].rollback();
+                        throw new RuntimeException(e);
+                    }
                 }));
             }
 
             futures.forEach(future -> {
                 try {
                     future.get();
-                    } catch (Exception e) {
+                } catch (ExecutionException | InterruptedException e) {
-                        logger.error(e.getMessage(), e);
+                    throw new RuntimeException(e);
                 }
             });
-
-                for (int i = 0; i < threads; i++) {
-                    dmlsPartition[i].clear();
-                }
-            }
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
         }
     }
 
@@ -133,7 +136,6 @@ public class RdbSyncService {
      * @param dmls 批量 DML
      */
     public void sync(Map<String, Map<String, MappingConfig>> mappingConfig, List<Dml> dmls) {
-        try {
         sync(dmls, dml -> {
             if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) {
                 // DDL
@@ -144,8 +146,7 @@ public class RdbSyncService {
             String destination = StringUtils.trimToEmpty(dml.getDestination());
             String database = dml.getDatabase();
             String table = dml.getTable();
-                    Map<String, MappingConfig> configMap = mappingConfig
+            Map<String, MappingConfig> configMap = mappingConfig.get(destination + "." + database + "." + table);
-                        .get(destination + "." + database + "." + table);
 
             if (configMap == null) {
                 return false;
@@ -173,9 +174,6 @@ public class RdbSyncService {
             return executed;
         }
     }   );
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
     }
 
     /**
@@ -186,8 +184,8 @@ public class RdbSyncService {
      * @param dml DML
      */
     public void sync(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) {
-        try {
         if (config != null) {
+            try {
                 String type = dml.getType();
                 if (type != null && type.equalsIgnoreCase("INSERT")) {
                     insert(batchExecutor, config, dml);
@@ -199,9 +197,9 @@ public class RdbSyncService {
                 if (logger.isDebugEnabled()) {
                     logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
                 }
+            } catch (SQLException e) {
+                throw new RuntimeException(e);
             }
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
         }
     }
 
@@ -211,7 +209,7 @@ public class RdbSyncService {
      * @param config 配置项
      * @param dml DML数据
      */
-    private void insert(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) {
+    private void insert(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
         Map<String, Object> data = dml.getData();
         if (data == null || data.isEmpty()) {
             return;
@@ -219,7 +217,6 @@ public class RdbSyncService {
 
         DbMapping dbMapping = config.getDbMapping();
 
-        try {
         Map<String, String> columnsMap = SyncUtil.getColumnsMap(dbMapping, data);
 
         StringBuilder insertSql = new StringBuilder();
@@ -246,20 +243,28 @@ public class RdbSyncService {
             }
 
             Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
-
+            if (type == null) {
+                throw new RuntimeException("Target column: " + targetColumnName + " not matched");
+            }
             Object value = data.get(srcColumnName);
-
             BatchExecutor.setValue(values, type, value);
         }
 
+        try {
             batchExecutor.execute(insertSql.toString(), values);
+        } catch (SQLException e) {
+            if (skipDupException
+                && (e.getMessage().contains("Duplicate entry") || e.getMessage().startsWith("ORA-00001: 违反唯一约束条件"))) {
+                // ignore
+                // TODO 增加更多关系数据库的主键冲突的错误码
+            } else {
+                throw e;
+            }
+        }
         if (logger.isTraceEnabled()) {
             logger.trace("Insert into target table, sql: {}", insertSql);
         }
 
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
     }
 
     /**
@@ -268,7 +273,7 @@ public class RdbSyncService {
      * @param config 配置项
      * @param dml DML数据
      */
-    private void update(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) {
+    private void update(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
         Map<String, Object> data = dml.getData();
         if (data == null || data.isEmpty()) {
             return;
@@ -281,7 +286,6 @@ public class RdbSyncService {
 
         DbMapping dbMapping = config.getDbMapping();
 
-        try {
         Map<String, String> columnsMap = SyncUtil.getColumnsMap(dbMapping, data);
 
         Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
@@ -301,6 +305,9 @@ public class RdbSyncService {
                 for (String targetColumnName : targetColumnNames) {
                     updateSql.append(targetColumnName).append("=?, ");
                     Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
+                    if (type == null) {
+                        throw new RuntimeException("Target column: " + targetColumnName + " not matched");
+                    }
                     BatchExecutor.setValue(values, type, data.get(srcColumnName));
                 }
             }
@@ -310,15 +317,10 @@ public class RdbSyncService {
 
         // 拼接主键
         appendCondition(dbMapping, updateSql, ctype, values, data, old);
-
         batchExecutor.execute(updateSql.toString(), values);
-
         if (logger.isTraceEnabled()) {
             logger.trace("Update target table, sql: {}", updateSql);
         }
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
     }
 
     /**
@@ -327,7 +329,7 @@ public class RdbSyncService {
      * @param config
      * @param dml
      */
-    private void delete(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) {
+    private void delete(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
         Map<String, Object> data = dml.getData();
         if (data == null || data.isEmpty()) {
             return;
@@ -335,7 +337,6 @@ public class RdbSyncService {
 
         DbMapping dbMapping = config.getDbMapping();
 
-        try {
         Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
 
         StringBuilder sql = new StringBuilder();
@@ -344,15 +345,10 @@ public class RdbSyncService {
         List<Map<String, ?>> values = new ArrayList<>();
         // 拼接主键
         appendCondition(dbMapping, sql, ctype, values, data);
-
         batchExecutor.execute(sql.toString(), values);
-
         if (logger.isTraceEnabled()) {
             logger.trace("Delete from target table, sql: {}", sql);
         }
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
     }
 
     /**
@@ -410,6 +406,9 @@ public class RdbSyncService {
             }
             sql.append(targetColumnName).append("=? AND ");
             Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
+            if (type == null) {
+                throw new RuntimeException("Target column: " + targetColumnName + " not matched");
+            }
             // 如果有修改主键的情况
             if (o != null && o.containsKey(srcColumnName)) {
                 BatchExecutor.setValue(values, type, o.get(srcColumnName));

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/BatchExecutor.java

@@ -16,27 +16,12 @@ public class BatchExecutor implements Closeable {
 
     private static final Logger logger = LoggerFactory.getLogger(BatchExecutor.class);
 
-    private Integer             key;
     private Connection          conn;
     private AtomicInteger       idx    = new AtomicInteger(0);
 
-    public BatchExecutor(Connection conn){
+    public BatchExecutor(Connection conn) throws SQLException{
-        this(1, conn);
-    }
-
-    public BatchExecutor(Integer key, Connection conn){
-        this.key = key;
         this.conn = conn;
-
-        try {
         this.conn.setAutoCommit(false);
-        } catch (SQLException e) {
-            logger.error(e.getMessage(), e);
-        }
-    }
-
-    public Integer getKey() {
-        return key;
     }
 
     public Connection getConn() {
@@ -50,8 +35,8 @@ public class BatchExecutor implements Closeable {
         values.add(valueItem);
     }
 
-    public void execute(String sql, List<Map<String, ?>> values) {
+    public void execute(String sql, List<Map<String, ?>> values) throws SQLException {
-        try (PreparedStatement pstmt = conn.prepareStatement(sql)) {
+        PreparedStatement pstmt = conn.prepareStatement(sql);
         int len = values.size();
         for (int i = 0; i < len; i++) {
             int type = (Integer) values.get(i).get("type");
@@ -61,30 +46,32 @@ public class BatchExecutor implements Closeable {
 
         pstmt.execute();
         idx.incrementAndGet();
-        } catch (SQLException e) {
-            logger.error(e.getMessage(), e);
-        }
     }
 
-    public void commit() {
+    public void commit() throws SQLException {
-        try {
         conn.commit();
         if (logger.isTraceEnabled()) {
-                logger.trace("Batch executor: " + key + " commit " + idx.get() + " rows");
+            logger.trace("Batch executor commit " + idx.get() + " rows");
         }
         idx.set(0);
-        } catch (SQLException e) {
-            logger.error(e.getMessage(), e);
     }
+
+    public void rollback() throws SQLException {
+        conn.rollback();
+        if (logger.isTraceEnabled()) {
+            logger.trace("Batch executor rollback " + idx.get() + " rows");
+        }
+        idx.set(0);
     }
 
     @Override
     public void close() {
         if (conn != null) {
             try {
+                if (conn != null) {
                     conn.close();
-            } catch (SQLException e) {
+                }
-                logger.error(e.getMessage(), e);
+            } catch (SQLException ioe) {
             }
         }
     }

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java

@@ -86,9 +86,7 @@ public class SyncUtil {
                 }
                 break;
             case Types.TINYINT:
-                if (value instanceof Byte || value instanceof Short || value instanceof Integer) {
+                 if (value instanceof Number) {
-                    pstmt.setByte(i, (byte) value);
-                } else if (value instanceof Number) {
                     pstmt.setByte(i, ((Number) value).byteValue());
                 } else if (value instanceof String) {
                     pstmt.setByte(i, Byte.parseByte((String) value));
@@ -97,9 +95,7 @@ public class SyncUtil {
                 }
                 break;
             case Types.SMALLINT:
-                if (value instanceof Byte || value instanceof Short || value instanceof Integer) {
+                if (value instanceof Number) {
-                    pstmt.setShort(i, (short) value);
-                } else if (value instanceof Number) {
                     pstmt.setShort(i, ((Number) value).shortValue());
                 } else if (value instanceof String) {
                     pstmt.setShort(i, Short.parseShort((String) value));
@@ -108,10 +104,7 @@ public class SyncUtil {
                 }
                 break;
             case Types.INTEGER:
-                if (value instanceof Byte || value instanceof Short || value instanceof Integer
+                if (value instanceof Number) {
-                    || value instanceof Long) {
-                    pstmt.setInt(i, (int) value);
-                } else if (value instanceof Number) {
                     pstmt.setInt(i, ((Number) value).intValue());
                 } else if (value instanceof String) {
                     pstmt.setInt(i, Integer.parseInt((String) value));
@@ -120,10 +113,7 @@ public class SyncUtil {
                 }
                 break;
             case Types.BIGINT:
-                if (value instanceof Byte || value instanceof Short || value instanceof Integer
+                if (value instanceof Number) {
-                    || value instanceof Long) {
-                    pstmt.setLong(i, (long) value);
-                } else if (value instanceof Number) {
                     pstmt.setLong(i, ((Number) value).longValue());
                 } else if (value instanceof String) {
                     pstmt.setLong(i, Long.parseLong((String) value));
@@ -136,13 +126,13 @@ public class SyncUtil {
                 if (value instanceof BigDecimal) {
                     pstmt.setBigDecimal(i, (BigDecimal) value);
                 } else if (value instanceof Byte) {
-                    pstmt.setInt(i, (int) value);
+                    pstmt.setInt(i, ((Byte) value).intValue());
                 } else if (value instanceof Short) {
-                    pstmt.setInt(i, (int) value);
+                    pstmt.setInt(i, ((Short) value).intValue());
                 } else if (value instanceof Integer) {
-                    pstmt.setInt(i, (int) value);
+                    pstmt.setInt(i, (Integer) value);
                 } else if (value instanceof Long) {
-                    pstmt.setLong(i, (long) value);
+                    pstmt.setLong(i, (Long) value);
                 } else if (value instanceof Float) {
                     pstmt.setBigDecimal(i, new BigDecimal((float) value));
                 } else if (value instanceof Double) {
@@ -154,10 +144,7 @@ public class SyncUtil {
                 }
                 break;
             case Types.REAL:
-                if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long
+                if (value instanceof Number) {
-                    || value instanceof Float || value instanceof Double) {
-                    pstmt.setFloat(i, (float) value);
-                } else if (value instanceof Number) {
                     pstmt.setFloat(i, ((Number) value).floatValue());
                 } else if (value instanceof String) {
                     pstmt.setFloat(i, Float.parseFloat((String) value));
@@ -167,10 +154,7 @@ public class SyncUtil {
                 break;
             case Types.FLOAT:
             case Types.DOUBLE:
-                if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long
+                if (value instanceof Number) {
-                    || value instanceof Float || value instanceof Double) {
-                    pstmt.setDouble(i, (double) value);
-                } else if (value instanceof Number) {
                     pstmt.setDouble(i, ((Number) value).doubleValue());
                 } else if (value instanceof String) {
                     pstmt.setDouble(i, Double.parseDouble((String) value));

client-adapter/rdb/src/main/resources/rdb/mytest_user.yml

@@ -8,13 +8,13 @@ dbMapping:
   targetTable: mytest.tb_user
   targetPk:
     id: id
-  mapAll: true
+#  mapAll: true
-#  targetColumns:
+  targetColumns:
-#    id:
+    id:
-#    name:
+    name:
-#    role_id:
+    role_id:
-#    c_time:
+    c_time:
-#    test1:
+    test1:
 
 
 # Mirror schema synchronize config

client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java

@@ -1,9 +1,6 @@
 package com.alibaba.otter.canal.client.kafka;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Collections;
-import java.util.List;
-import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.kafka.clients.consumer.ConsumerRecord;
@@ -33,14 +30,16 @@ import com.google.common.collect.Lists;
  */
 public class KafkaCanalConnector implements CanalMQConnector {
 
-    private KafkaConsumer<String, Message> kafkaConsumer;
+    protected KafkaConsumer<String, Message> kafkaConsumer;
-    private KafkaConsumer<String, String>  kafkaConsumer2;   // 用于扁平message的数据消费
+    protected KafkaConsumer<String, String>  kafkaConsumer2;                  // 用于扁平message的数据消费
-    private String                         topic;
+    protected String                         topic;
-    private Integer                        partition;
+    protected Integer                        partition;
-    private Properties                     properties;
+    protected Properties                     properties;
-    private volatile boolean               connected = false;
+    protected volatile boolean               connected      = false;
-    private volatile boolean               running   = false;
+    protected volatile boolean               running        = false;
-    private boolean                        flatMessage;
+    protected boolean                        flatMessage;
+
+    private Map<Integer, Long>               currentOffsets = new HashMap<>();
 
     public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize,
                                boolean flatMessage){
@@ -71,6 +70,7 @@ public class KafkaCanalConnector implements CanalMQConnector {
     /**
      * 打开连接
      */
+    @Override
     public void connect() {
         if (connected) {
             return;
@@ -79,6 +79,7 @@ public class KafkaCanalConnector implements CanalMQConnector {
         connected = true;
         if (kafkaConsumer == null && !flatMessage) {
             kafkaConsumer = new KafkaConsumer<String, Message>(properties);
+
         }
         if (kafkaConsumer2 == null && flatMessage) {
             kafkaConsumer2 = new KafkaConsumer<String, String>(properties);
@@ -88,6 +89,7 @@ public class KafkaCanalConnector implements CanalMQConnector {
     /**
      * 关闭链接
      */
+    @Override
     public void disconnect() {
         if (kafkaConsumer != null) {
             kafkaConsumer.close();
@@ -101,10 +103,11 @@ public class KafkaCanalConnector implements CanalMQConnector {
         connected = false;
     }
 
-    private void waitClientRunning() {
+    protected void waitClientRunning() {
         running = true;
     }
 
+    @Override
     public boolean checkValid() {
         return true;// 默认都放过
     }
@@ -112,6 +115,7 @@ public class KafkaCanalConnector implements CanalMQConnector {
     /**
      * 订阅topic
      */
+    @Override
     public void subscribe() {
         waitClientRunning();
         if (!running) {
@@ -139,6 +143,7 @@ public class KafkaCanalConnector implements CanalMQConnector {
     /**
      * 取消订阅
      */
+    @Override
     public void unsubscribe() {
         waitClientRunning();
         if (!running) {
@@ -176,6 +181,11 @@ public class KafkaCanalConnector implements CanalMQConnector {
 
         ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));
 
+        currentOffsets.clear();
+        for (TopicPartition topicPartition : records.partitions()) {
+            currentOffsets.put(topicPartition.partition(), kafkaConsumer.position(topicPartition));
+        }
+
         if (!records.isEmpty()) {
             List<Message> messages = new ArrayList<>();
             for (ConsumerRecord<String, Message> record : records) {
@@ -208,6 +218,12 @@ public class KafkaCanalConnector implements CanalMQConnector {
         }
 
         ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
+
+        currentOffsets.clear();
+        for (TopicPartition topicPartition : records.partitions()) {
+            currentOffsets.put(topicPartition.partition(), kafkaConsumer2.position(topicPartition));
+        }
+
         if (!records.isEmpty()) {
             List<FlatMessage> flatMessages = new ArrayList<>();
             for (ConsumerRecord<String, String> record : records) {
@@ -222,12 +238,28 @@ public class KafkaCanalConnector implements CanalMQConnector {
     }
 
     @Override
-    public void rollback() throws CanalClientException {
+    public void rollback() {
+        waitClientRunning();
+        if (!running) {
+            return;
+        }
+        // 回滚所有分区
+        if (kafkaConsumer != null) {
+            for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
+                kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1);
+            }
+        }
+        if (kafkaConsumer2 != null) {
+            for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
+                kafkaConsumer2.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1);
+            }
+        }
     }
 
     /**
      * 提交offset，如果超过 session.timeout.ms 设置的时间没有ack则会抛出异常，ack失败
      */
+    @Override
     public void ack() {
         waitClientRunning();
         if (!running) {

client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaOffsetCanalConnector.java

@@ -0,0 +1,112 @@
+package com.alibaba.otter.canal.client.kafka;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.client.kafka.protocol.KafkaFlatMessage;
+import com.alibaba.otter.canal.client.kafka.protocol.KafkaMessage;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.common.TopicPartition;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * kafka带消息offset的连接器
+ *
+ * @Author panjianping
+ * @Email ipanjianping@qq.com
+ * @Date 2018/12/17
+ */
+public class KafkaOffsetCanalConnector extends KafkaCanalConnector {
+
+    public KafkaOffsetCanalConnector(String servers, String topic, Integer partition, String groupId, boolean flatMessage) {
+        super(servers, topic, partition, groupId, 100, flatMessage);
+        // 启动时从未消费的消息位置开始
+        properties.put("auto.offset.reset", "earliest");
+    }
+
+    /**
+     * 获取Kafka消息，不确认
+     *
+     * @param timeout
+     * @param unit
+     * @param offset  消息偏移地址（-1为不偏移）
+     * @return
+     * @throws CanalClientException
+     */
+    public List<KafkaMessage> getListWithoutAck(Long timeout, TimeUnit unit, long offset) throws CanalClientException {
+        waitClientRunning();
+        if (!running) {
+            return Lists.newArrayList();
+        }
+
+        if (offset > -1) {
+            TopicPartition tp = new TopicPartition(topic, partition == null ? 0 : partition);
+            kafkaConsumer.seek(tp, offset);
+        }
+
+        ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));
+
+        if (!records.isEmpty()) {
+            List<KafkaMessage> messages = new ArrayList<>();
+            for (ConsumerRecord<String, Message> record : records) {
+                KafkaMessage message = new KafkaMessage(record.value(), record.offset());
+                messages.add(message);
+            }
+            return messages;
+        }
+        return Lists.newArrayList();
+    }
+
+    /**
+     * 获取Kafka消息，不确认
+     *
+     * @param timeout
+     * @param unit
+     * @param offset  消息偏移地址（-1为不偏移）
+     * @return
+     * @throws CanalClientException
+     */
+    public List<KafkaFlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit, long offset) throws CanalClientException {
+        waitClientRunning();
+        if (!running) {
+            return Lists.newArrayList();
+        }
+
+        if (offset > -1) {
+            TopicPartition tp = new TopicPartition(topic, partition == null ? 0 : partition);
+            kafkaConsumer2.seek(tp, offset);
+        }
+
+        ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
+        if (!records.isEmpty()) {
+            List<KafkaFlatMessage> flatMessages = new ArrayList<>();
+            for (ConsumerRecord<String, String> record : records) {
+                String flatMessageJson = record.value();
+                FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class);
+                KafkaFlatMessage message = new KafkaFlatMessage(flatMessage, record.offset());
+                flatMessages.add(message);
+            }
+
+            return flatMessages;
+        }
+        return Lists.newArrayList();
+    }
+
+    /**
+     * 重新设置AutoOffsetReset（默认 earliest ）
+     *
+     * @param value
+     */
+    public void setAutoOffsetReset(String value) {
+        if (StringUtils.isNotBlank(value)) {
+            properties.put("auto.offset.reset", value);
+        }
+    }
+}

client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaFlatMessage.java

@@ -0,0 +1,32 @@
+package com.alibaba.otter.canal.client.kafka.protocol;
+
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import org.springframework.beans.BeanUtils;
+
+/**
+ * 消息对象（Kafka）
+ *
+ * @Author panjianping
+ * @Email ipanjianping@qq.com
+ * @Date 2018/12/17
+ */
+public class KafkaFlatMessage extends FlatMessage {
+    /**
+     * Kafka 消息 offset
+     */
+    private long offset;
+
+    public KafkaFlatMessage(FlatMessage message, long offset) {
+        super(message.getId());
+        BeanUtils.copyProperties(message, this);
+        this.offset = offset;
+    }
+
+    public long getOffset() {
+        return offset;
+    }
+
+    public void setOffset(long offset) {
+        this.offset = offset;
+    }
+}

client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaMessage.java

@@ -0,0 +1,33 @@
+package com.alibaba.otter.canal.client.kafka.protocol;
+
+import com.alibaba.otter.canal.protocol.Message;
+import org.springframework.beans.BeanUtils;
+
+/**
+ * 消息对象（Kafka）
+ *
+ * @Author panjianping
+ * @Email ipanjianping@qq.com
+ * @Date 2018/12/17
+ */
+public class KafkaMessage extends Message {
+    /**
+     * Kafka 消息 offset
+     */
+    private long offset;
+
+    public KafkaMessage(Message message, long offset) {
+        super(message.getId());
+        BeanUtils.copyProperties(message, this);
+        this.offset = offset;
+    }
+
+
+    public long getOffset() {
+        return offset;
+    }
+
+    public void setOffset(long offset) {
+        this.offset = offset;
+    }
+}

client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaOffsetClientExample.java

@@ -0,0 +1,168 @@
+package com.alibaba.otter.canal.client.running.kafka;
+
+import com.alibaba.otter.canal.client.kafka.KafkaOffsetCanalConnector;
+import com.alibaba.otter.canal.client.kafka.protocol.KafkaMessage;
+import org.apache.kafka.common.errors.WakeupException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * KafkaOffsetCanalConnector 使用示例
+ * <p>KafkaOffsetCanalConnector 与 KafkaCanalConnector 的另一区别是 auto.offset.reset 默认值不同；</p>
+ * <p>KafkaOffsetCanalConnector 默认为 earliest；canal-kafka-client重启后从未被消费的记录开始拉取消息，同时提供了修改 auto.offset.reset 的方法 setAutoOffsetReset</p>
+ *
+ * @author panjianping @ 2018-12-18
+ * @version 1.1.3
+ */
+public class CanalKafkaOffsetClientExample {
+
+    protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaOffsetClientExample.class);
+
+    private KafkaOffsetCanalConnector connector;
+
+    private static volatile boolean running = false;
+
+    private Thread thread = null;
+
+    private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
+
+        public void uncaughtException(Thread t, Throwable e) {
+            logger.error("parse events has an error", e);
+        }
+    };
+
+    public CanalKafkaOffsetClientExample(String servers, String topic, Integer partition, String groupId) {
+        connector = new KafkaOffsetCanalConnector(servers, topic, partition, groupId, false);
+    }
+
+    public static void main(String[] args) {
+        try {
+            final CanalKafkaOffsetClientExample kafkaCanalClientExample = new CanalKafkaOffsetClientExample(
+                    AbstractKafkaTest.servers,
+                    AbstractKafkaTest.topic,
+                    AbstractKafkaTest.partition,
+                    AbstractKafkaTest.groupId);
+            logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
+            kafkaCanalClientExample.start();
+            logger.info("## the canal kafka consumer is running now ......");
+            Runtime.getRuntime().addShutdownHook(new Thread() {
+
+                public void run() {
+                    try {
+                        logger.info("## stop the kafka consumer");
+                        kafkaCanalClientExample.stop();
+                    } catch (Throwable e) {
+                        logger.warn("##something goes wrong when stopping kafka consumer:", e);
+                    } finally {
+                        logger.info("## kafka consumer is down.");
+                    }
+                }
+
+            });
+            while (running)
+                ;
+        } catch (Throwable e) {
+            logger.error("## Something goes wrong when starting up the kafka consumer:", e);
+            System.exit(0);
+        }
+    }
+
+    public void start() {
+        Assert.notNull(connector, "connector is null");
+        thread = new Thread(new Runnable() {
+
+            public void run() {
+                process();
+            }
+        });
+        thread.setUncaughtExceptionHandler(handler);
+        thread.start();
+        running = true;
+    }
+
+    public void stop() {
+        if (!running) {
+            return;
+        }
+        running = false;
+        if (thread != null) {
+            try {
+                thread.join();
+            } catch (InterruptedException e) {
+                // ignore
+            }
+        }
+    }
+
+    private void process() {
+        while (!running)
+            ;
+        while (running) {
+            try {
+                // 修改 AutoOffsetReset 的值，默认（earliest）
+                //connector.setAutoOffsetReset(null);
+                connector.connect();
+                connector.subscribe();
+                // 消息起始偏移地址
+                long offset = -1;
+                // 错误次数
+                int errorCount = 0;
+                while (running) {
+                    try {
+                        // 错误重试次数超过3次后，每30秒递增重试
+                        if (errorCount > 2) {
+                            Thread.sleep((errorCount - 2) * 1000 * 30);
+                        }
+
+                        List<KafkaMessage> messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS, offset); // 获取message
+                        if (messages == null) {
+                            continue;
+                        }
+                        for (KafkaMessage message : messages) {
+                            long batchId = message.getId();
+                            int size = message.getEntries().size();
+
+                            if (batchId == -1 || size == 0) {
+                                continue;
+                            }
+
+                            // 记录第一条消息的offset，用于处理数据异常时重新从此位置获取消息
+                            if (offset < 0) {
+                                offset = message.getOffset();
+                            }
+
+                            // printSummary(message, batchId, size);
+                            // printEntry(message.getEntries());
+                            logger.info(message.toString());
+                        }
+
+                        connector.ack(); // 提交确认
+                        // 还原offset
+                        offset = -1;
+                        errorCount = 0;
+                    } catch (Exception e) {
+                        errorCount++;
+                        logger.error(e.getMessage(), e);
+                        if (errorCount == 3) {
+                            // 重试3次后发送邮件提醒异常
+                            // mailService.sendMail("同步数据异常，请及时处理", "错误消息");
+                        }
+                    }
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        try {
+            connector.unsubscribe();
+        } catch (WakeupException e) {
+            // No-op. Continue process
+        }
+        connector.disconnect();
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java

@@ -32,7 +32,7 @@ public final class CharsetConversion {
     }
 
     // Character set data used in lookups. The array will be sparse.
-    static final Entry[] entries = new Entry[0xff];
+    static final Entry[] entries = new Entry[2048];
 
     static Entry getEntry(final int id) {
         if (id >= 0 && id < entries.length) {
@@ -95,7 +95,7 @@ public final class CharsetConversion {
         putEntry(34, "cp1250", "cp1250_czech_cs", "Cp1250");
         putEntry(35, "ucs2", "ucs2_general_ci", "UnicodeBig");
         putEntry(36, "cp866", "cp866_general_ci", "Cp866");
-        putEntry(37, "keybcs2", "keybcs2_general_ci", "Cp895");
+        putEntry(37, "keybcs2", "keybcs2_general_ci", "Cp852");
         putEntry(38, "macce", "macce_general_ci", "MacCentralEurope");
         putEntry(39, "macroman", "macroman_general_ci", "MacRoman");
         putEntry(40, "cp852", "cp852_general_ci", "Cp852");
@@ -129,7 +129,7 @@ public final class CharsetConversion {
         putEntry(70, "greek", "greek_bin", "ISO8859_7");
         putEntry(71, "hebrew", "hebrew_bin", "ISO8859_8");
         putEntry(72, "hp8", "hp8_bin", "US-ASCII");
-        putEntry(73, "keybcs2", "keybcs2_bin", "Cp895");
+        putEntry(73, "keybcs2", "keybcs2_bin", "Cp852");
         putEntry(74, "koi8r", "koi8r_bin", "KOI8_R");
         putEntry(75, "koi8u", "koi8u_bin", "KOI8_U");
         putEntry(77, "latin2", "latin2_bin", "ISO8859_2");
@@ -260,8 +260,86 @@ public final class CharsetConversion {
         putEntry(241, "utf8mb4", "utf8mb4_esperanto_ci", "UTF-8");
         putEntry(242, "utf8mb4", "utf8mb4_hungarian_ci", "UTF-8");
         putEntry(243, "utf8mb4", "utf8mb4_sinhala_ci", "UTF-8");
+        putEntry(244, "utf8mb4", "utf8mb4_german2_ci", "UTF-8");
+        putEntry(245, "utf8mb4", "utf8mb4_croatian_ci", "UTF-8");
+        putEntry(246, "utf8mb4", "utf8mb4_unicode_520_ci", "UTF-8");
+        putEntry(247, "utf8mb4", "utf8mb4_vietnamese_ci", "UTF-8");
+        putEntry(248, "gb18030", "gb18030_chinese_ci", "GB18030");
+        putEntry(249, "gb18030", "gb18030_bin", "GB18030");
+        putEntry(250, "gb18030", "gb18030_unicode_520_ci", "GB18030");
 
         putEntry(254, "utf8", "utf8_general_cs", "UTF-8");
+        putEntry(255, "utf8mb4", "utf8mb4_0900_ai_ci", "UTF-8");
+        putEntry(256, "utf8mb4", "utf8mb4_de_pb_0900_ai_ci", "UTF-8");
+        putEntry(257, "utf8mb4", "utf8mb4_is_0900_ai_ci", "UTF-8");
+        putEntry(258, "utf8mb4", "utf8mb4_lv_0900_ai_ci", "UTF-8");
+        putEntry(259, "utf8mb4", "utf8mb4_ro_0900_ai_ci", "UTF-8");
+        putEntry(260, "utf8mb4", "utf8mb4_sl_0900_ai_ci", "UTF-8");
+        putEntry(261, "utf8mb4", "utf8mb4_pl_0900_ai_ci", "UTF-8");
+        putEntry(262, "utf8mb4", "utf8mb4_et_0900_ai_ci", "UTF-8");
+        putEntry(263, "utf8mb4", "utf8mb4_es_0900_ai_ci", "UTF-8");
+        putEntry(264, "utf8mb4", "utf8mb4_sv_0900_ai_ci", "UTF-8");
+        putEntry(265, "utf8mb4", "utf8mb4_tr_0900_ai_ci", "UTF-8");
+        putEntry(266, "utf8mb4", "utf8mb4_cs_0900_ai_ci", "UTF-8");
+        putEntry(267, "utf8mb4", "utf8mb4_da_0900_ai_ci", "UTF-8");
+        putEntry(268, "utf8mb4", "utf8mb4_lt_0900_ai_ci", "UTF-8");
+        putEntry(269, "utf8mb4", "utf8mb4_sk_0900_ai_ci", "UTF-8");
+        putEntry(270, "utf8mb4", "utf8mb4_es_trad_0900_ai_ci", "UTF-8");
+        putEntry(271, "utf8mb4", "utf8mb4_la_0900_ai_ci", "UTF-8");
+
+        putEntry(273, "utf8mb4", "utf8mb4_eo_0900_ai_ci", "UTF-8");
+        putEntry(274, "utf8mb4", "utf8mb4_hu_0900_ai_ci", "UTF-8");
+        putEntry(275, "utf8mb4", "utf8mb4_hr_0900_ai_ci", "UTF-8");
+
+        putEntry(277, "utf8mb4", "utf8mb4_vi_0900_ai_ci", "UTF-8");
+        putEntry(278, "utf8mb4", "utf8mb4_0900_as_cs", "UTF-8");
+        putEntry(279, "utf8mb4", "utf8mb4_de_pb_0900_as_cs", "UTF-8");
+        putEntry(280, "utf8mb4", "utf8mb4_is_0900_as_cs", "UTF-8");
+        putEntry(281, "utf8mb4", "utf8mb4_lv_0900_as_cs", "UTF-8");
+        putEntry(282, "utf8mb4", "utf8mb4_ro_0900_as_cs", "UTF-8");
+        putEntry(283, "utf8mb4", "utf8mb4_sl_0900_as_cs", "UTF-8");
+        putEntry(284, "utf8mb4", "utf8mb4_pl_0900_as_cs", "UTF-8");
+        putEntry(285, "utf8mb4", "utf8mb4_et_0900_as_cs", "UTF-8");
+        putEntry(286, "utf8mb4", "utf8mb4_es_0900_as_cs", "UTF-8");
+        putEntry(287, "utf8mb4", "utf8mb4_sv_0900_as_cs", "UTF-8");
+        putEntry(288, "utf8mb4", "utf8mb4_tr_0900_as_cs", "UTF-8");
+        putEntry(289, "utf8mb4", "utf8mb4_cs_0900_as_cs", "UTF-8");
+        putEntry(290, "utf8mb4", "utf8mb4_da_0900_as_cs", "UTF-8");
+        putEntry(291, "utf8mb4", "utf8mb4_lt_0900_as_cs", "UTF-8");
+        putEntry(292, "utf8mb4", "utf8mb4_sk_0900_as_cs", "UTF-8");
+        putEntry(293, "utf8mb4", "utf8mb4_es_trad_0900_as_cs", "UTF-8");
+        putEntry(294, "utf8mb4", "utf8mb4_la_0900_as_cs", "UTF-8");
+
+        putEntry(296, "utf8mb4", "utf8mb4_eo_0900_as_cs", "UTF-8");
+        putEntry(297, "utf8mb4", "utf8mb4_hu_0900_as_cs", "UTF-8");
+        putEntry(298, "utf8mb4", "utf8mb4_hr_0900_as_cs", "UTF-8");
+
+        putEntry(300, "utf8mb4", "utf8mb4_vi_0900_as_cs", "UTF-8");
+        putEntry(303, "utf8mb4", "utf8mb4_ja_0900_as_cs", "UTF-8");
+        putEntry(304, "utf8mb4", "utf8mb4_ja_0900_as_cs_ks", "UTF-8");
+        putEntry(305, "utf8mb4", "utf8mb4_0900_as_ci", "UTF-8");
+        putEntry(306, "utf8mb4", "utf8mb4_ru_0900_ai_ci", "UTF-8");
+        putEntry(307, "utf8mb4", "utf8mb4_ru_0900_as_cs", "UTF-8");
+
+        putEntry(326, "utf8mb4", "utf8mb4_test_ci", "UTF-8");
+        putEntry(327, "utf16", "utf16_test_ci", "UTF-16");
+        putEntry(328, "utf8mb4", "utf8mb4_test_400_ci", "UTF-8");
+
+        putEntry(336, "utf8", "utf8_bengali_standard_ci", "UTF-8");
+        putEntry(337, "utf8", "utf8_bengali_standard_ci", "UTF-8");
+        putEntry(352, "utf8", "utf8_phone_ci", "UTF-8");
+        putEntry(353, "utf8", "utf8_test_ci", "UTF-8");
+        putEntry(354, "utf8", "utf8_5624_1", "UTF-8");
+        putEntry(355, "utf8", "utf8_5624_2", "UTF-8");
+        putEntry(356, "utf8", "utf8_5624_3", "UTF-8");
+        putEntry(357, "utf8", "utf8_5624_4", "UTF-8");
+        putEntry(358, "ucs2", "ucs2_test_ci", "UnicodeBig");
+        putEntry(359, "ucs2", "ucs2_vn_ci", "UnicodeBig");
+        putEntry(360, "ucs2", "ucs2_5624_1", "UnicodeBig");
+
+        putEntry(368, "utf8", "utf8_5624_5", "UTF-8");
+        putEntry(391, "utf32", "utf32_test_ci", "UTF-32");
+        putEntry(2047, "utf8", "utf8_maxuserid_ci", "UTF-8");
     }
 
     /**

deployer/manager_ddl.sql

@@ -0,0 +1,37 @@
+CREATE SCHEMA `canal_manager` DEFAULT CHARACTER SET utf8mb4 ;
+
+CREATE TABLE `canal_config` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `name` varchar(45) NOT NULL,
+  `content` text NOT NULL,
+  `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `name_UNIQUE` (`name`)
+) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 ;
+
+CREATE TABLE `canal_instance_config` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `name` varchar(45) NOT NULL,
+  `content` text NOT NULL,
+  `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `name_UNIQUE` (`name`)
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 ;
+
+CREATE TABLE `canal_adapter_config` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `category` varchar(45) NOT NULL,
+  `name` varchar(45) NOT NULL,
+  `content` text NOT NULL,
+  `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 ;
+
+INSERT INTO `canal_config` VALUES ('1', 'canal.properties', '#################################################\r\n######### 		common argument		############# \r\n#################################################\r\n#canal.manager.jdbc.url=jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8\r\n#canal.manager.jdbc.username=root\r\n#canal.manager.jdbc.password=121212\r\ncanal.id = 1\r\ncanal.ip =\r\ncanal.port = 11111\r\ncanal.metrics.pull.port = 11112\r\ncanal.zkServers =\r\n# flush data to zk\r\ncanal.zookeeper.flush.period = 1000\r\ncanal.withoutNetty = false\r\n# tcp, kafka, RocketMQ\r\ncanal.serverMode = tcp\r\n# flush meta cursor/parse position to file\r\ncanal.file.data.dir = ${canal.conf.dir}\r\ncanal.file.flush.period = 1000\r\n## memory store RingBuffer size, should be Math.pow(2,n)\r\ncanal.instance.memory.buffer.size = 16384\r\n## memory store RingBuffer used memory unit size , default 1kb\r\ncanal.instance.memory.buffer.memunit = 1024 \r\n## meory store gets mode used MEMSIZE or ITEMSIZE\r\ncanal.instance.memory.batch.mode = MEMSIZE\r\ncanal.instance.memory.rawEntry = true\r\n\r\n## detecing config\r\ncanal.instance.detecting.enable = false\r\n#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now()\r\ncanal.instance.detecting.sql = select 1\r\ncanal.instance.detecting.interval.time = 3\r\ncanal.instance.detecting.retry.threshold = 3\r\ncanal.instance.detecting.heartbeatHaEnable = false\r\n\r\n# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery\r\ncanal.instance.transaction.size =  1024\r\n# mysql fallback connected to new master should fallback times\r\ncanal.instance.fallbackIntervalInSeconds = 60\r\n\r\n# network config\r\ncanal.instance.network.receiveBufferSize = 16384\r\ncanal.instance.network.sendBufferSize = 16384\r\ncanal.instance.network.soTimeout = 30\r\n\r\n# binlog filter config\r\ncanal.instance.filter.druid.ddl = true\r\ncanal.instance.filter.query.dcl = false\r\ncanal.instance.filter.query.dml = false\r\ncanal.instance.filter.query.ddl = false\r\ncanal.instance.filter.table.error = false\r\ncanal.instance.filter.rows = false\r\ncanal.instance.filter.transaction.entry = false\r\n\r\n# binlog format/image check\r\ncanal.instance.binlog.format = ROW,STATEMENT,MIXED \r\ncanal.instance.binlog.image = FULL,MINIMAL,NOBLOB\r\n\r\n# binlog ddl isolation\r\ncanal.instance.get.ddl.isolation = false\r\n\r\n# parallel parser config\r\ncanal.instance.parser.parallel = true\r\n## concurrent thread number, default 60% available processors, suggest not to exceed Runtime.getRuntime().availableProcessors()\r\n#canal.instance.parser.parallelThreadSize = 16\r\n## disruptor ringbuffer size, must be power of 2\r\ncanal.instance.parser.parallelBufferSize = 256\r\n\r\n# table meta tsdb info\r\ncanal.instance.tsdb.enable = true\r\ncanal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}\r\ncanal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;\r\ncanal.instance.tsdb.dbUsername = canal\r\ncanal.instance.tsdb.dbPassword = canal\r\n# dump snapshot interval, default 24 hour\r\ncanal.instance.tsdb.snapshot.interval = 24\r\n# purge snapshot expire , default 360 hour(15 days)\r\ncanal.instance.tsdb.snapshot.expire = 360\r\n\r\n# aliyun ak/sk , support rds/mq\r\ncanal.aliyun.accesskey =\r\ncanal.aliyun.secretkey =\r\n\r\n#################################################\r\n######### 		destinations		############# \r\n#################################################\r\ncanal.destinations = example\r\n# conf root dir\r\ncanal.conf.dir = ../conf\r\n# auto scan instance dir add/remove and start/stop instance\r\ncanal.auto.scan = true\r\ncanal.auto.scan.interval = 5\r\n\r\ncanal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml\r\n#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml\r\n\r\ncanal.instance.global.mode = spring\r\ncanal.instance.global.lazy = false\r\n#canal.instance.global.manager.address = 127.0.0.1:1099\r\n#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml\r\ncanal.instance.global.spring.xml = classpath:spring/file-instance.xml\r\n#canal.instance.global.spring.xml = classpath:spring/default-instance.xml\r\n\r\n##################################################\r\n######### 		     MQ 		     #############\r\n##################################################\r\ncanal.mq.servers = 127.0.0.1:6667\r\ncanal.mq.retries = 0\r\ncanal.mq.batchSize = 16384\r\ncanal.mq.maxRequestSize = 1048576\r\ncanal.mq.lingerMs = 1\r\ncanal.mq.bufferMemory = 33554432\r\ncanal.mq.canalBatchSize = 50\r\ncanal.mq.canalGetTimeout = 100\r\ncanal.mq.flatMessage = true\r\ncanal.mq.compressionType = none\r\ncanal.mq.acks = all\r\n', '2018-12-30 16:46:00');
+INSERT INTO `canal_config` VALUES ('2', 'application.yml', 'server:\n  port: 8081\nlogging:\n  level:\n    org.springframework: WARN\n    com.alibaba.otter.canal.client.adapter.hbase: DEBUG\n    com.alibaba.otter.canal.client.adapter.es: DEBUG\n    com.alibaba.otter.canal.client.adapter.rdb: DEBUG\nspring:\n  jackson:\n    date-format: yyyy-MM-dd HH:mm:ss\n    time-zone: GMT+8\n    default-property-inclusion: non_null\n\ncanal.conf:\n  canalServerHost: 127.0.0.1:11111\n#  zookeeperHosts: slave1:2181\n#  mqServers: 127.0.0.1:9092 #or rocketmq\n#  flatMessage: true\n  batchSize: 500\n  syncBatchSize: 1000\n  retries: 0\n  timeout:\n  accessKey:\n  secretKey:\n  mode: tcp # kafka rocketMQ\n#  srcDataSources:\n#    defaultDS:\n#      url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true\n#      username: root\n#      password: 121212\n  canalAdapters:\n  - instance: example # canal instance Name or mq topic name\n    groups:\n    - groupId: g1\n      outerAdapters:\n      - name: logger\n#      - name: rdb\n#        key: mysql1\n#        properties:\n#          jdbc.driverClassName: com.mysql.jdbc.Driver\n#          jdbc.url: jdbc:mysql://127.0.0.1:3306/mytest2?useUnicode=true\n#          jdbc.username: root\n#          jdbc.password: 121212\n#      - name: rdb\n#        key: oracle1\n#        properties:\n#          jdbc.driverClassName: oracle.jdbc.OracleDriver\n#          jdbc.url: jdbc:oracle:thin:@localhost:49161:XE\n#          jdbc.username: mytest\n#          jdbc.password: m121212\n#      - name: rdb\n#        key: postgres1\n#        properties:\n#          jdbc.driverClassName: org.postgresql.Driver\n#          jdbc.url: jdbc:postgresql://localhost:5432/postgres\n#          jdbc.username: postgres\n#          jdbc.password: 121212\n#          threads: 1\n#          commitSize: 3000\n#      - name: hbase\n#        properties:\n#          hbase.zookeeper.quorum: 127.0.0.1\n#          hbase.zookeeper.property.clientPort: 2181\n#          zookeeper.znode.parent: /hbase\n#      - name: es\n#        hosts: 127.0.0.1:9300\n#        properties:\n#          cluster.name: elasticsearch\n', '2019-01-05 01:37:43');
+
+INSERT INTO `canal_instance_config` VALUES ('1', 'example', '#################################################\r\n## mysql serverId , v1.0.26+ will autoGen\r\n# canal.instance.mysql.slaveId=0\r\n\r\n# enable gtid use true/false\r\ncanal.instance.gtidon=false\r\n\r\n# position info\r\ncanal.instance.master.address=127.0.0.1:3306\r\ncanal.instance.master.journal.name=\r\ncanal.instance.master.position=\r\ncanal.instance.master.timestamp=\r\ncanal.instance.master.gtid=\r\n\r\n# rds oss binlog\r\ncanal.instance.rds.accesskey=\r\ncanal.instance.rds.secretkey=\r\ncanal.instance.rds.instanceId=\r\n\r\n# table meta tsdb info\r\ncanal.instance.tsdb.enable=true\r\n#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb\r\n#canal.instance.tsdb.dbUsername=canal\r\n#canal.instance.tsdb.dbPassword=canal\r\n\r\n#canal.instance.standby.address =\r\n#canal.instance.standby.journal.name =\r\n#canal.instance.standby.position =\r\n#canal.instance.standby.timestamp =\r\n#canal.instance.standby.gtid=\r\n\r\n# username/password\r\ncanal.instance.dbUsername=canal\r\ncanal.instance.dbPassword=canal\r\ncanal.instance.connectionCharset = UTF-8\r\ncanal.instance.defaultDatabaseName =test\r\n# enable druid Decrypt database password\r\ncanal.instance.enableDruid=false\r\n#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==\r\n\r\n# table regex\r\ncanal.instance.filter.regex=.*\\\\..*\r\n# table black regex\r\ncanal.instance.filter.black.regex=\r\n\r\n# mq config\r\ncanal.mq.topic=example\r\n# 动态topic, 需mq支持动态创建topic\r\n#canal.mq.dynamicTopic=.*,mytest\\\\..*,mytest2.user\r\ncanal.mq.partition=0\r\n# hash partition config\r\n#canal.mq.partitionsNum=3\r\n#canal.mq.partitionHash=test.table:id^name,.*\\\\..*\r\n#################################################\r\n', '2018-12-30 16:46:16');
+
+INSERT INTO `canal_adapter_config` VALUES ('1', 'es', 'mytest_user.yml', 'dataSourceKey: defaultDS\ndestination: example\nesMapping:\n  _index: mytest_user\n  _type: _doc\n  _id: _id\n#  pk: id\n  sql: \"select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name,\n        a.c_time as _c_time, c.labels as _labels from user a\n        left join role b on b.id=a.role_id\n        left join (select user_id, group_concat(label order by id desc separator \';\') as labels from label\n        group by user_id) c on c.user_id=a.id\"\n#  objFields:\n#    _labels: array:;\n  etlCondition: \"where a.c_time>=\'{0}\'\"\n  commitBatch: 3000\n', '2019-01-01 17:05:43');
+INSERT INTO `canal_adapter_config` VALUES ('2', 'hhase', 'mytest_person2.yml', 'dataSourceKey: defaultDS\ndestination: example\nhbaseMapping:\n  mode: STRING  #NATIVE   #PHOENIX\n  database: mytest  # 数据库名\n  table: person2     # 数据库表名\n  hbaseTable: MYTEST.PERSON2   # HBase表名\n  family: CF  # 默认统一Family名称\n  uppercaseQualifier: true  # 字段名转大写, 默认为true\n  commitBatch: 3000 # 批量提交的大小\n  #rowKey: id,type  # 复合字段rowKey不能和columns中的rowKey重复\n  columns:\n    # 数据库字段:HBase对应字段\n    id: ROWKEY LEN:15\n    name: NAME\n    email: EMAIL\n    type:\n    c_time: C_TIME\n    birthday: BIRTHDAY\n#  excludeColumns:\n#    - lat   # 忽略字段\n\n# -- NATIVE类型\n# $DEFAULT\n# $STRING\n# $INTEGER\n# $LONG\n# $SHORT\n# $BOOLEAN\n# $FLOAT\n# $DOUBLE\n# $BIGDECIMAL\n# $DATE\n# $BYTE\n# $BYTES\n\n# -- PHOENIX类型\n# $DEFAULT                  对应PHOENIX里的VARCHAR\n# $UNSIGNED_INT             对应PHOENIX里的UNSIGNED_INT           4字节\n# $UNSIGNED_LONG            对应PHOENIX里的UNSIGNED_LONG          8字节\n# $UNSIGNED_TINYINT         对应PHOENIX里的UNSIGNED_TINYINT       1字节\n# $UNSIGNED_SMALLINT        对应PHOENIX里的UNSIGNED_SMALLINT      2字节\n# $UNSIGNED_FLOAT           对应PHOENIX里的UNSIGNED_FLOAT         4字节\n# $UNSIGNED_DOUBLE          对应PHOENIX里的UNSIGNED_DOUBLE        8字节\n# $INTEGER                  对应PHOENIX里的INTEGER                4字节\n# $BIGINT                   对应PHOENIX里的BIGINT                 8字节\n# $TINYINT                  对应PHOENIX里的TINYINT                1字节\n# $SMALLINT                 对应PHOENIX里的SMALLINT               2字节\n# $FLOAT                    对应PHOENIX里的FLOAT                  4字节\n# $DOUBLE                    对应PHOENIX里的DOUBLE                 8字节\n# $BOOLEAN                  对应PHOENIX里的BOOLEAN                1字节\n# $TIME                     对应PHOENIX里的TIME                   8字节\n# $DATE                     对应PHOENIX里的DATE                   8字节\n# $TIMESTAMP                对应PHOENIX里的TIMESTAMP              12字节\n# $UNSIGNED_TIME            对应PHOENIX里的UNSIGNED_TIME          8字节\n# $UNSIGNED_DATE            对应PHOENIX里的UNSIGNED_DATE          8字节\n# $UNSIGNED_TIMESTAMP       对应PHOENIX里的UNSIGNED_TIMESTAMP     12字节\n# $VARCHAR                  对应PHOENIX里的VARCHAR                动态长度\n# $VARBINARY                对应PHOENIX里的VARBINARY              动态长度\n# $DECIMAL                  对应PHOENIX里的DECIMAL                动态长度', '2019-01-01 17:08:08');
+INSERT INTO `canal_adapter_config` VALUES ('3', 'rdb', 'mytest_user.yml', 'dataSourceKey: defaultDS\ndestination: example\nouterAdapterKey: oracle1\nconcurrent: true\ndbMapping:\n  database: mytest\n  table: user\n  targetTable: mytest.tb_user\n  targetPk:\n    id: id\n#  mapAll: true\n  targetColumns:\n    id:\n    name:\n    role_id:\n    c_time:\n    test1:\n\n\n# Mirror schema synchronize config\n#dataSourceKey: defaultDS\n#destination: example\n#outerAdapterKey: mysql1\n#concurrent: true\n#dbMapping:\n#  mirrorDb: true\n#  database: mytest\n', '2019-01-01 17:08:50');

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java

@@ -7,11 +7,7 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.otter.canal.common.MQProperties;
+import com.alibaba.otter.canal.deployer.monitor.ManagerRemoteConfigMonitor;
-import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
-import com.alibaba.otter.canal.rocketmq.CanalRocketMQProducer;
-import com.alibaba.otter.canal.server.CanalMQStarter;
-import com.alibaba.otter.canal.spi.CanalMQProducer;
 
 /**
  * canal独立版本启动的入口类
@@ -23,15 +19,18 @@ public class CanalLauncher {
 
     private static final String    CLASSPATH_URL_PREFIX = "classpath:";
     private static final Logger    logger               = LoggerFactory.getLogger(CanalLauncher.class);
+    public static volatile boolean running              = false;
 
-    public static void main(String[] args) throws Throwable {
+    public static void main(String[] args) {
         try {
+            running = true;
             logger.info("## set default uncaught exception handler");
             setGlobalUncaughtExceptionHandler();
 
             logger.info("## load canal configurations");
             String conf = System.getProperty("canal.conf", "classpath:canal.properties");
             Properties properties = new Properties();
+            ManagerRemoteConfigMonitor managerDbConfigMonitor = null;
             if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
                 conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
                 properties.load(CanalLauncher.class.getClassLoader().getResourceAsStream(conf));
@@ -39,110 +38,56 @@ public class CanalLauncher {
                 properties.load(new FileInputStream(conf));
             }
 
-            CanalMQProducer canalMQProducer = null;
+            String jdbcUrl = properties.getProperty("canal.manager.jdbc.url");
-            String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE);
+            if (!StringUtils.isEmpty(jdbcUrl)) {
-            if (serverMode.equalsIgnoreCase("kafka")) {
+                logger.info("## load remote canal configurations");
-                canalMQProducer = new CanalKafkaProducer();
+                // load remote config
-            } else if (serverMode.equalsIgnoreCase("rocketmq")) {
+                String jdbcUsername = properties.getProperty("canal.manager.jdbc.username");
-                canalMQProducer = new CanalRocketMQProducer();
+                String jdbcPassword = properties.getProperty("canal.manager.jdbc.password");
+                managerDbConfigMonitor = new ManagerRemoteConfigMonitor(jdbcUrl, jdbcUsername, jdbcPassword);
+                // 加载远程canal.properties
+                Properties remoteConfig = managerDbConfigMonitor.loadRemoteConfig();
+                // 加载remote instance配置
+                managerDbConfigMonitor.loadRemoteInstanceConfigs();
+                if (remoteConfig != null) {
+                    properties = remoteConfig;
+                } else {
+                    managerDbConfigMonitor = null;
+                }
+            } else {
+                logger.info("## load canal configurations");
             }
 
-            if (canalMQProducer != null) {
+            final CanalStater canalStater = new CanalStater();
-                // disable netty
+            canalStater.start(properties);
-                System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true");
-                System.setProperty(CanalConstants.CANAL_DESTINATIONS,
-                    properties.getProperty(CanalConstants.CANAL_DESTINATIONS));
-            }
 
-            logger.info("## start the canal server.");
+            if (managerDbConfigMonitor != null) {
-            final CanalController controller = new CanalController(properties);
+                managerDbConfigMonitor.start(new ManagerRemoteConfigMonitor.Listener<Properties>() {
-            controller.start();
-            logger.info("## the canal server is running now ......");
-            Runtime.getRuntime().addShutdownHook(new Thread() {
 
-                public void run() {
+                    @Override
+                    public void onChange(Properties properties) {
                         try {
-                        logger.info("## stop the canal server");
+                            // 远程配置canal.properties修改重新加载整个应用
-                        controller.stop();
+                            canalStater.destroy();
-                    } catch (Throwable e) {
+                            canalStater.start(properties);
-                        logger.warn("##something goes wrong when stopping canal Server:", e);
+                        } catch (Throwable throwable) {
-                    } finally {
+                            logger.error(throwable.getMessage(), throwable);
-                        logger.info("## canal server is down.");
                         }
                     }
-
                 });
+            }
 
-            if (canalMQProducer != null) {
+            while (running)
-                CanalMQStarter canalMQStarter = new CanalMQStarter(canalMQProducer);
+                ;
-                MQProperties mqProperties = buildMQPosition(properties);
+
-                canalMQStarter.start(mqProperties);
+            if (managerDbConfigMonitor != null) {
-                controller.setCanalMQStarter(canalMQStarter);
+                managerDbConfigMonitor.destroy();
             }
         } catch (Throwable e) {
             logger.error("## Something goes wrong when starting up the canal Server:", e);
-            System.exit(0);
         }
     }
 
-    private static MQProperties buildMQPosition(Properties properties) {
-        MQProperties mqProperties = new MQProperties();
-        String servers = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_SERVERS);
-        if (!StringUtils.isEmpty(servers)) {
-            mqProperties.setServers(servers);
-        }
-        String retires = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_RETRIES);
-        if (!StringUtils.isEmpty(retires)) {
-            mqProperties.setRetries(Integer.valueOf(retires));
-        }
-        String batchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BATCHSIZE);
-        if (!StringUtils.isEmpty(batchSize)) {
-            mqProperties.setBatchSize(Integer.valueOf(batchSize));
-        }
-        String lingerMs = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_LINGERMS);
-        if (!StringUtils.isEmpty(lingerMs)) {
-            mqProperties.setLingerMs(Integer.valueOf(lingerMs));
-        }
-        String maxRequestSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_MAXREQUESTSIZE);
-        if (!StringUtils.isEmpty(maxRequestSize)) {
-            mqProperties.setMaxRequestSize(Integer.valueOf(maxRequestSize));
-        }
-        String bufferMemory = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BUFFERMEMORY);
-        if (!StringUtils.isEmpty(bufferMemory)) {
-            mqProperties.setBufferMemory(Long.valueOf(bufferMemory));
-        }
-        String canalBatchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALBATCHSIZE);
-        if (!StringUtils.isEmpty(canalBatchSize)) {
-            mqProperties.setCanalBatchSize(Integer.valueOf(canalBatchSize));
-        }
-        String canalGetTimeout = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALGETTIMEOUT);
-        if (!StringUtils.isEmpty(canalGetTimeout)) {
-            mqProperties.setCanalGetTimeout(Long.valueOf(canalGetTimeout));
-        }
-        String flatMessage = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_FLATMESSAGE);
-        if (!StringUtils.isEmpty(flatMessage)) {
-            mqProperties.setFlatMessage(Boolean.valueOf(flatMessage));
-        }
-        String compressionType = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_COMPRESSION_TYPE);
-        if (!StringUtils.isEmpty(compressionType)) {
-            mqProperties.setCompressionType(compressionType);
-        }
-        String acks = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACKS);
-        if (!StringUtils.isEmpty(acks)) {
-            mqProperties.setAcks(acks);
-        }
-
-        String aliyunAccessKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_ACCESSKEY);
-        if (!StringUtils.isEmpty(aliyunAccessKey)) {
-            mqProperties.setAliyunAccessKey(aliyunAccessKey);
-        }
-        String aliyunSecretKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_SECRETKEY);
-        if (!StringUtils.isEmpty(aliyunSecretKey)) {
-            mqProperties.setAliyunSecretKey(aliyunSecretKey);
-        }
-        return mqProperties;
-    }
-
     private static void setGlobalUncaughtExceptionHandler() {
         Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
 
@@ -152,4 +97,5 @@ public class CanalLauncher {
             }
         });
     }
+
 }

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStater.java

@@ -0,0 +1,165 @@
+package com.alibaba.otter.canal.deployer;
+
+import java.util.Properties;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.common.MQProperties;
+import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
+import com.alibaba.otter.canal.rocketmq.CanalRocketMQProducer;
+import com.alibaba.otter.canal.server.CanalMQStarter;
+import com.alibaba.otter.canal.spi.CanalMQProducer;
+
+/**
+ * Canal server 启动类
+ *
+ * @author rewerma 2018-12-30 下午05:12:16
+ * @version 1.0.1
+ */
+public class CanalStater {
+
+    private static final Logger logger          = LoggerFactory.getLogger(CanalStater.class);
+
+    private CanalController     controller      = null;
+    private CanalMQProducer     canalMQProducer = null;
+    private Thread              shutdownThread  = null;
+    private CanalMQStarter      canalMQStarter  = null;
+
+    /**
+     * 启动方法
+     * 
+     * @param properties canal.properties 配置
+     * @throws Throwable
+     */
+    synchronized void start(Properties properties) throws Throwable {
+        String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE);
+        if (serverMode.equalsIgnoreCase("kafka")) {
+            canalMQProducer = new CanalKafkaProducer();
+        } else if (serverMode.equalsIgnoreCase("rocketmq")) {
+            canalMQProducer = new CanalRocketMQProducer();
+        }
+
+        if (canalMQProducer != null) {
+            // disable netty
+            System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true");
+            System.setProperty(CanalConstants.CANAL_DESTINATIONS,
+                properties.getProperty(CanalConstants.CANAL_DESTINATIONS));
+        }
+
+        logger.info("## start the canal server.");
+        controller = new CanalController(properties);
+        controller.start();
+        logger.info("## the canal server is running now ......");
+        shutdownThread = new Thread() {
+
+            public void run() {
+                try {
+                    logger.info("## stop the canal server");
+                    controller.stop();
+                    CanalLauncher.running = false;
+                } catch (Throwable e) {
+                    logger.warn("##something goes wrong when stopping canal Server:", e);
+                } finally {
+                    logger.info("## canal server is down.");
+                }
+            }
+
+        };
+        Runtime.getRuntime().addShutdownHook(shutdownThread);
+
+        if (canalMQProducer != null) {
+            canalMQStarter = new CanalMQStarter(canalMQProducer);
+            MQProperties mqProperties = buildMQProperties(properties);
+            canalMQStarter.start(mqProperties);
+            controller.setCanalMQStarter(canalMQStarter);
+        }
+    }
+
+    /**
+     * 销毁方法，远程配置变更时调用
+     * 
+     * @throws Throwable
+     */
+    synchronized void destroy() throws Throwable {
+        if (controller != null) {
+            controller.stop();
+            controller = null;
+        }
+        if (shutdownThread != null) {
+            Runtime.getRuntime().removeShutdownHook(shutdownThread);
+            shutdownThread = null;
+        }
+        if (canalMQProducer != null && canalMQStarter != null) {
+            canalMQStarter.destroy();
+            canalMQStarter = null;
+            canalMQProducer = null;
+        }
+    }
+
+    /**
+     * 构造MQ对应的配置
+     * 
+     * @param properties canal.properties 配置
+     * @return
+     */
+    private static MQProperties buildMQProperties(Properties properties) {
+        MQProperties mqProperties = new MQProperties();
+        String servers = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_SERVERS);
+        if (!StringUtils.isEmpty(servers)) {
+            mqProperties.setServers(servers);
+        }
+        String retires = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_RETRIES);
+        if (!StringUtils.isEmpty(retires)) {
+            mqProperties.setRetries(Integer.valueOf(retires));
+        }
+        String batchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BATCHSIZE);
+        if (!StringUtils.isEmpty(batchSize)) {
+            mqProperties.setBatchSize(Integer.valueOf(batchSize));
+        }
+        String lingerMs = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_LINGERMS);
+        if (!StringUtils.isEmpty(lingerMs)) {
+            mqProperties.setLingerMs(Integer.valueOf(lingerMs));
+        }
+        String maxRequestSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_MAXREQUESTSIZE);
+        if (!StringUtils.isEmpty(maxRequestSize)) {
+            mqProperties.setMaxRequestSize(Integer.valueOf(maxRequestSize));
+        }
+        String bufferMemory = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BUFFERMEMORY);
+        if (!StringUtils.isEmpty(bufferMemory)) {
+            mqProperties.setBufferMemory(Long.valueOf(bufferMemory));
+        }
+        String canalBatchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALBATCHSIZE);
+        if (!StringUtils.isEmpty(canalBatchSize)) {
+            mqProperties.setCanalBatchSize(Integer.valueOf(canalBatchSize));
+        }
+        String canalGetTimeout = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALGETTIMEOUT);
+        if (!StringUtils.isEmpty(canalGetTimeout)) {
+            mqProperties.setCanalGetTimeout(Long.valueOf(canalGetTimeout));
+        }
+        String flatMessage = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_FLATMESSAGE);
+        if (!StringUtils.isEmpty(flatMessage)) {
+            mqProperties.setFlatMessage(Boolean.valueOf(flatMessage));
+        }
+        String compressionType = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_COMPRESSION_TYPE);
+        if (!StringUtils.isEmpty(compressionType)) {
+            mqProperties.setCompressionType(compressionType);
+        }
+        String acks = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACKS);
+        if (!StringUtils.isEmpty(acks)) {
+            mqProperties.setAcks(acks);
+        }
+
+        String aliyunAccessKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_ACCESSKEY);
+        if (!StringUtils.isEmpty(aliyunAccessKey)) {
+            mqProperties.setAliyunAccessKey(aliyunAccessKey);
+        }
+        String aliyunSecretKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_SECRETKEY);
+        if (!StringUtils.isEmpty(aliyunSecretKey)) {
+            mqProperties.setAliyunSecretKey(aliyunSecretKey);
+        }
+        return mqProperties;
+    }
+
+}

deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/ManagerRemoteConfigMonitor.java

@@ -0,0 +1,358 @@
+package com.alibaba.otter.canal.deployer.monitor;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileWriter;
+import java.nio.charset.StandardCharsets;
+import java.sql.*;
+import java.util.*;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
+import com.alibaba.otter.canal.deployer.CanalConstants;
+import com.google.common.base.Joiner;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 远程配置装载监控
+ * 
+ * @author rewerma 2018-12-30 下午05:12:16
+ * @version 1.0.1
+ */
+public class ManagerRemoteConfigMonitor {
+
+    private static final Logger      logger                 = LoggerFactory.getLogger(ManagerRemoteConfigMonitor.class);
+
+    private Map<String, ConfigItem>  remoteInstanceConfigs  = new MapMaker().makeMap();
+
+    private Connection               conn;
+    private String                   jdbcUrl;
+    private String                   jdbcUsername;
+    private String                   jdbcPassword;
+
+    private long                     currentConfigTimestamp = 0;
+
+    private long                     scanIntervalInSecond   = 5;
+    private ScheduledExecutorService executor               = Executors.newScheduledThreadPool(2,
+        new NamedThreadFactory("remote-canal-config-scan"));
+
+    public ManagerRemoteConfigMonitor(String jdbcUrl, String jdbcUsername, String jdbcPassword){
+        this.jdbcUrl = jdbcUrl;
+        this.jdbcUsername = jdbcUsername;
+        this.jdbcPassword = jdbcPassword;
+    }
+
+    public void setScanIntervalInSecond(long scanIntervalInSecond) {
+        this.scanIntervalInSecond = scanIntervalInSecond;
+    }
+
+    private Connection getConn() throws Exception {
+        if (conn == null || conn.isClosed()) {
+            Class.forName("com.mysql.jdbc.Driver");
+            conn = DriverManager.getConnection(jdbcUrl, jdbcUsername, jdbcPassword);
+        }
+        return conn;
+    }
+
+    /**
+     * 加载远程 canal.properties文件 覆盖本地
+     * 
+     * @return 远程配置的properties
+     */
+    public Properties loadRemoteConfig() {
+        Properties properties = null;
+        try {
+            // 加载远程canal配置
+            ConfigItem configItem = getRemoteCanalConfig();
+            if (configItem != null) {
+                if (configItem.getModifiedTime() != currentConfigTimestamp) {
+                    currentConfigTimestamp = configItem.getModifiedTime();
+                    overrideLocalCanalConfig(configItem.getContent());
+                    properties = new Properties();
+                    properties.load(new ByteArrayInputStream(configItem.getContent().getBytes(StandardCharsets.UTF_8)));
+                    scanIntervalInSecond = Integer
+                        .valueOf(properties.getProperty(CanalConstants.CANAL_AUTO_SCAN_INTERVAL, "5"));
+                }
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+        return properties;
+    }
+
+    /**
+     * 加载远程的instance配置 覆盖本地
+     */
+    public void loadRemoteInstanceConfigs() {
+        try {
+            // 加载远程instance配置
+            Map<String, ConfigItem>[] modifiedConfigs = getModifiedInstanceConfigs();
+            if (modifiedConfigs != null) {
+                overrideLocalInstanceConfigs(modifiedConfigs);
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 获取远程canal.properties配置内容
+     * 
+     * @return 内容对象
+     */
+    private ConfigItem getRemoteCanalConfig() {
+        String sql = "select name, content, modified_time from canal_config where id=1";
+        try (Statement stmt = getConn().createStatement(); ResultSet rs = stmt.executeQuery(sql)) {
+            if (rs.next()) {
+                ConfigItem configItem = new ConfigItem();
+                configItem.setId(1L);
+                configItem.setName(rs.getString("name"));
+                configItem.setContent(rs.getString("content"));
+                configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
+                return configItem;
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+        return null;
+    }
+
+    /**
+     * 覆盖本地 canal.properties
+     * 
+     * @param content 远程配置内容文本
+     */
+    private void overrideLocalCanalConfig(String content) {
+        try (FileWriter writer = new FileWriter(getConfPath() + "canal.properties")) {
+            writer.write(content);
+            writer.flush();
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 获取远程instance新增、修改、删除配置
+     *
+     * @return Map[0]：新增或修改的instance配置; Map[1]：删除的instance配置
+     */
+    @SuppressWarnings("unchecked")
+    private Map<String, ConfigItem>[] getModifiedInstanceConfigs() {
+        Map<String, ConfigItem>[] res = new Map[2];
+        Map<String, ConfigItem> remoteConfigStatus = new HashMap<>();
+        String sql = "select id, name, modified_time from canal_instance_config";
+        try (Statement stmt = getConn().createStatement(); ResultSet rs = stmt.executeQuery(sql)) {
+            while (rs.next()) {
+                ConfigItem configItem = new ConfigItem();
+                configItem.setId(rs.getLong("id"));
+                configItem.setName(rs.getString("name"));
+                configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
+                remoteConfigStatus.put(configItem.getName(), configItem);
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            return null;
+        }
+
+        if (!remoteConfigStatus.isEmpty()) {
+            List<Long> changedIds = new ArrayList<>();
+
+            for (ConfigItem remoteConfigStat : remoteConfigStatus.values()) {
+                ConfigItem currentConfig = remoteInstanceConfigs.get(remoteConfigStat.getName());
+                if (currentConfig == null) {
+                    // 新增
+                    changedIds.add(remoteConfigStat.getId());
+                } else {
+                    // 修改
+                    if (currentConfig.getModifiedTime() != remoteConfigStat.getModifiedTime()) {
+                        changedIds.add(remoteConfigStat.getId());
+                    }
+                }
+            }
+            if (!changedIds.isEmpty()) {
+                Map<String, ConfigItem> changedInstanceConfig = new HashMap<>();
+                String contentsSql = "select id, name, content, modified_time from canal_instance_config  where id in ("
+                                     + Joiner.on(",").join(changedIds) + ")";
+                try (Statement stmt = getConn().createStatement(); ResultSet rs = stmt.executeQuery(contentsSql)) {
+                    while (rs.next()) {
+                        ConfigItem configItemNew = new ConfigItem();
+                        configItemNew.setId(rs.getLong("id"));
+                        configItemNew.setName(rs.getString("name"));
+                        configItemNew.setContent(rs.getString("content"));
+                        configItemNew.setModifiedTime(rs.getTimestamp("modified_time").getTime());
+
+                        remoteInstanceConfigs.put(configItemNew.getName(), configItemNew);
+                        changedInstanceConfig.put(configItemNew.getName(), configItemNew);
+                    }
+
+                    res[0] = changedInstanceConfig.isEmpty() ? null : changedInstanceConfig;
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+        }
+
+        Map<String, ConfigItem> removedInstanceConfig = new HashMap<>();
+        for (String name : remoteInstanceConfigs.keySet()) {
+            if (!remoteConfigStatus.containsKey(name)) {
+                // 删除
+                remoteInstanceConfigs.remove(name);
+                removedInstanceConfig.put(name, null);
+            }
+        }
+        res[1] = removedInstanceConfig.isEmpty() ? null : removedInstanceConfig;
+
+        if (res[0] == null && res[1] == null) {
+            return null;
+        } else {
+            return res;
+        }
+    }
+
+    /**
+     * 覆盖本地instance配置
+     * 
+     * @param modifiedInstanceConfigs 有变更的配置项
+     */
+    private void overrideLocalInstanceConfigs(Map<String, ConfigItem>[] modifiedInstanceConfigs) {
+        Map<String, ConfigItem> changedInstanceConfigs = modifiedInstanceConfigs[0];
+        if (changedInstanceConfigs != null) {
+            for (ConfigItem configItem : changedInstanceConfigs.values()) {
+                try (FileWriter writer = new FileWriter(
+                    getConfPath() + configItem.getName() + "/instance.properties")) {
+                    writer.write(configItem.getContent());
+                    writer.flush();
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+        }
+        Map<String, ConfigItem> removedInstanceConfigs = modifiedInstanceConfigs[1];
+        if (removedInstanceConfigs != null) {
+            for (String name : removedInstanceConfigs.keySet()) {
+                File file = new File(getConfPath() + name + "/");
+                if (file.exists()) {
+                    file.delete();
+                }
+            }
+        }
+    }
+
+    /**
+     * 监听 canal 主配置和 instance 配置变化
+     * 
+     * @param listener 监听回调方法
+     */
+    public void start(final Listener<Properties> listener) {
+        // 监听canal.properties变化
+        executor.scheduleWithFixedDelay(new Runnable() {
+
+            public void run() {
+                try {
+                    Properties properties = loadRemoteConfig();
+                    if (properties != null) {
+                        listener.onChange(properties);
+                    }
+                } catch (Throwable e) {
+                    logger.error("scan failed", e);
+                }
+            }
+
+        }, 10, scanIntervalInSecond, TimeUnit.SECONDS);
+
+        // 监听instance变化
+        executor.scheduleWithFixedDelay(new Runnable() {
+
+            public void run() {
+                try {
+                    loadRemoteInstanceConfigs();
+                } catch (Throwable e) {
+                    logger.error("scan failed", e);
+                }
+            }
+
+        }, 10, 3, TimeUnit.SECONDS);
+    }
+
+    /**
+     * 销毁
+     */
+    public void destroy() {
+        executor.shutdownNow();
+        if (conn != null) {
+            try {
+                conn.close();
+            } catch (SQLException e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+    }
+
+    /**
+     * 获取conf文件夹所在路径
+     * 
+     * @return 路径地址
+     */
+    private String getConfPath() {
+        String classpath = this.getClass().getResource("/").getPath();
+        String confPath = classpath + ".." + File.separator + "conf" + File.separator;
+        if (new File(confPath).exists()) {
+            return confPath;
+        } else {
+            return classpath;
+        }
+    }
+
+    /**
+     * 配置对应对象
+     */
+    public static class ConfigItem {
+
+        private Long   id;
+        private String name;
+        private String content;
+        private long   modifiedTime;
+
+        public Long getId() {
+            return id;
+        }
+
+        public void setId(Long id) {
+            this.id = id;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        public String getContent() {
+            return content;
+        }
+
+        public void setContent(String content) {
+            this.content = content;
+        }
+
+        public long getModifiedTime() {
+            return modifiedTime;
+        }
+
+        public void setModifiedTime(long modifiedTime) {
+            this.modifiedTime = modifiedTime;
+        }
+    }
+
+    public interface Listener<Properties> {
+
+        void onChange(Properties properties);
+    }
+}

deployer/src/main/resources/canal.properties

@@ -1,6 +1,9 @@
 #################################################
 ######### 		common argument		############# 
 #################################################
+#canal.manager.jdbc.url=jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8
+#canal.manager.jdbc.username=root
+#canal.manager.jdbc.password=121212
 canal.id = 1
 canal.ip =
 canal.port = 11111

deployer/src/main/resources/example/instance.properties

@@ -45,6 +45,8 @@ canal.instance.filter.black.regex=
 
 # mq config
 canal.mq.topic=example
+# dynamic topic route by table regex
+#canal.mq.dynamicTopic=.*,mytest\\..*,mytest2.user
 canal.mq.partition=0
 # hash partition config
 #canal.mq.partitionsNum=3

deployer/src/main/resources/spring/default-instance.xml

@@ -193,6 +193,7 @@
 
 	<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
 		<property name="topic" value="${canal.mq.topic}" />
+		<property name="dynamicTopic" value="${canal.mq.dynamicTopic}" />
 		<property name="partition" value="${canal.mq.partition}" />
 		<property name="partitionsNum" value="${canal.mq.partitionsNum}" />
 		<property name="partitionHash" value="${canal.mq.partitionHash}" />

deployer/src/main/resources/spring/file-instance.xml

@@ -178,6 +178,7 @@
 
 	<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
         <property name="topic" value="${canal.mq.topic}" />
+		<property name="dynamicTopic" value="${canal.mq.dynamicTopic}" />
         <property name="partition" value="${canal.mq.partition}" />
         <property name="partitionsNum" value="${canal.mq.partitionsNum}" />
         <property name="partitionHash" value="${canal.mq.partitionHash}" />

deployer/src/main/resources/spring/group-instance.xml

@@ -268,6 +268,7 @@
 
     <bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
         <property name="topic" value="${canal.mq.topic}" />
+		<property name="dynamicTopic" value="${canal.mq.dynamicTopic}" />
         <property name="partition" value="${canal.mq.partition}" />
         <property name="partitionsNum" value="${canal.mq.partitionsNum}" />
         <property name="partitionHash" value="${canal.mq.partitionHash}" />

deployer/src/main/resources/spring/memory-instance.xml

@@ -166,6 +166,7 @@
 
 	<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
 		<property name="topic" value="${canal.mq.topic}" />
+		<property name="dynamicTopic" value="${canal.mq.dynamicTopic}" />
 		<property name="partition" value="${canal.mq.partition}" />
 		<property name="partitionsNum" value="${canal.mq.partitionsNum}" />
 		<property name="partitionHash" value="${canal.mq.partitionHash}" />

example/src/main/bin/startup.bat

@@ -8,7 +8,7 @@ if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0%
 set conf_dir=%ENV_PATH%\..\conf
 set logback_configurationFile=%conf_dir%\logback.xml
 set client_mode=Simple
-if "%1%" != "" set client_mode=%1%
+if not "%1" == "" set client_mode=%1
 
 set CLASSPATH=%conf_dir%
 set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH%
@@ -20,7 +20,8 @@ set CANAL_OPTS= -DappName=otter-canal-example -Dlogback.configurationFile="%logb
 
 set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS%
 
-if "%client_mode%" == "Cluster"  
+if "%client_mode%" == "Cluster" (
 	java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.example.ClusterCanalClientTest
-else 
+) else (
 	java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.example.SimpleCanalClientTest
+)

instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalMQConfig.java

@@ -6,6 +6,7 @@ public class CanalMQConfig {
     private Integer partition;
     private Integer partitionsNum;
     private String  partitionHash;
+    private String  dynamicTopic;
 
     public String getTopic() {
         return topic;
@@ -39,4 +40,11 @@ public class CanalMQConfig {
         this.partitionHash = partitionHash;
     }
 
+    public String getDynamicTopic() {
+        return dynamicTopic;
+    }
+
+    public void setDynamicTopic(String dynamicTopic) {
+        this.dynamicTopic = dynamicTopic;
+    }
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java

@@ -20,7 +20,7 @@ public interface ErosaConnection {
     /**
      * 用于快速数据查找,和dump的区别在于，seek会只给出部分的数据
      */
-    public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException;
+    public void seek(String binlogfilename, Long binlogPosition, String gtid, SinkFunction func) throws IOException;
 
     public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException;
 

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java

@@ -75,7 +75,7 @@ public class LocalBinLogConnection implements ErosaConnection {
         return running;
     }
 
-    public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
+    public void seek(String binlogfilename, Long binlogPosition, String gtid, SinkFunction func) throws IOException {
     }
 
     public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java

@@ -10,6 +10,7 @@ import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
+import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.math.NumberUtils;
 import org.slf4j.Logger;
@@ -118,7 +119,7 @@ public class MysqlConnection implements ErosaConnection {
     /**
      * 加速主备切换时的查找速度，做一些特殊优化，比如只解析事务头或者尾
      */
-    public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
+    public void seek(String binlogfilename, Long binlogPosition, String gtid, SinkFunction func) throws IOException {
         updateSettings();
         loadBinlogChecksum();
         sendBinlogDump(binlogfilename, binlogPosition);
@@ -130,6 +131,13 @@ public class MysqlConnection implements ErosaConnection {
         decoder.handle(LogEvent.QUERY_EVENT);
         decoder.handle(LogEvent.XID_EVENT);
         LogContext context = new LogContext();
+        // 若entry position存在gtid，则使用传入的gtid作为gtidSet 拼接的标准,否则同时开启gtid和tsdb时，会导致丢失gtid
+        // 而当源端数据库gtid 有purged时会有如下类似报错
+        // 'errno = 1236, sqlstate = HY000 errmsg = The slave is connecting using CHANGE MASTER TO MASTER_AUTO_POSITION = 1 ...
+        if (StringUtils.isNotEmpty(gtid)) {
+            decoder.handle(LogEvent.GTID_LOG_EVENT);
+            context.setGtidSet(MysqlGTIDSet.parse(gtid));
+        }
         context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
         while (fetcher.fetch()) {
             accumulateReceivedBytes(fetcher.limit());

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java

@@ -527,7 +527,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
         // 针对开始的第一条为非Begin记录，需要从该binlog扫描
         final java.util.concurrent.atomic.AtomicLong preTransactionStartPosition = new java.util.concurrent.atomic.AtomicLong(0L);
         mysqlConnection.reconnect();
-        mysqlConnection.seek(entryPosition.getJournalName(), 4L, new SinkFunction<LogEvent>() {
+        mysqlConnection.seek(entryPosition.getJournalName(), 4L, entryPosition.getGtid(), new SinkFunction<LogEvent>() {
 
             private LogPosition lastPosition;
 
@@ -740,7 +740,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
         try {
             mysqlConnection.reconnect();
             // 开始遍历文件
-            mysqlConnection.seek(searchBinlogFile, 4L, new SinkFunction<LogEvent>() {
+            mysqlConnection.seek(searchBinlogFile, 4L, endPosition.getGtid(), new SinkFunction<LogEvent>() {
 
                 private LogPosition lastPosition;
 
@@ -754,6 +754,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                                 event.getLogPos() - event.getEventLen(),
                                 event.getWhen() * 1000,
                                 event.getServerId());
+                            entryPosition.setGtid(event.getHeader().getGtidSetStr());
                             logPosition.setPostion(entryPosition);
                         }
 
@@ -789,6 +790,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                                     entryPosition);
                             }
                             logPosition.setPostion(entryPosition);
+                            entryPosition.setGtid(entry.getHeader().getGtid());
                         } else if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) {
                             // 当前事务开始位点
                             entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp, serverId);
@@ -796,6 +798,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                                 logger.debug("set {} to be pending start position before finding another proper one...",
                                     entryPosition);
                             }
+                            entryPosition.setGtid(entry.getHeader().getGtid());
                             logPosition.setPostion(entryPosition);
                         }
 

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java

@@ -160,7 +160,7 @@ public abstract class AbstractRequest<T> {
     }
 
     public final String formatUTCTZ(Date date) {
-        SimpleDateFormat sdf = new SimpleDateFormat("YYYY-MM-dd'T'HH:mm:ss'Z'");
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
         sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
         return sdf.format(date);
     }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java

@@ -30,6 +30,7 @@ import com.alibaba.fastsql.sql.ast.statement.SQLSelectOrderByItem;
 import com.alibaba.fastsql.sql.ast.statement.SQLTableElement;
 import com.alibaba.fastsql.sql.dialect.mysql.ast.MySqlPrimaryKey;
 import com.alibaba.fastsql.sql.dialect.mysql.ast.MySqlUnique;
+import com.alibaba.fastsql.sql.dialect.mysql.ast.expr.MySqlOrderingExpr;
 import com.alibaba.fastsql.sql.repository.Schema;
 import com.alibaba.fastsql.sql.repository.SchemaObject;
 import com.alibaba.fastsql.sql.repository.SchemaRepository;
@@ -258,6 +259,8 @@ public class MemoryTableMeta implements TableMetaTSDB {
             return ((SQLCharExpr) sqlName).getText();
         } else if (sqlName instanceof SQLMethodInvokeExpr) {
             return DruidDdlParser.unescapeName(((SQLMethodInvokeExpr) sqlName).getMethodName());
+        } else if (sqlName instanceof MySqlOrderingExpr) {
+            return getSqlName(((MySqlOrderingExpr) sqlName).getExpr());
         } else {
             return sqlName.toString();
         }

server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java

@@ -1,6 +1,7 @@
 package com.alibaba.otter.canal.common;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -67,6 +68,85 @@ public class MQMessageUtils {
                                                                                  }
                                                                              });
 
+    @SuppressWarnings("deprecation")
+    private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
+                                                                             new Function<String, List<DynamicTopicData>>() {
+
+                                                                                 public List<DynamicTopicData> apply(String pkHashConfigs) {
+                                                                                     List<DynamicTopicData> datas = Lists.newArrayList();
+                                                                                     String[] dynamicTopicArray = StringUtils.split(pkHashConfigs,
+                                                                                         ",");
+                                                                                     // schema.table
+                                                                                     for (String dynamicTopic : dynamicTopicArray) {
+                                                                                         DynamicTopicData data = new DynamicTopicData();
+
+                                                                                         if (!isWildCard(dynamicTopic)) {
+                                                                                             data.simpleName = dynamicTopic;
+                                                                                         } else {
+                                                                                             if (dynamicTopic.contains("\\.")) {
+                                                                                                 data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
+                                                                                             } else {
+                                                                                                 data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
+                                                                                             }
+                                                                                         }
+                                                                                         datas.add(data);
+                                                                                     }
+
+                                                                                     return datas;
+                                                                                 }
+                                                                             });
+
+    /**
+     * 按 schema 或者 schema+table 将 message 分配到对应topic
+     *
+     * @param message 原message
+     * @param defaultTopic 默认topic
+     * @param dynamicTopicConfigs 动态topic规则
+     * @return 分隔后的message map
+     */
+    public static Map<String, Message> messageTopics(Message message, String defaultTopic, String dynamicTopicConfigs) {
+        List<CanalEntry.Entry> entries;
+        if (message.isRaw()) {
+            List<ByteString> rawEntries = message.getRawEntries();
+            entries = new ArrayList<>(rawEntries.size());
+            for (ByteString byteString : rawEntries) {
+                CanalEntry.Entry entry;
+                try {
+                    entry = CanalEntry.Entry.parseFrom(byteString);
+                } catch (InvalidProtocolBufferException e) {
+                    throw new RuntimeException(e);
+                }
+                entries.add(entry);
+            }
+        } else {
+            entries = message.getEntries();
+        }
+        Map<String, Message> messages = new HashMap<>();
+        for (CanalEntry.Entry entry : entries) {
+            if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
+                continue;
+            }
+
+            String schemaName = entry.getHeader().getSchemaName();
+            String tableName = entry.getHeader().getTableName();
+
+            if (StringUtils.isEmpty(schemaName) || StringUtils.isEmpty(tableName)) {
+                put2MapMessage(messages, message.getId(), defaultTopic, entry);
+            } else {
+                if (matchDynamicTopic(schemaName + "." + tableName, dynamicTopicConfigs)) {
+                    put2MapMessage(messages, message.getId(), schemaName + "." + tableName, entry);
+                } else if (matchDynamicTopic(schemaName, dynamicTopicConfigs)) {
+                    put2MapMessage(messages, message.getId(), schemaName, entry);
+                } else {
+                    put2MapMessage(messages, message.getId(), defaultTopic, entry);
+                }
+            }
+
+        }
+        return messages;
+    }
+
     /**
      * 将 message 分区
      *
@@ -116,7 +196,7 @@ public class MQMessageUtils {
                 if (rowChange.getRowDatasList() != null && !rowChange.getRowDatasList().isEmpty()) {
                     String database = entry.getHeader().getSchemaName();
                     String table = entry.getHeader().getTableName();
-                    HashMode hashMode = getParitionHashColumns(database + "." + table, pkHashConfigs);
+                    HashMode hashMode = getPartitionHashColumns(database + "." + table, pkHashConfigs);
                     if (hashMode == null) {
                         // 如果都没有匹配，发送到第一个分区
                         partitionEntries[0].add(entry);
@@ -143,6 +223,9 @@ public class MQMessageUtils {
                             partitionEntries[pkHash].add(entry);
                         }
                     }
+                } else {
+                    // 针对stmt/mixed binlog格式的query事件
+                    partitionEntries[0].add(entry);
                 }
             }
         }
@@ -307,7 +390,7 @@ public class MQMessageUtils {
             if (flatMessage.getData() != null && !flatMessage.getData().isEmpty()) {
                 String database = flatMessage.getDatabase();
                 String table = flatMessage.getTable();
-                HashMode hashMode = getParitionHashColumns(database + "." + table, pkHashConfigs);
+                HashMode hashMode = getPartitionHashColumns(database + "." + table, pkHashConfigs);
                 if (hashMode == null) {
                     // 如果都没有匹配，发送到第一个分区
                     partitionMessages[0] = flatMessage;
@@ -365,6 +448,9 @@ public class MQMessageUtils {
                         idx++;
                     }
                 }
+            } else {
+                // 针对stmt/mixed binlog格式的query事件
+                partitionMessages[0] = flatMessage;
             }
         }
         return partitionMessages;
@@ -373,7 +459,7 @@ public class MQMessageUtils {
     /**
      * match return List , not match return null
      */
-    public static HashMode getParitionHashColumns(String name, String pkHashConfigs) {
+    public static HashMode getPartitionHashColumns(String name, String pkHashConfigs) {
         if (StringUtils.isEmpty(pkHashConfigs)) {
             return null;
         }
@@ -394,6 +480,34 @@ public class MQMessageUtils {
         return null;
     }
 
+    public static boolean matchDynamicTopic(String name, String dynamicTopicConfigs) {
+        if (StringUtils.isEmpty(dynamicTopicConfigs)) {
+            return false;
+        }
+
+        boolean res = false;
+        List<DynamicTopicData> datas = dynamicTopicDatas.get(dynamicTopicConfigs);
+        for (DynamicTopicData data : datas) {
+            if (data.simpleName != null) {
+                if (data.simpleName.equalsIgnoreCase(name)) {
+                    res = true;
+                    break;
+                }
+            } else if (name.contains(".")) {
+                if (data.tableRegexFilter != null && data.tableRegexFilter.filter(name)) {
+                    res = true;
+                    break;
+                }
+            } else {
+                if (data.schemaRegexFilter != null && data.schemaRegexFilter.filter(name)) {
+                    res = true;
+                    break;
+                }
+            }
+        }
+        return res;
+    }
+
     public static boolean checkPkNamesHasContain(List<String> pkNames, String name) {
         for (String pkName : pkNames) {
             if (pkName.equalsIgnoreCase(name)) {
@@ -410,6 +524,16 @@ public class MQMessageUtils {
                 '^' });
     }
 
+    private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName,
+                                       CanalEntry.Entry entry) {
+        Message message = messageMap.get(topicName);
+        if (message == null) {
+            message = new Message(messageId, new ArrayList<CanalEntry.Entry>());
+            messageMap.put(topicName, message);
+        }
+        message.getEntries().add(entry);
+    }
+
     public static class PartitionData {
 
         public String             simpleName;
@@ -424,4 +548,10 @@ public class MQMessageUtils {
         public List<String> pkNames    = Lists.newArrayList();
     }
 
+    public static class DynamicTopicData {
+
+        public String             simpleName;
+        public AviaterRegexFilter schemaRegexFilter;
+        public AviaterRegexFilter tableRegexFilter;
+    }
 }

server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java

@@ -1,6 +1,5 @@
 package com.alibaba.otter.canal.common;
 
-
 /**
  * kafka 配置项
  *
@@ -32,6 +31,7 @@ public class MQProperties {
         private Integer partition;
         private Integer partitionsNum;
         private String  partitionHash;
+        private String  dynamicTopic;
 
         public String getCanalDestination() {
             return canalDestination;
@@ -72,6 +72,14 @@ public class MQProperties {
         public void setPartitionHash(String partitionHash) {
             this.partitionHash = partitionHash;
         }
+
+        public String getDynamicTopic() {
+            return dynamicTopic;
+        }
+
+        public void setDynamicTopic(String dynamicTopic) {
+            this.dynamicTopic = dynamicTopic;
+        }
     }
 
     public String getServers() {

server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java

@@ -1,8 +1,10 @@
 package com.alibaba.otter.canal.kafka;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.clients.producer.ProducerRecord;
@@ -78,15 +80,32 @@ public class CanalKafkaProducer implements CanalMQProducer {
     @Override
     public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) {
 
+        if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) {
+            // 动态topic
+            Map<String, Message> messageMap = MQMessageUtils
+                .messageTopics(message, canalDestination.getTopic(), canalDestination.getDynamicTopic());
+
+            for (Map.Entry<String, Message> entry : messageMap.entrySet()) {
+                String topicName = entry.getKey();
+                Message messageSub = entry.getValue();
+                if (logger.isDebugEnabled()) {
+                    logger.debug("## Send message to kafka topic: " + topicName);
+                }
+                send(canalDestination, topicName, messageSub, callback);
+            }
+        } else {
+            send(canalDestination, canalDestination.getTopic(), message, callback);
+        }
+    }
+
+    private void send(MQProperties.CanalDestination canalDestination, String topicName, Message message,
+                      Callback callback) {
         // producer.beginTransaction();
         if (!kafkaProperties.getFlatMessage()) {
             try {
                 ProducerRecord<String, Message> record = null;
                 if (canalDestination.getPartition() != null) {
-                    record = new ProducerRecord<>(canalDestination.getTopic(),
+                    record = new ProducerRecord<>(topicName, canalDestination.getPartition(), null, message);
-                        canalDestination.getPartition(),
-                        null,
-                        message);
                 } else {
                     if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) {
                         Message[] messages = MQMessageUtils.messagePartition(message,
@@ -96,21 +115,20 @@ public class CanalKafkaProducer implements CanalMQProducer {
                         for (int i = 0; i < length; i++) {
                             Message messagePartition = messages[i];
                             if (messagePartition != null) {
-                                record = new ProducerRecord<>(canalDestination.getTopic(), i, null, messagePartition);
+                                record = new ProducerRecord<>(topicName, i, null, messagePartition);
                             }
                         }
                     } else {
-                        record = new ProducerRecord<>(canalDestination.getTopic(), 0, null, message);
+                        record = new ProducerRecord<>(topicName, 0, null, message);
                     }
                 }
 
                 if (record != null) {
+                    // 同步发送原生message
                     producer.send(record).get();
 
                     if (logger.isDebugEnabled()) {
-                        logger.debug("Send  message to kafka topic: [{}], packet: {}",
+                        logger.debug("Send  message to kafka topic: [{}], packet: {}", topicName, message.toString());
-                            canalDestination.getTopic(),
-                            message.toString());
                     }
                 }
             } catch (Exception e) {
@@ -124,13 +142,17 @@ public class CanalKafkaProducer implements CanalMQProducer {
             List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(message);
             if (flatMessages != null) {
                 for (FlatMessage flatMessage : flatMessages) {
-                    if (canalDestination.getPartition() != null) {
+                    if (StringUtils.isEmpty(canalDestination.getPartitionHash())) {
                         try {
-                            ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination.getTopic(),
+                            Integer partition = canalDestination.getPartition();
-                                canalDestination.getPartition(),
+                            if (partition == null) {
+                                partition = 0;
+                            }
+                            ProducerRecord<String, String> record = new ProducerRecord<String, String>(topicName,
+                                partition,
                                 null,
                                 JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue));
-                            producer2.send(record).get();
+                            producer2.send(record);
                         } catch (Exception e) {
                             logger.error(e.getMessage(), e);
                             // producer.abortTransaction();
@@ -138,8 +160,6 @@ public class CanalKafkaProducer implements CanalMQProducer {
                             return;
                         }
                     } else {
-                        if (canalDestination.getPartitionHash() != null
-                            && !canalDestination.getPartitionHash().isEmpty()) {
                         FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
                             canalDestination.getPartitionsNum(),
                             canalDestination.getPartitionHash());
@@ -148,11 +168,12 @@ public class CanalKafkaProducer implements CanalMQProducer {
                             FlatMessage flatMessagePart = partitionFlatMessage[i];
                             if (flatMessagePart != null) {
                                 try {
-                                        ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination.getTopic(),
+                                    ProducerRecord<String, String> record = new ProducerRecord<String, String>(
+                                        topicName,
                                         i,
                                         null,
                                         JSON.toJSONString(flatMessagePart, SerializerFeature.WriteMapNullValue));
-                                        producer2.send(record).get();
+                                    producer2.send(record);
                                 } catch (Exception e) {
                                     logger.error(e.getMessage(), e);
                                     // producer.abortTransaction();
@@ -161,24 +182,10 @@ public class CanalKafkaProducer implements CanalMQProducer {
                                 }
                             }
                         }
-                        } else {
-                            try {
-                                ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination.getTopic(),
-                                    0,
-                                    null,
-                                    JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue));
-                                producer2.send(record).get();
-                            } catch (Exception e) {
-                                logger.error(e.getMessage(), e);
-                                // producer.abortTransaction();
-                                callback.rollback();
-                                return;
-                            }
-                        }
                     }
                     if (logger.isDebugEnabled()) {
                         logger.debug("Send flat message to kafka topic: [{}], packet: {}",
-                            canalDestination.getTopic(),
+                            topicName,
                             JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue));
                     }
                 }
@@ -187,7 +194,6 @@ public class CanalKafkaProducer implements CanalMQProducer {
 
         // producer.commitTransaction();
         callback.commit();
-
     }
 
 }

server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java

@@ -1,7 +1,9 @@
 package com.alibaba.otter.canal.rocketmq;
 
 import java.util.List;
+import java.util.Map;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.client.producer.DefaultMQProducer;
@@ -59,11 +61,28 @@ public class CanalRocketMQProducer implements CanalMQProducer {
     @Override
     public void send(final MQProperties.CanalDestination destination, com.alibaba.otter.canal.protocol.Message data,
                      Callback callback) {
+        if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
+            // 动态topic
+            Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils
+                .messageTopics(data, destination.getTopic(), destination.getDynamicTopic());
+
+            for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
+                String topicName = entry.getKey();
+                com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
+                send(destination, topicName, messageSub, callback);
+            }
+        } else {
+            send(destination, destination.getTopic(), data, callback);
+        }
+    }
+
+    public void send(final MQProperties.CanalDestination destination, String topicName,
+                     com.alibaba.otter.canal.protocol.Message data, Callback callback) {
         if (!mqProperties.getFlatMessage()) {
             try {
                 if (destination.getPartition() != null) {
-                    Message message = new Message(destination.getTopic(), CanalMessageSerializer.serializer(data,
+                    Message message = new Message(topicName,
-                        mqProperties.isFilterTransactionEntry()));
+                        CanalMessageSerializer.serializer(data, mqProperties.isFilterTransactionEntry()));
                     if (logger.isDebugEnabled()) {
                         logger.debug("send message:{} to destination:{}, partition: {}",
                             message,
@@ -83,9 +102,8 @@ public class CanalRocketMQProducer implements CanalMQProducer {
                     }, null);
                 } else {
                     if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
-                        com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(data,
+                        com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils
-                            destination.getPartitionsNum(),
+                            .messagePartition(data, destination.getPartitionsNum(), destination.getPartitionHash());
-                            destination.getPartitionHash());
                         int length = messages.length;
                         for (int i = 0; i < length; i++) {
                             com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
@@ -97,7 +115,7 @@ public class CanalRocketMQProducer implements CanalMQProducer {
                                 }
                                 final int index = i;
                                 try {
-                                    Message message = new Message(destination.getTopic(),
+                                    Message message = new Message(topicName,
                                         CanalMessageSerializer.serializer(dataPartition,
                                             mqProperties.isFilterTransactionEntry()));
                                     this.defaultMQProducer.send(message, new MessageQueueSelector() {
@@ -135,11 +153,11 @@ public class CanalRocketMQProducer implements CanalMQProducer {
                             if (logger.isDebugEnabled()) {
                                 logger.debug("send message: {} to topic: {} fixed partition: {}",
                                     JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue),
-                                    destination.getTopic(),
+                                    topicName,
                                     destination.getPartition());
                             }
-                            Message message = new Message(destination.getTopic(), JSON.toJSONString(flatMessage,
+                            Message message = new Message(topicName,
-                                SerializerFeature.WriteMapNullValue).getBytes());
+                                JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue).getBytes());
                             this.defaultMQProducer.send(message, new MessageQueueSelector() {
 
                                 @Override
@@ -168,22 +186,23 @@ public class CanalRocketMQProducer implements CanalMQProducer {
                                     }
                                     final int index = i;
                                     try {
-                                        Message message = new Message(destination.getTopic(),
+                                        Message message = new Message(topicName,
                                             JSON.toJSONString(flatMessagePart, SerializerFeature.WriteMapNullValue)
                                                 .getBytes());
                                         this.defaultMQProducer.send(message, new MessageQueueSelector() {
 
                                             @Override
-                                            public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
+                                            public MessageQueue select(List<MessageQueue> mqs, Message msg,
+                                                                       Object arg) {
                                                 if (index > mqs.size()) {
-                                                    throw new CanalServerException("partition number is error,config num:"
+                                                    throw new CanalServerException(
+                                                        "partition number is error,config num:"
                                                                                    + destination.getPartitionsNum()
                                                                                    + ", mq num: " + mqs.size());
                                                 }
                                                 return mqs.get(index);
                                             }
-                                        },
+                                        }, null);
-                                            null);
                                     } catch (Exception e) {
                                         logger.error("send flat message to hashed partition error", e);
                                         callback.rollback();

server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java

@@ -35,6 +35,8 @@ public class CanalMQStarter {
 
     private Map<String, CanalMQRunnable> canalMQWorks   = new ConcurrentHashMap<>();
 
+    private static Thread                shutdownThread = null;
+
     public CanalMQStarter(CanalMQProducer canalMQProducer){
         this.canalMQProducer = canalMQProducer;
     }
@@ -72,7 +74,8 @@ public class CanalMQStarter {
 
             running = true;
             logger.info("## the MQ workers is running now ......");
-            Runtime.getRuntime().addShutdownHook(new Thread() {
+
+            shutdownThread = new Thread() {
 
                 public void run() {
                     try {
@@ -87,11 +90,25 @@ public class CanalMQStarter {
                     }
                 }
 
-            });
+            };
 
+            Runtime.getRuntime().addShutdownHook(shutdownThread);
         } catch (Throwable e) {
             logger.error("## Something goes wrong when starting up the canal MQ workers:", e);
-            System.exit(0);
+        }
+    }
+
+    public synchronized void destroy() {
+        running = false;
+        if (executorService != null) {
+            executorService.shutdown();
+        }
+        if (canalMQProducer != null) {
+            canalMQProducer.stop();
+        }
+        if (shutdownThread != null) {
+            Runtime.getRuntime().removeShutdownHook(shutdownThread);
+            shutdownThread = null;
         }
     }
 
@@ -137,6 +154,7 @@ public class CanalMQStarter {
                 CanalMQConfig mqConfig = canalInstance.getMqConfig();
                 canalDestination.setTopic(mqConfig.getTopic());
                 canalDestination.setPartition(mqConfig.getPartition());
+                canalDestination.setDynamicTopic(mqConfig.getDynamicTopic());
                 canalDestination.setPartitionsNum(mqConfig.getPartitionsNum());
                 canalDestination.setPartitionHash(mqConfig.getPartitionHash());