chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed admin

Browse Source
This commit is contained in:
agapple
2019-08-31 12:24:41 +08:00
parent 274e91f8aa
commit 4e0b8a0278
59 changed files with 523 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
deployer/src/main/bin/startup.bat
View File

@@ -7,7 +7,8 @@ if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0%
set conf_dir=%ENV_PATH%\..\conf
set canal_conf=%conf_dir%\canal.properties
set logback_configurationFile=%conf_dir%\logback.xml
set canal_local_conf=%conf_dir%\canal_local.properties
@rem set logback_configurationFile=%conf_dir%\logback.xml
set CLASSPATH=%conf_dir%
set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH%

31
deployer/src/main/bin/startup.sh
View File

@@ -11,6 +11,7 @@ case "`uname`" in
esac
base=${bin_abs_path}/..
canal_conf=$base/conf/canal.properties
canal_local_conf=$base/conf/canal_local.properties
logback_configurationFile=$base/conf/logback.xml
export LANG=en_US.UTF-8
export BASE=$base
@@ -48,21 +49,29 @@ in
;;
1 )
var=$*
if [ -f $var ] ; then
canal_conf=$var
if [ "$var" = "local" ]; then
canal_conf=$canal_local_conf
else
echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN."
exit
if [ -f $var ] ; then
canal_conf=$var
else
echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN."
exit
fi
fi;;
2 )
var=$1
if [ -f $var ] ; then
canal_conf=$var
else
if [ "$1" = "debug" ]; then
DEBUG_PORT=$2
DEBUG_SUSPEND="n"
JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND"
if [ "$var" = "local" ]; then
canal_conf=$canal_local_conf
else
if [ -f $var ] ; then
canal_conf=$var
else
if [ "$1" = "debug" ]; then
DEBUG_PORT=$2
DEBUG_SUSPEND="n"
JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND"
fi
fi
fi;;
* )

185
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
View File

@@ -46,7 +46,7 @@ import com.google.common.collect.MigrateMap;
*/
public class CanalController {
private static final Logger logger = LoggerFactory.getLogger(CanalController.class);
private static final Logger logger = LoggerFactory.getLogger(CanalController.class);
private Long cid;
private String ip;
private String registerIp;
@@ -54,11 +54,10 @@ public class CanalController {
private int adminPort;
// 默认使用spring的方式载入
private Map<String, InstanceConfig> instanceConfigs;
private Map<String, String> runningInstances = new MapMaker().makeMap();
private InstanceConfig globalInstanceConfig;
private Map<String, PlainCanalConfigClient> managerClients;
// 监听instance config的变化
private boolean autoScan = true;
private boolean autoScan = true;
private InstanceAction defaultAction;
private Map<InstanceMode, InstanceConfigMonitor> instanceConfigMonitors;
private CanalServerWithEmbedded embededCanalServer;
@@ -153,91 +152,88 @@ public class CanalController {
zkclientx.createPersistent(ZookeeperPathUtils.CANAL_CLUSTER_ROOT_NODE, true);
}
final ServerRunningData serverData = new ServerRunningData(cid, registerIp + ":" + port);
final ServerRunningData serverData = new ServerRunningData(registerIp + ":" + port);
ServerRunningMonitors.setServerData(serverData);
ServerRunningMonitors
.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
public ServerRunningMonitor apply(final String destination) {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
runningMonitor.setDestination(destination);
runningMonitor.setListener(new ServerRunningListener() {
public ServerRunningMonitor apply(final String destination) {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
runningMonitor.setDestination(destination);
runningMonitor.setListener(new ServerRunningListener() {
public void processActiveEnter() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.start(destination);
if (canalMQStarter != null) {
canalMQStarter.startDestination(destination);
}
runningInstances.put(destination, destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
public void processActiveEnter() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.start(destination);
if (canalMQStarter != null) {
canalMQStarter.startDestination(destination);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
public void processActiveExit() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (canalMQStarter != null) {
canalMQStarter.stopDestination(destination);
}
embededCanalServer.stop(destination);
runningInstances.remove(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
releaseCid(path);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
}
// 触发创建一下cid节点
runningMonitor.init();
return runningMonitor;
public void processActiveExit() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (canalMQStarter != null) {
canalMQStarter.stopDestination(destination);
}
embededCanalServer.stop(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
releaseCid(path);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
}
}));
// 触发创建一下cid节点
runningMonitor.init();
return runningMonitor;
}
}));
// 初始化monitor机制
autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN));
@@ -284,13 +280,26 @@ public class CanalController {
logger.info("auto notify reload {} successful.", destination);
}
@Override
public void release(String destination) {
// 此处的release，代表强制释放，主要针对HA机制释放运行，让给其他机器抢占
InstanceConfig config = instanceConfigs.get(destination);
if (config != null) {
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
if (runningMonitor.isStart()) {
runningMonitor.release();
}
}
logger.info("auto notify release {} successful.", destination);
}
};
instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
public InstanceConfigMonitor apply(InstanceMode mode) {
int scanInterval = Integer
.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
if (mode.isSpring()) {
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
@@ -366,7 +375,7 @@ public class CanalController {
}
if (config.getMode().isManager()) {
PlainCanalInstanceGenerator instanceGenerator = new PlainCanalInstanceGenerator(properties);
PlainCanalInstanceGenerator instanceGenerator = new PlainCanalInstanceGenerator();
instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress()));
instanceGenerator.setSpringXml(config.getSpringXml());
return instanceGenerator.generate(destination);
@@ -386,7 +395,7 @@ public class CanalController {
}
private PlainCanalConfigClient getManagerClient(String managerAddress) {
return new PlainCanalConfigClient(managerAddress, this.adminUser, this.adminPasswd, null, adminPort);
return new PlainCanalConfigClient(managerAddress, this.adminUser, this.adminPasswd, this.registerIp, adminPort);
}
private void initInstanceConfig(Properties properties) {
@@ -398,8 +407,7 @@ public class CanalController {
InstanceConfig oldConfig = instanceConfigs.put(destination, config);
if (oldConfig != null) {
logger
.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
logger.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
}
}
}
@@ -595,7 +603,4 @@ public class CanalController {
return instanceConfigs;
}
public Map<String, String> getRunningInstances() {
return runningInstances;
}
}

89
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
View File

@@ -11,6 +11,7 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.common.utils.AddressUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.instance.manager.plain.PlainCanal;
import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient;
@@ -27,7 +28,7 @@ public class CanalLauncher {
private static final Logger logger = LoggerFactory.getLogger(CanalLauncher.class);
public static final CountDownLatch runningLatch = new CountDownLatch(1);
private static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
new NamedThreadFactory("canal-server-scan"));
new NamedThreadFactory("canal-server-scan"));
public static void main(String[] args) {
try {
@@ -49,55 +50,54 @@ public class CanalLauncher {
if (StringUtils.isNotEmpty(managerAddress)) {
String user = properties.getProperty(CanalConstants.CANAL_ADMIN_USER);
String passwd = properties.getProperty(CanalConstants.CANAL_ADMIN_PASSWD);
String adminPort = properties.getProperty(CanalConstants.CANAL_ADMIN_PORT, "11110");
String adminPort = properties.getProperty(CanalConstants.CANAL_ADMIN_PORT);
String registerIp = properties.getProperty(CanalConstants.CANAL_REGISTER_IP);
if (StringUtils.isEmpty(adminPort)) {
adminPort = "11110";
}
if (StringUtils.isEmpty(registerIp)) {
registerIp = AddressUtils.getHostIp();
}
final PlainCanalConfigClient configClient = new PlainCanalConfigClient(managerAddress,
user,
passwd,
"",
registerIp,
Integer.parseInt(adminPort));
PlainCanal canalConfig = configClient.findServer(null);
if (canalConfig != null) {
properties = canalConfig.getProperties();
// 用本地配置覆盖
properties.put(CanalConstants.CANAL_ADMIN_MANAGER, managerAddress);
properties.put(CanalConstants.CANAL_ADMIN_USER, user);
properties.put(CanalConstants.CANAL_ADMIN_PASSWD, passwd);
properties.put(CanalConstants.CANAL_ADMIN_PORT, adminPort);
int scanIntervalInSecond = Integer
.parseInt(properties.getProperty(CanalConstants.CANAL_AUTO_SCAN_INTERVAL, "5"));
executor.scheduleWithFixedDelay(new Runnable() {
private PlainCanal lastCanalConfig;
public void run() {
try {
if (lastCanalConfig == null) {
lastCanalConfig = configClient.findServer(null);
} else {
PlainCanal newCanalConfig = configClient.findServer(lastCanalConfig.getMd5());
if (newCanalConfig != null) {
// 远程配置canal.properties修改重新加载整个应用
canalStater.stop();
Properties properties1 = newCanalConfig.getProperties();
// 用本地配置覆盖
properties1.put(CanalConstants.CANAL_ADMIN_MANAGER, managerAddress);
properties1.put(CanalConstants.CANAL_ADMIN_USER, user);
properties1.put(CanalConstants.CANAL_ADMIN_PASSWD, passwd);
properties1.put(CanalConstants.CANAL_ADMIN_PORT, adminPort);
canalStater.setProperties(properties1);
canalStater.start();
lastCanalConfig = newCanalConfig;
}
}
} catch (Throwable e) {
logger.error("scan failed", e);
}
}
}, 0, scanIntervalInSecond, TimeUnit.SECONDS);
if (canalConfig == null) {
throw new IllegalArgumentException("managerAddress:" + managerAddress
+ " can't not found config for [" + registerIp + ":" + adminPort
+ "]");
}
properties = canalConfig.getProperties();
int scanIntervalInSecond = Integer.valueOf(properties.getProperty(CanalConstants.CANAL_AUTO_SCAN_INTERVAL,
"5"));
executor.scheduleWithFixedDelay(new Runnable() {
private PlainCanal lastCanalConfig;
public void run() {
try {
if (lastCanalConfig == null) {
lastCanalConfig = configClient.findServer(null);
} else {
PlainCanal newCanalConfig = configClient.findServer(lastCanalConfig.getMd5());
if (newCanalConfig != null) {
// 远程配置canal.properties修改重新加载整个应用
canalStater.stop();
canalStater.setProperties(newCanalConfig.getProperties());
canalStater.start();
lastCanalConfig = newCanalConfig;
}
}
} catch (Throwable e) {
logger.error("scan failed", e);
}
}
}, 0, scanIntervalInSecond, TimeUnit.SECONDS);
}
canalStater.setProperties(properties);
@@ -118,4 +118,5 @@ public class CanalLauncher {
}
});
}
}

42
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStater.java
View File

@@ -1,9 +1,5 @@
package com.alibaba.otter.canal.deployer;
import java.io.File;
import java.io.FileFilter;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
@@ -17,9 +13,6 @@ import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
import com.alibaba.otter.canal.rocketmq.CanalRocketMQProducer;
import com.alibaba.otter.canal.server.CanalMQStarter;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
/**
* Canal server 启动类
@@ -76,40 +69,8 @@ public class CanalStater {
if (canalMQProducer != null) {
// disable netty
System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true");
String autoScan = CanalController.getProperty(properties, CanalConstants.CANAL_AUTO_SCAN);
// 设置为raw避免ByteString->Entry的二次解析
System.setProperty("canal.instance.memory.rawEntry", "false");
if ("true".equals(autoScan)) {
String rootDir = CanalController.getProperty(properties, CanalConstants.CANAL_CONF_DIR);
if (StringUtils.isEmpty(rootDir)) {
rootDir = "../conf";
}
File rootdir = new File(rootDir);
if (rootdir.exists()) {
File[] instanceDirs = rootdir.listFiles(new FileFilter() {
public boolean accept(File pathname) {
String filename = pathname.getName();
return pathname.isDirectory() && !"spring".equalsIgnoreCase(filename)
&& !"metrics".equalsIgnoreCase(filename);
}
});
if (instanceDirs != null && instanceDirs.length > 0) {
List<String> instances = Lists.transform(Arrays.asList(instanceDirs),
new Function<File, String>() {
@Override
public String apply(File instanceDir) {
return instanceDir.getName();
}
});
System.setProperty(CanalConstants.CANAL_DESTINATIONS, Joiner.on(",").join(instances));
}
}
} else {
String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
System.setProperty(CanalConstants.CANAL_DESTINATIONS, destinations);
}
}
logger.info("## start the canal server.");
@@ -136,7 +97,8 @@ public class CanalStater {
if (canalMQProducer != null) {
canalMQStarter = new CanalMQStarter(canalMQProducer);
MQProperties mqProperties = buildMQProperties(properties);
canalMQStarter.start(mqProperties);
String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
canalMQStarter.start(mqProperties, destinations);
controller.setCanalMQStarter(canalMQStarter);
}

48
deployer/src/main/java/com/alibaba/otter/canal/deployer/admin/CanalAdminController.java
View File

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.deployer.admin;
import java.io.File;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -14,7 +15,6 @@ import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.admin.CanalAdmin;
import com.alibaba.otter.canal.common.utils.FileUtils;
import com.alibaba.otter.canal.deployer.CanalController;
import com.alibaba.otter.canal.deployer.CanalStater;
import com.alibaba.otter.canal.deployer.InstanceConfig;
import com.alibaba.otter.canal.deployer.monitor.InstanceAction;
@@ -28,7 +28,7 @@ import com.google.common.base.Joiner;
/**
* 提供canal admin的管理操作
*
*
* @author agapple 2019年8月24日 下午11:39:01
* @since 1.1.4
*/
@@ -102,32 +102,18 @@ public class CanalAdminController implements CanalAdmin {
return false;
}
@Override
public String getInstances() {
try {
CanalController controller = canalStater.getController();
if (controller != null) {
Map<String, InstanceConfig> instanceConfigs = controller.getInstanceConfigs();
if (instanceConfigs != null) {
return Joiner.on(",").join(instanceConfigs.keySet());
}
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return "";
}
@Override
public String getRunningInstances() {
try {
CanalController controller = canalStater.getController();
if (controller != null) {
Map<String, String> instanceConfigs = controller.getRunningInstances();
if (instanceConfigs != null) {
return Joiner.on(",").join(instanceConfigs.keySet());
Map<String, CanalInstance> instances = CanalServerWithEmbedded.instance().getCanalInstances();
List<String> runningInstances = new ArrayList<String>();
instances.forEach((destination, instance) -> {
if (instance.isStart()) {
runningInstances.add(destination);
}
}
});
return Joiner.on(",").join(runningInstances);
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
@@ -173,6 +159,20 @@ public class CanalAdminController implements CanalAdmin {
return false;
}
@Override
public boolean releaseInstance(String destination) {
try {
InstanceAction instanceAction = getInstanceAction(destination);
if (instanceAction != null) {
instanceAction.release(destination);
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}
@Override
public boolean restartInstance(String destination) {
try {

5
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceAction.java
View File

@@ -13,6 +13,11 @@ public interface InstanceAction {
*/
void start(String destination);
/**
* 主动释放destination运行
*/
void release(String destination);
/**
* 停止destination
*/

7
deployer/src/main/resources/canal.properties
View File

@@ -1,7 +1,6 @@
#################################################
######### common argument #############
#################################################
canal.id = 1
# tcp bind ip
canal.ip =
# register ip to zookeeper
@@ -12,12 +11,6 @@ canal.metrics.pull.port = 11112
# canal.user = canal
# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
# canal admin config
#canal.admin.manager = 127.0.0.1:8089
canal.admin.port = 11110
canal.admin.user = admin
canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
canal.zkServers =
# flush data to zk
canal.zookeeper.flush.period = 1000

8
deployer/src/main/resources/canal_local.properties Normal file
View File

@@ -0,0 +1,8 @@
# register ip
canal.register.ip =
# canal admin config
canal.admin.manager = 127.0.0.1:8089
canal.admin.port = 11110
canal.admin.user = admin
canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441

139
deployer/src/main/resources/canal_manager.properties Normal file
View File

@@ -0,0 +1,139 @@
#################################################
######### common argument #############
#################################################
# tcp bind ip
canal.ip =
# register ip to zookeeper
canal.register.ip =
canal.port = 11111
canal.metrics.pull.port = 11112
# canal instance user/passwd
canal.user = canal
canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
# canal admin config
canal.admin.manager = 127.0.0.1:8089
canal.admin.port = 11110
canal.admin.user = admin
canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
canal.zkServers =
# flush data to zk
canal.zookeeper.flush.period = 1000
canal.withoutNetty = false
# tcp, kafka, RocketMQ
canal.serverMode = tcp
# flush meta cursor/parse position to file
canal.file.data.dir = ${canal.conf.dir}
canal.file.flush.period = 1000
## memory store RingBuffer size, should be Math.pow(2,n)
canal.instance.memory.buffer.size = 16384
## memory store RingBuffer used memory unit size , default 1kb
canal.instance.memory.buffer.memunit = 1024
## meory store gets mode used MEMSIZE or ITEMSIZE
canal.instance.memory.batch.mode = MEMSIZE
canal.instance.memory.rawEntry = true
## detecing config
canal.instance.detecting.enable = false
#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now()
canal.instance.detecting.sql = select 1
canal.instance.detecting.interval.time = 3
canal.instance.detecting.retry.threshold = 3
canal.instance.detecting.heartbeatHaEnable = false
# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery
canal.instance.transaction.size = 1024
# mysql fallback connected to new master should fallback times
canal.instance.fallbackIntervalInSeconds = 60
# network config
canal.instance.network.receiveBufferSize = 16384
canal.instance.network.sendBufferSize = 16384
canal.instance.network.soTimeout = 30
# binlog filter config
canal.instance.filter.druid.ddl = true
canal.instance.filter.query.dcl = false
canal.instance.filter.query.dml = false
canal.instance.filter.query.ddl = false
canal.instance.filter.table.error = false
canal.instance.filter.rows = false
canal.instance.filter.transaction.entry = false
# binlog format/image check
canal.instance.binlog.format = ROW,STATEMENT,MIXED
canal.instance.binlog.image = FULL,MINIMAL,NOBLOB
# binlog ddl isolation
canal.instance.get.ddl.isolation = false
# parallel parser config
canal.instance.parser.parallel = true
## concurrent thread number, default 60% available processors, suggest not to exceed Runtime.getRuntime().availableProcessors()
#canal.instance.parser.parallelThreadSize = 16
## disruptor ringbuffer size, must be power of 2
canal.instance.parser.parallelBufferSize = 256
# table meta tsdb info
canal.instance.tsdb.enable = true
canal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}
canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
canal.instance.tsdb.dbUsername = canal
canal.instance.tsdb.dbPassword = canal
# dump snapshot interval, default 24 hour
canal.instance.tsdb.snapshot.interval = 24
# purge snapshot expire , default 360 hour(15 days)
canal.instance.tsdb.snapshot.expire = 360
# aliyun ak/sk , support rds/mq
canal.aliyun.accessKey =
canal.aliyun.secretKey =
#################################################
######### destinations #############
#################################################
canal.destinations =
# conf root dir
canal.conf.dir = ../conf
# auto scan instance dir add/remove and start/stop instance
canal.auto.scan = true
canal.auto.scan.interval = 5
canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml
canal.instance.global.mode = manager
canal.instance.global.lazy = false
canal.instance.global.manager.address = ${canal.admin.manager}
#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
canal.instance.global.spring.xml = classpath:spring/file-instance.xml
#canal.instance.global.spring.xml = classpath:spring/default-instance.xml
##################################################
######### MQ #############
##################################################
canal.mq.servers = 127.0.0.1:6667
canal.mq.retries = 0
canal.mq.batchSize = 16384
canal.mq.maxRequestSize = 1048576
canal.mq.lingerMs = 100
canal.mq.bufferMemory = 33554432
canal.mq.canalBatchSize = 50
canal.mq.canalGetTimeout = 100
canal.mq.flatMessage = true
canal.mq.compressionType = none
canal.mq.acks = all
#canal.mq.properties. =
canal.mq.producerGroup = test
# Set this value to "cloud", if you want open message trace feature in aliyun.
canal.mq.accessChannel = local
# aliyun mq namespace
#canal.mq.namespace =
##################################################
######### Kafka Kerberos Info #############
##################################################
canal.mq.kafka.kerberos.enable = false
canal.mq.kafka.kerberos.krb5FilePath = "../conf/kerberos/krb5.conf"
canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf"