mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
fixed issue #2099 , support remote config poll
This commit is contained in:
agapple
@@ -8,9 +8,7 @@
|
||||
<version>1.1.4-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>canal-admin-server</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
@@ -199,6 +197,39 @@
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<version>3.0.2</version>
|
||||
<configuration>
|
||||
<excludes>
|
||||
<exclude>application.yml</exclude>
|
||||
<exclude>canal_manager.sql</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.2.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>assemble</id>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
<phase>package</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<appendAssemblyId>false</appendAssemblyId>
|
||||
<attach>false</attach>
|
||||
<descriptors>
|
||||
<descriptor>${basedir}/src/main/assembly/release.xml</descriptor>
|
||||
</descriptors>
|
||||
<finalName>canal-admin-${project.version}</finalName>
|
||||
<outputDirectory>${project.basedir}/../../target</outputDirectory>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
|
||||
41
canal-admin/canal-admin-server/src/main/assembly/release.xml
Normal file
41
canal-admin/canal-admin-server/src/main/assembly/release.xml
Normal file
@@ -0,0 +1,41 @@
|
||||
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||
<id>dist</id>
|
||||
<formats>
|
||||
<format>tar.gz</format>
|
||||
</formats>
|
||||
<includeBaseDirectory>false</includeBaseDirectory>
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory>.</directory>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
<includes>
|
||||
<include>README*</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>./src/main/bin</directory>
|
||||
<outputDirectory>bin</outputDirectory>
|
||||
<includes>
|
||||
<include>**/*</include>
|
||||
</includes>
|
||||
<fileMode>0755</fileMode>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>./src/main/resources</directory>
|
||||
<outputDirectory>/conf</outputDirectory>
|
||||
<includes>
|
||||
<include>application.yml</include>
|
||||
<include>canal_manager.sql</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>./target</directory>
|
||||
<outputDirectory>/lib</outputDirectory>
|
||||
<includes>
|
||||
<include>canal-admin-*.jar</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
</assembly>
|
||||
5
canal-admin/canal-admin-server/src/main/bin/restart.sh
Normal file
5
canal-admin/canal-admin-server/src/main/bin/restart.sh
Normal file
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
|
||||
sh stop.sh
|
||||
|
||||
sh startup.sh
|
||||
22
canal-admin/canal-admin-server/src/main/bin/startup.bat
Executable file
22
canal-admin/canal-admin-server/src/main/bin/startup.bat
Executable file
@@ -0,0 +1,22 @@
|
||||
@echo off
|
||||
@if not "%ECHO%" == "" echo %ECHO%
|
||||
@if "%OS%" == "Windows_NT" setlocal
|
||||
|
||||
set ENV_PATH=.\
|
||||
if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0%
|
||||
|
||||
set conf_dir=%ENV_PATH%\..\conf
|
||||
|
||||
set CLASSPATH=%conf_dir%
|
||||
set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH%
|
||||
|
||||
set JAVA_MEM_OPTS= -Xms128m -Xmx512m
|
||||
set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8
|
||||
set ADAPTER_OPTS= -DappName=canal-admin
|
||||
|
||||
set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %ADAPTER_OPTS%
|
||||
|
||||
set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.admin.CanalAdminApplication
|
||||
echo start cmd : %CMD_STR%
|
||||
|
||||
java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.admin.CanalAdminApplication
|
||||
84
canal-admin/canal-admin-server/src/main/bin/startup.sh
Normal file
84
canal-admin/canal-admin-server/src/main/bin/startup.sh
Normal file
@@ -0,0 +1,84 @@
|
||||
#!/bin/bash
|
||||
|
||||
current_path=`pwd`
|
||||
case "`uname`" in
|
||||
Linux)
|
||||
bin_abs_path=$(readlink -f $(dirname $0))
|
||||
;;
|
||||
*)
|
||||
bin_abs_path=`cd $(dirname $0); pwd`
|
||||
;;
|
||||
esac
|
||||
base=${bin_abs_path}/..
|
||||
export LANG=en_US.UTF-8
|
||||
export BASE=$base
|
||||
|
||||
if [ -f $base/bin/adapter.pid ] ; then
|
||||
echo "found adapter.pid , Please run stop.sh first ,then startup.sh" 2>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d $base/logs ] ; then
|
||||
mkdir -p $base/logs
|
||||
fi
|
||||
|
||||
## set java path
|
||||
if [ -z "$JAVA" ] ; then
|
||||
JAVA=$(which java)
|
||||
fi
|
||||
|
||||
ALIBABA_JAVA="/usr/alibaba/java/bin/java"
|
||||
TAOBAO_JAVA="/opt/taobao/java/bin/java"
|
||||
if [ -z "$JAVA" ]; then
|
||||
if [ -f $ALIBABA_JAVA ] ; then
|
||||
JAVA=$ALIBABA_JAVA
|
||||
elif [ -f $TAOBAO_JAVA ] ; then
|
||||
JAVA=$TAOBAO_JAVA
|
||||
else
|
||||
echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
case "$#"
|
||||
in
|
||||
0 )
|
||||
;;
|
||||
2 )
|
||||
if [ "$1" = "debug" ]; then
|
||||
DEBUG_PORT=$2
|
||||
DEBUG_SUSPEND="n"
|
||||
JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND"
|
||||
fi
|
||||
;;
|
||||
* )
|
||||
echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN."
|
||||
exit;;
|
||||
esac
|
||||
|
||||
str=`file -L $JAVA | grep 64-bit`
|
||||
if [ -n "$str" ]; then
|
||||
JAVA_OPTS="-server -Xms2048m -Xmx3072m"
|
||||
else
|
||||
JAVA_OPTS="-server -Xms1024m -Xmx1024m"
|
||||
fi
|
||||
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent -XX:+PrintAdaptiveSizePolicy -XX:+PrintTenuringDistribution"
|
||||
JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8"
|
||||
CANAL_OPTS="-DappName=canal-admin"
|
||||
|
||||
for i in $base/lib/*;
|
||||
do CLASSPATH=$i:"$CLASSPATH";
|
||||
done
|
||||
|
||||
CLASSPATH="$base/conf:$CLASSPATH";
|
||||
|
||||
echo "cd to $bin_abs_path for workaround relative path"
|
||||
cd $bin_abs_path
|
||||
|
||||
echo CLASSPATH :$CLASSPATH
|
||||
$JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.admin.CanalAdminApplication 1>>/dev/null 2>&1 &
|
||||
echo $! > $base/bin/admin.pid
|
||||
|
||||
echo "cd to $current_path for continue"
|
||||
cd $current_path
|
||||
65
canal-admin/canal-admin-server/src/main/bin/stop.sh
Normal file
65
canal-admin/canal-admin-server/src/main/bin/stop.sh
Normal file
@@ -0,0 +1,65 @@
|
||||
#!/bin/bash
|
||||
|
||||
cygwin=false;
|
||||
linux=false;
|
||||
case "`uname`" in
|
||||
CYGWIN*)
|
||||
cygwin=true
|
||||
;;
|
||||
Linux*)
|
||||
linux=true
|
||||
;;
|
||||
esac
|
||||
|
||||
get_pid() {
|
||||
STR=$1
|
||||
PID=$2
|
||||
if $cygwin; then
|
||||
JAVA_CMD="$JAVA_HOME\bin\java"
|
||||
JAVA_CMD=`cygpath --path --unix $JAVA_CMD`
|
||||
JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'`
|
||||
else
|
||||
if $linux; then
|
||||
if [ ! -z "$PID" ]; then
|
||||
JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
|
||||
else
|
||||
JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
|
||||
fi
|
||||
else
|
||||
if [ ! -z "$PID" ]; then
|
||||
JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
|
||||
else
|
||||
JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'`
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
echo $JAVA_PID;
|
||||
}
|
||||
|
||||
base=`dirname $0`/..
|
||||
pidfile=$base/bin/admin.pid
|
||||
if [ ! -f "$pidfile" ];then
|
||||
echo "canal-admin is not running. exists"
|
||||
exit
|
||||
fi
|
||||
|
||||
pid=`cat $pidfile`
|
||||
if [ "$pid" == "" ] ; then
|
||||
pid=`get_pid "appName=canal-admin"`
|
||||
fi
|
||||
|
||||
echo -e "`hostname`: stopping canal $pid ... "
|
||||
kill $pid
|
||||
|
||||
LOOPS=0
|
||||
while (true);
|
||||
do
|
||||
gpid=`get_pid "appName=canal-admin" "$pid"`
|
||||
if [ "$gpid" == "" ] ; then
|
||||
echo "Oook! cost:$LOOPS"
|
||||
`rm $pidfile`
|
||||
break;
|
||||
fi
|
||||
let LOOPS=LOOPS+1
|
||||
sleep 1
|
||||
done
|
||||
@@ -74,8 +74,8 @@ public class WebConfig implements WebMvcConfigurer {
|
||||
httpServletResponse.setContentType("application/json;charset=UTF-8");
|
||||
PrintWriter out = httpServletResponse.getWriter();
|
||||
out.print(json);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
} catch (Throwable e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -84,6 +84,7 @@ public class WebConfig implements WebMvcConfigurer {
|
||||
}
|
||||
})
|
||||
.addPathPatterns("/api/**")
|
||||
.excludePathPatterns("/api/**/config/**")
|
||||
.excludePathPatterns("/api/**/user/login")
|
||||
.excludePathPatterns("/api/**/user/logout")
|
||||
.excludePathPatterns("/api/**/user/info");
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
package com.alibaba.otter.canal.admin.controller;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.alibaba.otter.canal.admin.model.BaseModel;
|
||||
import com.alibaba.otter.canal.admin.model.CanalConfig;
|
||||
@@ -31,17 +36,6 @@ public class CanalConfigController {
|
||||
return BaseModel.getInstance(canalConfigService.getCanalConfig());
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取配置信息摘要(无配置内容)
|
||||
*
|
||||
* @param env 环境变量
|
||||
* @return 配置信息摘要
|
||||
*/
|
||||
@GetMapping(value = "/config/summary")
|
||||
public BaseModel<CanalConfig> canalConfigSummary(@PathVariable String env) {
|
||||
return BaseModel.getInstance(canalConfigService.getCanalConfigSummary());
|
||||
}
|
||||
|
||||
/**
|
||||
* 修改配置
|
||||
*
|
||||
@@ -54,4 +48,5 @@ public class CanalConfigController {
|
||||
canalConfigService.updateContent(canalConfig);
|
||||
return BaseModel.getInstance("success");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
package com.alibaba.otter.canal.admin.controller;
|
||||
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestHeader;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.alibaba.otter.canal.admin.model.BaseModel;
|
||||
import com.alibaba.otter.canal.admin.model.CanalConfig;
|
||||
import com.alibaba.otter.canal.admin.model.CanalInstanceConfig;
|
||||
import com.alibaba.otter.canal.admin.service.CanalConfigService;
|
||||
import com.alibaba.otter.canal.admin.service.CanalInstanceService;
|
||||
import com.alibaba.otter.canal.protocol.SecurityUtil;
|
||||
import com.google.common.base.Joiner;
|
||||
|
||||
/**
|
||||
* Canal Instance配置管理控制层
|
||||
*
|
||||
* @author rewerma 2019-07-13 下午05:12:16
|
||||
* @version 1.0.0
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/{env}/config")
|
||||
public class PollConfigController {
|
||||
|
||||
private static final byte[] seeds = "canal is best!".getBytes();
|
||||
|
||||
@Autowired
|
||||
CanalInstanceService canalInstanceConfigService;
|
||||
|
||||
@Autowired
|
||||
CanalConfigService canalConfigService;
|
||||
|
||||
@Value(value = "${canal.adminUser}")
|
||||
String user;
|
||||
|
||||
@Value(value = "${canal.adminPasswd}")
|
||||
String passwd;
|
||||
|
||||
/**
|
||||
* 获取server全局配置
|
||||
*/
|
||||
@GetMapping(value = "/server_poll")
|
||||
public BaseModel<CanalConfig> canalConfigPoll(@RequestHeader String user, @RequestHeader String passwd,
|
||||
@PathVariable String env, @RequestParam String md5) {
|
||||
if (!auth(user, passwd)) {
|
||||
throw new RuntimeException("auth :" + user + " is failed");
|
||||
}
|
||||
|
||||
CanalConfig config = canalConfigService.getCanalConfig();
|
||||
if (StringUtils.isEmpty(md5)) {
|
||||
return BaseModel.getInstance(config);
|
||||
} else {
|
||||
|
||||
try {
|
||||
String newMd5 = SecurityUtil.md5String(config.getContent());
|
||||
if (StringUtils.equals(md5, newMd5)) {
|
||||
config.setContent(null);
|
||||
}
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
}
|
||||
|
||||
return BaseModel.getInstance(config);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取单个instance的配置
|
||||
*/
|
||||
@GetMapping(value = "/instance_poll/{destination}")
|
||||
public BaseModel<CanalInstanceConfig> instanceConfigPoll(@RequestHeader String user, @RequestHeader String passwd,
|
||||
@PathVariable String env,
|
||||
@PathVariable String destination, @RequestParam String md5) {
|
||||
if (!auth(user, passwd)) {
|
||||
throw new RuntimeException("auth :" + user + " is failed");
|
||||
}
|
||||
|
||||
CanalInstanceConfig config = canalInstanceConfigService.findOne(destination);
|
||||
if (StringUtils.isEmpty(md5)) {
|
||||
return BaseModel.getInstance(config);
|
||||
} else {
|
||||
try {
|
||||
String newMd5 = SecurityUtil.md5String(config.getContent());
|
||||
if (StringUtils.equals(md5, newMd5)) {
|
||||
config.setContent(null);
|
||||
}
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
}
|
||||
|
||||
return BaseModel.getInstance(config);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取对应server(ip+port)所需要运行的instance列表
|
||||
*/
|
||||
@GetMapping(value = "/instances_poll")
|
||||
public BaseModel<CanalInstanceConfig> instancesPoll(@RequestHeader String user, @RequestHeader String passwd,
|
||||
@PathVariable String env, @RequestParam String ip,
|
||||
@RequestParam String port, @RequestParam String md5) {
|
||||
if (!auth(user, passwd)) {
|
||||
throw new RuntimeException("auth :" + user + " is failed");
|
||||
}
|
||||
|
||||
CanalInstanceConfig canalInstanceConfig = new CanalInstanceConfig();
|
||||
List<CanalInstanceConfig> configs = canalInstanceConfigService.findList(canalInstanceConfig);
|
||||
List<String> instances = configs.stream().map(config -> config.getName()).collect(Collectors.toList());
|
||||
String data = Joiner.on(',').join(instances);
|
||||
canalInstanceConfig.setContent(data);
|
||||
if (StringUtils.isEmpty(md5)) {
|
||||
return BaseModel.getInstance(canalInstanceConfig);
|
||||
} else {
|
||||
try {
|
||||
String newMd5 = SecurityUtil.md5String(canalInstanceConfig.getContent());
|
||||
if (StringUtils.equals(md5, newMd5)) {
|
||||
canalInstanceConfig.setContent(null);
|
||||
}
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
}
|
||||
|
||||
return BaseModel.getInstance(canalInstanceConfig);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean auth(String user, String passwd) {
|
||||
// 如果user/passwd密码为空,则任何用户账户都能登录
|
||||
if ((StringUtils.isEmpty(this.user) || StringUtils.equals(this.user, user))) {
|
||||
if (StringUtils.isEmpty(this.passwd)) {
|
||||
return true;
|
||||
} else if (StringUtils.isEmpty(passwd)) {
|
||||
// 如果server密码有配置,客户端密码为空,则拒绝
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
// manager这里保存了原始密码，反过来和canal发送过来的进行校验
|
||||
byte[] passForClient = SecurityUtil.scramble411(this.passwd.getBytes(), seeds);
|
||||
return SecurityUtil.scrambleServerAuth(passForClient, SecurityUtil.hexStr2Bytes(passwd), seeds);
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,7 @@ public class CanalConfig extends Model {
|
||||
private Long id;
|
||||
private String name;
|
||||
private String content;
|
||||
private String status;
|
||||
private Date modifiedTime;
|
||||
|
||||
public Long getId() {
|
||||
@@ -66,4 +67,13 @@ public class CanalConfig extends Model {
|
||||
public void setModifiedTime(Date modifiedTime) {
|
||||
this.modifiedTime = modifiedTime;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -2,10 +2,11 @@ package com.alibaba.otter.canal.admin.model;
|
||||
|
||||
import io.ebean.Finder;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Transient;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* Canal实例配置信息实体类
|
||||
@@ -33,6 +34,7 @@ public class CanalInstanceConfig extends Model {
|
||||
private Long id;
|
||||
private String name;
|
||||
private String content;
|
||||
private String status;
|
||||
private Date modifiedTime;
|
||||
|
||||
@Transient
|
||||
@@ -64,6 +66,14 @@ public class CanalInstanceConfig extends Model {
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public Date getModifiedTime() {
|
||||
return modifiedTime;
|
||||
}
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
package com.alibaba.otter.canal.admin.model;
|
||||
|
||||
import io.ebean.Finder;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
|
||||
import io.ebean.Finder;
|
||||
|
||||
/**
|
||||
* 节点信息实体类
|
||||
*
|
||||
@@ -35,13 +35,14 @@ public class NodeServer extends Model {
|
||||
private Long id;
|
||||
private String name;
|
||||
private String ip;
|
||||
private Integer port;
|
||||
private Integer port2;
|
||||
private Integer status;
|
||||
private Integer adminPort;
|
||||
private Integer metricPort;
|
||||
private Integer tcpPort;
|
||||
private String status;
|
||||
private Date modifiedTime;
|
||||
|
||||
public void init() {
|
||||
status = -1;
|
||||
status = "-1";
|
||||
}
|
||||
|
||||
public Long getId() {
|
||||
@@ -68,27 +69,35 @@ public class NodeServer extends Model {
|
||||
this.ip = ip;
|
||||
}
|
||||
|
||||
public Integer getPort() {
|
||||
return port;
|
||||
public Integer getAdminPort() {
|
||||
return adminPort;
|
||||
}
|
||||
|
||||
public void setPort(Integer port) {
|
||||
this.port = port;
|
||||
public void setAdminPort(Integer adminPort) {
|
||||
this.adminPort = adminPort;
|
||||
}
|
||||
|
||||
public Integer getPort2() {
|
||||
return port2;
|
||||
public Integer getMetricPort() {
|
||||
return metricPort;
|
||||
}
|
||||
|
||||
public void setPort2(Integer port2) {
|
||||
this.port2 = port2;
|
||||
public void setMetricPort(Integer metricPort) {
|
||||
this.metricPort = metricPort;
|
||||
}
|
||||
|
||||
public Integer getStatus() {
|
||||
public Integer getTcpPort() {
|
||||
return tcpPort;
|
||||
}
|
||||
|
||||
public void setTcpPort(Integer tcpPort) {
|
||||
this.tcpPort = tcpPort;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(Integer status) {
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package com.alibaba.otter.canal.admin.service;
|
||||
|
||||
import com.alibaba.otter.canal.admin.model.CanalInstanceConfig;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.alibaba.otter.canal.admin.model.CanalInstanceConfig;
|
||||
|
||||
/**
|
||||
* Canal实例配置信息业务层接口
|
||||
*
|
||||
@@ -19,6 +19,8 @@ public interface CanalInstanceService {
|
||||
|
||||
CanalInstanceConfig detail(Long id);
|
||||
|
||||
CanalInstanceConfig findOne(String name);
|
||||
|
||||
void updateContent(CanalInstanceConfig canalInstanceConfig);
|
||||
|
||||
void delete(Long id);
|
||||
|
||||
@@ -40,7 +40,7 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
|
||||
List<NodeServer> nodeServers = NodeServer.find.query().findList();
|
||||
for (NodeServer nodeServer : nodeServers) {
|
||||
String runningInstances = SimpleAdminConnectors.execute(nodeServer.getIp(),
|
||||
nodeServer.getPort(),
|
||||
nodeServer.getAdminPort(),
|
||||
AdminConnector::getRunningInstances);
|
||||
if (runningInstances == null) {
|
||||
continue;
|
||||
@@ -79,6 +79,16 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanalInstanceConfig findOne(String name) {
|
||||
CanalInstanceConfig config = CanalInstanceConfig.find.query()
|
||||
.setDisableLazyLoading(true)
|
||||
.where()
|
||||
.eq("name", name)
|
||||
.findOne();
|
||||
return config;
|
||||
}
|
||||
|
||||
public Map<String, String> remoteInstanceLog(Long id, Long nodeId) {
|
||||
Map<String, String> result = new HashMap<>();
|
||||
|
||||
@@ -92,7 +102,7 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
|
||||
}
|
||||
|
||||
String log = SimpleAdminConnectors.execute(nodeServer.getIp(),
|
||||
nodeServer.getPort(),
|
||||
nodeServer.getAdminPort(),
|
||||
adminConnector -> adminConnector.instanceLog(canalInstanceConfig.getName(), null, 100));
|
||||
|
||||
result.put("instance", canalInstanceConfig.getName());
|
||||
@@ -124,11 +134,11 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
|
||||
Boolean resutl = null;
|
||||
if ("start".equals(option)) {
|
||||
resutl = SimpleAdminConnectors.execute(nodeServer.getIp(),
|
||||
nodeServer.getPort(),
|
||||
nodeServer.getAdminPort(),
|
||||
adminConnector -> adminConnector.startInstance(canalInstanceConfig.getName()));
|
||||
} else if ("stop".equals(option)) {
|
||||
resutl = SimpleAdminConnectors.execute(nodeServer.getIp(),
|
||||
nodeServer.getPort(),
|
||||
nodeServer.getAdminPort(),
|
||||
adminConnector -> adminConnector.stopInstance(canalInstanceConfig.getName()));
|
||||
} else {
|
||||
return false;
|
||||
@@ -139,4 +149,5 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
|
||||
}
|
||||
return resutl;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ public class NodeServerServiceImpl implements NodeServerService {
|
||||
int cnt = NodeServer.find.query()
|
||||
.where()
|
||||
.eq("ip", nodeServer.getIp())
|
||||
.eq("port", nodeServer.getPort())
|
||||
.eq("admin_port", nodeServer.getAdminPort())
|
||||
.findCount();
|
||||
if (cnt > 0) {
|
||||
throw new ServiceException("节点信息已存在");
|
||||
@@ -48,14 +48,14 @@ public class NodeServerServiceImpl implements NodeServerService {
|
||||
int cnt = NodeServer.find.query()
|
||||
.where()
|
||||
.eq("ip", nodeServer.getIp())
|
||||
.eq("port", nodeServer.getPort())
|
||||
.eq("admin_port", nodeServer.getAdminPort())
|
||||
.ne("id", nodeServer.getId())
|
||||
.findCount();
|
||||
if (cnt > 0) {
|
||||
throw new ServiceException("节点信息已存在");
|
||||
}
|
||||
|
||||
nodeServer.update("name", "ip", "port", "port2");
|
||||
nodeServer.update("name", "ip", "admin_port", "tcp_port", "metric_port");
|
||||
}
|
||||
|
||||
public void delete(Long id) {
|
||||
@@ -86,8 +86,8 @@ public class NodeServerServiceImpl implements NodeServerService {
|
||||
// get all nodes status
|
||||
for (NodeServer ns : nodeServers) {
|
||||
futures.add(executorService.submit(() -> {
|
||||
boolean status = SimpleAdminConnectors.execute(ns.getIp(), ns.getPort(), AdminConnector::check);
|
||||
ns.setStatus(status ? 1 : 0);
|
||||
boolean status = SimpleAdminConnectors.execute(ns.getIp(), ns.getAdminPort(), AdminConnector::check);
|
||||
ns.setStatus(status ? "1" : "0");
|
||||
return !status;
|
||||
}));
|
||||
}
|
||||
@@ -115,7 +115,7 @@ public class NodeServerServiceImpl implements NodeServerService {
|
||||
return "";
|
||||
}
|
||||
return SimpleAdminConnectors.execute(nodeServer.getIp(),
|
||||
nodeServer.getPort(),
|
||||
nodeServer.getAdminPort(),
|
||||
adminConnector -> adminConnector.canalLog(100));
|
||||
}
|
||||
|
||||
@@ -126,9 +126,9 @@ public class NodeServerServiceImpl implements NodeServerService {
|
||||
}
|
||||
Boolean result = null;
|
||||
if ("start".equals(option)) {
|
||||
result = SimpleAdminConnectors.execute(nodeServer.getIp(), nodeServer.getPort(), AdminConnector::start);
|
||||
result = SimpleAdminConnectors.execute(nodeServer.getIp(), nodeServer.getAdminPort(), AdminConnector::start);
|
||||
} else if ("stop".equals(option)) {
|
||||
result = SimpleAdminConnectors.execute(nodeServer.getIp(), nodeServer.getPort(), AdminConnector::stop);
|
||||
result = SimpleAdminConnectors.execute(nodeServer.getIp(), nodeServer.getAdminPort(), AdminConnector::stop);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -13,3 +13,7 @@ spring.datasource:
|
||||
hikari:
|
||||
maximum-pool-size: 10
|
||||
minimum-idle: 1
|
||||
|
||||
canal:
|
||||
adminUser: admin
|
||||
adminPasswd: admin
|
||||
|
||||
@@ -13,10 +13,11 @@ CREATE TABLE `canal_adapter_config` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
||||
`category` varchar(45) NOT NULL,
|
||||
`name` varchar(45) NOT NULL,
|
||||
`status` varchar(45) DEFAULT NULL,
|
||||
`content` text NOT NULL,
|
||||
`modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4;
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for canal_config
|
||||
@@ -25,11 +26,12 @@ DROP TABLE IF EXISTS `canal_config`;
|
||||
CREATE TABLE `canal_config` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
||||
`name` varchar(45) NOT NULL,
|
||||
`status` varchar(45) DEFAULT NULL,
|
||||
`content` text NOT NULL,
|
||||
`modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `name_UNIQUE` (`name`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4;
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for canal_instance_config
|
||||
@@ -38,11 +40,12 @@ DROP TABLE IF EXISTS `canal_instance_config`;
|
||||
CREATE TABLE `canal_instance_config` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
||||
`name` varchar(45) NOT NULL,
|
||||
`status` varchar(45) DEFAULT NULL,
|
||||
`content` text NOT NULL,
|
||||
`modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `name_UNIQUE` (`name`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4;
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for canal_node_server
|
||||
@@ -52,9 +55,10 @@ CREATE TABLE `canal_node_server` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
||||
`name` varchar(63) NOT NULL,
|
||||
`ip` varchar(63) NOT NULL,
|
||||
`port` int(11) DEFAULT NULL,
|
||||
`port2` int(11) DEFAULT NULL,
|
||||
`status` int(11) NOT NULL,
|
||||
`admin_port` int(11) DEFAULT NULL,
|
||||
`tcp_port` int(11) DEFAULT NULL,
|
||||
`metric_port` int(11) DEFAULT NULL,
|
||||
`status` varchar(45) DEFAULT NULL,
|
||||
`modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;
|
||||
@@ -73,7 +77,7 @@ CREATE TABLE `canal_user` (
|
||||
`avatar` varchar(255) DEFAULT NULL,
|
||||
`creation_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4;
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
-- ----------------------------
|
||||
-- Records of canal_user
|
||||
|
||||
@@ -1 +1 @@
|
||||
<!DOCTYPE html><html><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=viewport content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><link rel=icon href=/favicon.ico><title>Canal Admin</title><link href=/static/css/chunk-elementUI.18b11d0e.css rel=stylesheet><link href=/static/css/chunk-libs.5cf311f0.css rel=stylesheet><link href=/static/css/app.bb951cb3.css rel=stylesheet></head><body><noscript><strong>We're sorry but Canal Admin doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id=app></div><script src=/static/js/chunk-elementUI.667f4c87.js></script><script src=/static/js/chunk-libs.c04beefc.js></script><script>(function(e){function n(n){for(var r,c,f=n[0],a=n[1],i=n[2],h=0,d=[];h<f.length;h++)c=f[h],u[c]&&d.push(u[c][0]),u[c]=0;for(r in a)Object.prototype.hasOwnProperty.call(a,r)&&(e[r]=a[r]);l&&l(n);while(d.length)d.shift()();return o.push.apply(o,i||[]),t()}function t(){for(var e,n=0;n<o.length;n++){for(var t=o[n],r=!0,c=1;c<t.length;c++){var f=t[c];0!==u[f]&&(r=!1)}r&&(o.splice(n--,1),e=a(a.s=t[0]))}return e}var r={},c={runtime:0},u={runtime:0},o=[];function f(e){return a.p+"static/js/"+({}[e]||e)+"."+{"chunk-101fc062":"372a5ca5","chunk-137eaf06":"539d27c7","chunk-238a81e9":"da844a27","chunk-37c49cbf":"92ebe0ae","chunk-3fcdf643":"4b7133b8","chunk-1ef583f0":"c799c803","chunk-5560c396":"5eb91155","chunk-6f6d6145":"385d3c06","chunk-4f09fed2":"d107437b","chunk-69386cf0":"bdbe3f0c","chunk-7ec889b7":"192dc04c","chunk-e1a839e4":"7ee86dd8"}[e]+".js"}function a(n){if(r[n])return r[n].exports;var t=r[n]={i:n,l:!1,exports:{}};return e[n].call(t.exports,t,t.exports,a),t.l=!0,t.exports}a.e=function(e){var n=[],t={"chunk-101fc062":1,"chunk-238a81e9":1,"chunk-37c49cbf":1,"chunk-1ef583f0":1,"chunk-5560c396":1,"chunk-6f6d6145":1,"chunk-4f09fed2":1,"chunk-69386cf0":1,"chunk-7ec889b7":1};c[e]?n.push(c[e]):0!==c[e]&&t[e]&&n.push(c[e]=new Promise(function(n,t){for(var r="static/css/"+({}[e]||e)+"."+{"chunk-101fc062":"fad9926f","chunk-137eaf06":"31d6cfe0","chunk-238a81e9":"e8e2beee","chunk-37c49cbf":"efc21a9c","chunk-3fcdf643":"31d6cfe0","chunk-1ef583f0":"ec3a43f5","chunk-5560c396":"b660603c","chunk-6f6d6145":"bf63041a","chunk-4f09fed2":"70ec0b86","chunk-69386cf0":"741ff14e","chunk-7ec889b7":"c0585512","chunk-e1a839e4":"31d6cfe0"}[e]+".css",u=a.p+r,o=document.getElementsByTagName("link"),f=0;f<o.length;f++){var i=o[f],h=i.getAttribute("data-href")||i.getAttribute("href");if("stylesheet"===i.rel&&(h===r||h===u))return n()}var d=document.getElementsByTagName("style");for(f=0;f<d.length;f++){i=d[f],h=i.getAttribute("data-href");if(h===r||h===u)return n()}var l=document.createElement("link");l.rel="stylesheet",l.type="text/css",l.onload=n,l.onerror=function(n){var r=n&&n.target&&n.target.src||u,o=new Error("Loading CSS chunk "+e+" failed.\n("+r+")");o.code="CSS_CHUNK_LOAD_FAILED",o.request=r,delete c[e],l.parentNode.removeChild(l),t(o)},l.href=u;var s=document.getElementsByTagName("head")[0];s.appendChild(l)}).then(function(){c[e]=0}));var r=u[e];if(0!==r)if(r)n.push(r[2]);else{var o=new Promise(function(n,t){r=u[e]=[n,t]});n.push(r[2]=o);var i,h=document.createElement("script");h.charset="utf-8",h.timeout=120,a.nc&&h.setAttribute("nonce",a.nc),h.src=f(e),i=function(n){h.onerror=h.onload=null,clearTimeout(d);var t=u[e];if(0!==t){if(t){var r=n&&("load"===n.type?"missing":n.type),c=n&&n.target&&n.target.src,o=new Error("Loading chunk "+e+" failed.\n("+r+": "+c+")");o.type=r,o.request=c,t[1](o)}u[e]=void 0}};var d=setTimeout(function(){i({type:"timeout",target:h})},12e4);h.onerror=h.onload=i,document.head.appendChild(h)}return Promise.all(n)},a.m=e,a.c=r,a.d=function(e,n,t){a.o(e,n)||Object.defineProperty(e,n,{enumerable:!0,get:t})},a.r=function(e){"undefined"!==typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},a.t=function(e,n){if(1&n&&(e=a(e)),8&n)return e;if(4&n&&"object"===typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(a.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&n&&"string"!=typeof e)for(var r in e)a.d(t,r,function(n){return e[n]}.bind(null,r));return t},a.n=function(e){var n=e&&e.__esModule?function(){return e["default"]}:function(){return e};return a.d(n,"a",n),n},a.o=function(e,n){return Object.prototype.hasOwnProperty.call(e,n)},a.p="/",a.oe=function(e){throw console.error(e),e};var i=window["webpackJsonp"]=window["webpackJsonp"]||[],h=i.push.bind(i);i.push=n,i=i.slice();for(var d=0;d<i.length;d++)n(i[d]);var l=h;t()})([]);</script><script src=/static/js/app.38307ef0.js></script></body></html>
|
||||
<!DOCTYPE html><html><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=viewport content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><link rel=icon href=/favicon.ico><title>Canal Admin</title><link href=/static/css/chunk-elementUI.18b11d0e.css rel=stylesheet><link href=/static/css/chunk-libs.5cf311f0.css rel=stylesheet><link href=/static/css/app.bb951cb3.css rel=stylesheet></head><body><noscript><strong>We're sorry but Canal Admin doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id=app></div><script src=/static/js/chunk-elementUI.667f4c87.js></script><script src=/static/js/chunk-libs.c04beefc.js></script><script>(function(e){function n(n){for(var r,c,f=n[0],a=n[1],i=n[2],h=0,l=[];h<f.length;h++)c=f[h],u[c]&&l.push(u[c][0]),u[c]=0;for(r in a)Object.prototype.hasOwnProperty.call(a,r)&&(e[r]=a[r]);d&&d(n);while(l.length)l.shift()();return o.push.apply(o,i||[]),t()}function t(){for(var e,n=0;n<o.length;n++){for(var t=o[n],r=!0,c=1;c<t.length;c++){var f=t[c];0!==u[f]&&(r=!1)}r&&(o.splice(n--,1),e=a(a.s=t[0]))}return e}var r={},c={runtime:0},u={runtime:0},o=[];function f(e){return a.p+"static/js/"+({}[e]||e)+"."+{"chunk-101fc062":"372a5ca5","chunk-137eaf06":"539d27c7","chunk-238a81e9":"da844a27","chunk-37c49cbf":"92ebe0ae","chunk-3fcdf643":"4b7133b8","chunk-1ef583f0":"c799c803","chunk-5560c396":"5eb91155","chunk-6f6d6145":"385d3c06","chunk-4f09fed2":"d107437b","chunk-69386cf0":"bdbe3f0c","chunk-7ec889b7":"192dc04c","chunk-e1a839e4":"e9eb1b06"}[e]+".js"}function a(n){if(r[n])return r[n].exports;var t=r[n]={i:n,l:!1,exports:{}};return e[n].call(t.exports,t,t.exports,a),t.l=!0,t.exports}a.e=function(e){var n=[],t={"chunk-101fc062":1,"chunk-238a81e9":1,"chunk-37c49cbf":1,"chunk-1ef583f0":1,"chunk-5560c396":1,"chunk-6f6d6145":1,"chunk-4f09fed2":1,"chunk-69386cf0":1,"chunk-7ec889b7":1};c[e]?n.push(c[e]):0!==c[e]&&t[e]&&n.push(c[e]=new Promise(function(n,t){for(var r="static/css/"+({}[e]||e)+"."+{"chunk-101fc062":"fad9926f","chunk-137eaf06":"31d6cfe0","chunk-238a81e9":"e8e2beee","chunk-37c49cbf":"efc21a9c","chunk-3fcdf643":"31d6cfe0","chunk-1ef583f0":"ec3a43f5","chunk-5560c396":"b660603c","chunk-6f6d6145":"bf63041a","chunk-4f09fed2":"70ec0b86","chunk-69386cf0":"741ff14e","chunk-7ec889b7":"c0585512","chunk-e1a839e4":"31d6cfe0"}[e]+".css",u=a.p+r,o=document.getElementsByTagName("link"),f=0;f<o.length;f++){var i=o[f],h=i.getAttribute("data-href")||i.getAttribute("href");if("stylesheet"===i.rel&&(h===r||h===u))return n()}var l=document.getElementsByTagName("style");for(f=0;f<l.length;f++){i=l[f],h=i.getAttribute("data-href");if(h===r||h===u)return n()}var d=document.createElement("link");d.rel="stylesheet",d.type="text/css",d.onload=n,d.onerror=function(n){var r=n&&n.target&&n.target.src||u,o=new Error("Loading CSS chunk "+e+" failed.\n("+r+")");o.code="CSS_CHUNK_LOAD_FAILED",o.request=r,delete c[e],d.parentNode.removeChild(d),t(o)},d.href=u;var s=document.getElementsByTagName("head")[0];s.appendChild(d)}).then(function(){c[e]=0}));var r=u[e];if(0!==r)if(r)n.push(r[2]);else{var o=new Promise(function(n,t){r=u[e]=[n,t]});n.push(r[2]=o);var i,h=document.createElement("script");h.charset="utf-8",h.timeout=120,a.nc&&h.setAttribute("nonce",a.nc),h.src=f(e),i=function(n){h.onerror=h.onload=null,clearTimeout(l);var t=u[e];if(0!==t){if(t){var r=n&&("load"===n.type?"missing":n.type),c=n&&n.target&&n.target.src,o=new Error("Loading chunk "+e+" failed.\n("+r+": "+c+")");o.type=r,o.request=c,t[1](o)}u[e]=void 0}};var l=setTimeout(function(){i({type:"timeout",target:h})},12e4);h.onerror=h.onload=i,document.head.appendChild(h)}return Promise.all(n)},a.m=e,a.c=r,a.d=function(e,n,t){a.o(e,n)||Object.defineProperty(e,n,{enumerable:!0,get:t})},a.r=function(e){"undefined"!==typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},a.t=function(e,n){if(1&n&&(e=a(e)),8&n)return e;if(4&n&&"object"===typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(a.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&n&&"string"!=typeof e)for(var r in e)a.d(t,r,function(n){return e[n]}.bind(null,r));return t},a.n=function(e){var n=e&&e.__esModule?function(){return e["default"]}:function(){return e};return a.d(n,"a",n),n},a.o=function(e,n){return Object.prototype.hasOwnProperty.call(e,n)},a.p="/",a.oe=function(e){throw console.error(e),e};var i=window["webpackJsonp"]=window["webpackJsonp"]||[],h=i.push.bind(i);i.push=n,i=i.slice();for(var l=0;l<i.length;l++)n(i[l]);var d=h;t()})([]);</script><script src=/static/js/app.38307ef0.js></script></body></html>
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -25,14 +25,19 @@
|
||||
<span>{{ scope.row.ip }}</span>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="Admin 端口" min-width="100" align="center">
|
||||
<el-table-column label="admin 端口" min-width="100" align="center">
|
||||
<template slot-scope="scope">
|
||||
{{ scope.row.port }}
|
||||
{{ scope.row.adminPort }}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="监控端口" min-width="100" align="center">
|
||||
<el-table-column label="tcp 端口" min-width="100" align="center">
|
||||
<template slot-scope="scope">
|
||||
{{ scope.row.port2 }}
|
||||
{{ scope.row.tcpPort }}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="metric 端口" min-width="100" align="center">
|
||||
<template slot-scope="scope">
|
||||
{{ scope.row.metricPort }}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column class-name="status-col" label="状态" min-width="150" align="center">
|
||||
@@ -65,11 +70,14 @@
|
||||
<el-form-item label="Server IP" prop="ip">
|
||||
<el-input v-model="nodeModel.ip" />
|
||||
</el-form-item>
|
||||
<el-form-item label="Admin 端口" prop="port">
|
||||
<el-input v-model="nodeModel.port" placeholder="11110" type="number" />
|
||||
<el-form-item label="admin 端口" prop="adminPort">
|
||||
<el-input v-model="nodeModel.adminPort" placeholder="11110" type="number" />
|
||||
</el-form-item>
|
||||
<el-form-item label="监控端口" prop="port2">
|
||||
<el-input v-model="nodeModel.port2" placeholder="11112" type="number" />
|
||||
<el-form-item label="tcp 端口" prop="tcpPort">
|
||||
<el-input v-model="nodeModel.tcpPort" placeholder="11111" type="number" />
|
||||
</el-form-item>
|
||||
<el-form-item label="metric 端口" prop="metricPort">
|
||||
<el-input v-model="nodeModel.metricPort" placeholder="11112" type="number" />
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
<div slot="footer" class="dialog-footer">
|
||||
@@ -119,13 +127,14 @@ export default {
|
||||
id: undefined,
|
||||
name: null,
|
||||
ip: null,
|
||||
port: 11110,
|
||||
port2: 11112
|
||||
adminPort: 11110,
|
||||
tcpPort: 11111,
|
||||
metricPort: 11112
|
||||
},
|
||||
rules: {
|
||||
name: [{ required: true, message: 'Server 名称不能为空', trigger: 'change' }],
|
||||
ip: [{ required: true, message: 'Server IP不能为空', trigger: 'change' }],
|
||||
port: [{ required: true, message: 'Server Admin端口不能为空', trigger: 'change' }]
|
||||
port: [{ required: true, message: 'Server admin端口不能为空', trigger: 'change' }]
|
||||
},
|
||||
dialogStatus: 'create'
|
||||
}
|
||||
@@ -147,8 +156,9 @@ export default {
|
||||
id: undefined,
|
||||
name: null,
|
||||
ip: null,
|
||||
port: null,
|
||||
port2: null
|
||||
adminPort: null,
|
||||
tcpPort: null,
|
||||
metricPort: null
|
||||
}
|
||||
},
|
||||
handleCreate() {
|
||||
@@ -222,7 +232,7 @@ export default {
|
||||
})
|
||||
},
|
||||
handleStart(row) {
|
||||
if (row.status !== 0) {
|
||||
if (row.status !== '0') {
|
||||
this.$message({ message: '当前Server不是停止状态，无法启动', type: 'error' })
|
||||
return
|
||||
}
|
||||
@@ -248,7 +258,7 @@ export default {
|
||||
})
|
||||
},
|
||||
handleStop(row) {
|
||||
if (row.status !== 1) {
|
||||
if (row.status !== '1') {
|
||||
this.$message({ message: '当前Server不是启动状态，无法停止', type: 'error' })
|
||||
return
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.client</artifactId>
|
||||
<version>1.1.4-SNAPSHOT</version>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.yaml</groupId>
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
CREATE SCHEMA `canal_manager` DEFAULT CHARACTER SET utf8mb4 ;
|
||||
|
||||
CREATE TABLE `canal_config` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
||||
`name` varchar(45) NOT NULL,
|
||||
`content` text NOT NULL,
|
||||
`modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `name_UNIQUE` (`name`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 ;
|
||||
|
||||
CREATE TABLE `canal_instance_config` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
||||
`name` varchar(45) NOT NULL,
|
||||
`content` text NOT NULL,
|
||||
`modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `name_UNIQUE` (`name`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 ;
|
||||
|
||||
CREATE TABLE `canal_adapter_config` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
||||
`category` varchar(45) NOT NULL,
|
||||
`name` varchar(45) NOT NULL,
|
||||
`content` text NOT NULL,
|
||||
`modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 ;
|
||||
|
||||
INSERT INTO `canal_config` VALUES ('1', 'canal.properties', '#################################################\r\n######### common argument ############# \r\n#################################################\r\n#canal.manager.jdbc.url=jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8\r\n#canal.manager.jdbc.username=root\r\n#canal.manager.jdbc.password=121212\r\ncanal.id = 1\r\ncanal.ip =\r\ncanal.port = 11111\r\ncanal.metrics.pull.port = 11112\r\ncanal.zkServers =\r\n# flush data to zk\r\ncanal.zookeeper.flush.period = 1000\r\ncanal.withoutNetty = false\r\n# tcp, kafka, RocketMQ\r\ncanal.serverMode = tcp\r\n# flush meta cursor/parse position to file\r\ncanal.file.data.dir = ${canal.conf.dir}\r\ncanal.file.flush.period = 1000\r\n## memory store RingBuffer size, should be Math.pow(2,n)\r\ncanal.instance.memory.buffer.size = 16384\r\n## memory store RingBuffer used memory unit size , default 1kb\r\ncanal.instance.memory.buffer.memunit = 1024 \r\n## meory store gets mode used MEMSIZE or ITEMSIZE\r\ncanal.instance.memory.batch.mode = MEMSIZE\r\ncanal.instance.memory.rawEntry = true\r\n\r\n## detecing config\r\ncanal.instance.detecting.enable = false\r\n#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now()\r\ncanal.instance.detecting.sql = select 1\r\ncanal.instance.detecting.interval.time = 3\r\ncanal.instance.detecting.retry.threshold = 3\r\ncanal.instance.detecting.heartbeatHaEnable = false\r\n\r\n# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery\r\ncanal.instance.transaction.size = 1024\r\n# mysql fallback connected to new master should fallback times\r\ncanal.instance.fallbackIntervalInSeconds = 60\r\n\r\n# network config\r\ncanal.instance.network.receiveBufferSize = 16384\r\ncanal.instance.network.sendBufferSize = 16384\r\ncanal.instance.network.soTimeout = 30\r\n\r\n# binlog filter config\r\ncanal.instance.filter.druid.ddl = true\r\ncanal.instance.filter.query.dcl = false\r\ncanal.instance.filter.query.dml = false\r\ncanal.instance.filter.query.ddl = false\r\ncanal.instance.filter.table.error = false\r\ncanal.instance.filter.rows = false\r\ncanal.instance.filter.transaction.entry = false\r\n\r\n# binlog format/image check\r\ncanal.instance.binlog.format = ROW,STATEMENT,MIXED \r\ncanal.instance.binlog.image = FULL,MINIMAL,NOBLOB\r\n\r\n# binlog ddl isolation\r\ncanal.instance.get.ddl.isolation = false\r\n\r\n# parallel parser config\r\ncanal.instance.parser.parallel = true\r\n## concurrent thread number, default 60% available processors, suggest not to exceed Runtime.getRuntime().availableProcessors()\r\n#canal.instance.parser.parallelThreadSize = 16\r\n## disruptor ringbuffer size, must be power of 2\r\ncanal.instance.parser.parallelBufferSize = 256\r\n\r\n# table meta tsdb info\r\ncanal.instance.tsdb.enable = true\r\ncanal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}\r\ncanal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;\r\ncanal.instance.tsdb.dbUsername = canal\r\ncanal.instance.tsdb.dbPassword = canal\r\n# dump snapshot interval, default 24 hour\r\ncanal.instance.tsdb.snapshot.interval = 24\r\n# purge snapshot expire , default 360 hour(15 days)\r\ncanal.instance.tsdb.snapshot.expire = 360\r\n\r\n# aliyun ak/sk , support rds/mq\r\ncanal.aliyun.accesskey =\r\ncanal.aliyun.secretkey =\r\n\r\n#################################################\r\n######### destinations ############# \r\n#################################################\r\ncanal.destinations = example\r\n# conf root dir\r\ncanal.conf.dir = ../conf\r\n# auto scan instance dir add/remove and start/stop instance\r\ncanal.auto.scan = true\r\ncanal.auto.scan.interval = 5\r\n\r\ncanal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml\r\n#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml\r\n\r\ncanal.instance.global.mode = spring\r\ncanal.instance.global.lazy = false\r\n#canal.instance.global.manager.address = 127.0.0.1:1099\r\n#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml\r\ncanal.instance.global.spring.xml = classpath:spring/file-instance.xml\r\n#canal.instance.global.spring.xml = classpath:spring/default-instance.xml\r\n\r\n##################################################\r\n######### MQ #############\r\n##################################################\r\ncanal.mq.servers = 127.0.0.1:6667\r\ncanal.mq.retries = 0\r\ncanal.mq.batchSize = 16384\r\ncanal.mq.maxRequestSize = 1048576\r\ncanal.mq.lingerMs = 1\r\ncanal.mq.bufferMemory = 33554432\r\ncanal.mq.canalBatchSize = 50\r\ncanal.mq.canalGetTimeout = 100\r\ncanal.mq.flatMessage = true\r\ncanal.mq.compressionType = none\r\ncanal.mq.acks = all\r\n', '2018-12-30 16:46:00');
|
||||
INSERT INTO `canal_config` VALUES ('2', 'application.yml', 'server:\n port: 8081\nlogging:\n level:\n org.springframework: WARN\n com.alibaba.otter.canal.client.adapter.hbase: DEBUG\n com.alibaba.otter.canal.client.adapter.es: DEBUG\n com.alibaba.otter.canal.client.adapter.rdb: DEBUG\nspring:\n jackson:\n date-format: yyyy-MM-dd HH:mm:ss\n time-zone: GMT+8\n default-property-inclusion: non_null\n\ncanal.conf:\n canalServerHost: 127.0.0.1:11111\n# zookeeperHosts: slave1:2181\n# mqServers: 127.0.0.1:9092 #or rocketmq\n# flatMessage: true\n batchSize: 500\n syncBatchSize: 1000\n retries: 0\n timeout:\n accessKey:\n secretKey:\n mode: tcp # kafka rocketMQ\n# srcDataSources:\n# defaultDS:\n# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true\n# username: root\n# password: 121212\n canalAdapters:\n - instance: example # canal instance Name or mq topic name\n groups:\n - groupId: g1\n outerAdapters:\n - name: logger\n# - name: rdb\n# key: mysql1\n# properties:\n# jdbc.driverClassName: com.mysql.jdbc.Driver\n# jdbc.url: jdbc:mysql://127.0.0.1:3306/mytest2?useUnicode=true\n# jdbc.username: root\n# jdbc.password: 121212\n# - name: rdb\n# key: oracle1\n# properties:\n# jdbc.driverClassName: oracle.jdbc.OracleDriver\n# jdbc.url: jdbc:oracle:thin:@localhost:49161:XE\n# jdbc.username: mytest\n# jdbc.password: m121212\n# - name: rdb\n# key: postgres1\n# properties:\n# jdbc.driverClassName: org.postgresql.Driver\n# jdbc.url: jdbc:postgresql://localhost:5432/postgres\n# jdbc.username: postgres\n# jdbc.password: 121212\n# threads: 1\n# commitSize: 3000\n# - name: hbase\n# properties:\n# hbase.zookeeper.quorum: 127.0.0.1\n# hbase.zookeeper.property.clientPort: 2181\n# zookeeper.znode.parent: /hbase\n# - name: es\n# hosts: 127.0.0.1:9300\n# properties:\n# cluster.name: elasticsearch\n', '2019-01-05 01:37:43');
|
||||
|
||||
INSERT INTO `canal_instance_config` VALUES ('1', 'example', '#################################################\r\n## mysql serverId , v1.0.26+ will autoGen\r\n# canal.instance.mysql.slaveId=0\r\n\r\n# enable gtid use true/false\r\ncanal.instance.gtidon=false\r\n\r\n# position info\r\ncanal.instance.master.address=127.0.0.1:3306\r\ncanal.instance.master.journal.name=\r\ncanal.instance.master.position=\r\ncanal.instance.master.timestamp=\r\ncanal.instance.master.gtid=\r\n\r\n# rds oss binlog\r\ncanal.instance.rds.accesskey=\r\ncanal.instance.rds.secretkey=\r\ncanal.instance.rds.instanceId=\r\n\r\n# table meta tsdb info\r\ncanal.instance.tsdb.enable=true\r\n#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb\r\n#canal.instance.tsdb.dbUsername=canal\r\n#canal.instance.tsdb.dbPassword=canal\r\n\r\n#canal.instance.standby.address =\r\n#canal.instance.standby.journal.name =\r\n#canal.instance.standby.position =\r\n#canal.instance.standby.timestamp =\r\n#canal.instance.standby.gtid=\r\n\r\n# username/password\r\ncanal.instance.dbUsername=canal\r\ncanal.instance.dbPassword=canal\r\ncanal.instance.connectionCharset = UTF-8\r\ncanal.instance.defaultDatabaseName =test\r\n# enable druid Decrypt database password\r\ncanal.instance.enableDruid=false\r\n#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==\r\n\r\n# table regex\r\ncanal.instance.filter.regex=.*\\\\..*\r\n# table black regex\r\ncanal.instance.filter.black.regex=\r\n\r\n# mq config\r\ncanal.mq.topic=example\r\n# 动态topic, 需mq支持动态创建topic\r\n#canal.mq.dynamicTopic=.*,mytest\\\\..*,mytest2.user\r\ncanal.mq.partition=0\r\n# hash partition config\r\n#canal.mq.partitionsNum=3\r\n#canal.mq.partitionHash=test.table:id^name,.*\\\\..*\r\n#################################################\r\n', '2018-12-30 16:46:16');
|
||||
|
||||
INSERT INTO `canal_adapter_config` VALUES ('1', 'es', 'mytest_user.yml', 'dataSourceKey: defaultDS\ndestination: example\nesMapping:\n _index: mytest_user\n _type: _doc\n _id: _id\n# pk: id\n sql: \"select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name,\n a.c_time as _c_time, c.labels as _labels from user a\n left join role b on b.id=a.role_id\n left join (select user_id, group_concat(label order by id desc separator \';\') as labels from label\n group by user_id) c on c.user_id=a.id\"\n# objFields:\n# _labels: array:;\n etlCondition: \"where a.c_time>=\'{0}\'\"\n commitBatch: 3000\n', '2019-01-01 17:05:43');
|
||||
INSERT INTO `canal_adapter_config` VALUES ('2', 'hbase', 'mytest_person2.yml', 'dataSourceKey: defaultDS\ndestination: example\nhbaseMapping:\n mode: STRING #NATIVE #PHOENIX\n database: mytest # 数据库名\n table: person2 # 数据库表名\n hbaseTable: MYTEST.PERSON2 # HBase表名\n family: CF # 默认统一Family名称\n uppercaseQualifier: true # 字段名转大写, 默认为true\n commitBatch: 3000 # 批量提交的大小\n #rowKey: id,type # 复合字段rowKey不能和columns中的rowKey重复\n columns:\n # 数据库字段:HBase对应字段\n id: ROWKEY LEN:15\n name: NAME\n email: EMAIL\n type:\n c_time: C_TIME\n birthday: BIRTHDAY\n# excludeColumns:\n# - lat # 忽略字段\n\n# -- NATIVE类型\n# $DEFAULT\n# $STRING\n# $INTEGER\n# $LONG\n# $SHORT\n# $BOOLEAN\n# $FLOAT\n# $DOUBLE\n# $BIGDECIMAL\n# $DATE\n# $BYTE\n# $BYTES\n\n# -- PHOENIX类型\n# $DEFAULT 对应PHOENIX里的VARCHAR\n# $UNSIGNED_INT 对应PHOENIX里的UNSIGNED_INT 4字节\n# $UNSIGNED_LONG 对应PHOENIX里的UNSIGNED_LONG 8字节\n# $UNSIGNED_TINYINT 对应PHOENIX里的UNSIGNED_TINYINT 1字节\n# $UNSIGNED_SMALLINT 对应PHOENIX里的UNSIGNED_SMALLINT 2字节\n# $UNSIGNED_FLOAT 对应PHOENIX里的UNSIGNED_FLOAT 4字节\n# $UNSIGNED_DOUBLE 对应PHOENIX里的UNSIGNED_DOUBLE 8字节\n# $INTEGER 对应PHOENIX里的INTEGER 4字节\n# $BIGINT 对应PHOENIX里的BIGINT 8字节\n# $TINYINT 对应PHOENIX里的TINYINT 1字节\n# $SMALLINT 对应PHOENIX里的SMALLINT 2字节\n# $FLOAT 对应PHOENIX里的FLOAT 4字节\n# $DOUBLE 对应PHOENIX里的DOUBLE 8字节\n# $BOOLEAN 对应PHOENIX里的BOOLEAN 1字节\n# $TIME 对应PHOENIX里的TIME 8字节\n# $DATE 对应PHOENIX里的DATE 8字节\n# $TIMESTAMP 对应PHOENIX里的TIMESTAMP 12字节\n# $UNSIGNED_TIME 对应PHOENIX里的UNSIGNED_TIME 8字节\n# $UNSIGNED_DATE 对应PHOENIX里的UNSIGNED_DATE 8字节\n# $UNSIGNED_TIMESTAMP 对应PHOENIX里的UNSIGNED_TIMESTAMP 12字节\n# $VARCHAR 对应PHOENIX里的VARCHAR 动态长度\n# $VARBINARY 对应PHOENIX里的VARBINARY 动态长度\n# $DECIMAL 对应PHOENIX里的DECIMAL 动态长度', '2019-01-01 17:08:08');
|
||||
INSERT INTO `canal_adapter_config` VALUES ('3', 'rdb', 'mytest_user.yml', 'dataSourceKey: defaultDS\ndestination: example\nouterAdapterKey: oracle1\nconcurrent: true\ndbMapping:\n database: mytest\n table: user\n targetTable: mytest.tb_user\n targetPk:\n id: id\n# mapAll: true\n targetColumns:\n id:\n name:\n role_id:\n c_time:\n test1:\n\n\n# Mirror schema synchronize config\n#dataSourceKey: defaultDS\n#destination: example\n#outerAdapterKey: mysql1\n#concurrent: true\n#dbMapping:\n# mirrorDb: true\n# database: mytest\n', '2019-01-01 17:08:50');
|
||||
@@ -19,6 +19,7 @@ public class CanalConstants {
|
||||
public static final String CANAL_USER = ROOT + "." + "user";
|
||||
public static final String CANAL_PASSWD = ROOT + "." + "passwd";
|
||||
public static final String CANAL_METRICS_PULL_PORT = ROOT + "." + "metrics.pull.port";
|
||||
public static final String CANAL_ADMIN_MANAGER = ROOT + "." + "admin.manager";
|
||||
public static final String CANAL_ADMIN_PORT = ROOT + "." + "admin.port";
|
||||
public static final String CANAL_ADMIN_USER = ROOT + "." + "admin.user";
|
||||
public static final String CANAL_ADMIN_PASSWD = ROOT + "." + "admin.passwd";
|
||||
|
||||
@@ -12,11 +12,7 @@ import org.apache.zookeeper.Watcher.Event.KeeperState;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.slf4j.MDC;
|
||||
import org.springframework.beans.factory.BeanFactory;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
|
||||
import com.alibaba.otter.canal.common.CanalException;
|
||||
import com.alibaba.otter.canal.common.utils.AddressUtils;
|
||||
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
|
||||
@@ -31,10 +27,9 @@ import com.alibaba.otter.canal.deployer.monitor.ManagerInstanceConfigMonitor;
|
||||
import com.alibaba.otter.canal.deployer.monitor.SpringInstanceConfigMonitor;
|
||||
import com.alibaba.otter.canal.instance.core.CanalInstance;
|
||||
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
|
||||
import com.alibaba.otter.canal.instance.manager.CanalConfigClient;
|
||||
import com.alibaba.otter.canal.instance.manager.ManagerCanalInstanceGenerator;
|
||||
import com.alibaba.otter.canal.instance.manager.PlainCanalInstanceGenerator;
|
||||
import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient;
|
||||
import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator;
|
||||
import com.alibaba.otter.canal.parse.CanalEventParser;
|
||||
import com.alibaba.otter.canal.server.CanalMQStarter;
|
||||
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
|
||||
import com.alibaba.otter.canal.server.exception.CanalServerException;
|
||||
@@ -56,10 +51,11 @@ public class CanalController {
|
||||
private String ip;
|
||||
private String registerIp;
|
||||
private int port;
|
||||
private int adminPort;
|
||||
// 默认使用spring的方式载入
|
||||
private Map<String, InstanceConfig> instanceConfigs;
|
||||
private InstanceConfig globalInstanceConfig;
|
||||
private Map<String, CanalConfigClient> managerClients;
|
||||
private Map<String, PlainCanalConfigClient> managerClients;
|
||||
// 监听instance config的变化
|
||||
private boolean autoScan = true;
|
||||
private InstanceAction defaultAction;
|
||||
@@ -71,15 +67,17 @@ public class CanalController {
|
||||
private ZkClientx zkclientx;
|
||||
|
||||
private CanalMQStarter canalMQStarter;
|
||||
private String adminUser;
|
||||
private String adminPasswd;
|
||||
|
||||
public CanalController(){
|
||||
this(System.getProperties());
|
||||
}
|
||||
|
||||
public CanalController(final Properties properties){
|
||||
managerClients = MigrateMap.makeComputingMap(new Function<String, CanalConfigClient>() {
|
||||
managerClients = MigrateMap.makeComputingMap(new Function<String, PlainCanalConfigClient>() {
|
||||
|
||||
public CanalConfigClient apply(String managerAddress) {
|
||||
public PlainCanalConfigClient apply(String managerAddress) {
|
||||
return getManagerClient(managerAddress);
|
||||
}
|
||||
});
|
||||
@@ -111,6 +109,7 @@ public class CanalController {
|
||||
ip = getProperty(properties, CanalConstants.CANAL_IP);
|
||||
registerIp = getProperty(properties, CanalConstants.CANAL_REGISTER_IP);
|
||||
port = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_PORT));
|
||||
adminPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_ADMIN_PORT));
|
||||
embededCanalServer = CanalServerWithEmbedded.instance();
|
||||
embededCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator
|
||||
try {
|
||||
@@ -121,6 +120,8 @@ public class CanalController {
|
||||
embededCanalServer.setMetricsPort(11112);
|
||||
}
|
||||
|
||||
this.adminUser = getProperty(properties, CanalConstants.CANAL_ADMIN_USER);
|
||||
this.adminPasswd = getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD);
|
||||
embededCanalServer.setUser(getProperty(properties, CanalConstants.CANAL_USER));
|
||||
embededCanalServer.setPasswd(getProperty(properties, CanalConstants.CANAL_PASSWD));
|
||||
|
||||
@@ -254,6 +255,8 @@ public class CanalController {
|
||||
runningMonitor.start();
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("auto notify start {} successful.", destination);
|
||||
}
|
||||
|
||||
public void stop(String destination) {
|
||||
@@ -266,12 +269,16 @@ public class CanalController {
|
||||
runningMonitor.stop();
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("auto notify stop {} successful.", destination);
|
||||
}
|
||||
|
||||
public void reload(String destination) {
|
||||
// 目前任何配置变化，直接重启，简单处理
|
||||
stop(destination);
|
||||
start(destination);
|
||||
|
||||
logger.info("auto notify reload {} successful.", destination);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -298,7 +305,14 @@ public class CanalController {
|
||||
}
|
||||
return monitor;
|
||||
} else if (mode.isManager()) {
|
||||
return new ManagerInstanceConfigMonitor();
|
||||
ManagerInstanceConfigMonitor monitor = new ManagerInstanceConfigMonitor();
|
||||
monitor.setScanIntervalInSecond(scanInterval);
|
||||
monitor.setDefaultAction(defaultAction);
|
||||
String managerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
|
||||
monitor.setConfigClient(getManagerClient(managerAddress));
|
||||
monitor.setIp(registerIp);
|
||||
monitor.setPort(adminPort);
|
||||
return monitor;
|
||||
} else {
|
||||
throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor");
|
||||
}
|
||||
@@ -308,9 +322,13 @@ public class CanalController {
|
||||
}
|
||||
|
||||
private InstanceConfig initGlobalConfig(Properties properties) {
|
||||
String adminManagerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
|
||||
InstanceConfig globalConfig = new InstanceConfig();
|
||||
String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(CanalConstants.GLOBAL_NAME));
|
||||
if (StringUtils.isNotEmpty(modeStr)) {
|
||||
if (StringUtils.isNotEmpty(adminManagerAddress)) {
|
||||
// 如果指定了manager地址,则强制适用manager
|
||||
globalConfig.setMode(InstanceMode.MANAGER);
|
||||
} else if (StringUtils.isNotEmpty(modeStr)) {
|
||||
globalConfig.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr)));
|
||||
}
|
||||
|
||||
@@ -322,6 +340,10 @@ public class CanalController {
|
||||
String managerAddress = getProperty(properties,
|
||||
CanalConstants.getInstanceManagerAddressKey(CanalConstants.GLOBAL_NAME));
|
||||
if (StringUtils.isNotEmpty(managerAddress)) {
|
||||
if (StringUtils.equals(managerAddress, "${canal.admin.manager}")) {
|
||||
managerAddress = adminManagerAddress;
|
||||
}
|
||||
|
||||
globalConfig.setManagerAddress(managerAddress);
|
||||
}
|
||||
|
||||
@@ -335,28 +357,18 @@ public class CanalController {
|
||||
public CanalInstance generate(String destination) {
|
||||
InstanceConfig config = instanceConfigs.get(destination);
|
||||
if (config == null) {
|
||||
throw new CanalServerException("can't find destination:{}");
|
||||
throw new CanalServerException("can't find destination:" + destination);
|
||||
}
|
||||
|
||||
if (config.getMode().isManager()) {
|
||||
ManagerCanalInstanceGenerator instanceGenerator = new ManagerCanalInstanceGenerator();
|
||||
PlainCanalInstanceGenerator instanceGenerator = new PlainCanalInstanceGenerator();
|
||||
instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress()));
|
||||
instanceGenerator.setSpringXml(config.getSpringXml());
|
||||
return instanceGenerator.generate(destination);
|
||||
} else if (config.getMode().isSpring()) {
|
||||
SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator();
|
||||
synchronized (CanalEventParser.class) {
|
||||
try {
|
||||
// 设置当前正在加载的通道，加载spring查找文件时会用到该变量
|
||||
System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, destination);
|
||||
instanceGenerator.setBeanFactory(getBeanFactory(config.getSpringXml()));
|
||||
return instanceGenerator.generate(destination);
|
||||
} catch (Throwable e) {
|
||||
logger.error("generator instance failed.", e);
|
||||
throw new CanalException(e);
|
||||
} finally {
|
||||
System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, "");
|
||||
}
|
||||
}
|
||||
instanceGenerator.setSpringXml(config.getSpringXml());
|
||||
return instanceGenerator.generate(destination);
|
||||
} else {
|
||||
throw new UnsupportedOperationException("unknow mode :" + config.getMode());
|
||||
}
|
||||
@@ -368,13 +380,8 @@ public class CanalController {
|
||||
return globalConfig;
|
||||
}
|
||||
|
||||
private CanalConfigClient getManagerClient(String managerAddress) {
|
||||
return new CanalConfigClient();
|
||||
}
|
||||
|
||||
private BeanFactory getBeanFactory(String springXml) {
|
||||
ApplicationContext applicationContext = new ClassPathXmlApplicationContext(springXml);
|
||||
return applicationContext;
|
||||
private PlainCanalConfigClient getManagerClient(String managerAddress) {
|
||||
return new PlainCanalConfigClient(managerAddress, this.adminUser, this.adminPasswd);
|
||||
}
|
||||
|
||||
private void initInstanceConfig(Properties properties) {
|
||||
@@ -392,9 +399,13 @@ public class CanalController {
|
||||
}
|
||||
|
||||
private InstanceConfig parseInstanceConfig(Properties properties, String destination) {
|
||||
String adminManagerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
|
||||
InstanceConfig config = new InstanceConfig(globalInstanceConfig);
|
||||
String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(destination));
|
||||
if (!StringUtils.isEmpty(modeStr)) {
|
||||
if (StringUtils.isNotEmpty(adminManagerAddress)) {
|
||||
// 如果指定了manager地址,则强制适用manager
|
||||
config.setMode(InstanceMode.MANAGER);
|
||||
} else if (StringUtils.isNotEmpty(modeStr)) {
|
||||
config.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr)));
|
||||
}
|
||||
|
||||
@@ -406,6 +417,9 @@ public class CanalController {
|
||||
if (config.getMode().isManager()) {
|
||||
String managerAddress = getProperty(properties, CanalConstants.getInstanceManagerAddressKey(destination));
|
||||
if (StringUtils.isNotEmpty(managerAddress)) {
|
||||
if (StringUtils.equals(managerAddress, "${canal.admin.manager}")) {
|
||||
managerAddress = adminManagerAddress;
|
||||
}
|
||||
config.setManagerAddress(managerAddress);
|
||||
}
|
||||
} else if (config.getMode().isSpring()) {
|
||||
|
||||
@@ -3,14 +3,17 @@ package com.alibaba.otter.canal.deployer;
|
||||
import java.io.FileInputStream;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.deployer.monitor.remote.RemoteCanalConfigMonitor;
|
||||
import com.alibaba.otter.canal.deployer.monitor.remote.RemoteConfigLoader;
|
||||
import com.alibaba.otter.canal.deployer.monitor.remote.RemoteConfigLoaderFactory;
|
||||
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
|
||||
import com.alibaba.otter.canal.instance.manager.plain.PlainCanal;
|
||||
import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient;
|
||||
|
||||
/**
|
||||
* canal独立版本启动的入口类
|
||||
@@ -20,9 +23,11 @@ import com.alibaba.otter.canal.deployer.monitor.remote.RemoteConfigLoaderFactory
|
||||
*/
|
||||
public class CanalLauncher {
|
||||
|
||||
private static final String CLASSPATH_URL_PREFIX = "classpath:";
|
||||
private static final Logger logger = LoggerFactory.getLogger(CanalLauncher.class);
|
||||
public static final CountDownLatch runningLatch = new CountDownLatch(1);
|
||||
private static final String CLASSPATH_URL_PREFIX = "classpath:";
|
||||
private static final Logger logger = LoggerFactory.getLogger(CanalLauncher.class);
|
||||
public static final CountDownLatch runningLatch = new CountDownLatch(1);
|
||||
private static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
|
||||
new NamedThreadFactory("canal-server-scan"));
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
@@ -32,7 +37,6 @@ public class CanalLauncher {
|
||||
logger.info("## load canal configurations");
|
||||
String conf = System.getProperty("canal.conf", "classpath:canal.properties");
|
||||
Properties properties = new Properties();
|
||||
RemoteConfigLoader remoteConfigLoader = null;
|
||||
if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
|
||||
conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
|
||||
properties.load(CanalLauncher.class.getClassLoader().getResourceAsStream(conf));
|
||||
@@ -40,43 +44,48 @@ public class CanalLauncher {
|
||||
properties.load(new FileInputStream(conf));
|
||||
}
|
||||
|
||||
remoteConfigLoader = RemoteConfigLoaderFactory.getRemoteConfigLoader(properties);
|
||||
if (remoteConfigLoader != null) {
|
||||
// 加载远程canal.properties
|
||||
Properties remoteConfig = remoteConfigLoader.loadRemoteConfig();
|
||||
// 加载remote instance配置
|
||||
remoteConfigLoader.loadRemoteInstanceConfigs();
|
||||
if (remoteConfig != null) {
|
||||
properties = remoteConfig;
|
||||
} else {
|
||||
remoteConfigLoader = null;
|
||||
}
|
||||
}
|
||||
|
||||
final CanalStater canalStater = new CanalStater(properties);
|
||||
canalStater.start();
|
||||
String managerAddress = properties.getProperty(CanalConstants.CANAL_ADMIN_MANAGER);
|
||||
if (StringUtils.isNotEmpty(managerAddress)) {
|
||||
String user = properties.getProperty(CanalConstants.CANAL_ADMIN_USER);
|
||||
String passwd = properties.getProperty(CanalConstants.CANAL_ADMIN_PASSWD);
|
||||
final PlainCanalConfigClient configClient = new PlainCanalConfigClient(managerAddress, user, passwd);
|
||||
PlainCanal canalConfig = configClient.findServer(null);
|
||||
properties = canalConfig.getProperties();
|
||||
int scanIntervalInSecond = Integer.valueOf(properties.getProperty(CanalConstants.CANAL_AUTO_SCAN_INTERVAL,
|
||||
"5"));
|
||||
executor.scheduleWithFixedDelay(new Runnable() {
|
||||
|
||||
if (remoteConfigLoader != null) {
|
||||
remoteConfigLoader.startMonitor(new RemoteCanalConfigMonitor() {
|
||||
private PlainCanal lastCanalConfig;
|
||||
|
||||
@Override
|
||||
public void onChange(Properties properties) {
|
||||
public void run() {
|
||||
try {
|
||||
// 远程配置canal.properties修改重新加载整个应用
|
||||
canalStater.stop();
|
||||
canalStater.setProperties(properties);
|
||||
canalStater.start();
|
||||
} catch (Throwable throwable) {
|
||||
logger.error(throwable.getMessage(), throwable);
|
||||
if (lastCanalConfig == null) {
|
||||
lastCanalConfig = configClient.findServer(null);
|
||||
} else {
|
||||
PlainCanal newCanalConfig = configClient.findServer(lastCanalConfig.getMd5());
|
||||
if (newCanalConfig != null) {
|
||||
// 远程配置canal.properties修改重新加载整个应用
|
||||
canalStater.stop();
|
||||
canalStater.setProperties(newCanalConfig.getProperties());
|
||||
canalStater.start();
|
||||
|
||||
lastCanalConfig = newCanalConfig;
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Throwable e) {
|
||||
logger.error("scan failed", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}, 0, scanIntervalInSecond, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
canalStater.setProperties(properties);
|
||||
canalStater.start();
|
||||
runningLatch.await();
|
||||
if (remoteConfigLoader != null) {
|
||||
remoteConfigLoader.destroy();
|
||||
}
|
||||
executor.shutdownNow();
|
||||
} catch (Throwable e) {
|
||||
logger.error("## Something goes wrong when starting up the canal Server:", e);
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import com.alibaba.otter.canal.deployer.CanalStater;
|
||||
import com.alibaba.otter.canal.deployer.InstanceConfig;
|
||||
import com.alibaba.otter.canal.deployer.monitor.InstanceAction;
|
||||
import com.alibaba.otter.canal.deployer.monitor.InstanceConfigMonitor;
|
||||
import com.alibaba.otter.canal.deployer.monitor.ManagerInstanceConfigMonitor;
|
||||
import com.alibaba.otter.canal.deployer.monitor.SpringInstanceConfigMonitor;
|
||||
import com.alibaba.otter.canal.instance.core.CanalInstance;
|
||||
import com.alibaba.otter.canal.protocol.SecurityUtil;
|
||||
@@ -204,9 +205,24 @@ public class CanalAdminController implements CanalAdmin {
|
||||
private InstanceAction getInstanceAction(String destination) {
|
||||
Map<InstanceConfig.InstanceMode, InstanceConfigMonitor> monitors = canalStater.getController()
|
||||
.getInstanceConfigMonitors();
|
||||
SpringInstanceConfigMonitor monitor = (SpringInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.SPRING);
|
||||
Map<String, InstanceAction> instanceActions = monitor.getActions();
|
||||
return instanceActions.get(destination);
|
||||
|
||||
InstanceAction instanceAction = null;
|
||||
if (monitors.containsKey(InstanceConfig.InstanceMode.SPRING)) {
|
||||
SpringInstanceConfigMonitor monitor = (SpringInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.SPRING);
|
||||
Map<String, InstanceAction> instanceActions = monitor.getActions();
|
||||
instanceAction = instanceActions.get(destination);
|
||||
}
|
||||
|
||||
if (instanceAction != null) {
|
||||
return instanceAction;
|
||||
}
|
||||
|
||||
if (monitors.containsKey(InstanceConfig.InstanceMode.MANAGER)) {
|
||||
ManagerInstanceConfigMonitor monitor = (ManagerInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.MANAGER);
|
||||
Map<String, InstanceAction> instanceActions = monitor.getActions();
|
||||
instanceAction = instanceActions.get(destination);
|
||||
}
|
||||
return instanceAction;
|
||||
}
|
||||
|
||||
public void setUser(String user) {
|
||||
|
||||
@@ -1,20 +1,185 @@
|
||||
package com.alibaba.otter.canal.deployer.monitor;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
|
||||
import com.alibaba.otter.canal.common.CanalLifeCycle;
|
||||
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
|
||||
import com.alibaba.otter.canal.instance.manager.plain.PlainCanal;
|
||||
import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient;
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import com.google.common.collect.MigrateMap;
|
||||
|
||||
/**
|
||||
* @author jianghang 2013-2-18 下午03:19:06
|
||||
* @version 1.0.1
|
||||
* 基于manager配置的实现
|
||||
*
|
||||
* @author agapple 2019年8月26日 下午10:00:20
|
||||
* @since 1.1.4
|
||||
*/
|
||||
public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle {
|
||||
|
||||
public void register(String destination, InstanceAction action) {
|
||||
private static final Logger logger = LoggerFactory.getLogger(ManagerInstanceConfigMonitor.class);
|
||||
private long scanIntervalInSecond = 5;
|
||||
private InstanceAction defaultAction = null;
|
||||
private Map<String, InstanceAction> actions = new MapMaker().makeMap();
|
||||
private Map<String, PlainCanal> configs = MigrateMap.makeComputingMap(new Function<String, PlainCanal>() {
|
||||
|
||||
public PlainCanal apply(String destination) {
|
||||
return new PlainCanal();
|
||||
}
|
||||
});
|
||||
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
|
||||
new NamedThreadFactory("canal-instance-scan"));
|
||||
|
||||
private volatile boolean isFirst = true;
|
||||
private PlainCanalConfigClient configClient;
|
||||
private String ip;
|
||||
private int port;
|
||||
private String lastInstanceMD5;
|
||||
|
||||
public void start() {
|
||||
super.start();
|
||||
executor.scheduleWithFixedDelay(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
scan();
|
||||
if (isFirst) {
|
||||
isFirst = false;
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.error("scan failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
}, 0, scanIntervalInSecond, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
super.stop();
|
||||
executor.shutdownNow();
|
||||
actions.clear();
|
||||
}
|
||||
|
||||
public void register(String destination, InstanceAction action) {
|
||||
if (action != null) {
|
||||
actions.put(destination, action);
|
||||
} else {
|
||||
actions.put(destination, defaultAction);
|
||||
}
|
||||
}
|
||||
|
||||
public void unregister(String destination) {
|
||||
actions.remove(destination);
|
||||
}
|
||||
|
||||
private void scan() {
|
||||
String instances = configClient.findInstances(ip, String.valueOf(port), lastInstanceMD5);
|
||||
final List<String> is = Lists.newArrayList(StringUtils.split(instances, ','));
|
||||
List<String> start = Lists.newArrayList();
|
||||
List<String> stop = Lists.newArrayList();
|
||||
List<String> restart = Lists.newArrayList();
|
||||
for (String instance : is) {
|
||||
if (!configs.containsKey(instance)) {
|
||||
PlainCanal newPlainCanal = configClient.findInstance(instance, null);
|
||||
if (newPlainCanal != null) {
|
||||
configs.put(instance, newPlainCanal);
|
||||
start.add(instance);
|
||||
}
|
||||
} else {
|
||||
PlainCanal plainCanal = configs.get(instance);
|
||||
PlainCanal newPlainCanal = configClient.findInstance(instance, plainCanal.getMd5());
|
||||
if (newPlainCanal != null) {
|
||||
// 配置有变化
|
||||
restart.add(instance);
|
||||
configs.put(instance, newPlainCanal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
configs.forEach((instance, plainCanal) -> {
|
||||
if (!is.contains(instance)) {
|
||||
stop.add(instance);
|
||||
}
|
||||
});
|
||||
|
||||
stop.forEach(instance -> {
|
||||
notifyStop(instance);
|
||||
configs.remove(instance);
|
||||
});
|
||||
|
||||
restart.forEach(instance -> {
|
||||
notifyReload(instance);
|
||||
});
|
||||
|
||||
start.forEach(instance -> {
|
||||
notifyStart(instance);
|
||||
});
|
||||
}
|
||||
|
||||
private void notifyStart(String destination) {
|
||||
try {
|
||||
defaultAction.start(destination);
|
||||
actions.put(destination, defaultAction);
|
||||
// 启动成功后记录配置文件信息
|
||||
} catch (Throwable e) {
|
||||
logger.error(String.format("scan add found[%s] but start failed", destination), e);
|
||||
}
|
||||
}
|
||||
|
||||
private void notifyStop(String destination) {
|
||||
InstanceAction action = actions.remove(destination);
|
||||
try {
|
||||
action.stop(destination);
|
||||
} catch (Throwable e) {
|
||||
logger.error(String.format("scan delete found[%s] but stop failed", destination), e);
|
||||
actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除
|
||||
}
|
||||
}
|
||||
|
||||
private void notifyReload(String destination) {
|
||||
InstanceAction action = actions.get(destination);
|
||||
if (action != null) {
|
||||
try {
|
||||
action.reload(destination);
|
||||
} catch (Throwable e) {
|
||||
logger.error(String.format("scan reload found[%s] but reload failed", destination), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void setDefaultAction(InstanceAction defaultAction) {
|
||||
this.defaultAction = defaultAction;
|
||||
}
|
||||
|
||||
public void setScanIntervalInSecond(long scanIntervalInSecond) {
|
||||
this.scanIntervalInSecond = scanIntervalInSecond;
|
||||
}
|
||||
|
||||
public void setConfigClient(PlainCanalConfigClient configClient) {
|
||||
this.configClient = configClient;
|
||||
}
|
||||
|
||||
public void setIp(String ip) {
|
||||
this.ip = ip;
|
||||
}
|
||||
|
||||
public void setPort(int port) {
|
||||
this.port = port;
|
||||
}
|
||||
|
||||
public Map<String, InstanceAction> getActions() {
|
||||
return actions;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -186,8 +186,6 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
|
||||
newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified()));
|
||||
}
|
||||
lastFile.setInstanceFiles(newFileInfo);
|
||||
|
||||
logger.info("auto notify start {} successful.", destination);
|
||||
} catch (Throwable e) {
|
||||
logger.error(String.format("scan add found[%s] but start failed", destination), e);
|
||||
}
|
||||
@@ -198,7 +196,6 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
|
||||
try {
|
||||
action.stop(destination);
|
||||
lastFiles.remove(destination);
|
||||
logger.info("auto notify stop {} successful.", destination);
|
||||
} catch (Throwable e) {
|
||||
logger.error(String.format("scan delete found[%s] but stop failed", destination), e);
|
||||
actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除
|
||||
@@ -210,7 +207,6 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
|
||||
if (action != null) {
|
||||
try {
|
||||
action.reload(destination);
|
||||
logger.info("auto notify reload {} successful.", destination);
|
||||
} catch (Throwable e) {
|
||||
logger.error(String.format("scan reload found[%s] but reload failed", destination), e);
|
||||
}
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
package com.alibaba.otter.canal.deployer.monitor.remote;
|
||||
|
||||
/**
|
||||
* 配置对应对象
|
||||
*
|
||||
* @author rewerma 2019-01-25 下午05:20:16
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ConfigItem {
|
||||
|
||||
private Long id;
|
||||
private String name;
|
||||
private String content;
|
||||
private long modifiedTime;
|
||||
|
||||
public Long getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(Long id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getContent() {
|
||||
return content;
|
||||
}
|
||||
|
||||
public void setContent(String content) {
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
public long getModifiedTime() {
|
||||
return modifiedTime;
|
||||
}
|
||||
|
||||
public void setModifiedTime(long modifiedTime) {
|
||||
this.modifiedTime = modifiedTime;
|
||||
}
|
||||
}
|
||||
@@ -1,264 +0,0 @@
|
||||
package com.alibaba.otter.canal.deployer.monitor.remote;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.druid.pool.DruidDataSource;
|
||||
import com.alibaba.otter.canal.common.utils.CommonUtils;
|
||||
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
|
||||
import com.alibaba.otter.canal.deployer.CanalConstants;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.MapMaker;
|
||||
|
||||
/**
|
||||
* 基于数据库的远程配置装载器
|
||||
*
|
||||
* @author rewerma 2019-01-25 下午05:20:16
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class DbRemoteConfigLoader implements RemoteConfigLoader {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DbRemoteConfigLoader.class);
|
||||
|
||||
private Map<String, ConfigItem> remoteInstanceConfigs = new MapMaker().makeMap();
|
||||
|
||||
private DruidDataSource dataSource;
|
||||
|
||||
private long currentConfigTimestamp = 0;
|
||||
|
||||
private long scanIntervalInSecond = 5;
|
||||
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(2,
|
||||
new NamedThreadFactory("remote-canal-config-scan"));
|
||||
|
||||
private RemoteInstanceMonitor remoteInstanceMonitor = new RemoteInstanceMonitorImpl();
|
||||
|
||||
public DbRemoteConfigLoader(String driverName, String jdbcUrl, String jdbcUsername, String jdbcPassword){
|
||||
dataSource = new DruidDataSource();
|
||||
if (StringUtils.isEmpty(driverName)) {
|
||||
driverName = "com.mysql.jdbc.Driver";
|
||||
}
|
||||
dataSource.setDriverClassName(driverName);
|
||||
dataSource.setUrl(jdbcUrl);
|
||||
dataSource.setUsername(jdbcUsername);
|
||||
dataSource.setPassword(jdbcPassword);
|
||||
dataSource.setInitialSize(1);
|
||||
dataSource.setMinIdle(1);
|
||||
dataSource.setMaxActive(1);
|
||||
dataSource.setMaxWait(60000);
|
||||
dataSource.setTimeBetweenEvictionRunsMillis(60000);
|
||||
dataSource.setMinEvictableIdleTimeMillis(300000);
|
||||
try {
|
||||
dataSource.init();
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载远程 canal.properties文件
|
||||
*
|
||||
* @return 远程配置的properties
|
||||
*/
|
||||
@Override
|
||||
public Properties loadRemoteConfig() {
|
||||
Properties properties = null;
|
||||
try {
|
||||
// 加载远程canal配置
|
||||
ConfigItem configItem = getRemoteCanalConfig();
|
||||
if (configItem != null) {
|
||||
if (configItem.getModifiedTime() != currentConfigTimestamp) {
|
||||
currentConfigTimestamp = configItem.getModifiedTime();
|
||||
overrideLocalCanalConfig(configItem.getContent());
|
||||
properties = new Properties();
|
||||
properties.load(new ByteArrayInputStream(configItem.getContent().getBytes(StandardCharsets.UTF_8)));
|
||||
scanIntervalInSecond = Integer
|
||||
.parseInt(properties.getProperty(CanalConstants.CANAL_AUTO_SCAN_INTERVAL, "5"));
|
||||
logger.info("## Loaded remote canal config: canal.properties ");
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
|
||||
/**
|
||||
* 覆盖本地 canal.properties
|
||||
*
|
||||
* @param content 远程配置内容文本
|
||||
*/
|
||||
private void overrideLocalCanalConfig(String content) {
|
||||
try (FileWriter writer = new FileWriter(CommonUtils.getConfPath() + "canal.properties")) {
|
||||
writer.write(content);
|
||||
writer.flush();
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取远程canal.properties配置内容
|
||||
*
|
||||
* @return 内容对象
|
||||
*/
|
||||
private ConfigItem getRemoteCanalConfig() {
|
||||
String sql = "select name, content, modified_time from canal_config where id=1";
|
||||
try (Connection conn = dataSource.getConnection();
|
||||
Statement stmt = conn.createStatement();
|
||||
ResultSet rs = stmt.executeQuery(sql)) {
|
||||
if (rs.next()) {
|
||||
ConfigItem configItem = new ConfigItem();
|
||||
configItem.setId(1L);
|
||||
configItem.setName(rs.getString("name"));
|
||||
configItem.setContent(rs.getString("content"));
|
||||
configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
|
||||
return configItem;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载远程的instance配置
|
||||
*/
|
||||
@Override
|
||||
public void loadRemoteInstanceConfigs() {
|
||||
try {
|
||||
// 加载远程instance配置
|
||||
loadModifiedInstanceConfigs();
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载远程instance新增、修改、删除配置
|
||||
*/
|
||||
private void loadModifiedInstanceConfigs() {
|
||||
Map<String, ConfigItem> remoteConfigStatus = new HashMap<>();
|
||||
String sql = "select id, name, modified_time from canal_instance_config";
|
||||
try (Connection conn = dataSource.getConnection();
|
||||
Statement stmt = conn.createStatement();
|
||||
ResultSet rs = stmt.executeQuery(sql)) {
|
||||
while (rs.next()) {
|
||||
ConfigItem configItem = new ConfigItem();
|
||||
configItem.setId(rs.getLong("id"));
|
||||
configItem.setName(rs.getString("name"));
|
||||
configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
|
||||
remoteConfigStatus.put(configItem.getName(), configItem);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
if (!remoteConfigStatus.isEmpty()) {
|
||||
List<Long> changedIds = new ArrayList<>();
|
||||
|
||||
for (ConfigItem remoteConfigStat : remoteConfigStatus.values()) {
|
||||
ConfigItem currentConfig = remoteInstanceConfigs.get(remoteConfigStat.getName());
|
||||
if (currentConfig == null) {
|
||||
// 新增
|
||||
changedIds.add(remoteConfigStat.getId());
|
||||
} else {
|
||||
// 修改
|
||||
if (currentConfig.getModifiedTime() != remoteConfigStat.getModifiedTime()) {
|
||||
changedIds.add(remoteConfigStat.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!changedIds.isEmpty()) {
|
||||
String contentsSql = "select id, name, content, modified_time from canal_instance_config where id in ("
|
||||
+ Joiner.on(",").join(changedIds) + ")";
|
||||
try (Connection conn = dataSource.getConnection();
|
||||
Statement stmt = conn.createStatement();
|
||||
ResultSet rs = stmt.executeQuery(contentsSql)) {
|
||||
while (rs.next()) {
|
||||
ConfigItem configItemNew = new ConfigItem();
|
||||
configItemNew.setId(rs.getLong("id"));
|
||||
configItemNew.setName(rs.getString("name"));
|
||||
configItemNew.setContent(rs.getString("content"));
|
||||
configItemNew.setModifiedTime(rs.getTimestamp("modified_time").getTime());
|
||||
|
||||
remoteInstanceConfigs.put(configItemNew.getName(), configItemNew);
|
||||
remoteInstanceMonitor.onModify(configItemNew);
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (String name : remoteInstanceConfigs.keySet()) {
|
||||
if (!remoteConfigStatus.containsKey(name)) {
|
||||
// 删除
|
||||
remoteInstanceConfigs.remove(name);
|
||||
remoteInstanceMonitor.onDelete(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 监听 canal 主配置和 instance 配置变化
|
||||
*
|
||||
* @param remoteCanalConfigMonitor 监听回调方法
|
||||
*/
|
||||
public void startMonitor(final RemoteCanalConfigMonitor remoteCanalConfigMonitor) {
|
||||
// 监听canal.properties变化
|
||||
executor.scheduleWithFixedDelay(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
Properties properties = loadRemoteConfig();
|
||||
if (properties != null) {
|
||||
remoteCanalConfigMonitor.onChange(properties);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.error("Scan remote canal config failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
}, 10, scanIntervalInSecond, TimeUnit.SECONDS);
|
||||
|
||||
// 监听instance变化
|
||||
executor.scheduleWithFixedDelay(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
loadRemoteInstanceConfigs();
|
||||
} catch (Throwable e) {
|
||||
logger.error("Scan remote instance config failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
}, 10, 3, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
/**
|
||||
* 销毁
|
||||
*/
|
||||
public void destroy() {
|
||||
executor.shutdownNow();
|
||||
try {
|
||||
dataSource.close();
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package com.alibaba.otter.canal.deployer.monitor.remote;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
* 远程canal.properties配置监听器接口
|
||||
*
|
||||
* @author rewerma 2019-01-25 下午05:20:16
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public interface RemoteCanalConfigMonitor {
|
||||
void onChange(Properties properties);
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
package com.alibaba.otter.canal.deployer.monitor.remote;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
* 远程配置装载器接口
|
||||
*
|
||||
* @author rewerma 2019-01-25 下午05:20:16
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public interface RemoteConfigLoader {
|
||||
|
||||
/**
|
||||
* 加载远程 canal.properties文件
|
||||
*
|
||||
* @return 远程配置的properties
|
||||
*/
|
||||
Properties loadRemoteConfig();
|
||||
|
||||
/**
|
||||
* 加载远程的instance配置
|
||||
*/
|
||||
void loadRemoteInstanceConfigs();
|
||||
|
||||
/**
|
||||
* 启动监听 canal 主配置和 instance 配置变化
|
||||
*
|
||||
* @param remoteCanalConfigMonitor 监听回调方法
|
||||
*/
|
||||
void startMonitor(final RemoteCanalConfigMonitor remoteCanalConfigMonitor);
|
||||
|
||||
/**
|
||||
* 销毁
|
||||
*/
|
||||
void destroy();
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
package com.alibaba.otter.canal.deployer.monitor.remote;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import com.alibaba.otter.canal.deployer.monitor.remote.http.HttpRemoteConfigLoader;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* 远程配置装载器工厂类
|
||||
*
|
||||
* @author rewerma 2019-01-25 下午05:20:16
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class RemoteConfigLoaderFactory {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(RemoteConfigLoaderFactory.class);
|
||||
|
||||
public static RemoteConfigLoader getRemoteConfigLoader(Properties localProperties) {
|
||||
String jdbcUrl = localProperties.getProperty("canal.manager.jdbc.url");
|
||||
String httpUrl = localProperties.getProperty("canal.manager.http.url");
|
||||
if (!StringUtils.isEmpty(jdbcUrl)) {
|
||||
logger.info("## load remote db canal configurations");
|
||||
// load remote config
|
||||
String driverName = localProperties.getProperty("canal.manager.jdbc.driverName");
|
||||
String jdbcUsername = localProperties.getProperty("canal.manager.jdbc.username");
|
||||
String jdbcPassword = localProperties.getProperty("canal.manager.jdbc.password");
|
||||
return new DbRemoteConfigLoader(driverName, jdbcUrl, jdbcUsername, jdbcPassword);
|
||||
} else if (!StringUtils.isEmpty(httpUrl)) {
|
||||
logger.info("## load remote http canal configurations");
|
||||
String httpUsername = localProperties.getProperty("canal.manager.http.username");
|
||||
String httpPassword = localProperties.getProperty("canal.manager.http.password");
|
||||
return new HttpRemoteConfigLoader(httpUrl, httpUsername, httpPassword);
|
||||
}
|
||||
// 可扩展其它远程配置加载器
|
||||
|
||||
logger.info("## load local canal configurations");
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package com.alibaba.otter.canal.deployer.monitor.remote;
|
||||
|
||||
/**
|
||||
* 远程xxx/instance.properties配置监听器接口
|
||||
*
|
||||
* @author rewerma 2019-01-25 下午05:20:16
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public interface RemoteInstanceMonitor {
|
||||
|
||||
/**
|
||||
* 新增配置事件
|
||||
*
|
||||
* @param configItem 配置项
|
||||
*/
|
||||
void onAdd(ConfigItem configItem);
|
||||
|
||||
/**
|
||||
* 修改配置事件
|
||||
*
|
||||
* @param configItem 配置项
|
||||
*/
|
||||
void onModify(ConfigItem configItem);
|
||||
|
||||
/**
|
||||
* 删除配置事件
|
||||
*
|
||||
* @param instanceName 实例名
|
||||
*/
|
||||
void onDelete(String instanceName);
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package com.alibaba.otter.canal.deployer.monitor.remote;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.common.utils.CommonUtils;
|
||||
|
||||
/**
|
||||
* 远程xxx/instance.properties配置监听器实现
|
||||
*
|
||||
* @author rewerma 2019-01-25 下午05:20:16
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class RemoteInstanceMonitorImpl implements RemoteInstanceMonitor {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(RemoteInstanceMonitorImpl.class);
|
||||
|
||||
@Override
|
||||
public void onAdd(ConfigItem configItem) {
|
||||
this.onModify(configItem);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onModify(ConfigItem configItem) {
|
||||
String confDir = CommonUtils.getConfPath() + configItem.getName();
|
||||
File instanceDir = new File(confDir);
|
||||
if (!instanceDir.exists()) {
|
||||
boolean mkDirs = instanceDir.mkdirs();
|
||||
if (!mkDirs) {
|
||||
logger.info("## Error to create instance config dir: {}", configItem.getName());
|
||||
return;
|
||||
}
|
||||
}
|
||||
try (FileWriter writer = new FileWriter(confDir + "/instance.properties")) {
|
||||
writer.write(configItem.getContent());
|
||||
writer.flush();
|
||||
logger.info("## Loaded remote instance config: {}/instance.properties ", configItem.getName());
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onDelete(String instanceName) {
|
||||
File file = new File(CommonUtils.getConfPath() + instanceName + "/");
|
||||
if (file.exists()) {
|
||||
CommonUtils.deleteDir(file);
|
||||
logger.info("## Deleted and loaded remote instance config: {} ", instanceName);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,257 +0,0 @@
|
||||
package com.alibaba.otter.canal.deployer.monitor.remote.http;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.alibaba.fastjson.TypeReference;
|
||||
import com.alibaba.otter.canal.common.utils.CommonUtils;
|
||||
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
|
||||
import com.alibaba.otter.canal.deployer.CanalConstants;
|
||||
import com.alibaba.otter.canal.deployer.monitor.remote.*;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 基于HTTP的远程配置装载器
|
||||
*
|
||||
* @author rewerma 2019-08-26 上午09:40:36
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class HttpRemoteConfigLoader implements RemoteConfigLoader {
|
||||
|
||||
private final static Logger logger = LoggerFactory.getLogger(HttpRemoteConfigLoader.class);
|
||||
|
||||
private final static Integer REQUEST_TIMEOUT = 5000;
|
||||
|
||||
private String baseUrl;
|
||||
private String username;
|
||||
private String password;
|
||||
|
||||
private String token;
|
||||
|
||||
private HttpHelper httpHelper;
|
||||
|
||||
private long currentConfigTimestamp = 0;
|
||||
private Map<String, ConfigItem> remoteInstanceConfigs = new MapMaker().makeMap();
|
||||
|
||||
private RemoteInstanceMonitor remoteInstanceMonitor = new RemoteInstanceMonitorImpl();
|
||||
|
||||
private long scanIntervalInSecond = 5;
|
||||
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(2,
|
||||
new NamedThreadFactory("remote-http-canal-config-scan"));
|
||||
|
||||
public HttpRemoteConfigLoader(String baseUrl, String username, String password){
|
||||
this.baseUrl = baseUrl;
|
||||
this.username = username;
|
||||
this.password = password;
|
||||
httpHelper = new HttpHelper();
|
||||
}
|
||||
|
||||
private String login4Token(HttpHelper httpHelper) {
|
||||
Map<String, Object> reqBody = new HashMap<>();
|
||||
reqBody.put("username", username);
|
||||
reqBody.put("password", password);
|
||||
String response = httpHelper.post(baseUrl + "/api/v1/user/login", null, reqBody, REQUEST_TIMEOUT);
|
||||
ResponseModel<JSONObject> resp = JSONObject.parseObject(response,
|
||||
new TypeReference<ResponseModel<JSONObject>>() {
|
||||
});
|
||||
if (!HttpHelper.REST_STATE_OK.equals(resp.getCode())) {
|
||||
throw new RuntimeException("requestPost for login error: " + resp.getMessage());
|
||||
}
|
||||
return (String) resp.getData().get("token");
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized Properties loadRemoteConfig() {
|
||||
Properties properties = null;
|
||||
try {
|
||||
ConfigItem configItem = getRemoteCanalConfig();
|
||||
if (configItem != null) {
|
||||
if (configItem.getModifiedTime() != currentConfigTimestamp) { // 修改时就不同说明配置有更新
|
||||
Map<String, String> heads = new HashMap<>();
|
||||
heads.put("X-Token", token);
|
||||
String response = httpHelper.get(baseUrl + "/api/v1/canal/config", heads, REQUEST_TIMEOUT);
|
||||
ResponseModel<ConfigItem> resp = JSONObject.parseObject(response,
|
||||
new TypeReference<ResponseModel<ConfigItem>>() {
|
||||
});
|
||||
currentConfigTimestamp = configItem.getModifiedTime();
|
||||
overrideLocalCanalConfig(resp.getData().getContent());
|
||||
properties = new Properties();
|
||||
properties
|
||||
.load(new ByteArrayInputStream(resp.getData().getContent().getBytes(StandardCharsets.UTF_8)));
|
||||
scanIntervalInSecond = Integer
|
||||
.parseInt(properties.getProperty(CanalConstants.CANAL_AUTO_SCAN_INTERVAL, "5"));
|
||||
logger.info("## Loaded http remote canal config: canal.properties ");
|
||||
}
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
logger.error(t.getMessage(), t);
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取远程canal server配置摘要(无配置内容)
|
||||
*
|
||||
* @return 配置摘要
|
||||
*/
|
||||
private ConfigItem getRemoteCanalConfig() {
|
||||
if (StringUtils.isEmpty(token)) {
|
||||
token = login4Token(httpHelper);
|
||||
}
|
||||
|
||||
Map<String, String> heads = new HashMap<>();
|
||||
heads.put("X-Token", token);
|
||||
String response = httpHelper.get(baseUrl + "/api/v1/canal/config/summary", heads, REQUEST_TIMEOUT);
|
||||
ResponseModel<ConfigItem> resp = JSONObject.parseObject(response,
|
||||
new TypeReference<ResponseModel<ConfigItem>>() {
|
||||
});
|
||||
if (HttpHelper.REST_STATE_TOKEN_INVALID.equals(resp.getCode())) {
|
||||
// token 失效
|
||||
token = login4Token(httpHelper);
|
||||
heads.put("X-Token", token);
|
||||
response = httpHelper.get(baseUrl + "/api/v1/canal/config/summary", heads, REQUEST_TIMEOUT);
|
||||
resp = JSONObject.parseObject(response, new TypeReference<ResponseModel<ConfigItem>>() {
|
||||
});
|
||||
}
|
||||
if (!HttpHelper.REST_STATE_OK.equals(resp.getCode())) {
|
||||
throw new RuntimeException("requestGet for canal config error: " + resp.getMessage());
|
||||
}
|
||||
return resp.getData();
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void loadRemoteInstanceConfigs() {
|
||||
try {
|
||||
// 加载远程instance配置
|
||||
loadModifiedInstanceConfigs();
|
||||
} catch (Throwable t) {
|
||||
logger.error(t.getMessage(), t);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载远程instance新增、修改、删除配置
|
||||
*/
|
||||
private void loadModifiedInstanceConfigs() {
|
||||
if (StringUtils.isEmpty(token)) {
|
||||
token = login4Token(httpHelper);
|
||||
}
|
||||
|
||||
Map<String, ConfigItem> remoteConfigStatus = new HashMap<>();
|
||||
|
||||
Map<String, String> heads = new HashMap<>();
|
||||
heads.put("X-Token", token);
|
||||
String response = httpHelper.get(baseUrl + "/api/v1/canal/instances", heads, REQUEST_TIMEOUT);
|
||||
ResponseModel<List<ConfigItem>> resp = JSONObject.parseObject(response,
|
||||
new TypeReference<ResponseModel<List<ConfigItem>>>() {
|
||||
});
|
||||
if (HttpHelper.REST_STATE_TOKEN_INVALID.equals(resp.getCode())) {
|
||||
// token 失效
|
||||
token = login4Token(httpHelper);
|
||||
heads.put("X-Token", token);
|
||||
response = httpHelper.get(baseUrl + "/api/v1/canal/instances", heads, REQUEST_TIMEOUT);
|
||||
resp = JSONObject.parseObject(response, new TypeReference<ResponseModel<List<ConfigItem>>>() {
|
||||
});
|
||||
}
|
||||
if (!HttpHelper.REST_STATE_OK.equals(resp.getCode())) {
|
||||
throw new RuntimeException("requestGet for canal instances error: " + resp.getMessage());
|
||||
}
|
||||
for (ConfigItem configItem : resp.getData()) {
|
||||
remoteConfigStatus.put(configItem.getName(), configItem);
|
||||
}
|
||||
|
||||
if (!remoteConfigStatus.isEmpty()) {
|
||||
List<Long> changedIds = new ArrayList<>();
|
||||
|
||||
for (ConfigItem remoteConfigStat : remoteConfigStatus.values()) {
|
||||
ConfigItem currentConfig = remoteInstanceConfigs.get(remoteConfigStat.getName());
|
||||
if (currentConfig == null) {
|
||||
// 新增
|
||||
changedIds.add(remoteConfigStat.getId());
|
||||
} else {
|
||||
// 修改
|
||||
if (currentConfig.getModifiedTime() != remoteConfigStat.getModifiedTime()) {
|
||||
changedIds.add(remoteConfigStat.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!changedIds.isEmpty()) {
|
||||
for (Long changedId : changedIds) {
|
||||
String response2 = httpHelper
|
||||
.get(baseUrl + "/api/v1/canal/instance/" + changedId, heads, REQUEST_TIMEOUT);
|
||||
ResponseModel<ConfigItem> resp2 = JSONObject.parseObject(response2,
|
||||
new TypeReference<ResponseModel<ConfigItem>>() {
|
||||
});
|
||||
ConfigItem configItemNew = resp2.getData();
|
||||
remoteInstanceConfigs.put(configItemNew.getName(), configItemNew);
|
||||
remoteInstanceMonitor.onModify(configItemNew);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (String name : remoteInstanceConfigs.keySet()) {
|
||||
if (!remoteConfigStatus.containsKey(name)) {
|
||||
// 删除
|
||||
remoteInstanceConfigs.remove(name);
|
||||
remoteInstanceMonitor.onDelete(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void startMonitor(RemoteCanalConfigMonitor remoteCanalConfigMonitor) {
|
||||
|
||||
executor.scheduleWithFixedDelay(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
// 监听canal.properties变化
|
||||
try {
|
||||
Properties properties = loadRemoteConfig();
|
||||
if (properties != null) {
|
||||
remoteCanalConfigMonitor.onChange(properties);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.error("Scan remote canal config failed", e);
|
||||
}
|
||||
|
||||
// 监听instance变化
|
||||
try {
|
||||
loadRemoteInstanceConfigs();
|
||||
} catch (Throwable e) {
|
||||
logger.error("Scan remote instance config failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
}, 10, scanIntervalInSecond, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
/**
|
||||
* 覆盖本地 canal.properties
|
||||
*
|
||||
* @param content 远程配置内容文本
|
||||
*/
|
||||
private void overrideLocalCanalConfig(String content) {
|
||||
try (FileWriter writer = new FileWriter(CommonUtils.getConfPath() + "canal.properties")) {
|
||||
writer.write(content);
|
||||
writer.flush();
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void destroy() {
|
||||
if (httpHelper != null) {
|
||||
httpHelper.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package com.alibaba.otter.canal.deployer.monitor.remote.http;
|
||||
|
||||
/**
|
||||
* 响应类
|
||||
*
|
||||
* @author rewerma 2019-08-26 上午09:40:36
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ResponseModel<T> {
|
||||
|
||||
private Integer code;
|
||||
private String message;
|
||||
private T data;
|
||||
|
||||
public Integer getCode() {
|
||||
return code;
|
||||
}
|
||||
|
||||
public void setCode(Integer code) {
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
public String getMessage() {
|
||||
return message;
|
||||
}
|
||||
|
||||
public void setMessage(String message) {
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public T getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
public void setData(T data) {
|
||||
this.data = data;
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,6 @@
|
||||
#################################################
|
||||
######### common argument #############
|
||||
#################################################
|
||||
#canal.manager.jdbc.url=jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8
|
||||
#canal.manager.jdbc.username=root
|
||||
#canal.manager.jdbc.password=121212
|
||||
canal.manager.http.url=http://127.0.0.1:8089
|
||||
canal.manager.http.username=admin
|
||||
canal.manager.http.password=121212
|
||||
|
||||
canal.id = 1
|
||||
# tcp bind ip
|
||||
canal.ip =
|
||||
@@ -20,6 +13,7 @@ canal.metrics.pull.port = 11112
|
||||
# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
|
||||
|
||||
# canal admin config
|
||||
#canal.admin.manager = 127.0.0.1:8089
|
||||
canal.admin.port = 11110
|
||||
canal.admin.user = admin
|
||||
canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
|
||||
@@ -112,7 +106,7 @@ canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
|
||||
|
||||
canal.instance.global.mode = spring
|
||||
canal.instance.global.lazy = false
|
||||
#canal.instance.global.manager.address = 127.0.0.1:1099
|
||||
canal.instance.global.manager.address = ${canal.admin.manager}
|
||||
#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
|
||||
canal.instance.global.spring.xml = classpath:spring/file-instance.xml
|
||||
#canal.instance.global.spring.xml = classpath:spring/default-instance.xml
|
||||
|
||||
@@ -16,5 +16,16 @@
|
||||
<artifactId>canal.instance.core</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.instance.spring</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<!-- junit -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.alibaba.otter.canal.instance.manager;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.BeanFactory;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
|
||||
import com.alibaba.otter.canal.common.CanalException;
|
||||
import com.alibaba.otter.canal.instance.core.CanalInstance;
|
||||
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
|
||||
import com.alibaba.otter.canal.instance.manager.plain.PlainCanal;
|
||||
import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient;
|
||||
import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator;
|
||||
|
||||
/**
|
||||
* 基于manager生成对应的{@linkplain CanalInstance}
|
||||
*
|
||||
* @author jianghang 2012-7-12 下午05:37:09
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class PlainCanalInstanceGenerator implements CanalInstanceGenerator {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(SpringCanalInstanceGenerator.class);
|
||||
private String springXml;
|
||||
private PlainCanalConfigClient canalConfigClient;
|
||||
private String defaultName = "instance";
|
||||
private BeanFactory beanFactory;
|
||||
|
||||
public CanalInstance generate(String destination) {
|
||||
synchronized (CanalInstanceGenerator.class) {
|
||||
try {
|
||||
PlainCanal canal = canalConfigClient.findInstance(destination, null);
|
||||
if (canal == null) {
|
||||
throw new CanalException("instance : " + destination + " config is not found");
|
||||
}
|
||||
// 设置动态properties,替换掉本地properties
|
||||
com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer.propertiesLocal.set(canal.getProperties());
|
||||
// 设置当前正在加载的通道，加载spring查找文件时会用到该变量
|
||||
System.setProperty("canal.instance.destination", destination);
|
||||
this.beanFactory = getBeanFactory(springXml);
|
||||
String beanName = destination;
|
||||
if (!beanFactory.containsBean(beanName)) {
|
||||
beanName = defaultName;
|
||||
}
|
||||
|
||||
return (CanalInstance) beanFactory.getBean(beanName);
|
||||
} catch (Throwable e) {
|
||||
logger.error("generator instance failed.", e);
|
||||
throw new CanalException(e);
|
||||
} finally {
|
||||
System.setProperty("canal.instance.destination", "");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ================ setter / getter ================
|
||||
|
||||
private BeanFactory getBeanFactory(String springXml) {
|
||||
ApplicationContext applicationContext = new ClassPathXmlApplicationContext(springXml);
|
||||
return applicationContext;
|
||||
}
|
||||
|
||||
public void setCanalConfigClient(PlainCanalConfigClient canalConfigClient) {
|
||||
this.canalConfigClient = canalConfigClient;
|
||||
}
|
||||
|
||||
public void setSpringXml(String springXml) {
|
||||
this.springXml = springXml;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,9 +1,15 @@
|
||||
package com.alibaba.otter.canal.deployer.monitor.remote.http;
|
||||
package com.alibaba.otter.canal.instance.manager.plain;
|
||||
|
||||
import static org.apache.http.client.config.RequestConfig.custom;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.security.cert.CertificateException;
|
||||
import java.security.cert.X509Certificate;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.alibaba.fastjson.TypeReference;
|
||||
import com.alibaba.otter.canal.deployer.monitor.remote.ConfigItem;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
@@ -11,18 +17,22 @@ import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.apache.http.client.utils.URIBuilder;
|
||||
import org.apache.http.config.RegistryBuilder;
|
||||
import org.apache.http.conn.socket.ConnectionSocketFactory;
|
||||
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
|
||||
import org.apache.http.conn.ssl.NoopHostnameVerifier;
|
||||
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
|
||||
import org.apache.http.ssl.SSLContextBuilder;
|
||||
import org.apache.http.ssl.TrustStrategy;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.apache.http.client.config.RequestConfig.custom;
|
||||
import com.alibaba.fastjson.JSON;
|
||||
|
||||
/**
|
||||
* http client 工具类
|
||||
@@ -32,7 +42,7 @@ import static org.apache.http.client.config.RequestConfig.custom;
|
||||
*/
|
||||
public class HttpHelper {
|
||||
|
||||
private final static Logger logger = LoggerFactory.getLogger(HttpRemoteConfigLoader.class);
|
||||
private final static Logger logger = LoggerFactory.getLogger(HttpHelper.class);
|
||||
|
||||
public static final Integer REST_STATE_OK = 20000;
|
||||
public static final Integer REST_STATE_TOKEN_INVALID = 50014;
|
||||
@@ -44,16 +54,31 @@ public class HttpHelper {
|
||||
HttpClientBuilder builder = HttpClientBuilder.create();
|
||||
builder.setMaxConnPerRoute(50);
|
||||
builder.setMaxConnTotal(100);
|
||||
httpclient = builder.build();
|
||||
|
||||
// 创建支持忽略证书的https
|
||||
try {
|
||||
SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
|
||||
|
||||
@Override
|
||||
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
|
||||
return true;
|
||||
}
|
||||
}).build();
|
||||
|
||||
httpclient = HttpClientBuilder.create()
|
||||
.setSSLContext(sslContext)
|
||||
.setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory> create()
|
||||
.register("http", PlainConnectionSocketFactory.INSTANCE)
|
||||
.register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
|
||||
.build()))
|
||||
.build();
|
||||
} catch (Throwable e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
public String get(String url, Map<String, String> heads, int timeout) {
|
||||
// 支持采用https协议，忽略证书
|
||||
url = url.trim();
|
||||
if (url.startsWith("https")) {
|
||||
// FIXME
|
||||
return "";
|
||||
}
|
||||
CloseableHttpResponse response = null;
|
||||
HttpGet httpGet = null;
|
||||
try {
|
||||
@@ -97,17 +122,11 @@ public class HttpHelper {
|
||||
}
|
||||
|
||||
public String post(String url, Map<String, String> heads, Object requestBody, int timeout) {
|
||||
String json = JSON.toJSONString(requestBody);
|
||||
return post0(url, heads, json, timeout);
|
||||
return post0(url, heads, JSON.toJSONString(requestBody), timeout);
|
||||
}
|
||||
|
||||
public String post0(String url, Map<String, String> heads, String requestBody, int timeout) {
|
||||
url = url.trim();
|
||||
// 支持采用https协议，忽略证书
|
||||
if (url.startsWith("https")) {
|
||||
// FIXME
|
||||
return "";
|
||||
}
|
||||
HttpPost httpPost = null;
|
||||
CloseableHttpResponse response = null;
|
||||
try {
|
||||
@@ -0,0 +1,55 @@
|
||||
package com.alibaba.otter.canal.instance.manager.plain;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
* plain远程配置，提供基于properties纯文本的配置
|
||||
*
|
||||
* @author rewerma 2019-01-25 下午05:20:16
|
||||
* @author agapple
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class PlainCanal {
|
||||
|
||||
private Properties properties;
|
||||
private String md5;
|
||||
private String status;
|
||||
|
||||
public PlainCanal(){
|
||||
}
|
||||
|
||||
public PlainCanal(Properties properties, String status, String md5){
|
||||
this.properties = properties;
|
||||
this.md5 = md5;
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public Properties getProperties() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
public void setProperties(Properties properties) {
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
public String getMd5() {
|
||||
return md5;
|
||||
}
|
||||
|
||||
public void setMd5(String md5) {
|
||||
this.md5 = md5;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "PlainCanal [properties=" + properties + ", md5=" + md5 + ", status=" + status + "]";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package com.alibaba.otter.canal.instance.manager.plain;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.alibaba.fastjson.TypeReference;
|
||||
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
|
||||
import com.alibaba.otter.canal.common.CanalException;
|
||||
import com.alibaba.otter.canal.common.CanalLifeCycle;
|
||||
import com.alibaba.otter.canal.protocol.SecurityUtil;
|
||||
|
||||
/**
|
||||
* 远程配置获取
|
||||
*
|
||||
* @author rewerma 2019-01-25 下午05:20:16
|
||||
* @author agapple 2019年8月26日 下午7:52:06
|
||||
* @since 1.1.4
|
||||
*/
|
||||
public class PlainCanalConfigClient extends AbstractCanalLifeCycle implements CanalLifeCycle {
|
||||
|
||||
private final static Integer REQUEST_TIMEOUT = 5000;
|
||||
private String configURL;
|
||||
private String user;
|
||||
private String passwd;
|
||||
private HttpHelper httpHelper;
|
||||
|
||||
public PlainCanalConfigClient(String configURL, String user, String passwd){
|
||||
this.configURL = configURL;
|
||||
if (!StringUtils.startsWithIgnoreCase(configURL, "http")) {
|
||||
this.configURL = "http://" + configURL;
|
||||
} else {
|
||||
this.configURL = configURL;
|
||||
}
|
||||
this.user = user;
|
||||
this.passwd = passwd;
|
||||
this.httpHelper = new HttpHelper();
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载canal.properties文件
|
||||
*
|
||||
* @return 远程配置的properties
|
||||
*/
|
||||
public PlainCanal findServer(String md5) {
|
||||
if (StringUtils.isEmpty(md5)) {
|
||||
md5 = "";
|
||||
}
|
||||
String url = configURL + "/api/v1/config/server_poll?md5=" + md5;
|
||||
return queryConfig(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载远程的instance.properties
|
||||
*/
|
||||
public PlainCanal findInstance(String destination, String md5) {
|
||||
if (StringUtils.isEmpty(md5)) {
|
||||
md5 = "";
|
||||
}
|
||||
String url = configURL + "/api/v1/config/instance_poll/" + destination + "?md5=" + md5;
|
||||
return queryConfig(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回需要运行的instance列表
|
||||
*/
|
||||
public String findInstances(String ip, String port, String md5) {
|
||||
if (StringUtils.isEmpty(md5)) {
|
||||
md5 = "";
|
||||
}
|
||||
String url = configURL + "/api/v1/config/instances_poll?md5=" + md5 + "&ip=" + ip + "&port=" + port;
|
||||
ResponseModel<CanalConfig> config = doQuery(url);
|
||||
if (config.data != null) {
|
||||
return config.data.content;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private PlainCanal queryConfig(String url) {
|
||||
try {
|
||||
ResponseModel<CanalConfig> config = doQuery(url);
|
||||
return processData(config.data);
|
||||
} catch (Throwable e) {
|
||||
throw new CanalException("load manager config failed.", e);
|
||||
}
|
||||
}
|
||||
|
||||
private ResponseModel<CanalConfig> doQuery(String url) {
|
||||
Map<String, String> heads = new HashMap<>();
|
||||
heads.put("user", user);
|
||||
heads.put("passwd", passwd);
|
||||
String response = httpHelper.get(url, heads, REQUEST_TIMEOUT);
|
||||
ResponseModel<CanalConfig> resp = JSONObject.parseObject(response,
|
||||
new TypeReference<ResponseModel<CanalConfig>>() {
|
||||
});
|
||||
|
||||
if (!HttpHelper.REST_STATE_OK.equals(resp.code)) {
|
||||
throw new CanalException("requestGet for canal config error: " + resp.message);
|
||||
}
|
||||
|
||||
return resp;
|
||||
}
|
||||
|
||||
private PlainCanal processData(CanalConfig config) throws IOException, NoSuchAlgorithmException {
|
||||
Properties properties = new Properties();
|
||||
String md5 = null;
|
||||
String status = null;
|
||||
if (config != null && StringUtils.isNotEmpty(config.content)) {
|
||||
md5 = SecurityUtil.md5String(config.content);
|
||||
status = config.status;
|
||||
properties.load(new ByteArrayInputStream(config.content.getBytes(StandardCharsets.UTF_8)));
|
||||
} else {
|
||||
// null代表没有新配置变更
|
||||
return null;
|
||||
}
|
||||
|
||||
return new PlainCanal(properties, status, md5);
|
||||
}
|
||||
|
||||
private static class ResponseModel<T> {
|
||||
|
||||
public Integer code;
|
||||
public String message;
|
||||
public T data;
|
||||
}
|
||||
|
||||
private static class CanalConfig {
|
||||
|
||||
public String content;
|
||||
public String status;
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.alibaba.otter.canal.instance.manager;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
import com.alibaba.otter.canal.instance.manager.plain.PlainCanal;
|
||||
import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient;
|
||||
|
||||
public class PlainCanalConfigClientIntegration {
|
||||
|
||||
@Test
|
||||
public void testSimple() {
|
||||
PlainCanalConfigClient client = new PlainCanalConfigClient("http://127.0.0.1:8089",
|
||||
"admin",
|
||||
"4ACFE3202A5FF5CF467898FC58AAB1D615029441");
|
||||
|
||||
PlainCanal plain = client.findServer(null);
|
||||
Assert.notNull(plain);
|
||||
|
||||
plain = client.findServer(plain.getMd5());
|
||||
Assert.isNull(plain);
|
||||
|
||||
plain = client.findInstance("example", null);
|
||||
Assert.notNull(plain);
|
||||
|
||||
plain = client.findInstance("example", plain.getMd5());
|
||||
Assert.isNull(plain);
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,12 @@
|
||||
package com.alibaba.otter.canal.instance.spring;
|
||||
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.BeanFactory;
|
||||
import org.springframework.beans.factory.BeanFactoryAware;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
|
||||
import com.alibaba.otter.canal.common.CanalException;
|
||||
import com.alibaba.otter.canal.instance.core.CanalInstance;
|
||||
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
|
||||
|
||||
@@ -11,22 +14,40 @@ import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
|
||||
* @author zebin.xuzb @ 2012-7-12
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class SpringCanalInstanceGenerator implements CanalInstanceGenerator, BeanFactoryAware {
|
||||
public class SpringCanalInstanceGenerator implements CanalInstanceGenerator {
|
||||
|
||||
private String defaultName = "instance";
|
||||
private BeanFactory beanFactory;
|
||||
private static final Logger logger = LoggerFactory.getLogger(SpringCanalInstanceGenerator.class);
|
||||
private String springXml;
|
||||
private String defaultName = "instance";
|
||||
private BeanFactory beanFactory;
|
||||
|
||||
public CanalInstance generate(String destination) {
|
||||
String beanName = destination;
|
||||
if (!beanFactory.containsBean(beanName)) {
|
||||
beanName = defaultName;
|
||||
synchronized (CanalInstanceGenerator.class) {
|
||||
try {
|
||||
// 设置当前正在加载的通道，加载spring查找文件时会用到该变量
|
||||
System.setProperty("canal.instance.destination", destination);
|
||||
this.beanFactory = getBeanFactory(springXml);
|
||||
String beanName = destination;
|
||||
if (!beanFactory.containsBean(beanName)) {
|
||||
beanName = defaultName;
|
||||
}
|
||||
|
||||
return (CanalInstance) beanFactory.getBean(beanName);
|
||||
} catch (Throwable e) {
|
||||
logger.error("generator instance failed.", e);
|
||||
throw new CanalException(e);
|
||||
} finally {
|
||||
System.setProperty("canal.instance.destination", "");
|
||||
}
|
||||
}
|
||||
|
||||
return (CanalInstance) beanFactory.getBean(beanName);
|
||||
}
|
||||
|
||||
public void setBeanFactory(BeanFactory beanFactory) throws BeansException {
|
||||
this.beanFactory = beanFactory;
|
||||
private BeanFactory getBeanFactory(String springXml) {
|
||||
ApplicationContext applicationContext = new ClassPathXmlApplicationContext(springXml);
|
||||
return applicationContext;
|
||||
}
|
||||
|
||||
public void setSpringXml(String springXml) {
|
||||
this.springXml = springXml;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,10 +21,19 @@ import org.springframework.util.Assert;
|
||||
*/
|
||||
public class PropertyPlaceholderConfigurer extends org.springframework.beans.factory.config.PropertyPlaceholderConfigurer implements ResourceLoaderAware, InitializingBean {
|
||||
|
||||
private static final String PLACEHOLDER_PREFIX = "${";
|
||||
private static final String PLACEHOLDER_SUFFIX = "}";
|
||||
private ResourceLoader loader;
|
||||
private String[] locationNames;
|
||||
private static final String PLACEHOLDER_PREFIX = "${";
|
||||
private static final String PLACEHOLDER_SUFFIX = "}";
|
||||
public static ThreadLocal<Properties> propertiesLocal = new ThreadLocal<Properties>() {
|
||||
|
||||
@Override
|
||||
protected Properties initialValue() {
|
||||
return new Properties();
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
private ResourceLoader loader;
|
||||
private String[] locationNames;
|
||||
|
||||
public PropertyPlaceholderConfigurer(){
|
||||
setIgnoreUnresolvablePlaceholders(true);
|
||||
@@ -117,13 +126,31 @@ public class PropertyPlaceholderConfigurer extends org.springframework.beans.fac
|
||||
@Override
|
||||
protected String resolvePlaceholder(String placeholder, Properties props, int systemPropertiesMode) {
|
||||
DefaultablePlaceholder dp = new DefaultablePlaceholder(placeholder);
|
||||
String value = super.resolvePlaceholder(dp.placeholder, props, systemPropertiesMode);
|
||||
|
||||
if (value == null) {
|
||||
value = dp.defaultValue;
|
||||
String propVal = null;
|
||||
// 以system为准覆盖本地配置, 适用于docker
|
||||
if (systemPropertiesMode == SYSTEM_PROPERTIES_MODE_OVERRIDE) {
|
||||
propVal = resolveSystemProperty(dp.placeholder);
|
||||
}
|
||||
|
||||
return trimToEmpty(value);
|
||||
// 以threadlocal的为准覆盖file properties
|
||||
if (propVal == null) {
|
||||
Properties localProperties = propertiesLocal.get();
|
||||
propVal = resolvePlaceholder(dp.placeholder, localProperties);
|
||||
}
|
||||
|
||||
if (propVal == null) {
|
||||
propVal = resolvePlaceholder(dp.placeholder, props);
|
||||
}
|
||||
|
||||
if (propVal == null && systemPropertiesMode == SYSTEM_PROPERTIES_MODE_FALLBACK) {
|
||||
propVal = resolveSystemProperty(dp.placeholder);
|
||||
}
|
||||
|
||||
if (propVal == null) {
|
||||
propVal = dp.defaultValue;
|
||||
}
|
||||
|
||||
return trimToEmpty(propVal);
|
||||
}
|
||||
|
||||
private static class DefaultablePlaceholder {
|
||||
|
||||
@@ -3,6 +3,8 @@ package com.alibaba.otter.canal.protocol;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* <pre>
|
||||
@@ -19,6 +21,25 @@ import java.util.Arrays;
|
||||
*/
|
||||
public class SecurityUtil {
|
||||
|
||||
private static char[] digits = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c',
|
||||
'd', 'e', 'f' };
|
||||
|
||||
private static Map<Character, Integer> rDigits = new HashMap<Character, Integer>(16);
|
||||
static {
|
||||
for (int i = 0; i < digits.length; ++i) {
|
||||
rDigits.put(digits[i], i);
|
||||
}
|
||||
}
|
||||
|
||||
public static String md5String(String content) throws NoSuchAlgorithmException {
|
||||
MessageDigest md = MessageDigest.getInstance("md5");
|
||||
byte[] bt = md.digest(content.getBytes());
|
||||
if (null == bt || bt.length != 16) {
|
||||
throw new IllegalArgumentException("md5 need");
|
||||
}
|
||||
return byte2HexStr(bt);
|
||||
}
|
||||
|
||||
public static final String scrambleGenPass(byte[] pass) throws NoSuchAlgorithmException {
|
||||
MessageDigest md = MessageDigest.getInstance("SHA-1");
|
||||
byte[] pass1 = md.digest(pass);
|
||||
|
||||
Reference in New Issue
Block a user