mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
修复测试类报错 (#4516)
* optimize YAML config loader (#4332) * fix bug BASE TABLE as table name (#4217) * fix issues#4328 (#4329) * docs: add nodejs canal client support (#4260) Co-authored-by: zhangxunwei <zhangxunwei@dxy.cn> * fix destination not encoded (#4279) * 修复Canal指定时间戳启动失效，总是从最新的点位开始同步问题 (#4348) * 支持用户自定义的CanalAlarmHandler * RowsLogEvent增加对TableMapLogEvent判空检查，防止NPE异常 * 修复Canal指定时间戳启动失效，总是从最新的点位开始同步问题, issue: #4347 Co-authored-by: 云时 <mingya.wmy@alibaba-inc.com> * fixed 4334 , support jdk8/jdk11 * fixed issue #4266 , typo * fixed issue #4243 , support auto register for cluster = null * fixed issue #4225 , support mysql version >= 8.0.26 heartbeat v2 * fixed issue #4308 , support query_log_event for maraiadb 10.10.1 * ignore compression event * support jdk11 * support druid 1.2.12 * fixed mariadb 10.x * sync canal-template.properties * update fastjson & druid version (#4406) * 修复升级2.0.4导致兼容的问题 * update druid & fastjson version * update fastjson version * update druid & fastjson version * meta.dat文件数据丢失 (#4397) * update fastjson & druid version (#4438) * 局部变量线程安全，优先使用StringBuilder替换StringBuffer (#4472) Co-authored-by: 夏亮 <xialiang@newerabc.com> * 1. CanalController stop 需要同时将 embededCanalServer.stop (#4477) 2. ServerRunningMonitor 线程池未正常回收，线程池管理与 start/stop保持一致 * vuln-fix: Use HTTPS instead of HTTP to resolve dependencies (#4437) This fixes a security vulnerability in this project where the `build.gradle` files were configuring Gradle to resolve dependencies over HTTP instead of HTTPS. Weakness: CWE-829: Inclusion of Functionality from Untrusted Control Sphere Severity: High CVSSS: 8.1 Detection: OpenRewrite Reported-by: Jonathan Leitschuh <Jonathan.Leitschuh@gmail.com> Signed-off-by: Jonathan Leitschuh <Jonathan.Leitschuh@gmail.com> Bug-tracker: https://github.com/JLLeitschuh/security-research/issues/9 Co-authored-by: Moderne <team@moderne.io> Co-authored-by: Moderne <team@moderne.io> * fix_ETL同步mysql关键字报错 (#4346) Co-authored-by: foleyang <foleyang@cogobuy.com> * optimize code * use compact BigDecimal * performance optimize , 1. cache string names 2. cache Charset * performance optimize ,cache integer/long valueof * 测试类报错 Co-authored-by: gongchangyou <gongchangyou@gmail.com> Co-authored-by: ChanaLii <316529035@qq.com> Co-authored-by: zhangxunwei <zhangxunweia@gmail.com> Co-authored-by: zhangxunwei <zhangxunwei@dxy.cn> Co-authored-by: tianpeidong <39491687+tianpeidong@users.noreply.github.com> Co-authored-by: dataccs <dataccs@163.com> Co-authored-by: 云时 <mingya.wmy@alibaba-inc.com> Co-authored-by: jianghang.loujh <jianghang.loujh@alibaba-inc.com> Co-authored-by: 温绍锦 <shaojin.wensj@alibaba-inc.com> Co-authored-by: noaso <noasoso@gmail.com> Co-authored-by: HumanPassenger <40585855+HumanPassenger@users.noreply.github.com> Co-authored-by: 夏亮 <xialiang@newerabc.com> Co-authored-by: 华仔 <591327356@qq.com> Co-authored-by: Jonathan Leitschuh <jonathan.leitschuh@gmail.com> Co-authored-by: Moderne <team@moderne.io> Co-authored-by: 杰锅不是锅 <1105568074@qq.com> Co-authored-by: foleyang <foleyang@cogobuy.com>
This commit is contained in:
rewerma
@@ -85,6 +85,7 @@ canal 特别设计了 client-server 模式，交互协议使用 protobuf 3.0 , c
|
||||
- canal php客户端: [https://github.com/xingwenge/canal-php](https://github.com/xingwenge/canal-php)
|
||||
- canal Python客户端：[https://github.com/haozi3156666/canal-python](https://github.com/haozi3156666/canal-python)
|
||||
- canal Rust客户端：[https://github.com/laohanlinux/canal-rs](https://github.com/laohanlinux/canal-rs)
|
||||
- canal Nodejs客户端：[https://github.com/marmot-z/canal-nodejs](https://github.com/marmot-z/canal-nodejs)
|
||||
|
||||
canal 作为 MySQL binlog 增量获取和解析工具，可将变更记录投递到 MQ 系统中，比如 Kafka/RocketMQ，可以借助于 MQ 的多语言能力
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ export default {
|
||||
canalInstanceDetail(this.$route.query.id).then(response => {
|
||||
const data = response.data
|
||||
this.form.id = data.id
|
||||
this.form.name = data.name + '/instance.propertios'
|
||||
this.form.name = data.name + '/instance.properties'
|
||||
this.form.content = data.content
|
||||
this.form.clusterServerId = data.clusterServerId
|
||||
})
|
||||
|
||||
@@ -69,9 +69,12 @@ else
|
||||
fi
|
||||
|
||||
if [ -n "$str" ]; then
|
||||
# JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
|
||||
# For G1
|
||||
JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
|
||||
if [ $JavaVersion -ge 11 ] ; then
|
||||
# For G1
|
||||
JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
|
||||
else
|
||||
JAVA_OPTS="-server -Xms2g -Xmx3g -Xmn1g -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
|
||||
fi
|
||||
else
|
||||
JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS"
|
||||
fi
|
||||
|
||||
@@ -25,7 +25,7 @@ canal.zkServers =
|
||||
# flush data to zk
|
||||
canal.zookeeper.flush.period = 1000
|
||||
canal.withoutNetty = false
|
||||
# tcp, kafka, rocketMQ, rabbitMQ
|
||||
# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ
|
||||
canal.serverMode = tcp
|
||||
# flush meta cursor/parse position to file
|
||||
canal.file.data.dir = ${canal.conf.dir}
|
||||
@@ -96,7 +96,7 @@ canal.instance.tsdb.snapshot.expire = 360
|
||||
#################################################
|
||||
######### destinations #############
|
||||
#################################################
|
||||
canal.destinations =
|
||||
canal.destinations =
|
||||
# conf root dir
|
||||
canal.conf.dir = ../conf
|
||||
# auto scan instance dir add/remove and start/stop instance
|
||||
@@ -148,8 +148,13 @@ kafka.max.in.flight.requests.per.connection = 1
|
||||
kafka.retries = 0
|
||||
|
||||
kafka.kerberos.enable = false
|
||||
kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
|
||||
kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
|
||||
kafka.kerberos.krb5.file = ../conf/kerberos/krb5.conf
|
||||
kafka.kerberos.jaas.file = ../conf/kerberos/jaas.conf
|
||||
|
||||
# sasl demo
|
||||
# kafka.sasl.jaas.config = org.apache.kafka.common.security.scram.ScramLoginModule required \\n username=\"alice\" \\npassword="alice-secret\";
|
||||
# kafka.sasl.mechanism = SCRAM-SHA-512
|
||||
# kafka.security.protocol = SASL_PLAINTEXT
|
||||
|
||||
##################################################
|
||||
######### RocketMQ #############
|
||||
@@ -161,7 +166,7 @@ rocketmq.namespace =
|
||||
rocketmq.namesrv.addr = 127.0.0.1:9876
|
||||
rocketmq.retry.times.when.send.failed = 0
|
||||
rocketmq.vip.channel.enabled = false
|
||||
rocketmq.tag =
|
||||
rocketmq.tag =
|
||||
|
||||
##################################################
|
||||
######### RabbitMQ #############
|
||||
@@ -171,4 +176,12 @@ rabbitmq.virtual.host =
|
||||
rabbitmq.exchange =
|
||||
rabbitmq.username =
|
||||
rabbitmq.password =
|
||||
rabbitmq.deliveryMode =
|
||||
rabbitmq.deliveryMode =
|
||||
|
||||
|
||||
##################################################
|
||||
######### Pulsar #############
|
||||
##################################################
|
||||
pulsarmq.serverUrl =
|
||||
pulsarmq.roleToken =
|
||||
pulsarmq.topicTenantPrefix =
|
||||
|
||||
@@ -23,29 +23,8 @@
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-aop</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-context</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-jdbc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-orm</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-test</artifactId>
|
||||
<scope>test</scope>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>joda-time</groupId>
|
||||
@@ -55,10 +34,6 @@
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>druid</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.yaml</groupId>
|
||||
<artifactId>snakeyaml</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.powermock</groupId>
|
||||
<artifactId>powermock-api-mockito</artifactId>
|
||||
|
||||
@@ -1,155 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.springframework.util.PropertyPlaceholderHelper;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.bind.PropertiesConfigurationFactory;
|
||||
import com.alibaba.otter.canal.client.adapter.config.common.*;
|
||||
|
||||
/**
|
||||
* 将yaml内容绑定到指定对象, 遵循spring yml的绑定规范
|
||||
*
|
||||
* @author reweerma 2019-2-1 上午9:14:02
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class YmlConfigBinder {
|
||||
|
||||
/**
|
||||
* 将当前内容绑定到指定对象
|
||||
*
|
||||
* @param content yml内容
|
||||
* @param clazz 指定对象类型
|
||||
* @return 对象
|
||||
*/
|
||||
public static <T> T bindYmlToObj(String content, Class<T> clazz) {
|
||||
return bindYmlToObj(null, content, clazz, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将当前内容绑定到指定对象并指定内容编码格式
|
||||
*
|
||||
* @param content yml内容
|
||||
* @param clazz 指定对象类型
|
||||
* @param charset yml内容编码格式
|
||||
* @return 对象
|
||||
*/
|
||||
public static <T> T bindYmlToObj(String content, Class<T> clazz, String charset) {
|
||||
return bindYmlToObj(null, content, clazz, charset);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将当前内容指定前缀部分绑定到指定对象
|
||||
*
|
||||
* @param prefix 指定前缀
|
||||
* @param content yml内容
|
||||
* @param clazz 指定对象类型
|
||||
* @return 对象
|
||||
*/
|
||||
public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz) {
|
||||
return bindYmlToObj(prefix, content, clazz, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将当前内容指定前缀部分绑定到指定对象并指定内容编码格式
|
||||
*
|
||||
* @param prefix 指定前缀
|
||||
* @param content yml内容
|
||||
* @param clazz 指定对象类型
|
||||
* @param charset yml内容编码格式
|
||||
* @return 对象
|
||||
*/
|
||||
public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz, String charset) {
|
||||
return bindYmlToObj(prefix, content, clazz, charset, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: 当前内容有属性 zkServers: ${zookeeper.servers}
|
||||
* 在envProperties中有属性 zookeeper.servers:
|
||||
* 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 则当前内容 zkServers 会被替换为
|
||||
* zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 注: 假设绑定的类中
|
||||
* zkServers 属性是 List<String> 对象, 则会自动映射成List
|
||||
*
|
||||
* @param prefix 指定前缀
|
||||
* @param content yml内容
|
||||
* @param clazz 指定对象类型
|
||||
* @param charset yml内容编码格式
|
||||
* @return 对象
|
||||
*/
|
||||
public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz, String charset,
|
||||
Properties baseProperties) {
|
||||
try {
|
||||
byte[] contentBytes;
|
||||
if (charset == null) {
|
||||
contentBytes = content.getBytes("UTF-8");
|
||||
} else {
|
||||
contentBytes = content.getBytes(charset);
|
||||
}
|
||||
YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader();
|
||||
Resource configResource = new ByteArrayResource(contentBytes);
|
||||
PropertySource<?> propertySource = propertySourceLoader.load("manualBindConfig", configResource, null);
|
||||
|
||||
if (propertySource == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Properties properties = new Properties();
|
||||
Map<String, Object> propertiesRes = new LinkedHashMap<>();
|
||||
if (!StringUtils.isEmpty(prefix) && !prefix.endsWith(".")) {
|
||||
prefix = prefix + ".";
|
||||
}
|
||||
|
||||
properties.putAll((Map<?, ?>) propertySource.getSource());
|
||||
|
||||
if (baseProperties != null) {
|
||||
baseProperties.putAll(properties);
|
||||
properties = baseProperties;
|
||||
}
|
||||
|
||||
for (Map.Entry<?, ?> entry : ((Map<?, ?>) propertySource.getSource()).entrySet()) {
|
||||
String key = (String) entry.getKey();
|
||||
Object value = entry.getValue();
|
||||
|
||||
if (prefix != null) {
|
||||
if (key != null && key.startsWith(prefix)) {
|
||||
key = key.substring(prefix.length());
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (value instanceof String && ((String) value).contains("${") && ((String) value).contains("}")) {
|
||||
PropertyPlaceholderHelper propertyPlaceholderHelper = new PropertyPlaceholderHelper("${", "}");
|
||||
value = propertyPlaceholderHelper.replacePlaceholders((String) value, properties);
|
||||
}
|
||||
|
||||
propertiesRes.put(key, value);
|
||||
}
|
||||
|
||||
if (propertiesRes.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
propertySource = new MapPropertySource(propertySource.getName(), propertiesRes);
|
||||
|
||||
T target = clazz.newInstance();
|
||||
|
||||
MutablePropertySources propertySources = new MutablePropertySources();
|
||||
propertySources.addFirst(propertySource);
|
||||
|
||||
PropertiesConfigurationFactory<Object> factory = new PropertiesConfigurationFactory<>(target);
|
||||
factory.setPropertySources(propertySources);
|
||||
factory.setIgnoreInvalidFields(true);
|
||||
factory.setIgnoreUnknownFields(true);
|
||||
|
||||
factory.bindPropertiesToTarget();
|
||||
|
||||
return target;
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.bind;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* {@link PropertyNamePatternsMatcher} that matches when a property name exactly
|
||||
* matches one of the given names, or starts with one of the given names
|
||||
* followed by a delimiter. This implementation is optimized for frequent calls.
|
||||
*
|
||||
* @author Phillip Webb
|
||||
* @since 1.2.0
|
||||
*/
|
||||
class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher {
|
||||
|
||||
private final char[] delimiters;
|
||||
|
||||
private final boolean ignoreCase;
|
||||
|
||||
private final String[] names;
|
||||
|
||||
protected DefaultPropertyNamePatternsMatcher(char[] delimiters, String... names){
|
||||
this(delimiters, false, names);
|
||||
}
|
||||
|
||||
protected DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, String... names){
|
||||
this(delimiters, ignoreCase, new HashSet<>(Arrays.asList(names)));
|
||||
}
|
||||
|
||||
DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, Set<String> names){
|
||||
this.delimiters = delimiters;
|
||||
this.ignoreCase = ignoreCase;
|
||||
this.names = names.toArray(new String[names.size()]);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean matches(String propertyName) {
|
||||
char[] propertyNameChars = propertyName.toCharArray();
|
||||
boolean[] match = new boolean[this.names.length];
|
||||
boolean noneMatched = true;
|
||||
for (int i = 0; i < this.names.length; i++) {
|
||||
if (this.names[i].length() <= propertyNameChars.length) {
|
||||
match[i] = true;
|
||||
noneMatched = false;
|
||||
}
|
||||
}
|
||||
if (noneMatched) {
|
||||
return false;
|
||||
}
|
||||
for (int charIndex = 0; charIndex < propertyNameChars.length; charIndex++) {
|
||||
for (int nameIndex = 0; nameIndex < this.names.length; nameIndex++) {
|
||||
if (match[nameIndex]) {
|
||||
match[nameIndex] = false;
|
||||
if (charIndex < this.names[nameIndex].length()) {
|
||||
if (isCharMatch(this.names[nameIndex].charAt(charIndex), propertyNameChars[charIndex])) {
|
||||
match[nameIndex] = true;
|
||||
noneMatched = false;
|
||||
}
|
||||
} else {
|
||||
char charAfter = propertyNameChars[this.names[nameIndex].length()];
|
||||
if (isDelimiter(charAfter)) {
|
||||
match[nameIndex] = true;
|
||||
noneMatched = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (noneMatched) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (boolean b : match) {
|
||||
if (b) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean isCharMatch(char c1, char c2) {
|
||||
if (this.ignoreCase) {
|
||||
return Character.toLowerCase(c1) == Character.toLowerCase(c2);
|
||||
}
|
||||
return c1 == c2;
|
||||
}
|
||||
|
||||
private boolean isDelimiter(char c) {
|
||||
for (char delimiter : this.delimiters) {
|
||||
if (c == delimiter) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.bind;
|
||||
|
||||
import java.beans.PropertyEditorSupport;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
|
||||
/**
|
||||
* {@link PropertyNamePatternsMatcher} that matches when a property name exactly
|
||||
* matches one of the given names, or starts with one of the given names
|
||||
* followed by a delimiter. This implementation is optimized for frequent calls.
|
||||
*
|
||||
* @author Phillip Webb
|
||||
* @since 1.2.0
|
||||
*/
|
||||
class InetAddressEditor extends PropertyEditorSupport {
|
||||
|
||||
@Override
|
||||
public String getAsText() {
|
||||
return ((InetAddress) getValue()).getHostAddress();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
try {
|
||||
setValue(InetAddress.getByName(text));
|
||||
} catch (UnknownHostException ex) {
|
||||
throw new IllegalArgumentException("Cannot locate host", ex);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.bind;
|
||||
|
||||
import org.springframework.beans.PropertyValue;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.common.PropertySource;
|
||||
|
||||
/**
|
||||
* A {@link PropertyValue} that can provide information about its origin.
|
||||
*
|
||||
* @author Andy Wilkinson
|
||||
*/
|
||||
class OriginCapablePropertyValue extends PropertyValue {
|
||||
|
||||
private static final String ATTRIBUTE_PROPERTY_ORIGIN = "propertyOrigin";
|
||||
|
||||
private final PropertyOrigin origin;
|
||||
|
||||
OriginCapablePropertyValue(PropertyValue propertyValue){
|
||||
this(propertyValue.getName(),
|
||||
propertyValue.getValue(),
|
||||
(PropertyOrigin) propertyValue.getAttribute(ATTRIBUTE_PROPERTY_ORIGIN));
|
||||
}
|
||||
|
||||
OriginCapablePropertyValue(String name, Object value, String originName, PropertySource<?> originSource){
|
||||
this(name, value, new PropertyOrigin(originSource, originName));
|
||||
}
|
||||
|
||||
OriginCapablePropertyValue(String name, Object value, PropertyOrigin origin){
|
||||
super(name, value);
|
||||
this.origin = origin;
|
||||
setAttribute(ATTRIBUTE_PROPERTY_ORIGIN, origin);
|
||||
}
|
||||
|
||||
public PropertyOrigin getOrigin() {
|
||||
return this.origin;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
String name = (this.origin != null ? this.origin.getName() : this.getName());
|
||||
String source = (this.origin.getSource() != null ? this.origin.getSource().getName() : "unknown");
|
||||
return "'" + name + "' from '" + source + "'";
|
||||
}
|
||||
|
||||
public static PropertyOrigin getOrigin(PropertyValue propertyValue) {
|
||||
if (propertyValue instanceof OriginCapablePropertyValue) {
|
||||
return ((OriginCapablePropertyValue) propertyValue).getOrigin();
|
||||
}
|
||||
return new OriginCapablePropertyValue(propertyValue).getOrigin();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.bind;
|
||||
|
||||
import java.util.Collection;
|
||||
|
||||
import org.springframework.util.PatternMatchUtils;
|
||||
|
||||
/**
|
||||
* {@link PropertyNamePatternsMatcher} that delegates to
|
||||
* {@link PatternMatchUtils#simpleMatch(String[], String)}.
|
||||
*
|
||||
* @author Phillip Webb
|
||||
* @since 1.2.0
|
||||
*/
|
||||
class PatternPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher {
|
||||
|
||||
private final String[] patterns;
|
||||
|
||||
PatternPropertyNamePatternsMatcher(Collection<String> patterns){
|
||||
this.patterns = (patterns != null ? patterns.toArray(new String[patterns.size()]) : new String[] {});
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean matches(String propertyName) {
|
||||
return PatternMatchUtils.simpleMatch(this.patterns, propertyName);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,356 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.bind;
|
||||
|
||||
import java.beans.PropertyDescriptor;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.PropertyValues;
|
||||
import org.springframework.beans.factory.FactoryBean;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
import org.springframework.beans.support.ResourceEditorRegistrar;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ApplicationContextAware;
|
||||
import org.springframework.context.MessageSource;
|
||||
import org.springframework.context.MessageSourceAware;
|
||||
import org.springframework.core.convert.ConversionService;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.validation.*;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.common.PropertySources;
|
||||
|
||||
/**
|
||||
* Validate some {@link Properties} (or optionally
|
||||
* {@link org.springframework.core.env.PropertySources}) by binding them to an
|
||||
* object of a specified type and then optionally running a {@link Validator}
|
||||
* over it.
|
||||
*
|
||||
* @param <T> the target type
|
||||
* @author Dave Syer
|
||||
*/
|
||||
public class PropertiesConfigurationFactory<T> implements FactoryBean<T>, ApplicationContextAware, MessageSourceAware, InitializingBean {
|
||||
|
||||
private static final char[] EXACT_DELIMITERS = { '_', '.', '[' };
|
||||
|
||||
private static final char[] TARGET_NAME_DELIMITERS = { '_', '.' };
|
||||
|
||||
private static final Log logger = LogFactory.getLog(PropertiesConfigurationFactory.class);
|
||||
|
||||
private boolean ignoreUnknownFields = true;
|
||||
|
||||
private boolean ignoreInvalidFields;
|
||||
|
||||
private boolean exceptionIfInvalid = true;
|
||||
|
||||
private PropertySources propertySources;
|
||||
|
||||
private final T target;
|
||||
|
||||
private Validator validator;
|
||||
|
||||
private ApplicationContext applicationContext;
|
||||
|
||||
private MessageSource messageSource;
|
||||
|
||||
private boolean hasBeenBound = false;
|
||||
|
||||
private boolean ignoreNestedProperties = false;
|
||||
|
||||
private String targetName;
|
||||
|
||||
private ConversionService conversionService;
|
||||
|
||||
private boolean resolvePlaceholders = true;
|
||||
|
||||
/**
|
||||
* Create a new {@link PropertiesConfigurationFactory} instance.
|
||||
*
|
||||
* @param target the target object to bind too
|
||||
* @see #PropertiesConfigurationFactory(Class)
|
||||
*/
|
||||
public PropertiesConfigurationFactory(T target){
|
||||
Assert.notNull(target, "target must not be null");
|
||||
this.target = target;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new {@link PropertiesConfigurationFactory} instance.
|
||||
*
|
||||
* @param type the target type
|
||||
* @see #PropertiesConfigurationFactory(Class)
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public PropertiesConfigurationFactory(Class<?> type){
|
||||
Assert.notNull(type, "type must not be null");
|
||||
this.target = (T) BeanUtils.instantiate(type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Flag to disable binding of nested properties (i.e. those with period
|
||||
* separators in their paths). Can be useful to disable this if the name prefix
|
||||
* is empty and you don't want to ignore unknown fields.
|
||||
*
|
||||
* @param ignoreNestedProperties the flag to set (default false)
|
||||
*/
|
||||
public void setIgnoreNestedProperties(boolean ignoreNestedProperties) {
|
||||
this.ignoreNestedProperties = ignoreNestedProperties;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set whether to ignore unknown fields, that is, whether to ignore bind
|
||||
* parameters that do not have corresponding fields in the target object.
|
||||
* <p>
|
||||
* Default is "true". Turn this off to enforce that all bind parameters must
|
||||
* have a matching field in the target object.
|
||||
*
|
||||
* @param ignoreUnknownFields if unknown fields should be ignored
|
||||
*/
|
||||
public void setIgnoreUnknownFields(boolean ignoreUnknownFields) {
|
||||
this.ignoreUnknownFields = ignoreUnknownFields;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set whether to ignore invalid fields, that is, whether to ignore bind
|
||||
* parameters that have corresponding fields in the target object which are not
|
||||
* accessible (for example because of null values in the nested path).
|
||||
* <p>
|
||||
* Default is "false". Turn this on to ignore bind parameters for nested objects
|
||||
* in non-existing parts of the target object graph.
|
||||
*
|
||||
* @param ignoreInvalidFields if invalid fields should be ignored
|
||||
*/
|
||||
public void setIgnoreInvalidFields(boolean ignoreInvalidFields) {
|
||||
this.ignoreInvalidFields = ignoreInvalidFields;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the target name.
|
||||
*
|
||||
* @param targetName the target name
|
||||
*/
|
||||
public void setTargetName(String targetName) {
|
||||
this.targetName = targetName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationContext(ApplicationContext applicationContext) {
|
||||
this.applicationContext = applicationContext;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the message source.
|
||||
*
|
||||
* @param messageSource the message source
|
||||
*/
|
||||
@Override
|
||||
public void setMessageSource(MessageSource messageSource) {
|
||||
this.messageSource = messageSource;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the property sources.
|
||||
*
|
||||
* @param propertySources the property sources
|
||||
*/
|
||||
public void setPropertySources(PropertySources propertySources) {
|
||||
this.propertySources = propertySources;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the conversion service.
|
||||
*
|
||||
* @param conversionService the conversion service
|
||||
*/
|
||||
public void setConversionService(ConversionService conversionService) {
|
||||
this.conversionService = conversionService;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the validator.
|
||||
*
|
||||
* @param validator the validator
|
||||
*/
|
||||
public void setValidator(Validator validator) {
|
||||
this.validator = validator;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a flag to indicate that an exception should be raised if a Validator is
|
||||
* available and validation fails.
|
||||
*
|
||||
* @param exceptionIfInvalid the flag to set
|
||||
* @deprecated as of 1.5, do not specify a {@link Validator} if validation
|
||||
* should not occur
|
||||
*/
|
||||
@Deprecated
|
||||
public void setExceptionIfInvalid(boolean exceptionIfInvalid) {
|
||||
this.exceptionIfInvalid = exceptionIfInvalid;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flag to indicate that placeholders should be replaced during binding. Default
|
||||
* is true.
|
||||
*
|
||||
* @param resolvePlaceholders flag value
|
||||
*/
|
||||
public void setResolvePlaceholders(boolean resolvePlaceholders) {
|
||||
this.resolvePlaceholders = resolvePlaceholders;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void afterPropertiesSet() throws Exception {
|
||||
bindPropertiesToTarget();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<?> getObjectType() {
|
||||
if (this.target == null) {
|
||||
return Object.class;
|
||||
}
|
||||
return this.target.getClass();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isSingleton() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public T getObject() throws Exception {
|
||||
if (!this.hasBeenBound) {
|
||||
bindPropertiesToTarget();
|
||||
}
|
||||
return this.target;
|
||||
}
|
||||
|
||||
public void bindPropertiesToTarget() throws BindException {
|
||||
Assert.state(this.propertySources != null, "PropertySources should not be null");
|
||||
try {
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Property Sources: " + this.propertySources);
|
||||
|
||||
}
|
||||
this.hasBeenBound = true;
|
||||
doBindPropertiesToTarget();
|
||||
} catch (BindException ex) {
|
||||
if (this.exceptionIfInvalid) {
|
||||
throw ex;
|
||||
}
|
||||
logger.error("Failed to load Properties validation bean. " + "Your Properties may be invalid.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void doBindPropertiesToTarget() throws BindException {
|
||||
RelaxedDataBinder dataBinder = (this.targetName != null ? new RelaxedDataBinder(this.target,
|
||||
this.targetName) : new RelaxedDataBinder(this.target));
|
||||
if (this.validator != null && this.validator.supports(dataBinder.getTarget().getClass())) {
|
||||
dataBinder.setValidator(this.validator);
|
||||
}
|
||||
if (this.conversionService != null) {
|
||||
dataBinder.setConversionService(this.conversionService);
|
||||
}
|
||||
dataBinder.setAutoGrowCollectionLimit(Integer.MAX_VALUE);
|
||||
dataBinder.setIgnoreNestedProperties(this.ignoreNestedProperties);
|
||||
dataBinder.setIgnoreInvalidFields(this.ignoreInvalidFields);
|
||||
dataBinder.setIgnoreUnknownFields(this.ignoreUnknownFields);
|
||||
customizeBinder(dataBinder);
|
||||
if (this.applicationContext != null) {
|
||||
ResourceEditorRegistrar resourceEditorRegistrar = new ResourceEditorRegistrar(this.applicationContext,
|
||||
this.applicationContext.getEnvironment());
|
||||
resourceEditorRegistrar.registerCustomEditors(dataBinder);
|
||||
}
|
||||
Iterable<String> relaxedTargetNames = getRelaxedTargetNames();
|
||||
Set<String> names = getNames(relaxedTargetNames);
|
||||
PropertyValues propertyValues = getPropertySourcesPropertyValues(names, relaxedTargetNames);
|
||||
dataBinder.bind(propertyValues);
|
||||
if (this.validator != null) {
|
||||
dataBinder.validate();
|
||||
}
|
||||
checkForBindingErrors(dataBinder);
|
||||
}
|
||||
|
||||
private Iterable<String> getRelaxedTargetNames() {
|
||||
return (this.target != null
|
||||
&& StringUtils.hasLength(this.targetName) ? new RelaxedNames(this.targetName) : null);
|
||||
}
|
||||
|
||||
private Set<String> getNames(Iterable<String> prefixes) {
|
||||
Set<String> names = new LinkedHashSet<>();
|
||||
if (this.target != null) {
|
||||
PropertyDescriptor[] descriptors = BeanUtils.getPropertyDescriptors(this.target.getClass());
|
||||
for (PropertyDescriptor descriptor : descriptors) {
|
||||
String name = descriptor.getName();
|
||||
if (!name.equals("class")) {
|
||||
RelaxedNames relaxedNames = RelaxedNames.forCamelCase(name);
|
||||
if (prefixes == null) {
|
||||
for (String relaxedName : relaxedNames) {
|
||||
names.add(relaxedName);
|
||||
}
|
||||
} else {
|
||||
for (String prefix : prefixes) {
|
||||
for (String relaxedName : relaxedNames) {
|
||||
names.add(prefix + "." + relaxedName);
|
||||
names.add(prefix + "_" + relaxedName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
private PropertyValues getPropertySourcesPropertyValues(Set<String> names, Iterable<String> relaxedTargetNames) {
|
||||
PropertyNamePatternsMatcher includes = getPropertyNamePatternsMatcher(names, relaxedTargetNames);
|
||||
return new PropertySourcesPropertyValues(this.propertySources, names, includes, this.resolvePlaceholders);
|
||||
}
|
||||
|
||||
private PropertyNamePatternsMatcher getPropertyNamePatternsMatcher(Set<String> names,
|
||||
Iterable<String> relaxedTargetNames) {
|
||||
if (this.ignoreUnknownFields && !isMapTarget()) {
|
||||
// Since unknown fields are ignored we can filter them out early to save
|
||||
// unnecessary calls to the PropertySource.
|
||||
return new DefaultPropertyNamePatternsMatcher(EXACT_DELIMITERS, true, names);
|
||||
}
|
||||
if (relaxedTargetNames != null) {
|
||||
// We can filter properties to those starting with the target name, but
|
||||
// we can't do a complete filter since we need to trigger the
|
||||
// unknown fields check
|
||||
Set<String> relaxedNames = new HashSet<>();
|
||||
for (String relaxedTargetName : relaxedTargetNames) {
|
||||
relaxedNames.add(relaxedTargetName);
|
||||
}
|
||||
return new DefaultPropertyNamePatternsMatcher(TARGET_NAME_DELIMITERS, true, relaxedNames);
|
||||
}
|
||||
// Not ideal, we basically can't filter anything
|
||||
return PropertyNamePatternsMatcher.ALL;
|
||||
}
|
||||
|
||||
private boolean isMapTarget() {
|
||||
return this.target != null && Map.class.isAssignableFrom(this.target.getClass());
|
||||
}
|
||||
|
||||
private void checkForBindingErrors(RelaxedDataBinder dataBinder) throws BindException {
|
||||
BindingResult errors = dataBinder.getBindingResult();
|
||||
if (errors.hasErrors()) {
|
||||
logger.error("Properties configuration failed validation");
|
||||
for (ObjectError error : errors.getAllErrors()) {
|
||||
logger.error(this.messageSource != null ? this.messageSource.getMessage(error, Locale.getDefault())
|
||||
+ " (" + error + ")" : error);
|
||||
}
|
||||
if (this.exceptionIfInvalid) {
|
||||
throw new BindException(errors);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Customize the data binder.
|
||||
*
|
||||
* @param dataBinder the data binder that will be used to bind and validate
|
||||
*/
|
||||
protected void customizeBinder(DataBinder dataBinder) {
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.bind;
|
||||
|
||||
/**
|
||||
* Strategy interface used to check if a property name matches specific
|
||||
* criteria.
|
||||
*
|
||||
* @author Phillip Webb
|
||||
* @since 1.2.0
|
||||
*/
|
||||
interface PropertyNamePatternsMatcher {
|
||||
|
||||
PropertyNamePatternsMatcher ALL = propertyName -> true;
|
||||
|
||||
PropertyNamePatternsMatcher NONE = propertyName -> false;
|
||||
|
||||
/**
|
||||
* Return {@code true} of the property name matches.
|
||||
*
|
||||
* @param propertyName the property name
|
||||
* @return {@code true} if the property name matches
|
||||
*/
|
||||
boolean matches(String propertyName);
|
||||
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.bind;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.common.PropertySource;
|
||||
|
||||
/**
|
||||
* The origin of a property, specifically its source and its name before any
|
||||
* prefix was removed.
|
||||
*
|
||||
* @author Andy Wilkinson
|
||||
* @since 1.3.0
|
||||
*/
|
||||
public class PropertyOrigin {
|
||||
|
||||
private final PropertySource<?> source;
|
||||
|
||||
private final String name;
|
||||
|
||||
PropertyOrigin(PropertySource<?> source, String name){
|
||||
this.name = name;
|
||||
this.source = source;
|
||||
}
|
||||
|
||||
public PropertySource<?> getSource() {
|
||||
return this.source;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return this.name;
|
||||
}
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.bind;
|
||||
|
||||
import org.springframework.core.convert.ConversionException;
|
||||
import org.springframework.core.env.AbstractEnvironment;
|
||||
import org.springframework.core.env.AbstractPropertyResolver;
|
||||
import org.springframework.core.env.PropertyResolver;
|
||||
import org.springframework.util.ClassUtils;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.common.PropertySource;
|
||||
import com.alibaba.otter.canal.client.adapter.config.common.PropertySources;
|
||||
|
||||
/**
|
||||
* {@link PropertyResolver} implementation that resolves property values against
|
||||
* an underlying set of {@link PropertySources}.
|
||||
*
|
||||
* @author Chris Beams
|
||||
* @author Juergen Hoeller
|
||||
* @see PropertySource
|
||||
* @see PropertySources
|
||||
* @see AbstractEnvironment
|
||||
* @since 3.1
|
||||
*/
|
||||
public class PropertySourcesPropertyResolver extends AbstractPropertyResolver {
|
||||
|
||||
private final PropertySources propertySources;
|
||||
|
||||
/**
|
||||
* Create a new resolver against the given property sources.
|
||||
*
|
||||
* @param propertySources the set of {@link PropertySource} objects to use
|
||||
*/
|
||||
public PropertySourcesPropertyResolver(PropertySources propertySources){
|
||||
this.propertySources = propertySources;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean containsProperty(String key) {
|
||||
if (this.propertySources != null) {
|
||||
for (PropertySource<?> propertySource : this.propertySources) {
|
||||
if (propertySource.containsProperty(key)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getProperty(String key) {
|
||||
return getProperty(key, String.class, true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T getProperty(String key, Class<T> targetValueType) {
|
||||
return getProperty(key, targetValueType, true);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getPropertyAsRawString(String key) {
|
||||
return getProperty(key, String.class, false);
|
||||
}
|
||||
|
||||
protected <T> T getProperty(String key, Class<T> targetValueType, boolean resolveNestedPlaceholders) {
|
||||
if (this.propertySources != null) {
|
||||
for (PropertySource<?> propertySource : this.propertySources) {
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger
|
||||
.trace("Searching for key '" + key + "' in PropertySource '" + propertySource.getName() + "'");
|
||||
}
|
||||
Object value = propertySource.getProperty(key);
|
||||
if (value != null) {
|
||||
if (resolveNestedPlaceholders && value instanceof String) {
|
||||
value = resolveNestedPlaceholders((String) value);
|
||||
}
|
||||
logKeyFound(key, propertySource, value);
|
||||
return convertValueIfNecessary(value, targetValueType);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Could not find key '" + key + "' in any property source");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public <T> Class<T> getPropertyAsClass(String key, Class<T> targetValueType) {
|
||||
if (this.propertySources != null) {
|
||||
for (PropertySource<?> propertySource : this.propertySources) {
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace(String.format("Searching for key '%s' in [%s]", key, propertySource.getName()));
|
||||
}
|
||||
Object value = propertySource.getProperty(key);
|
||||
if (value != null) {
|
||||
logKeyFound(key, propertySource, value);
|
||||
Class<?> clazz;
|
||||
if (value instanceof String) {
|
||||
try {
|
||||
clazz = ClassUtils.forName((String) value, null);
|
||||
} catch (Exception ex) {
|
||||
throw new PropertySourcesPropertyResolver.ClassConversionException((String) value,
|
||||
targetValueType,
|
||||
ex);
|
||||
}
|
||||
} else if (value instanceof Class) {
|
||||
clazz = (Class<?>) value;
|
||||
} else {
|
||||
clazz = value.getClass();
|
||||
}
|
||||
if (!targetValueType.isAssignableFrom(clazz)) {
|
||||
throw new PropertySourcesPropertyResolver.ClassConversionException(clazz, targetValueType);
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
Class<T> targetClass = (Class<T>) clazz;
|
||||
return targetClass;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug(String.format("Could not find key '%s' in any property source", key));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Log the given key as found in the given {@link PropertySource}, resulting in
|
||||
* the given value.
|
||||
* <p>
|
||||
* The default implementation writes a debug log message with key and source. As
|
||||
* of 4.3.3, this does not log the value anymore in order to avoid accidental
|
||||
* logging of sensitive settings. Subclasses may override this method to change
|
||||
* the log level and/or log message, including the property's value if desired.
|
||||
*
|
||||
* @param key the key found
|
||||
* @param propertySource the {@code PropertySource} that the key has been found
|
||||
* in
|
||||
* @param value the corresponding value
|
||||
* @since 4.3.1
|
||||
*/
|
||||
protected void logKeyFound(String key, PropertySource<?> propertySource, Object value) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Found key '" + key + "' in PropertySource '" + propertySource.getName()
|
||||
+ "' with value of type " + value.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("serial")
|
||||
@Deprecated
|
||||
private static class ClassConversionException extends ConversionException {
|
||||
|
||||
public ClassConversionException(Class<?> actual, Class<?> expected){
|
||||
super(String
|
||||
.format("Actual type %s is not assignable to expected type %s", actual.getName(), expected.getName()));
|
||||
}
|
||||
|
||||
public ClassConversionException(String actual, Class<?> expected, Exception ex){
|
||||
super(
|
||||
String
|
||||
.format("Could not find/load class %s during attempt to convert to %s", actual, expected.getName()),
|
||||
ex);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,233 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.bind;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.beans.MutablePropertyValues;
|
||||
import org.springframework.beans.PropertyValue;
|
||||
import org.springframework.beans.PropertyValues;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.validation.DataBinder;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.common.CompositePropertySource;
|
||||
import com.alibaba.otter.canal.client.adapter.config.common.EnumerablePropertySource;
|
||||
import com.alibaba.otter.canal.client.adapter.config.common.PropertySource;
|
||||
import com.alibaba.otter.canal.client.adapter.config.common.PropertySources;
|
||||
|
||||
/**
|
||||
* A {@link PropertyValues} implementation backed by a {@link PropertySources},
|
||||
* bridging the two abstractions and allowing (for instance) a regular
|
||||
* {@link DataBinder} to be used with the latter.
|
||||
*
|
||||
* @author Dave Syer
|
||||
* @author Phillip Webb
|
||||
*/
|
||||
public class PropertySourcesPropertyValues implements PropertyValues {
|
||||
|
||||
private static final Pattern COLLECTION_PROPERTY = Pattern
|
||||
.compile("\\[(\\d+)\\](\\.\\S+)?");
|
||||
|
||||
private final PropertySources propertySources;
|
||||
|
||||
private final Collection<String> nonEnumerableFallbackNames;
|
||||
|
||||
private final PropertyNamePatternsMatcher includes;
|
||||
|
||||
private final Map<String, PropertyValue> propertyValues = new LinkedHashMap<>();
|
||||
|
||||
private final ConcurrentHashMap<String, PropertySource<?>> collectionOwners = new ConcurrentHashMap<>();
|
||||
|
||||
private final boolean resolvePlaceholders;
|
||||
|
||||
/**
|
||||
* Create a new PropertyValues from the given PropertySources.
|
||||
*
|
||||
* @param propertySources a PropertySources instance
|
||||
*/
|
||||
public PropertySourcesPropertyValues(PropertySources propertySources){
|
||||
this(propertySources, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new PropertyValues from the given PropertySources that will
|
||||
* optionally resolve placeholders.
|
||||
*
|
||||
* @param propertySources a PropertySources instance
|
||||
* @param resolvePlaceholders {@code true} if placeholders should be resolved.
|
||||
* @since 1.5.2
|
||||
*/
|
||||
public PropertySourcesPropertyValues(PropertySources propertySources, boolean resolvePlaceholders){
|
||||
this(propertySources, (Collection<String>) null, PropertyNamePatternsMatcher.ALL, resolvePlaceholders);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new PropertyValues from the given PropertySources.
|
||||
*
|
||||
* @param propertySources a PropertySources instance
|
||||
* @param includePatterns property name patterns to include from system
|
||||
* properties and environment variables
|
||||
* @param nonEnumerableFallbackNames the property names to try in lieu of an
|
||||
* {@link EnumerablePropertySource}.
|
||||
*/
|
||||
public PropertySourcesPropertyValues(PropertySources propertySources, Collection<String> includePatterns,
|
||||
Collection<String> nonEnumerableFallbackNames){
|
||||
this(propertySources,
|
||||
nonEnumerableFallbackNames,
|
||||
new PatternPropertyNamePatternsMatcher(includePatterns),
|
||||
true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new PropertyValues from the given PropertySources.
|
||||
*
|
||||
* @param propertySources a PropertySources instance
|
||||
* @param nonEnumerableFallbackNames the property names to try in lieu of an
|
||||
* {@link EnumerablePropertySource}.
|
||||
* @param includes the property name patterns to include
|
||||
* @param resolvePlaceholders flag to indicate the placeholders should be
|
||||
* resolved
|
||||
*/
|
||||
PropertySourcesPropertyValues(PropertySources propertySources, Collection<String> nonEnumerableFallbackNames,
|
||||
PropertyNamePatternsMatcher includes, boolean resolvePlaceholders){
|
||||
Assert.notNull(propertySources, "PropertySources must not be null");
|
||||
Assert.notNull(includes, "Includes must not be null");
|
||||
this.propertySources = propertySources;
|
||||
this.nonEnumerableFallbackNames = nonEnumerableFallbackNames;
|
||||
this.includes = includes;
|
||||
this.resolvePlaceholders = resolvePlaceholders;
|
||||
PropertySourcesPropertyResolver resolver = new PropertySourcesPropertyResolver(propertySources);
|
||||
for (PropertySource<?> source : propertySources) {
|
||||
processPropertySource(source, resolver);
|
||||
}
|
||||
}
|
||||
|
||||
private void processPropertySource(PropertySource<?> source, PropertySourcesPropertyResolver resolver) {
|
||||
if (source instanceof CompositePropertySource) {
|
||||
processCompositePropertySource((CompositePropertySource) source, resolver);
|
||||
} else if (source instanceof EnumerablePropertySource) {
|
||||
processEnumerablePropertySource((EnumerablePropertySource<?>) source, resolver, this.includes);
|
||||
} else {
|
||||
processNonEnumerablePropertySource(source, resolver);
|
||||
}
|
||||
}
|
||||
|
||||
private void processCompositePropertySource(CompositePropertySource source,
|
||||
PropertySourcesPropertyResolver resolver) {
|
||||
for (PropertySource<?> nested : source.getPropertySources()) {
|
||||
processPropertySource(nested, resolver);
|
||||
}
|
||||
}
|
||||
|
||||
private void processEnumerablePropertySource(EnumerablePropertySource<?> source,
|
||||
PropertySourcesPropertyResolver resolver,
|
||||
PropertyNamePatternsMatcher includes) {
|
||||
if (source.getPropertyNames().length > 0) {
|
||||
for (String propertyName : source.getPropertyNames()) {
|
||||
if (includes.matches(propertyName)) {
|
||||
Object value = getEnumerableProperty(source, resolver, propertyName);
|
||||
putIfAbsent(propertyName, value, source);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Object getEnumerableProperty(EnumerablePropertySource<?> source, PropertySourcesPropertyResolver resolver,
|
||||
String propertyName) {
|
||||
try {
|
||||
if (this.resolvePlaceholders) {
|
||||
return resolver.getProperty(propertyName, Object.class);
|
||||
}
|
||||
} catch (RuntimeException ex) {
|
||||
// Probably could not resolve placeholders, ignore it here
|
||||
}
|
||||
return source.getProperty(propertyName);
|
||||
}
|
||||
|
||||
private void processNonEnumerablePropertySource(PropertySource<?> source,
|
||||
PropertySourcesPropertyResolver resolver) {
|
||||
// We can only do exact matches for non-enumerable property names, but
|
||||
// that's better than nothing...
|
||||
if (this.nonEnumerableFallbackNames == null) {
|
||||
return;
|
||||
}
|
||||
for (String propertyName : this.nonEnumerableFallbackNames) {
|
||||
if (!source.containsProperty(propertyName)) {
|
||||
continue;
|
||||
}
|
||||
Object value = null;
|
||||
try {
|
||||
value = resolver.getProperty(propertyName, Object.class);
|
||||
} catch (RuntimeException ex) {
|
||||
// Probably could not convert to Object, weird, but ignorable
|
||||
}
|
||||
if (value == null) {
|
||||
value = source.getProperty(propertyName.toUpperCase(Locale.ENGLISH));
|
||||
}
|
||||
putIfAbsent(propertyName, value, source);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public PropertyValue[] getPropertyValues() {
|
||||
Collection<PropertyValue> values = this.propertyValues.values();
|
||||
return values.toArray(new PropertyValue[values.size()]);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PropertyValue getPropertyValue(String propertyName) {
|
||||
PropertyValue propertyValue = this.propertyValues.get(propertyName);
|
||||
if (propertyValue != null) {
|
||||
return propertyValue;
|
||||
}
|
||||
for (PropertySource<?> source : this.propertySources) {
|
||||
Object value = source.getProperty(propertyName);
|
||||
propertyValue = putIfAbsent(propertyName, value, source);
|
||||
if (propertyValue != null) {
|
||||
return propertyValue;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private PropertyValue putIfAbsent(String propertyName, Object value, PropertySource<?> source) {
|
||||
if (value != null && !this.propertyValues.containsKey(propertyName)) {
|
||||
PropertySource<?> collectionOwner = this.collectionOwners
|
||||
.putIfAbsent(COLLECTION_PROPERTY.matcher(propertyName).replaceAll("[]"), source);
|
||||
if (collectionOwner == null || collectionOwner == source) {
|
||||
PropertyValue propertyValue = new OriginCapablePropertyValue(propertyName, value, propertyName, source);
|
||||
this.propertyValues.put(propertyName, propertyValue);
|
||||
return propertyValue;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PropertyValues changesSince(PropertyValues old) {
|
||||
MutablePropertyValues changes = new MutablePropertyValues();
|
||||
// for each property value in the new set
|
||||
for (PropertyValue newValue : getPropertyValues()) {
|
||||
// if there wasn't an old one, add it
|
||||
PropertyValue oldValue = old.getPropertyValue(newValue.getName());
|
||||
if (oldValue == null || !oldValue.equals(newValue)) {
|
||||
changes.addPropertyValue(newValue);
|
||||
}
|
||||
}
|
||||
return changes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean contains(String propertyName) {
|
||||
return getPropertyValue(propertyName) != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
return this.propertyValues.isEmpty();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.bind;
|
||||
|
||||
import java.util.EnumSet;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.core.convert.ConversionFailedException;
|
||||
import org.springframework.core.convert.ConversionService;
|
||||
import org.springframework.core.convert.TypeDescriptor;
|
||||
import org.springframework.core.convert.converter.Converter;
|
||||
import org.springframework.core.convert.converter.ConverterFactory;
|
||||
import org.springframework.core.convert.support.DefaultConversionService;
|
||||
import org.springframework.core.convert.support.GenericConversionService;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
/**
|
||||
* Internal {@link ConversionService} used by {@link RelaxedDataBinder} to
|
||||
* support additional relaxed conversion.
|
||||
*
|
||||
* @author Phillip Webb
|
||||
* @author Stephane Nicoll
|
||||
* @since 1.1.0
|
||||
*/
|
||||
class RelaxedConversionService implements ConversionService {
|
||||
|
||||
private final ConversionService conversionService;
|
||||
|
||||
private final GenericConversionService additionalConverters;
|
||||
|
||||
/**
|
||||
* Create a new {@link RelaxedConversionService} instance.
|
||||
*
|
||||
* @param conversionService and option root conversion service
|
||||
*/
|
||||
RelaxedConversionService(ConversionService conversionService){
|
||||
this.conversionService = conversionService;
|
||||
this.additionalConverters = new GenericConversionService();
|
||||
DefaultConversionService.addCollectionConverters(this.additionalConverters);
|
||||
this.additionalConverters
|
||||
.addConverterFactory(new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory());
|
||||
this.additionalConverters.addConverter(new StringToCharArrayConverter());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean canConvert(Class<?> sourceType, Class<?> targetType) {
|
||||
return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType))
|
||||
|| this.additionalConverters.canConvert(sourceType, targetType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean canConvert(TypeDescriptor sourceType, TypeDescriptor targetType) {
|
||||
return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType))
|
||||
|| this.additionalConverters.canConvert(sourceType, targetType);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
public <T> T convert(Object source, Class<T> targetType) {
|
||||
Assert.notNull(targetType, "The targetType to convert to cannot be null");
|
||||
return (T) convert(source, TypeDescriptor.forObject(source), TypeDescriptor.valueOf(targetType));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object convert(Object source, TypeDescriptor sourceType, TypeDescriptor targetType) {
|
||||
if (this.conversionService != null) {
|
||||
try {
|
||||
return this.conversionService.convert(source, sourceType, targetType);
|
||||
} catch (ConversionFailedException ex) {
|
||||
// Ignore and try the additional converters
|
||||
}
|
||||
}
|
||||
return this.additionalConverters.convert(source, sourceType, targetType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clone of Spring's package private StringToEnumConverterFactory, but ignoring
|
||||
* the case of the source.
|
||||
*/
|
||||
@SuppressWarnings({ "unchecked", "rawtypes" })
|
||||
private static class StringToEnumIgnoringCaseConverterFactory implements ConverterFactory<String, Enum> {
|
||||
|
||||
@Override
|
||||
public <T extends Enum> Converter<String, T> getConverter(Class<T> targetType) {
|
||||
Class<?> enumType = targetType;
|
||||
while (enumType != null && !enumType.isEnum()) {
|
||||
enumType = enumType.getSuperclass();
|
||||
}
|
||||
Assert.notNull(enumType, "The target type " + targetType.getName() + " does not refer to an enum");
|
||||
return new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory.StringToEnum(enumType);
|
||||
}
|
||||
|
||||
private static class StringToEnum<T extends Enum> implements Converter<String, T> {
|
||||
|
||||
private final Class<T> enumType;
|
||||
|
||||
StringToEnum(Class<T> enumType){
|
||||
this.enumType = enumType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public T convert(String source) {
|
||||
if (source.isEmpty()) {
|
||||
// It's an empty enum identifier: reset the enum value to null.
|
||||
return null;
|
||||
}
|
||||
source = source.trim();
|
||||
for (T candidate : (Set<T>) EnumSet.allOf(this.enumType)) {
|
||||
RelaxedNames names = new RelaxedNames(
|
||||
candidate.name().replace('_', '-').toLowerCase(Locale.ENGLISH));
|
||||
for (String name : names) {
|
||||
if (name.equals(source)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
if (candidate.name().equalsIgnoreCase(source)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException(
|
||||
"No enum constant " + this.enumType.getCanonicalName() + "." + source);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,729 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.bind;
|
||||
|
||||
import java.beans.PropertyEditor;
|
||||
import java.net.InetAddress;
|
||||
import java.util.*;
|
||||
|
||||
import org.springframework.beans.*;
|
||||
import org.springframework.beans.propertyeditors.FileEditor;
|
||||
import org.springframework.core.convert.ConversionService;
|
||||
import org.springframework.core.convert.TypeDescriptor;
|
||||
import org.springframework.core.env.StandardEnvironment;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.validation.AbstractPropertyBindingResult;
|
||||
import org.springframework.validation.BeanPropertyBindingResult;
|
||||
import org.springframework.validation.DataBinder;
|
||||
|
||||
/**
|
||||
* Binder implementation that allows caller to bind to maps and also allows
|
||||
* property names to match a bit loosely (if underscores or dashes are removed
|
||||
* and replaced with camel case for example).
|
||||
*
|
||||
* @author Dave Syer
|
||||
* @author Phillip Webb
|
||||
* @author Stephane Nicoll
|
||||
* @author Andy Wilkinson
|
||||
* @see RelaxedNames
|
||||
*/
|
||||
public class RelaxedDataBinder extends DataBinder {
|
||||
|
||||
private static final Set<Class<?>> EXCLUDED_EDITORS;
|
||||
|
||||
static {
|
||||
Set<Class<?>> excluded = new HashSet<>();
|
||||
excluded.add(FileEditor.class);
|
||||
EXCLUDED_EDITORS = Collections.unmodifiableSet(excluded);
|
||||
}
|
||||
|
||||
private static final Object BLANK = new Object();
|
||||
|
||||
private String namePrefix;
|
||||
|
||||
private boolean ignoreNestedProperties;
|
||||
|
||||
private MultiValueMap<String, String> nameAliases = new LinkedMultiValueMap<>();
|
||||
|
||||
/**
|
||||
* Create a new {@link RelaxedDataBinder} instance.
|
||||
*
|
||||
* @param target the target into which properties are bound
|
||||
*/
|
||||
public RelaxedDataBinder(Object target){
|
||||
super(wrapTarget(target));
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new {@link RelaxedDataBinder} instance.
|
||||
*
|
||||
* @param target the target into which properties are bound
|
||||
* @param namePrefix An optional prefix to be used when reading properties
|
||||
*/
|
||||
public RelaxedDataBinder(Object target, String namePrefix){
|
||||
super(wrapTarget(target), (StringUtils.hasLength(namePrefix) ? namePrefix : DEFAULT_OBJECT_NAME));
|
||||
this.namePrefix = cleanNamePrefix(namePrefix);
|
||||
}
|
||||
|
||||
private String cleanNamePrefix(String namePrefix) {
|
||||
if (!StringUtils.hasLength(namePrefix)) {
|
||||
return null;
|
||||
}
|
||||
return (namePrefix.endsWith(".") ? namePrefix : namePrefix + ".");
|
||||
}
|
||||
|
||||
/**
|
||||
* Flag to disable binding of nested properties (i.e. those with period
|
||||
* separators in their paths). Can be useful to disable this if the name prefix
|
||||
* is empty and you don't want to ignore unknown fields.
|
||||
*
|
||||
* @param ignoreNestedProperties the flag to set (default false)
|
||||
*/
|
||||
public void setIgnoreNestedProperties(boolean ignoreNestedProperties) {
|
||||
this.ignoreNestedProperties = ignoreNestedProperties;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set name aliases.
|
||||
*
|
||||
* @param aliases a map of property name to aliases
|
||||
*/
|
||||
public void setNameAliases(Map<String, List<String>> aliases) {
|
||||
this.nameAliases = new LinkedMultiValueMap<>(aliases);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add aliases to the {@link DataBinder}.
|
||||
*
|
||||
* @param name the property name to alias
|
||||
* @param alias aliases for the property names
|
||||
* @return this instance
|
||||
*/
|
||||
public RelaxedDataBinder withAlias(String name, String... alias) {
|
||||
for (String value : alias) {
|
||||
this.nameAliases.add(name, value);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doBind(MutablePropertyValues propertyValues) {
|
||||
super.doBind(modifyProperties(propertyValues, getTarget()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Modify the property values so that period separated property paths are valid
|
||||
* for map keys. Also creates new maps for properties of map type that are null
|
||||
* (assuming all maps are potentially nested). The standard bracket {@code[...]}
|
||||
* dereferencing is also accepted.
|
||||
*
|
||||
* @param propertyValues the property values
|
||||
* @param target the target object
|
||||
* @return modified property values
|
||||
*/
|
||||
private MutablePropertyValues modifyProperties(MutablePropertyValues propertyValues, Object target) {
|
||||
propertyValues = getPropertyValuesForNamePrefix(propertyValues);
|
||||
if (target instanceof RelaxedDataBinder.MapHolder) {
|
||||
propertyValues = addMapPrefix(propertyValues);
|
||||
}
|
||||
BeanWrapper wrapper = new BeanWrapperImpl(target);
|
||||
wrapper.setConversionService(new RelaxedConversionService(getConversionService()));
|
||||
wrapper.setAutoGrowNestedPaths(true);
|
||||
List<PropertyValue> sortedValues = new ArrayList<>();
|
||||
Set<String> modifiedNames = new HashSet<>();
|
||||
List<String> sortedNames = getSortedPropertyNames(propertyValues);
|
||||
for (String name : sortedNames) {
|
||||
PropertyValue propertyValue = propertyValues.getPropertyValue(name);
|
||||
PropertyValue modifiedProperty = modifyProperty(wrapper, propertyValue);
|
||||
if (modifiedNames.add(modifiedProperty.getName())) {
|
||||
sortedValues.add(modifiedProperty);
|
||||
}
|
||||
}
|
||||
return new MutablePropertyValues(sortedValues);
|
||||
}
|
||||
|
||||
private List<String> getSortedPropertyNames(MutablePropertyValues propertyValues) {
|
||||
List<String> names = new LinkedList<>();
|
||||
for (PropertyValue propertyValue : propertyValues.getPropertyValueList()) {
|
||||
names.add(propertyValue.getName());
|
||||
}
|
||||
sortPropertyNames(names);
|
||||
return names;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort by name so that parent properties get processed first (e.g. 'foo.bar'
|
||||
* before 'foo.bar.spam'). Don't use Collections.sort() because the order might
|
||||
* be significant for other property names (it shouldn't be but who knows what
|
||||
* people might be relying on, e.g. HSQL has a JDBCXADataSource where
|
||||
* "databaseName" is a synonym for "url").
|
||||
*
|
||||
* @param names the names to sort
|
||||
*/
|
||||
private void sortPropertyNames(List<String> names) {
|
||||
for (String name : new ArrayList<>(names)) {
|
||||
int propertyIndex = names.indexOf(name);
|
||||
RelaxedDataBinder.BeanPath path = new RelaxedDataBinder.BeanPath(name);
|
||||
for (String prefix : path.prefixes()) {
|
||||
int prefixIndex = names.indexOf(prefix);
|
||||
if (prefixIndex >= propertyIndex) {
|
||||
// The child property has a parent in the list in the wrong order
|
||||
names.remove(name);
|
||||
names.add(prefixIndex, name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private MutablePropertyValues addMapPrefix(MutablePropertyValues propertyValues) {
|
||||
MutablePropertyValues rtn = new MutablePropertyValues();
|
||||
for (PropertyValue pv : propertyValues.getPropertyValues()) {
|
||||
rtn.add("map." + pv.getName(), pv.getValue());
|
||||
}
|
||||
return rtn;
|
||||
}
|
||||
|
||||
private MutablePropertyValues getPropertyValuesForNamePrefix(MutablePropertyValues propertyValues) {
|
||||
if (!StringUtils.hasText(this.namePrefix) && !this.ignoreNestedProperties) {
|
||||
return propertyValues;
|
||||
}
|
||||
MutablePropertyValues rtn = new MutablePropertyValues();
|
||||
for (PropertyValue value : propertyValues.getPropertyValues()) {
|
||||
String name = value.getName();
|
||||
for (String prefix : new RelaxedNames(stripLastDot(this.namePrefix))) {
|
||||
for (String separator : new String[] { ".", "_" }) {
|
||||
String candidate = (StringUtils.hasLength(prefix) ? prefix + separator : prefix);
|
||||
if (name.startsWith(candidate)) {
|
||||
name = name.substring(candidate.length());
|
||||
if (!(this.ignoreNestedProperties && name.contains("."))) {
|
||||
PropertyOrigin propertyOrigin = OriginCapablePropertyValue.getOrigin(value);
|
||||
rtn.addPropertyValue(
|
||||
new OriginCapablePropertyValue(name, value.getValue(), propertyOrigin));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rtn;
|
||||
}
|
||||
|
||||
private String stripLastDot(String string) {
|
||||
if (StringUtils.hasLength(string) && string.endsWith(".")) {
|
||||
string = string.substring(0, string.length() - 1);
|
||||
}
|
||||
return string;
|
||||
}
|
||||
|
||||
private PropertyValue modifyProperty(BeanWrapper target, PropertyValue propertyValue) {
|
||||
String name = propertyValue.getName();
|
||||
String normalizedName = normalizePath(target, name);
|
||||
if (!normalizedName.equals(name)) {
|
||||
return new PropertyValue(normalizedName, propertyValue.getValue());
|
||||
}
|
||||
return propertyValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a bean property path to a format understood by a BeanWrapper. This
|
||||
* is used so that
|
||||
* <ul>
|
||||
* <li>Fuzzy matching can be employed for bean property names</li>
|
||||
* <li>Period separators can be used instead of indexing ([...]) for map
|
||||
* keys</li>
|
||||
* </ul>
|
||||
*
|
||||
* @param wrapper a bean wrapper for the object to bind
|
||||
* @param path the bean path to bind
|
||||
* @return a transformed path with correct bean wrapper syntax
|
||||
*/
|
||||
protected String normalizePath(BeanWrapper wrapper, String path) {
|
||||
return initializePath(wrapper, new RelaxedDataBinder.BeanPath(path), 0);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected AbstractPropertyBindingResult createBeanPropertyBindingResult() {
|
||||
return new RelaxedDataBinder.RelaxedBeanPropertyBindingResult(getTarget(),
|
||||
getObjectName(),
|
||||
isAutoGrowNestedPaths(),
|
||||
getAutoGrowCollectionLimit(),
|
||||
getConversionService());
|
||||
}
|
||||
|
||||
private String initializePath(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) {
|
||||
String prefix = path.prefix(index);
|
||||
String key = path.name(index);
|
||||
if (path.isProperty(index)) {
|
||||
key = getActualPropertyName(wrapper, prefix, key);
|
||||
path.rename(index, key);
|
||||
}
|
||||
if (path.name(++index) == null) {
|
||||
return path.toString();
|
||||
}
|
||||
String name = path.prefix(index);
|
||||
TypeDescriptor descriptor = wrapper.getPropertyTypeDescriptor(name);
|
||||
if (descriptor == null || descriptor.isMap()) {
|
||||
if (isMapValueStringType(descriptor) || isBlanked(wrapper, name, path.name(index))) {
|
||||
path.collapseKeys(index);
|
||||
}
|
||||
path.mapIndex(index);
|
||||
extendMapIfNecessary(wrapper, path, index);
|
||||
} else if (descriptor.isCollection()) {
|
||||
extendCollectionIfNecessary(wrapper, path, index);
|
||||
} else if (descriptor.getType().equals(Object.class)) {
|
||||
if (isBlanked(wrapper, name, path.name(index))) {
|
||||
path.collapseKeys(index);
|
||||
}
|
||||
path.mapIndex(index);
|
||||
if (path.isLastNode(index)) {
|
||||
wrapper.setPropertyValue(path.toString(), BLANK);
|
||||
} else {
|
||||
String next = path.prefix(index + 1);
|
||||
if (wrapper.getPropertyValue(next) == null) {
|
||||
wrapper.setPropertyValue(next, new LinkedHashMap<String, Object>());
|
||||
}
|
||||
}
|
||||
}
|
||||
return initializePath(wrapper, path, index);
|
||||
}
|
||||
|
||||
private boolean isMapValueStringType(TypeDescriptor descriptor) {
|
||||
if (descriptor == null || descriptor.getMapValueTypeDescriptor() == null) {
|
||||
return false;
|
||||
}
|
||||
if (Properties.class.isAssignableFrom(descriptor.getObjectType())) {
|
||||
// Properties is declared as Map<Object,Object> but we know it's really
|
||||
// Map<String,String>
|
||||
return true;
|
||||
}
|
||||
Class<?> valueType = descriptor.getMapValueTypeDescriptor().getObjectType();
|
||||
return (valueType != null && CharSequence.class.isAssignableFrom(valueType));
|
||||
}
|
||||
|
||||
@SuppressWarnings("rawtypes")
|
||||
private boolean isBlanked(BeanWrapper wrapper, String propertyName, String key) {
|
||||
Object value = (wrapper.isReadableProperty(propertyName) ? wrapper.getPropertyValue(propertyName) : null);
|
||||
if (value instanceof Map) {
|
||||
if (((Map) value).get(key) == BLANK) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void extendCollectionIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) {
|
||||
String name = path.prefix(index);
|
||||
TypeDescriptor elementDescriptor = wrapper.getPropertyTypeDescriptor(name).getElementTypeDescriptor();
|
||||
if (!elementDescriptor.isMap() && !elementDescriptor.isCollection()
|
||||
&& !elementDescriptor.getType().equals(Object.class)) {
|
||||
return;
|
||||
}
|
||||
Object extend = new LinkedHashMap<String, Object>();
|
||||
if (!elementDescriptor.isMap() && path.isArrayIndex(index)) {
|
||||
extend = new ArrayList<>();
|
||||
}
|
||||
wrapper.setPropertyValue(path.prefix(index + 1), extend);
|
||||
}
|
||||
|
||||
private void extendMapIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) {
|
||||
String name = path.prefix(index);
|
||||
TypeDescriptor parent = wrapper.getPropertyTypeDescriptor(name);
|
||||
if (parent == null) {
|
||||
return;
|
||||
}
|
||||
TypeDescriptor descriptor = parent.getMapValueTypeDescriptor();
|
||||
if (descriptor == null) {
|
||||
descriptor = TypeDescriptor.valueOf(Object.class);
|
||||
}
|
||||
if (!descriptor.isMap() && !descriptor.isCollection() && !descriptor.getType().equals(Object.class)) {
|
||||
return;
|
||||
}
|
||||
String extensionName = path.prefix(index + 1);
|
||||
if (wrapper.isReadableProperty(extensionName)) {
|
||||
Object currentValue = wrapper.getPropertyValue(extensionName);
|
||||
if ((descriptor.isCollection() && currentValue instanceof Collection)
|
||||
|| (!descriptor.isCollection() && currentValue instanceof Map)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Object extend = new LinkedHashMap<String, Object>();
|
||||
if (descriptor.isCollection()) {
|
||||
extend = new ArrayList<>();
|
||||
}
|
||||
if (descriptor.getType().equals(Object.class) && path.isLastNode(index)) {
|
||||
extend = BLANK;
|
||||
}
|
||||
wrapper.setPropertyValue(extensionName, extend);
|
||||
}
|
||||
|
||||
private String getActualPropertyName(BeanWrapper target, String prefix, String name) {
|
||||
String propertyName = resolvePropertyName(target, prefix, name);
|
||||
if (propertyName == null) {
|
||||
propertyName = resolveNestedPropertyName(target, prefix, name);
|
||||
}
|
||||
return (propertyName != null ? propertyName : name);
|
||||
}
|
||||
|
||||
private String resolveNestedPropertyName(BeanWrapper target, String prefix, String name) {
|
||||
StringBuilder candidate = new StringBuilder();
|
||||
for (String field : name.split("[_\\-\\.]")) {
|
||||
candidate.append(candidate.length() > 0 ? "." : "");
|
||||
candidate.append(field);
|
||||
String nested = resolvePropertyName(target, prefix, candidate.toString());
|
||||
if (nested != null) {
|
||||
Class<?> type = target.getPropertyType(nested);
|
||||
if ((type != null) && Map.class.isAssignableFrom(type)) {
|
||||
// Special case for map property (gh-3836).
|
||||
return nested + "[" + name.substring(candidate.length() + 1) + "]";
|
||||
}
|
||||
String propertyName = resolvePropertyName(target,
|
||||
joinString(prefix, nested),
|
||||
name.substring(candidate.length() + 1));
|
||||
if (propertyName != null) {
|
||||
return joinString(nested, propertyName);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String resolvePropertyName(BeanWrapper target, String prefix, String name) {
|
||||
Iterable<String> names = getNameAndAliases(name);
|
||||
for (String nameOrAlias : names) {
|
||||
for (String candidate : new RelaxedNames(nameOrAlias)) {
|
||||
try {
|
||||
if (target.getPropertyType(joinString(prefix, candidate)) != null) {
|
||||
return candidate;
|
||||
}
|
||||
} catch (InvalidPropertyException ex) {
|
||||
// swallow and continue
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String joinString(String prefix, String name) {
|
||||
return (StringUtils.hasLength(prefix) ? prefix + "." + name : name);
|
||||
}
|
||||
|
||||
private Iterable<String> getNameAndAliases(String name) {
|
||||
List<String> aliases = this.nameAliases.get(name);
|
||||
if (aliases == null) {
|
||||
return Collections.singleton(name);
|
||||
}
|
||||
List<String> nameAndAliases = new ArrayList<>(aliases.size() + 1);
|
||||
nameAndAliases.add(name);
|
||||
nameAndAliases.addAll(aliases);
|
||||
return nameAndAliases;
|
||||
}
|
||||
|
||||
private static Object wrapTarget(Object target) {
|
||||
if (target instanceof Map) {
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> map = (Map<String, Object>) target;
|
||||
target = new RelaxedDataBinder.MapHolder(map);
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void registerCustomEditor(Class<?> requiredType, PropertyEditor propertyEditor) {
|
||||
if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) {
|
||||
super.registerCustomEditor(requiredType, propertyEditor);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void registerCustomEditor(Class<?> requiredType, String field, PropertyEditor propertyEditor) {
|
||||
if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) {
|
||||
super.registerCustomEditor(requiredType, field, propertyEditor);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Holder to allow Map targets to be bound.
|
||||
*/
|
||||
static class MapHolder {
|
||||
|
||||
private Map<String, Object> map;
|
||||
|
||||
MapHolder(Map<String, Object> map){
|
||||
this.map = map;
|
||||
}
|
||||
|
||||
public void setMap(Map<String, Object> map) {
|
||||
this.map = map;
|
||||
}
|
||||
|
||||
public Map<String, Object> getMap() {
|
||||
return this.map;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* A path though properties of a bean.
|
||||
*/
|
||||
private static class BeanPath {
|
||||
|
||||
private List<PathNode> nodes;
|
||||
|
||||
BeanPath(String path){
|
||||
this.nodes = splitPath(path);
|
||||
}
|
||||
|
||||
public List<String> prefixes() {
|
||||
List<String> prefixes = new ArrayList<>();
|
||||
for (int index = 1; index < this.nodes.size(); index++) {
|
||||
prefixes.add(prefix(index));
|
||||
}
|
||||
return prefixes;
|
||||
}
|
||||
|
||||
public boolean isLastNode(int index) {
|
||||
return index >= this.nodes.size() - 1;
|
||||
}
|
||||
|
||||
private List<PathNode> splitPath(String path) {
|
||||
List<PathNode> nodes = new ArrayList<>();
|
||||
String current = extractIndexedPaths(path, nodes);
|
||||
for (String name : StringUtils.delimitedListToStringArray(current, ".")) {
|
||||
if (StringUtils.hasText(name)) {
|
||||
nodes.add(new RelaxedDataBinder.BeanPath.PropertyNode(name));
|
||||
}
|
||||
}
|
||||
return nodes;
|
||||
}
|
||||
|
||||
private String extractIndexedPaths(String path, List<PathNode> nodes) {
|
||||
int startRef = path.indexOf("[");
|
||||
String current = path;
|
||||
while (startRef >= 0) {
|
||||
if (startRef > 0) {
|
||||
nodes.addAll(splitPath(current.substring(0, startRef)));
|
||||
}
|
||||
int endRef = current.indexOf("]", startRef);
|
||||
if (endRef > 0) {
|
||||
String sub = current.substring(startRef + 1, endRef);
|
||||
if (sub.matches("[0-9]+")) {
|
||||
nodes.add(new RelaxedDataBinder.BeanPath.ArrayIndexNode(sub));
|
||||
} else {
|
||||
nodes.add(new RelaxedDataBinder.BeanPath.MapIndexNode(sub));
|
||||
}
|
||||
}
|
||||
current = current.substring(endRef + 1);
|
||||
startRef = current.indexOf("[");
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
public void collapseKeys(int index) {
|
||||
List<PathNode> revised = new ArrayList<>();
|
||||
for (int i = 0; i < index; i++) {
|
||||
revised.add(this.nodes.get(i));
|
||||
}
|
||||
StringBuilder builder = new StringBuilder();
|
||||
for (int i = index; i < this.nodes.size(); i++) {
|
||||
if (i > index) {
|
||||
builder.append(".");
|
||||
}
|
||||
builder.append(this.nodes.get(i).name);
|
||||
}
|
||||
revised.add(new RelaxedDataBinder.BeanPath.PropertyNode(builder.toString()));
|
||||
this.nodes = revised;
|
||||
}
|
||||
|
||||
public void mapIndex(int index) {
|
||||
RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(index);
|
||||
if (node instanceof RelaxedDataBinder.BeanPath.PropertyNode) {
|
||||
node = ((RelaxedDataBinder.BeanPath.PropertyNode) node).mapIndex();
|
||||
}
|
||||
this.nodes.set(index, node);
|
||||
}
|
||||
|
||||
public String prefix(int index) {
|
||||
return range(0, index);
|
||||
}
|
||||
|
||||
public void rename(int index, String name) {
|
||||
this.nodes.get(index).name = name;
|
||||
}
|
||||
|
||||
public String name(int index) {
|
||||
if (index < this.nodes.size()) {
|
||||
return this.nodes.get(index).name;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String range(int start, int end) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
for (int i = start; i < end; i++) {
|
||||
RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(i);
|
||||
builder.append(node);
|
||||
}
|
||||
if (builder.toString().startsWith(("."))) {
|
||||
builder.replace(0, 1, "");
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public boolean isArrayIndex(int index) {
|
||||
return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.ArrayIndexNode;
|
||||
}
|
||||
|
||||
public boolean isProperty(int index) {
|
||||
return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.PropertyNode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return prefix(this.nodes.size());
|
||||
}
|
||||
|
||||
private static class PathNode {
|
||||
|
||||
protected String name;
|
||||
|
||||
PathNode(String name){
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static class ArrayIndexNode extends RelaxedDataBinder.BeanPath.PathNode {
|
||||
|
||||
ArrayIndexNode(String name){
|
||||
super(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "[" + this.name + "]";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static class MapIndexNode extends RelaxedDataBinder.BeanPath.PathNode {
|
||||
|
||||
MapIndexNode(String name){
|
||||
super(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "[" + this.name + "]";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static class PropertyNode extends RelaxedDataBinder.BeanPath.PathNode {
|
||||
|
||||
PropertyNode(String name){
|
||||
super(name);
|
||||
}
|
||||
|
||||
public RelaxedDataBinder.BeanPath.MapIndexNode mapIndex() {
|
||||
return new RelaxedDataBinder.BeanPath.MapIndexNode(this.name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "." + this.name;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Extended version of {@link BeanPropertyBindingResult} to support relaxed
|
||||
* binding.
|
||||
*/
|
||||
private static class RelaxedBeanPropertyBindingResult extends BeanPropertyBindingResult {
|
||||
|
||||
private RelaxedConversionService conversionService;
|
||||
|
||||
RelaxedBeanPropertyBindingResult(Object target, String objectName, boolean autoGrowNestedPaths,
|
||||
int autoGrowCollectionLimit, ConversionService conversionService){
|
||||
super(target, objectName, autoGrowNestedPaths, autoGrowCollectionLimit);
|
||||
this.conversionService = new RelaxedConversionService(conversionService);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected BeanWrapper createBeanWrapper() {
|
||||
BeanWrapper beanWrapper = new RelaxedDataBinder.RelaxedBeanWrapper(getTarget());
|
||||
beanWrapper.setConversionService(this.conversionService);
|
||||
beanWrapper.registerCustomEditor(InetAddress.class, new InetAddressEditor());
|
||||
return beanWrapper;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Extended version of {@link BeanWrapperImpl} to support relaxed binding.
|
||||
*/
|
||||
private static class RelaxedBeanWrapper extends BeanWrapperImpl {
|
||||
|
||||
private static final Set<String> BENIGN_PROPERTY_SOURCE_NAMES;
|
||||
|
||||
static {
|
||||
Set<String> names = new HashSet<>();
|
||||
names.add(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME);
|
||||
names.add(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME);
|
||||
BENIGN_PROPERTY_SOURCE_NAMES = Collections.unmodifiableSet(names);
|
||||
}
|
||||
|
||||
RelaxedBeanWrapper(Object target){
|
||||
super(target);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setPropertyValue(PropertyValue pv) throws BeansException {
|
||||
try {
|
||||
super.setPropertyValue(pv);
|
||||
} catch (NotWritablePropertyException ex) {
|
||||
PropertyOrigin origin = OriginCapablePropertyValue.getOrigin(pv);
|
||||
if (isBenign(origin)) {
|
||||
logger.debug("Ignoring benign property binding failure", ex);
|
||||
return;
|
||||
}
|
||||
if (origin == null) {
|
||||
throw ex;
|
||||
}
|
||||
throw new RelaxedBindingNotWritablePropertyException(ex, origin);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isBenign(PropertyOrigin origin) {
|
||||
String name = (origin != null ? origin.getSource().getName() : null);
|
||||
return BENIGN_PROPERTY_SOURCE_NAMES.contains(name);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class RelaxedBindingNotWritablePropertyException extends NotWritablePropertyException {
|
||||
|
||||
private final String message;
|
||||
|
||||
private final PropertyOrigin propertyOrigin;
|
||||
|
||||
RelaxedBindingNotWritablePropertyException(NotWritablePropertyException ex, PropertyOrigin propertyOrigin){
|
||||
super(ex.getBeanClass(), ex.getPropertyName());
|
||||
this.propertyOrigin = propertyOrigin;
|
||||
this.message = "Failed to bind '" + propertyOrigin.getName() + "' from '"
|
||||
+ propertyOrigin.getSource().getName() + "' to '" + ex.getPropertyName() + "' property on '"
|
||||
+ ex.getBeanClass().getName() + "'";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getMessage() {
|
||||
return this.message;
|
||||
}
|
||||
|
||||
public PropertyOrigin getPropertyOrigin() {
|
||||
return this.propertyOrigin;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,241 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.bind;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* Generates relaxed name variations from a given source.
|
||||
*
|
||||
* @author Phillip Webb
|
||||
* @author Dave Syer
|
||||
* @see RelaxedDataBinder
|
||||
*/
|
||||
public final class RelaxedNames implements Iterable<String> {
|
||||
|
||||
private static final Pattern CAMEL_CASE_PATTERN = Pattern.compile("([^A-Z-])([A-Z])");
|
||||
|
||||
private static final Pattern SEPARATED_TO_CAMEL_CASE_PATTERN = Pattern.compile("[_\\-.]");
|
||||
|
||||
private final String name;
|
||||
|
||||
private final Set<String> values = new LinkedHashSet<>();
|
||||
|
||||
/**
|
||||
* Create a new {@link RelaxedNames} instance.
|
||||
*
|
||||
* @param name the source name. For the maximum number of variations specify the
|
||||
* name using dashed notation (e.g. {@literal my-property-name}
|
||||
*/
|
||||
public RelaxedNames(String name){
|
||||
this.name = (name != null ? name : "");
|
||||
initialize(RelaxedNames.this.name, this.values);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<String> iterator() {
|
||||
return this.values.iterator();
|
||||
}
|
||||
|
||||
private void initialize(String name, Set<String> values) {
|
||||
if (values.contains(name)) {
|
||||
return;
|
||||
}
|
||||
for (RelaxedNames.Variation variation : RelaxedNames.Variation.values()) {
|
||||
for (RelaxedNames.Manipulation manipulation : RelaxedNames.Manipulation.values()) {
|
||||
String result = name;
|
||||
result = manipulation.apply(result);
|
||||
result = variation.apply(result);
|
||||
values.add(result);
|
||||
initialize(result, values);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Name variations.
|
||||
*/
|
||||
enum Variation {
|
||||
|
||||
NONE {
|
||||
|
||||
@Override
|
||||
public String apply(String value) {
|
||||
return value;
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
LOWERCASE {
|
||||
|
||||
@Override
|
||||
public String apply(String value) {
|
||||
return (value.isEmpty() ? value : value.toLowerCase(Locale.ENGLISH));
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
UPPERCASE {
|
||||
|
||||
@Override
|
||||
public String apply(String value) {
|
||||
return (value.isEmpty() ? value : value.toUpperCase(Locale.ENGLISH));
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
public abstract String apply(String value);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Name manipulations.
|
||||
*/
|
||||
enum Manipulation {
|
||||
|
||||
NONE {
|
||||
|
||||
@Override
|
||||
public String apply(String value) {
|
||||
return value;
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
HYPHEN_TO_UNDERSCORE {
|
||||
|
||||
@Override
|
||||
public String apply(String value) {
|
||||
return (value.indexOf('-') != -1 ? value.replace('-', '_') : value);
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
UNDERSCORE_TO_PERIOD {
|
||||
|
||||
@Override
|
||||
public String apply(String value) {
|
||||
return (value.indexOf('_') != -1 ? value.replace('_', '.') : value);
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
PERIOD_TO_UNDERSCORE {
|
||||
|
||||
@Override
|
||||
public String apply(String value) {
|
||||
return (value.indexOf('.') != -1 ? value.replace('.', '_') : value);
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
CAMELCASE_TO_UNDERSCORE {
|
||||
|
||||
@Override
|
||||
public String apply(String value) {
|
||||
if (value.isEmpty()) {
|
||||
return value;
|
||||
}
|
||||
Matcher matcher = CAMEL_CASE_PATTERN.matcher(value);
|
||||
if (!matcher.find()) {
|
||||
return value;
|
||||
}
|
||||
matcher = matcher.reset();
|
||||
StringBuffer result = new StringBuffer();
|
||||
while (matcher.find()) {
|
||||
matcher.appendReplacement(result,
|
||||
matcher.group(1) + '_' + StringUtils.uncapitalize(matcher.group(2)));
|
||||
}
|
||||
matcher.appendTail(result);
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
CAMELCASE_TO_HYPHEN {
|
||||
|
||||
@Override
|
||||
public String apply(String value) {
|
||||
if (value.isEmpty()) {
|
||||
return value;
|
||||
}
|
||||
Matcher matcher = CAMEL_CASE_PATTERN.matcher(value);
|
||||
if (!matcher.find()) {
|
||||
return value;
|
||||
}
|
||||
matcher = matcher.reset();
|
||||
StringBuffer result = new StringBuffer();
|
||||
while (matcher.find()) {
|
||||
matcher.appendReplacement(result,
|
||||
matcher.group(1) + '-' + StringUtils.uncapitalize(matcher.group(2)));
|
||||
}
|
||||
matcher.appendTail(result);
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
SEPARATED_TO_CAMELCASE {
|
||||
|
||||
@Override
|
||||
public String apply(String value) {
|
||||
return separatedToCamelCase(value, false);
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
CASE_INSENSITIVE_SEPARATED_TO_CAMELCASE {
|
||||
|
||||
@Override
|
||||
public String apply(String value) {
|
||||
return separatedToCamelCase(value, true);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
private static final char[] SUFFIXES = new char[] { '_', '-', '.' };
|
||||
|
||||
public abstract String apply(String value);
|
||||
|
||||
private static String separatedToCamelCase(String value, boolean caseInsensitive) {
|
||||
if (value.isEmpty()) {
|
||||
return value;
|
||||
}
|
||||
StringBuilder builder = new StringBuilder();
|
||||
for (String field : SEPARATED_TO_CAMEL_CASE_PATTERN.split(value)) {
|
||||
field = (caseInsensitive ? field.toLowerCase(Locale.ENGLISH) : field);
|
||||
builder.append(builder.length() != 0 ? StringUtils.capitalize(field) : field);
|
||||
}
|
||||
char lastChar = value.charAt(value.length() - 1);
|
||||
for (char suffix : SUFFIXES) {
|
||||
if (lastChar == suffix) {
|
||||
builder.append(suffix);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a {@link RelaxedNames} for the given source camelCase source name.
|
||||
*
|
||||
* @param name the source name in camelCase
|
||||
* @return the relaxed names
|
||||
*/
|
||||
public static RelaxedNames forCamelCase(String name) {
|
||||
StringBuilder result = new StringBuilder();
|
||||
for (char c : name.toCharArray()) {
|
||||
result.append(Character.isUpperCase(c) && result.length() > 0
|
||||
&& result.charAt(result.length() - 1) != '-' ? "-" + Character.toLowerCase(c) : c);
|
||||
}
|
||||
return new RelaxedNames(result.toString());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.bind;
|
||||
|
||||
import org.springframework.core.convert.converter.Converter;
|
||||
|
||||
/**
|
||||
* Converts a String to a Char Array.
|
||||
*
|
||||
* @author Phillip Webb
|
||||
*/
|
||||
class StringToCharArrayConverter implements Converter<String, char[]> {
|
||||
|
||||
@Override
|
||||
public char[] convert(String source) {
|
||||
return source.toCharArray();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,203 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.ResourceUtils;
|
||||
|
||||
/**
|
||||
* Convenience base class for {@link Resource} implementations, pre-implementing
|
||||
* typical behavior.
|
||||
* <p>
|
||||
* The "exists" method will check whether a File or InputStream can be opened;
|
||||
* "isOpen" will always return false; "getURL" and "getFile" throw an exception;
|
||||
* and "toString" will return the description.
|
||||
*
|
||||
* @author Juergen Hoeller
|
||||
* @since 28.12.2003
|
||||
*/
|
||||
public abstract class AbstractResource implements Resource {
|
||||
|
||||
/**
|
||||
* This implementation checks whether a File can be opened, falling back to
|
||||
* whether an InputStream can be opened. This will cover both directories and
|
||||
* content resources.
|
||||
*/
|
||||
@Override
|
||||
public boolean exists() {
|
||||
// Try file existence: can we find the file in the file system?
|
||||
try {
|
||||
return getFile().exists();
|
||||
} catch (IOException ex) {
|
||||
// Fall back to stream existence: can we open the stream?
|
||||
try {
|
||||
InputStream is = getInputStream();
|
||||
is.close();
|
||||
return true;
|
||||
} catch (Throwable isEx) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation always returns {@code true}.
|
||||
*/
|
||||
@Override
|
||||
public boolean isReadable() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation always returns {@code false}.
|
||||
*/
|
||||
@Override
|
||||
public boolean isOpen() {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation throws a FileNotFoundException, assuming that the
|
||||
* resource cannot be resolved to a URL.
|
||||
*/
|
||||
@Override
|
||||
public URL getURL() throws IOException {
|
||||
throw new FileNotFoundException(getDescription() + " cannot be resolved to URL");
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation builds a URI based on the URL returned by
|
||||
* {@link #getURL()}.
|
||||
*/
|
||||
@Override
|
||||
public URI getURI() throws IOException {
|
||||
URL url = getURL();
|
||||
try {
|
||||
return ResourceUtils.toURI(url);
|
||||
} catch (URISyntaxException ex) {
|
||||
throw new RuntimeException("Invalid URI [" + url + "]", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation throws a FileNotFoundException, assuming that the
|
||||
* resource cannot be resolved to an absolute file path.
|
||||
*/
|
||||
@Override
|
||||
public File getFile() throws IOException {
|
||||
throw new FileNotFoundException(getDescription() + " cannot be resolved to absolute file path");
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation reads the entire InputStream to calculate the content
|
||||
* length. Subclasses will almost always be able to provide a more optimal
|
||||
* version of this, e.g. checking a File length.
|
||||
*
|
||||
* @see #getInputStream()
|
||||
*/
|
||||
@Override
|
||||
public long contentLength() throws IOException {
|
||||
InputStream is = getInputStream();
|
||||
Assert.state(is != null, "Resource InputStream must not be null");
|
||||
try {
|
||||
long size = 0;
|
||||
byte[] buf = new byte[255];
|
||||
int read;
|
||||
while ((read = is.read(buf)) != -1) {
|
||||
size += read;
|
||||
}
|
||||
return size;
|
||||
} finally {
|
||||
try {
|
||||
is.close();
|
||||
} catch (IOException ex) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation checks the timestamp of the underlying File, if
|
||||
* available.
|
||||
*
|
||||
* @see #getFileForLastModifiedCheck()
|
||||
*/
|
||||
@Override
|
||||
public long lastModified() throws IOException {
|
||||
long lastModified = getFileForLastModifiedCheck().lastModified();
|
||||
if (lastModified == 0L) {
|
||||
throw new FileNotFoundException(
|
||||
getDescription() + " cannot be resolved in the file system for resolving its last-modified timestamp");
|
||||
}
|
||||
return lastModified;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the File to use for timestamp checking.
|
||||
* <p>
|
||||
* The default implementation delegates to {@link #getFile()}.
|
||||
*
|
||||
* @return the File to use for timestamp checking (never {@code null})
|
||||
* @throws FileNotFoundException if the resource cannot be resolved as an
|
||||
* absolute file path, i.e. is not available in a file system
|
||||
* @throws IOException in case of general resolution/reading failures
|
||||
*/
|
||||
protected File getFileForLastModifiedCheck() throws IOException {
|
||||
return getFile();
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation throws a FileNotFoundException, assuming that relative
|
||||
* resources cannot be created for this resource.
|
||||
*/
|
||||
@Override
|
||||
public org.springframework.core.io.Resource createRelative(String relativePath) throws IOException {
|
||||
throw new FileNotFoundException("Cannot create a relative resource for " + getDescription());
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation always returns {@code null}, assuming that this resource
|
||||
* type does not have a filename.
|
||||
*/
|
||||
@Override
|
||||
public String getFilename() {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation returns the description of this resource.
|
||||
*
|
||||
* @see #getDescription()
|
||||
*/
|
||||
@Override
|
||||
public String toString() {
|
||||
return getDescription();
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation compares description strings.
|
||||
*
|
||||
* @see #getDescription()
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
return (obj == this
|
||||
|| (obj instanceof org.springframework.core.io.Resource
|
||||
&& ((org.springframework.core.io.Resource) obj).getDescription().equals(getDescription())));
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation returns the description's hash code.
|
||||
*
|
||||
* @see #getDescription()
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return getDescription().hashCode();
|
||||
}
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
/**
|
||||
* {@link Resource} implementation for a given byte array.
|
||||
* <p>
|
||||
* Creates a {@link ByteArrayInputStream} for the given byte array.
|
||||
* <p>
|
||||
* Useful for loading content from any given byte array, without having to
|
||||
* resort to a single-use {@link InputStreamResource}. Particularly useful for
|
||||
* creating mail attachments from local content, where JavaMail needs to be able
|
||||
* to read the stream multiple times.
|
||||
*
|
||||
* @author Juergen Hoeller
|
||||
* @author Sam Brannen
|
||||
* @see ByteArrayInputStream
|
||||
* @see InputStreamResource
|
||||
* @since 1.2.3
|
||||
*/
|
||||
public class ByteArrayResource extends AbstractResource {
|
||||
|
||||
private final byte[] byteArray;
|
||||
|
||||
private final String description;
|
||||
|
||||
/**
|
||||
* Create a new {@code ByteArrayResource}.
|
||||
*
|
||||
* @param byteArray the byte array to wrap
|
||||
*/
|
||||
public ByteArrayResource(byte[] byteArray){
|
||||
this(byteArray, "resource loaded from byte array");
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new {@code ByteArrayResource} with a description.
|
||||
*
|
||||
* @param byteArray the byte array to wrap
|
||||
* @param description where the byte array comes from
|
||||
*/
|
||||
public ByteArrayResource(byte[] byteArray, String description){
|
||||
Assert.notNull(byteArray, "Byte array must not be null");
|
||||
this.byteArray = byteArray;
|
||||
this.description = (description != null ? description : "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the underlying byte array.
|
||||
*/
|
||||
public final byte[] getByteArray() {
|
||||
return this.byteArray;
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation always returns {@code true}.
|
||||
*/
|
||||
@Override
|
||||
public boolean exists() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation returns the length of the underlying byte array.
|
||||
*/
|
||||
@Override
|
||||
public long contentLength() {
|
||||
return this.byteArray.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation returns a ByteArrayInputStream for the underlying byte
|
||||
* array.
|
||||
*
|
||||
* @see ByteArrayInputStream
|
||||
*/
|
||||
@Override
|
||||
public InputStream getInputStream() throws IOException {
|
||||
return new ByteArrayInputStream(this.byteArray);
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation returns a description that includes the passed-in
|
||||
* {@code description}, if any.
|
||||
*/
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Byte array resource [" + this.description + "]";
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation compares the underlying byte array.
|
||||
*
|
||||
* @see Arrays#equals(byte[], byte[])
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
return (obj == this || (obj instanceof org.springframework.core.io.ByteArrayResource
|
||||
&& Arrays.equals(((ByteArrayResource) obj).byteArray, this.byteArray)));
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation returns the hash code based on the underlying byte array.
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return (byte[].class.hashCode() * 29 * this.byteArray.length);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* Composite {@link PropertySource} implementation that iterates over a set of
|
||||
* {@link PropertySource} instances. Necessary in cases where multiple property
|
||||
* sources share the same name, e.g. when multiple values are supplied to
|
||||
* {@code @PropertySource}.
|
||||
* <p>
|
||||
* As of Spring 4.1.2, this class extends {@link EnumerablePropertySource}
|
||||
* instead of plain {@link PropertySource}, exposing {@link #getPropertyNames()}
|
||||
* based on the accumulated property names from all contained sources (as far as
|
||||
* possible).
|
||||
*
|
||||
* @author Chris Beams
|
||||
* @author Juergen Hoeller
|
||||
* @author Phillip Webb
|
||||
* @since 3.1.1
|
||||
*/
|
||||
public class CompositePropertySource extends EnumerablePropertySource<Object> {
|
||||
|
||||
private final Set<PropertySource<?>> propertySources = new LinkedHashSet<>();
|
||||
|
||||
/**
|
||||
* Create a new {@code CompositePropertySource}.
|
||||
*
|
||||
* @param name the name of the property source
|
||||
*/
|
||||
public CompositePropertySource(String name){
|
||||
super(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getProperty(String name) {
|
||||
for (PropertySource<?> propertySource : this.propertySources) {
|
||||
Object candidate = propertySource.getProperty(name);
|
||||
if (candidate != null) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean containsProperty(String name) {
|
||||
for (PropertySource<?> propertySource : this.propertySources) {
|
||||
if (propertySource.containsProperty(name)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String[] getPropertyNames() {
|
||||
Set<String> names = new LinkedHashSet<>();
|
||||
for (PropertySource<?> propertySource : this.propertySources) {
|
||||
if (!(propertySource instanceof EnumerablePropertySource)) {
|
||||
throw new IllegalStateException(
|
||||
"Failed to enumerate property names due to non-enumerable property source: " + propertySource);
|
||||
}
|
||||
names.addAll(Arrays.asList(((EnumerablePropertySource<?>) propertySource).getPropertyNames()));
|
||||
}
|
||||
return StringUtils.toStringArray(names);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the given {@link PropertySource} to the end of the chain.
|
||||
*
|
||||
* @param propertySource the PropertySource to add
|
||||
*/
|
||||
public void addPropertySource(PropertySource<?> propertySource) {
|
||||
this.propertySources.add(propertySource);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the given {@link PropertySource} to the start of the chain.
|
||||
*
|
||||
* @param propertySource the PropertySource to add
|
||||
* @since 4.1
|
||||
*/
|
||||
public void addFirstPropertySource(PropertySource<?> propertySource) {
|
||||
List<PropertySource<?>> existing = new ArrayList<>(this.propertySources);
|
||||
this.propertySources.clear();
|
||||
this.propertySources.add(propertySource);
|
||||
this.propertySources.addAll(existing);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return all property sources that this composite source holds.
|
||||
*
|
||||
* @since 4.1.1
|
||||
*/
|
||||
public Collection<PropertySource<?>> getPropertySources() {
|
||||
return this.propertySources;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String
|
||||
.format("%s [name='%s', propertySources=%s]", getClass().getSimpleName(), this.name, this.propertySources);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
import org.springframework.util.ObjectUtils;
|
||||
|
||||
/**
|
||||
* A {@link PropertySource} implementation capable of interrogating its
|
||||
* underlying source object to enumerate all possible property name/value pairs.
|
||||
* Exposes the {@link #getPropertyNames()} method to allow callers to introspect
|
||||
* available properties without having to access the underlying source object.
|
||||
* This also facilitates a more efficient implementation of
|
||||
* {@link #containsProperty(String)}, in that it can call
|
||||
* {@link #getPropertyNames()} and iterate through the returned array rather
|
||||
* than attempting a call to {@link #getProperty(String)} which may be more
|
||||
* expensive. Implementations may consider caching the result of
|
||||
* {@link #getPropertyNames()} to fully exploit this performance opportunity.
|
||||
* <p>
|
||||
* Most framework-provided {@code PropertySource} implementations are
|
||||
* enumerable; a counter-example would be {@code JndiPropertySource} where, due
|
||||
* to the nature of JNDI it is not possible to determine all possible property
|
||||
* names at any given time; rather it is only possible to try to access a
|
||||
* property (via {@link #getProperty(String)}) in order to evaluate whether it
|
||||
* is present or not.
|
||||
*
|
||||
* @author Chris Beams
|
||||
* @author Juergen Hoeller
|
||||
* @since 3.1
|
||||
*/
|
||||
public abstract class EnumerablePropertySource<T> extends PropertySource<T> {
|
||||
|
||||
public EnumerablePropertySource(String name, T source){
|
||||
super(name, source);
|
||||
}
|
||||
|
||||
protected EnumerablePropertySource(String name){
|
||||
super(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return whether this {@code PropertySource} contains a property with the given
|
||||
* name.
|
||||
* <p>
|
||||
* This implementation checks for the presence of the given name within the
|
||||
* {@link #getPropertyNames()} array.
|
||||
*
|
||||
* @param name the name of the property to find
|
||||
*/
|
||||
@Override
|
||||
public boolean containsProperty(String name) {
|
||||
return ObjectUtils.containsElement(getPropertyNames(), name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the names of all properties contained by the {@linkplain #getSource()
|
||||
* source} object (never {@code null}).
|
||||
*/
|
||||
public abstract String[] getPropertyNames();
|
||||
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.core.env.PropertiesPropertySource;
|
||||
import org.springframework.core.env.PropertySource;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* {@link PropertySource} that reads keys and values from a {@code Map} object.
|
||||
*
|
||||
* @author Chris Beams
|
||||
* @author Juergen Hoeller
|
||||
* @since 3.1
|
||||
* @see PropertiesPropertySource
|
||||
*/
|
||||
public class MapPropertySource extends EnumerablePropertySource<Map<String, Object>> {
|
||||
|
||||
public MapPropertySource(String name, Map<String, Object> source){
|
||||
super(name, source);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getProperty(String name) {
|
||||
return this.source.get(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean containsProperty(String name) {
|
||||
return this.source.containsKey(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String[] getPropertyNames() {
|
||||
return StringUtils.toStringArray(this.source.keySet());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.springframework.core.env.PropertyResolver;
|
||||
import org.springframework.core.env.PropertySourcesPropertyResolver;
|
||||
|
||||
/**
|
||||
* Default implementation of the {@link PropertySources} interface. Allows
|
||||
* manipulation of contained property sources and provides a constructor for
|
||||
* copying an existing {@code PropertySources} instance.
|
||||
* <p>
|
||||
* Where <em>precedence</em> is mentioned in methods such as {@link #addFirst}
|
||||
* and {@link #addLast}, this is with regard to the order in which property
|
||||
* sources will be searched when resolving a given property with a
|
||||
* {@link PropertyResolver}.
|
||||
*
|
||||
* @author Chris Beams
|
||||
* @author Juergen Hoeller
|
||||
* @see PropertySourcesPropertyResolver
|
||||
* @since 3.1
|
||||
*/
|
||||
public class MutablePropertySources implements PropertySources {
|
||||
|
||||
private final Log logger;
|
||||
|
||||
private final List<PropertySource<?>> propertySourceList = new CopyOnWriteArrayList<>();
|
||||
|
||||
/**
|
||||
* Create a new {@link MutablePropertySources}
|
||||
* object.
|
||||
*/
|
||||
public MutablePropertySources(){
|
||||
this.logger = LogFactory.getLog(getClass());
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new {@code MutablePropertySources} from the given propertySources
|
||||
* object, preserving the original order of contained {@code PropertySource}
|
||||
* objects.
|
||||
*/
|
||||
public MutablePropertySources(PropertySources propertySources){
|
||||
this();
|
||||
for (PropertySource<?> propertySource : propertySources) {
|
||||
addLast(propertySource);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new {@link MutablePropertySources}
|
||||
* object and inherit the given logger, usually from an enclosing
|
||||
* {@link Environment}.
|
||||
*/
|
||||
MutablePropertySources(Log logger){
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean contains(String name) {
|
||||
return this.propertySourceList.contains(PropertySource.named(name));
|
||||
}
|
||||
|
||||
@Override
|
||||
public PropertySource<?> get(String name) {
|
||||
int index = this.propertySourceList.indexOf(PropertySource.named(name));
|
||||
return (index != -1 ? this.propertySourceList.get(index) : null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<PropertySource<?>> iterator() {
|
||||
return this.propertySourceList.iterator();
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the given property source object with highest precedence.
|
||||
*/
|
||||
public void addFirst(PropertySource<?> propertySource) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Adding PropertySource '" + propertySource.getName() + "' with highest search precedence");
|
||||
}
|
||||
removeIfPresent(propertySource);
|
||||
this.propertySourceList.add(0, propertySource);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the given property source object with lowest precedence.
|
||||
*/
|
||||
public void addLast(PropertySource<?> propertySource) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Adding PropertySource '" + propertySource.getName() + "' with lowest search precedence");
|
||||
}
|
||||
removeIfPresent(propertySource);
|
||||
this.propertySourceList.add(propertySource);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the given property source object with precedence immediately higher than
|
||||
* the named relative property source.
|
||||
*/
|
||||
public void addBefore(String relativePropertySourceName, PropertySource<?> propertySource) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Adding PropertySource '" + propertySource.getName()
|
||||
+ "' with search precedence immediately higher than '" + relativePropertySourceName + "'");
|
||||
}
|
||||
assertLegalRelativeAddition(relativePropertySourceName, propertySource);
|
||||
removeIfPresent(propertySource);
|
||||
int index = assertPresentAndGetIndex(relativePropertySourceName);
|
||||
addAtIndex(index, propertySource);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the given property source object with precedence immediately lower than
|
||||
* the named relative property source.
|
||||
*/
|
||||
public void addAfter(String relativePropertySourceName, PropertySource<?> propertySource) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Adding PropertySource '" + propertySource.getName()
|
||||
+ "' with search precedence immediately lower than '" + relativePropertySourceName + "'");
|
||||
}
|
||||
assertLegalRelativeAddition(relativePropertySourceName, propertySource);
|
||||
removeIfPresent(propertySource);
|
||||
int index = assertPresentAndGetIndex(relativePropertySourceName);
|
||||
addAtIndex(index + 1, propertySource);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the precedence of the given property source, {@code -1} if not found.
|
||||
*/
|
||||
public int precedenceOf(PropertySource<?> propertySource) {
|
||||
return this.propertySourceList.indexOf(propertySource);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove and return the property source with the given name, {@code null} if
|
||||
* not found.
|
||||
*
|
||||
* @param name the name of the property source to find and remove
|
||||
*/
|
||||
public PropertySource<?> remove(String name) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Removing PropertySource '" + name + "'");
|
||||
}
|
||||
int index = this.propertySourceList.indexOf(PropertySource.named(name));
|
||||
return (index != -1 ? this.propertySourceList.remove(index) : null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the property source with the given name with the given property
|
||||
* source object.
|
||||
*
|
||||
* @param name the name of the property source to find and replace
|
||||
* @param propertySource the replacement property source
|
||||
* @throws IllegalArgumentException if no property source with the given name is
|
||||
* present
|
||||
* @see #contains
|
||||
*/
|
||||
public void replace(String name, PropertySource<?> propertySource) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Replacing PropertySource '" + name + "' with '" + propertySource.getName() + "'");
|
||||
}
|
||||
int index = assertPresentAndGetIndex(name);
|
||||
this.propertySourceList.set(index, propertySource);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the number of {@link PropertySource} objects contained.
|
||||
*/
|
||||
public int size() {
|
||||
return this.propertySourceList.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return this.propertySourceList.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure that the given property source is not being added relative to itself.
|
||||
*/
|
||||
protected void assertLegalRelativeAddition(String relativePropertySourceName, PropertySource<?> propertySource) {
|
||||
String newPropertySourceName = propertySource.getName();
|
||||
if (relativePropertySourceName.equals(newPropertySourceName)) {
|
||||
throw new IllegalArgumentException(
|
||||
"PropertySource named '" + newPropertySourceName + "' cannot be added relative to itself");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove the given property source if it is present.
|
||||
*/
|
||||
protected void removeIfPresent(PropertySource<?> propertySource) {
|
||||
this.propertySourceList.remove(propertySource);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the given property source at a particular index in the list.
|
||||
*/
|
||||
private void addAtIndex(int index, PropertySource<?> propertySource) {
|
||||
removeIfPresent(propertySource);
|
||||
this.propertySourceList.add(index, propertySource);
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert that the named property source is present and return its index.
|
||||
*
|
||||
* @param name {@linkplain PropertySource#getName() name of the property source}
|
||||
* to find
|
||||
* @throws IllegalArgumentException if the named property source is not present
|
||||
*/
|
||||
private int assertPresentAndGetIndex(String name) {
|
||||
int index = this.propertySourceList.indexOf(PropertySource.named(name));
|
||||
if (index == -1) {
|
||||
throw new IllegalArgumentException("PropertySource named '" + name + "' does not exist");
|
||||
}
|
||||
return index;
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.springframework.core.env.PropertySource;
|
||||
|
||||
/**
|
||||
* {@link PropertySource} implementation that extracts properties from a
|
||||
* {@link Properties} object.
|
||||
* <p>
|
||||
* Note that because a {@code Properties} object is technically an
|
||||
* {@code <Object, Object>} {@link java.util.Hashtable Hashtable}, one may
|
||||
* contain non-{@code String} keys or values. This implementation, however is
|
||||
* restricted to accessing only {@code String}-based keys and values, in the
|
||||
* same fashion as {@link Properties#getProperty} and
|
||||
* {@link Properties#setProperty}.
|
||||
*
|
||||
* @author Chris Beams
|
||||
* @author Juergen Hoeller
|
||||
* @since 3.1
|
||||
*/
|
||||
public class PropertiesPropertySource extends MapPropertySource {
|
||||
|
||||
@SuppressWarnings({ "unchecked", "rawtypes" })
|
||||
public PropertiesPropertySource(String name, Properties source){
|
||||
super(name, (Map) source);
|
||||
}
|
||||
|
||||
protected PropertiesPropertySource(String name, Map<String, Object> source){
|
||||
super(name, source);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,239 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.ObjectUtils;
|
||||
|
||||
/**
|
||||
* Abstract base class representing a source of name/value property pairs. The
|
||||
* underlying {@linkplain #getSource() source object} may be of any type
|
||||
* {@code T} that encapsulates properties. Examples include
|
||||
* {@link java.util.Properties} objects, {@link java.util.Map} objects,
|
||||
* {@code ServletContext} and {@code ServletConfig} objects (for access to init
|
||||
* parameters). Explore the {@code PropertySource} type hierarchy to see
|
||||
* provided implementations.
|
||||
* <p>
|
||||
* {@code PropertySource} objects are not typically used in isolation, but
|
||||
* rather through a {@link PropertySources} object, which aggregates property
|
||||
* sources and in conjunction with a {@link PropertyResolver} implementation
|
||||
* that can perform precedence-based searches across the set of
|
||||
* {@code PropertySources}.
|
||||
* <p>
|
||||
* {@code PropertySource} identity is determined not based on the content of
|
||||
* encapsulated properties, but rather based on the {@link #getName() name} of
|
||||
* the {@code PropertySource} alone. This is useful for manipulating
|
||||
* {@code PropertySource} objects when in collection contexts. See operations in
|
||||
* {@link MutablePropertySources} as well as the {@link #named(String)} and
|
||||
* {@link #toString()} methods for details.
|
||||
* <p>
|
||||
* Note that when working
|
||||
* with @{@link org.springframework.context.annotation.Configuration
|
||||
* Configuration} classes that the @{@link PropertySource PropertySource}
|
||||
* annotation provides a convenient and declarative way of adding property
|
||||
* sources to the enclosing {@code Environment}.
|
||||
*
|
||||
* @author Chris Beams
|
||||
* @since 3.1
|
||||
* @see PropertySources
|
||||
* @see MutablePropertySources
|
||||
* @see PropertySource
|
||||
*/
|
||||
public abstract class PropertySource<T> {
|
||||
|
||||
protected final Log logger = LogFactory.getLog(getClass());
|
||||
|
||||
protected final String name;
|
||||
|
||||
protected final T source;
|
||||
|
||||
/**
|
||||
* Create a new {@code PropertySource} with the given name and source object.
|
||||
*/
|
||||
public PropertySource(String name, T source){
|
||||
Assert.hasText(name, "Property source name must contain at least one character");
|
||||
Assert.notNull(source, "Property source must not be null");
|
||||
this.name = name;
|
||||
this.source = source;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new {@code PropertySource} with the given name and with a new
|
||||
* {@code Object} instance as the underlying source.
|
||||
* <p>
|
||||
* Often useful in testing scenarios when creating anonymous implementations
|
||||
* that never query an actual source but rather return hard-coded values.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public PropertySource(String name){
|
||||
this(name, (T) new Object());
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the name of this {@code PropertySource}
|
||||
*/
|
||||
public String getName() {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the underlying source object for this {@code PropertySource}.
|
||||
*/
|
||||
public T getSource() {
|
||||
return this.source;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return whether this {@code PropertySource} contains the given name.
|
||||
* <p>
|
||||
* This implementation simply checks for a {@code null} return value from
|
||||
* {@link #getProperty(String)}. Subclasses may wish to implement a more
|
||||
* efficient algorithm if possible.
|
||||
*
|
||||
* @param name the property name to find
|
||||
*/
|
||||
public boolean containsProperty(String name) {
|
||||
return (getProperty(name) != null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the value associated with the given name, or {@code null} if not
|
||||
* found.
|
||||
*
|
||||
* @param name the property to find
|
||||
*/
|
||||
public abstract Object getProperty(String name);
|
||||
|
||||
/**
|
||||
* This {@code PropertySource} object is equal to the given object if:
|
||||
* <ul>
|
||||
* <li>they are the same instance
|
||||
* <li>the {@code name} properties for both objects are equal
|
||||
* </ul>
|
||||
* <p>
|
||||
* No properties other than {@code name} are evaluated.
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
return (this == obj || (obj instanceof PropertySource
|
||||
&& ObjectUtils.nullSafeEquals(this.name, ((PropertySource<?>) obj).name)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a hash code derived from the {@code name} property of this
|
||||
* {@code PropertySource} object.
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return ObjectUtils.nullSafeHashCode(this.name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Produce concise output (type and name) if the current log level does not
|
||||
* include debug. If debug is enabled, produce verbose output including the hash
|
||||
* code of the PropertySource instance and every name/value property pair.
|
||||
* <p>
|
||||
* This variable verbosity is useful as a property source such as system
|
||||
* properties or environment variables may contain an arbitrary number of
|
||||
* property pairs, potentially leading to difficult to read exception and log
|
||||
* messages.
|
||||
*
|
||||
* @see Log#isDebugEnabled()
|
||||
*/
|
||||
@Override
|
||||
public String toString() {
|
||||
if (logger.isDebugEnabled()) {
|
||||
return getClass().getSimpleName() + "@" + System.identityHashCode(this) + " {name='" + this.name
|
||||
+ "', properties=" + this.source + "}";
|
||||
} else {
|
||||
return getClass().getSimpleName() + " {name='" + this.name + "'}";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a {@code PropertySource} implementation intended for collection
|
||||
* comparison purposes only.
|
||||
* <p>
|
||||
* Primarily for internal use, but given a collection of {@code PropertySource}
|
||||
* objects, may be used as follows:
|
||||
*
|
||||
* <pre class="code">
|
||||
*
|
||||
* {
|
||||
* @code
|
||||
* List<PropertySource<?>> sources = new ArrayList<PropertySource<?>>();
|
||||
* sources.add(new MapPropertySource("sourceA", mapA));
|
||||
* sources.add(new MapPropertySource("sourceB", mapB));
|
||||
* assert sources.contains(PropertySource.named("sourceA"));
|
||||
* assert sources.contains(PropertySource.named("sourceB"));
|
||||
* assert !sources.contains(PropertySource.named("sourceC"));
|
||||
* }
|
||||
* </pre>
|
||||
*
|
||||
* The returned {@code PropertySource} will throw
|
||||
* {@code UnsupportedOperationException} if any methods other than
|
||||
* {@code equals(Object)}, {@code hashCode()}, and {@code toString()} are
|
||||
* called.
|
||||
*
|
||||
* @param name the name of the comparison {@code PropertySource} to be created
|
||||
* and returned.
|
||||
*/
|
||||
public static PropertySource<?> named(String name) {
|
||||
return new ComparisonPropertySource(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@code PropertySource} to be used as a placeholder in cases where an actual
|
||||
* property source cannot be eagerly initialized at application context creation
|
||||
* time. For example, a {@code ServletContext}-based property source must wait
|
||||
* until the {@code ServletContext} object is available to its enclosing
|
||||
* {@code ApplicationContext}. In such cases, a stub should be used to hold the
|
||||
* intended default position/order of the property source, then be replaced
|
||||
* during context refresh.
|
||||
*
|
||||
* @see org.springframework.web.context.support.StandardServletEnvironment
|
||||
* @see org.springframework.web.context.support.ServletContextPropertySource
|
||||
*/
|
||||
public static class StubPropertySource extends PropertySource<Object> {
|
||||
|
||||
public StubPropertySource(String name){
|
||||
super(name, new Object());
|
||||
}
|
||||
|
||||
/**
|
||||
* Always returns {@code null}.
|
||||
*/
|
||||
@Override
|
||||
public String getProperty(String name) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @see PropertySource#named(String)
|
||||
*/
|
||||
static class ComparisonPropertySource extends StubPropertySource {
|
||||
|
||||
private static final String USAGE_ERROR = "ComparisonPropertySource instances are for use with collection comparison only";
|
||||
|
||||
public ComparisonPropertySource(String name){
|
||||
super(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getSource() {
|
||||
throw new UnsupportedOperationException(USAGE_ERROR);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean containsProperty(String name) {
|
||||
throw new UnsupportedOperationException(USAGE_ERROR);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getProperty(String name) {
|
||||
throw new UnsupportedOperationException(USAGE_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.springframework.core.io.support.SpringFactoriesLoader;
|
||||
|
||||
/**
|
||||
* Strategy interface located via {@link SpringFactoriesLoader} and used to load
|
||||
* a {@link PropertySource}.
|
||||
*
|
||||
* @author Dave Syer
|
||||
* @author Phillip Webb
|
||||
*/
|
||||
public interface PropertySourceLoader {
|
||||
|
||||
/**
|
||||
* Returns the file extensions that the loader supports (excluding the '.').
|
||||
*
|
||||
* @return the file extensions
|
||||
*/
|
||||
String[] getFileExtensions();
|
||||
|
||||
/**
|
||||
* Load the resource into a property source.
|
||||
*
|
||||
* @param name the name of the property source
|
||||
* @param resource the resource to load
|
||||
* @param profile the name of the profile to load or {@code null}. The profile
|
||||
* can be used to load multi-document files (such as YAML). Simple property
|
||||
* formats should {@code null} when asked to load a profile.
|
||||
* @return a property source or {@code null}
|
||||
* @throws IOException if the source cannot be loaded
|
||||
*/
|
||||
PropertySource<?> load(String name, Resource resource, String profile) throws IOException;
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
/**
|
||||
* Holder containing one or more {@link PropertySource} objects.
|
||||
*
|
||||
* @author Chris Beams
|
||||
* @since 3.1
|
||||
*/
|
||||
public interface PropertySources extends Iterable<PropertySource<?>> {
|
||||
|
||||
/**
|
||||
* Return whether a property source with the given name is contained.
|
||||
*
|
||||
* @param name the {@linkplain PropertySource#getName() name of the property source} to find
|
||||
*/
|
||||
boolean contains(String name);
|
||||
|
||||
/**
|
||||
* Return the property source with the given name, {@code null} if not found.
|
||||
*
|
||||
* @param name the {@linkplain PropertySource#getName() name of the property source} to find
|
||||
*/
|
||||
PropertySource<?> get(String name);
|
||||
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URL;
|
||||
|
||||
import org.springframework.core.io.*;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
|
||||
/**
|
||||
* Interface for a resource descriptor that abstracts from the actual type of
|
||||
* underlying resource, such as a file or class path resource.
|
||||
* <p>
|
||||
* An InputStream can be opened for every resource if it exists in physical
|
||||
* form, but a URL or File handle can just be returned for certain resources.
|
||||
* The actual behavior is implementation-specific.
|
||||
*
|
||||
* @author Juergen Hoeller
|
||||
* @since 28.12.2003
|
||||
* @see #getInputStream()
|
||||
* @see #getURL()
|
||||
* @see #getURI()
|
||||
* @see #getFile()
|
||||
* @see WritableResource
|
||||
* @see ContextResource
|
||||
* @see UrlResource
|
||||
* @see ClassPathResource
|
||||
* @see FileSystemResource
|
||||
* @see PathResource
|
||||
* @see ByteArrayResource
|
||||
* @see InputStreamResource
|
||||
*/
|
||||
public interface Resource extends InputStreamSource {
|
||||
|
||||
boolean exists();
|
||||
|
||||
boolean isReadable();
|
||||
|
||||
boolean isOpen();
|
||||
|
||||
URL getURL() throws IOException;
|
||||
|
||||
URI getURI() throws IOException;
|
||||
|
||||
File getFile() throws IOException;
|
||||
|
||||
long contentLength() throws IOException;
|
||||
|
||||
long lastModified() throws IOException;
|
||||
|
||||
org.springframework.core.io.Resource createRelative(String var1) throws IOException;
|
||||
|
||||
String getFilename();
|
||||
|
||||
String getDescription();
|
||||
}
|
||||
@@ -1,182 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* {@link YamlProcessor.DocumentMatcher} backed by
|
||||
* {@link Environment#getActiveProfiles()}. A YAML document may define a
|
||||
* "spring.profiles" element as a comma-separated list of Spring profile names,
|
||||
* optionally negated using the {@code !} character. If both negated and
|
||||
* non-negated profiles are specified for a single document, at least one
|
||||
* non-negated profile must match and no negated profiles may match.
|
||||
*
|
||||
* @author Dave Syer
|
||||
* @author Matt Benson
|
||||
* @author Phillip Webb
|
||||
* @author Andy Wilkinson
|
||||
*/
|
||||
public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatcher {
|
||||
|
||||
private String[] activeProfiles = new String[0];
|
||||
|
||||
public SpringProfileDocumentMatcher(){
|
||||
}
|
||||
|
||||
public SpringProfileDocumentMatcher(String... profiles){
|
||||
addActiveProfiles(profiles);
|
||||
}
|
||||
|
||||
public void addActiveProfiles(String... profiles) {
|
||||
LinkedHashSet<String> set = new LinkedHashSet<>(Arrays.asList(this.activeProfiles));
|
||||
Collections.addAll(set, profiles);
|
||||
this.activeProfiles = set.toArray(new String[set.size()]);
|
||||
}
|
||||
|
||||
@Override
|
||||
public YamlProcessor.MatchStatus matches(Properties properties) {
|
||||
List<String> profiles = extractSpringProfiles(properties);
|
||||
ProfilesMatcher profilesMatcher = getProfilesMatcher();
|
||||
Set<String> negative = extractProfiles(profiles, ProfileType.NEGATIVE);
|
||||
Set<String> positive = extractProfiles(profiles, ProfileType.POSITIVE);
|
||||
if (!CollectionUtils.isEmpty(negative)) {
|
||||
if (profilesMatcher.matches(negative) == YamlProcessor.MatchStatus.FOUND) {
|
||||
return YamlProcessor.MatchStatus.NOT_FOUND;
|
||||
}
|
||||
if (CollectionUtils.isEmpty(positive)) {
|
||||
return YamlProcessor.MatchStatus.FOUND;
|
||||
}
|
||||
}
|
||||
return profilesMatcher.matches(positive);
|
||||
}
|
||||
|
||||
private List<String> extractSpringProfiles(Properties properties) {
|
||||
SpringProperties springProperties = new SpringProperties();
|
||||
MutablePropertySources propertySources = new MutablePropertySources();
|
||||
propertySources.addFirst(new PropertiesPropertySource("profiles", properties));
|
||||
// PropertyValues propertyValues = new PropertySourcesPropertyValues(
|
||||
// propertySources);
|
||||
// new RelaxedDataBinder(springProperties, "spring").bind(propertyValues);
|
||||
// TODO
|
||||
List<String> profiles = springProperties.getProfiles();
|
||||
return profiles;
|
||||
}
|
||||
|
||||
private ProfilesMatcher getProfilesMatcher() {
|
||||
return (this.activeProfiles.length != 0 ? new ActiveProfilesMatcher(
|
||||
new HashSet<>(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher());
|
||||
}
|
||||
|
||||
private Set<String> extractProfiles(List<String> profiles, ProfileType type) {
|
||||
if (CollectionUtils.isEmpty(profiles)) {
|
||||
return null;
|
||||
}
|
||||
Set<String> extractedProfiles = new HashSet<>();
|
||||
for (String candidate : profiles) {
|
||||
ProfileType candidateType = ProfileType.POSITIVE;
|
||||
if (candidate.startsWith("!")) {
|
||||
candidateType = ProfileType.NEGATIVE;
|
||||
}
|
||||
if (candidateType == type) {
|
||||
extractedProfiles.add(type != ProfileType.POSITIVE ? candidate.substring(1) : candidate);
|
||||
}
|
||||
}
|
||||
return extractedProfiles;
|
||||
}
|
||||
|
||||
/**
|
||||
* Profile match types.
|
||||
*/
|
||||
enum ProfileType {
|
||||
|
||||
POSITIVE, NEGATIVE
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Base class for profile matchers.
|
||||
*/
|
||||
private abstract static class ProfilesMatcher {
|
||||
|
||||
public final YamlProcessor.MatchStatus matches(Set<String> profiles) {
|
||||
if (CollectionUtils.isEmpty(profiles)) {
|
||||
return YamlProcessor.MatchStatus.ABSTAIN;
|
||||
}
|
||||
return doMatches(profiles);
|
||||
}
|
||||
|
||||
protected abstract YamlProcessor.MatchStatus doMatches(Set<String> profiles);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* {@link ProfilesMatcher} that matches when a value in {@code spring.profiles}
|
||||
* is also in {@code spring.profiles.active}.
|
||||
*/
|
||||
private static class ActiveProfilesMatcher extends ProfilesMatcher {
|
||||
|
||||
private final Set<String> activeProfiles;
|
||||
|
||||
ActiveProfilesMatcher(Set<String> activeProfiles){
|
||||
this.activeProfiles = activeProfiles;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected YamlProcessor.MatchStatus doMatches(Set<String> profiles) {
|
||||
if (profiles.isEmpty()) {
|
||||
return YamlProcessor.MatchStatus.NOT_FOUND;
|
||||
}
|
||||
for (String activeProfile : this.activeProfiles) {
|
||||
if (profiles.contains(activeProfile)) {
|
||||
return YamlProcessor.MatchStatus.FOUND;
|
||||
}
|
||||
}
|
||||
return YamlProcessor.MatchStatus.NOT_FOUND;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* {@link ProfilesMatcher} that matches when {@code
|
||||
* spring.profiles} is empty or contains a value with no text.
|
||||
*
|
||||
* @see StringUtils#hasText(String)
|
||||
*/
|
||||
private static class EmptyProfilesMatcher extends ProfilesMatcher {
|
||||
|
||||
@Override
|
||||
public YamlProcessor.MatchStatus doMatches(Set<String> springProfiles) {
|
||||
if (springProfiles.isEmpty()) {
|
||||
return YamlProcessor.MatchStatus.FOUND;
|
||||
}
|
||||
for (String profile : springProfiles) {
|
||||
if (!StringUtils.hasText(profile)) {
|
||||
return YamlProcessor.MatchStatus.FOUND;
|
||||
}
|
||||
}
|
||||
return YamlProcessor.MatchStatus.NOT_FOUND;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Class for binding {@code spring.profiles} property.
|
||||
*/
|
||||
static class SpringProperties {
|
||||
|
||||
private List<String> profiles = new ArrayList<>();
|
||||
|
||||
public List<String> getProfiles() {
|
||||
return this.profiles;
|
||||
}
|
||||
|
||||
public void setProfiles(List<String> profiles) {
|
||||
this.profiles = profiles;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,417 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.springframework.util.Assert;
|
||||
import org.yaml.snakeyaml.Yaml;
|
||||
import org.yaml.snakeyaml.constructor.Constructor;
|
||||
import org.yaml.snakeyaml.nodes.MappingNode;
|
||||
import org.yaml.snakeyaml.parser.ParserException;
|
||||
import org.yaml.snakeyaml.reader.UnicodeReader;
|
||||
|
||||
/**
|
||||
* Base class for YAML factories.
|
||||
*
|
||||
* @author Dave Syer
|
||||
* @author Juergen Hoeller
|
||||
* @since 4.1
|
||||
*/
|
||||
public abstract class YamlProcessor {
|
||||
|
||||
private final Log logger = LogFactory.getLog(getClass());
|
||||
|
||||
private ResolutionMethod resolutionMethod = ResolutionMethod.OVERRIDE;
|
||||
|
||||
private Resource[] resources = new Resource[0];
|
||||
|
||||
private List<DocumentMatcher> documentMatchers = Collections.emptyList();
|
||||
|
||||
private boolean matchDefault = true;
|
||||
|
||||
/**
|
||||
* A map of document matchers allowing callers to selectively use only some of
|
||||
* the documents in a YAML resource. In YAML documents are separated by
|
||||
* <code>---<code> lines, and each document is converted to properties before
|
||||
* the match is made. E.g.
|
||||
*
|
||||
* <pre class="code">
|
||||
* environment: dev
|
||||
* url: http://dev.bar.com
|
||||
* name: Developer Setup
|
||||
* ---
|
||||
* environment: prod
|
||||
* url:http://foo.bar.com
|
||||
* name: My Cool App
|
||||
* </pre>
|
||||
*
|
||||
* when mapped with
|
||||
*
|
||||
* <pre class="code">
|
||||
* setDocumentMatchers(properties -> ("prod"
|
||||
* .equals(properties.getProperty("environment")) ? MatchStatus.FOUND : MatchStatus.NOT_FOUND));
|
||||
* </pre>
|
||||
*
|
||||
* would end up as
|
||||
*
|
||||
* <pre class="code">
|
||||
* environment=prod
|
||||
* url=http://foo.bar.com
|
||||
* name=My Cool App
|
||||
* </pre>
|
||||
*/
|
||||
public void setDocumentMatchers(DocumentMatcher... matchers) {
|
||||
this.documentMatchers = Arrays.asList(matchers);
|
||||
}
|
||||
|
||||
/**
|
||||
* Flag indicating that a document for which all the
|
||||
* {@link #setDocumentMatchers(DocumentMatcher...) document matchers} abstain
|
||||
* will nevertheless match. Default is {@code true}.
|
||||
*/
|
||||
public void setMatchDefault(boolean matchDefault) {
|
||||
this.matchDefault = matchDefault;
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to use for resolving resources. Each resource will be converted to a
|
||||
* Map, so this property is used to decide which map entries to keep in the
|
||||
* final output from this factory. Default is {@link ResolutionMethod#OVERRIDE}.
|
||||
*/
|
||||
public void setResolutionMethod(ResolutionMethod resolutionMethod) {
|
||||
Assert.notNull(resolutionMethod, "ResolutionMethod must not be null");
|
||||
this.resolutionMethod = resolutionMethod;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set locations of YAML {@link Resource resources} to be loaded.
|
||||
*
|
||||
* @see ResolutionMethod
|
||||
*/
|
||||
public void setResources(Resource... resources) {
|
||||
this.resources = resources;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provide an opportunity for subclasses to process the Yaml parsed from the
|
||||
* supplied resources. Each resource is parsed in turn and the documents inside
|
||||
* checked against the {@link #setDocumentMatchers(DocumentMatcher...)
|
||||
* matchers}. If a document matches it is passed into the callback, along with
|
||||
* its representation as Properties. Depending on the
|
||||
* {@link #setResolutionMethod(ResolutionMethod)} not all of the documents will
|
||||
* be parsed.
|
||||
*
|
||||
* @param callback a callback to delegate to once matching documents are found
|
||||
* @see #createYaml()
|
||||
*/
|
||||
protected void process(MatchCallback callback) {
|
||||
Yaml yaml = createYaml();
|
||||
for (Resource resource : this.resources) {
|
||||
boolean found = process(callback, yaml, resource);
|
||||
if (this.resolutionMethod == ResolutionMethod.FIRST_FOUND && found) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the {@link Yaml} instance to use.
|
||||
*/
|
||||
protected Yaml createYaml() {
|
||||
return new Yaml(new StrictMapAppenderConstructor());
|
||||
}
|
||||
|
||||
private boolean process(MatchCallback callback, Yaml yaml, Resource resource) {
|
||||
int count = 0;
|
||||
try {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Loading from YAML: " + resource);
|
||||
}
|
||||
try (Reader reader = new UnicodeReader(resource.getInputStream())) {
|
||||
for (Object object : yaml.loadAll(reader)) {
|
||||
if (object != null && process(asMap(object), callback)) {
|
||||
count++;
|
||||
if (this.resolutionMethod == ResolutionMethod.FIRST_FOUND) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug(
|
||||
"Loaded " + count + " document" + (count > 1 ? "s" : "") + " from YAML resource: " + resource);
|
||||
}
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
handleProcessError(resource, ex);
|
||||
}
|
||||
return (count > 0);
|
||||
}
|
||||
|
||||
private void handleProcessError(Resource resource, IOException ex) {
|
||||
if (this.resolutionMethod != ResolutionMethod.FIRST_FOUND
|
||||
&& this.resolutionMethod != ResolutionMethod.OVERRIDE_AND_IGNORE) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
if (logger.isWarnEnabled()) {
|
||||
logger.warn("Could not load map from " + resource + ": " + ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private Map<String, Object> asMap(Object object) {
|
||||
// YAML can have numbers as keys
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
if (!(object instanceof Map)) {
|
||||
// A document can be a text literal
|
||||
result.put("document", object);
|
||||
return result;
|
||||
}
|
||||
|
||||
Map<Object, Object> map = (Map<Object, Object>) object;
|
||||
for (Map.Entry<Object, Object> entry : map.entrySet()) {
|
||||
Object value = entry.getValue();
|
||||
if (value instanceof Map) {
|
||||
value = asMap(value);
|
||||
}
|
||||
Object key = entry.getKey();
|
||||
if (key instanceof CharSequence) {
|
||||
result.put(key.toString(), value);
|
||||
} else {
|
||||
// It has to be a map key in this case
|
||||
result.put("[" + key.toString() + "]", value);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean process(Map<String, Object> map, MatchCallback callback) {
|
||||
@SuppressWarnings("serial")
|
||||
Properties properties = new Properties() {
|
||||
|
||||
@Override
|
||||
public String getProperty(String key) {
|
||||
Object value = get(key);
|
||||
return (value != null ? value.toString() : null);
|
||||
}
|
||||
};
|
||||
properties.putAll(getFlattenedMap(map));
|
||||
|
||||
if (this.documentMatchers.isEmpty()) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Merging document (no matchers set): " + map);
|
||||
}
|
||||
callback.process(properties, map);
|
||||
return true;
|
||||
}
|
||||
|
||||
MatchStatus result = MatchStatus.ABSTAIN;
|
||||
for (DocumentMatcher matcher : this.documentMatchers) {
|
||||
MatchStatus match = matcher.matches(properties);
|
||||
result = MatchStatus.getMostSpecific(match, result);
|
||||
if (match == MatchStatus.FOUND) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Matched document with document matcher: " + properties);
|
||||
}
|
||||
callback.process(properties, map);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (result == MatchStatus.ABSTAIN && this.matchDefault) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Matched document with default matcher: " + map);
|
||||
}
|
||||
callback.process(properties, map);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Unmatched document: " + map);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a flattened version of the given map, recursively following any nested
|
||||
* Map or Collection values. Entries from the resulting map retain the same
|
||||
* order as the source. When called with the Map from a {@link MatchCallback}
|
||||
* the result will contain the same values as the {@link MatchCallback}
|
||||
* Properties.
|
||||
*
|
||||
* @param source the source map
|
||||
* @return a flattened map
|
||||
* @since 4.1.3
|
||||
*/
|
||||
protected final Map<String, Object> getFlattenedMap(Map<String, Object> source) {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
buildFlattenedMap(result, source, null);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static boolean containsText(CharSequence str) {
|
||||
int strLen = str.length();
|
||||
for (int i = 0; i < strLen; i++) {
|
||||
if (!Character.isWhitespace(str.charAt(i))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void buildFlattenedMap(Map<String, Object> result, Map<String, Object> source, String path) {
|
||||
for (Map.Entry<String, Object> entry : source.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
if (path != null && !path.isEmpty() && containsText(path)) {
|
||||
if (key.startsWith("[")) {
|
||||
key = path + key;
|
||||
} else {
|
||||
key = path + '.' + key;
|
||||
}
|
||||
}
|
||||
Object value = entry.getValue();
|
||||
if (value instanceof String) {
|
||||
result.put(key, value);
|
||||
} else if (value instanceof Map) {
|
||||
// Need a compound key
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> map = (Map<String, Object>) value;
|
||||
buildFlattenedMap(result, map, key);
|
||||
} else if (value instanceof Collection) {
|
||||
// Need a compound key
|
||||
@SuppressWarnings("unchecked")
|
||||
Collection<Object> collection = (Collection<Object>) value;
|
||||
int count = 0;
|
||||
for (Object object : collection) {
|
||||
buildFlattenedMap(result, Collections.singletonMap("[" + (count++) + "]", object), key);
|
||||
}
|
||||
} else {
|
||||
result.put(key, (value != null ? value : ""));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Callback interface used to process the YAML parsing results.
|
||||
*/
|
||||
public interface MatchCallback {
|
||||
|
||||
/**
|
||||
* Process the given representation of the parsing results.
|
||||
*
|
||||
* @param properties the properties to process (as a flattened representation
|
||||
* with indexed keys in case of a collection or map)
|
||||
* @param map the result map (preserving the original value structure in the
|
||||
* YAML document)
|
||||
*/
|
||||
void process(Properties properties, Map<String, Object> map);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strategy interface used to test if properties match.
|
||||
*/
|
||||
public interface DocumentMatcher {
|
||||
|
||||
/**
|
||||
* Test if the given properties match.
|
||||
*
|
||||
* @param properties the properties to test
|
||||
* @return the status of the match
|
||||
*/
|
||||
MatchStatus matches(Properties properties);
|
||||
}
|
||||
|
||||
/**
|
||||
* Status returned from {@link DocumentMatcher#matches(Properties)}
|
||||
*/
|
||||
public enum MatchStatus {
|
||||
|
||||
/**
|
||||
* A match was found.
|
||||
*/
|
||||
FOUND,
|
||||
|
||||
/**
|
||||
* No match was found.
|
||||
*/
|
||||
NOT_FOUND,
|
||||
|
||||
/**
|
||||
* The matcher should not be considered.
|
||||
*/
|
||||
ABSTAIN;
|
||||
|
||||
/**
|
||||
* Compare two {@link MatchStatus} items, returning the most specific status.
|
||||
*/
|
||||
public static MatchStatus getMostSpecific(MatchStatus a, MatchStatus b) {
|
||||
return (a.ordinal() < b.ordinal() ? a : b);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to use for resolving resources.
|
||||
*/
|
||||
public enum ResolutionMethod {
|
||||
|
||||
/**
|
||||
* Replace values from earlier in the list.
|
||||
*/
|
||||
OVERRIDE,
|
||||
|
||||
/**
|
||||
* Replace values from earlier in the list, ignoring any failures.
|
||||
*/
|
||||
OVERRIDE_AND_IGNORE,
|
||||
|
||||
/**
|
||||
* Take the first resource in the list that exists and use just that.
|
||||
*/
|
||||
FIRST_FOUND
|
||||
}
|
||||
|
||||
/**
|
||||
* A specialized {@link Constructor} that checks for duplicate keys.
|
||||
*/
|
||||
protected static class StrictMapAppenderConstructor extends Constructor {
|
||||
|
||||
// Declared as public for use in subclasses
|
||||
public StrictMapAppenderConstructor(){
|
||||
super();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<Object, Object> constructMapping(MappingNode node) {
|
||||
try {
|
||||
return super.constructMapping(node);
|
||||
} catch (IllegalStateException ex) {
|
||||
throw new ParserException("while parsing MappingNode",
|
||||
node.getStartMark(),
|
||||
ex.getMessage(),
|
||||
node.getEndMark());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<Object, Object> createDefaultMap() {
|
||||
final Map<Object, Object> delegate = super.createDefaultMap();
|
||||
return new AbstractMap<Object, Object>() {
|
||||
|
||||
@Override
|
||||
public Object put(Object key, Object value) {
|
||||
if (delegate.containsKey(key)) {
|
||||
throw new IllegalStateException("Duplicate key: " + key);
|
||||
}
|
||||
return delegate.put(key, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Entry<Object, Object>> entrySet() {
|
||||
return delegate.entrySet();
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.config.common;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.util.ClassUtils;
|
||||
import org.yaml.snakeyaml.DumperOptions;
|
||||
import org.yaml.snakeyaml.Yaml;
|
||||
import org.yaml.snakeyaml.nodes.Tag;
|
||||
import org.yaml.snakeyaml.representer.Representer;
|
||||
import org.yaml.snakeyaml.resolver.Resolver;
|
||||
|
||||
/**
|
||||
* Strategy to load '.yml' (or '.yaml') files into a {@link PropertySource}.
|
||||
*
|
||||
* @author Dave Syer
|
||||
* @author Phillip Webb
|
||||
* @author Andy Wilkinson
|
||||
*/
|
||||
public class YamlPropertySourceLoader implements PropertySourceLoader {
|
||||
|
||||
@Override
|
||||
public String[] getFileExtensions() {
|
||||
return new String[] { "yml", "yaml" };
|
||||
}
|
||||
|
||||
@Override
|
||||
public PropertySource<?> load(String name, Resource resource, String profile) throws IOException {
|
||||
if (ClassUtils.isPresent("org.yaml.snakeyaml.Yaml", null)) {
|
||||
Processor processor = new Processor(resource, profile);
|
||||
Map<String, Object> source = processor.process();
|
||||
if (!source.isEmpty()) {
|
||||
return new MapPropertySource(name, source);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@link YamlProcessor} to create a {@link Map} containing the property values.
|
||||
* Similar to {@link YamlPropertiesFactoryBean} but retains the order of
|
||||
* entries.
|
||||
*/
|
||||
private static class Processor extends YamlProcessor {
|
||||
|
||||
Processor(Resource resource, String profile){
|
||||
if (profile == null) {
|
||||
setMatchDefault(true);
|
||||
setDocumentMatchers(new SpringProfileDocumentMatcher());
|
||||
} else {
|
||||
setMatchDefault(false);
|
||||
setDocumentMatchers(new SpringProfileDocumentMatcher(profile));
|
||||
}
|
||||
setResources(resource);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Yaml createYaml() {
|
||||
return new Yaml(new StrictMapAppenderConstructor(), new Representer(), new DumperOptions(), new Resolver() {
|
||||
|
||||
@Override
|
||||
public void addImplicitResolver(Tag tag, Pattern regexp, String first) {
|
||||
if (tag == Tag.TIMESTAMP) {
|
||||
return;
|
||||
}
|
||||
super.addImplicitResolver(tag, regexp, first);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public Map<String, Object> process() {
|
||||
final Map<String, Object> result = new LinkedHashMap<>();
|
||||
process((properties, map) -> result.putAll(getFlattenedMap(map)));
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package com.alibaba.otter.canal.client.adapter.support;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.springframework.boot.context.properties.bind.Bindable;
|
||||
import org.springframework.boot.context.properties.bind.Binder;
|
||||
import org.springframework.boot.context.properties.source.ConfigurationPropertySource;
|
||||
import org.springframework.boot.context.properties.source.MapConfigurationPropertySource;
|
||||
import org.springframework.boot.env.YamlPropertySourceLoader;
|
||||
import org.springframework.boot.origin.OriginTrackedValue;
|
||||
import org.springframework.core.env.PropertySource;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
|
||||
public class YamlUtils {
|
||||
|
||||
public static <T> T resourceYmlToObj(String resource, String prefix, Class<T> clazz) {
|
||||
ClassPathResource classPathResource = new ClassPathResource(resource);
|
||||
|
||||
String content;
|
||||
try (InputStream inputStream = classPathResource.getInputStream()) {
|
||||
|
||||
ByteArrayOutputStream result = new ByteArrayOutputStream();
|
||||
byte[] buffer = new byte[1024];
|
||||
int length;
|
||||
while ((length = inputStream.read(buffer)) != -1) {
|
||||
result.write(buffer, 0, length);
|
||||
}
|
||||
content = result.toString("UTF-8");
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return ymlToObj(prefix, content, clazz);
|
||||
}
|
||||
|
||||
public static <T> T ymlToObj(String content, Class<T> clazz) {
|
||||
return ymlToObj("", content, clazz, null, null);
|
||||
}
|
||||
|
||||
public static <T> T ymlToObj(String prefix, String content, Class<T> clazz) {
|
||||
return ymlToObj(prefix, content, clazz, null, null);
|
||||
}
|
||||
|
||||
public static <T> T ymlToObj(String prefix, String content, Class<T> clazz, String charset) {
|
||||
return ymlToObj(prefix, content, clazz, charset, null);
|
||||
}
|
||||
|
||||
public static <T> T ymlToObj(String prefix, String content, Class<T> clazz, String charset,
|
||||
Properties baseProperties) {
|
||||
try {
|
||||
prefix = StringUtils.trimToEmpty(prefix);
|
||||
byte[] contentBytes;
|
||||
if (charset == null) {
|
||||
contentBytes = content.getBytes(StandardCharsets.UTF_8);
|
||||
} else {
|
||||
contentBytes = content.getBytes(charset);
|
||||
}
|
||||
YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader();
|
||||
Resource configResource = new ByteArrayResource(contentBytes);
|
||||
List<PropertySource<?>> propertySources = propertySourceLoader.load("manualBindConfig", configResource);
|
||||
|
||||
if (propertySources == null || propertySources.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
PropertySource<?> propertySource = propertySources.get(0);
|
||||
|
||||
Properties properties = new Properties();
|
||||
|
||||
properties.putAll((Map<?, ?>) propertySource.getSource());
|
||||
|
||||
for (Map.Entry<Object, Object> entry : properties.entrySet()) {
|
||||
Object value = entry.getValue();
|
||||
if (value instanceof OriginTrackedValue) {
|
||||
entry.setValue(((OriginTrackedValue) value).getValue());
|
||||
}
|
||||
}
|
||||
|
||||
if (baseProperties != null) {
|
||||
baseProperties.putAll(properties);
|
||||
properties = baseProperties;
|
||||
}
|
||||
|
||||
ConfigurationPropertySource sources = new MapConfigurationPropertySource(properties);
|
||||
Binder binder = new Binder(sources);
|
||||
return binder.bind(prefix, Bindable.of(clazz)).get();
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
package com.alibaba.otter.canal.client.adapter.support;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class YamlUtilsTest {
|
||||
|
||||
@Test
|
||||
public void testLoadConfigToYml() {
|
||||
String configStr="dataSourceKey: defaultDS\n"
|
||||
+ "destination: example\n"
|
||||
+ "groupId: g1\n"
|
||||
+ "outerAdapterKey: mysql1\n"
|
||||
+ "concurrent: true\n"
|
||||
+ "dbMapping:\n"
|
||||
+ " database: mytest\n"
|
||||
+ " table: user\n"
|
||||
+ " targetTable: mytest2.user\n"
|
||||
+ " targetPk:\n"
|
||||
+ " id: id\n"
|
||||
+ "# mapAll: true\n"
|
||||
+ " targetColumns:\n"
|
||||
+ " id:\n"
|
||||
+ " name:\n"
|
||||
+ " role_id:\n"
|
||||
+ " c_time:\n"
|
||||
+ " test1:\n"
|
||||
+ " etlCondition: \"where c_time>={}\"\n"
|
||||
+ " commitBatch: 3000 # 批量提交的大小";
|
||||
|
||||
MappingConfig config = YamlUtils.ymlToObj(null, configStr, MappingConfig.class, null, new Properties());
|
||||
|
||||
Assert.assertNotNull(config);
|
||||
Assert.assertEquals(config.getDestination(), "example");
|
||||
Assert.assertEquals(config.getOuterAdapterKey(), "mysql1");
|
||||
Assert.assertEquals(config.getDbMapping().getDatabase(), "mytest");
|
||||
Assert.assertEquals(config.getDbMapping().getTargetColumns().size(), 5);
|
||||
}
|
||||
|
||||
private static class MappingConfig {
|
||||
private String dataSourceKey;
|
||||
|
||||
private String destination;
|
||||
|
||||
private String groupId;
|
||||
|
||||
private String outerAdapterKey;
|
||||
|
||||
private boolean concurrent = false;
|
||||
|
||||
private DbMapping dbMapping;
|
||||
|
||||
public String getDataSourceKey() {
|
||||
return dataSourceKey;
|
||||
}
|
||||
|
||||
public void setDataSourceKey(String dataSourceKey) {
|
||||
this.dataSourceKey = dataSourceKey;
|
||||
}
|
||||
|
||||
public String getDestination() {
|
||||
return destination;
|
||||
}
|
||||
|
||||
public void setDestination(String destination) {
|
||||
this.destination = destination;
|
||||
}
|
||||
|
||||
public String getGroupId() {
|
||||
return groupId;
|
||||
}
|
||||
|
||||
public void setGroupId(String groupId) {
|
||||
this.groupId = groupId;
|
||||
}
|
||||
|
||||
public String getOuterAdapterKey() {
|
||||
return outerAdapterKey;
|
||||
}
|
||||
|
||||
public void setOuterAdapterKey(String outerAdapterKey) {
|
||||
this.outerAdapterKey = outerAdapterKey;
|
||||
}
|
||||
|
||||
public boolean isConcurrent() {
|
||||
return concurrent;
|
||||
}
|
||||
|
||||
public void setConcurrent(boolean concurrent) {
|
||||
this.concurrent = concurrent;
|
||||
}
|
||||
|
||||
public DbMapping getDbMapping() {
|
||||
return dbMapping;
|
||||
}
|
||||
|
||||
public void setDbMapping(DbMapping dbMapping) {
|
||||
this.dbMapping = dbMapping;
|
||||
}
|
||||
}
|
||||
|
||||
private static class DbMapping {
|
||||
private boolean mirrorDb = false; // 是否镜像库
|
||||
private String database; // 数据库名或schema名
|
||||
private String table; // 表名
|
||||
private Map<String, String> targetPk = new LinkedHashMap<>(); // 目标表主键字段
|
||||
private boolean mapAll = false; // 映射所有字段
|
||||
private String targetDb; // 目标库名
|
||||
private String targetTable; // 目标表名
|
||||
private Map<String, String> targetColumns; // 目标表字段映射
|
||||
|
||||
private boolean caseInsensitive = false; // 目标表不区分大小写，默认是否
|
||||
|
||||
private String etlCondition; // etl条件sql
|
||||
|
||||
private int readBatch = 5000;
|
||||
private int commitBatch = 5000; // etl等批量提交大小
|
||||
|
||||
private Map<String, String> allMapColumns;
|
||||
|
||||
public boolean isMirrorDb() {
|
||||
return mirrorDb;
|
||||
}
|
||||
|
||||
public void setMirrorDb(boolean mirrorDb) {
|
||||
this.mirrorDb = mirrorDb;
|
||||
}
|
||||
|
||||
public String getDatabase() {
|
||||
return database;
|
||||
}
|
||||
|
||||
public void setDatabase(String database) {
|
||||
this.database = database;
|
||||
}
|
||||
|
||||
public String getTable() {
|
||||
return table;
|
||||
}
|
||||
|
||||
public void setTable(String table) {
|
||||
this.table = table;
|
||||
}
|
||||
|
||||
public Map<String, String> getTargetPk() {
|
||||
return targetPk;
|
||||
}
|
||||
|
||||
public void setTargetPk(Map<String, String> targetPk) {
|
||||
this.targetPk = targetPk;
|
||||
}
|
||||
|
||||
public boolean isMapAll() {
|
||||
return mapAll;
|
||||
}
|
||||
|
||||
public void setMapAll(boolean mapAll) {
|
||||
this.mapAll = mapAll;
|
||||
}
|
||||
|
||||
public String getTargetDb() {
|
||||
return targetDb;
|
||||
}
|
||||
|
||||
public void setTargetDb(String targetDb) {
|
||||
this.targetDb = targetDb;
|
||||
}
|
||||
|
||||
public String getTargetTable() {
|
||||
return targetTable;
|
||||
}
|
||||
|
||||
public void setTargetTable(String targetTable) {
|
||||
this.targetTable = targetTable;
|
||||
}
|
||||
|
||||
public Map<String, String> getTargetColumns() {
|
||||
return targetColumns;
|
||||
}
|
||||
|
||||
public void setTargetColumns(Map<String, String> targetColumns) {
|
||||
this.targetColumns = targetColumns;
|
||||
}
|
||||
|
||||
public boolean isCaseInsensitive() {
|
||||
return caseInsensitive;
|
||||
}
|
||||
|
||||
public void setCaseInsensitive(boolean caseInsensitive) {
|
||||
this.caseInsensitive = caseInsensitive;
|
||||
}
|
||||
|
||||
public String getEtlCondition() {
|
||||
return etlCondition;
|
||||
}
|
||||
|
||||
public void setEtlCondition(String etlCondition) {
|
||||
this.etlCondition = etlCondition;
|
||||
}
|
||||
|
||||
public int getReadBatch() {
|
||||
return readBatch;
|
||||
}
|
||||
|
||||
public void setReadBatch(int readBatch) {
|
||||
this.readBatch = readBatch;
|
||||
}
|
||||
|
||||
public int getCommitBatch() {
|
||||
return commitBatch;
|
||||
}
|
||||
|
||||
public void setCommitBatch(int commitBatch) {
|
||||
this.commitBatch = commitBatch;
|
||||
}
|
||||
|
||||
public Map<String, String> getAllMapColumns() {
|
||||
return allMapColumns;
|
||||
}
|
||||
|
||||
public void setAllMapColumns(Map<String, String> allMapColumns) {
|
||||
this.allMapColumns = allMapColumns;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.core.config;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
@@ -7,7 +8,6 @@ import java.util.Properties;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
|
||||
/**
|
||||
@@ -28,7 +28,7 @@ public class ESSyncConfigLoader {
|
||||
String esv = envProperties.getProperty("es.version");
|
||||
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs(esv);
|
||||
configContentMap.forEach((fileName, content) -> {
|
||||
ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null, content, ESSyncConfig.class, null, envProperties);
|
||||
ESSyncConfig config = YamlUtils.ymlToObj(null, content, ESSyncConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,12 +1,8 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.core.monitor;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||
import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||
import java.io.File;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.commons.io.filefilter.FileFilterUtils;
|
||||
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
|
||||
import org.apache.commons.io.monitor.FileAlterationMonitor;
|
||||
@@ -14,6 +10,14 @@ import org.apache.commons.io.monitor.FileAlterationObserver;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||
import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
|
||||
|
||||
;
|
||||
|
||||
public class ESConfigMonitor {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ESConfigMonitor.class);
|
||||
@@ -60,7 +64,7 @@ public class ESConfigMonitor {
|
||||
try {
|
||||
// 加载新增的配置文件
|
||||
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
|
||||
ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
|
||||
ESSyncConfig config = YamlUtils.ymlToObj(null,
|
||||
configContent,
|
||||
ESSyncConfig.class,
|
||||
null,
|
||||
@@ -93,7 +97,7 @@ public class ESConfigMonitor {
|
||||
onFileDelete(file);
|
||||
return;
|
||||
}
|
||||
ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
|
||||
ESSyncConfig config = YamlUtils.ymlToObj(null,
|
||||
configContent,
|
||||
ESSyncConfig.class,
|
||||
null,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.alibaba.otter.canal.client.adapter.hbase.config;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
@@ -7,7 +8,6 @@ import java.util.Properties;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
|
||||
/**
|
||||
@@ -32,8 +32,7 @@ public class MappingConfigLoader {
|
||||
|
||||
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("hbase");
|
||||
configContentMap.forEach((fileName, content) -> {
|
||||
MappingConfig config = YmlConfigBinder
|
||||
.bindYmlToObj(null, content, MappingConfig.class, null, envProperties);
|
||||
MappingConfig config = YamlUtils.ymlToObj(null, content, MappingConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package com.alibaba.otter.canal.client.adapter.hbase.monitor;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||
import com.alibaba.otter.canal.client.adapter.hbase.HbaseAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||
import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
|
||||
import java.io.File;
|
||||
import java.util.Properties;
|
||||
import org.apache.commons.io.filefilter.FileFilterUtils;
|
||||
@@ -59,8 +59,7 @@ public class HbaseConfigMonitor {
|
||||
try {
|
||||
// 加载新增的配置文件
|
||||
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
|
||||
MappingConfig config = YmlConfigBinder
|
||||
.bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties);
|
||||
MappingConfig config = YamlUtils.ymlToObj(null, configContent, MappingConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
@@ -88,8 +87,7 @@ public class HbaseConfigMonitor {
|
||||
onFileDelete(file);
|
||||
return;
|
||||
}
|
||||
MappingConfig config = YmlConfigBinder
|
||||
.bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties);
|
||||
MappingConfig config = YamlUtils.ymlToObj(null, configContent, MappingConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.alibaba.otter.canal.client.adapter.kudu.config;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
@@ -7,7 +8,6 @@ import java.util.Properties;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
|
||||
/**
|
||||
@@ -30,11 +30,7 @@ public class KuduMappingConfigLoader {
|
||||
|
||||
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("kudu");
|
||||
configContentMap.forEach((fileName, content) -> {
|
||||
KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
|
||||
content,
|
||||
KuduMappingConfig.class,
|
||||
null,
|
||||
envProperties);
|
||||
KuduMappingConfig config = YamlUtils.ymlToObj(null, content, KuduMappingConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package com.alibaba.otter.canal.client.adapter.kudu.monitor;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||
import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||
import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
|
||||
import java.io.File;
|
||||
import java.util.Properties;
|
||||
import org.apache.commons.io.filefilter.FileFilterUtils;
|
||||
@@ -36,7 +36,7 @@ public class KuduConfigMonitor {
|
||||
File confDir = Util.getConfDirPath(adapterName);
|
||||
try {
|
||||
FileAlterationObserver observer = new FileAlterationObserver(confDir,
|
||||
FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
|
||||
FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
|
||||
FileListener listener = new FileListener();
|
||||
observer.addListener(listener);
|
||||
fileMonitor = new FileAlterationMonitor(3000, observer);
|
||||
@@ -69,19 +69,15 @@ public class KuduConfigMonitor {
|
||||
try {
|
||||
// 加载新增的配置文件
|
||||
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
|
||||
KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
|
||||
configContent,
|
||||
KuduMappingConfig.class,
|
||||
null,
|
||||
envProperties);
|
||||
KuduMappingConfig config = YamlUtils
|
||||
.ymlToObj(null, configContent, KuduMappingConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
config.validate();
|
||||
boolean result = kuduAdapter.addConfig(file.getName(), config);
|
||||
if (result) {
|
||||
logger.info("Add a new kudu mapping config: {} to canal adapter",
|
||||
file.getName());
|
||||
logger.info("Add a new kudu mapping config: {} to canal adapter", file.getName());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
@@ -95,17 +91,14 @@ public class KuduConfigMonitor {
|
||||
try {
|
||||
if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
|
||||
// 加载配置文件
|
||||
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator
|
||||
+ file.getName());
|
||||
String configContent = MappingConfigsLoader
|
||||
.loadConfig(adapterName + File.separator + file.getName());
|
||||
if (configContent == null) {
|
||||
onFileDelete(file);
|
||||
return;
|
||||
}
|
||||
KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
|
||||
configContent,
|
||||
KuduMappingConfig.class,
|
||||
null,
|
||||
envProperties);
|
||||
KuduMappingConfig config = YamlUtils
|
||||
.ymlToObj(null, configContent, KuduMappingConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -69,9 +69,12 @@ else
|
||||
fi
|
||||
|
||||
if [ -n "$str" ]; then
|
||||
# JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
|
||||
# For G1
|
||||
JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
|
||||
if [ $JavaVersion -ge 11 ] ; then
|
||||
# For G1
|
||||
JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
|
||||
else
|
||||
JAVA_OPTS="-server -Xms2g -Xmx3g -Xmn1g -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
|
||||
fi
|
||||
else
|
||||
JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS"
|
||||
fi
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.alibaba.otter.canal.client.adapter.phoenix.config;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
@@ -28,8 +28,7 @@ public class ConfigLoader {
|
||||
|
||||
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("phoenix");
|
||||
configContentMap.forEach((fileName, content) -> {
|
||||
MappingConfig config = YmlConfigBinder
|
||||
.bindYmlToObj(null, content, MappingConfig.class, null, envProperties);
|
||||
MappingConfig config = YamlUtils.ymlToObj(null, content, MappingConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package com.alibaba.otter.canal.client.adapter.phoenix.monitor;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||
import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||
import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
|
||||
import java.io.File;
|
||||
import java.util.Properties;
|
||||
import org.apache.commons.io.filefilter.FileFilterUtils;
|
||||
@@ -19,15 +19,15 @@ import org.slf4j.LoggerFactory;
|
||||
*/
|
||||
public class PhoenixConfigMonitor {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PhoenixConfigMonitor.class);
|
||||
private static final Logger logger = LoggerFactory.getLogger(PhoenixConfigMonitor.class);
|
||||
|
||||
private static final String adapterName = "phoenix"; //相应组件名字
|
||||
private static final String adapterName = "phoenix"; // 相应组件名字
|
||||
|
||||
private String key;
|
||||
private String key;
|
||||
|
||||
private PhoenixAdapter phoenixAdapter; //相应适配器名实现类
|
||||
private PhoenixAdapter phoenixAdapter; // 相应适配器名实现类
|
||||
|
||||
private Properties envProperties;
|
||||
private Properties envProperties;
|
||||
|
||||
private FileAlterationMonitor fileMonitor;
|
||||
|
||||
@@ -38,7 +38,7 @@ public class PhoenixConfigMonitor {
|
||||
File confDir = Util.getConfDirPath(adapterName);
|
||||
try {
|
||||
FileAlterationObserver observer = new FileAlterationObserver(confDir,
|
||||
FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
|
||||
FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
|
||||
FileListener listener = new FileListener();
|
||||
observer.addListener(listener);
|
||||
fileMonitor = new FileAlterationMonitor(3000, observer);
|
||||
@@ -65,16 +65,15 @@ public class PhoenixConfigMonitor {
|
||||
try {
|
||||
// 加载新增的配置文件
|
||||
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
|
||||
MappingConfig config = YmlConfigBinder
|
||||
.bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties);
|
||||
MappingConfig config = YamlUtils
|
||||
.ymlToObj(null, configContent, MappingConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
config.validate();
|
||||
boolean result = phoenixAdapter.addConfig(file.getName(), config);
|
||||
if (result) {
|
||||
logger.info("Add a new phoenix mapping config: {} to canal adapter",
|
||||
file.getName());
|
||||
logger.info("Add a new phoenix mapping config: {} to canal adapter", file.getName());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
@@ -89,13 +88,13 @@ public class PhoenixConfigMonitor {
|
||||
if (phoenixAdapter.getPhoenixMapping().containsKey(file.getName())) {
|
||||
// 加载配置文件
|
||||
String configContent = MappingConfigsLoader
|
||||
.loadConfig(adapterName + File.separator + file.getName());
|
||||
.loadConfig(adapterName + File.separator + file.getName());
|
||||
if (configContent == null) {
|
||||
onFileDelete(file);
|
||||
return;
|
||||
}
|
||||
MappingConfig config = YmlConfigBinder
|
||||
.bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties);
|
||||
MappingConfig config = YamlUtils
|
||||
.ymlToObj(null, configContent, MappingConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -53,7 +53,7 @@
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>central</id>
|
||||
<url>http://repo1.maven.org/maven2</url>
|
||||
<url>https://repo1.maven.org/maven2</url>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
@@ -63,7 +63,7 @@
|
||||
</repository>
|
||||
<repository>
|
||||
<id>java.net</id>
|
||||
<url>http://download.java.net/maven/2/</url>
|
||||
<url>https://download.java.net/maven/2/</url>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
@@ -73,7 +73,7 @@
|
||||
</repository>
|
||||
<repository>
|
||||
<id>aliyun</id>
|
||||
<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
|
||||
<url>https://maven.aliyun.com/nexus/content/groups/public/</url>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
@@ -113,35 +113,9 @@
|
||||
<version>2.0.0.RELEASE</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-core</artifactId>
|
||||
<version>5.0.5.RELEASE</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-aop</artifactId>
|
||||
<version>5.0.5.RELEASE</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-context</artifactId>
|
||||
<version>5.0.5.RELEASE</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-jdbc</artifactId>
|
||||
<version>5.0.5.RELEASE</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-orm</artifactId>
|
||||
<version>5.0.5.RELEASE</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-test</artifactId>
|
||||
<version>5.0.5.RELEASE</version>
|
||||
<scope>test</scope>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot</artifactId>
|
||||
<version>2.0.1.RELEASE</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.alibaba.otter.canal.client.adapter.rdb.config;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
@@ -7,7 +8,6 @@ import java.util.Properties;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
|
||||
/**
|
||||
@@ -32,8 +32,7 @@ public class ConfigLoader {
|
||||
|
||||
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("rdb");
|
||||
configContentMap.forEach((fileName, content) -> {
|
||||
MappingConfig config = YmlConfigBinder
|
||||
.bindYmlToObj(null, content, MappingConfig.class, null, envProperties);
|
||||
MappingConfig config = YamlUtils.ymlToObj(null, content, MappingConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package com.alibaba.otter.canal.client.adapter.rdb.monitor;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||
import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||
import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
|
||||
import java.io.File;
|
||||
import java.util.Properties;
|
||||
import org.apache.commons.io.filefilter.FileFilterUtils;
|
||||
@@ -35,7 +35,7 @@ public class RdbConfigMonitor {
|
||||
File confDir = Util.getConfDirPath(adapterName);
|
||||
try {
|
||||
FileAlterationObserver observer = new FileAlterationObserver(confDir,
|
||||
FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
|
||||
FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
|
||||
FileListener listener = new FileListener();
|
||||
observer.addListener(listener);
|
||||
fileMonitor = new FileAlterationMonitor(3000, observer);
|
||||
@@ -62,16 +62,15 @@ public class RdbConfigMonitor {
|
||||
try {
|
||||
// 加载新增的配置文件
|
||||
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
|
||||
MappingConfig config = YmlConfigBinder
|
||||
.bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties);
|
||||
MappingConfig config = YamlUtils
|
||||
.ymlToObj(null, configContent, MappingConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
config.validate();
|
||||
boolean result = rdbAdapter.addConfig(file.getName(), config);
|
||||
if (result) {
|
||||
logger.info("Add a new rdb mapping config: {} to canal adapter",
|
||||
file.getName());
|
||||
logger.info("Add a new rdb mapping config: {} to canal adapter", file.getName());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
@@ -85,14 +84,14 @@ public class RdbConfigMonitor {
|
||||
try {
|
||||
if (rdbAdapter.getRdbMapping().containsKey(file.getName())) {
|
||||
// 加载配置文件
|
||||
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
|
||||
String configContent = MappingConfigsLoader
|
||||
.loadConfig(adapterName + File.separator + file.getName());
|
||||
if (configContent == null) {
|
||||
onFileDelete(file);
|
||||
return;
|
||||
}
|
||||
MappingConfig config = YmlConfigBinder
|
||||
.bindYmlToObj(null, configContent, MappingConfig.class, null,
|
||||
envProperties);
|
||||
MappingConfig config = YamlUtils
|
||||
.ymlToObj(null, configContent, MappingConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -58,6 +58,7 @@ public class RdbEtlService extends AbstractEtlService {
|
||||
Map<String, String> columnsMap = new LinkedHashMap<>();
|
||||
Map<String, Integer> columnType = new LinkedHashMap<>();
|
||||
DruidDataSource dataSource = (DruidDataSource) srcDS;
|
||||
String backtick = SyncUtil.getBacktickByDbType(dataSource.getDbType());
|
||||
|
||||
Util.sqlRS(targetDS,
|
||||
"SELECT * FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()) + " LIMIT 1 ",
|
||||
@@ -91,7 +92,7 @@ public class RdbEtlService extends AbstractEtlService {
|
||||
.append(SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()))
|
||||
.append(" (");
|
||||
columnsMap
|
||||
.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(","));
|
||||
.forEach((targetColumnName, srcColumnName) -> insertSql.append(backtick).append(targetColumnName).append(backtick).append(","));
|
||||
|
||||
int len = insertSql.length();
|
||||
insertSql.delete(len - 1, len).append(") VALUES (");
|
||||
@@ -101,6 +102,7 @@ public class RdbEtlService extends AbstractEtlService {
|
||||
}
|
||||
len = insertSql.length();
|
||||
insertSql.delete(len - 1, len).append(")");
|
||||
logger.info("executeSqlImport sql:{}",insertSql.toString());
|
||||
try (Connection connTarget = targetDS.getConnection();
|
||||
PreparedStatement pstmt = connTarget.prepareStatement(insertSql.toString())) {
|
||||
connTarget.setAutoCommit(false);
|
||||
@@ -114,7 +116,7 @@ public class RdbEtlService extends AbstractEtlService {
|
||||
Map<String, Object> pkVal = new LinkedHashMap<>();
|
||||
StringBuilder deleteSql = new StringBuilder(
|
||||
"DELETE FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()) + " WHERE ");
|
||||
appendCondition(dbMapping, deleteSql, pkVal, rs);
|
||||
appendCondition(dbMapping, deleteSql, pkVal, rs, backtick);
|
||||
try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) {
|
||||
int k = 1;
|
||||
for (Object val : pkVal.values()) {
|
||||
@@ -132,7 +134,6 @@ public class RdbEtlService extends AbstractEtlService {
|
||||
}
|
||||
|
||||
Integer type = columnType.get(targetClolumnName.toLowerCase());
|
||||
|
||||
Object value = rs.getObject(srcColumnName);
|
||||
if (value != null) {
|
||||
SyncUtil.setPStmt(type, pstmt, value, i);
|
||||
@@ -180,7 +181,7 @@ public class RdbEtlService extends AbstractEtlService {
|
||||
* 拼接目标表主键where条件
|
||||
*/
|
||||
private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values,
|
||||
ResultSet rs) throws SQLException {
|
||||
ResultSet rs, String backtick) throws SQLException {
|
||||
// 拼接主键
|
||||
for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
|
||||
String targetColumnName = entry.getKey();
|
||||
@@ -188,7 +189,7 @@ public class RdbEtlService extends AbstractEtlService {
|
||||
if (srcColumnName == null) {
|
||||
srcColumnName = targetColumnName;
|
||||
}
|
||||
sql.append(targetColumnName).append("=? AND ");
|
||||
sql.append(backtick).append(targetColumnName).append(backtick).append("=? AND ");
|
||||
values.put(targetColumnName, rs.getObject(srcColumnName));
|
||||
}
|
||||
int len = sql.length();
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.alibaba.otter.canal.client.adapter.tablestore.config;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.util.StringUtils;
|
||||
@@ -32,8 +32,7 @@ public class ConfigLoader {
|
||||
|
||||
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("tablestore");
|
||||
configContentMap.forEach((fileName, content) -> {
|
||||
MappingConfig config = YmlConfigBinder
|
||||
.bindYmlToObj(null, content, MappingConfig.class, null, envProperties);
|
||||
MappingConfig config = YamlUtils.ymlToObj(null, content, MappingConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
@@ -45,7 +44,8 @@ public class ConfigLoader {
|
||||
result.put(fileName, config);
|
||||
});
|
||||
|
||||
logger.info("## Tablestore mapping config loaded:" + StringUtils.collectionToCommaDelimitedString(result.keySet()));
|
||||
logger.info(
|
||||
"## Tablestore mapping config loaded:" + StringUtils.collectionToCommaDelimitedString(result.keySet()));
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
|
||||
private ServerRunningData serverData;
|
||||
// 当前实际运行的节点状态信息
|
||||
private volatile ServerRunningData activeData;
|
||||
private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1);
|
||||
private ScheduledExecutorService delayExecutor;
|
||||
private int delayTime = 5;
|
||||
private ServerRunningListener listener;
|
||||
|
||||
@@ -73,7 +73,7 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
|
||||
initRunning();
|
||||
} else {
|
||||
// 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作
|
||||
delayExector.schedule(() -> initRunning(), delayTime, TimeUnit.SECONDS);
|
||||
delayExecutor.schedule(() -> initRunning(), delayTime, TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,6 +90,7 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
|
||||
try {
|
||||
processStart();
|
||||
if (zkClient != null) {
|
||||
delayExecutor = Executors.newScheduledThreadPool(1);
|
||||
// 如果需要尽可能释放instance资源，不需要监听running节点，不然即使stop了这台机器，另一台机器立马会start
|
||||
String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
|
||||
zkClient.subscribeDataChanges(path, dataListener);
|
||||
@@ -122,6 +123,10 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
|
||||
if (zkClient != null) {
|
||||
String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
|
||||
zkClient.unsubscribeDataChanges(path, dataListener);
|
||||
if (delayExecutor != null) {
|
||||
delayExecutor.shutdown();
|
||||
delayExecutor = null;
|
||||
}
|
||||
|
||||
releaseRunning(); // 尝试一下release
|
||||
} else {
|
||||
|
||||
@@ -107,11 +107,11 @@ public class AviaterRegexFilter {
|
||||
private List<String> completionPattern(List<String> patterns) {
|
||||
List<String> result = new ArrayList<>();
|
||||
for (String pattern : patterns) {
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
stringBuffer.append("^");
|
||||
stringBuffer.append(pattern);
|
||||
stringBuffer.append("$");
|
||||
result.add(stringBuffer.toString());
|
||||
StringBuilder stringBuilder = new StringBuilder();
|
||||
stringBuilder.append("^");
|
||||
stringBuilder.append(pattern);
|
||||
stringBuilder.append("$");
|
||||
result.add(stringBuilder.toString());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>central</id>
|
||||
<url>http://repo1.maven.org/maven2</url>
|
||||
<url>https://repo1.maven.org/maven2</url>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
@@ -49,7 +49,7 @@
|
||||
</repository>
|
||||
<repository>
|
||||
<id>java.net</id>
|
||||
<url>http://download.java.net/maven/2/</url>
|
||||
<url>https://download.java.net/maven/2/</url>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
@@ -59,7 +59,7 @@
|
||||
</repository>
|
||||
<repository>
|
||||
<id>aliyun</id>
|
||||
<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
|
||||
<url>https://maven.aliyun.com/nexus/content/groups/public/</url>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
|
||||
@@ -21,6 +21,7 @@ public final class CharsetConversion {
|
||||
protected final String mysqlCharset;
|
||||
protected final String mysqlCollation;
|
||||
protected final String javaCharset;
|
||||
protected final Charset charset;
|
||||
|
||||
Entry(final int id, String mysqlCharset, // NL
|
||||
String mysqlCollation, String javaCharset){
|
||||
@@ -28,6 +29,7 @@ public final class CharsetConversion {
|
||||
this.mysqlCharset = mysqlCharset;
|
||||
this.mysqlCollation = mysqlCollation;
|
||||
this.javaCharset = javaCharset;
|
||||
this.charset = Charset.isSupported(javaCharset) ? Charset.forName(javaCharset) : null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -390,6 +392,23 @@ public final class CharsetConversion {
|
||||
}
|
||||
}
|
||||
|
||||
public static Charset getNioCharset(final int id) {
|
||||
Entry entry = getEntry(id);
|
||||
|
||||
if (entry != null) {
|
||||
if (entry.charset != null) {
|
||||
return entry.charset;
|
||||
} else {
|
||||
logger.warn("Unknown java charset for: id = " + id + ", name = " + entry.mysqlCharset + ", coll = "
|
||||
+ entry.mysqlCollation);
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
logger.warn("Unexpect mysql charset: " + id);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
for (int i = 0; i < entries.length; i++) {
|
||||
Entry entry = entries[i];
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package com.taobao.tddl.dbsync.binlog;
|
||||
|
||||
import java.nio.charset.Charset;
|
||||
|
||||
import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber2;
|
||||
import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber4;
|
||||
import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.usecondsToStr;
|
||||
@@ -57,23 +59,27 @@ public class JsonConversion {
|
||||
public static final int VALUE_ENTRY_SIZE_LARGE = (1 + LARGE_OFFSET_SIZE);
|
||||
|
||||
public static Json_Value parse_value(int type, LogBuffer buffer, long len, String charsetName) {
|
||||
return parse_value(type, buffer, len, Charset.forName(charsetName));
|
||||
}
|
||||
|
||||
public static Json_Value parse_value(int type, LogBuffer buffer, long len, Charset charset) {
|
||||
buffer = buffer.duplicate(buffer.position(), (int) len);
|
||||
switch (type) {
|
||||
case JSONB_TYPE_SMALL_OBJECT:
|
||||
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false, charsetName);
|
||||
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false, charset);
|
||||
case JSONB_TYPE_LARGE_OBJECT:
|
||||
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true, charsetName);
|
||||
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true, charset);
|
||||
case JSONB_TYPE_SMALL_ARRAY:
|
||||
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false, charsetName);
|
||||
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false, charset);
|
||||
case JSONB_TYPE_LARGE_ARRAY:
|
||||
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true, charsetName);
|
||||
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true, charset);
|
||||
default:
|
||||
return parse_scalar(type, buffer, len, charsetName);
|
||||
return parse_scalar(type, buffer, len, charset);
|
||||
}
|
||||
}
|
||||
|
||||
private static Json_Value parse_array_or_object(Json_enum_type type, LogBuffer buffer, long len, boolean large,
|
||||
String charsetName) {
|
||||
Charset charset) {
|
||||
long offset_size = large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
|
||||
if (len < 2 * offset_size) {
|
||||
throw new IllegalArgumentException("illegal json data");
|
||||
@@ -100,7 +106,7 @@ public class JsonConversion {
|
||||
return large ? buffer.getUint32() : buffer.getUint16();
|
||||
}
|
||||
|
||||
private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, String charsetName) {
|
||||
private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, Charset charset) {
|
||||
switch (type) {
|
||||
case JSONB_TYPE_LITERAL:
|
||||
/* purecov: inspected */
|
||||
@@ -156,7 +162,7 @@ public class JsonConversion {
|
||||
}
|
||||
return new Json_Value(Json_enum_type.STRING, buffer.rewind()
|
||||
.forward((int) n)
|
||||
.getFixString((int) str_len, charsetName));
|
||||
.getFixString((int) str_len, charset));
|
||||
case JSONB_TYPE_OPAQUE:
|
||||
/*
|
||||
* There should always be at least one byte, which tells the
|
||||
@@ -242,7 +248,7 @@ public class JsonConversion {
|
||||
this.m_large = large;
|
||||
}
|
||||
|
||||
public String key(int i, String charsetName) {
|
||||
public String key(int i, Charset charset) {
|
||||
m_data.rewind();
|
||||
int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
|
||||
int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
|
||||
@@ -255,10 +261,10 @@ public class JsonConversion {
|
||||
// entry, always two
|
||||
// bytes.
|
||||
long key_length = m_data.getUint16();
|
||||
return m_data.rewind().forward((int) key_offset).getFixString((int) key_length, charsetName);
|
||||
return m_data.rewind().forward((int) key_offset).getFixString((int) key_length, charset);
|
||||
}
|
||||
|
||||
public Json_Value element(int i, String charsetName) {
|
||||
public Json_Value element(int i, Charset charset) {
|
||||
m_data.rewind();
|
||||
int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
|
||||
int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
|
||||
@@ -271,13 +277,13 @@ public class JsonConversion {
|
||||
int type = m_data.forward(entry_offset).getUint8();
|
||||
if (type == JSONB_TYPE_INT16 || type == JSONB_TYPE_UINT16 || type == JSONB_TYPE_LITERAL
|
||||
|| (m_large && (type == JSONB_TYPE_INT32 || type == JSONB_TYPE_UINT32))) {
|
||||
return parse_scalar(type, m_data, value_entry_size - 1, charsetName);
|
||||
return parse_scalar(type, m_data, value_entry_size - 1, charset);
|
||||
}
|
||||
int value_offset = (int) read_offset_or_size(m_data, m_large);
|
||||
return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset, charsetName);
|
||||
return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset, charset);
|
||||
}
|
||||
|
||||
public StringBuilder toJsonString(StringBuilder buf, String charsetName) {
|
||||
public StringBuilder toJsonString(StringBuilder buf, Charset charset) {
|
||||
switch (m_type) {
|
||||
case OBJECT:
|
||||
buf.append("{");
|
||||
@@ -285,9 +291,9 @@ public class JsonConversion {
|
||||
if (i > 0) {
|
||||
buf.append(", ");
|
||||
}
|
||||
buf.append('"').append(key(i, charsetName)).append('"');
|
||||
buf.append('"').append(key(i, charset)).append('"');
|
||||
buf.append(": ");
|
||||
element(i, charsetName).toJsonString(buf, charsetName);
|
||||
element(i, charset).toJsonString(buf, charset);
|
||||
}
|
||||
buf.append("}");
|
||||
break;
|
||||
@@ -297,7 +303,7 @@ public class JsonConversion {
|
||||
if (i > 0) {
|
||||
buf.append(", ");
|
||||
}
|
||||
element(i, charsetName).toJsonString(buf, charsetName);
|
||||
element(i, charset).toJsonString(buf, charset);
|
||||
}
|
||||
buf.append("]");
|
||||
break;
|
||||
@@ -398,7 +404,7 @@ public class JsonConversion {
|
||||
}
|
||||
buf.append('"').append(text).append('"');
|
||||
} else {
|
||||
text = m_data.getFixString((int) m_length, charsetName);
|
||||
text = m_data.getFixString((int) m_length, charset);
|
||||
buf.append('"').append(escapse(text)).append('"');
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.taobao.tddl.dbsync.binlog;
|
||||
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@@ -36,6 +37,11 @@ public class JsonDiffConversion {
|
||||
|
||||
public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex,
|
||||
String charsetName) {
|
||||
return print_json_diff(buffer, len, columnName, columnIndex, Charset.forName(charsetName));
|
||||
}
|
||||
|
||||
public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex,
|
||||
Charset charset) {
|
||||
int position = buffer.position();
|
||||
List<String> operation_names = new ArrayList<>();
|
||||
while (buffer.hasRemaining()) {
|
||||
@@ -111,14 +117,14 @@ public class JsonDiffConversion {
|
||||
Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(),
|
||||
buffer,
|
||||
value_length - 1,
|
||||
charsetName);
|
||||
charset);
|
||||
buffer.forward((int) value_length - 1);
|
||||
// Read value
|
||||
if (jsonValue.m_type == Json_enum_type.ERROR) {
|
||||
throw new IllegalArgumentException("parsing json value");
|
||||
}
|
||||
StringBuilder jsonBuilder = new StringBuilder();
|
||||
jsonValue.toJsonString(jsonBuilder, charsetName);
|
||||
jsonValue.toJsonString(jsonBuilder, charset);
|
||||
builder.append(jsonBuilder);
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ import java.io.OutputStream;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.math.BigDecimal;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.BitSet;
|
||||
|
||||
@@ -16,6 +18,8 @@ import java.util.BitSet;
|
||||
*/
|
||||
public class LogBuffer {
|
||||
|
||||
static final BigDecimal DECIMAL_ZERO_1_SCALE = BigDecimal.valueOf(0, 1);
|
||||
static final BigDecimal DECIMAL_ONE_1_SCALE = BigDecimal.valueOf(10, 1);
|
||||
protected byte[] buffer;
|
||||
|
||||
protected int origin, limit;
|
||||
@@ -1038,29 +1042,27 @@ public class LogBuffer {
|
||||
}
|
||||
}
|
||||
|
||||
/* default ANSI charset */
|
||||
public static final String ISO_8859_1 = "ISO-8859-1";
|
||||
|
||||
/**
|
||||
* Return fix length string from buffer.
|
||||
*/
|
||||
public final String getFixString(final int pos, final int len) {
|
||||
return getFixString(pos, len, ISO_8859_1);
|
||||
return getFixString(pos, len, StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return next fix length string from buffer.
|
||||
*/
|
||||
public final String getFixString(final int len) {
|
||||
return getFixString(len, ISO_8859_1);
|
||||
return getFixString(len, StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return fix length string from buffer.
|
||||
*/
|
||||
public final String getFixString(final int pos, final int len, String charsetName) {
|
||||
if (pos + len > limit || pos < 0) throw new IllegalArgumentException("limit excceed: "
|
||||
+ (pos < 0 ? pos : (pos + len)));
|
||||
public final String getFixString(final int pos, final int len, Charset charset) {
|
||||
if (pos + len > limit || pos < 0) {
|
||||
throw new IllegalArgumentException("limit excceed: " + (pos < 0 ? pos : (pos + len)));
|
||||
}
|
||||
|
||||
final int from = origin + pos;
|
||||
final int end = from + len;
|
||||
@@ -1069,19 +1071,16 @@ public class LogBuffer {
|
||||
for (; (found < end) && buf[found] != '\0'; found++)
|
||||
/* empty loop */;
|
||||
|
||||
try {
|
||||
return new String(buf, from, found - from, charsetName);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new IllegalArgumentException("Unsupported encoding: " + charsetName, e);
|
||||
}
|
||||
return new String(buf, from, found - from, charset);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return next fix length string from buffer.
|
||||
*/
|
||||
public final String getFixString(final int len, String charsetName) {
|
||||
if (position + len > origin + limit) throw new IllegalArgumentException("limit excceed: "
|
||||
+ (position + len - origin));
|
||||
public final String getFixString(final int len, Charset charset) {
|
||||
if (position + len > origin + limit) {
|
||||
throw new IllegalArgumentException("limit excceed: " + (position + len - origin));
|
||||
}
|
||||
|
||||
final int from = position;
|
||||
final int end = from + len;
|
||||
@@ -1090,97 +1089,141 @@ public class LogBuffer {
|
||||
for (; (found < end) && buf[found] != '\0'; found++)
|
||||
/* empty loop */;
|
||||
|
||||
try {
|
||||
String string = new String(buf, from, found - from, charsetName);
|
||||
position += len;
|
||||
return string;
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new IllegalArgumentException("Unsupported encoding: " + charsetName, e);
|
||||
String string = new String(buf, from, found - from, charset);
|
||||
position += len;
|
||||
return string;
|
||||
}
|
||||
|
||||
public final String getFixName(final int len, Charset charset) {
|
||||
if (position + len > origin + limit) {
|
||||
throw new IllegalArgumentException("limit excceed: " + (position + len - origin));
|
||||
}
|
||||
|
||||
final int from = position;
|
||||
final int end = from + len;
|
||||
byte[] buf = buffer;
|
||||
int found = from;
|
||||
for (; (found < end) && buf[found] != '\0'; found++)
|
||||
/* empty loop */;
|
||||
|
||||
int length = found - from;
|
||||
String string = null;
|
||||
if (length <= 16) {
|
||||
string = NameCache.name(buf, from, length, charset);
|
||||
}
|
||||
if (string == null) {
|
||||
string = new String(buf, from, length, charset);
|
||||
}
|
||||
position += len;
|
||||
return string;
|
||||
}
|
||||
|
||||
public final String getFixName(final int len) {
|
||||
if (position + len > origin + limit) {
|
||||
throw new IllegalArgumentException("limit excceed: " + (position + len - origin));
|
||||
}
|
||||
|
||||
final int from = position;
|
||||
final int end = from + len;
|
||||
byte[] buf = buffer;
|
||||
int found = from;
|
||||
for (; (found < end) && buf[found] != '\0'; found++)
|
||||
/* empty loop */;
|
||||
|
||||
int length = found - from;
|
||||
String string = null;
|
||||
if (length <= 16) {
|
||||
string = NameCache.name(buf, from, length, StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
if (string == null) {
|
||||
string = new String(buf, from, length, StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
position += len;
|
||||
return string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return fix-length string from buffer without null-terminate checking. Fix
|
||||
* bug #17 {@link https://github.com/AlibabaTech/canal/issues/17 }
|
||||
* Return fix-length string from buffer without null-terminate checking.
|
||||
* Fix bug #17 {@link https://github.com/AlibabaTech/canal/issues/17 }
|
||||
*/
|
||||
public final String getFullString(final int pos, final int len, String charsetName) {
|
||||
if (pos + len > limit || pos < 0) throw new IllegalArgumentException("limit excceed: "
|
||||
+ (pos < 0 ? pos : (pos + len)));
|
||||
|
||||
try {
|
||||
return new String(buffer, origin + pos, len, charsetName);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new IllegalArgumentException("Unsupported encoding: " + charsetName, e);
|
||||
public final String getFullString(final int pos, final int len, Charset charset) {
|
||||
if (pos + len > limit || pos < 0) {
|
||||
throw new IllegalArgumentException("limit excceed: " + (pos < 0 ? pos : (pos + len)));
|
||||
}
|
||||
|
||||
return new String(buffer, origin + pos, len, charset);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return next fix-length string from buffer without null-terminate
|
||||
* checking. Fix bug #17 {@link https
|
||||
* ://github.com/AlibabaTech/canal/issues/17 }
|
||||
* checking.
|
||||
* Fix bug #17 {@link https://github.com/AlibabaTech/canal/issues/17 }
|
||||
*/
|
||||
public final String getFullString(final int len, String charsetName) {
|
||||
if (position + len > origin + limit) throw new IllegalArgumentException("limit excceed: "
|
||||
+ (position + len - origin));
|
||||
|
||||
try {
|
||||
String string = new String(buffer, position, len, charsetName);
|
||||
position += len;
|
||||
return string;
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new IllegalArgumentException("Unsupported encoding: " + charsetName, e);
|
||||
public final String getFullString(final int len, Charset charset) {
|
||||
if (position + len > origin + limit) {
|
||||
throw new IllegalArgumentException("limit excceed: " + (position + len - origin));
|
||||
}
|
||||
|
||||
String string = new String(buffer, position, len, charset);
|
||||
position += len;
|
||||
return string;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Return dynamic length string from buffer.
|
||||
*/
|
||||
public final String getString(final int pos) {
|
||||
return getString(pos, ISO_8859_1);
|
||||
if (pos >= limit || pos < 0) {
|
||||
throw new IllegalArgumentException("limit excceed: " + pos);
|
||||
}
|
||||
|
||||
byte[] buf = buffer;
|
||||
final int len = (0xff & buf[origin + pos]);
|
||||
if (pos + len + 1 > limit) {
|
||||
throw new IllegalArgumentException("limit excceed: " + (pos + len + 1));
|
||||
}
|
||||
|
||||
return new String(buf, origin + pos + 1, len, StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return next dynamic length string from buffer.
|
||||
*/
|
||||
public final String getString() {
|
||||
return getString(ISO_8859_1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return dynamic length string from buffer.
|
||||
*/
|
||||
public final String getString(final int pos, String charsetName) {
|
||||
if (pos >= limit || pos < 0) throw new IllegalArgumentException("limit excceed: " + pos);
|
||||
|
||||
byte[] buf = buffer;
|
||||
final int len = (0xff & buf[origin + pos]);
|
||||
if (pos + len + 1 > limit) throw new IllegalArgumentException("limit excceed: " + (pos + len + 1));
|
||||
|
||||
try {
|
||||
return new String(buf, origin + pos + 1, len, charsetName);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new IllegalArgumentException("Unsupported encoding: " + charsetName, e);
|
||||
if (position >= origin + limit) {
|
||||
throw new IllegalArgumentException("limit excceed: " + position);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return next dynamic length string from buffer.
|
||||
*/
|
||||
public final String getString(String charsetName) {
|
||||
if (position >= origin + limit) throw new IllegalArgumentException("limit excceed: " + position);
|
||||
|
||||
byte[] buf = buffer;
|
||||
final int len = (0xff & buf[position]);
|
||||
if (position + len + 1 > origin + limit) throw new IllegalArgumentException("limit excceed: "
|
||||
+ (position + len + 1 - origin));
|
||||
|
||||
try {
|
||||
String string = new String(buf, position + 1, len, charsetName);
|
||||
position += len + 1;
|
||||
return string;
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new IllegalArgumentException("Unsupported encoding: " + charsetName, e);
|
||||
if (position + len + 1 > origin + limit) {
|
||||
throw new IllegalArgumentException("limit excceed: "
|
||||
+ (position + len + 1 - origin));
|
||||
}
|
||||
|
||||
String string = new String(buf, position + 1, len, StandardCharsets.ISO_8859_1);
|
||||
position += len + 1;
|
||||
return string;
|
||||
}
|
||||
|
||||
public final String getName() {
|
||||
if (position >= origin + limit) {
|
||||
throw new IllegalArgumentException("limit excceed: " + position);
|
||||
}
|
||||
|
||||
byte[] buf = buffer;
|
||||
final int len = (0xff & buf[position]);
|
||||
if (position + len + 1 > origin + limit) {
|
||||
throw new IllegalArgumentException("limit excceed: " + (position + len + 1 - origin));
|
||||
}
|
||||
|
||||
String string = NameCache.name(buf, position + 1, len, StandardCharsets.ISO_8859_1);
|
||||
if (string == null) {
|
||||
string = new String(buf, position + 1, len, StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
position += len + 1;
|
||||
return string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1263,6 +1306,15 @@ public class LogBuffer {
|
||||
throw new IllegalArgumentException("limit excceed: " + (position + binSize - origin));
|
||||
}
|
||||
|
||||
if (precision <= 20 && intg <= 18 && frac <= 9) {
|
||||
// use compact BigDecimal
|
||||
BigDecimal compactDecimal = getBigDecimalCompact(precision, scale, intg, frac, intg0x);
|
||||
if (compactDecimal != null) {
|
||||
position += binSize;
|
||||
return compactDecimal;
|
||||
}
|
||||
}
|
||||
|
||||
BigDecimal decimal = getDecimal0(position, intg, frac, // NL
|
||||
intg0,
|
||||
frac0,
|
||||
@@ -1272,6 +1324,162 @@ public class LogBuffer {
|
||||
return decimal;
|
||||
}
|
||||
|
||||
/**
|
||||
* use compact BigDecimal
|
||||
*/
|
||||
private BigDecimal getBigDecimalCompact(int precision, int scale, int intg, int frac, int intg0x) {
|
||||
if (precision > 20 || intg > 18 && frac > 9) {
|
||||
return null;
|
||||
}
|
||||
|
||||
BigDecimal compactDecimal = null;
|
||||
long longValue = 0;
|
||||
int small = 0;
|
||||
int pos = position;
|
||||
byte b0 = buffer[position];
|
||||
final int mask = ((b0 & 0x80) == 0x80) ? 0 : -1;
|
||||
b0 ^= 0x80;
|
||||
|
||||
switch (intg0x) {
|
||||
case 0:
|
||||
if (intg == 0) {
|
||||
longValue = 0;
|
||||
} else {
|
||||
longValue = (b0 << 24)
|
||||
| ((0xff & buffer[pos + 1]) << 16)
|
||||
| ((0xff & buffer[pos + 2]) << 8)
|
||||
| (0xff & buffer[pos + 3]);
|
||||
longValue ^= mask;
|
||||
pos += 4;
|
||||
}
|
||||
break;
|
||||
case 1:
|
||||
case 2:
|
||||
longValue = b0;
|
||||
longValue ^= mask;
|
||||
pos += 1;
|
||||
break;
|
||||
case 3:
|
||||
case 4:
|
||||
longValue = (b0 << 8) | (0xff & buffer[pos + 1]);
|
||||
longValue ^= mask;
|
||||
pos += 2;
|
||||
break;
|
||||
case 5:
|
||||
case 6:
|
||||
longValue = (b0 << 16) | ((0xff & buffer[pos + 1]) << 8) | (0xff & buffer[pos + 2]);
|
||||
longValue ^= mask;
|
||||
pos += 3;
|
||||
break;
|
||||
case 7:
|
||||
case 8:
|
||||
longValue = (b0 << 24)
|
||||
| ((0xff & buffer[pos + 1]) << 16)
|
||||
| ((0xff & buffer[pos + 2]) << 8)
|
||||
| (0xff & buffer[pos + 3]);
|
||||
longValue ^= mask;
|
||||
pos += 4;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
if (intg > 9) {
|
||||
int value = getInt32BE(buffer, pos);
|
||||
value ^= mask;
|
||||
longValue *= 1_000_000_000;
|
||||
longValue += value;
|
||||
pos += 4;
|
||||
}
|
||||
|
||||
int power = 1;
|
||||
if (pos != position) {
|
||||
b0 = buffer[pos];
|
||||
}
|
||||
switch (frac) {
|
||||
case 0:
|
||||
small = 0;
|
||||
break;
|
||||
case 1:
|
||||
case 2:
|
||||
power = frac == 1 ? 10 : 100;
|
||||
small = b0;
|
||||
small ^= mask;
|
||||
break;
|
||||
case 3:
|
||||
case 4:
|
||||
power = frac == 3 ? 1_000 : 10_000;
|
||||
small = (b0 << 8) | (0xff & buffer[pos + 1]);
|
||||
small ^= mask;
|
||||
break;
|
||||
case 5:
|
||||
case 6:
|
||||
power = frac == 5 ? 100_000 : 1_000_000;
|
||||
small = (b0 << 16) | ((0xff & buffer[pos + 1]) << 8) | (0xff & buffer[pos + 2]);
|
||||
small ^= mask;
|
||||
break;
|
||||
case 7:
|
||||
case 8:
|
||||
case 9:
|
||||
power = frac == 7 ? 10_000_000 : frac == 8 ? 100_000_000 : 1_000_000_000;
|
||||
small = (b0 << 24)
|
||||
| ((0xff & buffer[pos + 1]) << 16)
|
||||
| ((0xff & buffer[pos + 2]) << 8)
|
||||
| (0xff & buffer[pos + 3]);
|
||||
small ^= mask;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
if (small == 0) {
|
||||
if (frac == 0) {
|
||||
if (mask != 0) {
|
||||
longValue = -longValue;
|
||||
}
|
||||
compactDecimal = BigDecimal.valueOf(longValue, 0);
|
||||
} else if (longValue == 0) {
|
||||
if (scale < 9) {
|
||||
compactDecimal = DECIMAL_ZERO_1_SCALE;
|
||||
} else {
|
||||
compactDecimal = BigDecimal.valueOf(longValue, scale);
|
||||
}
|
||||
} else {
|
||||
if (scale < 9) {
|
||||
long unscaleValue = longValue * 10;
|
||||
if (unscaleValue >= longValue) {
|
||||
if (mask != 0) {
|
||||
unscaleValue = -unscaleValue;
|
||||
}
|
||||
if (unscaleValue == 10) {
|
||||
compactDecimal = DECIMAL_ONE_1_SCALE;
|
||||
} else {
|
||||
compactDecimal = BigDecimal.valueOf(unscaleValue, 1);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
long unscaleValue = longValue * power;
|
||||
if (unscaleValue >= longValue) {
|
||||
if (mask != 0) {
|
||||
unscaleValue = -unscaleValue;
|
||||
}
|
||||
compactDecimal = BigDecimal.valueOf(unscaleValue, scale);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
long unscaleValue = longValue * power + small;
|
||||
if (unscaleValue >= longValue) {
|
||||
if (mask != 0) {
|
||||
unscaleValue = -unscaleValue;
|
||||
}
|
||||
compactDecimal = BigDecimal.valueOf(unscaleValue, scale);
|
||||
}
|
||||
}
|
||||
|
||||
return compactDecimal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return big decimal from buffer.
|
||||
*
|
||||
@@ -1445,8 +1653,9 @@ public class LogBuffer {
|
||||
}
|
||||
|
||||
d_copy[begin] ^= 0x80; /* restore sign */
|
||||
String decimal = String.valueOf(buf, 0, pos);
|
||||
return new BigDecimal(decimal);
|
||||
// String decimal = String.valueOf(buf, 0, pos);
|
||||
// return new BigDecimal(decimal);
|
||||
return new BigDecimal(buf, 0, pos);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -197,8 +197,9 @@ public final class LogDecoder {
|
||||
}
|
||||
case LogEvent.SLAVE_EVENT: /* can never happen (unused event) */
|
||||
{
|
||||
if (logger.isWarnEnabled()) logger.warn("Skipping unsupported SLAVE_EVENT from: "
|
||||
+ context.getLogPosition());
|
||||
if (logger.isWarnEnabled()) {
|
||||
logger.warn("Skipping unsupported SLAVE_EVENT from: " + context.getLogPosition());
|
||||
}
|
||||
break;
|
||||
}
|
||||
case LogEvent.CREATE_FILE_EVENT: {
|
||||
@@ -264,22 +265,25 @@ public final class LogDecoder {
|
||||
return descriptionEvent;
|
||||
}
|
||||
case LogEvent.PRE_GA_WRITE_ROWS_EVENT: {
|
||||
if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: "
|
||||
+ context.getLogPosition());
|
||||
if (logger.isWarnEnabled()) {
|
||||
logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: " + context.getLogPosition());
|
||||
}
|
||||
// ev = new Write_rows_log_event_old(buf, event_len,
|
||||
// description_event);
|
||||
break;
|
||||
}
|
||||
case LogEvent.PRE_GA_UPDATE_ROWS_EVENT: {
|
||||
if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: "
|
||||
+ context.getLogPosition());
|
||||
if (logger.isWarnEnabled()) {
|
||||
logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: " + context.getLogPosition());
|
||||
}
|
||||
// ev = new Update_rows_log_event_old(buf, event_len,
|
||||
// description_event);
|
||||
break;
|
||||
}
|
||||
case LogEvent.PRE_GA_DELETE_ROWS_EVENT: {
|
||||
if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: "
|
||||
+ context.getLogPosition());
|
||||
if (logger.isWarnEnabled()) {
|
||||
logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: " + context.getLogPosition());
|
||||
}
|
||||
// ev = new Delete_rows_log_event_old(buf, event_len,
|
||||
// description_event);
|
||||
break;
|
||||
@@ -356,10 +360,16 @@ public final class LogDecoder {
|
||||
return event;
|
||||
}
|
||||
case LogEvent.TRANSACTION_PAYLOAD_EVENT: {
|
||||
TransactionPayloadLogEvent event = new TransactionPayloadLogEvent(header, buffer, descriptionEvent);
|
||||
/* updating position in context */
|
||||
logPosition.position = header.getLogPos();
|
||||
return event;
|
||||
if (logger.isWarnEnabled()) {
|
||||
logger.warn("Skipping unsupported MySQL TRANSACTION_PAYLOAD_EVENT from: " + context.getLogPosition());
|
||||
}
|
||||
break;
|
||||
|
||||
// TransactionPayloadLogEvent event = new TransactionPayloadLogEvent(header,
|
||||
// buffer, descriptionEvent);
|
||||
// /* updating position in context */
|
||||
// logPosition.position = header.getLogPos();
|
||||
// return event;
|
||||
}
|
||||
case LogEvent.VIEW_CHANGE_EVENT: {
|
||||
ViewChangeEvent event = new ViewChangeEvent(header, buffer, descriptionEvent);
|
||||
@@ -418,6 +428,39 @@ public final class LogDecoder {
|
||||
logPosition.position = header.getLogPos();
|
||||
return event;
|
||||
}
|
||||
case LogEvent.HEARTBEAT_LOG_EVENT_V2: {
|
||||
HeartbeatV2LogEvent event = new HeartbeatV2LogEvent(header, buffer, descriptionEvent);
|
||||
/* updating position in context */
|
||||
logPosition.position = header.getLogPos();
|
||||
return event;
|
||||
}
|
||||
case LogEvent.QUERY_COMPRESSED_EVENT: {
|
||||
if (logger.isWarnEnabled()) {
|
||||
logger.warn("Skipping unsupported MaraiDB QUERY_COMPRESSED_EVENT from: " + context.getLogPosition());
|
||||
}
|
||||
break;
|
||||
}
|
||||
case LogEvent.WRITE_ROWS_COMPRESSED_EVENT_V1:
|
||||
case LogEvent.WRITE_ROWS_COMPRESSED_EVENT: {
|
||||
if (logger.isWarnEnabled()) {
|
||||
logger.warn("Skipping unsupported MaraiDB WRITE_ROWS_COMPRESSED_EVENT from: " + context.getLogPosition());
|
||||
}
|
||||
break;
|
||||
}
|
||||
case LogEvent.UPDATE_ROWS_COMPRESSED_EVENT_V1:
|
||||
case LogEvent.UPDATE_ROWS_COMPRESSED_EVENT: {
|
||||
if (logger.isWarnEnabled()) {
|
||||
logger.warn("Skipping unsupported MaraiDB UPDATE_ROWS_COMPRESSED_EVENT from: " + context.getLogPosition());
|
||||
}
|
||||
break;
|
||||
}
|
||||
case LogEvent.DELETE_ROWS_COMPRESSED_EVENT_V1:
|
||||
case LogEvent.DELETE_ROWS_COMPRESSED_EVENT: {
|
||||
if (logger.isWarnEnabled()) {
|
||||
logger.warn("Skipping unsupported MaraiDB DELETE_ROWS_COMPRESSED_EVENT from: " + context.getLogPosition());
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
/*
|
||||
* Create an object of Ignorable_log_event for unrecognized
|
||||
|
||||
@@ -178,7 +178,10 @@ public abstract class LogEvent {
|
||||
/* mysql 8.0.20 */
|
||||
public static final int TRANSACTION_PAYLOAD_EVENT = 40;
|
||||
|
||||
public static final int MYSQL_ENUM_END_EVENT = 41;
|
||||
/* mysql 8.0.26 */
|
||||
public static final int HEARTBEAT_LOG_EVENT_V2 = 41;
|
||||
|
||||
public static final int MYSQL_ENUM_END_EVENT = 42;
|
||||
|
||||
// mariaDb 5.5.34
|
||||
/* New MySQL/Sun events are to be added right above this comment */
|
||||
@@ -210,8 +213,23 @@ public abstract class LogEvent {
|
||||
|
||||
public static final int START_ENCRYPTION_EVENT = 164;
|
||||
|
||||
// mariadb 10.10.1
|
||||
/*
|
||||
* Compressed binlog event. Note that the order between WRITE/UPDATE/DELETE
|
||||
* events is significant; this is so that we can convert from the compressed to
|
||||
* the uncompressed event type with (type-WRITE_ROWS_COMPRESSED_EVENT +
|
||||
* WRITE_ROWS_EVENT) and similar for _V1.
|
||||
*/
|
||||
public static final int QUERY_COMPRESSED_EVENT = 165;
|
||||
public static final int WRITE_ROWS_COMPRESSED_EVENT_V1 = 166;
|
||||
public static final int UPDATE_ROWS_COMPRESSED_EVENT_V1 = 167;
|
||||
public static final int DELETE_ROWS_COMPRESSED_EVENT_V1 = 168;
|
||||
public static final int WRITE_ROWS_COMPRESSED_EVENT = 169;
|
||||
public static final int UPDATE_ROWS_COMPRESSED_EVENT = 170;
|
||||
public static final int DELETE_ROWS_COMPRESSED_EVENT = 171;
|
||||
|
||||
/** end marker */
|
||||
public static final int ENUM_END_EVENT = 165;
|
||||
public static final int ENUM_END_EVENT = 171;
|
||||
|
||||
/**
|
||||
* 1 byte length, 1 byte format Length is total length in bytes, including 2
|
||||
@@ -358,6 +376,7 @@ public abstract class LogEvent {
|
||||
case INCIDENT_EVENT:
|
||||
return "Incident";
|
||||
case HEARTBEAT_LOG_EVENT:
|
||||
case HEARTBEAT_LOG_EVENT_V2:
|
||||
return "Heartbeat";
|
||||
case IGNORABLE_LOG_EVENT:
|
||||
return "Ignorable";
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -18,6 +18,7 @@ public class GtidLogEvent extends LogEvent {
|
||||
// / Length of SID in event encoding
|
||||
public static final int ENCODED_SID_LENGTH = 16;
|
||||
public static final int LOGICAL_TIMESTAMP_TYPE_CODE = 2;
|
||||
public static final UUID UUID_ZERO = new UUID(0, 0);
|
||||
|
||||
private boolean commitFlag;
|
||||
private UUID sid;
|
||||
@@ -35,11 +36,13 @@ public class GtidLogEvent extends LogEvent {
|
||||
buffer.position(commonHeaderLen);
|
||||
commitFlag = (buffer.getUint8() != 0); // ENCODED_FLAG_LENGTH
|
||||
|
||||
byte[] bs = buffer.getData(ENCODED_SID_LENGTH);
|
||||
ByteBuffer bb = ByteBuffer.wrap(bs);
|
||||
long high = bb.getLong();
|
||||
long low = bb.getLong();
|
||||
sid = new UUID(high, low);
|
||||
long high = buffer.getBeLong64();
|
||||
long low = buffer.getBeLong64();
|
||||
if (high == 0 && low == 0) {
|
||||
sid = UUID_ZERO;
|
||||
} else {
|
||||
sid = new UUID(high, low);
|
||||
}
|
||||
|
||||
gno = buffer.getLong64();
|
||||
|
||||
@@ -73,11 +76,11 @@ public class GtidLogEvent extends LogEvent {
|
||||
return gno;
|
||||
}
|
||||
|
||||
public Long getLastCommitted() {
|
||||
public long getLastCommitted() {
|
||||
return lastCommitted;
|
||||
}
|
||||
|
||||
public Long getSequenceNumber() {
|
||||
public long getSequenceNumber() {
|
||||
return sequenceNumber;
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ package com.taobao.tddl.dbsync.binlog.event;
|
||||
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* <pre>
|
||||
* Replication event to ensure to slave that master is alive.
|
||||
@@ -35,7 +37,7 @@ public class HeartbeatLogEvent extends LogEvent {
|
||||
identLen = FN_REFLEN - 1;
|
||||
}
|
||||
|
||||
logIdent = buffer.getFullString(commonHeaderLen, identLen, LogBuffer.ISO_8859_1);
|
||||
logIdent = buffer.getFullString(commonHeaderLen, identLen, StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
|
||||
public int getIdentLen() {
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package com.taobao.tddl.dbsync.binlog.event;
|
||||
|
||||
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||
|
||||
/**
|
||||
* <pre>
|
||||
* Replication event to ensure to replica that source is alive.
|
||||
* The event is originated by source's dump thread and sent straight to
|
||||
* replica without being logged. Slave itself does not store it in relay log
|
||||
* but rather uses a data for immediate checks and throws away the event.
|
||||
* Two members of the class m_log_filename and m_log_position comprise
|
||||
* @see the rpl_event_coordinates instance. The coordinates that a heartbeat
|
||||
* instance carries correspond to the last event source has sent from
|
||||
* its binlog.
|
||||
* Also this event will be generated only for the source server with
|
||||
* version > 8.0.26
|
||||
* </pre>
|
||||
*
|
||||
* @author jianghang 2022-09-01 下午16:36:29
|
||||
* @version 1.1.6
|
||||
* @since mysql 8.0.26
|
||||
*/
|
||||
public class HeartbeatV2LogEvent extends LogEvent {
|
||||
|
||||
private byte[] payload;
|
||||
|
||||
public HeartbeatV2LogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
|
||||
super(header);
|
||||
|
||||
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
|
||||
int payloadLenth = buffer.limit() - commonHeaderLen;
|
||||
// see : https://github.com/mysql/mysql-server/commit/59e590738a772b74ad50c4a57c86aaa1bc6501c7#diff-184e9a7d8a58f080974e475d4199fe5c6da5518c8a2811cc5df5988c8f9e9797
|
||||
payload = buffer.getData(payloadLenth);
|
||||
}
|
||||
|
||||
public byte[] getPayload() {
|
||||
return payload;
|
||||
}
|
||||
}
|
||||
@@ -67,7 +67,7 @@ public final class IncidentLogEvent extends LogEvent {
|
||||
incident = incidentNumber;
|
||||
|
||||
buffer.position(commonHeaderLen + postHeaderLen);
|
||||
message = buffer.getString();
|
||||
message = buffer.getName();
|
||||
}
|
||||
|
||||
public final int getIncident() {
|
||||
|
||||
@@ -433,7 +433,7 @@ public class QueryLogEvent extends LogEvent {
|
||||
private int serverCollation = -1;
|
||||
private int tvSec = -1;
|
||||
private BigInteger ddlXid = BigInteger.valueOf(-1L);
|
||||
private String charsetName;
|
||||
private Charset charset;
|
||||
|
||||
private String timezone;
|
||||
|
||||
@@ -498,12 +498,12 @@ public class QueryLogEvent extends LogEvent {
|
||||
|
||||
/* A 2nd variable part; this is common to all versions */
|
||||
final int queryLen = dataLen - dbLen - 1;
|
||||
dbname = buffer.getFixString(dbLen + 1);
|
||||
dbname = buffer.getFixName(dbLen + 1);
|
||||
if (clientCharset >= 0) {
|
||||
charsetName = CharsetConversion.getJavaCharset(clientCharset);
|
||||
charset = CharsetConversion.getNioCharset(clientCharset);
|
||||
|
||||
if ((charsetName != null) && (Charset.isSupported(charsetName))) {
|
||||
query = buffer.getFixString(queryLen, charsetName);
|
||||
if (charset != null) {
|
||||
query = buffer.getFixString(queryLen, charset);
|
||||
} else {
|
||||
logger.warn("unsupported character set in query log: " + "\n ID = " + clientCharset + ", Charset = "
|
||||
+ CharsetConversion.getCharset(clientCharset) + ", Collation = "
|
||||
@@ -603,6 +603,13 @@ public class QueryLogEvent extends LogEvent {
|
||||
*/
|
||||
public static final int Q_HRNOW = 128;
|
||||
|
||||
/**
|
||||
* Support MariaDB 10.10.1
|
||||
*/
|
||||
public static final int Q_XID = 129;
|
||||
|
||||
public static final int Q_GTID_FLAGS3 = 130;
|
||||
|
||||
private final void unpackVariables(LogBuffer buffer, final int end) throws IOException {
|
||||
int code = -1;
|
||||
try {
|
||||
@@ -613,10 +620,10 @@ public class QueryLogEvent extends LogEvent {
|
||||
break;
|
||||
case Q_SQL_MODE_CODE:
|
||||
sql_mode = buffer.getLong64(); // QQ: Fix when sql_mode
|
||||
// is ulonglong
|
||||
// is ulonglong
|
||||
break;
|
||||
case Q_CATALOG_NZ_CODE:
|
||||
catalog = buffer.getString();
|
||||
catalog = buffer.getName();
|
||||
break;
|
||||
case Q_AUTO_INCREMENT:
|
||||
autoIncrementIncrement = buffer.getUint16();
|
||||
@@ -632,7 +639,7 @@ public class QueryLogEvent extends LogEvent {
|
||||
serverCollation = buffer.getUint16();
|
||||
break;
|
||||
case Q_TIME_ZONE_CODE:
|
||||
timezone = buffer.getString();
|
||||
timezone = buffer.getName();
|
||||
break;
|
||||
case Q_CATALOG_CODE: /* for 5.0.x where 0<=x<=3 masters */
|
||||
final int len = buffer.getUint8();
|
||||
@@ -656,8 +663,8 @@ public class QueryLogEvent extends LogEvent {
|
||||
buffer.forward(4);
|
||||
break;
|
||||
case Q_INVOKER:
|
||||
user = buffer.getString();
|
||||
host = buffer.getString();
|
||||
user = buffer.getName();
|
||||
host = buffer.getName();
|
||||
break;
|
||||
case Q_MICROSECONDS:
|
||||
// when.tv_usec= uint3korr(pos);
|
||||
@@ -678,7 +685,7 @@ public class QueryLogEvent extends LogEvent {
|
||||
String mtsAccessedDbNames[] = new String[mtsAccessedDbs];
|
||||
for (int i = 0; i < mtsAccessedDbs && buffer.position() < end; i++) {
|
||||
int length = end - buffer.position();
|
||||
mtsAccessedDbNames[i] = buffer.getFixString(length < NAME_LEN ? length : NAME_LEN);
|
||||
mtsAccessedDbNames[i] = buffer.getFixName(length < NAME_LEN ? length : NAME_LEN);
|
||||
}
|
||||
break;
|
||||
case Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP:
|
||||
@@ -701,6 +708,25 @@ public class QueryLogEvent extends LogEvent {
|
||||
// int when_sec_part = buffer.getUint24();
|
||||
buffer.forward(3);
|
||||
break;
|
||||
case Q_XID:
|
||||
// xid= uint8korr(pos);
|
||||
buffer.forward(8);
|
||||
break;
|
||||
case Q_GTID_FLAGS3:
|
||||
// gtid_flags_extra= *pos++;
|
||||
// if (gtid_flags_extra & (Gtid_log_event::FL_COMMIT_ALTER_E1 |
|
||||
// Gtid_log_event::FL_ROLLBACK_ALTER_E1)) {
|
||||
// sa_seq_no = uint8korr(pos);
|
||||
// pos+= 8;
|
||||
// }
|
||||
|
||||
int gtid_flags_extra = buffer.getUint8();
|
||||
final int FL_COMMIT_ALTER_E1= 4;
|
||||
final int FL_ROLLBACK_ALTER_E1= 8;
|
||||
if ((gtid_flags_extra & (FL_COMMIT_ALTER_E1 | FL_ROLLBACK_ALTER_E1))> 0) {
|
||||
buffer.forward(8);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
/*
|
||||
* That's why you must write status vars in growing
|
||||
@@ -740,16 +766,22 @@ public class QueryLogEvent extends LogEvent {
|
||||
return "Q_TABLE_MAP_FOR_UPDATE_CODE";
|
||||
case Q_MASTER_DATA_WRITTEN_CODE:
|
||||
return "Q_MASTER_DATA_WRITTEN_CODE";
|
||||
case Q_UPDATED_DB_NAMES:
|
||||
return "Q_UPDATED_DB_NAMES";
|
||||
case Q_INVOKER:
|
||||
return "case Q_INVOKER";
|
||||
case Q_MICROSECONDS:
|
||||
return "Q_MICROSECONDS";
|
||||
case Q_UPDATED_DB_NAMES:
|
||||
return "Q_UPDATED_DB_NAMES";
|
||||
case Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP:
|
||||
return "Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP";
|
||||
case Q_DDL_LOGGED_WITH_XID:
|
||||
return "Q_DDL_LOGGED_WITH_XID";
|
||||
case Q_DEFAULT_COLLATION_FOR_UTF8MB4:
|
||||
return "Q_DEFAULT_COLLATION_FOR_UTF8MB4";
|
||||
case Q_SQL_REQUIRE_PRIMARY_KEY:
|
||||
return "Q_SQL_REQUIRE_PRIMARY_KEY";
|
||||
case Q_HRNOW:
|
||||
return "Q_HRNOW";
|
||||
}
|
||||
return "CODE#" + code;
|
||||
}
|
||||
@@ -797,8 +829,8 @@ public class QueryLogEvent extends LogEvent {
|
||||
return autoIncrementOffset;
|
||||
}
|
||||
|
||||
public final String getCharsetName() {
|
||||
return charsetName;
|
||||
public final Charset getCharset() {
|
||||
return charset;
|
||||
}
|
||||
|
||||
public final String getTimezone() {
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
package com.taobao.tddl.dbsync.binlog.event;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.nio.charset.Charset;
|
||||
import java.sql.Timestamp;
|
||||
import java.sql.Types;
|
||||
import java.time.ZoneId;
|
||||
import java.time.zone.ZoneRules;
|
||||
import java.util.BitSet;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
@@ -23,7 +26,11 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||
*/
|
||||
public final class RowsLogBuffer {
|
||||
|
||||
protected static final Log logger = LogFactory.getLog(RowsLogBuffer.class);
|
||||
protected static final Log logger = LogFactory.getLog(RowsLogBuffer.class);
|
||||
public static final Long[] longCache = new Long[1024 * 128];
|
||||
public static final int longCacheLimit = longCache.length + 127;
|
||||
public static final Integer[] integerCache = new Integer[1024 * 128];
|
||||
public static final int integerCacheLimit = longCache.length + 127;
|
||||
|
||||
public static final long DATETIMEF_INT_OFS = 0x8000000000L;
|
||||
public static final long TIMEF_INT_OFS = 0x800000L;
|
||||
@@ -33,7 +40,7 @@ public final class RowsLogBuffer {
|
||||
private final LogBuffer buffer;
|
||||
private final int columnLen;
|
||||
private final int jsonColumnCount;
|
||||
private final String charsetName;
|
||||
private final Charset charset;
|
||||
|
||||
private final BitSet nullBits;
|
||||
private int nullBitIndex;
|
||||
@@ -47,10 +54,10 @@ public final class RowsLogBuffer {
|
||||
private int length;
|
||||
private Serializable value;
|
||||
|
||||
public RowsLogBuffer(LogBuffer buffer, final int columnLen, String charsetName, int jsonColumnCount, boolean partial){
|
||||
public RowsLogBuffer(LogBuffer buffer, final int columnLen, Charset charset, int jsonColumnCount, boolean partial){
|
||||
this.buffer = buffer;
|
||||
this.columnLen = columnLen;
|
||||
this.charsetName = charsetName;
|
||||
this.charset = charset;
|
||||
this.partial = partial;
|
||||
this.jsonColumnCount = jsonColumnCount;
|
||||
this.nullBits = new BitSet(columnLen);
|
||||
@@ -309,7 +316,7 @@ public final class RowsLogBuffer {
|
||||
// XXX: How to check signed / unsigned?
|
||||
// value = unsigned ? Long.valueOf(buffer.getUint32()) :
|
||||
// Integer.valueOf(buffer.getInt32());
|
||||
value = Integer.valueOf(buffer.getInt32());
|
||||
value = valueOf(buffer.getInt32());
|
||||
javaType = Types.INTEGER;
|
||||
length = 4;
|
||||
break;
|
||||
@@ -318,7 +325,7 @@ public final class RowsLogBuffer {
|
||||
// XXX: How to check signed / unsigned?
|
||||
// value = Integer.valueOf(unsigned ? buffer.getUint8() :
|
||||
// buffer.getInt8());
|
||||
value = Integer.valueOf(buffer.getInt8());
|
||||
value = valueOf(buffer.getInt8());
|
||||
javaType = Types.TINYINT; // java.sql.Types.INTEGER;
|
||||
length = 1;
|
||||
break;
|
||||
@@ -327,7 +334,7 @@ public final class RowsLogBuffer {
|
||||
// XXX: How to check signed / unsigned?
|
||||
// value = Integer.valueOf(unsigned ? buffer.getUint16() :
|
||||
// buffer.getInt16());
|
||||
value = Integer.valueOf((short) buffer.getInt16());
|
||||
value = valueOf((short) buffer.getInt16());
|
||||
javaType = Types.SMALLINT; // java.sql.Types.INTEGER;
|
||||
length = 2;
|
||||
break;
|
||||
@@ -336,7 +343,7 @@ public final class RowsLogBuffer {
|
||||
// XXX: How to check signed / unsigned?
|
||||
// value = Integer.valueOf(unsigned ? buffer.getUint24() :
|
||||
// buffer.getInt24());
|
||||
value = Integer.valueOf(buffer.getInt24());
|
||||
value = valueOf(buffer.getInt24());
|
||||
javaType = Types.INTEGER;
|
||||
length = 3;
|
||||
break;
|
||||
@@ -345,7 +352,7 @@ public final class RowsLogBuffer {
|
||||
// XXX: How to check signed / unsigned?
|
||||
// value = unsigned ? buffer.getUlong64()) :
|
||||
// Long.valueOf(buffer.getLong64());
|
||||
value = Long.valueOf(buffer.getLong64());
|
||||
value = valueOf(buffer.getLong64());
|
||||
javaType = Types.BIGINT; // Types.INTEGER;
|
||||
length = 8;
|
||||
break;
|
||||
@@ -870,7 +877,7 @@ public final class RowsLogBuffer {
|
||||
}
|
||||
// logger.warn("MYSQL_TYPE_ENUM : This enumeration value is "
|
||||
// + "only used internally and cannot exist in a binlog!");
|
||||
value = Integer.valueOf(int32);
|
||||
value = valueOf(int32);
|
||||
javaType = Types.INTEGER;
|
||||
length = len;
|
||||
break;
|
||||
@@ -1017,7 +1024,7 @@ public final class RowsLogBuffer {
|
||||
javaType = Types.VARBINARY;
|
||||
value = binary;
|
||||
} else {
|
||||
value = buffer.getFullString(len, charsetName);
|
||||
value = buffer.getFullString(len, charset);
|
||||
javaType = Types.VARCHAR;
|
||||
}
|
||||
|
||||
@@ -1039,7 +1046,7 @@ public final class RowsLogBuffer {
|
||||
javaType = Types.BINARY;
|
||||
value = binary;
|
||||
} else {
|
||||
value = buffer.getFullString(len, charsetName);
|
||||
value = buffer.getFullString(len, charset);
|
||||
javaType = Types.CHAR; // Types.VARCHAR;
|
||||
}
|
||||
length = len;
|
||||
@@ -1074,7 +1081,7 @@ public final class RowsLogBuffer {
|
||||
len,
|
||||
columnName,
|
||||
columnIndex,
|
||||
charsetName);
|
||||
charset);
|
||||
value = builder.toString();
|
||||
buffer.position(position + len);
|
||||
} else {
|
||||
@@ -1088,9 +1095,9 @@ public final class RowsLogBuffer {
|
||||
Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(),
|
||||
buffer,
|
||||
len - 1,
|
||||
charsetName);
|
||||
charset);
|
||||
StringBuilder builder = new StringBuilder();
|
||||
jsonValue.toJsonString(builder, charsetName);
|
||||
jsonValue.toJsonString(builder, charset);
|
||||
value = builder.toString();
|
||||
buffer.position(position + len);
|
||||
}
|
||||
@@ -1211,4 +1218,38 @@ public final class RowsLogBuffer {
|
||||
builder.append('0').append(digits[d]);
|
||||
}
|
||||
}
|
||||
|
||||
public static Long valueOf(long value) {
|
||||
if (value >= -128 & value <= 127) {
|
||||
// if (l >= -128 && l <= 127) {
|
||||
return Long.valueOf(value);
|
||||
}
|
||||
if (value > 127 && value < longCacheLimit) {
|
||||
int cacheIndex = (int) value - 127;
|
||||
Long cacheValue = longCache[cacheIndex];
|
||||
if (cacheValue == null) {
|
||||
cacheValue = new Long(value);
|
||||
longCache[cacheIndex] = cacheValue;
|
||||
}
|
||||
return cacheValue;
|
||||
}
|
||||
return new Long(value);
|
||||
}
|
||||
|
||||
public static Integer valueOf(int value) {
|
||||
if (value >= -128 & value <= 127) {
|
||||
// if (l >= -128 && l <= 127) {
|
||||
return Integer.valueOf(value);
|
||||
}
|
||||
if (value > 127 && value < integerCacheLimit) {
|
||||
int cacheIndex = (int) value - 127;
|
||||
Integer cacheValue = integerCache[cacheIndex];
|
||||
if (cacheValue == null) {
|
||||
cacheValue = new Integer(value);
|
||||
integerCache[cacheIndex] = cacheValue;
|
||||
}
|
||||
return cacheValue;
|
||||
}
|
||||
return new Integer(value);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.taobao.tddl.dbsync.binlog.event;
|
||||
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.BitSet;
|
||||
|
||||
import com.taobao.tddl.dbsync.binlog.exception.TableIdNotFoundException;
|
||||
@@ -219,8 +220,8 @@ public abstract class RowsLogEvent extends LogEvent {
|
||||
return changeColumns;
|
||||
}
|
||||
|
||||
public final RowsLogBuffer getRowsBuf(String charsetName) {
|
||||
return new RowsLogBuffer(rowsBuf, columnLen, charsetName, jsonColumnCount, partial);
|
||||
public final RowsLogBuffer getRowsBuf(Charset charset) {
|
||||
return new RowsLogBuffer(rowsBuf, columnLen, charset, jsonColumnCount, partial);
|
||||
}
|
||||
|
||||
public final int getFlags(final int flags) {
|
||||
|
||||
@@ -2,6 +2,8 @@ package com.taobao.tddl.dbsync.binlog.event;
|
||||
|
||||
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* @author jianghang 2013-4-8 上午12:36:29
|
||||
* @version 1.0.3
|
||||
@@ -23,7 +25,7 @@ public class RowsQueryLogEvent extends IgnorableLogEvent {
|
||||
*/
|
||||
int offset = commonHeaderLen + postHeaderLen + 1;
|
||||
int len = buffer.limit() - offset;
|
||||
rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1);
|
||||
rowsQuery = buffer.getFullString(offset, len, StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
|
||||
public String getRowsQuery() {
|
||||
|
||||
@@ -36,7 +36,7 @@ public class StartLogEventV3 extends LogEvent {
|
||||
|
||||
buffer.position(descriptionEvent.commonHeaderLen);
|
||||
binlogVersion = buffer.getUint16(); // ST_BINLOG_VER_OFFSET
|
||||
serverVersion = buffer.getFixString(ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
|
||||
serverVersion = buffer.getFixName(ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
|
||||
}
|
||||
|
||||
public StartLogEventV3(){
|
||||
|
||||
@@ -423,10 +423,10 @@ public final class TableMapLogEvent extends LogEvent {
|
||||
|
||||
/* Read the variable part of the event */
|
||||
buffer.position(commonHeaderLen + postHeaderLen);
|
||||
dbname = buffer.getString();
|
||||
dbname = buffer.getName();
|
||||
buffer.forward(1); /* termination null */
|
||||
// fixed issue #2714
|
||||
tblname = buffer.getString();
|
||||
tblname = buffer.getName();
|
||||
buffer.forward(1); /* termination null */
|
||||
|
||||
// Read column information from buffer
|
||||
|
||||
@@ -2,6 +2,7 @@ package com.taobao.tddl.dbsync.binlog.event;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
import java.nio.charset.Charset;
|
||||
|
||||
import com.taobao.tddl.dbsync.binlog.CharsetConversion;
|
||||
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||
@@ -100,8 +101,8 @@ public final class UserVarLogEvent extends LogEvent {
|
||||
value = buffer.getDecimal(precision, scale); // bin2decimal
|
||||
break;
|
||||
case STRING_RESULT:
|
||||
String charsetName = CharsetConversion.getJavaCharset(charsetNumber);
|
||||
value = buffer.getFixString(valueLen, charsetName);
|
||||
Charset charset = CharsetConversion.getNioCharset(charsetNumber);
|
||||
value = buffer.getFixString(valueLen, charset);
|
||||
break;
|
||||
case ROW_RESULT:
|
||||
// this seems to be banned in MySQL altogether
|
||||
|
||||
@@ -5,6 +5,8 @@ import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.LogHeader;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* mariadb的ANNOTATE_ROWS_EVENT类型
|
||||
*
|
||||
@@ -23,7 +25,7 @@ public class AnnotateRowsEvent extends IgnorableLogEvent {
|
||||
|
||||
int offset = commonHeaderLen + postHeaderLen;
|
||||
int len = buffer.limit() - offset;
|
||||
rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1);
|
||||
rowsQuery = buffer.getFullString(offset, len, StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
|
||||
public String getRowsQuery() {
|
||||
|
||||
@@ -52,7 +52,7 @@ public class BaseLogFetcherTest {
|
||||
event.getHeader().getLogPos() - event.getHeader().getEventLen(),
|
||||
event.getTable().getDbName(),
|
||||
event.getTable().getTableName()));
|
||||
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
|
||||
RowsLogBuffer buffer = event.getRowsBuf(charset);
|
||||
BitSet columns = event.getColumns();
|
||||
BitSet changeColumns = event.getChangeColumns();
|
||||
while (buffer.nextOneRow(columns)) {
|
||||
|
||||
@@ -92,9 +92,12 @@ else
|
||||
fi
|
||||
|
||||
if [ -n "$str" ]; then
|
||||
# JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
|
||||
# For G1
|
||||
JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
|
||||
if [ $JavaVersion -ge 11 ] ; then
|
||||
# For G1
|
||||
JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
|
||||
else
|
||||
JAVA_OPTS="-server -Xms2g -Xmx3g -Xmn1g -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
|
||||
fi
|
||||
else
|
||||
JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS"
|
||||
fi
|
||||
|
||||
@@ -558,6 +558,11 @@ public class CanalController {
|
||||
}
|
||||
|
||||
ZkClientx.clearClients();
|
||||
|
||||
// 需要释放 CanalServerWithEmbedded 否则主线程退出后，进程无法自动完整退出...
|
||||
if (embededCanalServer != null) {
|
||||
embededCanalServer.stop();
|
||||
}
|
||||
}
|
||||
|
||||
private void initCid(String path) {
|
||||
|
||||
@@ -148,8 +148,13 @@ kafka.max.in.flight.requests.per.connection = 1
|
||||
kafka.retries = 0
|
||||
|
||||
kafka.kerberos.enable = false
|
||||
kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
|
||||
kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
|
||||
kafka.kerberos.krb5.file = ../conf/kerberos/krb5.conf
|
||||
kafka.kerberos.jaas.file = ../conf/kerberos/jaas.conf
|
||||
|
||||
# sasl demo
|
||||
# kafka.sasl.jaas.config = org.apache.kafka.common.security.scram.ScramLoginModule required \\n username=\"alice\" \\npassword="alice-secret\";
|
||||
# kafka.sasl.mechanism = SCRAM-SHA-512
|
||||
# kafka.security.protocol = SASL_PLAINTEXT
|
||||
|
||||
##################################################
|
||||
######### RocketMQ #############
|
||||
@@ -179,4 +184,4 @@ rabbitmq.deliveryMode =
|
||||
##################################################
|
||||
pulsarmq.serverUrl =
|
||||
pulsarmq.roleToken =
|
||||
pulsarmq.topicTenantPrefix =
|
||||
pulsarmq.topicTenantPrefix =
|
||||
|
||||
@@ -181,9 +181,7 @@ public class UUIDSet {
|
||||
int j;
|
||||
for (j = i + 1; j < len; j++) {
|
||||
if (intervals.get(i).stop >= intervals.get(j).start) {
|
||||
if (intervals.get(i).stop < intervals.get(j).stop) {
|
||||
intervals.get(i).stop = intervals.get(j).stop;
|
||||
}
|
||||
intervals.get(i).stop = intervals.get(j).stop;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -42,10 +42,9 @@ public class MysqlGTIDSetTest {
|
||||
MysqlGTIDSet mysqlGTIDSet1 = MysqlGTIDSet.parse(gtid1);
|
||||
|
||||
String gtid2 = "726757ad-4455-11e8-ae04-0242ac110002:1-20304074";
|
||||
MysqlGTIDSet mysqlGTIDSet2 = MysqlGTIDSet.parse(gtid2);
|
||||
|
||||
mysqlGTIDSet1.update(gtid2);
|
||||
assertEquals("726757ad-4455-11e8-ae04-0242ac110002:1-25536412", mysqlGTIDSet1.toString());
|
||||
assertEquals("726757ad-4455-11e8-ae04-0242ac110002:1-20304074", mysqlGTIDSet1.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -71,9 +71,12 @@ else
|
||||
fi
|
||||
|
||||
if [ -n "$str" ]; then
|
||||
# JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
|
||||
# For G1
|
||||
JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
|
||||
if [ $JavaVersion -ge 11 ] ; then
|
||||
# For G1
|
||||
JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
|
||||
else
|
||||
JAVA_OPTS="-server -Xms2g -Xmx3g -Xmn1g -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
|
||||
fi
|
||||
else
|
||||
JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS"
|
||||
fi
|
||||
|
||||
@@ -109,11 +109,11 @@ public class AviaterRegexFilter implements CanalEventFilter<String> {
|
||||
private List<String> completionPattern(List<String> patterns) {
|
||||
List<String> result = new ArrayList<>();
|
||||
for (String pattern : patterns) {
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
stringBuffer.append("^");
|
||||
stringBuffer.append(pattern);
|
||||
stringBuffer.append("$");
|
||||
result.add(stringBuffer.toString());
|
||||
StringBuilder stringBuilder = new StringBuilder();
|
||||
stringBuilder.append("^");
|
||||
stringBuilder.append(pattern);
|
||||
stringBuilder.append("$");
|
||||
result.add(stringBuilder.toString());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
|
||||
import com.alibaba.otter.canal.common.CanalException;
|
||||
import com.alibaba.otter.canal.common.CanalLifeCycle;
|
||||
import com.alibaba.otter.canal.protocol.SecurityUtil;
|
||||
import com.google.common.net.UrlEscapers;
|
||||
|
||||
/**
|
||||
* 远程配置获取
|
||||
@@ -73,7 +74,7 @@ public class PlainCanalConfigClient extends AbstractCanalLifeCycle implements Ca
|
||||
md5 = "";
|
||||
}
|
||||
String url = configURL + "/api/v1/config/server_polling?ip=" + localIp + "&port=" + adminPort + "&md5=" + md5
|
||||
+ "®ister=" + (autoRegister ? 1 : 0) + "&cluster=" + autoCluster + "&name=" + name;
|
||||
+ "®ister=" + (autoRegister ? 1 : 0) + "&cluster=" + StringUtils.stripToEmpty(autoCluster) + "&name=" + StringUtils.stripToEmpty(name);
|
||||
return queryConfig(url);
|
||||
}
|
||||
|
||||
@@ -84,7 +85,7 @@ public class PlainCanalConfigClient extends AbstractCanalLifeCycle implements Ca
|
||||
if (StringUtils.isEmpty(md5)) {
|
||||
md5 = "";
|
||||
}
|
||||
String url = configURL + "/api/v1/config/instance_polling/" + destination + "?md5=" + md5;
|
||||
String url = configURL + "/api/v1/config/instance_polling/" + UrlEscapers.urlPathSegmentEscaper().escape(destination) + "?md5=" + md5;
|
||||
return queryConfig(url);
|
||||
}
|
||||
|
||||
|
||||
@@ -171,7 +171,7 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
|
||||
return null;
|
||||
}
|
||||
|
||||
String json = FileUtils.readFileToString(dataFile, charset.name());
|
||||
String json = FileUtils.readFileToString(dataFile, charset);
|
||||
return JsonUtils.unmarshalFromString(json, FileMetaInstanceData.class);
|
||||
} catch (IOException e) {
|
||||
throw new CanalMetaManagerException(e);
|
||||
@@ -209,7 +209,11 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
|
||||
|
||||
data.setClientDatas(clientDatas);
|
||||
}
|
||||
|
||||
//fixed issue https://github.com/alibaba/canal/issues/4312
|
||||
//客户端数据为空时不覆盖文件内容 （适合单客户端）
|
||||
if(data.getClientDatas().isEmpty()){
|
||||
return;
|
||||
}
|
||||
String json = JsonUtils.marshalToString(data);
|
||||
try {
|
||||
FileUtils.writeStringToFile(dataFile, json);
|
||||
|
||||
@@ -12,6 +12,7 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.taobao.tddl.dbsync.binlog.event.*;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.lang.exception.ExceptionUtils;
|
||||
import org.slf4j.Logger;
|
||||
@@ -40,23 +41,7 @@ import com.alibaba.otter.canal.protocol.CanalEntry.Type;
|
||||
import com.alibaba.otter.canal.protocol.position.EntryPosition;
|
||||
import com.google.protobuf.ByteString;
|
||||
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.GtidLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.HeartbeatLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.IntvarLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.LogHeader;
|
||||
import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.RandLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
|
||||
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
|
||||
import com.taobao.tddl.dbsync.binlog.event.UnknownLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.UserVarLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidListLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
|
||||
@@ -147,6 +132,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
|
||||
return parseGTIDLogEvent((GtidLogEvent) logEvent);
|
||||
case LogEvent.HEARTBEAT_LOG_EVENT:
|
||||
return parseHeartbeatLogEvent((HeartbeatLogEvent) logEvent);
|
||||
case LogEvent.HEARTBEAT_LOG_EVENT_V2:
|
||||
return parseHeartbeatV2LogEvent((HeartbeatV2LogEvent) logEvent);
|
||||
case LogEvent.GTID_EVENT:
|
||||
case LogEvent.GTID_LIST_EVENT:
|
||||
return parseMariaGTIDLogEvent(logEvent);
|
||||
@@ -173,13 +160,22 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
|
||||
return entryBuilder.build();
|
||||
}
|
||||
|
||||
private Entry parseHeartbeatV2LogEvent(HeartbeatV2LogEvent logEvent) {
|
||||
Header.Builder headerBuilder = Header.newBuilder();
|
||||
headerBuilder.setEventType(EventType.MHEARTBEAT);
|
||||
Entry.Builder entryBuilder = Entry.newBuilder();
|
||||
entryBuilder.setHeader(headerBuilder.build());
|
||||
entryBuilder.setEntryType(EntryType.HEARTBEAT);
|
||||
return entryBuilder.build();
|
||||
}
|
||||
|
||||
private Entry parseGTIDLogEvent(GtidLogEvent logEvent) {
|
||||
LogHeader logHeader = logEvent.getHeader();
|
||||
Pair.Builder builder = Pair.newBuilder();
|
||||
builder.setKey("gtid");
|
||||
builder.setValue(logEvent.getGtidStr());
|
||||
|
||||
if (logEvent.getLastCommitted() != null) {
|
||||
if (logEvent.getLastCommitted() != -1) {
|
||||
builder.setKey("lastCommitted");
|
||||
builder.setValue(String.valueOf(logEvent.getLastCommitted()));
|
||||
builder.setKey("sequenceNumber");
|
||||
@@ -396,7 +392,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
|
||||
// mysql5.6支持，需要设置binlog-rows-query-log-events=1，可详细打印原始DML语句
|
||||
String queryString = null;
|
||||
try {
|
||||
queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
|
||||
queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset);
|
||||
String tableName = null;
|
||||
if (useDruidDdlFilter) {
|
||||
List<DdlResult> results = DruidDdlParser.parse(queryString, null);
|
||||
@@ -418,7 +414,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
|
||||
// mariaDb支持，需要设置binlog_annotate_row_events=true，可详细打印原始DML语句
|
||||
String queryString = null;
|
||||
try {
|
||||
queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
|
||||
queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset);
|
||||
return buildQueryEntry(queryString, event.getHeader());
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new CanalParseException(e);
|
||||
@@ -512,10 +508,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
|
||||
|
||||
public void parseTableMapEvent(TableMapLogEvent event) {
|
||||
try {
|
||||
String charsetDbName = new String(event.getDbName().getBytes(ISO_8859_1), charset.name());
|
||||
String charsetDbName = new String(event.getDbName().getBytes(ISO_8859_1), charset);
|
||||
event.setDbname(charsetDbName);
|
||||
|
||||
String charsetTbName = new String(event.getTableName().getBytes(ISO_8859_1), charset.name());
|
||||
String charsetTbName = new String(event.getTableName().getBytes(ISO_8859_1), charset);
|
||||
event.setTblname(charsetTbName);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new CanalParseException(e);
|
||||
@@ -554,7 +550,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
|
||||
rowChangeBuider.setIsDdl(false);
|
||||
|
||||
rowChangeBuider.setEventType(eventType);
|
||||
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
|
||||
RowsLogBuffer buffer = event.getRowsBuf(charset);
|
||||
BitSet columns = event.getColumns();
|
||||
BitSet changeColumns = event.getChangeColumns();
|
||||
|
||||
|
||||
@@ -100,7 +100,7 @@ public class DdlResult {
|
||||
@Override
|
||||
public String toString() {
|
||||
DdlResult ddlResult = this;
|
||||
StringBuffer sb = new StringBuffer();
|
||||
StringBuilder sb = new StringBuilder();
|
||||
do {
|
||||
sb.append(String.format("DdlResult [schemaName=%s , tableName=%s , oriSchemaName=%s , oriTableName=%s , type=%s ];",
|
||||
ddlResult.schemaName,
|
||||
|
||||
@@ -209,6 +209,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
|
||||
for (String schema : schemas) {
|
||||
// filter views
|
||||
packet = connection.query("show full tables from `" + schema + "` where Table_type = 'BASE TABLE'");
|
||||
columnSize = packet.getFieldDescriptors().size();
|
||||
int tableNameColumnIndex = 0; // default index is 0
|
||||
List<String> tables = new ArrayList<>();
|
||||
for (int line = 0; line < packet.getFieldValues().size() / columnSize; line++) {
|
||||
|
||||
@@ -80,7 +80,8 @@ public class MemoryTableMeta implements TableMetaTSDB {
|
||||
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create user")
|
||||
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "alter user")
|
||||
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "drop user")
|
||||
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create database")) {
|
||||
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create database")
|
||||
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "alter schema")) {
|
||||
repository.console(ddl);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
@@ -145,7 +146,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
|
||||
public Map<String, String> snapshot() {
|
||||
Map<String, String> schemaDdls = new HashMap<>();
|
||||
for (Schema schema : repository.getSchemas()) {
|
||||
StringBuffer data = new StringBuffer(4 * 1024);
|
||||
StringBuilder data = new StringBuilder(4 * 1024);
|
||||
for (String table : schema.showTables()) {
|
||||
SchemaObject schemaObject = schema.findTable(table);
|
||||
schemaObject.getStatement().output(data);
|
||||
|
||||
@@ -182,7 +182,7 @@ public class FileMixedLogPositionManager extends AbstractLogPositionManager {
|
||||
return null;
|
||||
}
|
||||
|
||||
String json = FileUtils.readFileToString(dataFile, charset.name());
|
||||
String json = FileUtils.readFileToString(dataFile, charset);
|
||||
return JsonUtils.unmarshalFromString(json, LogPosition.class);
|
||||
} catch (IOException e) {
|
||||
throw new CanalMetaManagerException(e);
|
||||
|
||||
@@ -302,7 +302,7 @@ public class DirectLogFetcherTest {
|
||||
event.getHeader().getLogPos() - event.getHeader().getEventLen(),
|
||||
event.getTable().getDbName(),
|
||||
event.getTable().getTableName()));
|
||||
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
|
||||
RowsLogBuffer buffer = event.getRowsBuf(charset);
|
||||
BitSet columns = event.getColumns();
|
||||
BitSet changeColumns = event.getChangeColumns();
|
||||
while (buffer.nextOneRow(columns)) {
|
||||
|
||||
@@ -134,7 +134,7 @@ public class MysqlBinlogParsePerformanceTest {
|
||||
|
||||
public static void parseRowsEvent(RowsLogEvent event, AtomicLong sum) {
|
||||
try {
|
||||
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
|
||||
RowsLogBuffer buffer = event.getRowsBuf(charset);
|
||||
BitSet columns = event.getColumns();
|
||||
BitSet changeColumns = event.getChangeColumns();
|
||||
while (buffer.nextOneRow(columns)) {
|
||||
|
||||
6
pom.xml
6
pom.xml
@@ -58,7 +58,7 @@
|
||||
</repository>
|
||||
<repository>
|
||||
<id>java.net</id>
|
||||
<url>http://download.java.net/maven/2/</url>
|
||||
<url>https://download.java.net/maven/2/</url>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
@@ -222,7 +222,7 @@
|
||||
<dependency>
|
||||
<groupId>com.alibaba.fastjson2</groupId>
|
||||
<artifactId>fastjson2</artifactId>
|
||||
<version>2.0.7</version>
|
||||
<version>2.0.15</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
@@ -262,7 +262,7 @@
|
||||
<dependency>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>druid</artifactId>
|
||||
<version>1.2.11</version>
|
||||
<version>1.2.13</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lmax</groupId>
|
||||
|
||||
Reference in New Issue
Block a user