From f3bb0dd18d510b6c399cc08fb23c642f99d5f238 Mon Sep 17 00:00:00 2001 From: rewerma <29311598@qq.com> Date: Tue, 6 Dec 2022 16:41:17 +0800 Subject: [PATCH] =?UTF-8?q?Revert=20"=E4=BF=AE=E5=A4=8D=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=B1=BB=E6=8A=A5=E9=94=99=20(#4516)"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit d8999813544a06235dca13539f1d8e571b9600aa. --- README.md | 1 - .../views/canalServer/CanalInstanceUpdate.vue | 2 +- admin/admin-web/src/main/bin/startup.sh | 9 +- .../main/resources/canal-template.properties | 25 +- client-adapter/common/pom.xml | 29 +- .../adapter/config/YmlConfigBinder.java | 155 ++++ .../DefaultPropertyNamePatternsMatcher.java | 97 +++ .../config/bind/InetAddressEditor.java | 31 + .../bind/OriginCapablePropertyValue.java | 52 ++ .../PatternPropertyNamePatternsMatcher.java | 27 + .../bind/PropertiesConfigurationFactory.java | 356 +++++++++ .../bind/PropertyNamePatternsMatcher.java | 24 + .../adapter/config/bind/PropertyOrigin.java | 30 + .../bind/PropertySourcesPropertyResolver.java | 164 ++++ .../bind/PropertySourcesPropertyValues.java | 233 ++++++ .../config/bind/RelaxedConversionService.java | 127 +++ .../config/bind/RelaxedDataBinder.java | 729 ++++++++++++++++++ .../adapter/config/bind/RelaxedNames.java | 241 ++++++ .../bind/StringToCharArrayConverter.java | 17 + .../config/common/AbstractResource.java | 203 +++++ .../config/common/ByteArrayResource.java | 117 +++ .../common/CompositePropertySource.java | 107 +++ .../common/EnumerablePropertySource.java | 58 ++ .../config/common/MapPropertySource.java | 38 + .../config/common/MutablePropertySources.java | 221 ++++++ .../common/PropertiesPropertySource.java | 34 + .../adapter/config/common/PropertySource.java | 239 ++++++ .../config/common/PropertySourceLoader.java | 35 + .../config/common/PropertySources.java | 25 + .../adapter/config/common/Resource.java | 57 ++ .../common/SpringProfileDocumentMatcher.java | 182 +++++ .../adapter/config/common/YamlProcessor.java | 417 ++++++++++ .../common/YamlPropertySourceLoader.java | 80 ++ .../client/adapter/support/YamlUtils.java | 98 --- .../client/adapter/support/YamlUtilsTest.java | 229 ------ .../es/core/config/ESSyncConfigLoader.java | 4 +- .../es/core/monitor/ESConfigMonitor.java | 18 +- .../hbase/config/MappingConfigLoader.java | 5 +- .../hbase/monitor/HbaseConfigMonitor.java | 8 +- .../kudu/config/KuduMappingConfigLoader.java | 8 +- .../kudu/monitor/KuduConfigMonitor.java | 25 +- .../launcher/src/main/bin/startup.sh | 9 +- .../adapter/phoenix/config/ConfigLoader.java | 5 +- .../phoenix/monitor/PhoenixConfigMonitor.java | 27 +- client-adapter/pom.xml | 38 +- .../adapter/rdb/config/ConfigLoader.java | 5 +- .../adapter/rdb/monitor/RdbConfigMonitor.java | 19 +- .../adapter/rdb/service/RdbEtlService.java | 11 +- .../tablestore/config/ConfigLoader.java | 8 +- .../running/ServerRunningMonitor.java | 9 +- .../core/filter/AviaterRegexFilter.java | 10 +- connector/pom.xml | 6 +- .../tddl/dbsync/binlog/CharsetConversion.java | 19 - .../tddl/dbsync/binlog/JsonConversion.java | 42 +- .../dbsync/binlog/JsonDiffConversion.java | 10 +- .../taobao/tddl/dbsync/binlog/LogBuffer.java | 363 ++------- .../taobao/tddl/dbsync/binlog/LogDecoder.java | 67 +- .../taobao/tddl/dbsync/binlog/LogEvent.java | 23 +- .../taobao/tddl/dbsync/binlog/NameCache.java | 1 - .../dbsync/binlog/event/GtidLogEvent.java | 17 +- .../binlog/event/HeartbeatLogEvent.java | 4 +- .../binlog/event/HeartbeatV2LogEvent.java | 40 - .../dbsync/binlog/event/IncidentLogEvent.java | 2 +- .../dbsync/binlog/event/QueryLogEvent.java | 62 +- .../dbsync/binlog/event/RowsLogBuffer.java | 71 +- .../dbsync/binlog/event/RowsLogEvent.java | 5 +- .../binlog/event/RowsQueryLogEvent.java | 4 +- .../dbsync/binlog/event/StartLogEventV3.java | 2 +- .../dbsync/binlog/event/TableMapLogEvent.java | 4 +- .../dbsync/binlog/event/UserVarLogEvent.java | 5 +- .../event/mariadb/AnnotateRowsEvent.java | 4 +- .../dbsync/binlog/BaseLogFetcherTest.java | 2 +- deployer/src/main/bin/startup.sh | 9 +- .../otter/canal/deployer/CanalController.java | 5 - deployer/src/main/resources/canal.properties | 11 +- .../parse/driver/mysql/packets/UUIDSet.java | 4 +- .../parse/driver/mysql/MysqlGTIDSetTest.java | 3 +- example/src/main/bin/startup.sh | 9 +- .../filter/aviater/AviaterRegexFilter.java | 10 +- .../manager/plain/PlainCanalConfigClient.java | 5 +- .../canal/meta/FileMixedMetaManager.java | 8 +- .../inbound/mysql/dbsync/LogEventConvert.java | 40 +- .../parse/inbound/mysql/ddl/DdlResult.java | 2 +- .../inbound/mysql/tsdb/DatabaseTableMeta.java | 1 - .../inbound/mysql/tsdb/MemoryTableMeta.java | 5 +- .../index/FileMixedLogPositionManager.java | 2 +- .../canal/parse/DirectLogFetcherTest.java | 2 +- .../MysqlBinlogParsePerformanceTest.java | 2 +- pom.xml | 6 +- 89 files changed, 4470 insertions(+), 1105 deletions(-) create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java delete mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/YamlUtils.java delete mode 100644 client-adapter/common/src/test/java/com/alibaba/otter/canal/client/adapter/support/YamlUtilsTest.java delete mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/NameCache.java delete mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatV2LogEvent.java diff --git a/README.md b/README.md index f3e29051..b00d4a58 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,6 @@ canal 特别设计了 client-server 模式，交互协议使用 protobuf 3.0 , c - canal php客户端: [https://github.com/xingwenge/canal-php](https://github.com/xingwenge/canal-php) - canal Python客户端：[https://github.com/haozi3156666/canal-python](https://github.com/haozi3156666/canal-python) - canal Rust客户端：[https://github.com/laohanlinux/canal-rs](https://github.com/laohanlinux/canal-rs) -- canal Nodejs客户端：[https://github.com/marmot-z/canal-nodejs](https://github.com/marmot-z/canal-nodejs) canal 作为 MySQL binlog 增量获取和解析工具，可将变更记录投递到 MQ 系统中，比如 Kafka/RocketMQ，可以借助于 MQ 的多语言能力 diff --git a/admin/admin-ui/src/views/canalServer/CanalInstanceUpdate.vue b/admin/admin-ui/src/views/canalServer/CanalInstanceUpdate.vue index 814ff44c..cdb168f2 100644 --- a/admin/admin-ui/src/views/canalServer/CanalInstanceUpdate.vue +++ b/admin/admin-ui/src/views/canalServer/CanalInstanceUpdate.vue @@ -59,7 +59,7 @@ export default { canalInstanceDetail(this.$route.query.id).then(response => { const data = response.data this.form.id = data.id - this.form.name = data.name + '/instance.properties' + this.form.name = data.name + '/instance.propertios' this.form.content = data.content this.form.clusterServerId = data.clusterServerId }) diff --git a/admin/admin-web/src/main/bin/startup.sh b/admin/admin-web/src/main/bin/startup.sh index f631cbbe..2a6de188 100644 --- a/admin/admin-web/src/main/bin/startup.sh +++ b/admin/admin-web/src/main/bin/startup.sh @@ -69,12 +69,9 @@ else fi if [ -n "$str" ]; then - if [ $JavaVersion -ge 11 ] ; then - # For G1 - JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS" - else - JAVA_OPTS="-server -Xms2g -Xmx3g -Xmn1g -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS" - fi + # JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS" + # For G1 + JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS" else JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS" fi diff --git a/admin/admin-web/src/main/resources/canal-template.properties b/admin/admin-web/src/main/resources/canal-template.properties index 477bb957..9655f1e2 100644 --- a/admin/admin-web/src/main/resources/canal-template.properties +++ b/admin/admin-web/src/main/resources/canal-template.properties @@ -25,7 +25,7 @@ canal.zkServers = # flush data to zk canal.zookeeper.flush.period = 1000 canal.withoutNetty = false -# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ +# tcp, kafka, rocketMQ, rabbitMQ canal.serverMode = tcp # flush meta cursor/parse position to file canal.file.data.dir = ${canal.conf.dir} @@ -96,7 +96,7 @@ canal.instance.tsdb.snapshot.expire = 360 ################################################# ######### destinations ############# ################################################# -canal.destinations = +canal.destinations = # conf root dir canal.conf.dir = ../conf # auto scan instance dir add/remove and start/stop instance @@ -148,13 +148,8 @@ kafka.max.in.flight.requests.per.connection = 1 kafka.retries = 0 kafka.kerberos.enable = false -kafka.kerberos.krb5.file = ../conf/kerberos/krb5.conf -kafka.kerberos.jaas.file = ../conf/kerberos/jaas.conf - -# sasl demo -# kafka.sasl.jaas.config = org.apache.kafka.common.security.scram.ScramLoginModule required \\n username=\"alice\" \\npassword="alice-secret\"; -# kafka.sasl.mechanism = SCRAM-SHA-512 -# kafka.security.protocol = SASL_PLAINTEXT +kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf" +kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf" ################################################## ######### RocketMQ ############# @@ -166,7 +161,7 @@ rocketmq.namespace = rocketmq.namesrv.addr = 127.0.0.1:9876 rocketmq.retry.times.when.send.failed = 0 rocketmq.vip.channel.enabled = false -rocketmq.tag = +rocketmq.tag = ################################################## ######### RabbitMQ ############# @@ -176,12 +171,4 @@ rabbitmq.virtual.host = rabbitmq.exchange = rabbitmq.username = rabbitmq.password = -rabbitmq.deliveryMode = - - -################################################## -######### Pulsar ############# -################################################## -pulsarmq.serverUrl = -pulsarmq.roleToken = -pulsarmq.topicTenantPrefix = +rabbitmq.deliveryMode = \ No newline at end of file diff --git a/client-adapter/common/pom.xml b/client-adapter/common/pom.xml index 21213cf0..af213c24 100644 --- a/client-adapter/common/pom.xml +++ b/client-adapter/common/pom.xml @@ -23,8 +23,29 @@ ${project.version} - org.springframework.boot - spring-boot + org.springframework + spring-core + + + org.springframework + spring-aop + + + org.springframework + spring-context + + + org.springframework + spring-jdbc + + + org.springframework + spring-orm + + + org.springframework + spring-test + test joda-time @@ -34,6 +55,10 @@ com.alibaba druid + + org.yaml + snakeyaml + org.powermock powermock-api-mockito diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java new file mode 100644 index 00000000..59a48d61 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java @@ -0,0 +1,155 @@ +package com.alibaba.otter.canal.client.adapter.config; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +import org.springframework.util.PropertyPlaceholderHelper; +import org.springframework.util.StringUtils; + +import com.alibaba.otter.canal.client.adapter.config.bind.PropertiesConfigurationFactory; +import com.alibaba.otter.canal.client.adapter.config.common.*; + +/** + * 将yaml内容绑定到指定对象, 遵循spring yml的绑定规范 + * + * @author reweerma 2019-2-1 上午9:14:02 + * @version 1.0.0 + */ +public class YmlConfigBinder { + + /** + * 将当前内容绑定到指定对象 + * + * @param content yml内容 + * @param clazz 指定对象类型 + * @return 对象 + */ + public static T bindYmlToObj(String content, Class clazz) { + return bindYmlToObj(null, content, clazz, null); + } + + /** + * 将当前内容绑定到指定对象并指定内容编码格式 + * + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String content, Class clazz, String charset) { + return bindYmlToObj(null, content, clazz, charset); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象 + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz) { + return bindYmlToObj(prefix, content, clazz, null); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象并指定内容编码格式 + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz, String charset) { + return bindYmlToObj(prefix, content, clazz, charset, null); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: 当前内容有属性 zkServers: ${zookeeper.servers} + * 在envProperties中有属性 zookeeper.servers: + * 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 则当前内容 zkServers 会被替换为 + * zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 注: 假设绑定的类中 + * zkServers 属性是 List 对象, 则会自动映射成List + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz, String charset, + Properties baseProperties) { + try { + byte[] contentBytes; + if (charset == null) { + contentBytes = content.getBytes("UTF-8"); + } else { + contentBytes = content.getBytes(charset); + } + YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader(); + Resource configResource = new ByteArrayResource(contentBytes); + PropertySource propertySource = propertySourceLoader.load("manualBindConfig", configResource, null); + + if (propertySource == null) { + return null; + } + + Properties properties = new Properties(); + Map propertiesRes = new LinkedHashMap<>(); + if (!StringUtils.isEmpty(prefix) && !prefix.endsWith(".")) { + prefix = prefix + "."; + } + + properties.putAll((Map) propertySource.getSource()); + + if (baseProperties != null) { + baseProperties.putAll(properties); + properties = baseProperties; + } + + for (Map.Entry entry : ((Map) propertySource.getSource()).entrySet()) { + String key = (String) entry.getKey(); + Object value = entry.getValue(); + + if (prefix != null) { + if (key != null && key.startsWith(prefix)) { + key = key.substring(prefix.length()); + } else { + continue; + } + } + + if (value instanceof String && ((String) value).contains("${") && ((String) value).contains("}")) { + PropertyPlaceholderHelper propertyPlaceholderHelper = new PropertyPlaceholderHelper("${", "}"); + value = propertyPlaceholderHelper.replacePlaceholders((String) value, properties); + } + + propertiesRes.put(key, value); + } + + if (propertiesRes.isEmpty()) { + return null; + } + + propertySource = new MapPropertySource(propertySource.getName(), propertiesRes); + + T target = clazz.newInstance(); + + MutablePropertySources propertySources = new MutablePropertySources(); + propertySources.addFirst(propertySource); + + PropertiesConfigurationFactory factory = new PropertiesConfigurationFactory<>(target); + factory.setPropertySources(propertySources); + factory.setIgnoreInvalidFields(true); + factory.setIgnoreUnknownFields(true); + + factory.bindPropertiesToTarget(); + + return target; + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java new file mode 100644 index 00000000..289eda97 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java @@ -0,0 +1,97 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * {@link PropertyNamePatternsMatcher} that matches when a property name exactly + * matches one of the given names, or starts with one of the given names + * followed by a delimiter. This implementation is optimized for frequent calls. + * + * @author Phillip Webb + * @since 1.2.0 + */ +class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher { + + private final char[] delimiters; + + private final boolean ignoreCase; + + private final String[] names; + + protected DefaultPropertyNamePatternsMatcher(char[] delimiters, String... names){ + this(delimiters, false, names); + } + + protected DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, String... names){ + this(delimiters, ignoreCase, new HashSet<>(Arrays.asList(names))); + } + + DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, Set names){ + this.delimiters = delimiters; + this.ignoreCase = ignoreCase; + this.names = names.toArray(new String[names.size()]); + } + + @Override + public boolean matches(String propertyName) { + char[] propertyNameChars = propertyName.toCharArray(); + boolean[] match = new boolean[this.names.length]; + boolean noneMatched = true; + for (int i = 0; i < this.names.length; i++) { + if (this.names[i].length() <= propertyNameChars.length) { + match[i] = true; + noneMatched = false; + } + } + if (noneMatched) { + return false; + } + for (int charIndex = 0; charIndex < propertyNameChars.length; charIndex++) { + for (int nameIndex = 0; nameIndex < this.names.length; nameIndex++) { + if (match[nameIndex]) { + match[nameIndex] = false; + if (charIndex < this.names[nameIndex].length()) { + if (isCharMatch(this.names[nameIndex].charAt(charIndex), propertyNameChars[charIndex])) { + match[nameIndex] = true; + noneMatched = false; + } + } else { + char charAfter = propertyNameChars[this.names[nameIndex].length()]; + if (isDelimiter(charAfter)) { + match[nameIndex] = true; + noneMatched = false; + } + } + } + } + if (noneMatched) { + return false; + } + } + for (boolean b : match) { + if (b) { + return true; + } + } + return false; + } + + private boolean isCharMatch(char c1, char c2) { + if (this.ignoreCase) { + return Character.toLowerCase(c1) == Character.toLowerCase(c2); + } + return c1 == c2; + } + + private boolean isDelimiter(char c) { + for (char delimiter : this.delimiters) { + if (c == delimiter) { + return true; + } + } + return false; + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java new file mode 100644 index 00000000..c9b259e3 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java @@ -0,0 +1,31 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.beans.PropertyEditorSupport; +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * {@link PropertyNamePatternsMatcher} that matches when a property name exactly + * matches one of the given names, or starts with one of the given names + * followed by a delimiter. This implementation is optimized for frequent calls. + * + * @author Phillip Webb + * @since 1.2.0 + */ +class InetAddressEditor extends PropertyEditorSupport { + + @Override + public String getAsText() { + return ((InetAddress) getValue()).getHostAddress(); + } + + @Override + public void setAsText(String text) throws IllegalArgumentException { + try { + setValue(InetAddress.getByName(text)); + } catch (UnknownHostException ex) { + throw new IllegalArgumentException("Cannot locate host", ex); + } + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java new file mode 100644 index 00000000..b79a03e8 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java @@ -0,0 +1,52 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import org.springframework.beans.PropertyValue; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; + +/** + * A {@link PropertyValue} that can provide information about its origin. + * + * @author Andy Wilkinson + */ +class OriginCapablePropertyValue extends PropertyValue { + + private static final String ATTRIBUTE_PROPERTY_ORIGIN = "propertyOrigin"; + + private final PropertyOrigin origin; + + OriginCapablePropertyValue(PropertyValue propertyValue){ + this(propertyValue.getName(), + propertyValue.getValue(), + (PropertyOrigin) propertyValue.getAttribute(ATTRIBUTE_PROPERTY_ORIGIN)); + } + + OriginCapablePropertyValue(String name, Object value, String originName, PropertySource originSource){ + this(name, value, new PropertyOrigin(originSource, originName)); + } + + OriginCapablePropertyValue(String name, Object value, PropertyOrigin origin){ + super(name, value); + this.origin = origin; + setAttribute(ATTRIBUTE_PROPERTY_ORIGIN, origin); + } + + public PropertyOrigin getOrigin() { + return this.origin; + } + + @Override + public String toString() { + String name = (this.origin != null ? this.origin.getName() : this.getName()); + String source = (this.origin.getSource() != null ? this.origin.getSource().getName() : "unknown"); + return "'" + name + "' from '" + source + "'"; + } + + public static PropertyOrigin getOrigin(PropertyValue propertyValue) { + if (propertyValue instanceof OriginCapablePropertyValue) { + return ((OriginCapablePropertyValue) propertyValue).getOrigin(); + } + return new OriginCapablePropertyValue(propertyValue).getOrigin(); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java new file mode 100644 index 00000000..a041a673 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java @@ -0,0 +1,27 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Collection; + +import org.springframework.util.PatternMatchUtils; + +/** + * {@link PropertyNamePatternsMatcher} that delegates to + * {@link PatternMatchUtils#simpleMatch(String[], String)}. + * + * @author Phillip Webb + * @since 1.2.0 + */ +class PatternPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher { + + private final String[] patterns; + + PatternPropertyNamePatternsMatcher(Collection patterns){ + this.patterns = (patterns != null ? patterns.toArray(new String[patterns.size()]) : new String[] {}); + } + + @Override + public boolean matches(String propertyName) { + return PatternMatchUtils.simpleMatch(this.patterns, propertyName); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java new file mode 100644 index 00000000..e8162efa --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java @@ -0,0 +1,356 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.beans.PropertyDescriptor; +import java.util.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.PropertyValues; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.support.ResourceEditorRegistrar; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.MessageSource; +import org.springframework.context.MessageSourceAware; +import org.springframework.core.convert.ConversionService; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.validation.*; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySources; + +/** + * Validate some {@link Properties} (or optionally + * {@link org.springframework.core.env.PropertySources}) by binding them to an + * object of a specified type and then optionally running a {@link Validator} + * over it. + * + * @param the target type + * @author Dave Syer + */ +public class PropertiesConfigurationFactory implements FactoryBean, ApplicationContextAware, MessageSourceAware, InitializingBean { + + private static final char[] EXACT_DELIMITERS = { '_', '.', '[' }; + + private static final char[] TARGET_NAME_DELIMITERS = { '_', '.' }; + + private static final Log logger = LogFactory.getLog(PropertiesConfigurationFactory.class); + + private boolean ignoreUnknownFields = true; + + private boolean ignoreInvalidFields; + + private boolean exceptionIfInvalid = true; + + private PropertySources propertySources; + + private final T target; + + private Validator validator; + + private ApplicationContext applicationContext; + + private MessageSource messageSource; + + private boolean hasBeenBound = false; + + private boolean ignoreNestedProperties = false; + + private String targetName; + + private ConversionService conversionService; + + private boolean resolvePlaceholders = true; + + /** + * Create a new {@link PropertiesConfigurationFactory} instance. + * + * @param target the target object to bind too + * @see #PropertiesConfigurationFactory(Class) + */ + public PropertiesConfigurationFactory(T target){ + Assert.notNull(target, "target must not be null"); + this.target = target; + } + + /** + * Create a new {@link PropertiesConfigurationFactory} instance. + * + * @param type the target type + * @see #PropertiesConfigurationFactory(Class) + */ + @SuppressWarnings("unchecked") + public PropertiesConfigurationFactory(Class type){ + Assert.notNull(type, "type must not be null"); + this.target = (T) BeanUtils.instantiate(type); + } + + /** + * Flag to disable binding of nested properties (i.e. those with period + * separators in their paths). Can be useful to disable this if the name prefix + * is empty and you don't want to ignore unknown fields. + * + * @param ignoreNestedProperties the flag to set (default false) + */ + public void setIgnoreNestedProperties(boolean ignoreNestedProperties) { + this.ignoreNestedProperties = ignoreNestedProperties; + } + + /** + * Set whether to ignore unknown fields, that is, whether to ignore bind + * parameters that do not have corresponding fields in the target object. + *

+ * Default is "true". Turn this off to enforce that all bind parameters must + * have a matching field in the target object. + * + * @param ignoreUnknownFields if unknown fields should be ignored + */ + public void setIgnoreUnknownFields(boolean ignoreUnknownFields) { + this.ignoreUnknownFields = ignoreUnknownFields; + } + + /** + * Set whether to ignore invalid fields, that is, whether to ignore bind + * parameters that have corresponding fields in the target object which are not + * accessible (for example because of null values in the nested path). + *

+ * Default is "false". Turn this on to ignore bind parameters for nested objects + * in non-existing parts of the target object graph. + * + * @param ignoreInvalidFields if invalid fields should be ignored + */ + public void setIgnoreInvalidFields(boolean ignoreInvalidFields) { + this.ignoreInvalidFields = ignoreInvalidFields; + } + + /** + * Set the target name. + * + * @param targetName the target name + */ + public void setTargetName(String targetName) { + this.targetName = targetName; + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) { + this.applicationContext = applicationContext; + } + + /** + * Set the message source. + * + * @param messageSource the message source + */ + @Override + public void setMessageSource(MessageSource messageSource) { + this.messageSource = messageSource; + } + + /** + * Set the property sources. + * + * @param propertySources the property sources + */ + public void setPropertySources(PropertySources propertySources) { + this.propertySources = propertySources; + } + + /** + * Set the conversion service. + * + * @param conversionService the conversion service + */ + public void setConversionService(ConversionService conversionService) { + this.conversionService = conversionService; + } + + /** + * Set the validator. + * + * @param validator the validator + */ + public void setValidator(Validator validator) { + this.validator = validator; + } + + /** + * Set a flag to indicate that an exception should be raised if a Validator is + * available and validation fails. + * + * @param exceptionIfInvalid the flag to set + * @deprecated as of 1.5, do not specify a {@link Validator} if validation + * should not occur + */ + @Deprecated + public void setExceptionIfInvalid(boolean exceptionIfInvalid) { + this.exceptionIfInvalid = exceptionIfInvalid; + } + + /** + * Flag to indicate that placeholders should be replaced during binding. Default + * is true. + * + * @param resolvePlaceholders flag value + */ + public void setResolvePlaceholders(boolean resolvePlaceholders) { + this.resolvePlaceholders = resolvePlaceholders; + } + + @Override + public void afterPropertiesSet() throws Exception { + bindPropertiesToTarget(); + } + + @Override + public Class getObjectType() { + if (this.target == null) { + return Object.class; + } + return this.target.getClass(); + } + + @Override + public boolean isSingleton() { + return true; + } + + @Override + public T getObject() throws Exception { + if (!this.hasBeenBound) { + bindPropertiesToTarget(); + } + return this.target; + } + + public void bindPropertiesToTarget() throws BindException { + Assert.state(this.propertySources != null, "PropertySources should not be null"); + try { + if (logger.isTraceEnabled()) { + logger.trace("Property Sources: " + this.propertySources); + + } + this.hasBeenBound = true; + doBindPropertiesToTarget(); + } catch (BindException ex) { + if (this.exceptionIfInvalid) { + throw ex; + } + logger.error("Failed to load Properties validation bean. " + "Your Properties may be invalid.", ex); + } + } + + private void doBindPropertiesToTarget() throws BindException { + RelaxedDataBinder dataBinder = (this.targetName != null ? new RelaxedDataBinder(this.target, + this.targetName) : new RelaxedDataBinder(this.target)); + if (this.validator != null && this.validator.supports(dataBinder.getTarget().getClass())) { + dataBinder.setValidator(this.validator); + } + if (this.conversionService != null) { + dataBinder.setConversionService(this.conversionService); + } + dataBinder.setAutoGrowCollectionLimit(Integer.MAX_VALUE); + dataBinder.setIgnoreNestedProperties(this.ignoreNestedProperties); + dataBinder.setIgnoreInvalidFields(this.ignoreInvalidFields); + dataBinder.setIgnoreUnknownFields(this.ignoreUnknownFields); + customizeBinder(dataBinder); + if (this.applicationContext != null) { + ResourceEditorRegistrar resourceEditorRegistrar = new ResourceEditorRegistrar(this.applicationContext, + this.applicationContext.getEnvironment()); + resourceEditorRegistrar.registerCustomEditors(dataBinder); + } + Iterable relaxedTargetNames = getRelaxedTargetNames(); + Set names = getNames(relaxedTargetNames); + PropertyValues propertyValues = getPropertySourcesPropertyValues(names, relaxedTargetNames); + dataBinder.bind(propertyValues); + if (this.validator != null) { + dataBinder.validate(); + } + checkForBindingErrors(dataBinder); + } + + private Iterable getRelaxedTargetNames() { + return (this.target != null + && StringUtils.hasLength(this.targetName) ? new RelaxedNames(this.targetName) : null); + } + + private Set getNames(Iterable prefixes) { + Set names = new LinkedHashSet<>(); + if (this.target != null) { + PropertyDescriptor[] descriptors = BeanUtils.getPropertyDescriptors(this.target.getClass()); + for (PropertyDescriptor descriptor : descriptors) { + String name = descriptor.getName(); + if (!name.equals("class")) { + RelaxedNames relaxedNames = RelaxedNames.forCamelCase(name); + if (prefixes == null) { + for (String relaxedName : relaxedNames) { + names.add(relaxedName); + } + } else { + for (String prefix : prefixes) { + for (String relaxedName : relaxedNames) { + names.add(prefix + "." + relaxedName); + names.add(prefix + "_" + relaxedName); + } + } + } + } + } + } + return names; + } + + private PropertyValues getPropertySourcesPropertyValues(Set names, Iterable relaxedTargetNames) { + PropertyNamePatternsMatcher includes = getPropertyNamePatternsMatcher(names, relaxedTargetNames); + return new PropertySourcesPropertyValues(this.propertySources, names, includes, this.resolvePlaceholders); + } + + private PropertyNamePatternsMatcher getPropertyNamePatternsMatcher(Set names, + Iterable relaxedTargetNames) { + if (this.ignoreUnknownFields && !isMapTarget()) { + // Since unknown fields are ignored we can filter them out early to save + // unnecessary calls to the PropertySource. + return new DefaultPropertyNamePatternsMatcher(EXACT_DELIMITERS, true, names); + } + if (relaxedTargetNames != null) { + // We can filter properties to those starting with the target name, but + // we can't do a complete filter since we need to trigger the + // unknown fields check + Set relaxedNames = new HashSet<>(); + for (String relaxedTargetName : relaxedTargetNames) { + relaxedNames.add(relaxedTargetName); + } + return new DefaultPropertyNamePatternsMatcher(TARGET_NAME_DELIMITERS, true, relaxedNames); + } + // Not ideal, we basically can't filter anything + return PropertyNamePatternsMatcher.ALL; + } + + private boolean isMapTarget() { + return this.target != null && Map.class.isAssignableFrom(this.target.getClass()); + } + + private void checkForBindingErrors(RelaxedDataBinder dataBinder) throws BindException { + BindingResult errors = dataBinder.getBindingResult(); + if (errors.hasErrors()) { + logger.error("Properties configuration failed validation"); + for (ObjectError error : errors.getAllErrors()) { + logger.error(this.messageSource != null ? this.messageSource.getMessage(error, Locale.getDefault()) + + " (" + error + ")" : error); + } + if (this.exceptionIfInvalid) { + throw new BindException(errors); + } + } + } + + /** + * Customize the data binder. + * + * @param dataBinder the data binder that will be used to bind and validate + */ + protected void customizeBinder(DataBinder dataBinder) { + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java new file mode 100644 index 00000000..47d8a780 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java @@ -0,0 +1,24 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +/** + * Strategy interface used to check if a property name matches specific + * criteria. + * + * @author Phillip Webb + * @since 1.2.0 + */ +interface PropertyNamePatternsMatcher { + + PropertyNamePatternsMatcher ALL = propertyName -> true; + + PropertyNamePatternsMatcher NONE = propertyName -> false; + + /** + * Return {@code true} of the property name matches. + * + * @param propertyName the property name + * @return {@code true} if the property name matches + */ + boolean matches(String propertyName); + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java new file mode 100644 index 00000000..7ad35f1a --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java @@ -0,0 +1,30 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; + +/** + * The origin of a property, specifically its source and its name before any + * prefix was removed. + * + * @author Andy Wilkinson + * @since 1.3.0 + */ +public class PropertyOrigin { + + private final PropertySource source; + + private final String name; + + PropertyOrigin(PropertySource source, String name){ + this.name = name; + this.source = source; + } + + public PropertySource getSource() { + return this.source; + } + + public String getName() { + return this.name; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java new file mode 100644 index 00000000..520e857f --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java @@ -0,0 +1,164 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import org.springframework.core.convert.ConversionException; +import org.springframework.core.env.AbstractEnvironment; +import org.springframework.core.env.AbstractPropertyResolver; +import org.springframework.core.env.PropertyResolver; +import org.springframework.util.ClassUtils; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.PropertySources; + +/** + * {@link PropertyResolver} implementation that resolves property values against + * an underlying set of {@link PropertySources}. + * + * @author Chris Beams + * @author Juergen Hoeller + * @see PropertySource + * @see PropertySources + * @see AbstractEnvironment + * @since 3.1 + */ +public class PropertySourcesPropertyResolver extends AbstractPropertyResolver { + + private final PropertySources propertySources; + + /** + * Create a new resolver against the given property sources. + * + * @param propertySources the set of {@link PropertySource} objects to use + */ + public PropertySourcesPropertyResolver(PropertySources propertySources){ + this.propertySources = propertySources; + } + + @Override + public boolean containsProperty(String key) { + if (this.propertySources != null) { + for (PropertySource propertySource : this.propertySources) { + if (propertySource.containsProperty(key)) { + return true; + } + } + } + return false; + } + + @Override + public String getProperty(String key) { + return getProperty(key, String.class, true); + } + + @Override + public T getProperty(String key, Class targetValueType) { + return getProperty(key, targetValueType, true); + } + + @Override + protected String getPropertyAsRawString(String key) { + return getProperty(key, String.class, false); + } + + protected T getProperty(String key, Class targetValueType, boolean resolveNestedPlaceholders) { + if (this.propertySources != null) { + for (PropertySource propertySource : this.propertySources) { + if (logger.isTraceEnabled()) { + logger + .trace("Searching for key '" + key + "' in PropertySource '" + propertySource.getName() + "'"); + } + Object value = propertySource.getProperty(key); + if (value != null) { + if (resolveNestedPlaceholders && value instanceof String) { + value = resolveNestedPlaceholders((String) value); + } + logKeyFound(key, propertySource, value); + return convertValueIfNecessary(value, targetValueType); + } + } + } + if (logger.isDebugEnabled()) { + logger.debug("Could not find key '" + key + "' in any property source"); + } + return null; + } + + @Deprecated + public Class getPropertyAsClass(String key, Class targetValueType) { + if (this.propertySources != null) { + for (PropertySource propertySource : this.propertySources) { + if (logger.isTraceEnabled()) { + logger.trace(String.format("Searching for key '%s' in [%s]", key, propertySource.getName())); + } + Object value = propertySource.getProperty(key); + if (value != null) { + logKeyFound(key, propertySource, value); + Class clazz; + if (value instanceof String) { + try { + clazz = ClassUtils.forName((String) value, null); + } catch (Exception ex) { + throw new PropertySourcesPropertyResolver.ClassConversionException((String) value, + targetValueType, + ex); + } + } else if (value instanceof Class) { + clazz = (Class) value; + } else { + clazz = value.getClass(); + } + if (!targetValueType.isAssignableFrom(clazz)) { + throw new PropertySourcesPropertyResolver.ClassConversionException(clazz, targetValueType); + } + @SuppressWarnings("unchecked") + Class targetClass = (Class) clazz; + return targetClass; + } + } + } + if (logger.isDebugEnabled()) { + logger.debug(String.format("Could not find key '%s' in any property source", key)); + } + return null; + } + + /** + * Log the given key as found in the given {@link PropertySource}, resulting in + * the given value. + *

+ * The default implementation writes a debug log message with key and source. As + * of 4.3.3, this does not log the value anymore in order to avoid accidental + * logging of sensitive settings. Subclasses may override this method to change + * the log level and/or log message, including the property's value if desired. + * + * @param key the key found + * @param propertySource the {@code PropertySource} that the key has been found + * in + * @param value the corresponding value + * @since 4.3.1 + */ + protected void logKeyFound(String key, PropertySource propertySource, Object value) { + if (logger.isDebugEnabled()) { + logger.debug("Found key '" + key + "' in PropertySource '" + propertySource.getName() + + "' with value of type " + value.getClass().getSimpleName()); + } + } + + @SuppressWarnings("serial") + @Deprecated + private static class ClassConversionException extends ConversionException { + + public ClassConversionException(Class actual, Class expected){ + super(String + .format("Actual type %s is not assignable to expected type %s", actual.getName(), expected.getName())); + } + + public ClassConversionException(String actual, Class expected, Exception ex){ + super( + String + .format("Could not find/load class %s during attempt to convert to %s", actual, expected.getName()), + ex); + } + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java new file mode 100644 index 00000000..70ad5731 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java @@ -0,0 +1,233 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; + +import org.springframework.beans.MutablePropertyValues; +import org.springframework.beans.PropertyValue; +import org.springframework.beans.PropertyValues; +import org.springframework.util.Assert; +import org.springframework.validation.DataBinder; + +import com.alibaba.otter.canal.client.adapter.config.common.CompositePropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.EnumerablePropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.PropertySources; + +/** + * A {@link PropertyValues} implementation backed by a {@link PropertySources}, + * bridging the two abstractions and allowing (for instance) a regular + * {@link DataBinder} to be used with the latter. + * + * @author Dave Syer + * @author Phillip Webb + */ +public class PropertySourcesPropertyValues implements PropertyValues { + + private static final Pattern COLLECTION_PROPERTY = Pattern + .compile("\\[(\\d+)\\](\\.\\S+)?"); + + private final PropertySources propertySources; + + private final Collection nonEnumerableFallbackNames; + + private final PropertyNamePatternsMatcher includes; + + private final Map propertyValues = new LinkedHashMap<>(); + + private final ConcurrentHashMap> collectionOwners = new ConcurrentHashMap<>(); + + private final boolean resolvePlaceholders; + + /** + * Create a new PropertyValues from the given PropertySources. + * + * @param propertySources a PropertySources instance + */ + public PropertySourcesPropertyValues(PropertySources propertySources){ + this(propertySources, true); + } + + /** + * Create a new PropertyValues from the given PropertySources that will + * optionally resolve placeholders. + * + * @param propertySources a PropertySources instance + * @param resolvePlaceholders {@code true} if placeholders should be resolved. + * @since 1.5.2 + */ + public PropertySourcesPropertyValues(PropertySources propertySources, boolean resolvePlaceholders){ + this(propertySources, (Collection) null, PropertyNamePatternsMatcher.ALL, resolvePlaceholders); + } + + /** + * Create a new PropertyValues from the given PropertySources. + * + * @param propertySources a PropertySources instance + * @param includePatterns property name patterns to include from system + * properties and environment variables + * @param nonEnumerableFallbackNames the property names to try in lieu of an + * {@link EnumerablePropertySource}. + */ + public PropertySourcesPropertyValues(PropertySources propertySources, Collection includePatterns, + Collection nonEnumerableFallbackNames){ + this(propertySources, + nonEnumerableFallbackNames, + new PatternPropertyNamePatternsMatcher(includePatterns), + true); + } + + /** + * Create a new PropertyValues from the given PropertySources. + * + * @param propertySources a PropertySources instance + * @param nonEnumerableFallbackNames the property names to try in lieu of an + * {@link EnumerablePropertySource}. + * @param includes the property name patterns to include + * @param resolvePlaceholders flag to indicate the placeholders should be + * resolved + */ + PropertySourcesPropertyValues(PropertySources propertySources, Collection nonEnumerableFallbackNames, + PropertyNamePatternsMatcher includes, boolean resolvePlaceholders){ + Assert.notNull(propertySources, "PropertySources must not be null"); + Assert.notNull(includes, "Includes must not be null"); + this.propertySources = propertySources; + this.nonEnumerableFallbackNames = nonEnumerableFallbackNames; + this.includes = includes; + this.resolvePlaceholders = resolvePlaceholders; + PropertySourcesPropertyResolver resolver = new PropertySourcesPropertyResolver(propertySources); + for (PropertySource source : propertySources) { + processPropertySource(source, resolver); + } + } + + private void processPropertySource(PropertySource source, PropertySourcesPropertyResolver resolver) { + if (source instanceof CompositePropertySource) { + processCompositePropertySource((CompositePropertySource) source, resolver); + } else if (source instanceof EnumerablePropertySource) { + processEnumerablePropertySource((EnumerablePropertySource) source, resolver, this.includes); + } else { + processNonEnumerablePropertySource(source, resolver); + } + } + + private void processCompositePropertySource(CompositePropertySource source, + PropertySourcesPropertyResolver resolver) { + for (PropertySource nested : source.getPropertySources()) { + processPropertySource(nested, resolver); + } + } + + private void processEnumerablePropertySource(EnumerablePropertySource source, + PropertySourcesPropertyResolver resolver, + PropertyNamePatternsMatcher includes) { + if (source.getPropertyNames().length > 0) { + for (String propertyName : source.getPropertyNames()) { + if (includes.matches(propertyName)) { + Object value = getEnumerableProperty(source, resolver, propertyName); + putIfAbsent(propertyName, value, source); + } + } + } + } + + private Object getEnumerableProperty(EnumerablePropertySource source, PropertySourcesPropertyResolver resolver, + String propertyName) { + try { + if (this.resolvePlaceholders) { + return resolver.getProperty(propertyName, Object.class); + } + } catch (RuntimeException ex) { + // Probably could not resolve placeholders, ignore it here + } + return source.getProperty(propertyName); + } + + private void processNonEnumerablePropertySource(PropertySource source, + PropertySourcesPropertyResolver resolver) { + // We can only do exact matches for non-enumerable property names, but + // that's better than nothing... + if (this.nonEnumerableFallbackNames == null) { + return; + } + for (String propertyName : this.nonEnumerableFallbackNames) { + if (!source.containsProperty(propertyName)) { + continue; + } + Object value = null; + try { + value = resolver.getProperty(propertyName, Object.class); + } catch (RuntimeException ex) { + // Probably could not convert to Object, weird, but ignorable + } + if (value == null) { + value = source.getProperty(propertyName.toUpperCase(Locale.ENGLISH)); + } + putIfAbsent(propertyName, value, source); + } + } + + @Override + public PropertyValue[] getPropertyValues() { + Collection values = this.propertyValues.values(); + return values.toArray(new PropertyValue[values.size()]); + } + + @Override + public PropertyValue getPropertyValue(String propertyName) { + PropertyValue propertyValue = this.propertyValues.get(propertyName); + if (propertyValue != null) { + return propertyValue; + } + for (PropertySource source : this.propertySources) { + Object value = source.getProperty(propertyName); + propertyValue = putIfAbsent(propertyName, value, source); + if (propertyValue != null) { + return propertyValue; + } + } + return null; + } + + private PropertyValue putIfAbsent(String propertyName, Object value, PropertySource source) { + if (value != null && !this.propertyValues.containsKey(propertyName)) { + PropertySource collectionOwner = this.collectionOwners + .putIfAbsent(COLLECTION_PROPERTY.matcher(propertyName).replaceAll("[]"), source); + if (collectionOwner == null || collectionOwner == source) { + PropertyValue propertyValue = new OriginCapablePropertyValue(propertyName, value, propertyName, source); + this.propertyValues.put(propertyName, propertyValue); + return propertyValue; + } + } + return null; + } + + @Override + public PropertyValues changesSince(PropertyValues old) { + MutablePropertyValues changes = new MutablePropertyValues(); + // for each property value in the new set + for (PropertyValue newValue : getPropertyValues()) { + // if there wasn't an old one, add it + PropertyValue oldValue = old.getPropertyValue(newValue.getName()); + if (oldValue == null || !oldValue.equals(newValue)) { + changes.addPropertyValue(newValue); + } + } + return changes; + } + + @Override + public boolean contains(String propertyName) { + return getPropertyValue(propertyName) != null; + } + + @Override + public boolean isEmpty() { + return this.propertyValues.isEmpty(); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java new file mode 100644 index 00000000..ab8e5506 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java @@ -0,0 +1,127 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.EnumSet; +import java.util.Locale; +import java.util.Set; + +import org.springframework.core.convert.ConversionFailedException; +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.core.convert.converter.Converter; +import org.springframework.core.convert.converter.ConverterFactory; +import org.springframework.core.convert.support.DefaultConversionService; +import org.springframework.core.convert.support.GenericConversionService; +import org.springframework.util.Assert; + +/** + * Internal {@link ConversionService} used by {@link RelaxedDataBinder} to + * support additional relaxed conversion. + * + * @author Phillip Webb + * @author Stephane Nicoll + * @since 1.1.0 + */ +class RelaxedConversionService implements ConversionService { + + private final ConversionService conversionService; + + private final GenericConversionService additionalConverters; + + /** + * Create a new {@link RelaxedConversionService} instance. + * + * @param conversionService and option root conversion service + */ + RelaxedConversionService(ConversionService conversionService){ + this.conversionService = conversionService; + this.additionalConverters = new GenericConversionService(); + DefaultConversionService.addCollectionConverters(this.additionalConverters); + this.additionalConverters + .addConverterFactory(new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory()); + this.additionalConverters.addConverter(new StringToCharArrayConverter()); + } + + @Override + public boolean canConvert(Class sourceType, Class targetType) { + return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType)) + || this.additionalConverters.canConvert(sourceType, targetType); + } + + @Override + public boolean canConvert(TypeDescriptor sourceType, TypeDescriptor targetType) { + return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType)) + || this.additionalConverters.canConvert(sourceType, targetType); + } + + @Override + @SuppressWarnings("unchecked") + public T convert(Object source, Class targetType) { + Assert.notNull(targetType, "The targetType to convert to cannot be null"); + return (T) convert(source, TypeDescriptor.forObject(source), TypeDescriptor.valueOf(targetType)); + } + + @Override + public Object convert(Object source, TypeDescriptor sourceType, TypeDescriptor targetType) { + if (this.conversionService != null) { + try { + return this.conversionService.convert(source, sourceType, targetType); + } catch (ConversionFailedException ex) { + // Ignore and try the additional converters + } + } + return this.additionalConverters.convert(source, sourceType, targetType); + } + + /** + * Clone of Spring's package private StringToEnumConverterFactory, but ignoring + * the case of the source. + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + private static class StringToEnumIgnoringCaseConverterFactory implements ConverterFactory { + + @Override + public Converter getConverter(Class targetType) { + Class enumType = targetType; + while (enumType != null && !enumType.isEnum()) { + enumType = enumType.getSuperclass(); + } + Assert.notNull(enumType, "The target type " + targetType.getName() + " does not refer to an enum"); + return new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory.StringToEnum(enumType); + } + + private static class StringToEnum implements Converter { + + private final Class enumType; + + StringToEnum(Class enumType){ + this.enumType = enumType; + } + + @Override + public T convert(String source) { + if (source.isEmpty()) { + // It's an empty enum identifier: reset the enum value to null. + return null; + } + source = source.trim(); + for (T candidate : (Set) EnumSet.allOf(this.enumType)) { + RelaxedNames names = new RelaxedNames( + candidate.name().replace('_', '-').toLowerCase(Locale.ENGLISH)); + for (String name : names) { + if (name.equals(source)) { + return candidate; + } + } + if (candidate.name().equalsIgnoreCase(source)) { + return candidate; + } + } + throw new IllegalArgumentException( + "No enum constant " + this.enumType.getCanonicalName() + "." + source); + } + + } + + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java new file mode 100644 index 00000000..7723eb0f --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java @@ -0,0 +1,729 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.beans.PropertyEditor; +import java.net.InetAddress; +import java.util.*; + +import org.springframework.beans.*; +import org.springframework.beans.propertyeditors.FileEditor; +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.util.StringUtils; +import org.springframework.validation.AbstractPropertyBindingResult; +import org.springframework.validation.BeanPropertyBindingResult; +import org.springframework.validation.DataBinder; + +/** + * Binder implementation that allows caller to bind to maps and also allows + * property names to match a bit loosely (if underscores or dashes are removed + * and replaced with camel case for example). + * + * @author Dave Syer + * @author Phillip Webb + * @author Stephane Nicoll + * @author Andy Wilkinson + * @see RelaxedNames + */ +public class RelaxedDataBinder extends DataBinder { + + private static final Set> EXCLUDED_EDITORS; + + static { + Set> excluded = new HashSet<>(); + excluded.add(FileEditor.class); + EXCLUDED_EDITORS = Collections.unmodifiableSet(excluded); + } + + private static final Object BLANK = new Object(); + + private String namePrefix; + + private boolean ignoreNestedProperties; + + private MultiValueMap nameAliases = new LinkedMultiValueMap<>(); + + /** + * Create a new {@link RelaxedDataBinder} instance. + * + * @param target the target into which properties are bound + */ + public RelaxedDataBinder(Object target){ + super(wrapTarget(target)); + } + + /** + * Create a new {@link RelaxedDataBinder} instance. + * + * @param target the target into which properties are bound + * @param namePrefix An optional prefix to be used when reading properties + */ + public RelaxedDataBinder(Object target, String namePrefix){ + super(wrapTarget(target), (StringUtils.hasLength(namePrefix) ? namePrefix : DEFAULT_OBJECT_NAME)); + this.namePrefix = cleanNamePrefix(namePrefix); + } + + private String cleanNamePrefix(String namePrefix) { + if (!StringUtils.hasLength(namePrefix)) { + return null; + } + return (namePrefix.endsWith(".") ? namePrefix : namePrefix + "."); + } + + /** + * Flag to disable binding of nested properties (i.e. those with period + * separators in their paths). Can be useful to disable this if the name prefix + * is empty and you don't want to ignore unknown fields. + * + * @param ignoreNestedProperties the flag to set (default false) + */ + public void setIgnoreNestedProperties(boolean ignoreNestedProperties) { + this.ignoreNestedProperties = ignoreNestedProperties; + } + + /** + * Set name aliases. + * + * @param aliases a map of property name to aliases + */ + public void setNameAliases(Map> aliases) { + this.nameAliases = new LinkedMultiValueMap<>(aliases); + } + + /** + * Add aliases to the {@link DataBinder}. + * + * @param name the property name to alias + * @param alias aliases for the property names + * @return this instance + */ + public RelaxedDataBinder withAlias(String name, String... alias) { + for (String value : alias) { + this.nameAliases.add(name, value); + } + return this; + } + + @Override + protected void doBind(MutablePropertyValues propertyValues) { + super.doBind(modifyProperties(propertyValues, getTarget())); + } + + /** + * Modify the property values so that period separated property paths are valid + * for map keys. Also creates new maps for properties of map type that are null + * (assuming all maps are potentially nested). The standard bracket {@code[...]} + * dereferencing is also accepted. + * + * @param propertyValues the property values + * @param target the target object + * @return modified property values + */ + private MutablePropertyValues modifyProperties(MutablePropertyValues propertyValues, Object target) { + propertyValues = getPropertyValuesForNamePrefix(propertyValues); + if (target instanceof RelaxedDataBinder.MapHolder) { + propertyValues = addMapPrefix(propertyValues); + } + BeanWrapper wrapper = new BeanWrapperImpl(target); + wrapper.setConversionService(new RelaxedConversionService(getConversionService())); + wrapper.setAutoGrowNestedPaths(true); + List sortedValues = new ArrayList<>(); + Set modifiedNames = new HashSet<>(); + List sortedNames = getSortedPropertyNames(propertyValues); + for (String name : sortedNames) { + PropertyValue propertyValue = propertyValues.getPropertyValue(name); + PropertyValue modifiedProperty = modifyProperty(wrapper, propertyValue); + if (modifiedNames.add(modifiedProperty.getName())) { + sortedValues.add(modifiedProperty); + } + } + return new MutablePropertyValues(sortedValues); + } + + private List getSortedPropertyNames(MutablePropertyValues propertyValues) { + List names = new LinkedList<>(); + for (PropertyValue propertyValue : propertyValues.getPropertyValueList()) { + names.add(propertyValue.getName()); + } + sortPropertyNames(names); + return names; + } + + /** + * Sort by name so that parent properties get processed first (e.g. 'foo.bar' + * before 'foo.bar.spam'). Don't use Collections.sort() because the order might + * be significant for other property names (it shouldn't be but who knows what + * people might be relying on, e.g. HSQL has a JDBCXADataSource where + * "databaseName" is a synonym for "url"). + * + * @param names the names to sort + */ + private void sortPropertyNames(List names) { + for (String name : new ArrayList<>(names)) { + int propertyIndex = names.indexOf(name); + RelaxedDataBinder.BeanPath path = new RelaxedDataBinder.BeanPath(name); + for (String prefix : path.prefixes()) { + int prefixIndex = names.indexOf(prefix); + if (prefixIndex >= propertyIndex) { + // The child property has a parent in the list in the wrong order + names.remove(name); + names.add(prefixIndex, name); + } + } + } + } + + private MutablePropertyValues addMapPrefix(MutablePropertyValues propertyValues) { + MutablePropertyValues rtn = new MutablePropertyValues(); + for (PropertyValue pv : propertyValues.getPropertyValues()) { + rtn.add("map." + pv.getName(), pv.getValue()); + } + return rtn; + } + + private MutablePropertyValues getPropertyValuesForNamePrefix(MutablePropertyValues propertyValues) { + if (!StringUtils.hasText(this.namePrefix) && !this.ignoreNestedProperties) { + return propertyValues; + } + MutablePropertyValues rtn = new MutablePropertyValues(); + for (PropertyValue value : propertyValues.getPropertyValues()) { + String name = value.getName(); + for (String prefix : new RelaxedNames(stripLastDot(this.namePrefix))) { + for (String separator : new String[] { ".", "_" }) { + String candidate = (StringUtils.hasLength(prefix) ? prefix + separator : prefix); + if (name.startsWith(candidate)) { + name = name.substring(candidate.length()); + if (!(this.ignoreNestedProperties && name.contains("."))) { + PropertyOrigin propertyOrigin = OriginCapablePropertyValue.getOrigin(value); + rtn.addPropertyValue( + new OriginCapablePropertyValue(name, value.getValue(), propertyOrigin)); + } + } + } + } + } + return rtn; + } + + private String stripLastDot(String string) { + if (StringUtils.hasLength(string) && string.endsWith(".")) { + string = string.substring(0, string.length() - 1); + } + return string; + } + + private PropertyValue modifyProperty(BeanWrapper target, PropertyValue propertyValue) { + String name = propertyValue.getName(); + String normalizedName = normalizePath(target, name); + if (!normalizedName.equals(name)) { + return new PropertyValue(normalizedName, propertyValue.getValue()); + } + return propertyValue; + } + + /** + * Normalize a bean property path to a format understood by a BeanWrapper. This + * is used so that + *

    + *
  • Fuzzy matching can be employed for bean property names
    • + *
  • Period separators can be used instead of indexing ([...]) for map + * keys
    • + *
+ * + * @param wrapper a bean wrapper for the object to bind + * @param path the bean path to bind + * @return a transformed path with correct bean wrapper syntax + */ + protected String normalizePath(BeanWrapper wrapper, String path) { + return initializePath(wrapper, new RelaxedDataBinder.BeanPath(path), 0); + } + + @Override + protected AbstractPropertyBindingResult createBeanPropertyBindingResult() { + return new RelaxedDataBinder.RelaxedBeanPropertyBindingResult(getTarget(), + getObjectName(), + isAutoGrowNestedPaths(), + getAutoGrowCollectionLimit(), + getConversionService()); + } + + private String initializePath(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) { + String prefix = path.prefix(index); + String key = path.name(index); + if (path.isProperty(index)) { + key = getActualPropertyName(wrapper, prefix, key); + path.rename(index, key); + } + if (path.name(++index) == null) { + return path.toString(); + } + String name = path.prefix(index); + TypeDescriptor descriptor = wrapper.getPropertyTypeDescriptor(name); + if (descriptor == null || descriptor.isMap()) { + if (isMapValueStringType(descriptor) || isBlanked(wrapper, name, path.name(index))) { + path.collapseKeys(index); + } + path.mapIndex(index); + extendMapIfNecessary(wrapper, path, index); + } else if (descriptor.isCollection()) { + extendCollectionIfNecessary(wrapper, path, index); + } else if (descriptor.getType().equals(Object.class)) { + if (isBlanked(wrapper, name, path.name(index))) { + path.collapseKeys(index); + } + path.mapIndex(index); + if (path.isLastNode(index)) { + wrapper.setPropertyValue(path.toString(), BLANK); + } else { + String next = path.prefix(index + 1); + if (wrapper.getPropertyValue(next) == null) { + wrapper.setPropertyValue(next, new LinkedHashMap()); + } + } + } + return initializePath(wrapper, path, index); + } + + private boolean isMapValueStringType(TypeDescriptor descriptor) { + if (descriptor == null || descriptor.getMapValueTypeDescriptor() == null) { + return false; + } + if (Properties.class.isAssignableFrom(descriptor.getObjectType())) { + // Properties is declared as Map but we know it's really + // Map + return true; + } + Class valueType = descriptor.getMapValueTypeDescriptor().getObjectType(); + return (valueType != null && CharSequence.class.isAssignableFrom(valueType)); + } + + @SuppressWarnings("rawtypes") + private boolean isBlanked(BeanWrapper wrapper, String propertyName, String key) { + Object value = (wrapper.isReadableProperty(propertyName) ? wrapper.getPropertyValue(propertyName) : null); + if (value instanceof Map) { + if (((Map) value).get(key) == BLANK) { + return true; + } + } + return false; + } + + private void extendCollectionIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) { + String name = path.prefix(index); + TypeDescriptor elementDescriptor = wrapper.getPropertyTypeDescriptor(name).getElementTypeDescriptor(); + if (!elementDescriptor.isMap() && !elementDescriptor.isCollection() + && !elementDescriptor.getType().equals(Object.class)) { + return; + } + Object extend = new LinkedHashMap(); + if (!elementDescriptor.isMap() && path.isArrayIndex(index)) { + extend = new ArrayList<>(); + } + wrapper.setPropertyValue(path.prefix(index + 1), extend); + } + + private void extendMapIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) { + String name = path.prefix(index); + TypeDescriptor parent = wrapper.getPropertyTypeDescriptor(name); + if (parent == null) { + return; + } + TypeDescriptor descriptor = parent.getMapValueTypeDescriptor(); + if (descriptor == null) { + descriptor = TypeDescriptor.valueOf(Object.class); + } + if (!descriptor.isMap() && !descriptor.isCollection() && !descriptor.getType().equals(Object.class)) { + return; + } + String extensionName = path.prefix(index + 1); + if (wrapper.isReadableProperty(extensionName)) { + Object currentValue = wrapper.getPropertyValue(extensionName); + if ((descriptor.isCollection() && currentValue instanceof Collection) + || (!descriptor.isCollection() && currentValue instanceof Map)) { + return; + } + } + Object extend = new LinkedHashMap(); + if (descriptor.isCollection()) { + extend = new ArrayList<>(); + } + if (descriptor.getType().equals(Object.class) && path.isLastNode(index)) { + extend = BLANK; + } + wrapper.setPropertyValue(extensionName, extend); + } + + private String getActualPropertyName(BeanWrapper target, String prefix, String name) { + String propertyName = resolvePropertyName(target, prefix, name); + if (propertyName == null) { + propertyName = resolveNestedPropertyName(target, prefix, name); + } + return (propertyName != null ? propertyName : name); + } + + private String resolveNestedPropertyName(BeanWrapper target, String prefix, String name) { + StringBuilder candidate = new StringBuilder(); + for (String field : name.split("[_\\-\\.]")) { + candidate.append(candidate.length() > 0 ? "." : ""); + candidate.append(field); + String nested = resolvePropertyName(target, prefix, candidate.toString()); + if (nested != null) { + Class type = target.getPropertyType(nested); + if ((type != null) && Map.class.isAssignableFrom(type)) { + // Special case for map property (gh-3836). + return nested + "[" + name.substring(candidate.length() + 1) + "]"; + } + String propertyName = resolvePropertyName(target, + joinString(prefix, nested), + name.substring(candidate.length() + 1)); + if (propertyName != null) { + return joinString(nested, propertyName); + } + } + } + return null; + } + + private String resolvePropertyName(BeanWrapper target, String prefix, String name) { + Iterable names = getNameAndAliases(name); + for (String nameOrAlias : names) { + for (String candidate : new RelaxedNames(nameOrAlias)) { + try { + if (target.getPropertyType(joinString(prefix, candidate)) != null) { + return candidate; + } + } catch (InvalidPropertyException ex) { + // swallow and continue + } + } + } + return null; + } + + private String joinString(String prefix, String name) { + return (StringUtils.hasLength(prefix) ? prefix + "." + name : name); + } + + private Iterable getNameAndAliases(String name) { + List aliases = this.nameAliases.get(name); + if (aliases == null) { + return Collections.singleton(name); + } + List nameAndAliases = new ArrayList<>(aliases.size() + 1); + nameAndAliases.add(name); + nameAndAliases.addAll(aliases); + return nameAndAliases; + } + + private static Object wrapTarget(Object target) { + if (target instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) target; + target = new RelaxedDataBinder.MapHolder(map); + } + return target; + } + + @Override + public void registerCustomEditor(Class requiredType, PropertyEditor propertyEditor) { + if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) { + super.registerCustomEditor(requiredType, propertyEditor); + } + } + + @Override + public void registerCustomEditor(Class requiredType, String field, PropertyEditor propertyEditor) { + if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) { + super.registerCustomEditor(requiredType, field, propertyEditor); + } + } + + /** + * Holder to allow Map targets to be bound. + */ + static class MapHolder { + + private Map map; + + MapHolder(Map map){ + this.map = map; + } + + public void setMap(Map map) { + this.map = map; + } + + public Map getMap() { + return this.map; + } + + } + + /** + * A path though properties of a bean. + */ + private static class BeanPath { + + private List nodes; + + BeanPath(String path){ + this.nodes = splitPath(path); + } + + public List prefixes() { + List prefixes = new ArrayList<>(); + for (int index = 1; index < this.nodes.size(); index++) { + prefixes.add(prefix(index)); + } + return prefixes; + } + + public boolean isLastNode(int index) { + return index >= this.nodes.size() - 1; + } + + private List splitPath(String path) { + List nodes = new ArrayList<>(); + String current = extractIndexedPaths(path, nodes); + for (String name : StringUtils.delimitedListToStringArray(current, ".")) { + if (StringUtils.hasText(name)) { + nodes.add(new RelaxedDataBinder.BeanPath.PropertyNode(name)); + } + } + return nodes; + } + + private String extractIndexedPaths(String path, List nodes) { + int startRef = path.indexOf("["); + String current = path; + while (startRef >= 0) { + if (startRef > 0) { + nodes.addAll(splitPath(current.substring(0, startRef))); + } + int endRef = current.indexOf("]", startRef); + if (endRef > 0) { + String sub = current.substring(startRef + 1, endRef); + if (sub.matches("[0-9]+")) { + nodes.add(new RelaxedDataBinder.BeanPath.ArrayIndexNode(sub)); + } else { + nodes.add(new RelaxedDataBinder.BeanPath.MapIndexNode(sub)); + } + } + current = current.substring(endRef + 1); + startRef = current.indexOf("["); + } + return current; + } + + public void collapseKeys(int index) { + List revised = new ArrayList<>(); + for (int i = 0; i < index; i++) { + revised.add(this.nodes.get(i)); + } + StringBuilder builder = new StringBuilder(); + for (int i = index; i < this.nodes.size(); i++) { + if (i > index) { + builder.append("."); + } + builder.append(this.nodes.get(i).name); + } + revised.add(new RelaxedDataBinder.BeanPath.PropertyNode(builder.toString())); + this.nodes = revised; + } + + public void mapIndex(int index) { + RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(index); + if (node instanceof RelaxedDataBinder.BeanPath.PropertyNode) { + node = ((RelaxedDataBinder.BeanPath.PropertyNode) node).mapIndex(); + } + this.nodes.set(index, node); + } + + public String prefix(int index) { + return range(0, index); + } + + public void rename(int index, String name) { + this.nodes.get(index).name = name; + } + + public String name(int index) { + if (index < this.nodes.size()) { + return this.nodes.get(index).name; + } + return null; + } + + private String range(int start, int end) { + StringBuilder builder = new StringBuilder(); + for (int i = start; i < end; i++) { + RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(i); + builder.append(node); + } + if (builder.toString().startsWith(("."))) { + builder.replace(0, 1, ""); + } + return builder.toString(); + } + + public boolean isArrayIndex(int index) { + return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.ArrayIndexNode; + } + + public boolean isProperty(int index) { + return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.PropertyNode; + } + + @Override + public String toString() { + return prefix(this.nodes.size()); + } + + private static class PathNode { + + protected String name; + + PathNode(String name){ + this.name = name; + } + + } + + private static class ArrayIndexNode extends RelaxedDataBinder.BeanPath.PathNode { + + ArrayIndexNode(String name){ + super(name); + } + + @Override + public String toString() { + return "[" + this.name + "]"; + } + + } + + private static class MapIndexNode extends RelaxedDataBinder.BeanPath.PathNode { + + MapIndexNode(String name){ + super(name); + } + + @Override + public String toString() { + return "[" + this.name + "]"; + } + + } + + private static class PropertyNode extends RelaxedDataBinder.BeanPath.PathNode { + + PropertyNode(String name){ + super(name); + } + + public RelaxedDataBinder.BeanPath.MapIndexNode mapIndex() { + return new RelaxedDataBinder.BeanPath.MapIndexNode(this.name); + } + + @Override + public String toString() { + return "." + this.name; + } + + } + + } + + /** + * Extended version of {@link BeanPropertyBindingResult} to support relaxed + * binding. + */ + private static class RelaxedBeanPropertyBindingResult extends BeanPropertyBindingResult { + + private RelaxedConversionService conversionService; + + RelaxedBeanPropertyBindingResult(Object target, String objectName, boolean autoGrowNestedPaths, + int autoGrowCollectionLimit, ConversionService conversionService){ + super(target, objectName, autoGrowNestedPaths, autoGrowCollectionLimit); + this.conversionService = new RelaxedConversionService(conversionService); + } + + @Override + protected BeanWrapper createBeanWrapper() { + BeanWrapper beanWrapper = new RelaxedDataBinder.RelaxedBeanWrapper(getTarget()); + beanWrapper.setConversionService(this.conversionService); + beanWrapper.registerCustomEditor(InetAddress.class, new InetAddressEditor()); + return beanWrapper; + } + + } + + /** + * Extended version of {@link BeanWrapperImpl} to support relaxed binding. + */ + private static class RelaxedBeanWrapper extends BeanWrapperImpl { + + private static final Set BENIGN_PROPERTY_SOURCE_NAMES; + + static { + Set names = new HashSet<>(); + names.add(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME); + names.add(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME); + BENIGN_PROPERTY_SOURCE_NAMES = Collections.unmodifiableSet(names); + } + + RelaxedBeanWrapper(Object target){ + super(target); + } + + @Override + public void setPropertyValue(PropertyValue pv) throws BeansException { + try { + super.setPropertyValue(pv); + } catch (NotWritablePropertyException ex) { + PropertyOrigin origin = OriginCapablePropertyValue.getOrigin(pv); + if (isBenign(origin)) { + logger.debug("Ignoring benign property binding failure", ex); + return; + } + if (origin == null) { + throw ex; + } + throw new RelaxedBindingNotWritablePropertyException(ex, origin); + } + } + + private boolean isBenign(PropertyOrigin origin) { + String name = (origin != null ? origin.getSource().getName() : null); + return BENIGN_PROPERTY_SOURCE_NAMES.contains(name); + } + + } + + public static class RelaxedBindingNotWritablePropertyException extends NotWritablePropertyException { + + private final String message; + + private final PropertyOrigin propertyOrigin; + + RelaxedBindingNotWritablePropertyException(NotWritablePropertyException ex, PropertyOrigin propertyOrigin){ + super(ex.getBeanClass(), ex.getPropertyName()); + this.propertyOrigin = propertyOrigin; + this.message = "Failed to bind '" + propertyOrigin.getName() + "' from '" + + propertyOrigin.getSource().getName() + "' to '" + ex.getPropertyName() + "' property on '" + + ex.getBeanClass().getName() + "'"; + } + + @Override + public String getMessage() { + return this.message; + } + + public PropertyOrigin getPropertyOrigin() { + return this.propertyOrigin; + } + + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java new file mode 100644 index 00000000..2c9dd70c --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java @@ -0,0 +1,241 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.util.StringUtils; + +/** + * Generates relaxed name variations from a given source. + * + * @author Phillip Webb + * @author Dave Syer + * @see RelaxedDataBinder + */ +public final class RelaxedNames implements Iterable { + + private static final Pattern CAMEL_CASE_PATTERN = Pattern.compile("([^A-Z-])([A-Z])"); + + private static final Pattern SEPARATED_TO_CAMEL_CASE_PATTERN = Pattern.compile("[_\\-.]"); + + private final String name; + + private final Set values = new LinkedHashSet<>(); + + /** + * Create a new {@link RelaxedNames} instance. + * + * @param name the source name. For the maximum number of variations specify the + * name using dashed notation (e.g. {@literal my-property-name} + */ + public RelaxedNames(String name){ + this.name = (name != null ? name : ""); + initialize(RelaxedNames.this.name, this.values); + } + + @Override + public Iterator iterator() { + return this.values.iterator(); + } + + private void initialize(String name, Set values) { + if (values.contains(name)) { + return; + } + for (RelaxedNames.Variation variation : RelaxedNames.Variation.values()) { + for (RelaxedNames.Manipulation manipulation : RelaxedNames.Manipulation.values()) { + String result = name; + result = manipulation.apply(result); + result = variation.apply(result); + values.add(result); + initialize(result, values); + } + } + } + + /** + * Name variations. + */ + enum Variation { + + NONE { + + @Override + public String apply(String value) { + return value; + } + + }, + + LOWERCASE { + + @Override + public String apply(String value) { + return (value.isEmpty() ? value : value.toLowerCase(Locale.ENGLISH)); + } + + }, + + UPPERCASE { + + @Override + public String apply(String value) { + return (value.isEmpty() ? value : value.toUpperCase(Locale.ENGLISH)); + } + + }; + + public abstract String apply(String value); + + } + + /** + * Name manipulations. + */ + enum Manipulation { + + NONE { + + @Override + public String apply(String value) { + return value; + } + + }, + + HYPHEN_TO_UNDERSCORE { + + @Override + public String apply(String value) { + return (value.indexOf('-') != -1 ? value.replace('-', '_') : value); + } + + }, + + UNDERSCORE_TO_PERIOD { + + @Override + public String apply(String value) { + return (value.indexOf('_') != -1 ? value.replace('_', '.') : value); + } + + }, + + PERIOD_TO_UNDERSCORE { + + @Override + public String apply(String value) { + return (value.indexOf('.') != -1 ? value.replace('.', '_') : value); + } + + }, + + CAMELCASE_TO_UNDERSCORE { + + @Override + public String apply(String value) { + if (value.isEmpty()) { + return value; + } + Matcher matcher = CAMEL_CASE_PATTERN.matcher(value); + if (!matcher.find()) { + return value; + } + matcher = matcher.reset(); + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(result, + matcher.group(1) + '_' + StringUtils.uncapitalize(matcher.group(2))); + } + matcher.appendTail(result); + return result.toString(); + } + + }, + + CAMELCASE_TO_HYPHEN { + + @Override + public String apply(String value) { + if (value.isEmpty()) { + return value; + } + Matcher matcher = CAMEL_CASE_PATTERN.matcher(value); + if (!matcher.find()) { + return value; + } + matcher = matcher.reset(); + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(result, + matcher.group(1) + '-' + StringUtils.uncapitalize(matcher.group(2))); + } + matcher.appendTail(result); + return result.toString(); + } + + }, + + SEPARATED_TO_CAMELCASE { + + @Override + public String apply(String value) { + return separatedToCamelCase(value, false); + } + + }, + + CASE_INSENSITIVE_SEPARATED_TO_CAMELCASE { + + @Override + public String apply(String value) { + return separatedToCamelCase(value, true); + } + + }; + + private static final char[] SUFFIXES = new char[] { '_', '-', '.' }; + + public abstract String apply(String value); + + private static String separatedToCamelCase(String value, boolean caseInsensitive) { + if (value.isEmpty()) { + return value; + } + StringBuilder builder = new StringBuilder(); + for (String field : SEPARATED_TO_CAMEL_CASE_PATTERN.split(value)) { + field = (caseInsensitive ? field.toLowerCase(Locale.ENGLISH) : field); + builder.append(builder.length() != 0 ? StringUtils.capitalize(field) : field); + } + char lastChar = value.charAt(value.length() - 1); + for (char suffix : SUFFIXES) { + if (lastChar == suffix) { + builder.append(suffix); + break; + } + } + return builder.toString(); + } + + } + + /** + * Return a {@link RelaxedNames} for the given source camelCase source name. + * + * @param name the source name in camelCase + * @return the relaxed names + */ + public static RelaxedNames forCamelCase(String name) { + StringBuilder result = new StringBuilder(); + for (char c : name.toCharArray()) { + result.append(Character.isUpperCase(c) && result.length() > 0 + && result.charAt(result.length() - 1) != '-' ? "-" + Character.toLowerCase(c) : c); + } + return new RelaxedNames(result.toString()); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java new file mode 100644 index 00000000..00a2d808 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java @@ -0,0 +1,17 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import org.springframework.core.convert.converter.Converter; + +/** + * Converts a String to a Char Array. + * + * @author Phillip Webb + */ +class StringToCharArrayConverter implements Converter { + + @Override + public char[] convert(String source) { + return source.toCharArray(); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java new file mode 100644 index 00000000..9cb73e90 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java @@ -0,0 +1,203 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +import org.springframework.util.Assert; +import org.springframework.util.ResourceUtils; + +/** + * Convenience base class for {@link Resource} implementations, pre-implementing + * typical behavior. + *

+ * The "exists" method will check whether a File or InputStream can be opened; + * "isOpen" will always return false; "getURL" and "getFile" throw an exception; + * and "toString" will return the description. + * + * @author Juergen Hoeller + * @since 28.12.2003 + */ +public abstract class AbstractResource implements Resource { + + /** + * This implementation checks whether a File can be opened, falling back to + * whether an InputStream can be opened. This will cover both directories and + * content resources. + */ + @Override + public boolean exists() { + // Try file existence: can we find the file in the file system? + try { + return getFile().exists(); + } catch (IOException ex) { + // Fall back to stream existence: can we open the stream? + try { + InputStream is = getInputStream(); + is.close(); + return true; + } catch (Throwable isEx) { + return false; + } + } + } + + /** + * This implementation always returns {@code true}. + */ + @Override + public boolean isReadable() { + return true; + } + + /** + * This implementation always returns {@code false}. + */ + @Override + public boolean isOpen() { + return false; + } + + /** + * This implementation throws a FileNotFoundException, assuming that the + * resource cannot be resolved to a URL. + */ + @Override + public URL getURL() throws IOException { + throw new FileNotFoundException(getDescription() + " cannot be resolved to URL"); + } + + /** + * This implementation builds a URI based on the URL returned by + * {@link #getURL()}. + */ + @Override + public URI getURI() throws IOException { + URL url = getURL(); + try { + return ResourceUtils.toURI(url); + } catch (URISyntaxException ex) { + throw new RuntimeException("Invalid URI [" + url + "]", ex); + } + } + + /** + * This implementation throws a FileNotFoundException, assuming that the + * resource cannot be resolved to an absolute file path. + */ + @Override + public File getFile() throws IOException { + throw new FileNotFoundException(getDescription() + " cannot be resolved to absolute file path"); + } + + /** + * This implementation reads the entire InputStream to calculate the content + * length. Subclasses will almost always be able to provide a more optimal + * version of this, e.g. checking a File length. + * + * @see #getInputStream() + */ + @Override + public long contentLength() throws IOException { + InputStream is = getInputStream(); + Assert.state(is != null, "Resource InputStream must not be null"); + try { + long size = 0; + byte[] buf = new byte[255]; + int read; + while ((read = is.read(buf)) != -1) { + size += read; + } + return size; + } finally { + try { + is.close(); + } catch (IOException ex) { + } + } + } + + /** + * This implementation checks the timestamp of the underlying File, if + * available. + * + * @see #getFileForLastModifiedCheck() + */ + @Override + public long lastModified() throws IOException { + long lastModified = getFileForLastModifiedCheck().lastModified(); + if (lastModified == 0L) { + throw new FileNotFoundException( + getDescription() + " cannot be resolved in the file system for resolving its last-modified timestamp"); + } + return lastModified; + } + + /** + * Determine the File to use for timestamp checking. + *

+ * The default implementation delegates to {@link #getFile()}. + * + * @return the File to use for timestamp checking (never {@code null}) + * @throws FileNotFoundException if the resource cannot be resolved as an + * absolute file path, i.e. is not available in a file system + * @throws IOException in case of general resolution/reading failures + */ + protected File getFileForLastModifiedCheck() throws IOException { + return getFile(); + } + + /** + * This implementation throws a FileNotFoundException, assuming that relative + * resources cannot be created for this resource. + */ + @Override + public org.springframework.core.io.Resource createRelative(String relativePath) throws IOException { + throw new FileNotFoundException("Cannot create a relative resource for " + getDescription()); + } + + /** + * This implementation always returns {@code null}, assuming that this resource + * type does not have a filename. + */ + @Override + public String getFilename() { + return null; + } + + /** + * This implementation returns the description of this resource. + * + * @see #getDescription() + */ + @Override + public String toString() { + return getDescription(); + } + + /** + * This implementation compares description strings. + * + * @see #getDescription() + */ + @Override + public boolean equals(Object obj) { + return (obj == this + || (obj instanceof org.springframework.core.io.Resource + && ((org.springframework.core.io.Resource) obj).getDescription().equals(getDescription()))); + } + + /** + * This implementation returns the description's hash code. + * + * @see #getDescription() + */ + @Override + public int hashCode() { + return getDescription().hashCode(); + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java new file mode 100644 index 00000000..c8ec120b --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java @@ -0,0 +1,117 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.util.Assert; + +/** + * {@link Resource} implementation for a given byte array. + *

+ * Creates a {@link ByteArrayInputStream} for the given byte array. + *

+ * Useful for loading content from any given byte array, without having to + * resort to a single-use {@link InputStreamResource}. Particularly useful for + * creating mail attachments from local content, where JavaMail needs to be able + * to read the stream multiple times. + * + * @author Juergen Hoeller + * @author Sam Brannen + * @see ByteArrayInputStream + * @see InputStreamResource + * @since 1.2.3 + */ +public class ByteArrayResource extends AbstractResource { + + private final byte[] byteArray; + + private final String description; + + /** + * Create a new {@code ByteArrayResource}. + * + * @param byteArray the byte array to wrap + */ + public ByteArrayResource(byte[] byteArray){ + this(byteArray, "resource loaded from byte array"); + } + + /** + * Create a new {@code ByteArrayResource} with a description. + * + * @param byteArray the byte array to wrap + * @param description where the byte array comes from + */ + public ByteArrayResource(byte[] byteArray, String description){ + Assert.notNull(byteArray, "Byte array must not be null"); + this.byteArray = byteArray; + this.description = (description != null ? description : ""); + } + + /** + * Return the underlying byte array. + */ + public final byte[] getByteArray() { + return this.byteArray; + } + + /** + * This implementation always returns {@code true}. + */ + @Override + public boolean exists() { + return true; + } + + /** + * This implementation returns the length of the underlying byte array. + */ + @Override + public long contentLength() { + return this.byteArray.length; + } + + /** + * This implementation returns a ByteArrayInputStream for the underlying byte + * array. + * + * @see ByteArrayInputStream + */ + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(this.byteArray); + } + + /** + * This implementation returns a description that includes the passed-in + * {@code description}, if any. + */ + @Override + public String getDescription() { + return "Byte array resource [" + this.description + "]"; + } + + /** + * This implementation compares the underlying byte array. + * + * @see Arrays#equals(byte[], byte[]) + */ + @Override + public boolean equals(Object obj) { + return (obj == this || (obj instanceof org.springframework.core.io.ByteArrayResource + && Arrays.equals(((ByteArrayResource) obj).byteArray, this.byteArray))); + } + + /** + * This implementation returns the hash code based on the underlying byte array. + */ + @Override + public int hashCode() { + return (byte[].class.hashCode() * 29 * this.byteArray.length); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java new file mode 100644 index 00000000..a6096207 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java @@ -0,0 +1,107 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.*; + +import org.springframework.util.StringUtils; + +/** + * Composite {@link PropertySource} implementation that iterates over a set of + * {@link PropertySource} instances. Necessary in cases where multiple property + * sources share the same name, e.g. when multiple values are supplied to + * {@code @PropertySource}. + *

+ * As of Spring 4.1.2, this class extends {@link EnumerablePropertySource} + * instead of plain {@link PropertySource}, exposing {@link #getPropertyNames()} + * based on the accumulated property names from all contained sources (as far as + * possible). + * + * @author Chris Beams + * @author Juergen Hoeller + * @author Phillip Webb + * @since 3.1.1 + */ +public class CompositePropertySource extends EnumerablePropertySource { + + private final Set> propertySources = new LinkedHashSet<>(); + + /** + * Create a new {@code CompositePropertySource}. + * + * @param name the name of the property source + */ + public CompositePropertySource(String name){ + super(name); + } + + @Override + public Object getProperty(String name) { + for (PropertySource propertySource : this.propertySources) { + Object candidate = propertySource.getProperty(name); + if (candidate != null) { + return candidate; + } + } + return null; + } + + @Override + public boolean containsProperty(String name) { + for (PropertySource propertySource : this.propertySources) { + if (propertySource.containsProperty(name)) { + return true; + } + } + return false; + } + + @Override + public String[] getPropertyNames() { + Set names = new LinkedHashSet<>(); + for (PropertySource propertySource : this.propertySources) { + if (!(propertySource instanceof EnumerablePropertySource)) { + throw new IllegalStateException( + "Failed to enumerate property names due to non-enumerable property source: " + propertySource); + } + names.addAll(Arrays.asList(((EnumerablePropertySource) propertySource).getPropertyNames())); + } + return StringUtils.toStringArray(names); + } + + /** + * Add the given {@link PropertySource} to the end of the chain. + * + * @param propertySource the PropertySource to add + */ + public void addPropertySource(PropertySource propertySource) { + this.propertySources.add(propertySource); + } + + /** + * Add the given {@link PropertySource} to the start of the chain. + * + * @param propertySource the PropertySource to add + * @since 4.1 + */ + public void addFirstPropertySource(PropertySource propertySource) { + List> existing = new ArrayList<>(this.propertySources); + this.propertySources.clear(); + this.propertySources.add(propertySource); + this.propertySources.addAll(existing); + } + + /** + * Return all property sources that this composite source holds. + * + * @since 4.1.1 + */ + public Collection> getPropertySources() { + return this.propertySources; + } + + @Override + public String toString() { + return String + .format("%s [name='%s', propertySources=%s]", getClass().getSimpleName(), this.name, this.propertySources); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java new file mode 100644 index 00000000..e5ce8639 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java @@ -0,0 +1,58 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import org.springframework.util.ObjectUtils; + +/** + * A {@link PropertySource} implementation capable of interrogating its + * underlying source object to enumerate all possible property name/value pairs. + * Exposes the {@link #getPropertyNames()} method to allow callers to introspect + * available properties without having to access the underlying source object. + * This also facilitates a more efficient implementation of + * {@link #containsProperty(String)}, in that it can call + * {@link #getPropertyNames()} and iterate through the returned array rather + * than attempting a call to {@link #getProperty(String)} which may be more + * expensive. Implementations may consider caching the result of + * {@link #getPropertyNames()} to fully exploit this performance opportunity. + *

+ * Most framework-provided {@code PropertySource} implementations are + * enumerable; a counter-example would be {@code JndiPropertySource} where, due + * to the nature of JNDI it is not possible to determine all possible property + * names at any given time; rather it is only possible to try to access a + * property (via {@link #getProperty(String)}) in order to evaluate whether it + * is present or not. + * + * @author Chris Beams + * @author Juergen Hoeller + * @since 3.1 + */ +public abstract class EnumerablePropertySource extends PropertySource { + + public EnumerablePropertySource(String name, T source){ + super(name, source); + } + + protected EnumerablePropertySource(String name){ + super(name); + } + + /** + * Return whether this {@code PropertySource} contains a property with the given + * name. + *

+ * This implementation checks for the presence of the given name within the + * {@link #getPropertyNames()} array. + * + * @param name the name of the property to find + */ + @Override + public boolean containsProperty(String name) { + return ObjectUtils.containsElement(getPropertyNames(), name); + } + + /** + * Return the names of all properties contained by the {@linkplain #getSource() + * source} object (never {@code null}). + */ + public abstract String[] getPropertyNames(); + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java new file mode 100644 index 00000000..eecc0ee4 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java @@ -0,0 +1,38 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.Map; + +import org.springframework.core.env.PropertiesPropertySource; +import org.springframework.core.env.PropertySource; +import org.springframework.util.StringUtils; + +/** + * {@link PropertySource} that reads keys and values from a {@code Map} object. + * + * @author Chris Beams + * @author Juergen Hoeller + * @since 3.1 + * @see PropertiesPropertySource + */ +public class MapPropertySource extends EnumerablePropertySource> { + + public MapPropertySource(String name, Map source){ + super(name, source); + } + + @Override + public Object getProperty(String name) { + return this.source.get(name); + } + + @Override + public boolean containsProperty(String name) { + return this.source.containsKey(name); + } + + @Override + public String[] getPropertyNames() { + return StringUtils.toStringArray(this.source.keySet()); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java new file mode 100644 index 00000000..78de18c0 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java @@ -0,0 +1,221 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.core.env.PropertyResolver; +import org.springframework.core.env.PropertySourcesPropertyResolver; + +/** + * Default implementation of the {@link PropertySources} interface. Allows + * manipulation of contained property sources and provides a constructor for + * copying an existing {@code PropertySources} instance. + *

+ * Where precedence is mentioned in methods such as {@link #addFirst} + * and {@link #addLast}, this is with regard to the order in which property + * sources will be searched when resolving a given property with a + * {@link PropertyResolver}. + * + * @author Chris Beams + * @author Juergen Hoeller + * @see PropertySourcesPropertyResolver + * @since 3.1 + */ +public class MutablePropertySources implements PropertySources { + + private final Log logger; + + private final List> propertySourceList = new CopyOnWriteArrayList<>(); + + /** + * Create a new {@link MutablePropertySources} + * object. + */ + public MutablePropertySources(){ + this.logger = LogFactory.getLog(getClass()); + } + + /** + * Create a new {@code MutablePropertySources} from the given propertySources + * object, preserving the original order of contained {@code PropertySource} + * objects. + */ + public MutablePropertySources(PropertySources propertySources){ + this(); + for (PropertySource propertySource : propertySources) { + addLast(propertySource); + } + } + + /** + * Create a new {@link MutablePropertySources} + * object and inherit the given logger, usually from an enclosing + * {@link Environment}. + */ + MutablePropertySources(Log logger){ + this.logger = logger; + } + + @Override + public boolean contains(String name) { + return this.propertySourceList.contains(PropertySource.named(name)); + } + + @Override + public PropertySource get(String name) { + int index = this.propertySourceList.indexOf(PropertySource.named(name)); + return (index != -1 ? this.propertySourceList.get(index) : null); + } + + @Override + public Iterator> iterator() { + return this.propertySourceList.iterator(); + } + + /** + * Add the given property source object with highest precedence. + */ + public void addFirst(PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + "' with highest search precedence"); + } + removeIfPresent(propertySource); + this.propertySourceList.add(0, propertySource); + } + + /** + * Add the given property source object with lowest precedence. + */ + public void addLast(PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + "' with lowest search precedence"); + } + removeIfPresent(propertySource); + this.propertySourceList.add(propertySource); + } + + /** + * Add the given property source object with precedence immediately higher than + * the named relative property source. + */ + public void addBefore(String relativePropertySourceName, PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + + "' with search precedence immediately higher than '" + relativePropertySourceName + "'"); + } + assertLegalRelativeAddition(relativePropertySourceName, propertySource); + removeIfPresent(propertySource); + int index = assertPresentAndGetIndex(relativePropertySourceName); + addAtIndex(index, propertySource); + } + + /** + * Add the given property source object with precedence immediately lower than + * the named relative property source. + */ + public void addAfter(String relativePropertySourceName, PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + + "' with search precedence immediately lower than '" + relativePropertySourceName + "'"); + } + assertLegalRelativeAddition(relativePropertySourceName, propertySource); + removeIfPresent(propertySource); + int index = assertPresentAndGetIndex(relativePropertySourceName); + addAtIndex(index + 1, propertySource); + } + + /** + * Return the precedence of the given property source, {@code -1} if not found. + */ + public int precedenceOf(PropertySource propertySource) { + return this.propertySourceList.indexOf(propertySource); + } + + /** + * Remove and return the property source with the given name, {@code null} if + * not found. + * + * @param name the name of the property source to find and remove + */ + public PropertySource remove(String name) { + if (logger.isDebugEnabled()) { + logger.debug("Removing PropertySource '" + name + "'"); + } + int index = this.propertySourceList.indexOf(PropertySource.named(name)); + return (index != -1 ? this.propertySourceList.remove(index) : null); + } + + /** + * Replace the property source with the given name with the given property + * source object. + * + * @param name the name of the property source to find and replace + * @param propertySource the replacement property source + * @throws IllegalArgumentException if no property source with the given name is + * present + * @see #contains + */ + public void replace(String name, PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Replacing PropertySource '" + name + "' with '" + propertySource.getName() + "'"); + } + int index = assertPresentAndGetIndex(name); + this.propertySourceList.set(index, propertySource); + } + + /** + * Return the number of {@link PropertySource} objects contained. + */ + public int size() { + return this.propertySourceList.size(); + } + + @Override + public String toString() { + return this.propertySourceList.toString(); + } + + /** + * Ensure that the given property source is not being added relative to itself. + */ + protected void assertLegalRelativeAddition(String relativePropertySourceName, PropertySource propertySource) { + String newPropertySourceName = propertySource.getName(); + if (relativePropertySourceName.equals(newPropertySourceName)) { + throw new IllegalArgumentException( + "PropertySource named '" + newPropertySourceName + "' cannot be added relative to itself"); + } + } + + /** + * Remove the given property source if it is present. + */ + protected void removeIfPresent(PropertySource propertySource) { + this.propertySourceList.remove(propertySource); + } + + /** + * Add the given property source at a particular index in the list. + */ + private void addAtIndex(int index, PropertySource propertySource) { + removeIfPresent(propertySource); + this.propertySourceList.add(index, propertySource); + } + + /** + * Assert that the named property source is present and return its index. + * + * @param name {@linkplain PropertySource#getName() name of the property source} + * to find + * @throws IllegalArgumentException if the named property source is not present + */ + private int assertPresentAndGetIndex(String name) { + int index = this.propertySourceList.indexOf(PropertySource.named(name)); + if (index == -1) { + throw new IllegalArgumentException("PropertySource named '" + name + "' does not exist"); + } + return index; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java new file mode 100644 index 00000000..12fe686a --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java @@ -0,0 +1,34 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.Map; +import java.util.Properties; + +import org.springframework.core.env.PropertySource; + +/** + * {@link PropertySource} implementation that extracts properties from a + * {@link Properties} object. + *

+ * Note that because a {@code Properties} object is technically an + * {@code } {@link java.util.Hashtable Hashtable}, one may + * contain non-{@code String} keys or values. This implementation, however is + * restricted to accessing only {@code String}-based keys and values, in the + * same fashion as {@link Properties#getProperty} and + * {@link Properties#setProperty}. + * + * @author Chris Beams + * @author Juergen Hoeller + * @since 3.1 + */ +public class PropertiesPropertySource extends MapPropertySource { + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public PropertiesPropertySource(String name, Properties source){ + super(name, (Map) source); + } + + protected PropertiesPropertySource(String name, Map source){ + super(name, source); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java new file mode 100644 index 00000000..f7e4cc34 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java @@ -0,0 +1,239 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; + +/** + * Abstract base class representing a source of name/value property pairs. The + * underlying {@linkplain #getSource() source object} may be of any type + * {@code T} that encapsulates properties. Examples include + * {@link java.util.Properties} objects, {@link java.util.Map} objects, + * {@code ServletContext} and {@code ServletConfig} objects (for access to init + * parameters). Explore the {@code PropertySource} type hierarchy to see + * provided implementations. + *

+ * {@code PropertySource} objects are not typically used in isolation, but + * rather through a {@link PropertySources} object, which aggregates property + * sources and in conjunction with a {@link PropertyResolver} implementation + * that can perform precedence-based searches across the set of + * {@code PropertySources}. + *

+ * {@code PropertySource} identity is determined not based on the content of + * encapsulated properties, but rather based on the {@link #getName() name} of + * the {@code PropertySource} alone. This is useful for manipulating + * {@code PropertySource} objects when in collection contexts. See operations in + * {@link MutablePropertySources} as well as the {@link #named(String)} and + * {@link #toString()} methods for details. + *

+ * Note that when working + * with @{@link org.springframework.context.annotation.Configuration + * Configuration} classes that the @{@link PropertySource PropertySource} + * annotation provides a convenient and declarative way of adding property + * sources to the enclosing {@code Environment}. + * + * @author Chris Beams + * @since 3.1 + * @see PropertySources + * @see MutablePropertySources + * @see PropertySource + */ +public abstract class PropertySource { + + protected final Log logger = LogFactory.getLog(getClass()); + + protected final String name; + + protected final T source; + + /** + * Create a new {@code PropertySource} with the given name and source object. + */ + public PropertySource(String name, T source){ + Assert.hasText(name, "Property source name must contain at least one character"); + Assert.notNull(source, "Property source must not be null"); + this.name = name; + this.source = source; + } + + /** + * Create a new {@code PropertySource} with the given name and with a new + * {@code Object} instance as the underlying source. + *

+ * Often useful in testing scenarios when creating anonymous implementations + * that never query an actual source but rather return hard-coded values. + */ + @SuppressWarnings("unchecked") + public PropertySource(String name){ + this(name, (T) new Object()); + } + + /** + * Return the name of this {@code PropertySource} + */ + public String getName() { + return this.name; + } + + /** + * Return the underlying source object for this {@code PropertySource}. + */ + public T getSource() { + return this.source; + } + + /** + * Return whether this {@code PropertySource} contains the given name. + *

+ * This implementation simply checks for a {@code null} return value from + * {@link #getProperty(String)}. Subclasses may wish to implement a more + * efficient algorithm if possible. + * + * @param name the property name to find + */ + public boolean containsProperty(String name) { + return (getProperty(name) != null); + } + + /** + * Return the value associated with the given name, or {@code null} if not + * found. + * + * @param name the property to find + */ + public abstract Object getProperty(String name); + + /** + * This {@code PropertySource} object is equal to the given object if: + *

    + *
  • they are the same instance + *
  • the {@code name} properties for both objects are equal + *
+ *

+ * No properties other than {@code name} are evaluated. + */ + @Override + public boolean equals(Object obj) { + return (this == obj || (obj instanceof PropertySource + && ObjectUtils.nullSafeEquals(this.name, ((PropertySource) obj).name))); + } + + /** + * Return a hash code derived from the {@code name} property of this + * {@code PropertySource} object. + */ + @Override + public int hashCode() { + return ObjectUtils.nullSafeHashCode(this.name); + } + + /** + * Produce concise output (type and name) if the current log level does not + * include debug. If debug is enabled, produce verbose output including the hash + * code of the PropertySource instance and every name/value property pair. + *

+ * This variable verbosity is useful as a property source such as system + * properties or environment variables may contain an arbitrary number of + * property pairs, potentially leading to difficult to read exception and log + * messages. + * + * @see Log#isDebugEnabled() + */ + @Override + public String toString() { + if (logger.isDebugEnabled()) { + return getClass().getSimpleName() + "@" + System.identityHashCode(this) + " {name='" + this.name + + "', properties=" + this.source + "}"; + } else { + return getClass().getSimpleName() + " {name='" + this.name + "'}"; + } + } + + /** + * Return a {@code PropertySource} implementation intended for collection + * comparison purposes only. + *

+ * Primarily for internal use, but given a collection of {@code PropertySource} + * objects, may be used as follows: + * + * 

+     *
+     * {
+     *     @code
+     *     List> sources = new ArrayList>();
+     *     sources.add(new MapPropertySource("sourceA", mapA));
+     *     sources.add(new MapPropertySource("sourceB", mapB));
+     *     assert sources.contains(PropertySource.named("sourceA"));
+     *     assert sources.contains(PropertySource.named("sourceB"));
+     *     assert !sources.contains(PropertySource.named("sourceC"));
+     * }
+     *
+ * + * The returned {@code PropertySource} will throw + * {@code UnsupportedOperationException} if any methods other than + * {@code equals(Object)}, {@code hashCode()}, and {@code toString()} are + * called. + * + * @param name the name of the comparison {@code PropertySource} to be created + * and returned. + */ + public static PropertySource named(String name) { + return new ComparisonPropertySource(name); + } + + /** + * {@code PropertySource} to be used as a placeholder in cases where an actual + * property source cannot be eagerly initialized at application context creation + * time. For example, a {@code ServletContext}-based property source must wait + * until the {@code ServletContext} object is available to its enclosing + * {@code ApplicationContext}. In such cases, a stub should be used to hold the + * intended default position/order of the property source, then be replaced + * during context refresh. + * + * @see org.springframework.web.context.support.StandardServletEnvironment + * @see org.springframework.web.context.support.ServletContextPropertySource + */ + public static class StubPropertySource extends PropertySource { + + public StubPropertySource(String name){ + super(name, new Object()); + } + + /** + * Always returns {@code null}. + */ + @Override + public String getProperty(String name) { + return null; + } + } + + /** + * @see PropertySource#named(String) + */ + static class ComparisonPropertySource extends StubPropertySource { + + private static final String USAGE_ERROR = "ComparisonPropertySource instances are for use with collection comparison only"; + + public ComparisonPropertySource(String name){ + super(name); + } + + @Override + public Object getSource() { + throw new UnsupportedOperationException(USAGE_ERROR); + } + + @Override + public boolean containsProperty(String name) { + throw new UnsupportedOperationException(USAGE_ERROR); + } + + @Override + public String getProperty(String name) { + throw new UnsupportedOperationException(USAGE_ERROR); + } + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java new file mode 100644 index 00000000..982f61b8 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java @@ -0,0 +1,35 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.IOException; + +import org.springframework.core.io.support.SpringFactoriesLoader; + +/** + * Strategy interface located via {@link SpringFactoriesLoader} and used to load + * a {@link PropertySource}. + * + * @author Dave Syer + * @author Phillip Webb + */ +public interface PropertySourceLoader { + + /** + * Returns the file extensions that the loader supports (excluding the '.'). + * + * @return the file extensions + */ + String[] getFileExtensions(); + + /** + * Load the resource into a property source. + * + * @param name the name of the property source + * @param resource the resource to load + * @param profile the name of the profile to load or {@code null}. The profile + * can be used to load multi-document files (such as YAML). Simple property + * formats should {@code null} when asked to load a profile. + * @return a property source or {@code null} + * @throws IOException if the source cannot be loaded + */ + PropertySource load(String name, Resource resource, String profile) throws IOException; +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java new file mode 100644 index 00000000..3cd08c9a --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java @@ -0,0 +1,25 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +/** + * Holder containing one or more {@link PropertySource} objects. + * + * @author Chris Beams + * @since 3.1 + */ +public interface PropertySources extends Iterable> { + + /** + * Return whether a property source with the given name is contained. + * + * @param name the {@linkplain PropertySource#getName() name of the property source} to find + */ + boolean contains(String name); + + /** + * Return the property source with the given name, {@code null} if not found. + * + * @param name the {@linkplain PropertySource#getName() name of the property source} to find + */ + PropertySource get(String name); + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java new file mode 100644 index 00000000..88301f20 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java @@ -0,0 +1,57 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URL; + +import org.springframework.core.io.*; +import org.springframework.core.io.ByteArrayResource; + +/** + * Interface for a resource descriptor that abstracts from the actual type of + * underlying resource, such as a file or class path resource. + *

+ * An InputStream can be opened for every resource if it exists in physical + * form, but a URL or File handle can just be returned for certain resources. + * The actual behavior is implementation-specific. + * + * @author Juergen Hoeller + * @since 28.12.2003 + * @see #getInputStream() + * @see #getURL() + * @see #getURI() + * @see #getFile() + * @see WritableResource + * @see ContextResource + * @see UrlResource + * @see ClassPathResource + * @see FileSystemResource + * @see PathResource + * @see ByteArrayResource + * @see InputStreamResource + */ +public interface Resource extends InputStreamSource { + + boolean exists(); + + boolean isReadable(); + + boolean isOpen(); + + URL getURL() throws IOException; + + URI getURI() throws IOException; + + File getFile() throws IOException; + + long contentLength() throws IOException; + + long lastModified() throws IOException; + + org.springframework.core.io.Resource createRelative(String var1) throws IOException; + + String getFilename(); + + String getDescription(); +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java new file mode 100644 index 00000000..df25eba1 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java @@ -0,0 +1,182 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.*; + +import org.springframework.core.env.Environment; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +/** + * {@link YamlProcessor.DocumentMatcher} backed by + * {@link Environment#getActiveProfiles()}. A YAML document may define a + * "spring.profiles" element as a comma-separated list of Spring profile names, + * optionally negated using the {@code !} character. If both negated and + * non-negated profiles are specified for a single document, at least one + * non-negated profile must match and no negated profiles may match. + * + * @author Dave Syer + * @author Matt Benson + * @author Phillip Webb + * @author Andy Wilkinson + */ +public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatcher { + + private String[] activeProfiles = new String[0]; + + public SpringProfileDocumentMatcher(){ + } + + public SpringProfileDocumentMatcher(String... profiles){ + addActiveProfiles(profiles); + } + + public void addActiveProfiles(String... profiles) { + LinkedHashSet set = new LinkedHashSet<>(Arrays.asList(this.activeProfiles)); + Collections.addAll(set, profiles); + this.activeProfiles = set.toArray(new String[set.size()]); + } + + @Override + public YamlProcessor.MatchStatus matches(Properties properties) { + List profiles = extractSpringProfiles(properties); + ProfilesMatcher profilesMatcher = getProfilesMatcher(); + Set negative = extractProfiles(profiles, ProfileType.NEGATIVE); + Set positive = extractProfiles(profiles, ProfileType.POSITIVE); + if (!CollectionUtils.isEmpty(negative)) { + if (profilesMatcher.matches(negative) == YamlProcessor.MatchStatus.FOUND) { + return YamlProcessor.MatchStatus.NOT_FOUND; + } + if (CollectionUtils.isEmpty(positive)) { + return YamlProcessor.MatchStatus.FOUND; + } + } + return profilesMatcher.matches(positive); + } + + private List extractSpringProfiles(Properties properties) { + SpringProperties springProperties = new SpringProperties(); + MutablePropertySources propertySources = new MutablePropertySources(); + propertySources.addFirst(new PropertiesPropertySource("profiles", properties)); + // PropertyValues propertyValues = new PropertySourcesPropertyValues( + // propertySources); + // new RelaxedDataBinder(springProperties, "spring").bind(propertyValues); + // TODO + List profiles = springProperties.getProfiles(); + return profiles; + } + + private ProfilesMatcher getProfilesMatcher() { + return (this.activeProfiles.length != 0 ? new ActiveProfilesMatcher( + new HashSet<>(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher()); + } + + private Set extractProfiles(List profiles, ProfileType type) { + if (CollectionUtils.isEmpty(profiles)) { + return null; + } + Set extractedProfiles = new HashSet<>(); + for (String candidate : profiles) { + ProfileType candidateType = ProfileType.POSITIVE; + if (candidate.startsWith("!")) { + candidateType = ProfileType.NEGATIVE; + } + if (candidateType == type) { + extractedProfiles.add(type != ProfileType.POSITIVE ? candidate.substring(1) : candidate); + } + } + return extractedProfiles; + } + + /** + * Profile match types. + */ + enum ProfileType { + + POSITIVE, NEGATIVE + + } + + /** + * Base class for profile matchers. + */ + private abstract static class ProfilesMatcher { + + public final YamlProcessor.MatchStatus matches(Set profiles) { + if (CollectionUtils.isEmpty(profiles)) { + return YamlProcessor.MatchStatus.ABSTAIN; + } + return doMatches(profiles); + } + + protected abstract YamlProcessor.MatchStatus doMatches(Set profiles); + + } + + /** + * {@link ProfilesMatcher} that matches when a value in {@code spring.profiles} + * is also in {@code spring.profiles.active}. + */ + private static class ActiveProfilesMatcher extends ProfilesMatcher { + + private final Set activeProfiles; + + ActiveProfilesMatcher(Set activeProfiles){ + this.activeProfiles = activeProfiles; + } + + @Override + protected YamlProcessor.MatchStatus doMatches(Set profiles) { + if (profiles.isEmpty()) { + return YamlProcessor.MatchStatus.NOT_FOUND; + } + for (String activeProfile : this.activeProfiles) { + if (profiles.contains(activeProfile)) { + return YamlProcessor.MatchStatus.FOUND; + } + } + return YamlProcessor.MatchStatus.NOT_FOUND; + } + + } + + /** + * {@link ProfilesMatcher} that matches when {@code + * spring.profiles} is empty or contains a value with no text. + * + * @see StringUtils#hasText(String) + */ + private static class EmptyProfilesMatcher extends ProfilesMatcher { + + @Override + public YamlProcessor.MatchStatus doMatches(Set springProfiles) { + if (springProfiles.isEmpty()) { + return YamlProcessor.MatchStatus.FOUND; + } + for (String profile : springProfiles) { + if (!StringUtils.hasText(profile)) { + return YamlProcessor.MatchStatus.FOUND; + } + } + return YamlProcessor.MatchStatus.NOT_FOUND; + } + + } + + /** + * Class for binding {@code spring.profiles} property. + */ + static class SpringProperties { + + private List profiles = new ArrayList<>(); + + public List getProfiles() { + return this.profiles; + } + + public void setProfiles(List profiles) { + this.profiles = profiles; + } + + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java new file mode 100644 index 00000000..18c5ebb8 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java @@ -0,0 +1,417 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.IOException; +import java.io.Reader; +import java.util.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.util.Assert; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.parser.ParserException; +import org.yaml.snakeyaml.reader.UnicodeReader; + +/** + * Base class for YAML factories. + * + * @author Dave Syer + * @author Juergen Hoeller + * @since 4.1 + */ +public abstract class YamlProcessor { + + private final Log logger = LogFactory.getLog(getClass()); + + private ResolutionMethod resolutionMethod = ResolutionMethod.OVERRIDE; + + private Resource[] resources = new Resource[0]; + + private List documentMatchers = Collections.emptyList(); + + private boolean matchDefault = true; + + /** + * A map of document matchers allowing callers to selectively use only some of + * the documents in a YAML resource. In YAML documents are separated by + * --- lines, and each document is converted to properties before + * the match is made. E.g. + * + * 

+     * environment: dev
+     * url: http://dev.bar.com
+     * name: Developer Setup
+     * ---
+     * environment: prod
+     * url:http://foo.bar.com
+     * name: My Cool App
+     *
+ * + * when mapped with + * + * 
+     * setDocumentMatchers(properties -> ("prod"
+     *     .equals(properties.getProperty("environment")) ? MatchStatus.FOUND : MatchStatus.NOT_FOUND));
+     *
+ * + * would end up as + * + * 
+     * environment=prod
+     * url=http://foo.bar.com
+     * name=My Cool App
+     *
+ */ + public void setDocumentMatchers(DocumentMatcher... matchers) { + this.documentMatchers = Arrays.asList(matchers); + } + + /** + * Flag indicating that a document for which all the + * {@link #setDocumentMatchers(DocumentMatcher...) document matchers} abstain + * will nevertheless match. Default is {@code true}. + */ + public void setMatchDefault(boolean matchDefault) { + this.matchDefault = matchDefault; + } + + /** + * Method to use for resolving resources. Each resource will be converted to a + * Map, so this property is used to decide which map entries to keep in the + * final output from this factory. Default is {@link ResolutionMethod#OVERRIDE}. + */ + public void setResolutionMethod(ResolutionMethod resolutionMethod) { + Assert.notNull(resolutionMethod, "ResolutionMethod must not be null"); + this.resolutionMethod = resolutionMethod; + } + + /** + * Set locations of YAML {@link Resource resources} to be loaded. + * + * @see ResolutionMethod + */ + public void setResources(Resource... resources) { + this.resources = resources; + } + + /** + * Provide an opportunity for subclasses to process the Yaml parsed from the + * supplied resources. Each resource is parsed in turn and the documents inside + * checked against the {@link #setDocumentMatchers(DocumentMatcher...) + * matchers}. If a document matches it is passed into the callback, along with + * its representation as Properties. Depending on the + * {@link #setResolutionMethod(ResolutionMethod)} not all of the documents will + * be parsed. + * + * @param callback a callback to delegate to once matching documents are found + * @see #createYaml() + */ + protected void process(MatchCallback callback) { + Yaml yaml = createYaml(); + for (Resource resource : this.resources) { + boolean found = process(callback, yaml, resource); + if (this.resolutionMethod == ResolutionMethod.FIRST_FOUND && found) { + return; + } + } + } + + /** + * Create the {@link Yaml} instance to use. + */ + protected Yaml createYaml() { + return new Yaml(new StrictMapAppenderConstructor()); + } + + private boolean process(MatchCallback callback, Yaml yaml, Resource resource) { + int count = 0; + try { + if (logger.isDebugEnabled()) { + logger.debug("Loading from YAML: " + resource); + } + try (Reader reader = new UnicodeReader(resource.getInputStream())) { + for (Object object : yaml.loadAll(reader)) { + if (object != null && process(asMap(object), callback)) { + count++; + if (this.resolutionMethod == ResolutionMethod.FIRST_FOUND) { + break; + } + } + } + if (logger.isDebugEnabled()) { + logger.debug( + "Loaded " + count + " document" + (count > 1 ? "s" : "") + " from YAML resource: " + resource); + } + } + } catch (IOException ex) { + handleProcessError(resource, ex); + } + return (count > 0); + } + + private void handleProcessError(Resource resource, IOException ex) { + if (this.resolutionMethod != ResolutionMethod.FIRST_FOUND + && this.resolutionMethod != ResolutionMethod.OVERRIDE_AND_IGNORE) { + throw new IllegalStateException(ex); + } + if (logger.isWarnEnabled()) { + logger.warn("Could not load map from " + resource + ": " + ex.getMessage()); + } + } + + @SuppressWarnings("unchecked") + private Map asMap(Object object) { + // YAML can have numbers as keys + Map result = new LinkedHashMap<>(); + if (!(object instanceof Map)) { + // A document can be a text literal + result.put("document", object); + return result; + } + + Map map = (Map) object; + for (Map.Entry entry : map.entrySet()) { + Object value = entry.getValue(); + if (value instanceof Map) { + value = asMap(value); + } + Object key = entry.getKey(); + if (key instanceof CharSequence) { + result.put(key.toString(), value); + } else { + // It has to be a map key in this case + result.put("[" + key.toString() + "]", value); + } + } + return result; + } + + private boolean process(Map map, MatchCallback callback) { + @SuppressWarnings("serial") + Properties properties = new Properties() { + + @Override + public String getProperty(String key) { + Object value = get(key); + return (value != null ? value.toString() : null); + } + }; + properties.putAll(getFlattenedMap(map)); + + if (this.documentMatchers.isEmpty()) { + if (logger.isDebugEnabled()) { + logger.debug("Merging document (no matchers set): " + map); + } + callback.process(properties, map); + return true; + } + + MatchStatus result = MatchStatus.ABSTAIN; + for (DocumentMatcher matcher : this.documentMatchers) { + MatchStatus match = matcher.matches(properties); + result = MatchStatus.getMostSpecific(match, result); + if (match == MatchStatus.FOUND) { + if (logger.isDebugEnabled()) { + logger.debug("Matched document with document matcher: " + properties); + } + callback.process(properties, map); + return true; + } + } + + if (result == MatchStatus.ABSTAIN && this.matchDefault) { + if (logger.isDebugEnabled()) { + logger.debug("Matched document with default matcher: " + map); + } + callback.process(properties, map); + return true; + } + + if (logger.isDebugEnabled()) { + logger.debug("Unmatched document: " + map); + } + return false; + } + + /** + * Return a flattened version of the given map, recursively following any nested + * Map or Collection values. Entries from the resulting map retain the same + * order as the source. When called with the Map from a {@link MatchCallback} + * the result will contain the same values as the {@link MatchCallback} + * Properties. + * + * @param source the source map + * @return a flattened map + * @since 4.1.3 + */ + protected final Map getFlattenedMap(Map source) { + Map result = new LinkedHashMap<>(); + buildFlattenedMap(result, source, null); + return result; + } + + private static boolean containsText(CharSequence str) { + int strLen = str.length(); + for (int i = 0; i < strLen; i++) { + if (!Character.isWhitespace(str.charAt(i))) { + return true; + } + } + return false; + } + + private void buildFlattenedMap(Map result, Map source, String path) { + for (Map.Entry entry : source.entrySet()) { + String key = entry.getKey(); + if (path != null && !path.isEmpty() && containsText(path)) { + if (key.startsWith("[")) { + key = path + key; + } else { + key = path + '.' + key; + } + } + Object value = entry.getValue(); + if (value instanceof String) { + result.put(key, value); + } else if (value instanceof Map) { + // Need a compound key + @SuppressWarnings("unchecked") + Map map = (Map) value; + buildFlattenedMap(result, map, key); + } else if (value instanceof Collection) { + // Need a compound key + @SuppressWarnings("unchecked") + Collection collection = (Collection) value; + int count = 0; + for (Object object : collection) { + buildFlattenedMap(result, Collections.singletonMap("[" + (count++) + "]", object), key); + } + } else { + result.put(key, (value != null ? value : "")); + } + } + } + + /** + * Callback interface used to process the YAML parsing results. + */ + public interface MatchCallback { + + /** + * Process the given representation of the parsing results. + * + * @param properties the properties to process (as a flattened representation + * with indexed keys in case of a collection or map) + * @param map the result map (preserving the original value structure in the + * YAML document) + */ + void process(Properties properties, Map map); + } + + /** + * Strategy interface used to test if properties match. + */ + public interface DocumentMatcher { + + /** + * Test if the given properties match. + * + * @param properties the properties to test + * @return the status of the match + */ + MatchStatus matches(Properties properties); + } + + /** + * Status returned from {@link DocumentMatcher#matches(Properties)} + */ + public enum MatchStatus { + + /** + * A match was found. + */ + FOUND, + + /** + * No match was found. + */ + NOT_FOUND, + + /** + * The matcher should not be considered. + */ + ABSTAIN; + + /** + * Compare two {@link MatchStatus} items, returning the most specific status. + */ + public static MatchStatus getMostSpecific(MatchStatus a, MatchStatus b) { + return (a.ordinal() < b.ordinal() ? a : b); + } + } + + /** + * Method to use for resolving resources. + */ + public enum ResolutionMethod { + + /** + * Replace values from earlier in the list. + */ + OVERRIDE, + + /** + * Replace values from earlier in the list, ignoring any failures. + */ + OVERRIDE_AND_IGNORE, + + /** + * Take the first resource in the list that exists and use just that. + */ + FIRST_FOUND + } + + /** + * A specialized {@link Constructor} that checks for duplicate keys. + */ + protected static class StrictMapAppenderConstructor extends Constructor { + + // Declared as public for use in subclasses + public StrictMapAppenderConstructor(){ + super(); + } + + @Override + protected Map constructMapping(MappingNode node) { + try { + return super.constructMapping(node); + } catch (IllegalStateException ex) { + throw new ParserException("while parsing MappingNode", + node.getStartMark(), + ex.getMessage(), + node.getEndMark()); + } + } + + @Override + protected Map createDefaultMap() { + final Map delegate = super.createDefaultMap(); + return new AbstractMap() { + + @Override + public Object put(Object key, Object value) { + if (delegate.containsKey(key)) { + throw new IllegalStateException("Duplicate key: " + key); + } + return delegate.put(key, value); + } + + @Override + public Set> entrySet() { + return delegate.entrySet(); + } + }; + } + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java new file mode 100644 index 00000000..57c15d1b --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java @@ -0,0 +1,80 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.regex.Pattern; + +import org.springframework.util.ClassUtils; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.representer.Representer; +import org.yaml.snakeyaml.resolver.Resolver; + +/** + * Strategy to load '.yml' (or '.yaml') files into a {@link PropertySource}. + * + * @author Dave Syer + * @author Phillip Webb + * @author Andy Wilkinson + */ +public class YamlPropertySourceLoader implements PropertySourceLoader { + + @Override + public String[] getFileExtensions() { + return new String[] { "yml", "yaml" }; + } + + @Override + public PropertySource load(String name, Resource resource, String profile) throws IOException { + if (ClassUtils.isPresent("org.yaml.snakeyaml.Yaml", null)) { + Processor processor = new Processor(resource, profile); + Map source = processor.process(); + if (!source.isEmpty()) { + return new MapPropertySource(name, source); + } + } + return null; + } + + /** + * {@link YamlProcessor} to create a {@link Map} containing the property values. + * Similar to {@link YamlPropertiesFactoryBean} but retains the order of + * entries. + */ + private static class Processor extends YamlProcessor { + + Processor(Resource resource, String profile){ + if (profile == null) { + setMatchDefault(true); + setDocumentMatchers(new SpringProfileDocumentMatcher()); + } else { + setMatchDefault(false); + setDocumentMatchers(new SpringProfileDocumentMatcher(profile)); + } + setResources(resource); + } + + @Override + protected Yaml createYaml() { + return new Yaml(new StrictMapAppenderConstructor(), new Representer(), new DumperOptions(), new Resolver() { + + @Override + public void addImplicitResolver(Tag tag, Pattern regexp, String first) { + if (tag == Tag.TIMESTAMP) { + return; + } + super.addImplicitResolver(tag, regexp, first); + } + }); + } + + public Map process() { + final Map result = new LinkedHashMap<>(); + process((properties, map) -> result.putAll(getFlattenedMap(map))); + return result; + } + + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/YamlUtils.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/YamlUtils.java deleted file mode 100644 index b9d98f31..00000000 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/YamlUtils.java +++ /dev/null @@ -1,98 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.support; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import org.apache.commons.lang.StringUtils; -import org.springframework.boot.context.properties.bind.Bindable; -import org.springframework.boot.context.properties.bind.Binder; -import org.springframework.boot.context.properties.source.ConfigurationPropertySource; -import org.springframework.boot.context.properties.source.MapConfigurationPropertySource; -import org.springframework.boot.env.YamlPropertySourceLoader; -import org.springframework.boot.origin.OriginTrackedValue; -import org.springframework.core.env.PropertySource; -import org.springframework.core.io.ByteArrayResource; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.Resource; - -public class YamlUtils { - - public static T resourceYmlToObj(String resource, String prefix, Class clazz) { - ClassPathResource classPathResource = new ClassPathResource(resource); - - String content; - try (InputStream inputStream = classPathResource.getInputStream()) { - - ByteArrayOutputStream result = new ByteArrayOutputStream(); - byte[] buffer = new byte[1024]; - int length; - while ((length = inputStream.read(buffer)) != -1) { - result.write(buffer, 0, length); - } - content = result.toString("UTF-8"); - } catch (IOException e) { - throw new RuntimeException(e); - } - return ymlToObj(prefix, content, clazz); - } - - public static T ymlToObj(String content, Class clazz) { - return ymlToObj("", content, clazz, null, null); - } - - public static T ymlToObj(String prefix, String content, Class clazz) { - return ymlToObj(prefix, content, clazz, null, null); - } - - public static T ymlToObj(String prefix, String content, Class clazz, String charset) { - return ymlToObj(prefix, content, clazz, charset, null); - } - - public static T ymlToObj(String prefix, String content, Class clazz, String charset, - Properties baseProperties) { - try { - prefix = StringUtils.trimToEmpty(prefix); - byte[] contentBytes; - if (charset == null) { - contentBytes = content.getBytes(StandardCharsets.UTF_8); - } else { - contentBytes = content.getBytes(charset); - } - YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader(); - Resource configResource = new ByteArrayResource(contentBytes); - List> propertySources = propertySourceLoader.load("manualBindConfig", configResource); - - if (propertySources == null || propertySources.isEmpty()) { - return null; - } - - PropertySource propertySource = propertySources.get(0); - - Properties properties = new Properties(); - - properties.putAll((Map) propertySource.getSource()); - - for (Map.Entry entry : properties.entrySet()) { - Object value = entry.getValue(); - if (value instanceof OriginTrackedValue) { - entry.setValue(((OriginTrackedValue) value).getValue()); - } - } - - if (baseProperties != null) { - baseProperties.putAll(properties); - properties = baseProperties; - } - - ConfigurationPropertySource sources = new MapConfigurationPropertySource(properties); - Binder binder = new Binder(sources); - return binder.bind(prefix, Bindable.of(clazz)).get(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} diff --git a/client-adapter/common/src/test/java/com/alibaba/otter/canal/client/adapter/support/YamlUtilsTest.java b/client-adapter/common/src/test/java/com/alibaba/otter/canal/client/adapter/support/YamlUtilsTest.java deleted file mode 100644 index d64ffe26..00000000 --- a/client-adapter/common/src/test/java/com/alibaba/otter/canal/client/adapter/support/YamlUtilsTest.java +++ /dev/null @@ -1,229 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.support; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Properties; - -import org.junit.Assert; -import org.junit.Test; - -public class YamlUtilsTest { - - @Test - public void testLoadConfigToYml() { - String configStr="dataSourceKey: defaultDS\n" - + "destination: example\n" - + "groupId: g1\n" - + "outerAdapterKey: mysql1\n" - + "concurrent: true\n" - + "dbMapping:\n" - + " database: mytest\n" - + " table: user\n" - + " targetTable: mytest2.user\n" - + " targetPk:\n" - + " id: id\n" - + "# mapAll: true\n" - + " targetColumns:\n" - + " id:\n" - + " name:\n" - + " role_id:\n" - + " c_time:\n" - + " test1:\n" - + " etlCondition: \"where c_time>={}\"\n" - + " commitBatch: 3000 # 批量提交的大小"; - - MappingConfig config = YamlUtils.ymlToObj(null, configStr, MappingConfig.class, null, new Properties()); - - Assert.assertNotNull(config); - Assert.assertEquals(config.getDestination(), "example"); - Assert.assertEquals(config.getOuterAdapterKey(), "mysql1"); - Assert.assertEquals(config.getDbMapping().getDatabase(), "mytest"); - Assert.assertEquals(config.getDbMapping().getTargetColumns().size(), 5); - } - - private static class MappingConfig { - private String dataSourceKey; - - private String destination; - - private String groupId; - - private String outerAdapterKey; - - private boolean concurrent = false; - - private DbMapping dbMapping; - - public String getDataSourceKey() { - return dataSourceKey; - } - - public void setDataSourceKey(String dataSourceKey) { - this.dataSourceKey = dataSourceKey; - } - - public String getDestination() { - return destination; - } - - public void setDestination(String destination) { - this.destination = destination; - } - - public String getGroupId() { - return groupId; - } - - public void setGroupId(String groupId) { - this.groupId = groupId; - } - - public String getOuterAdapterKey() { - return outerAdapterKey; - } - - public void setOuterAdapterKey(String outerAdapterKey) { - this.outerAdapterKey = outerAdapterKey; - } - - public boolean isConcurrent() { - return concurrent; - } - - public void setConcurrent(boolean concurrent) { - this.concurrent = concurrent; - } - - public DbMapping getDbMapping() { - return dbMapping; - } - - public void setDbMapping(DbMapping dbMapping) { - this.dbMapping = dbMapping; - } - } - - private static class DbMapping { - private boolean mirrorDb = false; // 是否镜像库 - private String database; // 数据库名或schema名 - private String table; // 表名 - private Map targetPk = new LinkedHashMap<>(); // 目标表主键字段 - private boolean mapAll = false; // 映射所有字段 - private String targetDb; // 目标库名 - private String targetTable; // 目标表名 - private Map targetColumns; // 目标表字段映射 - - private boolean caseInsensitive = false; // 目标表不区分大小写，默认是否 - - private String etlCondition; // etl条件sql - - private int readBatch = 5000; - private int commitBatch = 5000; // etl等批量提交大小 - - private Map allMapColumns; - - public boolean isMirrorDb() { - return mirrorDb; - } - - public void setMirrorDb(boolean mirrorDb) { - this.mirrorDb = mirrorDb; - } - - public String getDatabase() { - return database; - } - - public void setDatabase(String database) { - this.database = database; - } - - public String getTable() { - return table; - } - - public void setTable(String table) { - this.table = table; - } - - public Map getTargetPk() { - return targetPk; - } - - public void setTargetPk(Map targetPk) { - this.targetPk = targetPk; - } - - public boolean isMapAll() { - return mapAll; - } - - public void setMapAll(boolean mapAll) { - this.mapAll = mapAll; - } - - public String getTargetDb() { - return targetDb; - } - - public void setTargetDb(String targetDb) { - this.targetDb = targetDb; - } - - public String getTargetTable() { - return targetTable; - } - - public void setTargetTable(String targetTable) { - this.targetTable = targetTable; - } - - public Map getTargetColumns() { - return targetColumns; - } - - public void setTargetColumns(Map targetColumns) { - this.targetColumns = targetColumns; - } - - public boolean isCaseInsensitive() { - return caseInsensitive; - } - - public void setCaseInsensitive(boolean caseInsensitive) { - this.caseInsensitive = caseInsensitive; - } - - public String getEtlCondition() { - return etlCondition; - } - - public void setEtlCondition(String etlCondition) { - this.etlCondition = etlCondition; - } - - public int getReadBatch() { - return readBatch; - } - - public void setReadBatch(int readBatch) { - this.readBatch = readBatch; - } - - public int getCommitBatch() { - return commitBatch; - } - - public void setCommitBatch(int commitBatch) { - this.commitBatch = commitBatch; - } - - public Map getAllMapColumns() { - return allMapColumns; - } - - public void setAllMapColumns(Map allMapColumns) { - this.allMapColumns = allMapColumns; - } - } -} diff --git a/client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfigLoader.java b/client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfigLoader.java index 3056e334..6b717a5d 100644 --- a/client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfigLoader.java +++ b/client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfigLoader.java @@ -1,6 +1,5 @@ package com.alibaba.otter.canal.client.adapter.es.core.config; -import com.alibaba.otter.canal.client.adapter.support.YamlUtils; import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; @@ -8,6 +7,7 @@ import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; /** @@ -28,7 +28,7 @@ public class ESSyncConfigLoader { String esv = envProperties.getProperty("es.version"); Map configContentMap = MappingConfigsLoader.loadConfigs(esv); configContentMap.forEach((fileName, content) -> { - ESSyncConfig config = YamlUtils.ymlToObj(null, content, ESSyncConfig.class, null, envProperties); + ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null, content, ESSyncConfig.class, null, envProperties); if (config == null) { return; } diff --git a/client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java b/client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java index fbda587b..fbd91651 100644 --- a/client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java +++ b/client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java @@ -1,8 +1,12 @@ package com.alibaba.otter.canal.client.adapter.es.core.monitor; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; +import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; +import com.alibaba.otter.canal.client.adapter.support.Util; import java.io.File; import java.util.Properties; - import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.io.monitor.FileAlterationListenerAdaptor; import org.apache.commons.io.monitor.FileAlterationMonitor; @@ -10,14 +14,6 @@ import org.apache.commons.io.monitor.FileAlterationObserver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter; -import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; -import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; -import com.alibaba.otter.canal.client.adapter.support.Util; -import com.alibaba.otter.canal.client.adapter.support.YamlUtils; - -; - public class ESConfigMonitor { private static final Logger logger = LoggerFactory.getLogger(ESConfigMonitor.class); @@ -64,7 +60,7 @@ public class ESConfigMonitor { try { // 加载新增的配置文件 String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); - ESSyncConfig config = YamlUtils.ymlToObj(null, + ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null, configContent, ESSyncConfig.class, null, @@ -97,7 +93,7 @@ public class ESConfigMonitor { onFileDelete(file); return; } - ESSyncConfig config = YamlUtils.ymlToObj(null, + ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null, configContent, ESSyncConfig.class, null, diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java index 02a1d376..2b43492f 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java @@ -1,6 +1,5 @@ package com.alibaba.otter.canal.client.adapter.hbase.config; -import com.alibaba.otter.canal.client.adapter.support.YamlUtils; import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; @@ -8,6 +7,7 @@ import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; /** @@ -32,7 +32,8 @@ public class MappingConfigLoader { Map configContentMap = MappingConfigsLoader.loadConfigs("hbase"); configContentMap.forEach((fileName, content) -> { - MappingConfig config = YamlUtils.ymlToObj(null, content, MappingConfig.class, null, envProperties); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); if (config == null) { return; } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/monitor/HbaseConfigMonitor.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/monitor/HbaseConfigMonitor.java index 0ac3f960..caa8c39d 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/monitor/HbaseConfigMonitor.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/monitor/HbaseConfigMonitor.java @@ -1,10 +1,10 @@ package com.alibaba.otter.canal.client.adapter.hbase.monitor; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.hbase.HbaseAdapter; import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; import com.alibaba.otter.canal.client.adapter.support.Util; -import com.alibaba.otter.canal.client.adapter.support.YamlUtils; import java.io.File; import java.util.Properties; import org.apache.commons.io.filefilter.FileFilterUtils; @@ -59,7 +59,8 @@ public class HbaseConfigMonitor { try { // 加载新增的配置文件 String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); - MappingConfig config = YamlUtils.ymlToObj(null, configContent, MappingConfig.class, null, envProperties); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties); if (config == null) { return; } @@ -87,7 +88,8 @@ public class HbaseConfigMonitor { onFileDelete(file); return; } - MappingConfig config = YamlUtils.ymlToObj(null, configContent, MappingConfig.class, null, envProperties); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties); if (config == null) { return; } diff --git a/client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/config/KuduMappingConfigLoader.java b/client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/config/KuduMappingConfigLoader.java index f1b389ac..f2ca45ad 100644 --- a/client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/config/KuduMappingConfigLoader.java +++ b/client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/config/KuduMappingConfigLoader.java @@ -1,6 +1,5 @@ package com.alibaba.otter.canal.client.adapter.kudu.config; -import com.alibaba.otter.canal.client.adapter.support.YamlUtils; import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; @@ -8,6 +7,7 @@ import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; /** @@ -30,7 +30,11 @@ public class KuduMappingConfigLoader { Map configContentMap = MappingConfigsLoader.loadConfigs("kudu"); configContentMap.forEach((fileName, content) -> { - KuduMappingConfig config = YamlUtils.ymlToObj(null, content, KuduMappingConfig.class, null, envProperties); + KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null, + content, + KuduMappingConfig.class, + null, + envProperties); if (config == null) { return; } diff --git a/client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/monitor/KuduConfigMonitor.java b/client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/monitor/KuduConfigMonitor.java index 7c4e6231..aae0bc8d 100644 --- a/client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/monitor/KuduConfigMonitor.java +++ b/client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/monitor/KuduConfigMonitor.java @@ -1,10 +1,10 @@ package com.alibaba.otter.canal.client.adapter.kudu.monitor; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter; import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; import com.alibaba.otter.canal.client.adapter.support.Util; -import com.alibaba.otter.canal.client.adapter.support.YamlUtils; import java.io.File; import java.util.Properties; import org.apache.commons.io.filefilter.FileFilterUtils; @@ -36,7 +36,7 @@ public class KuduConfigMonitor { File confDir = Util.getConfDirPath(adapterName); try { FileAlterationObserver observer = new FileAlterationObserver(confDir, - FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml"))); + FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml"))); FileListener listener = new FileListener(); observer.addListener(listener); fileMonitor = new FileAlterationMonitor(3000, observer); @@ -69,15 +69,19 @@ public class KuduConfigMonitor { try { // 加载新增的配置文件 String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); - KuduMappingConfig config = YamlUtils - .ymlToObj(null, configContent, KuduMappingConfig.class, null, envProperties); + KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null, + configContent, + KuduMappingConfig.class, + null, + envProperties); if (config == null) { return; } config.validate(); boolean result = kuduAdapter.addConfig(file.getName(), config); if (result) { - logger.info("Add a new kudu mapping config: {} to canal adapter", file.getName()); + logger.info("Add a new kudu mapping config: {} to canal adapter", + file.getName()); } } catch (Exception e) { logger.error(e.getMessage(), e); @@ -91,14 +95,17 @@ public class KuduConfigMonitor { try { if (kuduAdapter.getKuduMapping().containsKey(file.getName())) { // 加载配置文件 - String configContent = MappingConfigsLoader - .loadConfig(adapterName + File.separator + file.getName()); + String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + + file.getName()); if (configContent == null) { onFileDelete(file); return; } - KuduMappingConfig config = YamlUtils - .ymlToObj(null, configContent, KuduMappingConfig.class, null, envProperties); + KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null, + configContent, + KuduMappingConfig.class, + null, + envProperties); if (config == null) { return; } diff --git a/client-adapter/launcher/src/main/bin/startup.sh b/client-adapter/launcher/src/main/bin/startup.sh index 102060a0..a478099f 100644 --- a/client-adapter/launcher/src/main/bin/startup.sh +++ b/client-adapter/launcher/src/main/bin/startup.sh @@ -69,12 +69,9 @@ else fi if [ -n "$str" ]; then - if [ $JavaVersion -ge 11 ] ; then - # For G1 - JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS" - else - JAVA_OPTS="-server -Xms2g -Xmx3g -Xmn1g -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS" - fi + # JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS" + # For G1 + JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS" else JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS" fi diff --git a/client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/config/ConfigLoader.java b/client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/config/ConfigLoader.java index 0e60e7c4..f5c753b3 100644 --- a/client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/config/ConfigLoader.java +++ b/client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/config/ConfigLoader.java @@ -1,7 +1,7 @@ package com.alibaba.otter.canal.client.adapter.phoenix.config; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; -import com.alibaba.otter.canal.client.adapter.support.YamlUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,7 +28,8 @@ public class ConfigLoader { Map configContentMap = MappingConfigsLoader.loadConfigs("phoenix"); configContentMap.forEach((fileName, content) -> { - MappingConfig config = YamlUtils.ymlToObj(null, content, MappingConfig.class, null, envProperties); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); if (config == null) { return; } diff --git a/client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/monitor/PhoenixConfigMonitor.java b/client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/monitor/PhoenixConfigMonitor.java index 8f9f1271..e8c2f978 100644 --- a/client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/monitor/PhoenixConfigMonitor.java +++ b/client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/monitor/PhoenixConfigMonitor.java @@ -1,10 +1,10 @@ package com.alibaba.otter.canal.client.adapter.phoenix.monitor; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter; import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; import com.alibaba.otter.canal.client.adapter.support.Util; -import com.alibaba.otter.canal.client.adapter.support.YamlUtils; import java.io.File; import java.util.Properties; import org.apache.commons.io.filefilter.FileFilterUtils; @@ -19,15 +19,15 @@ import org.slf4j.LoggerFactory; */ public class PhoenixConfigMonitor { - private static final Logger logger = LoggerFactory.getLogger(PhoenixConfigMonitor.class); + private static final Logger logger = LoggerFactory.getLogger(PhoenixConfigMonitor.class); - private static final String adapterName = "phoenix"; // 相应组件名字 + private static final String adapterName = "phoenix"; //相应组件名字 - private String key; + private String key; - private PhoenixAdapter phoenixAdapter; // 相应适配器名实现类 + private PhoenixAdapter phoenixAdapter; //相应适配器名实现类 - private Properties envProperties; + private Properties envProperties; private FileAlterationMonitor fileMonitor; @@ -38,7 +38,7 @@ public class PhoenixConfigMonitor { File confDir = Util.getConfDirPath(adapterName); try { FileAlterationObserver observer = new FileAlterationObserver(confDir, - FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml"))); + FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml"))); FileListener listener = new FileListener(); observer.addListener(listener); fileMonitor = new FileAlterationMonitor(3000, observer); @@ -65,15 +65,16 @@ public class PhoenixConfigMonitor { try { // 加载新增的配置文件 String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); - MappingConfig config = YamlUtils - .ymlToObj(null, configContent, MappingConfig.class, null, envProperties); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties); if (config == null) { return; } config.validate(); boolean result = phoenixAdapter.addConfig(file.getName(), config); if (result) { - logger.info("Add a new phoenix mapping config: {} to canal adapter", file.getName()); + logger.info("Add a new phoenix mapping config: {} to canal adapter", + file.getName()); } } catch (Exception e) { logger.error(e.getMessage(), e); @@ -88,13 +89,13 @@ public class PhoenixConfigMonitor { if (phoenixAdapter.getPhoenixMapping().containsKey(file.getName())) { // 加载配置文件 String configContent = MappingConfigsLoader - .loadConfig(adapterName + File.separator + file.getName()); + .loadConfig(adapterName + File.separator + file.getName()); if (configContent == null) { onFileDelete(file); return; } - MappingConfig config = YamlUtils - .ymlToObj(null, configContent, MappingConfig.class, null, envProperties); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties); if (config == null) { return; } diff --git a/client-adapter/pom.xml b/client-adapter/pom.xml index 463ea2e6..7f3a3dd5 100644 --- a/client-adapter/pom.xml +++ b/client-adapter/pom.xml @@ -53,7 +53,7 @@ central - https://repo1.maven.org/maven2 + http://repo1.maven.org/maven2 true @@ -63,7 +63,7 @@ java.net - https://download.java.net/maven/2/ + http://download.java.net/maven/2/ true @@ -73,7 +73,7 @@ aliyun - https://maven.aliyun.com/nexus/content/groups/public/ + http://maven.aliyun.com/nexus/content/groups/public/ true @@ -113,9 +113,35 @@ 2.0.0.RELEASE - org.springframework.boot - spring-boot - 2.0.1.RELEASE + org.springframework + spring-core + 5.0.5.RELEASE + + + org.springframework + spring-aop + 5.0.5.RELEASE + + + org.springframework + spring-context + 5.0.5.RELEASE + + + org.springframework + spring-jdbc + 5.0.5.RELEASE + + + org.springframework + spring-orm + 5.0.5.RELEASE + + + org.springframework + spring-test + 5.0.5.RELEASE + test com.h2database diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java index cf57c75d..d65e0ec5 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java @@ -1,6 +1,5 @@ package com.alibaba.otter.canal.client.adapter.rdb.config; -import com.alibaba.otter.canal.client.adapter.support.YamlUtils; import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; @@ -8,6 +7,7 @@ import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; /** @@ -32,7 +32,8 @@ public class ConfigLoader { Map configContentMap = MappingConfigsLoader.loadConfigs("rdb"); configContentMap.forEach((fileName, content) -> { - MappingConfig config = YamlUtils.ymlToObj(null, content, MappingConfig.class, null, envProperties); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); if (config == null) { return; } diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java index 794b021a..f726ab69 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java @@ -1,10 +1,10 @@ package com.alibaba.otter.canal.client.adapter.rdb.monitor; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter; import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; import com.alibaba.otter.canal.client.adapter.support.Util; -import com.alibaba.otter.canal.client.adapter.support.YamlUtils; import java.io.File; import java.util.Properties; import org.apache.commons.io.filefilter.FileFilterUtils; @@ -35,7 +35,7 @@ public class RdbConfigMonitor { File confDir = Util.getConfDirPath(adapterName); try { FileAlterationObserver observer = new FileAlterationObserver(confDir, - FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml"))); + FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml"))); FileListener listener = new FileListener(); observer.addListener(listener); fileMonitor = new FileAlterationMonitor(3000, observer); @@ -62,15 +62,16 @@ public class RdbConfigMonitor { try { // 加载新增的配置文件 String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); - MappingConfig config = YamlUtils - .ymlToObj(null, configContent, MappingConfig.class, null, envProperties); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties); if (config == null) { return; } config.validate(); boolean result = rdbAdapter.addConfig(file.getName(), config); if (result) { - logger.info("Add a new rdb mapping config: {} to canal adapter", file.getName()); + logger.info("Add a new rdb mapping config: {} to canal adapter", + file.getName()); } } catch (Exception e) { logger.error(e.getMessage(), e); @@ -84,14 +85,14 @@ public class RdbConfigMonitor { try { if (rdbAdapter.getRdbMapping().containsKey(file.getName())) { // 加载配置文件 - String configContent = MappingConfigsLoader - .loadConfig(adapterName + File.separator + file.getName()); + String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); if (configContent == null) { onFileDelete(file); return; } - MappingConfig config = YamlUtils - .ymlToObj(null, configContent, MappingConfig.class, null, envProperties); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, configContent, MappingConfig.class, null, + envProperties); if (config == null) { return; } diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java index 18a73404..d1a511bd 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java @@ -58,7 +58,6 @@ public class RdbEtlService extends AbstractEtlService { Map columnsMap = new LinkedHashMap<>(); Map columnType = new LinkedHashMap<>(); DruidDataSource dataSource = (DruidDataSource) srcDS; - String backtick = SyncUtil.getBacktickByDbType(dataSource.getDbType()); Util.sqlRS(targetDS, "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()) + " LIMIT 1 ", @@ -92,7 +91,7 @@ public class RdbEtlService extends AbstractEtlService { .append(SyncUtil.getDbTableName(dbMapping, dataSource.getDbType())) .append(" ("); columnsMap - .forEach((targetColumnName, srcColumnName) -> insertSql.append(backtick).append(targetColumnName).append(backtick).append(",")); + .forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(",")); int len = insertSql.length(); insertSql.delete(len - 1, len).append(") VALUES ("); @@ -102,7 +101,6 @@ public class RdbEtlService extends AbstractEtlService { } len = insertSql.length(); insertSql.delete(len - 1, len).append(")"); - logger.info("executeSqlImport sql:{}",insertSql.toString()); try (Connection connTarget = targetDS.getConnection(); PreparedStatement pstmt = connTarget.prepareStatement(insertSql.toString())) { connTarget.setAutoCommit(false); @@ -116,7 +114,7 @@ public class RdbEtlService extends AbstractEtlService { Map pkVal = new LinkedHashMap<>(); StringBuilder deleteSql = new StringBuilder( "DELETE FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()) + " WHERE "); - appendCondition(dbMapping, deleteSql, pkVal, rs, backtick); + appendCondition(dbMapping, deleteSql, pkVal, rs); try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) { int k = 1; for (Object val : pkVal.values()) { @@ -134,6 +132,7 @@ public class RdbEtlService extends AbstractEtlService { } Integer type = columnType.get(targetClolumnName.toLowerCase()); + Object value = rs.getObject(srcColumnName); if (value != null) { SyncUtil.setPStmt(type, pstmt, value, i); @@ -181,7 +180,7 @@ public class RdbEtlService extends AbstractEtlService { * 拼接目标表主键where条件 */ private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map values, - ResultSet rs, String backtick) throws SQLException { + ResultSet rs) throws SQLException { // 拼接主键 for (Map.Entry entry : dbMapping.getTargetPk().entrySet()) { String targetColumnName = entry.getKey(); @@ -189,7 +188,7 @@ public class RdbEtlService extends AbstractEtlService { if (srcColumnName == null) { srcColumnName = targetColumnName; } - sql.append(backtick).append(targetColumnName).append(backtick).append("=? AND "); + sql.append(targetColumnName).append("=? AND "); values.put(targetColumnName, rs.getObject(srcColumnName)); } int len = sql.length(); diff --git a/client-adapter/tablestore/src/main/java/com/alibaba/otter/canal/client/adapter/tablestore/config/ConfigLoader.java b/client-adapter/tablestore/src/main/java/com/alibaba/otter/canal/client/adapter/tablestore/config/ConfigLoader.java index 19b4a8f0..f064fdfa 100644 --- a/client-adapter/tablestore/src/main/java/com/alibaba/otter/canal/client/adapter/tablestore/config/ConfigLoader.java +++ b/client-adapter/tablestore/src/main/java/com/alibaba/otter/canal/client/adapter/tablestore/config/ConfigLoader.java @@ -1,7 +1,7 @@ package com.alibaba.otter.canal.client.adapter.tablestore.config; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; -import com.alibaba.otter.canal.client.adapter.support.YamlUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.StringUtils; @@ -32,7 +32,8 @@ public class ConfigLoader { Map configContentMap = MappingConfigsLoader.loadConfigs("tablestore"); configContentMap.forEach((fileName, content) -> { - MappingConfig config = YamlUtils.ymlToObj(null, content, MappingConfig.class, null, envProperties); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); if (config == null) { return; } @@ -44,8 +45,7 @@ public class ConfigLoader { result.put(fileName, config); }); - logger.info( - "## Tablestore mapping config loaded:" + StringUtils.collectionToCommaDelimitedString(result.keySet())); + logger.info("## Tablestore mapping config loaded:" + StringUtils.collectionToCommaDelimitedString(result.keySet())); return result; } } diff --git a/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java index bc068abd..fe983dc2 100644 --- a/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java +++ b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java @@ -38,7 +38,7 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle { private ServerRunningData serverData; // 当前实际运行的节点状态信息 private volatile ServerRunningData activeData; - private ScheduledExecutorService delayExecutor; + private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1); private int delayTime = 5; private ServerRunningListener listener; @@ -73,7 +73,7 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle { initRunning(); } else { // 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作 - delayExecutor.schedule(() -> initRunning(), delayTime, TimeUnit.SECONDS); + delayExector.schedule(() -> initRunning(), delayTime, TimeUnit.SECONDS); } } @@ -90,7 +90,6 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle { try { processStart(); if (zkClient != null) { - delayExecutor = Executors.newScheduledThreadPool(1); // 如果需要尽可能释放instance资源，不需要监听running节点，不然即使stop了这台机器，另一台机器立马会start String path = ZookeeperPathUtils.getDestinationServerRunning(destination); zkClient.subscribeDataChanges(path, dataListener); @@ -123,10 +122,6 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle { if (zkClient != null) { String path = ZookeeperPathUtils.getDestinationServerRunning(destination); zkClient.unsubscribeDataChanges(path, dataListener); - if (delayExecutor != null) { - delayExecutor.shutdown(); - delayExecutor = null; - } releaseRunning(); // 尝试一下release } else { diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java index 6d7bb283..d95087c6 100644 --- a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java @@ -107,11 +107,11 @@ public class AviaterRegexFilter { private List completionPattern(List patterns) { List result = new ArrayList<>(); for (String pattern : patterns) { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("^"); - stringBuilder.append(pattern); - stringBuilder.append("$"); - result.add(stringBuilder.toString()); + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append("^"); + stringBuffer.append(pattern); + stringBuffer.append("$"); + result.add(stringBuffer.toString()); } return result; } diff --git a/connector/pom.xml b/connector/pom.xml index 951da13d..744c8e20 100644 --- a/connector/pom.xml +++ b/connector/pom.xml @@ -39,7 +39,7 @@ central - https://repo1.maven.org/maven2 + http://repo1.maven.org/maven2 true @@ -49,7 +49,7 @@ java.net - https://download.java.net/maven/2/ + http://download.java.net/maven/2/ true @@ -59,7 +59,7 @@ aliyun - https://maven.aliyun.com/nexus/content/groups/public/ + http://maven.aliyun.com/nexus/content/groups/public/ true diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java index c81dcc6f..725abe60 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java @@ -21,7 +21,6 @@ public final class CharsetConversion { protected final String mysqlCharset; protected final String mysqlCollation; protected final String javaCharset; - protected final Charset charset; Entry(final int id, String mysqlCharset, // NL String mysqlCollation, String javaCharset){ @@ -29,7 +28,6 @@ public final class CharsetConversion { this.mysqlCharset = mysqlCharset; this.mysqlCollation = mysqlCollation; this.javaCharset = javaCharset; - this.charset = Charset.isSupported(javaCharset) ? Charset.forName(javaCharset) : null; } } @@ -392,23 +390,6 @@ public final class CharsetConversion { } } - public static Charset getNioCharset(final int id) { - Entry entry = getEntry(id); - - if (entry != null) { - if (entry.charset != null) { - return entry.charset; - } else { - logger.warn("Unknown java charset for: id = " + id + ", name = " + entry.mysqlCharset + ", coll = " - + entry.mysqlCollation); - return null; - } - } else { - logger.warn("Unexpect mysql charset: " + id); - return null; - } - } - public static void main(String[] args) { for (int i = 0; i < entries.length; i++) { Entry entry = entries[i]; diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java index bcfd26b6..f04daa1d 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java @@ -1,7 +1,5 @@ package com.taobao.tddl.dbsync.binlog; -import java.nio.charset.Charset; - import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber2; import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber4; import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.usecondsToStr; @@ -59,27 +57,23 @@ public class JsonConversion { public static final int VALUE_ENTRY_SIZE_LARGE = (1 + LARGE_OFFSET_SIZE); public static Json_Value parse_value(int type, LogBuffer buffer, long len, String charsetName) { - return parse_value(type, buffer, len, Charset.forName(charsetName)); - } - - public static Json_Value parse_value(int type, LogBuffer buffer, long len, Charset charset) { buffer = buffer.duplicate(buffer.position(), (int) len); switch (type) { case JSONB_TYPE_SMALL_OBJECT: - return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false, charset); + return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false, charsetName); case JSONB_TYPE_LARGE_OBJECT: - return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true, charset); + return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true, charsetName); case JSONB_TYPE_SMALL_ARRAY: - return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false, charset); + return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false, charsetName); case JSONB_TYPE_LARGE_ARRAY: - return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true, charset); + return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true, charsetName); default: - return parse_scalar(type, buffer, len, charset); + return parse_scalar(type, buffer, len, charsetName); } } private static Json_Value parse_array_or_object(Json_enum_type type, LogBuffer buffer, long len, boolean large, - Charset charset) { + String charsetName) { long offset_size = large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE; if (len < 2 * offset_size) { throw new IllegalArgumentException("illegal json data"); @@ -106,7 +100,7 @@ public class JsonConversion { return large ? buffer.getUint32() : buffer.getUint16(); } - private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, Charset charset) { + private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, String charsetName) { switch (type) { case JSONB_TYPE_LITERAL: /* purecov: inspected */ @@ -162,7 +156,7 @@ public class JsonConversion { } return new Json_Value(Json_enum_type.STRING, buffer.rewind() .forward((int) n) - .getFixString((int) str_len, charset)); + .getFixString((int) str_len, charsetName)); case JSONB_TYPE_OPAQUE: /* * There should always be at least one byte, which tells the @@ -248,7 +242,7 @@ public class JsonConversion { this.m_large = large; } - public String key(int i, Charset charset) { + public String key(int i, String charsetName) { m_data.rewind(); int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE; int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL; @@ -261,10 +255,10 @@ public class JsonConversion { // entry, always two // bytes. long key_length = m_data.getUint16(); - return m_data.rewind().forward((int) key_offset).getFixString((int) key_length, charset); + return m_data.rewind().forward((int) key_offset).getFixString((int) key_length, charsetName); } - public Json_Value element(int i, Charset charset) { + public Json_Value element(int i, String charsetName) { m_data.rewind(); int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE; int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL; @@ -277,13 +271,13 @@ public class JsonConversion { int type = m_data.forward(entry_offset).getUint8(); if (type == JSONB_TYPE_INT16 || type == JSONB_TYPE_UINT16 || type == JSONB_TYPE_LITERAL || (m_large && (type == JSONB_TYPE_INT32 || type == JSONB_TYPE_UINT32))) { - return parse_scalar(type, m_data, value_entry_size - 1, charset); + return parse_scalar(type, m_data, value_entry_size - 1, charsetName); } int value_offset = (int) read_offset_or_size(m_data, m_large); - return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset, charset); + return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset, charsetName); } - public StringBuilder toJsonString(StringBuilder buf, Charset charset) { + public StringBuilder toJsonString(StringBuilder buf, String charsetName) { switch (m_type) { case OBJECT: buf.append("{"); @@ -291,9 +285,9 @@ public class JsonConversion { if (i > 0) { buf.append(", "); } - buf.append('"').append(key(i, charset)).append('"'); + buf.append('"').append(key(i, charsetName)).append('"'); buf.append(": "); - element(i, charset).toJsonString(buf, charset); + element(i, charsetName).toJsonString(buf, charsetName); } buf.append("}"); break; @@ -303,7 +297,7 @@ public class JsonConversion { if (i > 0) { buf.append(", "); } - element(i, charset).toJsonString(buf, charset); + element(i, charsetName).toJsonString(buf, charsetName); } buf.append("]"); break; @@ -404,7 +398,7 @@ public class JsonConversion { } buf.append('"').append(text).append('"'); } else { - text = m_data.getFixString((int) m_length, charset); + text = m_data.getFixString((int) m_length, charsetName); buf.append('"').append(escapse(text)).append('"'); } diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java index eb8f6dde..878a0751 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java @@ -1,6 +1,5 @@ package com.taobao.tddl.dbsync.binlog; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; @@ -37,11 +36,6 @@ public class JsonDiffConversion { public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex, String charsetName) { - return print_json_diff(buffer, len, columnName, columnIndex, Charset.forName(charsetName)); - } - - public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex, - Charset charset) { int position = buffer.position(); List operation_names = new ArrayList<>(); while (buffer.hasRemaining()) { @@ -117,14 +111,14 @@ public class JsonDiffConversion { Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(), buffer, value_length - 1, - charset); + charsetName); buffer.forward((int) value_length - 1); // Read value if (jsonValue.m_type == Json_enum_type.ERROR) { throw new IllegalArgumentException("parsing json value"); } StringBuilder jsonBuilder = new StringBuilder(); - jsonValue.toJsonString(jsonBuilder, charset); + jsonValue.toJsonString(jsonBuilder, charsetName); builder.append(jsonBuilder); } diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java index a2594948..408eaa4e 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java @@ -5,8 +5,6 @@ import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.math.BigInteger; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.BitSet; @@ -18,8 +16,6 @@ import java.util.BitSet; */ public class LogBuffer { - static final BigDecimal DECIMAL_ZERO_1_SCALE = BigDecimal.valueOf(0, 1); - static final BigDecimal DECIMAL_ONE_1_SCALE = BigDecimal.valueOf(10, 1); protected byte[] buffer; protected int origin, limit; @@ -1042,27 +1038,29 @@ public class LogBuffer { } } + /* default ANSI charset */ + public static final String ISO_8859_1 = "ISO-8859-1"; + /** * Return fix length string from buffer. */ public final String getFixString(final int pos, final int len) { - return getFixString(pos, len, StandardCharsets.ISO_8859_1); + return getFixString(pos, len, ISO_8859_1); } /** * Return next fix length string from buffer. */ public final String getFixString(final int len) { - return getFixString(len, StandardCharsets.ISO_8859_1); + return getFixString(len, ISO_8859_1); } /** * Return fix length string from buffer. */ - public final String getFixString(final int pos, final int len, Charset charset) { - if (pos + len > limit || pos < 0) { - throw new IllegalArgumentException("limit excceed: " + (pos < 0 ? pos : (pos + len))); - } + public final String getFixString(final int pos, final int len, String charsetName) { + if (pos + len > limit || pos < 0) throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + len))); final int from = origin + pos; final int end = from + len; @@ -1071,16 +1069,19 @@ public class LogBuffer { for (; (found < end) && buf[found] != '\0'; found++) /* empty loop */; - return new String(buf, from, found - from, charset); + try { + return new String(buf, from, found - from, charsetName); + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException("Unsupported encoding: " + charsetName, e); + } } /** * Return next fix length string from buffer. */ - public final String getFixString(final int len, Charset charset) { - if (position + len > origin + limit) { - throw new IllegalArgumentException("limit excceed: " + (position + len - origin)); - } + public final String getFixString(final int len, String charsetName) { + if (position + len > origin + limit) throw new IllegalArgumentException("limit excceed: " + + (position + len - origin)); final int from = position; final int end = from + len; @@ -1089,141 +1090,97 @@ public class LogBuffer { for (; (found < end) && buf[found] != '\0'; found++) /* empty loop */; - String string = new String(buf, from, found - from, charset); - position += len; - return string; - } - - public final String getFixName(final int len, Charset charset) { - if (position + len > origin + limit) { - throw new IllegalArgumentException("limit excceed: " + (position + len - origin)); + try { + String string = new String(buf, from, found - from, charsetName); + position += len; + return string; + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException("Unsupported encoding: " + charsetName, e); } - - final int from = position; - final int end = from + len; - byte[] buf = buffer; - int found = from; - for (; (found < end) && buf[found] != '\0'; found++) - /* empty loop */; - - int length = found - from; - String string = null; - if (length <= 16) { - string = NameCache.name(buf, from, length, charset); - } - if (string == null) { - string = new String(buf, from, length, charset); - } - position += len; - return string; - } - - public final String getFixName(final int len) { - if (position + len > origin + limit) { - throw new IllegalArgumentException("limit excceed: " + (position + len - origin)); - } - - final int from = position; - final int end = from + len; - byte[] buf = buffer; - int found = from; - for (; (found < end) && buf[found] != '\0'; found++) - /* empty loop */; - - int length = found - from; - String string = null; - if (length <= 16) { - string = NameCache.name(buf, from, length, StandardCharsets.ISO_8859_1); - } - if (string == null) { - string = new String(buf, from, length, StandardCharsets.ISO_8859_1); - } - position += len; - return string; } /** - * Return fix-length string from buffer without null-terminate checking. - * Fix bug #17 {@link https://github.com/AlibabaTech/canal/issues/17 } + * Return fix-length string from buffer without null-terminate checking. Fix + * bug #17 {@link https://github.com/AlibabaTech/canal/issues/17 } */ - public final String getFullString(final int pos, final int len, Charset charset) { - if (pos + len > limit || pos < 0) { - throw new IllegalArgumentException("limit excceed: " + (pos < 0 ? pos : (pos + len))); - } + public final String getFullString(final int pos, final int len, String charsetName) { + if (pos + len > limit || pos < 0) throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + len))); - return new String(buffer, origin + pos, len, charset); + try { + return new String(buffer, origin + pos, len, charsetName); + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException("Unsupported encoding: " + charsetName, e); + } } /** * Return next fix-length string from buffer without null-terminate - * checking. - * Fix bug #17 {@link https://github.com/AlibabaTech/canal/issues/17 } + * checking. Fix bug #17 {@link https + * ://github.com/AlibabaTech/canal/issues/17 } */ - public final String getFullString(final int len, Charset charset) { - if (position + len > origin + limit) { - throw new IllegalArgumentException("limit excceed: " + (position + len - origin)); + public final String getFullString(final int len, String charsetName) { + if (position + len > origin + limit) throw new IllegalArgumentException("limit excceed: " + + (position + len - origin)); + + try { + String string = new String(buffer, position, len, charsetName); + position += len; + return string; + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException("Unsupported encoding: " + charsetName, e); } - - String string = new String(buffer, position, len, charset); - position += len; - return string; } - /** * Return dynamic length string from buffer. */ public final String getString(final int pos) { - if (pos >= limit || pos < 0) { - throw new IllegalArgumentException("limit excceed: " + pos); - } - - byte[] buf = buffer; - final int len = (0xff & buf[origin + pos]); - if (pos + len + 1 > limit) { - throw new IllegalArgumentException("limit excceed: " + (pos + len + 1)); - } - - return new String(buf, origin + pos + 1, len, StandardCharsets.ISO_8859_1); + return getString(pos, ISO_8859_1); } /** * Return next dynamic length string from buffer. */ public final String getString() { - if (position >= origin + limit) { - throw new IllegalArgumentException("limit excceed: " + position); - } - - byte[] buf = buffer; - final int len = (0xff & buf[position]); - if (position + len + 1 > origin + limit) { - throw new IllegalArgumentException("limit excceed: " - + (position + len + 1 - origin)); - } - - String string = new String(buf, position + 1, len, StandardCharsets.ISO_8859_1); - position += len + 1; - return string; + return getString(ISO_8859_1); } - public final String getName() { - if (position >= origin + limit) { - throw new IllegalArgumentException("limit excceed: " + position); + /** + * Return dynamic length string from buffer. + */ + public final String getString(final int pos, String charsetName) { + if (pos >= limit || pos < 0) throw new IllegalArgumentException("limit excceed: " + pos); + + byte[] buf = buffer; + final int len = (0xff & buf[origin + pos]); + if (pos + len + 1 > limit) throw new IllegalArgumentException("limit excceed: " + (pos + len + 1)); + + try { + return new String(buf, origin + pos + 1, len, charsetName); + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException("Unsupported encoding: " + charsetName, e); } + } + + /** + * Return next dynamic length string from buffer. + */ + public final String getString(String charsetName) { + if (position >= origin + limit) throw new IllegalArgumentException("limit excceed: " + position); byte[] buf = buffer; final int len = (0xff & buf[position]); - if (position + len + 1 > origin + limit) { - throw new IllegalArgumentException("limit excceed: " + (position + len + 1 - origin)); - } + if (position + len + 1 > origin + limit) throw new IllegalArgumentException("limit excceed: " + + (position + len + 1 - origin)); - String string = NameCache.name(buf, position + 1, len, StandardCharsets.ISO_8859_1); - if (string == null) { - string = new String(buf, position + 1, len, StandardCharsets.ISO_8859_1); + try { + String string = new String(buf, position + 1, len, charsetName); + position += len + 1; + return string; + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException("Unsupported encoding: " + charsetName, e); } - position += len + 1; - return string; } /** @@ -1306,15 +1263,6 @@ public class LogBuffer { throw new IllegalArgumentException("limit excceed: " + (position + binSize - origin)); } - if (precision <= 20 && intg <= 18 && frac <= 9) { - // use compact BigDecimal - BigDecimal compactDecimal = getBigDecimalCompact(precision, scale, intg, frac, intg0x); - if (compactDecimal != null) { - position += binSize; - return compactDecimal; - } - } - BigDecimal decimal = getDecimal0(position, intg, frac, // NL intg0, frac0, @@ -1324,162 +1272,6 @@ public class LogBuffer { return decimal; } - /** - * use compact BigDecimal - */ - private BigDecimal getBigDecimalCompact(int precision, int scale, int intg, int frac, int intg0x) { - if (precision > 20 || intg > 18 && frac > 9) { - return null; - } - - BigDecimal compactDecimal = null; - long longValue = 0; - int small = 0; - int pos = position; - byte b0 = buffer[position]; - final int mask = ((b0 & 0x80) == 0x80) ? 0 : -1; - b0 ^= 0x80; - - switch (intg0x) { - case 0: - if (intg == 0) { - longValue = 0; - } else { - longValue = (b0 << 24) - | ((0xff & buffer[pos + 1]) << 16) - | ((0xff & buffer[pos + 2]) << 8) - | (0xff & buffer[pos + 3]); - longValue ^= mask; - pos += 4; - } - break; - case 1: - case 2: - longValue = b0; - longValue ^= mask; - pos += 1; - break; - case 3: - case 4: - longValue = (b0 << 8) | (0xff & buffer[pos + 1]); - longValue ^= mask; - pos += 2; - break; - case 5: - case 6: - longValue = (b0 << 16) | ((0xff & buffer[pos + 1]) << 8) | (0xff & buffer[pos + 2]); - longValue ^= mask; - pos += 3; - break; - case 7: - case 8: - longValue = (b0 << 24) - | ((0xff & buffer[pos + 1]) << 16) - | ((0xff & buffer[pos + 2]) << 8) - | (0xff & buffer[pos + 3]); - longValue ^= mask; - pos += 4; - break; - default: - break; - } - - if (intg > 9) { - int value = getInt32BE(buffer, pos); - value ^= mask; - longValue *= 1_000_000_000; - longValue += value; - pos += 4; - } - - int power = 1; - if (pos != position) { - b0 = buffer[pos]; - } - switch (frac) { - case 0: - small = 0; - break; - case 1: - case 2: - power = frac == 1 ? 10 : 100; - small = b0; - small ^= mask; - break; - case 3: - case 4: - power = frac == 3 ? 1_000 : 10_000; - small = (b0 << 8) | (0xff & buffer[pos + 1]); - small ^= mask; - break; - case 5: - case 6: - power = frac == 5 ? 100_000 : 1_000_000; - small = (b0 << 16) | ((0xff & buffer[pos + 1]) << 8) | (0xff & buffer[pos + 2]); - small ^= mask; - break; - case 7: - case 8: - case 9: - power = frac == 7 ? 10_000_000 : frac == 8 ? 100_000_000 : 1_000_000_000; - small = (b0 << 24) - | ((0xff & buffer[pos + 1]) << 16) - | ((0xff & buffer[pos + 2]) << 8) - | (0xff & buffer[pos + 3]); - small ^= mask; - break; - default: - break; - } - - if (small == 0) { - if (frac == 0) { - if (mask != 0) { - longValue = -longValue; - } - compactDecimal = BigDecimal.valueOf(longValue, 0); - } else if (longValue == 0) { - if (scale < 9) { - compactDecimal = DECIMAL_ZERO_1_SCALE; - } else { - compactDecimal = BigDecimal.valueOf(longValue, scale); - } - } else { - if (scale < 9) { - long unscaleValue = longValue * 10; - if (unscaleValue >= longValue) { - if (mask != 0) { - unscaleValue = -unscaleValue; - } - if (unscaleValue == 10) { - compactDecimal = DECIMAL_ONE_1_SCALE; - } else { - compactDecimal = BigDecimal.valueOf(unscaleValue, 1); - } - } - } else { - long unscaleValue = longValue * power; - if (unscaleValue >= longValue) { - if (mask != 0) { - unscaleValue = -unscaleValue; - } - compactDecimal = BigDecimal.valueOf(unscaleValue, scale); - } - } - } - } else { - long unscaleValue = longValue * power + small; - if (unscaleValue >= longValue) { - if (mask != 0) { - unscaleValue = -unscaleValue; - } - compactDecimal = BigDecimal.valueOf(unscaleValue, scale); - } - } - - return compactDecimal; - } - /** * Return big decimal from buffer. * @@ -1653,9 +1445,8 @@ public class LogBuffer { } d_copy[begin] ^= 0x80; /* restore sign */ - // String decimal = String.valueOf(buf, 0, pos); - // return new BigDecimal(decimal); - return new BigDecimal(buf, 0, pos); + String decimal = String.valueOf(buf, 0, pos); + return new BigDecimal(decimal); } /** diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java index af237e27..dbb19701 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java @@ -197,9 +197,8 @@ public final class LogDecoder { } case LogEvent.SLAVE_EVENT: /* can never happen (unused event) */ { - if (logger.isWarnEnabled()) { - logger.warn("Skipping unsupported SLAVE_EVENT from: " + context.getLogPosition()); - } + if (logger.isWarnEnabled()) logger.warn("Skipping unsupported SLAVE_EVENT from: " + + context.getLogPosition()); break; } case LogEvent.CREATE_FILE_EVENT: { @@ -265,25 +264,22 @@ public final class LogDecoder { return descriptionEvent; } case LogEvent.PRE_GA_WRITE_ROWS_EVENT: { - if (logger.isWarnEnabled()) { - logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: " + context.getLogPosition()); - } + if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: " + + context.getLogPosition()); // ev = new Write_rows_log_event_old(buf, event_len, // description_event); break; } case LogEvent.PRE_GA_UPDATE_ROWS_EVENT: { - if (logger.isWarnEnabled()) { - logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: " + context.getLogPosition()); - } + if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: " + + context.getLogPosition()); // ev = new Update_rows_log_event_old(buf, event_len, // description_event); break; } case LogEvent.PRE_GA_DELETE_ROWS_EVENT: { - if (logger.isWarnEnabled()) { - logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: " + context.getLogPosition()); - } + if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: " + + context.getLogPosition()); // ev = new Delete_rows_log_event_old(buf, event_len, // description_event); break; @@ -360,16 +356,10 @@ public final class LogDecoder { return event; } case LogEvent.TRANSACTION_PAYLOAD_EVENT: { - if (logger.isWarnEnabled()) { - logger.warn("Skipping unsupported MySQL TRANSACTION_PAYLOAD_EVENT from: " + context.getLogPosition()); - } - break; - - // TransactionPayloadLogEvent event = new TransactionPayloadLogEvent(header, - // buffer, descriptionEvent); - // /* updating position in context */ - // logPosition.position = header.getLogPos(); - // return event; + TransactionPayloadLogEvent event = new TransactionPayloadLogEvent(header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; } case LogEvent.VIEW_CHANGE_EVENT: { ViewChangeEvent event = new ViewChangeEvent(header, buffer, descriptionEvent); @@ -428,39 +418,6 @@ public final class LogDecoder { logPosition.position = header.getLogPos(); return event; } - case LogEvent.HEARTBEAT_LOG_EVENT_V2: { - HeartbeatV2LogEvent event = new HeartbeatV2LogEvent(header, buffer, descriptionEvent); - /* updating position in context */ - logPosition.position = header.getLogPos(); - return event; - } - case LogEvent.QUERY_COMPRESSED_EVENT: { - if (logger.isWarnEnabled()) { - logger.warn("Skipping unsupported MaraiDB QUERY_COMPRESSED_EVENT from: " + context.getLogPosition()); - } - break; - } - case LogEvent.WRITE_ROWS_COMPRESSED_EVENT_V1: - case LogEvent.WRITE_ROWS_COMPRESSED_EVENT: { - if (logger.isWarnEnabled()) { - logger.warn("Skipping unsupported MaraiDB WRITE_ROWS_COMPRESSED_EVENT from: " + context.getLogPosition()); - } - break; - } - case LogEvent.UPDATE_ROWS_COMPRESSED_EVENT_V1: - case LogEvent.UPDATE_ROWS_COMPRESSED_EVENT: { - if (logger.isWarnEnabled()) { - logger.warn("Skipping unsupported MaraiDB UPDATE_ROWS_COMPRESSED_EVENT from: " + context.getLogPosition()); - } - break; - } - case LogEvent.DELETE_ROWS_COMPRESSED_EVENT_V1: - case LogEvent.DELETE_ROWS_COMPRESSED_EVENT: { - if (logger.isWarnEnabled()) { - logger.warn("Skipping unsupported MaraiDB DELETE_ROWS_COMPRESSED_EVENT from: " + context.getLogPosition()); - } - break; - } default: /* * Create an object of Ignorable_log_event for unrecognized diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java index ace5f2fd..74e83c33 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java @@ -178,10 +178,7 @@ public abstract class LogEvent { /* mysql 8.0.20 */ public static final int TRANSACTION_PAYLOAD_EVENT = 40; - /* mysql 8.0.26 */ - public static final int HEARTBEAT_LOG_EVENT_V2 = 41; - - public static final int MYSQL_ENUM_END_EVENT = 42; + public static final int MYSQL_ENUM_END_EVENT = 41; // mariaDb 5.5.34 /* New MySQL/Sun events are to be added right above this comment */ @@ -213,23 +210,8 @@ public abstract class LogEvent { public static final int START_ENCRYPTION_EVENT = 164; - // mariadb 10.10.1 - /* - * Compressed binlog event. Note that the order between WRITE/UPDATE/DELETE - * events is significant; this is so that we can convert from the compressed to - * the uncompressed event type with (type-WRITE_ROWS_COMPRESSED_EVENT + - * WRITE_ROWS_EVENT) and similar for _V1. - */ - public static final int QUERY_COMPRESSED_EVENT = 165; - public static final int WRITE_ROWS_COMPRESSED_EVENT_V1 = 166; - public static final int UPDATE_ROWS_COMPRESSED_EVENT_V1 = 167; - public static final int DELETE_ROWS_COMPRESSED_EVENT_V1 = 168; - public static final int WRITE_ROWS_COMPRESSED_EVENT = 169; - public static final int UPDATE_ROWS_COMPRESSED_EVENT = 170; - public static final int DELETE_ROWS_COMPRESSED_EVENT = 171; - /** end marker */ - public static final int ENUM_END_EVENT = 171; + public static final int ENUM_END_EVENT = 165; /** * 1 byte length, 1 byte format Length is total length in bytes, including 2 @@ -376,7 +358,6 @@ public abstract class LogEvent { case INCIDENT_EVENT: return "Incident"; case HEARTBEAT_LOG_EVENT: - case HEARTBEAT_LOG_EVENT_V2: return "Heartbeat"; case IGNORABLE_LOG_EVENT: return "Ignorable"; diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/NameCache.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/NameCache.java deleted file mode 100644 index e10a8beb..00000000 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/NameCache.java +++ /dev/null @@ -1 +0,0 @@ -package com.taobao.tddl.dbsync.binlog; import java.nio.charset.Charset; /** * string cache * @since 1.1.7 */ public class NameCache { static final NameCacheEntry[] NAME_CACHE = new NameCacheEntry[8192]; static final NameCacheEntry2[] NAME_CACHE2 = new NameCacheEntry2[8192]; static final class NameCacheEntry { final String name; final long value; public NameCacheEntry(String name, long value){ this.name = name; this.value = value; } } static final class NameCacheEntry2 { final String name; final long value0; final long value1; public NameCacheEntry2(String name, long value0, long value1){ this.name = name; this.value0 = value0; this.value1 = value1; } } static String name(byte[] buf, int from, int length, Charset charset) { long nameValue0 = -1, nameValue1 = -1; switch (length) { case 1: nameValue0 = buf[from]; break; case 2: nameValue0 = (buf[from + 1] << 8) + (buf[from] & 0xFF); break; case 3: nameValue0 = (buf[from + 2] << 16) + ((buf[from + 1] & 0xFF) << 8) + (buf[from] & 0xFF); break; case 4: nameValue0 = (buf[from + 3] << 24) + ((buf[from + 2] & 0xFF) << 16) + ((buf[from + 1] & 0xFF) << 8) + (buf[from] & 0xFF); break; case 5: nameValue0 = (((long) buf[from + 4]) << 32) + ((buf[from + 3] & 0xFFL) << 24) + ((buf[from + 2] & 0xFFL) << 16) + ((buf[from + 1] & 0xFFL) << 8) + (buf[from] & 0xFFL); break; case 6: nameValue0 = (((long) buf[from + 5]) << 40) + ((buf[from + 4] & 0xFFL) << 32) + ((buf[from + 3] & 0xFFL) << 24) + ((buf[from + 2] & 0xFFL) << 16) + ((buf[from + 1] & 0xFFL) << 8) + (buf[from] & 0xFFL); break; case 7: nameValue0 = (((long) buf[from + 6]) << 48) + ((buf[from + 5] & 0xFFL) << 40) + ((buf[from + 4] & 0xFFL) << 32) + ((buf[from + 3] & 0xFFL) << 24) + ((buf[from + 2] & 0xFFL) << 16) + ((buf[from + 1] & 0xFFL) << 8) + (buf[from] & 0xFFL); break; case 8: nameValue0 = (((long) buf[from + 7]) << 56) + ((buf[from + 6] & 0xFFL) << 48) + ((buf[from + 5] & 0xFFL) << 40) + ((buf[from + 4] & 0xFFL) << 32) + ((buf[from + 3] & 0xFFL) << 24) + ((buf[from + 2] & 0xFFL) << 16) + ((buf[from + 1] & 0xFFL) << 8) + (buf[from] & 0xFFL); break; case 9: nameValue0 = buf[from]; nameValue1 = (((long) buf[from + 8]) << 56) + ((buf[from + 7] & 0xFFL) << 48) + ((buf[from + 6] & 0xFFL) << 40) + ((buf[from + 5] & 0xFFL) << 32) + ((buf[from + 4] & 0xFFL) << 24) + ((buf[from + 3] & 0xFFL) << 16) + ((buf[from + 2] & 0xFFL) << 8) + (buf[from + 1] & 0xFFL); break; case 10: nameValue0 = (buf[from + 1] << 8) + (buf[from]); nameValue1 = (((long) buf[from + 9]) << 56) + ((buf[from + 8] & 0xFFL) << 48) + ((buf[from + 7] & 0xFFL) << 40) + ((buf[from + 6] & 0xFFL) << 32) + ((buf[from + 5] & 0xFFL) << 24) + ((buf[from + 4] & 0xFFL) << 16) + ((buf[from + 3] & 0xFFL) << 8) + (buf[from + 2] & 0xFFL); break; case 11: nameValue0 = (buf[from + 2] << 16) + (buf[from + 1] << 8) + (buf[from]); nameValue1 = (((long) buf[from + 10]) << 56) + ((buf[from + 9] & 0xFFL) << 48) + ((buf[from + 8] & 0xFFL) << 40) + ((buf[from + 7] & 0xFFL) << 32) + ((buf[from + 6] & 0xFFL) << 24) + ((buf[from + 5] & 0xFFL) << 16) + ((buf[from + 4] & 0xFFL) << 8) + (buf[from + 3] & 0xFFL); break; case 12: nameValue0 = (buf[from + 3] << 24) + (buf[from + 2] << 16) + (buf[from + 1] << 8) + (buf[from]); nameValue1 = (((long) buf[from + 11]) << 56) + ((buf[from + 10] & 0xFFL) << 48) + ((buf[from + 9] & 0xFFL) << 40) + ((buf[from + 8] & 0xFFL) << 32) + ((buf[from + 7] & 0xFFL) << 24) + ((buf[from + 6] & 0xFFL) << 16) + ((buf[from + 5] & 0xFFL) << 8) + (buf[from + 4] & 0xFFL); break; case 13: nameValue0 = (((long) buf[from + 4]) << 32) + (((long) buf[from + 3]) << 24) + (((long) buf[from + 2]) << 16) + (((long) buf[from + 1]) << 8) + ((long) buf[from]); nameValue1 = (((long) buf[from + 12]) << 56) + ((buf[from + 11] & 0xFFL) << 48) + ((buf[from + 10] & 0xFFL) << 40) + ((buf[from + 9] & 0xFFL) << 32) + ((buf[from + 8] & 0xFFL) << 24) + ((buf[from + 7] & 0xFFL) << 16) + ((buf[from + 6] & 0xFFL) << 8) + (buf[from + 5] & 0xFFL); break; case 14: nameValue0 = (((long) buf[from + 5]) << 40) + ((buf[from + 4] & 0xFFL) << 32) + ((buf[from + 3] & 0xFFL) << 24) + ((buf[from + 2] & 0xFFL) << 16) + ((buf[from + 1] & 0xFFL) << 8) + (buf[from] & 0xFFL); nameValue1 = (((long) buf[from + 13]) << 56) + ((buf[from + 12] & 0xFFL) << 48) + ((buf[from + 11] & 0xFFL) << 40) + ((buf[from + 10] & 0xFFL) << 32) + ((buf[from + 9] & 0xFFL) << 24) + ((buf[from + 8] & 0xFFL) << 16) + ((buf[from + 7] & 0xFFL) << 8) + (buf[from + 6] & 0xFFL); break; case 15: nameValue0 = (((long) buf[from + 6]) << 48) + ((buf[from + 5] & 0xFFL) << 40) + ((buf[from + 4] & 0xFFL) << 32) + ((buf[from + 3] & 0xFFL) << 24) + ((buf[from + 2] & 0xFFL) << 16) + ((buf[from + 1] & 0xFFL) << 8) + (buf[from] & 0xFFL); nameValue1 = (((long) buf[from + 14]) << 56) + ((buf[from + 13] & 0xFFL) << 48) + ((buf[from + 12] & 0xFFL) << 40) + ((buf[from + 11] & 0xFFL) << 32) + ((buf[from + 10] & 0xFFL) << 24) + ((buf[from + 9] & 0xFFL) << 16) + ((buf[from + 8] & 0xFFL) << 8) + (buf[from + 7] & 0xFFL); break; case 16: nameValue0 = (((long) buf[from + 7]) << 56) + ((buf[from + 6] & 0xFFL) << 48) + ((buf[from + 5] & 0xFFL) << 40) + ((buf[from + 4] & 0xFFL) << 32) + ((buf[from + 3] & 0xFFL) << 24) + ((buf[from + 2] & 0xFFL) << 16) + ((buf[from + 1] & 0xFFL) << 8) + (buf[from] & 0xFFL); nameValue1 = (((long) buf[from + 15]) << 56) + ((buf[from + 14] & 0xFFL) << 48) + ((buf[from + 13] & 0xFFL) << 40) + ((buf[from + 12] & 0xFFL) << 32) + ((buf[from + 11] & 0xFFL) << 24) + ((buf[from + 10] & 0xFFL) << 16) + ((buf[from + 9] & 0xFFL) << 8) + (buf[from + 8] & 0xFFL); break; default: break; } if (nameValue0 != -1) { if (nameValue1 != -1) { int indexMask = ((int) nameValue1) & (NameCache.NAME_CACHE2.length - 1); NameCache.NameCacheEntry2 entry = NameCache.NAME_CACHE2[indexMask]; if (entry == null) { String name = new String(buf, from, length, charset); NameCache.NAME_CACHE2[indexMask] = new NameCacheEntry2(name, nameValue0, nameValue1); return name; } else if (entry.value0 == nameValue0 && entry.value1 == nameValue1) { return entry.name; } } else { int indexMask = ((int) nameValue0) & (NAME_CACHE.length - 1); NameCacheEntry entry = NAME_CACHE[indexMask]; if (entry == null) { String name = new String(buf, from, length, charset); NAME_CACHE[indexMask] = new NameCacheEntry(name, nameValue0); return name; } else if (entry.value == nameValue0) { return entry.name; } } } return null; } } \ No newline at end of file diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java index 3081eedc..49557fc6 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java @@ -18,7 +18,6 @@ public class GtidLogEvent extends LogEvent { // / Length of SID in event encoding public static final int ENCODED_SID_LENGTH = 16; public static final int LOGICAL_TIMESTAMP_TYPE_CODE = 2; - public static final UUID UUID_ZERO = new UUID(0, 0); private boolean commitFlag; private UUID sid; @@ -36,13 +35,11 @@ public class GtidLogEvent extends LogEvent { buffer.position(commonHeaderLen); commitFlag = (buffer.getUint8() != 0); // ENCODED_FLAG_LENGTH - long high = buffer.getBeLong64(); - long low = buffer.getBeLong64(); - if (high == 0 && low == 0) { - sid = UUID_ZERO; - } else { - sid = new UUID(high, low); - } + byte[] bs = buffer.getData(ENCODED_SID_LENGTH); + ByteBuffer bb = ByteBuffer.wrap(bs); + long high = bb.getLong(); + long low = bb.getLong(); + sid = new UUID(high, low); gno = buffer.getLong64(); @@ -76,11 +73,11 @@ public class GtidLogEvent extends LogEvent { return gno; } - public long getLastCommitted() { + public Long getLastCommitted() { return lastCommitted; } - public long getSequenceNumber() { + public Long getSequenceNumber() { return sequenceNumber; } diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatLogEvent.java index ea8c519e..f3948d7e 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatLogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatLogEvent.java @@ -3,8 +3,6 @@ package com.taobao.tddl.dbsync.binlog.event; import com.taobao.tddl.dbsync.binlog.LogBuffer; import com.taobao.tddl.dbsync.binlog.LogEvent; -import java.nio.charset.StandardCharsets; - /** * 
  * Replication event to ensure to slave that master is alive.
@@ -37,7 +35,7 @@ public class HeartbeatLogEvent extends LogEvent {
             identLen = FN_REFLEN - 1;
         }
 
-        logIdent = buffer.getFullString(commonHeaderLen, identLen, StandardCharsets.ISO_8859_1);
+        logIdent = buffer.getFullString(commonHeaderLen, identLen, LogBuffer.ISO_8859_1);
     }
 
     public int getIdentLen() {
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatV2LogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatV2LogEvent.java
deleted file mode 100644
index e6619c7c..00000000
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatV2LogEvent.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package com.taobao.tddl.dbsync.binlog.event;
-
-import com.taobao.tddl.dbsync.binlog.LogBuffer;
-import com.taobao.tddl.dbsync.binlog.LogEvent;
-
-/**
- * 
- *   Replication event to ensure to replica that source is alive.
- *   The event is originated by source's dump thread and sent straight to
- *   replica without being logged. Slave itself does not store it in relay log
- *   but rather uses a data for immediate checks and throws away the event.
- *   Two members of the class m_log_filename and m_log_position comprise
- *   @see the rpl_event_coordinates instance. The coordinates that a heartbeat
- *   instance carries correspond to the last event source has sent from
- *   its binlog.
- *   Also this event will be generated only for the source server with
- *   version > 8.0.26
- * 

- * 
- * @author jianghang 2022-09-01 下午16:36:29
- * @version 1.1.6
- * @since mysql 8.0.26
- */
-public class HeartbeatV2LogEvent extends LogEvent {
-
-    private byte[]          payload;
-
-    public HeartbeatV2LogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-        super(header);
-
-        final int commonHeaderLen = descriptionEvent.commonHeaderLen;
-        int payloadLenth = buffer.limit() - commonHeaderLen;
-        // see : https://github.com/mysql/mysql-server/commit/59e590738a772b74ad50c4a57c86aaa1bc6501c7#diff-184e9a7d8a58f080974e475d4199fe5c6da5518c8a2811cc5df5988c8f9e9797
-        payload = buffer.getData(payloadLenth);
-    }
-
-    public byte[] getPayload() {
-        return payload;
-    }
-}
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java
index f1a4b73d..f0c03e56 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java
@@ -67,7 +67,7 @@ public final class IncidentLogEvent extends LogEvent {
         incident = incidentNumber;
 
         buffer.position(commonHeaderLen + postHeaderLen);
-        message = buffer.getName();
+        message = buffer.getString();
     }
 
     public final int getIncident() {
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java
index b4fb5096..8cc49795 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java
@@ -433,7 +433,7 @@ public class QueryLogEvent extends LogEvent {
     private int             serverCollation           = -1;
     private int             tvSec                     = -1;
     private BigInteger      ddlXid                    = BigInteger.valueOf(-1L);
-    private Charset         charset;
+    private String          charsetName;
 
     private String          timezone;
 
@@ -498,12 +498,12 @@ public class QueryLogEvent extends LogEvent {
 
         /* A 2nd variable part; this is common to all versions */
         final int queryLen = dataLen - dbLen - 1;
-        dbname = buffer.getFixName(dbLen + 1);
+        dbname = buffer.getFixString(dbLen + 1);
         if (clientCharset >= 0) {
-            charset = CharsetConversion.getNioCharset(clientCharset);
+            charsetName = CharsetConversion.getJavaCharset(clientCharset);
 
-            if (charset != null) {
-                query = buffer.getFixString(queryLen, charset);
+            if ((charsetName != null) && (Charset.isSupported(charsetName))) {
+                query = buffer.getFixString(queryLen, charsetName);
             } else {
                 logger.warn("unsupported character set in query log: " + "\n    ID = " + clientCharset + ", Charset = "
                             + CharsetConversion.getCharset(clientCharset) + ", Collation = "
@@ -603,13 +603,6 @@ public class QueryLogEvent extends LogEvent {
      */
     public static final int Q_HRNOW                           = 128;
 
-    /**
-     * Support MariaDB 10.10.1
-     */
-    public static final int Q_XID                             = 129;
-
-    public static final int Q_GTID_FLAGS3                     = 130;
-
     private final void unpackVariables(LogBuffer buffer, final int end) throws IOException {
         int code = -1;
         try {
@@ -620,10 +613,10 @@ public class QueryLogEvent extends LogEvent {
                         break;
                     case Q_SQL_MODE_CODE:
                         sql_mode = buffer.getLong64(); // QQ: Fix when sql_mode
-                        // is ulonglong
+                                                       // is ulonglong
                         break;
                     case Q_CATALOG_NZ_CODE:
-                        catalog = buffer.getName();
+                        catalog = buffer.getString();
                         break;
                     case Q_AUTO_INCREMENT:
                         autoIncrementIncrement = buffer.getUint16();
@@ -639,7 +632,7 @@ public class QueryLogEvent extends LogEvent {
                         serverCollation = buffer.getUint16();
                         break;
                     case Q_TIME_ZONE_CODE:
-                        timezone = buffer.getName();
+                        timezone = buffer.getString();
                         break;
                     case Q_CATALOG_CODE: /* for 5.0.x where 0<=x<=3 masters */
                         final int len = buffer.getUint8();
@@ -663,8 +656,8 @@ public class QueryLogEvent extends LogEvent {
                         buffer.forward(4);
                         break;
                     case Q_INVOKER:
-                        user = buffer.getName();
-                        host = buffer.getName();
+                        user = buffer.getString();
+                        host = buffer.getString();
                         break;
                     case Q_MICROSECONDS:
                         // when.tv_usec= uint3korr(pos);
@@ -685,7 +678,7 @@ public class QueryLogEvent extends LogEvent {
                         String mtsAccessedDbNames[] = new String[mtsAccessedDbs];
                         for (int i = 0; i < mtsAccessedDbs && buffer.position() < end; i++) {
                             int length = end - buffer.position();
-                            mtsAccessedDbNames[i] = buffer.getFixName(length < NAME_LEN ? length : NAME_LEN);
+                            mtsAccessedDbNames[i] = buffer.getFixString(length < NAME_LEN ? length : NAME_LEN);
                         }
                         break;
                     case Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP:
@@ -708,25 +701,6 @@ public class QueryLogEvent extends LogEvent {
                         // int when_sec_part = buffer.getUint24();
                         buffer.forward(3);
                         break;
-                    case Q_XID:
-                        // xid= uint8korr(pos);
-                        buffer.forward(8);
-                        break;
-                    case Q_GTID_FLAGS3:
-                        // gtid_flags_extra= *pos++;
-                        // if (gtid_flags_extra & (Gtid_log_event::FL_COMMIT_ALTER_E1 |
-                        // Gtid_log_event::FL_ROLLBACK_ALTER_E1)) {
-                        // sa_seq_no = uint8korr(pos);
-                        // pos+= 8;
-                        // }
-
-                        int gtid_flags_extra = buffer.getUint8();
-                        final int FL_COMMIT_ALTER_E1= 4;
-                        final int FL_ROLLBACK_ALTER_E1= 8;
-                        if ((gtid_flags_extra & (FL_COMMIT_ALTER_E1 | FL_ROLLBACK_ALTER_E1))> 0) {
-                            buffer.forward(8);
-                        }
-                        break;
                     default:
                         /*
                          * That's why you must write status vars in growing
@@ -766,22 +740,16 @@ public class QueryLogEvent extends LogEvent {
                 return "Q_TABLE_MAP_FOR_UPDATE_CODE";
             case Q_MASTER_DATA_WRITTEN_CODE:
                 return "Q_MASTER_DATA_WRITTEN_CODE";
-            case Q_INVOKER:
-                return "case Q_INVOKER";
-            case Q_MICROSECONDS:
-                return "Q_MICROSECONDS";
             case Q_UPDATED_DB_NAMES:
                 return "Q_UPDATED_DB_NAMES";
-            case Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP:
-                return "Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP";
+            case Q_MICROSECONDS:
+                return "Q_MICROSECONDS";
             case Q_DDL_LOGGED_WITH_XID:
                 return "Q_DDL_LOGGED_WITH_XID";
             case Q_DEFAULT_COLLATION_FOR_UTF8MB4:
                 return "Q_DEFAULT_COLLATION_FOR_UTF8MB4";
             case Q_SQL_REQUIRE_PRIMARY_KEY:
                 return "Q_SQL_REQUIRE_PRIMARY_KEY";
-            case Q_HRNOW:
-                return "Q_HRNOW";
         }
         return "CODE#" + code;
     }
@@ -829,8 +797,8 @@ public class QueryLogEvent extends LogEvent {
         return autoIncrementOffset;
     }
 
-    public final Charset getCharset() {
-        return charset;
+    public final String getCharsetName() {
+        return charsetName;
     }
 
     public final String getTimezone() {
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java
index 79335e11..30f05a45 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java
@@ -1,11 +1,8 @@
 package com.taobao.tddl.dbsync.binlog.event;
 
 import java.io.Serializable;
-import java.nio.charset.Charset;
 import java.sql.Timestamp;
 import java.sql.Types;
-import java.time.ZoneId;
-import java.time.zone.ZoneRules;
 import java.util.BitSet;
 
 import org.apache.commons.logging.Log;
@@ -26,11 +23,7 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  */
 public final class RowsLogBuffer {
 
-    protected static final Log    logger            = LogFactory.getLog(RowsLogBuffer.class);
-    public static final Long[]    longCache         = new Long[1024 * 128];
-    public static final int       longCacheLimit    = longCache.length + 127;
-    public static final Integer[] integerCache      = new Integer[1024 * 128];
-    public static final int       integerCacheLimit = longCache.length + 127;
+    protected static final Log logger            = LogFactory.getLog(RowsLogBuffer.class);
 
     public static final long   DATETIMEF_INT_OFS = 0x8000000000L;
     public static final long   TIMEF_INT_OFS     = 0x800000L;
@@ -40,7 +33,7 @@ public final class RowsLogBuffer {
     private final LogBuffer    buffer;
     private final int          columnLen;
     private final int          jsonColumnCount;
-    private final Charset      charset;
+    private final String       charsetName;
 
     private final BitSet       nullBits;
     private int                nullBitIndex;
@@ -54,10 +47,10 @@ public final class RowsLogBuffer {
     private int                length;
     private Serializable       value;
 
-    public RowsLogBuffer(LogBuffer buffer, final int columnLen, Charset charset, int jsonColumnCount, boolean partial){
+    public RowsLogBuffer(LogBuffer buffer, final int columnLen, String charsetName, int jsonColumnCount, boolean partial){
         this.buffer = buffer;
         this.columnLen = columnLen;
-        this.charset = charset;
+        this.charsetName = charsetName;
         this.partial = partial;
         this.jsonColumnCount = jsonColumnCount;
         this.nullBits = new BitSet(columnLen);
@@ -316,7 +309,7 @@ public final class RowsLogBuffer {
                 // XXX: How to check signed / unsigned?
                 // value = unsigned ? Long.valueOf(buffer.getUint32()) :
                 // Integer.valueOf(buffer.getInt32());
-                value = valueOf(buffer.getInt32());
+                value = Integer.valueOf(buffer.getInt32());
                 javaType = Types.INTEGER;
                 length = 4;
                 break;
@@ -325,7 +318,7 @@ public final class RowsLogBuffer {
                 // XXX: How to check signed / unsigned?
                 // value = Integer.valueOf(unsigned ? buffer.getUint8() :
                 // buffer.getInt8());
-                value = valueOf(buffer.getInt8());
+                value = Integer.valueOf(buffer.getInt8());
                 javaType = Types.TINYINT; // java.sql.Types.INTEGER;
                 length = 1;
                 break;
@@ -334,7 +327,7 @@ public final class RowsLogBuffer {
                 // XXX: How to check signed / unsigned?
                 // value = Integer.valueOf(unsigned ? buffer.getUint16() :
                 // buffer.getInt16());
-                value = valueOf((short) buffer.getInt16());
+                value = Integer.valueOf((short) buffer.getInt16());
                 javaType = Types.SMALLINT; // java.sql.Types.INTEGER;
                 length = 2;
                 break;
@@ -343,7 +336,7 @@ public final class RowsLogBuffer {
                 // XXX: How to check signed / unsigned?
                 // value = Integer.valueOf(unsigned ? buffer.getUint24() :
                 // buffer.getInt24());
-                value = valueOf(buffer.getInt24());
+                value = Integer.valueOf(buffer.getInt24());
                 javaType = Types.INTEGER;
                 length = 3;
                 break;
@@ -352,7 +345,7 @@ public final class RowsLogBuffer {
                 // XXX: How to check signed / unsigned?
                 // value = unsigned ? buffer.getUlong64()) :
                 // Long.valueOf(buffer.getLong64());
-                value = valueOf(buffer.getLong64());
+                value = Long.valueOf(buffer.getLong64());
                 javaType = Types.BIGINT; // Types.INTEGER;
                 length = 8;
                 break;
@@ -877,7 +870,7 @@ public final class RowsLogBuffer {
                 }
                 // logger.warn("MYSQL_TYPE_ENUM : This enumeration value is "
                 // + "only used internally and cannot exist in a binlog!");
-                value = valueOf(int32);
+                value = Integer.valueOf(int32);
                 javaType = Types.INTEGER;
                 length = len;
                 break;
@@ -1024,7 +1017,7 @@ public final class RowsLogBuffer {
                     javaType = Types.VARBINARY;
                     value = binary;
                 } else {
-                    value = buffer.getFullString(len, charset);
+                    value = buffer.getFullString(len, charsetName);
                     javaType = Types.VARCHAR;
                 }
 
@@ -1046,7 +1039,7 @@ public final class RowsLogBuffer {
                     javaType = Types.BINARY;
                     value = binary;
                 } else {
-                    value = buffer.getFullString(len, charset);
+                    value = buffer.getFullString(len, charsetName);
                     javaType = Types.CHAR; // Types.VARCHAR;
                 }
                 length = len;
@@ -1081,7 +1074,7 @@ public final class RowsLogBuffer {
                         len,
                         columnName,
                         columnIndex,
-                            charset);
+                        charsetName);
                     value = builder.toString();
                     buffer.position(position + len);
                 } else {
@@ -1095,9 +1088,9 @@ public final class RowsLogBuffer {
                         Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(),
                             buffer,
                             len - 1,
-                                charset);
+                            charsetName);
                         StringBuilder builder = new StringBuilder();
-                        jsonValue.toJsonString(builder, charset);
+                        jsonValue.toJsonString(builder, charsetName);
                         value = builder.toString();
                         buffer.position(position + len);
                     }
@@ -1218,38 +1211,4 @@ public final class RowsLogBuffer {
             builder.append('0').append(digits[d]);
         }
     }
-
-    public static Long valueOf(long value) {
-        if (value >= -128 & value <= 127) {
-            // if (l >= -128 && l <= 127) {
-            return Long.valueOf(value);
-        }
-        if (value > 127 && value < longCacheLimit) {
-            int cacheIndex = (int) value - 127;
-            Long cacheValue = longCache[cacheIndex];
-            if (cacheValue == null) {
-                cacheValue = new Long(value);
-                longCache[cacheIndex] = cacheValue;
-            }
-            return cacheValue;
-        }
-        return new Long(value);
-    }
-
-    public static Integer valueOf(int value) {
-        if (value >= -128 & value <= 127) {
-            // if (l >= -128 && l <= 127) {
-            return Integer.valueOf(value);
-        }
-        if (value > 127 && value < integerCacheLimit) {
-            int cacheIndex = (int) value - 127;
-            Integer cacheValue = integerCache[cacheIndex];
-            if (cacheValue == null) {
-                cacheValue = new Integer(value);
-                integerCache[cacheIndex] = cacheValue;
-            }
-            return cacheValue;
-        }
-        return new Integer(value);
-    }
 }
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java
index 2683217b..e1e78cd1 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java
@@ -1,6 +1,5 @@
 package com.taobao.tddl.dbsync.binlog.event;
 
-import java.nio.charset.Charset;
 import java.util.BitSet;
 
 import com.taobao.tddl.dbsync.binlog.exception.TableIdNotFoundException;
@@ -220,8 +219,8 @@ public abstract class RowsLogEvent extends LogEvent {
         return changeColumns;
     }
 
-    public final RowsLogBuffer getRowsBuf(Charset charset) {
-        return new RowsLogBuffer(rowsBuf, columnLen, charset, jsonColumnCount, partial);
+    public final RowsLogBuffer getRowsBuf(String charsetName) {
+        return new RowsLogBuffer(rowsBuf, columnLen, charsetName, jsonColumnCount, partial);
     }
 
     public final int getFlags(final int flags) {
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsQueryLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsQueryLogEvent.java
index d4ba23c6..16293ae1 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsQueryLogEvent.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsQueryLogEvent.java
@@ -2,8 +2,6 @@ package com.taobao.tddl.dbsync.binlog.event;
 
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 
-import java.nio.charset.StandardCharsets;
-
 /**
  * @author jianghang 2013-4-8 上午12:36:29
  * @version 1.0.3
@@ -25,7 +23,7 @@ public class RowsQueryLogEvent extends IgnorableLogEvent {
          */
         int offset = commonHeaderLen + postHeaderLen + 1;
         int len = buffer.limit() - offset;
-        rowsQuery = buffer.getFullString(offset, len, StandardCharsets.ISO_8859_1);
+        rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1);
     }
 
     public String getRowsQuery() {
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java
index 37ed1ad9..f230759b 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java
@@ -36,7 +36,7 @@ public class StartLogEventV3 extends LogEvent {
 
         buffer.position(descriptionEvent.commonHeaderLen);
         binlogVersion = buffer.getUint16(); // ST_BINLOG_VER_OFFSET
-        serverVersion = buffer.getFixName(ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
+        serverVersion = buffer.getFixString(ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
     }
 
     public StartLogEventV3(){
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java
index ead820ee..4f64a7d5 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java
@@ -423,10 +423,10 @@ public final class TableMapLogEvent extends LogEvent {
 
         /* Read the variable part of the event */
         buffer.position(commonHeaderLen + postHeaderLen);
-        dbname = buffer.getName();
+        dbname = buffer.getString();
         buffer.forward(1); /* termination null */
         // fixed issue #2714
-        tblname = buffer.getName();
+        tblname = buffer.getString();
         buffer.forward(1); /* termination null */
 
         // Read column information from buffer
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java
index 19d5320e..d6acb94c 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java
@@ -2,7 +2,6 @@ package com.taobao.tddl.dbsync.binlog.event;
 
 import java.io.IOException;
 import java.io.Serializable;
-import java.nio.charset.Charset;
 
 import com.taobao.tddl.dbsync.binlog.CharsetConversion;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
@@ -101,8 +100,8 @@ public final class UserVarLogEvent extends LogEvent {
                     value = buffer.getDecimal(precision, scale); // bin2decimal
                     break;
                 case STRING_RESULT:
-                    Charset charset = CharsetConversion.getNioCharset(charsetNumber);
-                    value = buffer.getFixString(valueLen, charset);
+                    String charsetName = CharsetConversion.getJavaCharset(charsetNumber);
+                    value = buffer.getFixString(valueLen, charsetName);
                     break;
                 case ROW_RESULT:
                     // this seems to be banned in MySQL altogether
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/AnnotateRowsEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/AnnotateRowsEvent.java
index 19992028..cbb83871 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/AnnotateRowsEvent.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/AnnotateRowsEvent.java
@@ -5,8 +5,6 @@ import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.LogHeader;
 
-import java.nio.charset.StandardCharsets;
-
 /**
  * mariadb的ANNOTATE_ROWS_EVENT类型
  * 
@@ -25,7 +23,7 @@ public class AnnotateRowsEvent extends IgnorableLogEvent {
 
         int offset = commonHeaderLen + postHeaderLen;
         int len = buffer.limit() - offset;
-        rowsQuery = buffer.getFullString(offset, len, StandardCharsets.ISO_8859_1);
+        rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1);
     }
 
     public String getRowsQuery() {
diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java
index 4980a292..27ecf3db 100644
--- a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java
+++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java
@@ -52,7 +52,7 @@ public class BaseLogFetcherTest {
                 event.getHeader().getLogPos() - event.getHeader().getEventLen(),
                 event.getTable().getDbName(),
                 event.getTable().getTableName()));
-            RowsLogBuffer buffer = event.getRowsBuf(charset);
+            RowsLogBuffer buffer = event.getRowsBuf(charset.name());
             BitSet columns = event.getColumns();
             BitSet changeColumns = event.getChangeColumns();
             while (buffer.nextOneRow(columns)) {
diff --git a/deployer/src/main/bin/startup.sh b/deployer/src/main/bin/startup.sh
index 583c080a..b8c0ed3e 100644
--- a/deployer/src/main/bin/startup.sh
+++ b/deployer/src/main/bin/startup.sh
@@ -92,12 +92,9 @@ else
 fi
 
 if [ -n "$str" ]; then
-  if [ $JavaVersion -ge 11 ] ; then
-    # For G1
-    JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
-  else
-	  JAVA_OPTS="-server -Xms2g -Xmx3g -Xmn1g -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
-	fi
+	# JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
+  # For G1
+  JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
 else
 	JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS"
 fi
diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
index 33e5e1cb..53de822c 100644
--- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
@@ -558,11 +558,6 @@ public class CanalController {
         }
 
         ZkClientx.clearClients();
-
-        // 需要释放 CanalServerWithEmbedded 否则主线程退出后，进程无法自动完整退出...
-        if (embededCanalServer != null) {
-            embededCanalServer.stop();
-        }
     }
 
     private void initCid(String path) {
diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties
index e6bdc4cc..82992bcc 100644
--- a/deployer/src/main/resources/canal.properties
+++ b/deployer/src/main/resources/canal.properties
@@ -148,13 +148,8 @@ kafka.max.in.flight.requests.per.connection = 1
 kafka.retries = 0
 
 kafka.kerberos.enable = false
-kafka.kerberos.krb5.file = ../conf/kerberos/krb5.conf
-kafka.kerberos.jaas.file = ../conf/kerberos/jaas.conf
-
-# sasl demo
-# kafka.sasl.jaas.config = org.apache.kafka.common.security.scram.ScramLoginModule required \\n username=\"alice\" \\npassword="alice-secret\";
-# kafka.sasl.mechanism = SCRAM-SHA-512
-# kafka.security.protocol = SASL_PLAINTEXT
+kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
+kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
 
 ##################################################
 ######### 		    RocketMQ	     #############
@@ -184,4 +179,4 @@ rabbitmq.deliveryMode =
 ##################################################
 pulsarmq.serverUrl =
 pulsarmq.roleToken =
-pulsarmq.topicTenantPrefix =
+pulsarmq.topicTenantPrefix =
\ No newline at end of file
diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/UUIDSet.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/UUIDSet.java
index 0eaea9cd..5926a4fa 100644
--- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/UUIDSet.java
+++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/UUIDSet.java
@@ -181,7 +181,9 @@ public class UUIDSet {
             int j;
             for (j = i + 1; j < len; j++) {
                 if (intervals.get(i).stop >= intervals.get(j).start) {
-                    intervals.get(i).stop = intervals.get(j).stop;
+                    if (intervals.get(i).stop < intervals.get(j).stop) {
+                        intervals.get(i).stop = intervals.get(j).stop;
+                    }
                 } else {
                     break;
                 }
diff --git a/driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlGTIDSetTest.java b/driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlGTIDSetTest.java
index 1c8ba0e9..bf29f7d9 100644
--- a/driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlGTIDSetTest.java
+++ b/driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlGTIDSetTest.java
@@ -42,9 +42,10 @@ public class MysqlGTIDSetTest {
         MysqlGTIDSet mysqlGTIDSet1 = MysqlGTIDSet.parse(gtid1);
 
         String gtid2 = "726757ad-4455-11e8-ae04-0242ac110002:1-20304074";
+        MysqlGTIDSet mysqlGTIDSet2 = MysqlGTIDSet.parse(gtid2);
 
         mysqlGTIDSet1.update(gtid2);
-        assertEquals("726757ad-4455-11e8-ae04-0242ac110002:1-20304074", mysqlGTIDSet1.toString());
+        assertEquals("726757ad-4455-11e8-ae04-0242ac110002:1-25536412", mysqlGTIDSet1.toString());
     }
 
     @Test
diff --git a/example/src/main/bin/startup.sh b/example/src/main/bin/startup.sh
index e83d7440..1fd8bf69 100644
--- a/example/src/main/bin/startup.sh
+++ b/example/src/main/bin/startup.sh
@@ -71,12 +71,9 @@ else
 fi
 
 if [ -n "$str" ]; then
-	if [ $JavaVersion -ge 11 ] ; then
-    # For G1
-    JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
-  else
-	  JAVA_OPTS="-server -Xms2g -Xmx3g -Xmn1g -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
-	fi
+	# JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
+  # For G1
+  JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
 else
 	JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS"
 fi
diff --git a/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java
index cb20762a..48b51b25 100644
--- a/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java
+++ b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java
@@ -109,11 +109,11 @@ public class AviaterRegexFilter implements CanalEventFilter {
     private List completionPattern(List patterns) {
         List result = new ArrayList<>();
         for (String pattern : patterns) {
-            StringBuilder stringBuilder = new StringBuilder();
-            stringBuilder.append("^");
-            stringBuilder.append(pattern);
-            stringBuilder.append("$");
-            result.add(stringBuilder.toString());
+            StringBuffer stringBuffer = new StringBuffer();
+            stringBuffer.append("^");
+            stringBuffer.append(pattern);
+            stringBuffer.append("$");
+            result.add(stringBuffer.toString());
         }
         return result;
     }
diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/plain/PlainCanalConfigClient.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/plain/PlainCanalConfigClient.java
index f8fb4355..2f509569 100644
--- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/plain/PlainCanalConfigClient.java
+++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/plain/PlainCanalConfigClient.java
@@ -16,7 +16,6 @@ import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
 import com.alibaba.otter.canal.common.CanalException;
 import com.alibaba.otter.canal.common.CanalLifeCycle;
 import com.alibaba.otter.canal.protocol.SecurityUtil;
-import com.google.common.net.UrlEscapers;
 
 /**
  * 远程配置获取
@@ -74,7 +73,7 @@ public class PlainCanalConfigClient extends AbstractCanalLifeCycle implements Ca
             md5 = "";
         }
         String url = configURL + "/api/v1/config/server_polling?ip=" + localIp + "&port=" + adminPort + "&md5=" + md5
-                     + "®ister=" + (autoRegister ? 1 : 0) + "&cluster=" + StringUtils.stripToEmpty(autoCluster) + "&name=" + StringUtils.stripToEmpty(name);
+                     + "®ister=" + (autoRegister ? 1 : 0) + "&cluster=" + autoCluster + "&name=" + name;
         return queryConfig(url);
     }
 
@@ -85,7 +84,7 @@ public class PlainCanalConfigClient extends AbstractCanalLifeCycle implements Ca
         if (StringUtils.isEmpty(md5)) {
             md5 = "";
         }
-        String url = configURL + "/api/v1/config/instance_polling/" + UrlEscapers.urlPathSegmentEscaper().escape(destination) + "?md5=" + md5;
+        String url = configURL + "/api/v1/config/instance_polling/" + destination + "?md5=" + md5;
         return queryConfig(url);
     }
 
diff --git a/meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java b/meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java
index 052e923c..53d92701 100644
--- a/meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java
+++ b/meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java
@@ -171,7 +171,7 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
                 return null;
             }
 
-            String json = FileUtils.readFileToString(dataFile, charset);
+            String json = FileUtils.readFileToString(dataFile, charset.name());
             return JsonUtils.unmarshalFromString(json, FileMetaInstanceData.class);
         } catch (IOException e) {
             throw new CanalMetaManagerException(e);
@@ -209,11 +209,7 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
 
                 data.setClientDatas(clientDatas);
             }
-            //fixed issue https://github.com/alibaba/canal/issues/4312
-            //客户端数据为空时不覆盖文件内容 （适合单客户端）
-            if(data.getClientDatas().isEmpty()){
-                return;
-            }
+
             String json = JsonUtils.marshalToString(data);
             try {
                 FileUtils.writeStringToFile(dataFile, json);
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
index 00f3d68e..8636acfd 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
@@ -12,7 +12,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import com.taobao.tddl.dbsync.binlog.event.*;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
@@ -41,7 +40,23 @@ import com.alibaba.otter.canal.protocol.CanalEntry.Type;
 import com.alibaba.otter.canal.protocol.position.EntryPosition;
 import com.google.protobuf.ByteString;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
+import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.GtidLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.HeartbeatLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.IntvarLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.LogHeader;
+import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RandLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
+import com.taobao.tddl.dbsync.binlog.event.UnknownLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.UserVarLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
 import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidListLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
@@ -132,8 +147,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                 return parseGTIDLogEvent((GtidLogEvent) logEvent);
             case LogEvent.HEARTBEAT_LOG_EVENT:
                 return parseHeartbeatLogEvent((HeartbeatLogEvent) logEvent);
-            case LogEvent.HEARTBEAT_LOG_EVENT_V2:
-                return parseHeartbeatV2LogEvent((HeartbeatV2LogEvent) logEvent);
             case LogEvent.GTID_EVENT:
             case LogEvent.GTID_LIST_EVENT:
                 return parseMariaGTIDLogEvent(logEvent);
@@ -160,22 +173,13 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
         return entryBuilder.build();
     }
 
-    private Entry parseHeartbeatV2LogEvent(HeartbeatV2LogEvent logEvent) {
-        Header.Builder headerBuilder = Header.newBuilder();
-        headerBuilder.setEventType(EventType.MHEARTBEAT);
-        Entry.Builder entryBuilder = Entry.newBuilder();
-        entryBuilder.setHeader(headerBuilder.build());
-        entryBuilder.setEntryType(EntryType.HEARTBEAT);
-        return entryBuilder.build();
-    }
-
     private Entry parseGTIDLogEvent(GtidLogEvent logEvent) {
         LogHeader logHeader = logEvent.getHeader();
         Pair.Builder builder = Pair.newBuilder();
         builder.setKey("gtid");
         builder.setValue(logEvent.getGtidStr());
 
-        if (logEvent.getLastCommitted() != -1) {
+        if (logEvent.getLastCommitted() != null) {
             builder.setKey("lastCommitted");
             builder.setValue(String.valueOf(logEvent.getLastCommitted()));
             builder.setKey("sequenceNumber");
@@ -392,7 +396,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
         // mysql5.6支持，需要设置binlog-rows-query-log-events=1，可详细打印原始DML语句
         String queryString = null;
         try {
-            queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset);
+            queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
             String tableName = null;
             if (useDruidDdlFilter) {
                 List results = DruidDdlParser.parse(queryString, null);
@@ -414,7 +418,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
         // mariaDb支持，需要设置binlog_annotate_row_events=true，可详细打印原始DML语句
         String queryString = null;
         try {
-            queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset);
+            queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
             return buildQueryEntry(queryString, event.getHeader());
         } catch (UnsupportedEncodingException e) {
             throw new CanalParseException(e);
@@ -508,10 +512,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
 
     public void parseTableMapEvent(TableMapLogEvent event) {
         try {
-            String charsetDbName = new String(event.getDbName().getBytes(ISO_8859_1), charset);
+            String charsetDbName = new String(event.getDbName().getBytes(ISO_8859_1), charset.name());
             event.setDbname(charsetDbName);
 
-            String charsetTbName = new String(event.getTableName().getBytes(ISO_8859_1), charset);
+            String charsetTbName = new String(event.getTableName().getBytes(ISO_8859_1), charset.name());
             event.setTblname(charsetTbName);
         } catch (UnsupportedEncodingException e) {
             throw new CanalParseException(e);
@@ -550,7 +554,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
             rowChangeBuider.setIsDdl(false);
 
             rowChangeBuider.setEventType(eventType);
-            RowsLogBuffer buffer = event.getRowsBuf(charset);
+            RowsLogBuffer buffer = event.getRowsBuf(charset.name());
             BitSet columns = event.getColumns();
             BitSet changeColumns = event.getChangeColumns();
 
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java
index 889b359a..9a7a5bb5 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java
@@ -100,7 +100,7 @@ public class DdlResult {
     @Override
     public String toString() {
         DdlResult ddlResult = this;
-        StringBuilder sb = new StringBuilder();
+        StringBuffer sb = new StringBuffer();
         do {
             sb.append(String.format("DdlResult [schemaName=%s , tableName=%s , oriSchemaName=%s , oriTableName=%s , type=%s ];",
                 ddlResult.schemaName,
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
index 0a2eb20e..b4c7ea2f 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
@@ -209,7 +209,6 @@ public class DatabaseTableMeta implements TableMetaTSDB {
             for (String schema : schemas) {
                 // filter views
                 packet = connection.query("show full tables from `" + schema + "` where Table_type = 'BASE TABLE'");
-                columnSize = packet.getFieldDescriptors().size();
                 int tableNameColumnIndex = 0; // default index is 0
                 List tables = new ArrayList<>();
                 for (int line = 0; line < packet.getFieldValues().size() / columnSize; line++) {
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
index 1d27f370..55aa702d 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
@@ -80,8 +80,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
                     && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create user")
                     && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "alter user")
                     && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "drop user")
-                    && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create database")
-                    && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "alter schema")) {
+                    && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create database")) {
                     repository.console(ddl);
                 }
             } catch (Throwable e) {
@@ -146,7 +145,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
     public Map snapshot() {
         Map schemaDdls = new HashMap<>();
         for (Schema schema : repository.getSchemas()) {
-            StringBuilder data = new StringBuilder(4 * 1024);
+            StringBuffer data = new StringBuffer(4 * 1024);
             for (String table : schema.showTables()) {
                 SchemaObject schemaObject = schema.findTable(table);
                 schemaObject.getStatement().output(data);
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java
index 4a0ab4a8..3a0fedd6 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java
@@ -182,7 +182,7 @@ public class FileMixedLogPositionManager extends AbstractLogPositionManager {
                 return null;
             }
 
-            String json = FileUtils.readFileToString(dataFile, charset);
+            String json = FileUtils.readFileToString(dataFile, charset.name());
             return JsonUtils.unmarshalFromString(json, LogPosition.class);
         } catch (IOException e) {
             throw new CanalMetaManagerException(e);
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java
index 1fe0010b..8207f4b7 100644
--- a/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java
@@ -302,7 +302,7 @@ public class DirectLogFetcherTest {
                 event.getHeader().getLogPos() - event.getHeader().getEventLen(),
                 event.getTable().getDbName(),
                 event.getTable().getTableName()));
-            RowsLogBuffer buffer = event.getRowsBuf(charset);
+            RowsLogBuffer buffer = event.getRowsBuf(charset.name());
             BitSet columns = event.getColumns();
             BitSet changeColumns = event.getChangeColumns();
             while (buffer.nextOneRow(columns)) {
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java
index c8cef505..58a9a9a4 100644
--- a/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java
@@ -134,7 +134,7 @@ public class MysqlBinlogParsePerformanceTest {
 
     public static void parseRowsEvent(RowsLogEvent event, AtomicLong sum) {
         try {
-            RowsLogBuffer buffer = event.getRowsBuf(charset);
+            RowsLogBuffer buffer = event.getRowsBuf(charset.name());
             BitSet columns = event.getColumns();
             BitSet changeColumns = event.getChangeColumns();
             while (buffer.nextOneRow(columns)) {
diff --git a/pom.xml b/pom.xml
index 0416fcbc..1597f67d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -58,7 +58,7 @@
         
         
             java.net
-            https://download.java.net/maven/2/
+            http://download.java.net/maven/2/
             
                 true
             
@@ -222,7 +222,7 @@
             
                 com.alibaba.fastjson2
                 fastjson2
-                2.0.15
+                2.0.7
             
             
                 com.google.guava
@@ -262,7 +262,7 @@
             
                 com.alibaba
                 druid
-                1.2.13
+                1.2.11
             
             
                 com.lmax