diff --git a/.github/issue_template.md b/.github/issue_template.md new file mode 100644 index 00000000..df5f2769 --- /dev/null +++ b/.github/issue_template.md @@ -0,0 +1,13 @@ +### environment + +* canal version +* mysql version + +### Issue Description + + +### Steps to reproduce + +### Expected behaviour + +### Actual behaviour diff --git a/README.md b/README.md index 86ef8755..8941c8fb 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,6 @@
- -

最新更新

-
    -
  1. canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。
    2. -
  2. canal消费端项目开源: Otter(分布式数据库同步系统)，地址：https://github.com/alibaba/otter
    3. -
  3. Canal已在阿里云推出商业化版本 数据传输服务DTS， 开通即用，免去部署维护的昂贵使用成本。DTS针对阿里云RDS、DRDS等产品进行了适配，解决了Binlog日志回收，主备切换、VPC网络切换等场景下的订阅高可用问题。同时，针对RDS进行了针对性的性能优化。出于稳定性、性能及成本的考虑，强烈推荐阿里云用户使用DTS产品。DTS产品使用文档
    4. -DTS支持阿里云RDS&DRDS的Binlog日志实时订阅，现推出首月免费体验，限时限量，立即体验>>> -
- +

背景

早期，阿里巴巴B2B公司因为存在杭州和美国双机房部署，存在跨机房同步的业务需求。不过早期的数据库同步业务，主要是基于trigger的方式获取增量变更，不过从2010年开始，阿里系公司开始逐步的尝试基于数据库的日志解析，获取增量变更进行同步，由此衍生出了增量订阅&消费的业务，从此开启了一段新纪元。

ps. 目前内部版本已经支持mysql和oracle部分版本的日志解析，当前的canal开源版本支持5.7及以下的版本(阿里内部mysql 5.7.13, 5.6.10, mysql 5.5.18和5.1.40/48)

@@ -46,6 +38,16 @@ DTS支持阿里云RDS&DRDS的Binlog日志实时订阅，现推出首月免费体
  • canal解析binary log对象(原始为byte流)
    • +

    重要版本更新说明

    + +canal 1.1.x系列，参考release文档：版本发布信息 + +1. 整体性能测试&优化,提升了150%. #726 参考: 【[Performance](https://github.com/alibaba/canal/wiki/Performance)】 +2. 原生支持prometheus监控 #765 【[Prometheus QuickStart](https://github.com/alibaba/canal/wiki/Prometheus-QuickStart)】 +3. 原生支持kafka消息投递 #695 【[Canal Kafka/RocketMQ QuickStart](https://github.com/alibaba/canal/wiki/Canal-Kafka-RocketMQ-QuickStart)】 +4. 原生支持aliyun rds的binlog订阅 (解决自动主备切换/oss binlog离线解析) 参考: 【[Aliyun RDS QuickStart](https://github.com/alibaba/canal/wiki/aliyun-RDS-QuickStart)】 +5. 原生支持docker镜像 #801 参考: 【[Docker QuickStart](https://github.com/alibaba/canal/wiki/Docker-QuickStart)】 +

    相关文档

    See the wiki page for : wiki文档 @@ -54,20 +56,41 @@ See the wiki page for : wiki文 +

    多语言业务

    + +1. canal整体交互协议设计上使用了protobuf3.0，理论上可以支持绝大部分的多语言场景，欢迎大家提交多客户端的PR + * canal java客户端: https://github.com/alibaba/canal/wiki/ClientExample + * canal c#客户端开源项目地址： https://github.com/CanalSharp/CanalSharp + * canal go客户端开源项目地址： https://github.com/CanalClient/canal-go +2. canal作为MySQL binlog的增量获取工具，可以将数据投递到MQ系统中，比如Kafka/RocketMQ，可以借助于MQ的多语言能力 + * 参考文档: [Canal Kafka/RocketMQ QuickStart](https://github.com/alibaba/canal/wiki/Canal-Kafka-RocketMQ-QuickStart) +

    相关资料

    * ADC阿里技术嘉年华分享ppt (放在google docs上，可能需要翻墙): ppt下载 -* [与阿里巴巴的RocketMQ配合使用](https://github.com/alibaba/RocketMQ) +* [与阿里巴巴的RocketMQ配合使用](https://github.com/apache/RocketMQ)

    相关开源

      @@ -91,9 +114,13 @@ See the wiki page for : wiki文
    1. 报告issue：issues
    -
    -【招聘】阿里巴巴中间件团队招聘JAVA高级工程师
-岗位主要为技术型内容(非业务部门)，阿里中间件整个体系对于未来想在技术上有所沉淀的同学还是非常有帮助的
-工作地点：杭州、北京均可. ps. 阿里待遇向来都是不错的，有意者可以QQ、微博私聊. 
-具体招聘内容：https://job.alibaba.com/zhaopin/position_detail.htm?positionId=32666
-
    +

    最新更新

    +
      +
    1. canal发布重大版本更新1.1.0，具体releaseNode参考：https://github.com/alibaba/canal/releases/tag/canal-1.1.0
      2. +
    2. canal c#客户端开源项目地址： https://github.com/CanalClient/CanalSharp ，推荐!
      3. +
    3. canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。
      4. +
    4. canal消费端项目开源: Otter(分布式数据库同步系统)，地址：https://github.com/alibaba/otter
      5. + +
    5. Canal已在阿里云推出商业化版本 数据传输服务DTS， 开通即用，免去部署维护的昂贵使用成本。DTS针对阿里云RDS、DRDS等产品进行了适配，解决了Binlog日志回收，主备切换、VPC网络切换等场景下的订阅高可用问题。同时，针对RDS进行了针对性的性能优化。出于稳定性、性能及成本的考虑，强烈推荐阿里云用户使用DTS产品。DTS产品使用文档
      6. +DTS支持阿里云RDS&DRDS的Binlog日志实时订阅，现推出首月免费体验，限时限量，立即体验>>> +
    diff --git a/client-adapter/README.md b/client-adapter/README.md new file mode 100644 index 00000000..80c62c0b --- /dev/null +++ b/client-adapter/README.md @@ -0,0 +1,545 @@ +## 基本说明 +canal 1.1.1版本之后, 增加客户端数据落地的适配及启动功能, 目前支持功能: +* 客户端启动器 +* 同步管理REST接口 +* 日志适配器, 作为DEMO +* HBase的数据同步(表对表同步), ETL功能 +* (后续支持) ElasticSearch多表数据同步,ETL功能 + +## 环境版本 +* 操作系统：无要求 +* java版本: jdk1.8 以上 +* canal 版本: 请下载最新的安装包，本文以当前v1.1.1 的canal.deployer-1.1.1.tar.gz为例 +* MySQL版本 ：5.7.18 +* HBase版本: Apache HBase 1.1.2, 若和服务端版本不一致可自行替换客户端HBase依赖 + +## 一、适配器启动器 +client-adapter分为适配器和启动器两部分, 适配器为多个fat jar, 每个适配器会将自己所需的依赖打成一个包, 以SPI的方式让启动器动态加载 + + +启动器为 SpringBoot 项目, 支持canal-client启动的同时提供相关REST管理接口, 运行目录结构为: +``` +- bin + restart.sh + startup.bat + startup.sh + stop.sh +- lib + client-adapter.logger-1.1.1-jar-with-dependencies.jar + client-adapter.hbase-1.1.1-jar-with-dependencies.jar + ... +- conf + application.yml + - hbase + mytest_person2.yml +- logs +``` +以上目录结构最终会打包成 canal-adapter-launcher.tar.gz 压缩包 + +## 二、启动器 +### 2.1 启动器配置 application.yml +#### canal相关配置部分说明 +``` +canal.conf: + canalServerHost: 127.0.0.1:11111 # 对应单机模式下的canal server的ip:port + zookeeperHosts: slave1:2181 # 对应集群模式下的zk地址, 如果配置了canalServerHost, 则以canalServerHost为准 + bootstrapServers: slave1:6667 # kafka或rocketMQ地址, 与canalServerHost不能并存 + flatMessage: true # 扁平message开关, 是否以json字符串形式投递数据, 仅在kafka/rocketMQ模式下有效 + canalInstances: # canal实例组, 如果是tcp模式可配置此项 + - instance: example # 对应canal destination + groups: # 对应适配器分组, 分组间的适配器并行运行 + - outAdapters: # 适配器列表, 分组内的适配串行运行 + - name: logger # 适配器SPI名 + - name: hbase + properties: # HBase相关连接参数 + hbase.zookeeper.quorum: slave1 + hbase.zookeeper.property.clientPort: 2181 + zookeeper.znode.parent: /hbase + mqTopics: # MQ topic组, 如果是kafka或者rockeMQ模式可配置此项, 与canalInstances不能并存 + - mqMode: kafka # MQ的模式: kafak/rocketMQ + topic: example # MQ topic + groups: # group组 + - groupId: g2 # group id + outAdapters: # 适配器列表, 以下配置和canalInstances中一样 + - name: logger +``` +#### 适配器相关配置部分说明 +``` +adapter.conf: + datasourceConfigs: # 数据源配置列表, 数据源将在适配器中用于ETL、数据同步回查等使用 + defaultDS: # 数据源 dataSourceKey, 适配器中通过该值获取指定数据源 + url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true + username: root + password: 121212 + adapterConfigs: # 适配器内部配置列表 + - hbase/mytest_person2.yml # 类型/配置文件名, 这里示例就是对应HBase适配器hbase目录下的mytest_person2.yml文件 +``` +## 2.2 同步管理REST接口 +#### 2.2.1 查询所有订阅同步的canal destination或MQ topic +``` +curl http://127.0.0.1:8081/destinations +``` +#### 2.2.2 数据同步开关 +``` +curl http://127.0.0.1:8081/syncSwitch/example/off -X PUT +``` +针对 example 这个canal destination/MQ topic 进行开关操作. off代表关闭, 该destination/topic下的同步将阻塞或者断开连接不再接收数据, on代表开启 + +注: 如果在配置文件中配置了 zookeeperHosts 项, 则会使用分布式锁来控制HA中的数据同步开关, 如果是单机模式则使用本地锁来控制开关 +#### 2.2.3 数据同步开关状态 +``` +curl http://127.0.0.1:8081/syncSwitch/example +``` +查看指定 canal destination/MQ topic 的数据同步开关状态 +#### 2.2.4 手动ETL +``` +curl http://127.0.0.1:8081/etl/hbase/mytest_person2.yml -X POST -d "params=2018-10-21 00:00:00" +``` +导入数据到指定类型的库 +#### 2.2.5 查看相关库总数据 +``` +curl http://127.0.0.1:8081/count/hbase/mytest_person2.yml +``` +### 2.3 启动canal-adapter示例 +#### 2.3.1 启动canal server (单机模式), 参考: [Canal QuickStart](https://github.com/alibaba/canal/wiki/QuickStart) +#### 2.3.2 修改conf/application.yml为: +``` +server: + port: 8081 +logging: + level: + com.alibaba.otter.canal.client.adapter.hbase: DEBUG +spring: + jackson: + date-format: yyyy-MM-dd HH:mm:ss + time-zone: GMT+8 + default-property-inclusion: non_null + +canal.conf: + canalServerHost: 127.0.0.1:11111 + flatMessage: true + canalInstances: + - instance: example + adapterGroups: + - outAdapters: + - name: logger +``` +启动 +``` +bin/startup.sh +``` + +## 三、HBase适配器 +### 3.1 修改启动器配置: application.yml +``` +server: + port: 8081 +logging: + level: + com.alibaba.otter.canal.client.adapter.hbase: DEBUG +spring: + jackson: + date-format: yyyy-MM-dd HH:mm:ss + time-zone: GMT+8 + default-property-inclusion: non_null + +canal.conf: + canalServerHost: 127.0.0.1:11111 + flatMessage: true + srcDataSources: + defaultDS: + url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true + username: root + password: 121212 + canalInstances: + - instance: example + adapterGroups: + - outAdapters: + - name: hbase + properties: + hbase.zookeeper.quorum: slave1 + hbase.zookeeper.property.clientPort: 2181 + zookeeper.znode.parent: /hbase +``` +adapter将会自动加载 conf/hbase 下的所有.yml结尾的配置文件 +### 3.2 适配器表映射文件 +修改 conf/hbase/mytest_person.yml文件: +``` +dataSourceKey: defaultDS # 对应application.yml中的datasourceConfigs下的配置 +hbaseMapping: # mysql--HBase的单表映射配置 + mode: STRING # HBase中的存储类型, 默认统一存为String, 可选: #PHOENIX #NATIVE #STRING + # NATIVE: 以java类型为主, PHOENIX: 将类型转换为Phoenix对应的类型 + destination: example # 对应 canal destination/MQ topic 名称 + database: mytest # 数据库名/schema名 + table: person # 表名 + hbaseTable: MYTEST.PERSON # HBase表名 + family: CF # 默认统一Column Family名称 + uppercaseQualifier: true # 字段名转大写, 默认为true + commitBatch: 3000 # 批量提交的大小, ETL中用到 + #rowKey: id,type # 复合字段rowKey不能和columns中的rowKey并存 + # 复合rowKey会以 '|' 分隔 + columns: # 字段映射, 如果不配置将自动映射所有字段, + # 并取第一个字段为rowKey, HBase字段名以mysql字段名为主 + id: ROWKE + name: CF:NAME + email: EMAIL # 如果column family为默认CF, 则可以省略 + type: # 如果HBase字段和mysql字段名一致, 则可以省略 + c_time: + birthday: +``` +如果涉及到类型转换,可以如下形式: +``` +... + columns: + id: ROWKE$STRING + ... + type: TYPE$BYTE + ... +``` +类型转换涉及到Java类型和Phoenix类型两种, 分别定义如下: +``` +#Java 类型转换, 对应配置 mode: NATIVE +$DEFAULT +$STRING +$INTEGER +$LONG +$SHORT +$BOOLEAN +$FLOAT +$DOUBLE +$BIGDECIMAL +$DATE +$BYTE +$BYTES +``` +``` +#Phoenix 类型转换, 对应配置 mode: PHOENIX +$DEFAULT 对应PHOENIX里的VARCHAR +$UNSIGNED_INT 对应PHOENIX里的UNSIGNED_INT 4字节 +$UNSIGNED_LONG 对应PHOENIX里的UNSIGNED_LONG 8字节 +$UNSIGNED_TINYINT 对应PHOENIX里的UNSIGNED_TINYINT 1字节 +$UNSIGNED_SMALLINT 对应PHOENIX里的UNSIGNED_SMALLINT 2字节 +$UNSIGNED_FLOAT 对应PHOENIX里的UNSIGNED_FLOAT 4字节 +$UNSIGNED_DOUBLE 对应PHOENIX里的UNSIGNED_DOUBLE 8字节 +$INTEGER 对应PHOENIX里的INTEGER 4字节 +$BIGINT 对应PHOENIX里的BIGINT 8字节 +$TINYINT 对应PHOENIX里的TINYINT 1字节 +$SMALLINT 对应PHOENIX里的SMALLINT 2字节 +$FLOAT 对应PHOENIX里的FLOAT 4字节 +$DOUBLE 对应PHOENIX里的DOUBLE 8字节 +$BOOLEAN 对应PHOENIX里的BOOLEAN 1字节 +$TIME 对应PHOENIX里的TIME 8字节 +$DATE 对应PHOENIX里的DATE 8字节 +$TIMESTAMP 对应PHOENIX里的TIMESTAMP 12字节 +$UNSIGNED_TIME 对应PHOENIX里的UNSIGNED_TIME 8字节 +$UNSIGNED_DATE 对应PHOENIX里的UNSIGNED_DATE 8字节 +$UNSIGNED_TIMESTAMP 对应PHOENIX里的UNSIGNED_TIMESTAMP 12字节 +$VARCHAR 对应PHOENIX里的VARCHAR 动态长度 +$VARBINARY 对应PHOENIX里的VARBINARY 动态长度 +$DECIMAL 对应PHOENIX里的DECIMAL 动态长度 +``` +如果不配置将以java对象原生类型默认映射转换 +### 3.3 启动HBase数据同步 +#### 创建HBase表 +在HBase shell中运行: +``` +create 'MYTEST.PERSON', {NAME=>'CF'} +``` +#### 启动canal-adapter启动器 +``` +bin/startup.sh +``` +#### 验证 +修改mysql mytest.person表的数据, 将会自动同步到HBase的MYTEST.PERSON表下面, 并会打出DML的log + + +## 四、关系型数据库适配器 + +RDB adapter 用于适配mysql到任意关系型数据库(需支持jdbc)的数据同步及导入 + +### 4.1 修改启动器配置: application.yml, 这里以oracle目标库为例 +``` +server: + port: 8081 +logging: + level: + com.alibaba.otter.canal.client.adapter.rdb: DEBUG +...... + +canal.conf: + canalServerHost: 127.0.0.1:11111 + srcDataSources: + defaultDS: + url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true + username: root + password: 121212 + canalInstances: + - instance: example + groups: + - outAdapters: + - name: rdb # 指定为rdb类型同步 + key: oracle1 # 指定adapter的唯一key, 与表映射配置中outerAdapterKey对应 + properties: + jdbc.driverClassName: oracle.jdbc.OracleDriver # jdbc驱动名, jdbc的jar包需要自行放致lib目录下 + jdbc.url: jdbc:oracle:thin:@localhost:49161:XE # jdbc url + jdbc.username: mytest # jdbc username + jdbc.password: m121212 # jdbc password + threads: 5 # 并行执行的线程数, 默认为1 + commitSize: 3000 # 批次提交的最大行数 +``` +其中 outAdapter 的配置: name统一为rdb, key为对应的数据源的唯一标识需和下面的表映射文件中的outerAdapterKey对应, properties为目标库jdb的相关参数 +adapter将会自动加载 conf/rdb 下的所有.yml结尾的表映射配置文件 + +### 4.2 适配器表映射文件 +修改 conf/rdb/mytest_user.yml文件: +``` +dataSourceKey: defaultDS # 源数据源的key, 对应上面配置的srcDataSources中的值 +destination: example # cannal的instance或者MQ的topic +outerAdapterKey: oracle1 # adapter key, 对应上面配置outAdapters中的key +concurrent: true # 是否按主键hase并行同步, 并行同步的表必须保证主键不会更改及主键不能为其他同步表的外键!! +dbMapping: + database: mytest # 源数据源的database/shcema + table: user # 源数据源表名 + targetTable: mytest.tb_user # 目标数据源的库名.表名 + targetPk: # 主键映射 + id: id # 如果是复合主键可以换行映射多个 +# mapAll: true # 是否整表映射, 要求源表和目标表字段名一模一样 (如果targetColumns也配置了映射,则以targetColumns配置为准) + targetColumns: # 字段映射, 格式: 目标表字段: 源表字段, 如果字段名一样源表字段名可不填 + id: + name: + role_id: + c_time: + test1: +``` +导入的类型以目标表的元类型为准, 将自动转换 + +### 4.3 启动RDB数据同步 +#### 将目标库的jdbc jar包放入lib文件夹, 这里放入ojdbc6.jar + +#### 启动canal-adapter启动器 +``` +bin/startup.sh +``` +#### 验证 +修改mysql mytest.user表的数据, 将会自动同步到Oracle的MYTEST.TB_USER表下面, 并会打出DML的log + + +## 五、ElasticSearch适配器 +### 5.1 修改启动器配置: application.yml +``` +server: + port: 8081 +logging: + level: + com.alibaba.otter.canal.client.adapter.es: DEBUG +spring: + jackson: + date-format: yyyy-MM-dd HH:mm:ss + time-zone: GMT+8 + default-property-inclusion: non_null + +canal.conf: + canalServerHost: 127.0.0.1:11111 + flatMessage: true + srcDataSources: + defaultDS: + url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true + username: root + password: 121212 + canalInstances: + - instance: example + adapterGroups: + - outAdapters: + - name: es + hosts: 127.0.0.1:9300 # es 集群地址, 逗号分隔 + properties: + cluster.name: elasticsearch # es cluster name +``` +adapter将会自动加载 conf/es 下的所有.yml结尾的配置文件 +### 5.2 适配器表映射文件 +修改 conf/es/mytest_user.yml文件: +``` +dataSourceKey: defaultDS # 源数据源的key, 对应上面配置的srcDataSources中的值 +destination: example # cannal的instance或者MQ的topic +esMapping: + _index: mytest_user # es 的索引名称 + _type: _doc # es 的doc名称 + _id: _id # es 的_id, 如果不配置该项必须配置下面的pk项_id则会由es自动分配 +# pk: id # 如果不需要_id, 则需要指定一个属性为主键属性 + # sql映射 + sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name, + a.c_time as _c_time, c.labels as _labels from user a + left join role b on b.id=a.role_id + left join (select user_id, group_concat(label order by id desc separator ';') as labels from label + group by user_id) c on c.user_id=a.id" +# objFields: +# _labels: array:; # 数组或者对象属性, array:; 代表以;字段里面是以;分隔的 +# _obj: obj:{"test":"123"} + etlCondition: "where a.c_time>='{0}'" # etl 的条件参数 + commitBatch: 3000 # 提交批大小 +``` +sql映射说明: + +sql支持多表关联自由组合, 但是有一定的限制: +1. 主表不能为子查询语句 +2. 只能使用left outer join即最左表一定要是主表 +3. 关联从表如果是子查询不能有多张表 +4. 主sql中不能有where查询条件(从表子查询中可以有where条件但是不推荐, 可能会造成数据同步的不一致, 比如修改了where条件中的字段内容) +5. 关联条件只允许主外键的'='操作不能出现其他常量判断比如: on a.role_id=b.id and b.statues=1 +6. 关联条件必须要有一个字段出现在主查询语句中比如: on a.role_id=b.id 其中的 a.role_id 或者 b.id 必须出现在主select语句中 + + +Elastic Search的mapping 属性与sql的查询值将一一对应(不支持 select *), 比如: select a.id as _id, a.name, a.email as _email from user, 其中name将映射到es mapping的name field, _email将 +映射到mapping的_email field, 这里以别名(如果有别名)作为最终的映射字段. 这里的_id可以填写到配置文件的 _id: _id映射. + +#### 5.2.1.单表映射索引示例sql: +``` +select a.id as _id, a.name, a.role_id, a.c_time from user a +``` +该sql对应的es mapping示例: +``` +{ + "mytest_user": { + "mappings": { + "_doc": { + "properties": { + "name": { + "type": "text" + }, + "role_id": { + "type": "long" + }, + "c_time": { + "type": "date" + } + } + } + } + } +} +``` + +#### 5.2.2.单表映射索引示例sql带函数或运算操作: +``` +select a.id as _id, concat(a.name,'_test') as name, a.role_id+10000 as role_id, a.c_time from user a +``` +函数字段后必须跟上别名, 该sql对应的es mapping示例: +``` +{ + "mytest_user": { + "mappings": { + "_doc": { + "properties": { + "name": { + "type": "text" + }, + "role_id": { + "type": "long" + }, + "c_time": { + "type": "date" + } + } + } + } + } +} +``` + +#### 5.2.3.多表映射(一对一, 多对一)索引示例sql: +``` +select a.id as _id, a.name, a.role_id, b.role_name, a.c_time from user a +left join role b on b.id = a.role_id +``` +注:这里join操作只能是left outer join, 第一张表必须为主表!! + +该sql对应的es mapping示例: +``` +{ + "mytest_user": { + "mappings": { + "_doc": { + "properties": { + "name": { + "type": "text" + }, + "role_id": { + "type": "long" + }, + "role_name": { + "type": "text" + }, + "c_time": { + "type": "date" + } + } + } + } + } +} +``` + +#### 5.2.4.多表映射(一对多)索引示例sql: +``` +select a.id as _id, a.name, a.role_id, c.labels, a.c_time from user a +left join (select user_id, group_concat(label order by id desc separator ';') as labels from label + group by user_id) c on c.user_id=a.id +``` +注:left join 后的子查询只允许一张表, 即子查询中不能再包含子查询或者关联!! + +该sql对应的es mapping示例: +``` +{ + "mytest_user": { + "mappings": { + "_doc": { + "properties": { + "name": { + "type": "text" + }, + "role_id": { + "type": "long" + }, + "c_time": { + "type": "date" + }, + "labels": { + "type": "text" + } + } + } + } + } +} +``` + +#### 5.2.5.其它类型的sql示例: +- geo type +``` +select ... concat(IFNULL(a.latitude, 0), ',', IFNULL(a.longitude, 0)) AS location, ... +``` +- 复合主键 +``` +select concat(a.id,'_',b.type) as _id, ... from user a left join role b on b.id=a.role_id +``` +- 数组字段 +``` +select a.id as _id, a.name, a.role_id, c.labels, a.c_time from user a +left join (select user_id, group_concat(label order by id desc separator ';') as labels from label + group by user_id) c on c.user_id=a.id +``` +配置中使用: +``` +objFields: + labels: array:; +``` + +### 5.3 启动ES数据同步 +#### 启动canal-adapter启动器 +``` +bin/startup.sh +``` +#### 验证 +1. 新增mysql mytest.user表的数据, 将会自动同步到es的mytest_user索引下面, 并会打出DML的log +2. 修改mysql mytest.role表的role_name, 将会自动同步es的mytest_user索引中的role_name数据 +3. 新增或者修改mysql mytest.label表的label, 将会自动同步es的mytest_user索引中的labels数据 diff --git a/client-adapter/common/pom.xml b/client-adapter/common/pom.xml index 93793d09..ec3ab9b8 100644 --- a/client-adapter/common/pom.xml +++ b/client-adapter/common/pom.xml @@ -1,11 +1,10 @@ - canal.client-adapter com.alibaba.otter - 1.1.1-SNAPSHOT + 1.1.3-SNAPSHOT 4.0.0 com.alibaba.otter @@ -16,18 +15,28 @@ com.alibaba.otter canal.protocol - ${project.version} - - - org.slf4j - slf4j-api - 1.7.12 + 1.1.3-SNAPSHOT joda-time joda-time 2.9.4 + + com.alibaba + druid + 1.1.9 + + + org.springframework + spring-context + 5.0.5.RELEASE + + + org.yaml + snakeyaml + 1.19 + diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/CanalOuterAdapter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/CanalOuterAdapter.java deleted file mode 100644 index 0871eb74..00000000 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/CanalOuterAdapter.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.alibaba.otter.canal.client.adapter; - -import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.client.adapter.support.SPI; -import com.alibaba.otter.canal.protocol.FlatMessage; -import com.alibaba.otter.canal.protocol.Message; - -/** - * 外部适配器接口 - * - * @author machengyuan 2018-8-18 下午10:14:02 - * @version 1.0.0 - */ -@SPI("logger") -public interface CanalOuterAdapter { - - /** - * 外部适配器初始化接口 - * - * @param configuration 外部适配器配置信息 - */ - void init(CanalOuterAdapterConfiguration configuration); - - /** - * 往适配器中写入数据 - * - * @param dml 数据包 - */ - void writeOut(Dml dml); - - // void writeOut(FlatMessage flatMessage); - - /** - * 外部适配器销毁接口 - */ - void destroy(); -} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java new file mode 100644 index 00000000..31004f16 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java @@ -0,0 +1,70 @@ +package com.alibaba.otter.canal.client.adapter; + +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import com.alibaba.otter.canal.client.adapter.support.Dml; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; +import com.alibaba.otter.canal.client.adapter.support.SPI; + +/** + * 外部适配器接口 + * + * @author reweerma 2018-8-18 下午10:14:02 + * @version 1.0.0 + */ +@SPI("logger") +public interface OuterAdapter { + + /** + * 外部适配器初始化接口 + * + * @param configuration 外部适配器配置信息 + * @param envProperties 环境变量的配置属性 + */ + void init(OuterAdapterConfig configuration, Properties envProperties); + + /** + * 往适配器中同步数据 + * + * @param dmls 数据包 + */ + void sync(List dmls); + + /** + * 外部适配器销毁接口 + */ + void destroy(); + + /** + * Etl操作 + * + * @param task 任务名, 对应配置名 + * @param params etl筛选条件 + */ + default EtlResult etl(String task, List params) { + throw new UnsupportedOperationException("unsupported operation"); + } + + /** + * 计算总数 + * + * @param task 任务名, 对应配置名 + * @return 总数 + */ + default Map count(String task) { + throw new UnsupportedOperationException("unsupported operation"); + } + + /** + * 通过task获取对应的destination + * + * @param task 任务名, 对应配置名 + * @return destination + */ + default String getDestination(String task) { + return null; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java new file mode 100644 index 00000000..2cb0890d --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java @@ -0,0 +1,156 @@ +package com.alibaba.otter.canal.client.adapter.config; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +import org.springframework.util.PropertyPlaceholderHelper; +import org.springframework.util.StringUtils; + +import com.alibaba.otter.canal.client.adapter.config.bind.PropertiesConfigurationFactory; +import com.alibaba.otter.canal.client.adapter.config.common.*; + +/** + * 将yaml内容绑定到指定对象, 遵循spring yml的绑定规范 + * + * @author reweerma 2019-2-1 上午9:14:02 + * @version 1.0.0 + */ +public class YmlConfigBinder { + + /** + * 将当前内容绑定到指定对象 + * + * @param content yml内容 + * @param clazz 指定对象类型 + * @return 对象 + */ + public static T bindYmlToObj(String content, Class clazz) { + return bindYmlToObj(null, content, clazz, null); + } + + /** + * 将当前内容绑定到指定对象并指定内容编码格式 + * + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String content, Class clazz, String charset) { + return bindYmlToObj(null, content, clazz, charset); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象 + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz) { + return bindYmlToObj(prefix, content, clazz, null); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象并指定内容编码格式 + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz, String charset) { + return bindYmlToObj(prefix, content, clazz, charset, null); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: 当前内容有属性 zkServers: + * ${zookeeper.servers} 在envProperties中有属性 zookeeper.servers: + * 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 则当前内容 zkServers 会被替换为 + * zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 注: 假设绑定的类中 + * zkServers 属性是 List 对象, 则会自动映射成List + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz, String charset, + Properties baseProperties) { + try { + byte[] contentBytes; + if (charset == null) { + contentBytes = content.getBytes(); + } else { + contentBytes = content.getBytes(charset); + } + YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader(); + Resource configResource = new ByteArrayResource(contentBytes); + PropertySource propertySource = propertySourceLoader.load("manualBindConfig", configResource, null); + + if (propertySource == null) { + return null; + } + + Properties properties = new Properties(); + Map propertiesRes = new LinkedHashMap<>(); + if (!StringUtils.isEmpty(prefix) && !prefix.endsWith(".")) { + prefix = prefix + "."; + } + + properties.putAll((Map) propertySource.getSource()); + + if (baseProperties != null) { + baseProperties.putAll(properties); + properties = baseProperties; + } + + for (Object o : ((Map) propertySource.getSource()).entrySet()) { + Map.Entry entry = (Map.Entry) o; + String key = (String) entry.getKey(); + Object value = entry.getValue(); + + if (prefix != null) { + if (key != null && key.startsWith(prefix)) { + key = key.substring(prefix.length()); + } else { + continue; + } + } + + if (value != null && value.toString().contains("${")) { + PropertyPlaceholderHelper propertyPlaceholderHelper = new PropertyPlaceholderHelper("${", "}"); + value = propertyPlaceholderHelper.replacePlaceholders(value.toString(), properties); + } + + propertiesRes.put(key, value); + } + + if (propertiesRes.isEmpty()) { + return null; + } + + propertySource = new MapPropertySource(propertySource.getName(), propertiesRes); + + T target = clazz.newInstance(); + + MutablePropertySources propertySources = new MutablePropertySources(); + propertySources.addFirst(propertySource); + + PropertiesConfigurationFactory factory = new PropertiesConfigurationFactory(target); + factory.setPropertySources(propertySources); + factory.setIgnoreInvalidFields(true); + factory.setIgnoreUnknownFields(true); + + factory.bindPropertiesToTarget(); + + return target; + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java new file mode 100644 index 00000000..0b63ed2c --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java @@ -0,0 +1,97 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * {@link PropertyNamePatternsMatcher} that matches when a property name exactly + * matches one of the given names, or starts with one of the given names + * followed by a delimiter. This implementation is optimized for frequent calls. + * + * @author Phillip Webb + * @since 1.2.0 + */ +class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher { + + private final char[] delimiters; + + private final boolean ignoreCase; + + private final String[] names; + + protected DefaultPropertyNamePatternsMatcher(char[] delimiters, String... names){ + this(delimiters, false, names); + } + + protected DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, String... names){ + this(delimiters, ignoreCase, new HashSet(Arrays.asList(names))); + } + + DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, Set names){ + this.delimiters = delimiters; + this.ignoreCase = ignoreCase; + this.names = names.toArray(new String[names.size()]); + } + + @Override + public boolean matches(String propertyName) { + char[] propertyNameChars = propertyName.toCharArray(); + boolean[] match = new boolean[this.names.length]; + boolean noneMatched = true; + for (int i = 0; i < this.names.length; i++) { + if (this.names[i].length() <= propertyNameChars.length) { + match[i] = true; + noneMatched = false; + } + } + if (noneMatched) { + return false; + } + for (int charIndex = 0; charIndex < propertyNameChars.length; charIndex++) { + for (int nameIndex = 0; nameIndex < this.names.length; nameIndex++) { + if (match[nameIndex]) { + match[nameIndex] = false; + if (charIndex < this.names[nameIndex].length()) { + if (isCharMatch(this.names[nameIndex].charAt(charIndex), propertyNameChars[charIndex])) { + match[nameIndex] = true; + noneMatched = false; + } + } else { + char charAfter = propertyNameChars[this.names[nameIndex].length()]; + if (isDelimiter(charAfter)) { + match[nameIndex] = true; + noneMatched = false; + } + } + } + } + if (noneMatched) { + return false; + } + } + for (int i = 0; i < match.length; i++) { + if (match[i]) { + return true; + } + } + return false; + } + + private boolean isCharMatch(char c1, char c2) { + if (this.ignoreCase) { + return Character.toLowerCase(c1) == Character.toLowerCase(c2); + } + return c1 == c2; + } + + private boolean isDelimiter(char c) { + for (char delimiter : this.delimiters) { + if (c == delimiter) { + return true; + } + } + return false; + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java new file mode 100644 index 00000000..c9b259e3 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java @@ -0,0 +1,31 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.beans.PropertyEditorSupport; +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * {@link PropertyNamePatternsMatcher} that matches when a property name exactly + * matches one of the given names, or starts with one of the given names + * followed by a delimiter. This implementation is optimized for frequent calls. + * + * @author Phillip Webb + * @since 1.2.0 + */ +class InetAddressEditor extends PropertyEditorSupport { + + @Override + public String getAsText() { + return ((InetAddress) getValue()).getHostAddress(); + } + + @Override + public void setAsText(String text) throws IllegalArgumentException { + try { + setValue(InetAddress.getByName(text)); + } catch (UnknownHostException ex) { + throw new IllegalArgumentException("Cannot locate host", ex); + } + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java new file mode 100644 index 00000000..b79a03e8 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java @@ -0,0 +1,52 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import org.springframework.beans.PropertyValue; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; + +/** + * A {@link PropertyValue} that can provide information about its origin. + * + * @author Andy Wilkinson + */ +class OriginCapablePropertyValue extends PropertyValue { + + private static final String ATTRIBUTE_PROPERTY_ORIGIN = "propertyOrigin"; + + private final PropertyOrigin origin; + + OriginCapablePropertyValue(PropertyValue propertyValue){ + this(propertyValue.getName(), + propertyValue.getValue(), + (PropertyOrigin) propertyValue.getAttribute(ATTRIBUTE_PROPERTY_ORIGIN)); + } + + OriginCapablePropertyValue(String name, Object value, String originName, PropertySource originSource){ + this(name, value, new PropertyOrigin(originSource, originName)); + } + + OriginCapablePropertyValue(String name, Object value, PropertyOrigin origin){ + super(name, value); + this.origin = origin; + setAttribute(ATTRIBUTE_PROPERTY_ORIGIN, origin); + } + + public PropertyOrigin getOrigin() { + return this.origin; + } + + @Override + public String toString() { + String name = (this.origin != null ? this.origin.getName() : this.getName()); + String source = (this.origin.getSource() != null ? this.origin.getSource().getName() : "unknown"); + return "'" + name + "' from '" + source + "'"; + } + + public static PropertyOrigin getOrigin(PropertyValue propertyValue) { + if (propertyValue instanceof OriginCapablePropertyValue) { + return ((OriginCapablePropertyValue) propertyValue).getOrigin(); + } + return new OriginCapablePropertyValue(propertyValue).getOrigin(); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java new file mode 100644 index 00000000..a041a673 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java @@ -0,0 +1,27 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Collection; + +import org.springframework.util.PatternMatchUtils; + +/** + * {@link PropertyNamePatternsMatcher} that delegates to + * {@link PatternMatchUtils#simpleMatch(String[], String)}. + * + * @author Phillip Webb + * @since 1.2.0 + */ +class PatternPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher { + + private final String[] patterns; + + PatternPropertyNamePatternsMatcher(Collection patterns){ + this.patterns = (patterns != null ? patterns.toArray(new String[patterns.size()]) : new String[] {}); + } + + @Override + public boolean matches(String propertyName) { + return PatternMatchUtils.simpleMatch(this.patterns, propertyName); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java new file mode 100644 index 00000000..76a97c0b --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java @@ -0,0 +1,356 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.beans.PropertyDescriptor; +import java.util.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.PropertyValues; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.support.ResourceEditorRegistrar; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.MessageSource; +import org.springframework.context.MessageSourceAware; +import org.springframework.core.convert.ConversionService; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.validation.*; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySources; + +/** + * Validate some {@link Properties} (or optionally + * {@link org.springframework.core.env.PropertySources}) by binding them to an + * object of a specified type and then optionally running a {@link Validator} + * over it. + * + * @param the target type + * @author Dave Syer + */ +public class PropertiesConfigurationFactory implements FactoryBean, ApplicationContextAware, MessageSourceAware, InitializingBean { + + private static final char[] EXACT_DELIMITERS = { '_', '.', '[' }; + + private static final char[] TARGET_NAME_DELIMITERS = { '_', '.' }; + + private static final Log logger = LogFactory.getLog(PropertiesConfigurationFactory.class); + + private boolean ignoreUnknownFields = true; + + private boolean ignoreInvalidFields; + + private boolean exceptionIfInvalid = true; + + private PropertySources propertySources; + + private final T target; + + private Validator validator; + + private ApplicationContext applicationContext; + + private MessageSource messageSource; + + private boolean hasBeenBound = false; + + private boolean ignoreNestedProperties = false; + + private String targetName; + + private ConversionService conversionService; + + private boolean resolvePlaceholders = true; + + /** + * Create a new {@link PropertiesConfigurationFactory} instance. + * + * @param target the target object to bind too + * @see #PropertiesConfigurationFactory(Class) + */ + public PropertiesConfigurationFactory(T target){ + Assert.notNull(target, "target must not be null"); + this.target = target; + } + + /** + * Create a new {@link PropertiesConfigurationFactory} instance. + * + * @param type the target type + * @see #PropertiesConfigurationFactory(Class) + */ + @SuppressWarnings("unchecked") + public PropertiesConfigurationFactory(Class type){ + Assert.notNull(type, "type must not be null"); + this.target = (T) BeanUtils.instantiate(type); + } + + /** + * Flag to disable binding of nested properties (i.e. those with period + * separators in their paths). Can be useful to disable this if the name prefix + * is empty and you don't want to ignore unknown fields. + * + * @param ignoreNestedProperties the flag to set (default false) + */ + public void setIgnoreNestedProperties(boolean ignoreNestedProperties) { + this.ignoreNestedProperties = ignoreNestedProperties; + } + + /** + * Set whether to ignore unknown fields, that is, whether to ignore bind + * parameters that do not have corresponding fields in the target object. + *

    + * Default is "true". Turn this off to enforce that all bind parameters must + * have a matching field in the target object. + * + * @param ignoreUnknownFields if unknown fields should be ignored + */ + public void setIgnoreUnknownFields(boolean ignoreUnknownFields) { + this.ignoreUnknownFields = ignoreUnknownFields; + } + + /** + * Set whether to ignore invalid fields, that is, whether to ignore bind + * parameters that have corresponding fields in the target object which are not + * accessible (for example because of null values in the nested path). + *

    + * Default is "false". Turn this on to ignore bind parameters for nested objects + * in non-existing parts of the target object graph. + * + * @param ignoreInvalidFields if invalid fields should be ignored + */ + public void setIgnoreInvalidFields(boolean ignoreInvalidFields) { + this.ignoreInvalidFields = ignoreInvalidFields; + } + + /** + * Set the target name. + * + * @param targetName the target name + */ + public void setTargetName(String targetName) { + this.targetName = targetName; + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) { + this.applicationContext = applicationContext; + } + + /** + * Set the message source. + * + * @param messageSource the message source + */ + @Override + public void setMessageSource(MessageSource messageSource) { + this.messageSource = messageSource; + } + + /** + * Set the property sources. + * + * @param propertySources the property sources + */ + public void setPropertySources(PropertySources propertySources) { + this.propertySources = propertySources; + } + + /** + * Set the conversion service. + * + * @param conversionService the conversion service + */ + public void setConversionService(ConversionService conversionService) { + this.conversionService = conversionService; + } + + /** + * Set the validator. + * + * @param validator the validator + */ + public void setValidator(Validator validator) { + this.validator = validator; + } + + /** + * Set a flag to indicate that an exception should be raised if a Validator is + * available and validation fails. + * + * @param exceptionIfInvalid the flag to set + * @deprecated as of 1.5, do not specify a {@link Validator} if validation + * should not occur + */ + @Deprecated + public void setExceptionIfInvalid(boolean exceptionIfInvalid) { + this.exceptionIfInvalid = exceptionIfInvalid; + } + + /** + * Flag to indicate that placeholders should be replaced during binding. Default + * is true. + * + * @param resolvePlaceholders flag value + */ + public void setResolvePlaceholders(boolean resolvePlaceholders) { + this.resolvePlaceholders = resolvePlaceholders; + } + + @Override + public void afterPropertiesSet() throws Exception { + bindPropertiesToTarget(); + } + + @Override + public Class getObjectType() { + if (this.target == null) { + return Object.class; + } + return this.target.getClass(); + } + + @Override + public boolean isSingleton() { + return true; + } + + @Override + public T getObject() throws Exception { + if (!this.hasBeenBound) { + bindPropertiesToTarget(); + } + return this.target; + } + + public void bindPropertiesToTarget() throws BindException { + Assert.state(this.propertySources != null, "PropertySources should not be null"); + try { + if (logger.isTraceEnabled()) { + logger.trace("Property Sources: " + this.propertySources); + + } + this.hasBeenBound = true; + doBindPropertiesToTarget(); + } catch (BindException ex) { + if (this.exceptionIfInvalid) { + throw ex; + } + logger.error("Failed to load Properties validation bean. " + "Your Properties may be invalid.", ex); + } + } + + private void doBindPropertiesToTarget() throws BindException { + RelaxedDataBinder dataBinder = (this.targetName != null ? new RelaxedDataBinder(this.target, + this.targetName) : new RelaxedDataBinder(this.target)); + if (this.validator != null && this.validator.supports(dataBinder.getTarget().getClass())) { + dataBinder.setValidator(this.validator); + } + if (this.conversionService != null) { + dataBinder.setConversionService(this.conversionService); + } + dataBinder.setAutoGrowCollectionLimit(Integer.MAX_VALUE); + dataBinder.setIgnoreNestedProperties(this.ignoreNestedProperties); + dataBinder.setIgnoreInvalidFields(this.ignoreInvalidFields); + dataBinder.setIgnoreUnknownFields(this.ignoreUnknownFields); + customizeBinder(dataBinder); + if (this.applicationContext != null) { + ResourceEditorRegistrar resourceEditorRegistrar = new ResourceEditorRegistrar(this.applicationContext, + this.applicationContext.getEnvironment()); + resourceEditorRegistrar.registerCustomEditors(dataBinder); + } + Iterable relaxedTargetNames = getRelaxedTargetNames(); + Set names = getNames(relaxedTargetNames); + PropertyValues propertyValues = getPropertySourcesPropertyValues(names, relaxedTargetNames); + dataBinder.bind(propertyValues); + if (this.validator != null) { + dataBinder.validate(); + } + checkForBindingErrors(dataBinder); + } + + private Iterable getRelaxedTargetNames() { + return (this.target != null + && StringUtils.hasLength(this.targetName) ? new RelaxedNames(this.targetName) : null); + } + + private Set getNames(Iterable prefixes) { + Set names = new LinkedHashSet(); + if (this.target != null) { + PropertyDescriptor[] descriptors = BeanUtils.getPropertyDescriptors(this.target.getClass()); + for (PropertyDescriptor descriptor : descriptors) { + String name = descriptor.getName(); + if (!name.equals("class")) { + RelaxedNames relaxedNames = RelaxedNames.forCamelCase(name); + if (prefixes == null) { + for (String relaxedName : relaxedNames) { + names.add(relaxedName); + } + } else { + for (String prefix : prefixes) { + for (String relaxedName : relaxedNames) { + names.add(prefix + "." + relaxedName); + names.add(prefix + "_" + relaxedName); + } + } + } + } + } + } + return names; + } + + private PropertyValues getPropertySourcesPropertyValues(Set names, Iterable relaxedTargetNames) { + PropertyNamePatternsMatcher includes = getPropertyNamePatternsMatcher(names, relaxedTargetNames); + return new PropertySourcesPropertyValues(this.propertySources, names, includes, this.resolvePlaceholders); + } + + private PropertyNamePatternsMatcher getPropertyNamePatternsMatcher(Set names, + Iterable relaxedTargetNames) { + if (this.ignoreUnknownFields && !isMapTarget()) { + // Since unknown fields are ignored we can filter them out early to save + // unnecessary calls to the PropertySource. + return new DefaultPropertyNamePatternsMatcher(EXACT_DELIMITERS, true, names); + } + if (relaxedTargetNames != null) { + // We can filter properties to those starting with the target name, but + // we can't do a complete filter since we need to trigger the + // unknown fields check + Set relaxedNames = new HashSet(); + for (String relaxedTargetName : relaxedTargetNames) { + relaxedNames.add(relaxedTargetName); + } + return new DefaultPropertyNamePatternsMatcher(TARGET_NAME_DELIMITERS, true, relaxedNames); + } + // Not ideal, we basically can't filter anything + return PropertyNamePatternsMatcher.ALL; + } + + private boolean isMapTarget() { + return this.target != null && Map.class.isAssignableFrom(this.target.getClass()); + } + + private void checkForBindingErrors(RelaxedDataBinder dataBinder) throws BindException { + BindingResult errors = dataBinder.getBindingResult(); + if (errors.hasErrors()) { + logger.error("Properties configuration failed validation"); + for (ObjectError error : errors.getAllErrors()) { + logger.error(this.messageSource != null ? this.messageSource.getMessage(error, Locale.getDefault()) + + " (" + error + ")" : error); + } + if (this.exceptionIfInvalid) { + throw new BindException(errors); + } + } + } + + /** + * Customize the data binder. + * + * @param dataBinder the data binder that will be used to bind and validate + */ + protected void customizeBinder(DataBinder dataBinder) { + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java new file mode 100644 index 00000000..0d7d4fe4 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java @@ -0,0 +1,38 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +/** + * Strategy interface used to check if a property name matches specific + * criteria. + * + * @author Phillip Webb + * @since 1.2.0 + */ +interface PropertyNamePatternsMatcher { + + PropertyNamePatternsMatcher ALL = new PropertyNamePatternsMatcher() { + + @Override + public boolean matches(String propertyName) { + return true; + } + + }; + + PropertyNamePatternsMatcher NONE = new PropertyNamePatternsMatcher() { + + @Override + public boolean matches(String propertyName) { + return false; + } + + }; + + /** + * Return {@code true} of the property name matches. + * + * @param propertyName the property name + * @return {@code true} if the property name matches + */ + boolean matches(String propertyName); + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java new file mode 100644 index 00000000..7ad35f1a --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java @@ -0,0 +1,30 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; + +/** + * The origin of a property, specifically its source and its name before any + * prefix was removed. + * + * @author Andy Wilkinson + * @since 1.3.0 + */ +public class PropertyOrigin { + + private final PropertySource source; + + private final String name; + + PropertyOrigin(PropertySource source, String name){ + this.name = name; + this.source = source; + } + + public PropertySource getSource() { + return this.source; + } + + public String getName() { + return this.name; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java new file mode 100644 index 00000000..520e857f --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java @@ -0,0 +1,164 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import org.springframework.core.convert.ConversionException; +import org.springframework.core.env.AbstractEnvironment; +import org.springframework.core.env.AbstractPropertyResolver; +import org.springframework.core.env.PropertyResolver; +import org.springframework.util.ClassUtils; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.PropertySources; + +/** + * {@link PropertyResolver} implementation that resolves property values against + * an underlying set of {@link PropertySources}. + * + * @author Chris Beams + * @author Juergen Hoeller + * @see PropertySource + * @see PropertySources + * @see AbstractEnvironment + * @since 3.1 + */ +public class PropertySourcesPropertyResolver extends AbstractPropertyResolver { + + private final PropertySources propertySources; + + /** + * Create a new resolver against the given property sources. + * + * @param propertySources the set of {@link PropertySource} objects to use + */ + public PropertySourcesPropertyResolver(PropertySources propertySources){ + this.propertySources = propertySources; + } + + @Override + public boolean containsProperty(String key) { + if (this.propertySources != null) { + for (PropertySource propertySource : this.propertySources) { + if (propertySource.containsProperty(key)) { + return true; + } + } + } + return false; + } + + @Override + public String getProperty(String key) { + return getProperty(key, String.class, true); + } + + @Override + public T getProperty(String key, Class targetValueType) { + return getProperty(key, targetValueType, true); + } + + @Override + protected String getPropertyAsRawString(String key) { + return getProperty(key, String.class, false); + } + + protected T getProperty(String key, Class targetValueType, boolean resolveNestedPlaceholders) { + if (this.propertySources != null) { + for (PropertySource propertySource : this.propertySources) { + if (logger.isTraceEnabled()) { + logger + .trace("Searching for key '" + key + "' in PropertySource '" + propertySource.getName() + "'"); + } + Object value = propertySource.getProperty(key); + if (value != null) { + if (resolveNestedPlaceholders && value instanceof String) { + value = resolveNestedPlaceholders((String) value); + } + logKeyFound(key, propertySource, value); + return convertValueIfNecessary(value, targetValueType); + } + } + } + if (logger.isDebugEnabled()) { + logger.debug("Could not find key '" + key + "' in any property source"); + } + return null; + } + + @Deprecated + public Class getPropertyAsClass(String key, Class targetValueType) { + if (this.propertySources != null) { + for (PropertySource propertySource : this.propertySources) { + if (logger.isTraceEnabled()) { + logger.trace(String.format("Searching for key '%s' in [%s]", key, propertySource.getName())); + } + Object value = propertySource.getProperty(key); + if (value != null) { + logKeyFound(key, propertySource, value); + Class clazz; + if (value instanceof String) { + try { + clazz = ClassUtils.forName((String) value, null); + } catch (Exception ex) { + throw new PropertySourcesPropertyResolver.ClassConversionException((String) value, + targetValueType, + ex); + } + } else if (value instanceof Class) { + clazz = (Class) value; + } else { + clazz = value.getClass(); + } + if (!targetValueType.isAssignableFrom(clazz)) { + throw new PropertySourcesPropertyResolver.ClassConversionException(clazz, targetValueType); + } + @SuppressWarnings("unchecked") + Class targetClass = (Class) clazz; + return targetClass; + } + } + } + if (logger.isDebugEnabled()) { + logger.debug(String.format("Could not find key '%s' in any property source", key)); + } + return null; + } + + /** + * Log the given key as found in the given {@link PropertySource}, resulting in + * the given value. + *

    + * The default implementation writes a debug log message with key and source. As + * of 4.3.3, this does not log the value anymore in order to avoid accidental + * logging of sensitive settings. Subclasses may override this method to change + * the log level and/or log message, including the property's value if desired. + * + * @param key the key found + * @param propertySource the {@code PropertySource} that the key has been found + * in + * @param value the corresponding value + * @since 4.3.1 + */ + protected void logKeyFound(String key, PropertySource propertySource, Object value) { + if (logger.isDebugEnabled()) { + logger.debug("Found key '" + key + "' in PropertySource '" + propertySource.getName() + + "' with value of type " + value.getClass().getSimpleName()); + } + } + + @SuppressWarnings("serial") + @Deprecated + private static class ClassConversionException extends ConversionException { + + public ClassConversionException(Class actual, Class expected){ + super(String + .format("Actual type %s is not assignable to expected type %s", actual.getName(), expected.getName())); + } + + public ClassConversionException(String actual, Class expected, Exception ex){ + super( + String + .format("Could not find/load class %s during attempt to convert to %s", actual, expected.getName()), + ex); + } + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java new file mode 100644 index 00000000..d41cb27b --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java @@ -0,0 +1,233 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; + +import org.springframework.beans.MutablePropertyValues; +import org.springframework.beans.PropertyValue; +import org.springframework.beans.PropertyValues; +import org.springframework.util.Assert; +import org.springframework.validation.DataBinder; + +import com.alibaba.otter.canal.client.adapter.config.common.CompositePropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.EnumerablePropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.PropertySources; + +/** + * A {@link PropertyValues} implementation backed by a {@link PropertySources}, + * bridging the two abstractions and allowing (for instance) a regular + * {@link DataBinder} to be used with the latter. + * + * @author Dave Syer + * @author Phillip Webb + */ +public class PropertySourcesPropertyValues implements PropertyValues { + + private static final Pattern COLLECTION_PROPERTY = Pattern + .compile("\\[(\\d+)\\](\\.\\S+)?"); + + private final PropertySources propertySources; + + private final Collection nonEnumerableFallbackNames; + + private final PropertyNamePatternsMatcher includes; + + private final Map propertyValues = new LinkedHashMap(); + + private final ConcurrentHashMap> collectionOwners = new ConcurrentHashMap>(); + + private final boolean resolvePlaceholders; + + /** + * Create a new PropertyValues from the given PropertySources. + * + * @param propertySources a PropertySources instance + */ + public PropertySourcesPropertyValues(PropertySources propertySources){ + this(propertySources, true); + } + + /** + * Create a new PropertyValues from the given PropertySources that will + * optionally resolve placeholders. + * + * @param propertySources a PropertySources instance + * @param resolvePlaceholders {@code true} if placeholders should be resolved. + * @since 1.5.2 + */ + public PropertySourcesPropertyValues(PropertySources propertySources, boolean resolvePlaceholders){ + this(propertySources, (Collection) null, PropertyNamePatternsMatcher.ALL, resolvePlaceholders); + } + + /** + * Create a new PropertyValues from the given PropertySources. + * + * @param propertySources a PropertySources instance + * @param includePatterns property name patterns to include from system + * properties and environment variables + * @param nonEnumerableFallbackNames the property names to try in lieu of an + * {@link EnumerablePropertySource}. + */ + public PropertySourcesPropertyValues(PropertySources propertySources, Collection includePatterns, + Collection nonEnumerableFallbackNames){ + this(propertySources, + nonEnumerableFallbackNames, + new PatternPropertyNamePatternsMatcher(includePatterns), + true); + } + + /** + * Create a new PropertyValues from the given PropertySources. + * + * @param propertySources a PropertySources instance + * @param nonEnumerableFallbackNames the property names to try in lieu of an + * {@link EnumerablePropertySource}. + * @param includes the property name patterns to include + * @param resolvePlaceholders flag to indicate the placeholders should be + * resolved + */ + PropertySourcesPropertyValues(PropertySources propertySources, Collection nonEnumerableFallbackNames, + PropertyNamePatternsMatcher includes, boolean resolvePlaceholders){ + Assert.notNull(propertySources, "PropertySources must not be null"); + Assert.notNull(includes, "Includes must not be null"); + this.propertySources = propertySources; + this.nonEnumerableFallbackNames = nonEnumerableFallbackNames; + this.includes = includes; + this.resolvePlaceholders = resolvePlaceholders; + PropertySourcesPropertyResolver resolver = new PropertySourcesPropertyResolver(propertySources); + for (PropertySource source : propertySources) { + processPropertySource(source, resolver); + } + } + + private void processPropertySource(PropertySource source, PropertySourcesPropertyResolver resolver) { + if (source instanceof CompositePropertySource) { + processCompositePropertySource((CompositePropertySource) source, resolver); + } else if (source instanceof EnumerablePropertySource) { + processEnumerablePropertySource((EnumerablePropertySource) source, resolver, this.includes); + } else { + processNonEnumerablePropertySource(source, resolver); + } + } + + private void processCompositePropertySource(CompositePropertySource source, + PropertySourcesPropertyResolver resolver) { + for (PropertySource nested : source.getPropertySources()) { + processPropertySource(nested, resolver); + } + } + + private void processEnumerablePropertySource(EnumerablePropertySource source, + PropertySourcesPropertyResolver resolver, + PropertyNamePatternsMatcher includes) { + if (source.getPropertyNames().length > 0) { + for (String propertyName : source.getPropertyNames()) { + if (includes.matches(propertyName)) { + Object value = getEnumerableProperty(source, resolver, propertyName); + putIfAbsent(propertyName, value, source); + } + } + } + } + + private Object getEnumerableProperty(EnumerablePropertySource source, PropertySourcesPropertyResolver resolver, + String propertyName) { + try { + if (this.resolvePlaceholders) { + return resolver.getProperty(propertyName, Object.class); + } + } catch (RuntimeException ex) { + // Probably could not resolve placeholders, ignore it here + } + return source.getProperty(propertyName); + } + + private void processNonEnumerablePropertySource(PropertySource source, + PropertySourcesPropertyResolver resolver) { + // We can only do exact matches for non-enumerable property names, but + // that's better than nothing... + if (this.nonEnumerableFallbackNames == null) { + return; + } + for (String propertyName : this.nonEnumerableFallbackNames) { + if (!source.containsProperty(propertyName)) { + continue; + } + Object value = null; + try { + value = resolver.getProperty(propertyName, Object.class); + } catch (RuntimeException ex) { + // Probably could not convert to Object, weird, but ignorable + } + if (value == null) { + value = source.getProperty(propertyName.toUpperCase(Locale.ENGLISH)); + } + putIfAbsent(propertyName, value, source); + } + } + + @Override + public PropertyValue[] getPropertyValues() { + Collection values = this.propertyValues.values(); + return values.toArray(new PropertyValue[values.size()]); + } + + @Override + public PropertyValue getPropertyValue(String propertyName) { + PropertyValue propertyValue = this.propertyValues.get(propertyName); + if (propertyValue != null) { + return propertyValue; + } + for (PropertySource source : this.propertySources) { + Object value = source.getProperty(propertyName); + propertyValue = putIfAbsent(propertyName, value, source); + if (propertyValue != null) { + return propertyValue; + } + } + return null; + } + + private PropertyValue putIfAbsent(String propertyName, Object value, PropertySource source) { + if (value != null && !this.propertyValues.containsKey(propertyName)) { + PropertySource collectionOwner = this.collectionOwners + .putIfAbsent(COLLECTION_PROPERTY.matcher(propertyName).replaceAll("[]"), source); + if (collectionOwner == null || collectionOwner == source) { + PropertyValue propertyValue = new OriginCapablePropertyValue(propertyName, value, propertyName, source); + this.propertyValues.put(propertyName, propertyValue); + return propertyValue; + } + } + return null; + } + + @Override + public PropertyValues changesSince(PropertyValues old) { + MutablePropertyValues changes = new MutablePropertyValues(); + // for each property value in the new set + for (PropertyValue newValue : getPropertyValues()) { + // if there wasn't an old one, add it + PropertyValue oldValue = old.getPropertyValue(newValue.getName()); + if (oldValue == null || !oldValue.equals(newValue)) { + changes.addPropertyValue(newValue); + } + } + return changes; + } + + @Override + public boolean contains(String propertyName) { + return getPropertyValue(propertyName) != null; + } + + @Override + public boolean isEmpty() { + return this.propertyValues.isEmpty(); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java new file mode 100644 index 00000000..00a288b6 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java @@ -0,0 +1,127 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.EnumSet; +import java.util.Locale; +import java.util.Set; + +import org.springframework.core.convert.ConversionFailedException; +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.core.convert.converter.Converter; +import org.springframework.core.convert.converter.ConverterFactory; +import org.springframework.core.convert.support.DefaultConversionService; +import org.springframework.core.convert.support.GenericConversionService; +import org.springframework.util.Assert; + +/** + * Internal {@link ConversionService} used by {@link RelaxedDataBinder} to + * support additional relaxed conversion. + * + * @author Phillip Webb + * @author Stephane Nicoll + * @since 1.1.0 + */ +class RelaxedConversionService implements ConversionService { + + private final ConversionService conversionService; + + private final GenericConversionService additionalConverters; + + /** + * Create a new {@link RelaxedConversionService} instance. + * + * @param conversionService and option root conversion service + */ + RelaxedConversionService(ConversionService conversionService){ + this.conversionService = conversionService; + this.additionalConverters = new GenericConversionService(); + DefaultConversionService.addCollectionConverters(this.additionalConverters); + this.additionalConverters + .addConverterFactory(new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory()); + this.additionalConverters.addConverter(new StringToCharArrayConverter()); + } + + @Override + public boolean canConvert(Class sourceType, Class targetType) { + return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType)) + || this.additionalConverters.canConvert(sourceType, targetType); + } + + @Override + public boolean canConvert(TypeDescriptor sourceType, TypeDescriptor targetType) { + return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType)) + || this.additionalConverters.canConvert(sourceType, targetType); + } + + @Override + @SuppressWarnings("unchecked") + public T convert(Object source, Class targetType) { + Assert.notNull(targetType, "The targetType to convert to cannot be null"); + return (T) convert(source, TypeDescriptor.forObject(source), TypeDescriptor.valueOf(targetType)); + } + + @Override + public Object convert(Object source, TypeDescriptor sourceType, TypeDescriptor targetType) { + if (this.conversionService != null) { + try { + return this.conversionService.convert(source, sourceType, targetType); + } catch (ConversionFailedException ex) { + // Ignore and try the additional converters + } + } + return this.additionalConverters.convert(source, sourceType, targetType); + } + + /** + * Clone of Spring's package private StringToEnumConverterFactory, but ignoring + * the case of the source. + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + private static class StringToEnumIgnoringCaseConverterFactory implements ConverterFactory { + + @Override + public Converter getConverter(Class targetType) { + Class enumType = targetType; + while (enumType != null && !enumType.isEnum()) { + enumType = enumType.getSuperclass(); + } + Assert.notNull(enumType, "The target type " + targetType.getName() + " does not refer to an enum"); + return new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory.StringToEnum(enumType); + } + + private class StringToEnum implements Converter { + + private final Class enumType; + + StringToEnum(Class enumType){ + this.enumType = enumType; + } + + @Override + public T convert(String source) { + if (source.isEmpty()) { + // It's an empty enum identifier: reset the enum value to null. + return null; + } + source = source.trim(); + for (T candidate : (Set) EnumSet.allOf(this.enumType)) { + RelaxedNames names = new RelaxedNames( + candidate.name().replace('_', '-').toLowerCase(Locale.ENGLISH)); + for (String name : names) { + if (name.equals(source)) { + return candidate; + } + } + if (candidate.name().equalsIgnoreCase(source)) { + return candidate; + } + } + throw new IllegalArgumentException( + "No enum constant " + this.enumType.getCanonicalName() + "." + source); + } + + } + + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java new file mode 100644 index 00000000..dbcc7130 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java @@ -0,0 +1,729 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.beans.PropertyEditor; +import java.net.InetAddress; +import java.util.*; + +import org.springframework.beans.*; +import org.springframework.beans.propertyeditors.FileEditor; +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.util.StringUtils; +import org.springframework.validation.AbstractPropertyBindingResult; +import org.springframework.validation.BeanPropertyBindingResult; +import org.springframework.validation.DataBinder; + +/** + * Binder implementation that allows caller to bind to maps and also allows + * property names to match a bit loosely (if underscores or dashes are removed + * and replaced with camel case for example). + * + * @author Dave Syer + * @author Phillip Webb + * @author Stephane Nicoll + * @author Andy Wilkinson + * @see RelaxedNames + */ +public class RelaxedDataBinder extends DataBinder { + + private static final Set> EXCLUDED_EDITORS; + + static { + Set> excluded = new HashSet>(); + excluded.add(FileEditor.class); + EXCLUDED_EDITORS = Collections.unmodifiableSet(excluded); + } + + private static final Object BLANK = new Object(); + + private String namePrefix; + + private boolean ignoreNestedProperties; + + private MultiValueMap nameAliases = new LinkedMultiValueMap(); + + /** + * Create a new {@link RelaxedDataBinder} instance. + * + * @param target the target into which properties are bound + */ + public RelaxedDataBinder(Object target){ + super(wrapTarget(target)); + } + + /** + * Create a new {@link RelaxedDataBinder} instance. + * + * @param target the target into which properties are bound + * @param namePrefix An optional prefix to be used when reading properties + */ + public RelaxedDataBinder(Object target, String namePrefix){ + super(wrapTarget(target), (StringUtils.hasLength(namePrefix) ? namePrefix : DEFAULT_OBJECT_NAME)); + this.namePrefix = cleanNamePrefix(namePrefix); + } + + private String cleanNamePrefix(String namePrefix) { + if (!StringUtils.hasLength(namePrefix)) { + return null; + } + return (namePrefix.endsWith(".") ? namePrefix : namePrefix + "."); + } + + /** + * Flag to disable binding of nested properties (i.e. those with period + * separators in their paths). Can be useful to disable this if the name prefix + * is empty and you don't want to ignore unknown fields. + * + * @param ignoreNestedProperties the flag to set (default false) + */ + public void setIgnoreNestedProperties(boolean ignoreNestedProperties) { + this.ignoreNestedProperties = ignoreNestedProperties; + } + + /** + * Set name aliases. + * + * @param aliases a map of property name to aliases + */ + public void setNameAliases(Map> aliases) { + this.nameAliases = new LinkedMultiValueMap(aliases); + } + + /** + * Add aliases to the {@link DataBinder}. + * + * @param name the property name to alias + * @param alias aliases for the property names + * @return this instance + */ + public RelaxedDataBinder withAlias(String name, String... alias) { + for (String value : alias) { + this.nameAliases.add(name, value); + } + return this; + } + + @Override + protected void doBind(MutablePropertyValues propertyValues) { + super.doBind(modifyProperties(propertyValues, getTarget())); + } + + /** + * Modify the property values so that period separated property paths are valid + * for map keys. Also creates new maps for properties of map type that are null + * (assuming all maps are potentially nested). The standard bracket {@code[...]} + * dereferencing is also accepted. + * + * @param propertyValues the property values + * @param target the target object + * @return modified property values + */ + private MutablePropertyValues modifyProperties(MutablePropertyValues propertyValues, Object target) { + propertyValues = getPropertyValuesForNamePrefix(propertyValues); + if (target instanceof RelaxedDataBinder.MapHolder) { + propertyValues = addMapPrefix(propertyValues); + } + BeanWrapper wrapper = new BeanWrapperImpl(target); + wrapper.setConversionService(new RelaxedConversionService(getConversionService())); + wrapper.setAutoGrowNestedPaths(true); + List sortedValues = new ArrayList(); + Set modifiedNames = new HashSet(); + List sortedNames = getSortedPropertyNames(propertyValues); + for (String name : sortedNames) { + PropertyValue propertyValue = propertyValues.getPropertyValue(name); + PropertyValue modifiedProperty = modifyProperty(wrapper, propertyValue); + if (modifiedNames.add(modifiedProperty.getName())) { + sortedValues.add(modifiedProperty); + } + } + return new MutablePropertyValues(sortedValues); + } + + private List getSortedPropertyNames(MutablePropertyValues propertyValues) { + List names = new LinkedList(); + for (PropertyValue propertyValue : propertyValues.getPropertyValueList()) { + names.add(propertyValue.getName()); + } + sortPropertyNames(names); + return names; + } + + /** + * Sort by name so that parent properties get processed first (e.g. 'foo.bar' + * before 'foo.bar.spam'). Don't use Collections.sort() because the order might + * be significant for other property names (it shouldn't be but who knows what + * people might be relying on, e.g. HSQL has a JDBCXADataSource where + * "databaseName" is a synonym for "url"). + * + * @param names the names to sort + */ + private void sortPropertyNames(List names) { + for (String name : new ArrayList(names)) { + int propertyIndex = names.indexOf(name); + RelaxedDataBinder.BeanPath path = new RelaxedDataBinder.BeanPath(name); + for (String prefix : path.prefixes()) { + int prefixIndex = names.indexOf(prefix); + if (prefixIndex >= propertyIndex) { + // The child property has a parent in the list in the wrong order + names.remove(name); + names.add(prefixIndex, name); + } + } + } + } + + private MutablePropertyValues addMapPrefix(MutablePropertyValues propertyValues) { + MutablePropertyValues rtn = new MutablePropertyValues(); + for (PropertyValue pv : propertyValues.getPropertyValues()) { + rtn.add("map." + pv.getName(), pv.getValue()); + } + return rtn; + } + + private MutablePropertyValues getPropertyValuesForNamePrefix(MutablePropertyValues propertyValues) { + if (!StringUtils.hasText(this.namePrefix) && !this.ignoreNestedProperties) { + return propertyValues; + } + MutablePropertyValues rtn = new MutablePropertyValues(); + for (PropertyValue value : propertyValues.getPropertyValues()) { + String name = value.getName(); + for (String prefix : new RelaxedNames(stripLastDot(this.namePrefix))) { + for (String separator : new String[] { ".", "_" }) { + String candidate = (StringUtils.hasLength(prefix) ? prefix + separator : prefix); + if (name.startsWith(candidate)) { + name = name.substring(candidate.length()); + if (!(this.ignoreNestedProperties && name.contains("."))) { + PropertyOrigin propertyOrigin = OriginCapablePropertyValue.getOrigin(value); + rtn.addPropertyValue( + new OriginCapablePropertyValue(name, value.getValue(), propertyOrigin)); + } + } + } + } + } + return rtn; + } + + private String stripLastDot(String string) { + if (StringUtils.hasLength(string) && string.endsWith(".")) { + string = string.substring(0, string.length() - 1); + } + return string; + } + + private PropertyValue modifyProperty(BeanWrapper target, PropertyValue propertyValue) { + String name = propertyValue.getName(); + String normalizedName = normalizePath(target, name); + if (!normalizedName.equals(name)) { + return new PropertyValue(normalizedName, propertyValue.getValue()); + } + return propertyValue; + } + + /** + * Normalize a bean property path to a format understood by a BeanWrapper. This + * is used so that + *

      + *
    • Fuzzy matching can be employed for bean property names
      • + *
    • Period separators can be used instead of indexing ([...]) for map + * keys
      • + *
    + * + * @param wrapper a bean wrapper for the object to bind + * @param path the bean path to bind + * @return a transformed path with correct bean wrapper syntax + */ + protected String normalizePath(BeanWrapper wrapper, String path) { + return initializePath(wrapper, new RelaxedDataBinder.BeanPath(path), 0); + } + + @Override + protected AbstractPropertyBindingResult createBeanPropertyBindingResult() { + return new RelaxedDataBinder.RelaxedBeanPropertyBindingResult(getTarget(), + getObjectName(), + isAutoGrowNestedPaths(), + getAutoGrowCollectionLimit(), + getConversionService()); + } + + private String initializePath(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) { + String prefix = path.prefix(index); + String key = path.name(index); + if (path.isProperty(index)) { + key = getActualPropertyName(wrapper, prefix, key); + path.rename(index, key); + } + if (path.name(++index) == null) { + return path.toString(); + } + String name = path.prefix(index); + TypeDescriptor descriptor = wrapper.getPropertyTypeDescriptor(name); + if (descriptor == null || descriptor.isMap()) { + if (isMapValueStringType(descriptor) || isBlanked(wrapper, name, path.name(index))) { + path.collapseKeys(index); + } + path.mapIndex(index); + extendMapIfNecessary(wrapper, path, index); + } else if (descriptor.isCollection()) { + extendCollectionIfNecessary(wrapper, path, index); + } else if (descriptor.getType().equals(Object.class)) { + if (isBlanked(wrapper, name, path.name(index))) { + path.collapseKeys(index); + } + path.mapIndex(index); + if (path.isLastNode(index)) { + wrapper.setPropertyValue(path.toString(), BLANK); + } else { + String next = path.prefix(index + 1); + if (wrapper.getPropertyValue(next) == null) { + wrapper.setPropertyValue(next, new LinkedHashMap()); + } + } + } + return initializePath(wrapper, path, index); + } + + private boolean isMapValueStringType(TypeDescriptor descriptor) { + if (descriptor == null || descriptor.getMapValueTypeDescriptor() == null) { + return false; + } + if (Properties.class.isAssignableFrom(descriptor.getObjectType())) { + // Properties is declared as Map but we know it's really + // Map + return true; + } + Class valueType = descriptor.getMapValueTypeDescriptor().getObjectType(); + return (valueType != null && CharSequence.class.isAssignableFrom(valueType)); + } + + @SuppressWarnings("rawtypes") + private boolean isBlanked(BeanWrapper wrapper, String propertyName, String key) { + Object value = (wrapper.isReadableProperty(propertyName) ? wrapper.getPropertyValue(propertyName) : null); + if (value instanceof Map) { + if (((Map) value).get(key) == BLANK) { + return true; + } + } + return false; + } + + private void extendCollectionIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) { + String name = path.prefix(index); + TypeDescriptor elementDescriptor = wrapper.getPropertyTypeDescriptor(name).getElementTypeDescriptor(); + if (!elementDescriptor.isMap() && !elementDescriptor.isCollection() + && !elementDescriptor.getType().equals(Object.class)) { + return; + } + Object extend = new LinkedHashMap(); + if (!elementDescriptor.isMap() && path.isArrayIndex(index)) { + extend = new ArrayList(); + } + wrapper.setPropertyValue(path.prefix(index + 1), extend); + } + + private void extendMapIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) { + String name = path.prefix(index); + TypeDescriptor parent = wrapper.getPropertyTypeDescriptor(name); + if (parent == null) { + return; + } + TypeDescriptor descriptor = parent.getMapValueTypeDescriptor(); + if (descriptor == null) { + descriptor = TypeDescriptor.valueOf(Object.class); + } + if (!descriptor.isMap() && !descriptor.isCollection() && !descriptor.getType().equals(Object.class)) { + return; + } + String extensionName = path.prefix(index + 1); + if (wrapper.isReadableProperty(extensionName)) { + Object currentValue = wrapper.getPropertyValue(extensionName); + if ((descriptor.isCollection() && currentValue instanceof Collection) + || (!descriptor.isCollection() && currentValue instanceof Map)) { + return; + } + } + Object extend = new LinkedHashMap(); + if (descriptor.isCollection()) { + extend = new ArrayList(); + } + if (descriptor.getType().equals(Object.class) && path.isLastNode(index)) { + extend = BLANK; + } + wrapper.setPropertyValue(extensionName, extend); + } + + private String getActualPropertyName(BeanWrapper target, String prefix, String name) { + String propertyName = resolvePropertyName(target, prefix, name); + if (propertyName == null) { + propertyName = resolveNestedPropertyName(target, prefix, name); + } + return (propertyName != null ? propertyName : name); + } + + private String resolveNestedPropertyName(BeanWrapper target, String prefix, String name) { + StringBuilder candidate = new StringBuilder(); + for (String field : name.split("[_\\-\\.]")) { + candidate.append(candidate.length() > 0 ? "." : ""); + candidate.append(field); + String nested = resolvePropertyName(target, prefix, candidate.toString()); + if (nested != null) { + Class type = target.getPropertyType(nested); + if ((type != null) && Map.class.isAssignableFrom(type)) { + // Special case for map property (gh-3836). + return nested + "[" + name.substring(candidate.length() + 1) + "]"; + } + String propertyName = resolvePropertyName(target, + joinString(prefix, nested), + name.substring(candidate.length() + 1)); + if (propertyName != null) { + return joinString(nested, propertyName); + } + } + } + return null; + } + + private String resolvePropertyName(BeanWrapper target, String prefix, String name) { + Iterable names = getNameAndAliases(name); + for (String nameOrAlias : names) { + for (String candidate : new RelaxedNames(nameOrAlias)) { + try { + if (target.getPropertyType(joinString(prefix, candidate)) != null) { + return candidate; + } + } catch (InvalidPropertyException ex) { + // swallow and continue + } + } + } + return null; + } + + private String joinString(String prefix, String name) { + return (StringUtils.hasLength(prefix) ? prefix + "." + name : name); + } + + private Iterable getNameAndAliases(String name) { + List aliases = this.nameAliases.get(name); + if (aliases == null) { + return Collections.singleton(name); + } + List nameAndAliases = new ArrayList(aliases.size() + 1); + nameAndAliases.add(name); + nameAndAliases.addAll(aliases); + return nameAndAliases; + } + + private static Object wrapTarget(Object target) { + if (target instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) target; + target = new RelaxedDataBinder.MapHolder(map); + } + return target; + } + + @Override + public void registerCustomEditor(Class requiredType, PropertyEditor propertyEditor) { + if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) { + super.registerCustomEditor(requiredType, propertyEditor); + } + } + + @Override + public void registerCustomEditor(Class requiredType, String field, PropertyEditor propertyEditor) { + if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) { + super.registerCustomEditor(requiredType, field, propertyEditor); + } + } + + /** + * Holder to allow Map targets to be bound. + */ + static class MapHolder { + + private Map map; + + MapHolder(Map map){ + this.map = map; + } + + public void setMap(Map map) { + this.map = map; + } + + public Map getMap() { + return this.map; + } + + } + + /** + * A path though properties of a bean. + */ + private static class BeanPath { + + private List nodes; + + BeanPath(String path){ + this.nodes = splitPath(path); + } + + public List prefixes() { + List prefixes = new ArrayList(); + for (int index = 1; index < this.nodes.size(); index++) { + prefixes.add(prefix(index)); + } + return prefixes; + } + + public boolean isLastNode(int index) { + return index >= this.nodes.size() - 1; + } + + private List splitPath(String path) { + List nodes = new ArrayList(); + String current = extractIndexedPaths(path, nodes); + for (String name : StringUtils.delimitedListToStringArray(current, ".")) { + if (StringUtils.hasText(name)) { + nodes.add(new RelaxedDataBinder.BeanPath.PropertyNode(name)); + } + } + return nodes; + } + + private String extractIndexedPaths(String path, List nodes) { + int startRef = path.indexOf("["); + String current = path; + while (startRef >= 0) { + if (startRef > 0) { + nodes.addAll(splitPath(current.substring(0, startRef))); + } + int endRef = current.indexOf("]", startRef); + if (endRef > 0) { + String sub = current.substring(startRef + 1, endRef); + if (sub.matches("[0-9]+")) { + nodes.add(new RelaxedDataBinder.BeanPath.ArrayIndexNode(sub)); + } else { + nodes.add(new RelaxedDataBinder.BeanPath.MapIndexNode(sub)); + } + } + current = current.substring(endRef + 1); + startRef = current.indexOf("["); + } + return current; + } + + public void collapseKeys(int index) { + List revised = new ArrayList(); + for (int i = 0; i < index; i++) { + revised.add(this.nodes.get(i)); + } + StringBuilder builder = new StringBuilder(); + for (int i = index; i < this.nodes.size(); i++) { + if (i > index) { + builder.append("."); + } + builder.append(this.nodes.get(i).name); + } + revised.add(new RelaxedDataBinder.BeanPath.PropertyNode(builder.toString())); + this.nodes = revised; + } + + public void mapIndex(int index) { + RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(index); + if (node instanceof RelaxedDataBinder.BeanPath.PropertyNode) { + node = ((RelaxedDataBinder.BeanPath.PropertyNode) node).mapIndex(); + } + this.nodes.set(index, node); + } + + public String prefix(int index) { + return range(0, index); + } + + public void rename(int index, String name) { + this.nodes.get(index).name = name; + } + + public String name(int index) { + if (index < this.nodes.size()) { + return this.nodes.get(index).name; + } + return null; + } + + private String range(int start, int end) { + StringBuilder builder = new StringBuilder(); + for (int i = start; i < end; i++) { + RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(i); + builder.append(node); + } + if (builder.toString().startsWith(("."))) { + builder.replace(0, 1, ""); + } + return builder.toString(); + } + + public boolean isArrayIndex(int index) { + return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.ArrayIndexNode; + } + + public boolean isProperty(int index) { + return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.PropertyNode; + } + + @Override + public String toString() { + return prefix(this.nodes.size()); + } + + private static class PathNode { + + protected String name; + + PathNode(String name){ + this.name = name; + } + + } + + private static class ArrayIndexNode extends RelaxedDataBinder.BeanPath.PathNode { + + ArrayIndexNode(String name){ + super(name); + } + + @Override + public String toString() { + return "[" + this.name + "]"; + } + + } + + private static class MapIndexNode extends RelaxedDataBinder.BeanPath.PathNode { + + MapIndexNode(String name){ + super(name); + } + + @Override + public String toString() { + return "[" + this.name + "]"; + } + + } + + private static class PropertyNode extends RelaxedDataBinder.BeanPath.PathNode { + + PropertyNode(String name){ + super(name); + } + + public RelaxedDataBinder.BeanPath.MapIndexNode mapIndex() { + return new RelaxedDataBinder.BeanPath.MapIndexNode(this.name); + } + + @Override + public String toString() { + return "." + this.name; + } + + } + + } + + /** + * Extended version of {@link BeanPropertyBindingResult} to support relaxed + * binding. + */ + private static class RelaxedBeanPropertyBindingResult extends BeanPropertyBindingResult { + + private RelaxedConversionService conversionService; + + RelaxedBeanPropertyBindingResult(Object target, String objectName, boolean autoGrowNestedPaths, + int autoGrowCollectionLimit, ConversionService conversionService){ + super(target, objectName, autoGrowNestedPaths, autoGrowCollectionLimit); + this.conversionService = new RelaxedConversionService(conversionService); + } + + @Override + protected BeanWrapper createBeanWrapper() { + BeanWrapper beanWrapper = new RelaxedDataBinder.RelaxedBeanWrapper(getTarget()); + beanWrapper.setConversionService(this.conversionService); + beanWrapper.registerCustomEditor(InetAddress.class, new InetAddressEditor()); + return beanWrapper; + } + + } + + /** + * Extended version of {@link BeanWrapperImpl} to support relaxed binding. + */ + private static class RelaxedBeanWrapper extends BeanWrapperImpl { + + private static final Set BENIGN_PROPERTY_SOURCE_NAMES; + + static { + Set names = new HashSet(); + names.add(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME); + names.add(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME); + BENIGN_PROPERTY_SOURCE_NAMES = Collections.unmodifiableSet(names); + } + + RelaxedBeanWrapper(Object target){ + super(target); + } + + @Override + public void setPropertyValue(PropertyValue pv) throws BeansException { + try { + super.setPropertyValue(pv); + } catch (NotWritablePropertyException ex) { + PropertyOrigin origin = OriginCapablePropertyValue.getOrigin(pv); + if (isBenign(origin)) { + logger.debug("Ignoring benign property binding failure", ex); + return; + } + if (origin == null) { + throw ex; + } + throw new RelaxedBindingNotWritablePropertyException(ex, origin); + } + } + + private boolean isBenign(PropertyOrigin origin) { + String name = (origin != null ? origin.getSource().getName() : null); + return BENIGN_PROPERTY_SOURCE_NAMES.contains(name); + } + + } + + public static class RelaxedBindingNotWritablePropertyException extends NotWritablePropertyException { + + private final String message; + + private final PropertyOrigin propertyOrigin; + + RelaxedBindingNotWritablePropertyException(NotWritablePropertyException ex, PropertyOrigin propertyOrigin){ + super(ex.getBeanClass(), ex.getPropertyName()); + this.propertyOrigin = propertyOrigin; + this.message = "Failed to bind '" + propertyOrigin.getName() + "' from '" + + propertyOrigin.getSource().getName() + "' to '" + ex.getPropertyName() + "' property on '" + + ex.getBeanClass().getName() + "'"; + } + + @Override + public String getMessage() { + return this.message; + } + + public PropertyOrigin getPropertyOrigin() { + return this.propertyOrigin; + } + + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java new file mode 100644 index 00000000..f1b1628f --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java @@ -0,0 +1,241 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.util.StringUtils; + +/** + * Generates relaxed name variations from a given source. + * + * @author Phillip Webb + * @author Dave Syer + * @see RelaxedDataBinder + */ +public final class RelaxedNames implements Iterable { + + private static final Pattern CAMEL_CASE_PATTERN = Pattern.compile("([^A-Z-])([A-Z])"); + + private static final Pattern SEPARATED_TO_CAMEL_CASE_PATTERN = Pattern.compile("[_\\-.]"); + + private final String name; + + private final Set values = new LinkedHashSet(); + + /** + * Create a new {@link RelaxedNames} instance. + * + * @param name the source name. For the maximum number of variations specify the + * name using dashed notation (e.g. {@literal my-property-name} + */ + public RelaxedNames(String name){ + this.name = (name != null ? name : ""); + initialize(RelaxedNames.this.name, this.values); + } + + @Override + public Iterator iterator() { + return this.values.iterator(); + } + + private void initialize(String name, Set values) { + if (values.contains(name)) { + return; + } + for (RelaxedNames.Variation variation : RelaxedNames.Variation.values()) { + for (RelaxedNames.Manipulation manipulation : RelaxedNames.Manipulation.values()) { + String result = name; + result = manipulation.apply(result); + result = variation.apply(result); + values.add(result); + initialize(result, values); + } + } + } + + /** + * Name variations. + */ + enum Variation { + + NONE { + + @Override + public String apply(String value) { + return value; + } + + }, + + LOWERCASE { + + @Override + public String apply(String value) { + return (value.isEmpty() ? value : value.toLowerCase(Locale.ENGLISH)); + } + + }, + + UPPERCASE { + + @Override + public String apply(String value) { + return (value.isEmpty() ? value : value.toUpperCase(Locale.ENGLISH)); + } + + }; + + public abstract String apply(String value); + + } + + /** + * Name manipulations. + */ + enum Manipulation { + + NONE { + + @Override + public String apply(String value) { + return value; + } + + }, + + HYPHEN_TO_UNDERSCORE { + + @Override + public String apply(String value) { + return (value.indexOf('-') != -1 ? value.replace('-', '_') : value); + } + + }, + + UNDERSCORE_TO_PERIOD { + + @Override + public String apply(String value) { + return (value.indexOf('_') != -1 ? value.replace('_', '.') : value); + } + + }, + + PERIOD_TO_UNDERSCORE { + + @Override + public String apply(String value) { + return (value.indexOf('.') != -1 ? value.replace('.', '_') : value); + } + + }, + + CAMELCASE_TO_UNDERSCORE { + + @Override + public String apply(String value) { + if (value.isEmpty()) { + return value; + } + Matcher matcher = CAMEL_CASE_PATTERN.matcher(value); + if (!matcher.find()) { + return value; + } + matcher = matcher.reset(); + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(result, + matcher.group(1) + '_' + StringUtils.uncapitalize(matcher.group(2))); + } + matcher.appendTail(result); + return result.toString(); + } + + }, + + CAMELCASE_TO_HYPHEN { + + @Override + public String apply(String value) { + if (value.isEmpty()) { + return value; + } + Matcher matcher = CAMEL_CASE_PATTERN.matcher(value); + if (!matcher.find()) { + return value; + } + matcher = matcher.reset(); + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(result, + matcher.group(1) + '-' + StringUtils.uncapitalize(matcher.group(2))); + } + matcher.appendTail(result); + return result.toString(); + } + + }, + + SEPARATED_TO_CAMELCASE { + + @Override + public String apply(String value) { + return separatedToCamelCase(value, false); + } + + }, + + CASE_INSENSITIVE_SEPARATED_TO_CAMELCASE { + + @Override + public String apply(String value) { + return separatedToCamelCase(value, true); + } + + }; + + private static final char[] SUFFIXES = new char[] { '_', '-', '.' }; + + public abstract String apply(String value); + + private static String separatedToCamelCase(String value, boolean caseInsensitive) { + if (value.isEmpty()) { + return value; + } + StringBuilder builder = new StringBuilder(); + for (String field : SEPARATED_TO_CAMEL_CASE_PATTERN.split(value)) { + field = (caseInsensitive ? field.toLowerCase(Locale.ENGLISH) : field); + builder.append(builder.length() != 0 ? StringUtils.capitalize(field) : field); + } + char lastChar = value.charAt(value.length() - 1); + for (char suffix : SUFFIXES) { + if (lastChar == suffix) { + builder.append(suffix); + break; + } + } + return builder.toString(); + } + + } + + /** + * Return a {@link RelaxedNames} for the given source camelCase source name. + * + * @param name the source name in camelCase + * @return the relaxed names + */ + public static RelaxedNames forCamelCase(String name) { + StringBuilder result = new StringBuilder(); + for (char c : name.toCharArray()) { + result.append(Character.isUpperCase(c) && result.length() > 0 + && result.charAt(result.length() - 1) != '-' ? "-" + Character.toLowerCase(c) : c); + } + return new RelaxedNames(result.toString()); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java new file mode 100644 index 00000000..00a2d808 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java @@ -0,0 +1,17 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import org.springframework.core.convert.converter.Converter; + +/** + * Converts a String to a Char Array. + * + * @author Phillip Webb + */ +class StringToCharArrayConverter implements Converter { + + @Override + public char[] convert(String source) { + return source.toCharArray(); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java new file mode 100644 index 00000000..9cb73e90 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java @@ -0,0 +1,203 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +import org.springframework.util.Assert; +import org.springframework.util.ResourceUtils; + +/** + * Convenience base class for {@link Resource} implementations, pre-implementing + * typical behavior. + *

    + * The "exists" method will check whether a File or InputStream can be opened; + * "isOpen" will always return false; "getURL" and "getFile" throw an exception; + * and "toString" will return the description. + * + * @author Juergen Hoeller + * @since 28.12.2003 + */ +public abstract class AbstractResource implements Resource { + + /** + * This implementation checks whether a File can be opened, falling back to + * whether an InputStream can be opened. This will cover both directories and + * content resources. + */ + @Override + public boolean exists() { + // Try file existence: can we find the file in the file system? + try { + return getFile().exists(); + } catch (IOException ex) { + // Fall back to stream existence: can we open the stream? + try { + InputStream is = getInputStream(); + is.close(); + return true; + } catch (Throwable isEx) { + return false; + } + } + } + + /** + * This implementation always returns {@code true}. + */ + @Override + public boolean isReadable() { + return true; + } + + /** + * This implementation always returns {@code false}. + */ + @Override + public boolean isOpen() { + return false; + } + + /** + * This implementation throws a FileNotFoundException, assuming that the + * resource cannot be resolved to a URL. + */ + @Override + public URL getURL() throws IOException { + throw new FileNotFoundException(getDescription() + " cannot be resolved to URL"); + } + + /** + * This implementation builds a URI based on the URL returned by + * {@link #getURL()}. + */ + @Override + public URI getURI() throws IOException { + URL url = getURL(); + try { + return ResourceUtils.toURI(url); + } catch (URISyntaxException ex) { + throw new RuntimeException("Invalid URI [" + url + "]", ex); + } + } + + /** + * This implementation throws a FileNotFoundException, assuming that the + * resource cannot be resolved to an absolute file path. + */ + @Override + public File getFile() throws IOException { + throw new FileNotFoundException(getDescription() + " cannot be resolved to absolute file path"); + } + + /** + * This implementation reads the entire InputStream to calculate the content + * length. Subclasses will almost always be able to provide a more optimal + * version of this, e.g. checking a File length. + * + * @see #getInputStream() + */ + @Override + public long contentLength() throws IOException { + InputStream is = getInputStream(); + Assert.state(is != null, "Resource InputStream must not be null"); + try { + long size = 0; + byte[] buf = new byte[255]; + int read; + while ((read = is.read(buf)) != -1) { + size += read; + } + return size; + } finally { + try { + is.close(); + } catch (IOException ex) { + } + } + } + + /** + * This implementation checks the timestamp of the underlying File, if + * available. + * + * @see #getFileForLastModifiedCheck() + */ + @Override + public long lastModified() throws IOException { + long lastModified = getFileForLastModifiedCheck().lastModified(); + if (lastModified == 0L) { + throw new FileNotFoundException( + getDescription() + " cannot be resolved in the file system for resolving its last-modified timestamp"); + } + return lastModified; + } + + /** + * Determine the File to use for timestamp checking. + *

    + * The default implementation delegates to {@link #getFile()}. + * + * @return the File to use for timestamp checking (never {@code null}) + * @throws FileNotFoundException if the resource cannot be resolved as an + * absolute file path, i.e. is not available in a file system + * @throws IOException in case of general resolution/reading failures + */ + protected File getFileForLastModifiedCheck() throws IOException { + return getFile(); + } + + /** + * This implementation throws a FileNotFoundException, assuming that relative + * resources cannot be created for this resource. + */ + @Override + public org.springframework.core.io.Resource createRelative(String relativePath) throws IOException { + throw new FileNotFoundException("Cannot create a relative resource for " + getDescription()); + } + + /** + * This implementation always returns {@code null}, assuming that this resource + * type does not have a filename. + */ + @Override + public String getFilename() { + return null; + } + + /** + * This implementation returns the description of this resource. + * + * @see #getDescription() + */ + @Override + public String toString() { + return getDescription(); + } + + /** + * This implementation compares description strings. + * + * @see #getDescription() + */ + @Override + public boolean equals(Object obj) { + return (obj == this + || (obj instanceof org.springframework.core.io.Resource + && ((org.springframework.core.io.Resource) obj).getDescription().equals(getDescription()))); + } + + /** + * This implementation returns the description's hash code. + * + * @see #getDescription() + */ + @Override + public int hashCode() { + return getDescription().hashCode(); + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java new file mode 100644 index 00000000..c8ec120b --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java @@ -0,0 +1,117 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.util.Assert; + +/** + * {@link Resource} implementation for a given byte array. + *

    + * Creates a {@link ByteArrayInputStream} for the given byte array. + *

    + * Useful for loading content from any given byte array, without having to + * resort to a single-use {@link InputStreamResource}. Particularly useful for + * creating mail attachments from local content, where JavaMail needs to be able + * to read the stream multiple times. + * + * @author Juergen Hoeller + * @author Sam Brannen + * @see ByteArrayInputStream + * @see InputStreamResource + * @since 1.2.3 + */ +public class ByteArrayResource extends AbstractResource { + + private final byte[] byteArray; + + private final String description; + + /** + * Create a new {@code ByteArrayResource}. + * + * @param byteArray the byte array to wrap + */ + public ByteArrayResource(byte[] byteArray){ + this(byteArray, "resource loaded from byte array"); + } + + /** + * Create a new {@code ByteArrayResource} with a description. + * + * @param byteArray the byte array to wrap + * @param description where the byte array comes from + */ + public ByteArrayResource(byte[] byteArray, String description){ + Assert.notNull(byteArray, "Byte array must not be null"); + this.byteArray = byteArray; + this.description = (description != null ? description : ""); + } + + /** + * Return the underlying byte array. + */ + public final byte[] getByteArray() { + return this.byteArray; + } + + /** + * This implementation always returns {@code true}. + */ + @Override + public boolean exists() { + return true; + } + + /** + * This implementation returns the length of the underlying byte array. + */ + @Override + public long contentLength() { + return this.byteArray.length; + } + + /** + * This implementation returns a ByteArrayInputStream for the underlying byte + * array. + * + * @see ByteArrayInputStream + */ + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(this.byteArray); + } + + /** + * This implementation returns a description that includes the passed-in + * {@code description}, if any. + */ + @Override + public String getDescription() { + return "Byte array resource [" + this.description + "]"; + } + + /** + * This implementation compares the underlying byte array. + * + * @see Arrays#equals(byte[], byte[]) + */ + @Override + public boolean equals(Object obj) { + return (obj == this || (obj instanceof org.springframework.core.io.ByteArrayResource + && Arrays.equals(((ByteArrayResource) obj).byteArray, this.byteArray))); + } + + /** + * This implementation returns the hash code based on the underlying byte array. + */ + @Override + public int hashCode() { + return (byte[].class.hashCode() * 29 * this.byteArray.length); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java new file mode 100644 index 00000000..106f8f90 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java @@ -0,0 +1,107 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.*; + +import org.springframework.util.StringUtils; + +/** + * Composite {@link PropertySource} implementation that iterates over a set of + * {@link PropertySource} instances. Necessary in cases where multiple property + * sources share the same name, e.g. when multiple values are supplied to + * {@code @PropertySource}. + *

    + * As of Spring 4.1.2, this class extends {@link EnumerablePropertySource} + * instead of plain {@link PropertySource}, exposing {@link #getPropertyNames()} + * based on the accumulated property names from all contained sources (as far as + * possible). + * + * @author Chris Beams + * @author Juergen Hoeller + * @author Phillip Webb + * @since 3.1.1 + */ +public class CompositePropertySource extends EnumerablePropertySource { + + private final Set> propertySources = new LinkedHashSet>(); + + /** + * Create a new {@code CompositePropertySource}. + * + * @param name the name of the property source + */ + public CompositePropertySource(String name){ + super(name); + } + + @Override + public Object getProperty(String name) { + for (PropertySource propertySource : this.propertySources) { + Object candidate = propertySource.getProperty(name); + if (candidate != null) { + return candidate; + } + } + return null; + } + + @Override + public boolean containsProperty(String name) { + for (PropertySource propertySource : this.propertySources) { + if (propertySource.containsProperty(name)) { + return true; + } + } + return false; + } + + @Override + public String[] getPropertyNames() { + Set names = new LinkedHashSet(); + for (PropertySource propertySource : this.propertySources) { + if (!(propertySource instanceof EnumerablePropertySource)) { + throw new IllegalStateException( + "Failed to enumerate property names due to non-enumerable property source: " + propertySource); + } + names.addAll(Arrays.asList(((EnumerablePropertySource) propertySource).getPropertyNames())); + } + return StringUtils.toStringArray(names); + } + + /** + * Add the given {@link PropertySource} to the end of the chain. + * + * @param propertySource the PropertySource to add + */ + public void addPropertySource(PropertySource propertySource) { + this.propertySources.add(propertySource); + } + + /** + * Add the given {@link PropertySource} to the start of the chain. + * + * @param propertySource the PropertySource to add + * @since 4.1 + */ + public void addFirstPropertySource(PropertySource propertySource) { + List> existing = new ArrayList>(this.propertySources); + this.propertySources.clear(); + this.propertySources.add(propertySource); + this.propertySources.addAll(existing); + } + + /** + * Return all property sources that this composite source holds. + * + * @since 4.1.1 + */ + public Collection> getPropertySources() { + return this.propertySources; + } + + @Override + public String toString() { + return String + .format("%s [name='%s', propertySources=%s]", getClass().getSimpleName(), this.name, this.propertySources); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java new file mode 100644 index 00000000..e5ce8639 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java @@ -0,0 +1,58 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import org.springframework.util.ObjectUtils; + +/** + * A {@link PropertySource} implementation capable of interrogating its + * underlying source object to enumerate all possible property name/value pairs. + * Exposes the {@link #getPropertyNames()} method to allow callers to introspect + * available properties without having to access the underlying source object. + * This also facilitates a more efficient implementation of + * {@link #containsProperty(String)}, in that it can call + * {@link #getPropertyNames()} and iterate through the returned array rather + * than attempting a call to {@link #getProperty(String)} which may be more + * expensive. Implementations may consider caching the result of + * {@link #getPropertyNames()} to fully exploit this performance opportunity. + *

    + * Most framework-provided {@code PropertySource} implementations are + * enumerable; a counter-example would be {@code JndiPropertySource} where, due + * to the nature of JNDI it is not possible to determine all possible property + * names at any given time; rather it is only possible to try to access a + * property (via {@link #getProperty(String)}) in order to evaluate whether it + * is present or not. + * + * @author Chris Beams + * @author Juergen Hoeller + * @since 3.1 + */ +public abstract class EnumerablePropertySource extends PropertySource { + + public EnumerablePropertySource(String name, T source){ + super(name, source); + } + + protected EnumerablePropertySource(String name){ + super(name); + } + + /** + * Return whether this {@code PropertySource} contains a property with the given + * name. + *

    + * This implementation checks for the presence of the given name within the + * {@link #getPropertyNames()} array. + * + * @param name the name of the property to find + */ + @Override + public boolean containsProperty(String name) { + return ObjectUtils.containsElement(getPropertyNames(), name); + } + + /** + * Return the names of all properties contained by the {@linkplain #getSource() + * source} object (never {@code null}). + */ + public abstract String[] getPropertyNames(); + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java new file mode 100644 index 00000000..eecc0ee4 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java @@ -0,0 +1,38 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.Map; + +import org.springframework.core.env.PropertiesPropertySource; +import org.springframework.core.env.PropertySource; +import org.springframework.util.StringUtils; + +/** + * {@link PropertySource} that reads keys and values from a {@code Map} object. + * + * @author Chris Beams + * @author Juergen Hoeller + * @since 3.1 + * @see PropertiesPropertySource + */ +public class MapPropertySource extends EnumerablePropertySource> { + + public MapPropertySource(String name, Map source){ + super(name, source); + } + + @Override + public Object getProperty(String name) { + return this.source.get(name); + } + + @Override + public boolean containsProperty(String name) { + return this.source.containsKey(name); + } + + @Override + public String[] getPropertyNames() { + return StringUtils.toStringArray(this.source.keySet()); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java new file mode 100644 index 00000000..d8f32d3c --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java @@ -0,0 +1,221 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.core.env.PropertyResolver; +import org.springframework.core.env.PropertySourcesPropertyResolver; + +/** + * Default implementation of the {@link PropertySources} interface. Allows + * manipulation of contained property sources and provides a constructor for + * copying an existing {@code PropertySources} instance. + *

    + * Where precedence is mentioned in methods such as {@link #addFirst} + * and {@link #addLast}, this is with regard to the order in which property + * sources will be searched when resolving a given property with a + * {@link PropertyResolver}. + * + * @author Chris Beams + * @author Juergen Hoeller + * @see PropertySourcesPropertyResolver + * @since 3.1 + */ +public class MutablePropertySources implements PropertySources { + + private final Log logger; + + private final List> propertySourceList = new CopyOnWriteArrayList>(); + + /** + * Create a new {@link MutablePropertySources} + * object. + */ + public MutablePropertySources(){ + this.logger = LogFactory.getLog(getClass()); + } + + /** + * Create a new {@code MutablePropertySources} from the given propertySources + * object, preserving the original order of contained {@code PropertySource} + * objects. + */ + public MutablePropertySources(PropertySources propertySources){ + this(); + for (PropertySource propertySource : propertySources) { + addLast(propertySource); + } + } + + /** + * Create a new {@link MutablePropertySources} + * object and inherit the given logger, usually from an enclosing + * {@link Environment}. + */ + MutablePropertySources(Log logger){ + this.logger = logger; + } + + @Override + public boolean contains(String name) { + return this.propertySourceList.contains(PropertySource.named(name)); + } + + @Override + public PropertySource get(String name) { + int index = this.propertySourceList.indexOf(PropertySource.named(name)); + return (index != -1 ? this.propertySourceList.get(index) : null); + } + + @Override + public Iterator> iterator() { + return this.propertySourceList.iterator(); + } + + /** + * Add the given property source object with highest precedence. + */ + public void addFirst(PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + "' with highest search precedence"); + } + removeIfPresent(propertySource); + this.propertySourceList.add(0, propertySource); + } + + /** + * Add the given property source object with lowest precedence. + */ + public void addLast(PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + "' with lowest search precedence"); + } + removeIfPresent(propertySource); + this.propertySourceList.add(propertySource); + } + + /** + * Add the given property source object with precedence immediately higher than + * the named relative property source. + */ + public void addBefore(String relativePropertySourceName, PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + + "' with search precedence immediately higher than '" + relativePropertySourceName + "'"); + } + assertLegalRelativeAddition(relativePropertySourceName, propertySource); + removeIfPresent(propertySource); + int index = assertPresentAndGetIndex(relativePropertySourceName); + addAtIndex(index, propertySource); + } + + /** + * Add the given property source object with precedence immediately lower than + * the named relative property source. + */ + public void addAfter(String relativePropertySourceName, PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + + "' with search precedence immediately lower than '" + relativePropertySourceName + "'"); + } + assertLegalRelativeAddition(relativePropertySourceName, propertySource); + removeIfPresent(propertySource); + int index = assertPresentAndGetIndex(relativePropertySourceName); + addAtIndex(index + 1, propertySource); + } + + /** + * Return the precedence of the given property source, {@code -1} if not found. + */ + public int precedenceOf(PropertySource propertySource) { + return this.propertySourceList.indexOf(propertySource); + } + + /** + * Remove and return the property source with the given name, {@code null} if + * not found. + * + * @param name the name of the property source to find and remove + */ + public PropertySource remove(String name) { + if (logger.isDebugEnabled()) { + logger.debug("Removing PropertySource '" + name + "'"); + } + int index = this.propertySourceList.indexOf(PropertySource.named(name)); + return (index != -1 ? this.propertySourceList.remove(index) : null); + } + + /** + * Replace the property source with the given name with the given property + * source object. + * + * @param name the name of the property source to find and replace + * @param propertySource the replacement property source + * @throws IllegalArgumentException if no property source with the given name is + * present + * @see #contains + */ + public void replace(String name, PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Replacing PropertySource '" + name + "' with '" + propertySource.getName() + "'"); + } + int index = assertPresentAndGetIndex(name); + this.propertySourceList.set(index, propertySource); + } + + /** + * Return the number of {@link PropertySource} objects contained. + */ + public int size() { + return this.propertySourceList.size(); + } + + @Override + public String toString() { + return this.propertySourceList.toString(); + } + + /** + * Ensure that the given property source is not being added relative to itself. + */ + protected void assertLegalRelativeAddition(String relativePropertySourceName, PropertySource propertySource) { + String newPropertySourceName = propertySource.getName(); + if (relativePropertySourceName.equals(newPropertySourceName)) { + throw new IllegalArgumentException( + "PropertySource named '" + newPropertySourceName + "' cannot be added relative to itself"); + } + } + + /** + * Remove the given property source if it is present. + */ + protected void removeIfPresent(PropertySource propertySource) { + this.propertySourceList.remove(propertySource); + } + + /** + * Add the given property source at a particular index in the list. + */ + private void addAtIndex(int index, PropertySource propertySource) { + removeIfPresent(propertySource); + this.propertySourceList.add(index, propertySource); + } + + /** + * Assert that the named property source is present and return its index. + * + * @param name {@linkplain PropertySource#getName() name of the property source} + * to find + * @throws IllegalArgumentException if the named property source is not present + */ + private int assertPresentAndGetIndex(String name) { + int index = this.propertySourceList.indexOf(PropertySource.named(name)); + if (index == -1) { + throw new IllegalArgumentException("PropertySource named '" + name + "' does not exist"); + } + return index; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java new file mode 100644 index 00000000..12fe686a --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java @@ -0,0 +1,34 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.Map; +import java.util.Properties; + +import org.springframework.core.env.PropertySource; + +/** + * {@link PropertySource} implementation that extracts properties from a + * {@link Properties} object. + *

    + * Note that because a {@code Properties} object is technically an + * {@code } {@link java.util.Hashtable Hashtable}, one may + * contain non-{@code String} keys or values. This implementation, however is + * restricted to accessing only {@code String}-based keys and values, in the + * same fashion as {@link Properties#getProperty} and + * {@link Properties#setProperty}. + * + * @author Chris Beams + * @author Juergen Hoeller + * @since 3.1 + */ +public class PropertiesPropertySource extends MapPropertySource { + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public PropertiesPropertySource(String name, Properties source){ + super(name, (Map) source); + } + + protected PropertiesPropertySource(String name, Map source){ + super(name, source); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java new file mode 100644 index 00000000..f7e4cc34 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java @@ -0,0 +1,239 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; + +/** + * Abstract base class representing a source of name/value property pairs. The + * underlying {@linkplain #getSource() source object} may be of any type + * {@code T} that encapsulates properties. Examples include + * {@link java.util.Properties} objects, {@link java.util.Map} objects, + * {@code ServletContext} and {@code ServletConfig} objects (for access to init + * parameters). Explore the {@code PropertySource} type hierarchy to see + * provided implementations. + *

    + * {@code PropertySource} objects are not typically used in isolation, but + * rather through a {@link PropertySources} object, which aggregates property + * sources and in conjunction with a {@link PropertyResolver} implementation + * that can perform precedence-based searches across the set of + * {@code PropertySources}. + *

    + * {@code PropertySource} identity is determined not based on the content of + * encapsulated properties, but rather based on the {@link #getName() name} of + * the {@code PropertySource} alone. This is useful for manipulating + * {@code PropertySource} objects when in collection contexts. See operations in + * {@link MutablePropertySources} as well as the {@link #named(String)} and + * {@link #toString()} methods for details. + *

    + * Note that when working + * with @{@link org.springframework.context.annotation.Configuration + * Configuration} classes that the @{@link PropertySource PropertySource} + * annotation provides a convenient and declarative way of adding property + * sources to the enclosing {@code Environment}. + * + * @author Chris Beams + * @since 3.1 + * @see PropertySources + * @see MutablePropertySources + * @see PropertySource + */ +public abstract class PropertySource { + + protected final Log logger = LogFactory.getLog(getClass()); + + protected final String name; + + protected final T source; + + /** + * Create a new {@code PropertySource} with the given name and source object. + */ + public PropertySource(String name, T source){ + Assert.hasText(name, "Property source name must contain at least one character"); + Assert.notNull(source, "Property source must not be null"); + this.name = name; + this.source = source; + } + + /** + * Create a new {@code PropertySource} with the given name and with a new + * {@code Object} instance as the underlying source. + *

    + * Often useful in testing scenarios when creating anonymous implementations + * that never query an actual source but rather return hard-coded values. + */ + @SuppressWarnings("unchecked") + public PropertySource(String name){ + this(name, (T) new Object()); + } + + /** + * Return the name of this {@code PropertySource} + */ + public String getName() { + return this.name; + } + + /** + * Return the underlying source object for this {@code PropertySource}. + */ + public T getSource() { + return this.source; + } + + /** + * Return whether this {@code PropertySource} contains the given name. + *

    + * This implementation simply checks for a {@code null} return value from + * {@link #getProperty(String)}. Subclasses may wish to implement a more + * efficient algorithm if possible. + * + * @param name the property name to find + */ + public boolean containsProperty(String name) { + return (getProperty(name) != null); + } + + /** + * Return the value associated with the given name, or {@code null} if not + * found. + * + * @param name the property to find + */ + public abstract Object getProperty(String name); + + /** + * This {@code PropertySource} object is equal to the given object if: + *

      + *
    • they are the same instance + *
    • the {@code name} properties for both objects are equal + *
    + *

    + * No properties other than {@code name} are evaluated. + */ + @Override + public boolean equals(Object obj) { + return (this == obj || (obj instanceof PropertySource + && ObjectUtils.nullSafeEquals(this.name, ((PropertySource) obj).name))); + } + + /** + * Return a hash code derived from the {@code name} property of this + * {@code PropertySource} object. + */ + @Override + public int hashCode() { + return ObjectUtils.nullSafeHashCode(this.name); + } + + /** + * Produce concise output (type and name) if the current log level does not + * include debug. If debug is enabled, produce verbose output including the hash + * code of the PropertySource instance and every name/value property pair. + *

    + * This variable verbosity is useful as a property source such as system + * properties or environment variables may contain an arbitrary number of + * property pairs, potentially leading to difficult to read exception and log + * messages. + * + * @see Log#isDebugEnabled() + */ + @Override + public String toString() { + if (logger.isDebugEnabled()) { + return getClass().getSimpleName() + "@" + System.identityHashCode(this) + " {name='" + this.name + + "', properties=" + this.source + "}"; + } else { + return getClass().getSimpleName() + " {name='" + this.name + "'}"; + } + } + + /** + * Return a {@code PropertySource} implementation intended for collection + * comparison purposes only. + *

    + * Primarily for internal use, but given a collection of {@code PropertySource} + * objects, may be used as follows: + * + * 

    +     *
+     * {
+     *     @code
+     *     List> sources = new ArrayList>();
+     *     sources.add(new MapPropertySource("sourceA", mapA));
+     *     sources.add(new MapPropertySource("sourceB", mapB));
+     *     assert sources.contains(PropertySource.named("sourceA"));
+     *     assert sources.contains(PropertySource.named("sourceB"));
+     *     assert !sources.contains(PropertySource.named("sourceC"));
+     * }
+     *
    + * + * The returned {@code PropertySource} will throw + * {@code UnsupportedOperationException} if any methods other than + * {@code equals(Object)}, {@code hashCode()}, and {@code toString()} are + * called. + * + * @param name the name of the comparison {@code PropertySource} to be created + * and returned. + */ + public static PropertySource named(String name) { + return new ComparisonPropertySource(name); + } + + /** + * {@code PropertySource} to be used as a placeholder in cases where an actual + * property source cannot be eagerly initialized at application context creation + * time. For example, a {@code ServletContext}-based property source must wait + * until the {@code ServletContext} object is available to its enclosing + * {@code ApplicationContext}. In such cases, a stub should be used to hold the + * intended default position/order of the property source, then be replaced + * during context refresh. + * + * @see org.springframework.web.context.support.StandardServletEnvironment + * @see org.springframework.web.context.support.ServletContextPropertySource + */ + public static class StubPropertySource extends PropertySource { + + public StubPropertySource(String name){ + super(name, new Object()); + } + + /** + * Always returns {@code null}. + */ + @Override + public String getProperty(String name) { + return null; + } + } + + /** + * @see PropertySource#named(String) + */ + static class ComparisonPropertySource extends StubPropertySource { + + private static final String USAGE_ERROR = "ComparisonPropertySource instances are for use with collection comparison only"; + + public ComparisonPropertySource(String name){ + super(name); + } + + @Override + public Object getSource() { + throw new UnsupportedOperationException(USAGE_ERROR); + } + + @Override + public boolean containsProperty(String name) { + throw new UnsupportedOperationException(USAGE_ERROR); + } + + @Override + public String getProperty(String name) { + throw new UnsupportedOperationException(USAGE_ERROR); + } + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java new file mode 100644 index 00000000..982f61b8 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java @@ -0,0 +1,35 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.IOException; + +import org.springframework.core.io.support.SpringFactoriesLoader; + +/** + * Strategy interface located via {@link SpringFactoriesLoader} and used to load + * a {@link PropertySource}. + * + * @author Dave Syer + * @author Phillip Webb + */ +public interface PropertySourceLoader { + + /** + * Returns the file extensions that the loader supports (excluding the '.'). + * + * @return the file extensions + */ + String[] getFileExtensions(); + + /** + * Load the resource into a property source. + * + * @param name the name of the property source + * @param resource the resource to load + * @param profile the name of the profile to load or {@code null}. The profile + * can be used to load multi-document files (such as YAML). Simple property + * formats should {@code null} when asked to load a profile. + * @return a property source or {@code null} + * @throws IOException if the source cannot be loaded + */ + PropertySource load(String name, Resource resource, String profile) throws IOException; +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java new file mode 100644 index 00000000..3cd08c9a --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java @@ -0,0 +1,25 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +/** + * Holder containing one or more {@link PropertySource} objects. + * + * @author Chris Beams + * @since 3.1 + */ +public interface PropertySources extends Iterable> { + + /** + * Return whether a property source with the given name is contained. + * + * @param name the {@linkplain PropertySource#getName() name of the property source} to find + */ + boolean contains(String name); + + /** + * Return the property source with the given name, {@code null} if not found. + * + * @param name the {@linkplain PropertySource#getName() name of the property source} to find + */ + PropertySource get(String name); + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java new file mode 100644 index 00000000..88301f20 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java @@ -0,0 +1,57 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URL; + +import org.springframework.core.io.*; +import org.springframework.core.io.ByteArrayResource; + +/** + * Interface for a resource descriptor that abstracts from the actual type of + * underlying resource, such as a file or class path resource. + *

    + * An InputStream can be opened for every resource if it exists in physical + * form, but a URL or File handle can just be returned for certain resources. + * The actual behavior is implementation-specific. + * + * @author Juergen Hoeller + * @since 28.12.2003 + * @see #getInputStream() + * @see #getURL() + * @see #getURI() + * @see #getFile() + * @see WritableResource + * @see ContextResource + * @see UrlResource + * @see ClassPathResource + * @see FileSystemResource + * @see PathResource + * @see ByteArrayResource + * @see InputStreamResource + */ +public interface Resource extends InputStreamSource { + + boolean exists(); + + boolean isReadable(); + + boolean isOpen(); + + URL getURL() throws IOException; + + URI getURI() throws IOException; + + File getFile() throws IOException; + + long contentLength() throws IOException; + + long lastModified() throws IOException; + + org.springframework.core.io.Resource createRelative(String var1) throws IOException; + + String getFilename(); + + String getDescription(); +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java new file mode 100644 index 00000000..f59c6983 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java @@ -0,0 +1,182 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.*; + +import org.springframework.core.env.Environment; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +/** + * {@link YamlProcessor.DocumentMatcher} backed by + * {@link Environment#getActiveProfiles()}. A YAML document may define a + * "spring.profiles" element as a comma-separated list of Spring profile names, + * optionally negated using the {@code !} character. If both negated and + * non-negated profiles are specified for a single document, at least one + * non-negated profile must match and no negated profiles may match. + * + * @author Dave Syer + * @author Matt Benson + * @author Phillip Webb + * @author Andy Wilkinson + */ +public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatcher { + + private String[] activeProfiles = new String[0]; + + public SpringProfileDocumentMatcher(){ + } + + public SpringProfileDocumentMatcher(String... profiles){ + addActiveProfiles(profiles); + } + + public void addActiveProfiles(String... profiles) { + LinkedHashSet set = new LinkedHashSet(Arrays.asList(this.activeProfiles)); + Collections.addAll(set, profiles); + this.activeProfiles = set.toArray(new String[set.size()]); + } + + @Override + public YamlProcessor.MatchStatus matches(Properties properties) { + List profiles = extractSpringProfiles(properties); + ProfilesMatcher profilesMatcher = getProfilesMatcher(); + Set negative = extractProfiles(profiles, ProfileType.NEGATIVE); + Set positive = extractProfiles(profiles, ProfileType.POSITIVE); + if (!CollectionUtils.isEmpty(negative)) { + if (profilesMatcher.matches(negative) == YamlProcessor.MatchStatus.FOUND) { + return YamlProcessor.MatchStatus.NOT_FOUND; + } + if (CollectionUtils.isEmpty(positive)) { + return YamlProcessor.MatchStatus.FOUND; + } + } + return profilesMatcher.matches(positive); + } + + private List extractSpringProfiles(Properties properties) { + SpringProperties springProperties = new SpringProperties(); + MutablePropertySources propertySources = new MutablePropertySources(); + propertySources.addFirst(new PropertiesPropertySource("profiles", properties)); + // PropertyValues propertyValues = new PropertySourcesPropertyValues( + // propertySources); + // new RelaxedDataBinder(springProperties, "spring").bind(propertyValues); + // TODO + List profiles = springProperties.getProfiles(); + return profiles; + } + + private ProfilesMatcher getProfilesMatcher() { + return (this.activeProfiles.length != 0 ? new ActiveProfilesMatcher( + new HashSet(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher()); + } + + private Set extractProfiles(List profiles, ProfileType type) { + if (CollectionUtils.isEmpty(profiles)) { + return null; + } + Set extractedProfiles = new HashSet(); + for (String candidate : profiles) { + ProfileType candidateType = ProfileType.POSITIVE; + if (candidate.startsWith("!")) { + candidateType = ProfileType.NEGATIVE; + } + if (candidateType == type) { + extractedProfiles.add(type != ProfileType.POSITIVE ? candidate.substring(1) : candidate); + } + } + return extractedProfiles; + } + + /** + * Profile match types. + */ + enum ProfileType { + + POSITIVE, NEGATIVE + + } + + /** + * Base class for profile matchers. + */ + private abstract static class ProfilesMatcher { + + public final YamlProcessor.MatchStatus matches(Set profiles) { + if (CollectionUtils.isEmpty(profiles)) { + return YamlProcessor.MatchStatus.ABSTAIN; + } + return doMatches(profiles); + } + + protected abstract YamlProcessor.MatchStatus doMatches(Set profiles); + + } + + /** + * {@link ProfilesMatcher} that matches when a value in {@code spring.profiles} + * is also in {@code spring.profiles.active}. + */ + private static class ActiveProfilesMatcher extends ProfilesMatcher { + + private final Set activeProfiles; + + ActiveProfilesMatcher(Set activeProfiles){ + this.activeProfiles = activeProfiles; + } + + @Override + protected YamlProcessor.MatchStatus doMatches(Set profiles) { + if (profiles.isEmpty()) { + return YamlProcessor.MatchStatus.NOT_FOUND; + } + for (String activeProfile : this.activeProfiles) { + if (profiles.contains(activeProfile)) { + return YamlProcessor.MatchStatus.FOUND; + } + } + return YamlProcessor.MatchStatus.NOT_FOUND; + } + + } + + /** + * {@link ProfilesMatcher} that matches when {@code + * spring.profiles} is empty or contains a value with no text. + * + * @see StringUtils#hasText(String) + */ + private static class EmptyProfilesMatcher extends ProfilesMatcher { + + @Override + public YamlProcessor.MatchStatus doMatches(Set springProfiles) { + if (springProfiles.isEmpty()) { + return YamlProcessor.MatchStatus.FOUND; + } + for (String profile : springProfiles) { + if (!StringUtils.hasText(profile)) { + return YamlProcessor.MatchStatus.FOUND; + } + } + return YamlProcessor.MatchStatus.NOT_FOUND; + } + + } + + /** + * Class for binding {@code spring.profiles} property. + */ + static class SpringProperties { + + private List profiles = new ArrayList(); + + public List getProfiles() { + return this.profiles; + } + + public void setProfiles(List profiles) { + this.profiles = profiles; + } + + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java new file mode 100644 index 00000000..819c99fc --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java @@ -0,0 +1,419 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.IOException; +import java.io.Reader; +import java.util.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.util.Assert; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.parser.ParserException; +import org.yaml.snakeyaml.reader.UnicodeReader; + +/** + * Base class for YAML factories. + * + * @author Dave Syer + * @author Juergen Hoeller + * @since 4.1 + */ +public abstract class YamlProcessor { + + private final Log logger = LogFactory.getLog(getClass()); + + private ResolutionMethod resolutionMethod = ResolutionMethod.OVERRIDE; + + private Resource[] resources = new Resource[0]; + + private List documentMatchers = Collections.emptyList(); + + private boolean matchDefault = true; + + /** + * A map of document matchers allowing callers to selectively use only some of + * the documents in a YAML resource. In YAML documents are separated by + * --- lines, and each document is converted to properties before + * the match is made. E.g. + * + * 

    +     * environment: dev
+     * url: http://dev.bar.com
+     * name: Developer Setup
+     * ---
+     * environment: prod
+     * url:http://foo.bar.com
+     * name: My Cool App
+     *
    + * + * when mapped with + * + * 
    +     * setDocumentMatchers(properties -> ("prod"
+     *     .equals(properties.getProperty("environment")) ? MatchStatus.FOUND : MatchStatus.NOT_FOUND));
+     *
    + * + * would end up as + * + * 
    +     * environment=prod
+     * url=http://foo.bar.com
+     * name=My Cool App
+     *
    + */ + public void setDocumentMatchers(DocumentMatcher... matchers) { + this.documentMatchers = Arrays.asList(matchers); + } + + /** + * Flag indicating that a document for which all the + * {@link #setDocumentMatchers(DocumentMatcher...) document matchers} abstain + * will nevertheless match. Default is {@code true}. + */ + public void setMatchDefault(boolean matchDefault) { + this.matchDefault = matchDefault; + } + + /** + * Method to use for resolving resources. Each resource will be converted to a + * Map, so this property is used to decide which map entries to keep in the + * final output from this factory. Default is {@link ResolutionMethod#OVERRIDE}. + */ + public void setResolutionMethod(ResolutionMethod resolutionMethod) { + Assert.notNull(resolutionMethod, "ResolutionMethod must not be null"); + this.resolutionMethod = resolutionMethod; + } + + /** + * Set locations of YAML {@link Resource resources} to be loaded. + * + * @see ResolutionMethod + */ + public void setResources(Resource... resources) { + this.resources = resources; + } + + /** + * Provide an opportunity for subclasses to process the Yaml parsed from the + * supplied resources. Each resource is parsed in turn and the documents inside + * checked against the {@link #setDocumentMatchers(DocumentMatcher...) + * matchers}. If a document matches it is passed into the callback, along with + * its representation as Properties. Depending on the + * {@link #setResolutionMethod(ResolutionMethod)} not all of the documents will + * be parsed. + * + * @param callback a callback to delegate to once matching documents are found + * @see #createYaml() + */ + protected void process(MatchCallback callback) { + Yaml yaml = createYaml(); + for (Resource resource : this.resources) { + boolean found = process(callback, yaml, resource); + if (this.resolutionMethod == ResolutionMethod.FIRST_FOUND && found) { + return; + } + } + } + + /** + * Create the {@link Yaml} instance to use. + */ + protected Yaml createYaml() { + return new Yaml(new StrictMapAppenderConstructor()); + } + + private boolean process(MatchCallback callback, Yaml yaml, Resource resource) { + int count = 0; + try { + if (logger.isDebugEnabled()) { + logger.debug("Loading from YAML: " + resource); + } + Reader reader = new UnicodeReader(resource.getInputStream()); + try { + for (Object object : yaml.loadAll(reader)) { + if (object != null && process(asMap(object), callback)) { + count++; + if (this.resolutionMethod == ResolutionMethod.FIRST_FOUND) { + break; + } + } + } + if (logger.isDebugEnabled()) { + logger.debug( + "Loaded " + count + " document" + (count > 1 ? "s" : "") + " from YAML resource: " + resource); + } + } finally { + reader.close(); + } + } catch (IOException ex) { + handleProcessError(resource, ex); + } + return (count > 0); + } + + private void handleProcessError(Resource resource, IOException ex) { + if (this.resolutionMethod != ResolutionMethod.FIRST_FOUND + && this.resolutionMethod != ResolutionMethod.OVERRIDE_AND_IGNORE) { + throw new IllegalStateException(ex); + } + if (logger.isWarnEnabled()) { + logger.warn("Could not load map from " + resource + ": " + ex.getMessage()); + } + } + + @SuppressWarnings("unchecked") + private Map asMap(Object object) { + // YAML can have numbers as keys + Map result = new LinkedHashMap(); + if (!(object instanceof Map)) { + // A document can be a text literal + result.put("document", object); + return result; + } + + Map map = (Map) object; + for (Map.Entry entry : map.entrySet()) { + Object value = entry.getValue(); + if (value instanceof Map) { + value = asMap(value); + } + Object key = entry.getKey(); + if (key instanceof CharSequence) { + result.put(key.toString(), value); + } else { + // It has to be a map key in this case + result.put("[" + key.toString() + "]", value); + } + } + return result; + } + + private boolean process(Map map, MatchCallback callback) { + Properties properties = new Properties() { + + @Override + public String getProperty(String key) { + Object value = get(key); + return (value != null ? value.toString() : null); + } + }; + properties.putAll(getFlattenedMap(map)); + + if (this.documentMatchers.isEmpty()) { + if (logger.isDebugEnabled()) { + logger.debug("Merging document (no matchers set): " + map); + } + callback.process(properties, map); + return true; + } + + MatchStatus result = MatchStatus.ABSTAIN; + for (DocumentMatcher matcher : this.documentMatchers) { + MatchStatus match = matcher.matches(properties); + result = MatchStatus.getMostSpecific(match, result); + if (match == MatchStatus.FOUND) { + if (logger.isDebugEnabled()) { + logger.debug("Matched document with document matcher: " + properties); + } + callback.process(properties, map); + return true; + } + } + + if (result == MatchStatus.ABSTAIN && this.matchDefault) { + if (logger.isDebugEnabled()) { + logger.debug("Matched document with default matcher: " + map); + } + callback.process(properties, map); + return true; + } + + if (logger.isDebugEnabled()) { + logger.debug("Unmatched document: " + map); + } + return false; + } + + /** + * Return a flattened version of the given map, recursively following any nested + * Map or Collection values. Entries from the resulting map retain the same + * order as the source. When called with the Map from a {@link MatchCallback} + * the result will contain the same values as the {@link MatchCallback} + * Properties. + * + * @param source the source map + * @return a flattened map + * @since 4.1.3 + */ + protected final Map getFlattenedMap(Map source) { + Map result = new LinkedHashMap(); + buildFlattenedMap(result, source, null); + return result; + } + + private static boolean containsText(CharSequence str) { + int strLen = str.length(); + for (int i = 0; i < strLen; i++) { + if (!Character.isWhitespace(str.charAt(i))) { + return true; + } + } + return false; + } + + private void buildFlattenedMap(Map result, Map source, String path) { + for (Map.Entry entry : source.entrySet()) { + String key = entry.getKey(); + if (path != null && !path.isEmpty() && containsText(path)) { + if (key.startsWith("[")) { + key = path + key; + } else { + key = path + '.' + key; + } + } + Object value = entry.getValue(); + if (value instanceof String) { + result.put(key, value); + } else if (value instanceof Map) { + // Need a compound key + @SuppressWarnings("unchecked") + Map map = (Map) value; + buildFlattenedMap(result, map, key); + } else if (value instanceof Collection) { + // Need a compound key + @SuppressWarnings("unchecked") + Collection collection = (Collection) value; + int count = 0; + for (Object object : collection) { + buildFlattenedMap(result, Collections.singletonMap("[" + (count++) + "]", object), key); + } + } else { + result.put(key, (value != null ? value : "")); + } + } + } + + /** + * Callback interface used to process the YAML parsing results. + */ + public interface MatchCallback { + + /** + * Process the given representation of the parsing results. + * + * @param properties the properties to process (as a flattened representation + * with indexed keys in case of a collection or map) + * @param map the result map (preserving the original value structure in the + * YAML document) + */ + void process(Properties properties, Map map); + } + + /** + * Strategy interface used to test if properties match. + */ + public interface DocumentMatcher { + + /** + * Test if the given properties match. + * + * @param properties the properties to test + * @return the status of the match + */ + MatchStatus matches(Properties properties); + } + + /** + * Status returned from {@link DocumentMatcher#matches(Properties)} + */ + public enum MatchStatus { + + /** + * A match was found. + */ + FOUND, + + /** + * No match was found. + */ + NOT_FOUND, + + /** + * The matcher should not be considered. + */ + ABSTAIN; + + /** + * Compare two {@link MatchStatus} items, returning the most specific status. + */ + public static MatchStatus getMostSpecific(MatchStatus a, MatchStatus b) { + return (a.ordinal() < b.ordinal() ? a : b); + } + } + + /** + * Method to use for resolving resources. + */ + public enum ResolutionMethod { + + /** + * Replace values from earlier in the list. + */ + OVERRIDE, + + /** + * Replace values from earlier in the list, ignoring any failures. + */ + OVERRIDE_AND_IGNORE, + + /** + * Take the first resource in the list that exists and use just that. + */ + FIRST_FOUND + } + + /** + * A specialized {@link Constructor} that checks for duplicate keys. + */ + protected static class StrictMapAppenderConstructor extends Constructor { + + // Declared as public for use in subclasses + public StrictMapAppenderConstructor(){ + super(); + } + + @Override + protected Map constructMapping(MappingNode node) { + try { + return super.constructMapping(node); + } catch (IllegalStateException ex) { + throw new ParserException("while parsing MappingNode", + node.getStartMark(), + ex.getMessage(), + node.getEndMark()); + } + } + + @Override + protected Map createDefaultMap() { + final Map delegate = super.createDefaultMap(); + return new AbstractMap() { + + @Override + public Object put(Object key, Object value) { + if (delegate.containsKey(key)) { + throw new IllegalStateException("Duplicate key: " + key); + } + return delegate.put(key, value); + } + + @Override + public Set> entrySet() { + return delegate.entrySet(); + } + }; + } + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java new file mode 100644 index 00000000..936bdba6 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java @@ -0,0 +1,87 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Pattern; + +import org.springframework.util.ClassUtils; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.representer.Representer; +import org.yaml.snakeyaml.resolver.Resolver; + +/** + * Strategy to load '.yml' (or '.yaml') files into a {@link PropertySource}. + * + * @author Dave Syer + * @author Phillip Webb + * @author Andy Wilkinson + */ +public class YamlPropertySourceLoader implements PropertySourceLoader { + + @Override + public String[] getFileExtensions() { + return new String[] { "yml", "yaml" }; + } + + @Override + public PropertySource load(String name, Resource resource, String profile) throws IOException { + if (ClassUtils.isPresent("org.yaml.snakeyaml.Yaml", null)) { + Processor processor = new Processor(resource, profile); + Map source = processor.process(); + if (!source.isEmpty()) { + return new MapPropertySource(name, source); + } + } + return null; + } + + /** + * {@link YamlProcessor} to create a {@link Map} containing the property values. + * Similar to {@link YamlPropertiesFactoryBean} but retains the order of + * entries. + */ + private static class Processor extends YamlProcessor { + + Processor(Resource resource, String profile){ + if (profile == null) { + setMatchDefault(true); + setDocumentMatchers(new SpringProfileDocumentMatcher()); + } else { + setMatchDefault(false); + setDocumentMatchers(new SpringProfileDocumentMatcher(profile)); + } + setResources(resource); + } + + @Override + protected Yaml createYaml() { + return new Yaml(new StrictMapAppenderConstructor(), new Representer(), new DumperOptions(), new Resolver() { + + @Override + public void addImplicitResolver(Tag tag, Pattern regexp, String first) { + if (tag == Tag.TIMESTAMP) { + return; + } + super.addImplicitResolver(tag, regexp, first); + } + }); + } + + public Map process() { + final Map result = new LinkedHashMap(); + process(new MatchCallback() { + + @Override + public void process(Properties properties, Map map) { + result.putAll(getFlattenedMap(map)); + } + }); + return result; + } + + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java index 25b339fe..3574b678 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java @@ -1,30 +1,41 @@ package com.alibaba.otter.canal.client.adapter.support; -import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; -import java.util.Properties; +import java.util.Map; /** * 配置信息类 * - * @author machengyuan 2018-8-18 下午10:40:12 + * @author rewerma 2018-8-18 下午10:40:12 * @version 1.0.0 */ public class CanalClientConfig { - private String canalServerHost; + // 单机模式下canal server的ip:port + private String canalServerHost; + // 集群模式下的zk地址,如果配置了单机地址则以单机为准!! + private String zookeeperHosts; + // kafka or rocket mq 地址 + private String mqServers; + // 是否已flatMessage模式传输,只适用于mq模式 + private Boolean flatMessage = true; + // 批大小 + private Integer batchSize; + // 同步分批提交大小 + private Integer syncBatchSize = 1000; + // 重试次数 + private Integer retries; + // 消费超时时间 + private Long timeout; + // 模式 tcp kafka rocketMQ + private String mode = "tcp"; + // aliyun ak/sk + private String accessKey; + private String secretKey; - private String zookeeperHosts; - - private Properties properties; - - private String bootstrapServers; - - private Boolean flatMessage = true; - - private List kafkaTopics; - - private List canalInstances; + // canal adapters 配置 + private List canalAdapters; public String getCanalServerHost() { return canalServerHost; @@ -42,20 +53,12 @@ public class CanalClientConfig { this.zookeeperHosts = zookeeperHosts; } - public Properties getProperties() { - return properties; + public String getMqServers() { + return mqServers; } - public void setProperties(Properties properties) { - this.properties = properties; - } - - public String getBootstrapServers() { - return bootstrapServers; - } - - public void setBootstrapServers(String bootstrapServers) { - this.bootstrapServers = bootstrapServers; + public void setMqServers(String mqServers) { + this.mqServers = mqServers; } public Boolean getFlatMessage() { @@ -66,27 +69,75 @@ public class CanalClientConfig { this.flatMessage = flatMessage; } - public List getKafkaTopics() { - return kafkaTopics; + public Integer getBatchSize() { + return batchSize; } - public void setKafkaTopics(List kafkaTopics) { - this.kafkaTopics = kafkaTopics; + public void setBatchSize(Integer batchSize) { + this.batchSize = batchSize; } - public List getCanalInstances() { - return canalInstances; + public Integer getRetries() { + return retries; } - public void setCanalInstances(List canalInstances) { - this.canalInstances = canalInstances; + public Integer getSyncBatchSize() { + return syncBatchSize; } - public static class CanalInstance { + public void setSyncBatchSize(Integer syncBatchSize) { + this.syncBatchSize = syncBatchSize; + } - private String instance; + public void setRetries(Integer retries) { + this.retries = retries; + } - private List adapterGroups; + public Long getTimeout() { + return timeout; + } + + public void setTimeout(Long timeout) { + this.timeout = timeout; + } + + public String getMode() { + return mode; + } + + public void setMode(String mode) { + this.mode = mode; + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public List getCanalAdapters() { + return canalAdapters; + } + + public void setCanalAdapters(List canalAdapters) { + this.canalAdapters = canalAdapters; + } + + public static class CanalAdapter { + + private String instance; // 实例名 + + private List groups; // 适配器分组列表 public String getInstance() { return instance; @@ -98,42 +149,6 @@ public class CanalClientConfig { } } - public List getAdapterGroups() { - return adapterGroups; - } - - public void setAdapterGroups(List adapterGroups) { - this.adapterGroups = adapterGroups; - } - } - - public static class AdapterGroup { - - private List outAdapters; - - public List getOutAdapters() { - return outAdapters; - } - - public void setOutAdapters(List outAdapters) { - this.outAdapters = outAdapters; - } - } - - public static class KafkaTopic { - - private String topic; - - private List groups = new ArrayList<>(); - - public String getTopic() { - return topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - public List getGroups() { return groups; } @@ -145,11 +160,10 @@ public class CanalClientConfig { public static class Group { - private String groupId; - - // private List adapters = new ArrayList<>(); - - private List outAdapters; + // group id + private String groupId = "default"; + private List outerAdapters; // 适配器列表 + private Map outerAdaptersMap = new LinkedHashMap<>(); public String getGroupId() { return groupId; @@ -159,33 +173,20 @@ public class CanalClientConfig { this.groupId = groupId; } - public List getOutAdapters() { - return outAdapters; + public List getOuterAdapters() { + return outerAdapters; } - public void setOutAdapters(List outAdapters) { - this.outAdapters = outAdapters; + public void setOuterAdapters(List outerAdapters) { + this.outerAdapters = outerAdapters; } - // public List getAdapters() { - // return adapters; - // } - // - // public void setAdapters(List adapters) { - // this.adapters = adapters; - // } + public Map getOuterAdaptersMap() { + return outerAdaptersMap; + } + + public void setOuterAdaptersMap(Map outerAdaptersMap) { + this.outerAdaptersMap = outerAdaptersMap; + } } - - // public static class Adaptor { - // private List outAdapters; - // - // public List getOutAdapters() { - // return outAdapters; - // } - // - // public void setOutAdapters(List outAdapters) - // { - // this.outAdapters = outAdapters; - // } - // } } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalOuterAdapterConfiguration.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalOuterAdapterConfiguration.java deleted file mode 100644 index e6e34b67..00000000 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalOuterAdapterConfiguration.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.support; - -import java.util.Properties; - -/** - * 外部适配器配置信息类 - * - * @author machengyuan 2018-8-18 下午10:15:12 - * @version 1.0.0 - */ -public class CanalOuterAdapterConfiguration { - - private String name; // 适配器名称, 如: logger, hbase, es - - private String hosts; // 适配器内部的地址, 比如对应es该参数可以填写es的server地址 - - private String zkHosts; // 适配器内部的ZK地址, 比如对应HBase该参数可以填写HBase对应的ZK地址 - - private Properties properties; // 其余参数, 可填写适配器中的所需的配置信息 - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getHosts() { - return hosts; - } - - public void setHosts(String hosts) { - this.hosts = hosts; - } - - public Properties getProperties() { - return properties; - } - - public void setProperties(Properties properties) { - this.properties = properties; - } - - public String getZkHosts() { - return zkHosts; - } - - public void setZkHosts(String zkHosts) { - this.zkHosts = zkHosts; - } -} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Constant.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Constant.java new file mode 100644 index 00000000..9a75f765 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Constant.java @@ -0,0 +1,6 @@ +package com.alibaba.otter.canal.client.adapter.support; + +public class Constant { + + public static final String CONF_DIR = "conf"; +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/DatasourceConfig.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/DatasourceConfig.java new file mode 100644 index 00000000..acc07dd1 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/DatasourceConfig.java @@ -0,0 +1,81 @@ +package com.alibaba.otter.canal.client.adapter.support; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import com.alibaba.druid.pool.DruidDataSource; + +/** + * 数据源配置 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +public class DatasourceConfig { + + public final static Map DATA_SOURCES = new ConcurrentHashMap<>(); // key对应的数据源 + + private String driver = "com.mysql.jdbc.Driver"; // 默认为mysql jdbc驱动 + private String url; // jdbc url + private String database; // jdbc database + private String type = "mysql"; // 类型, 默认为mysql + private String username; // jdbc username + private String password; // jdbc password + private Integer maxActive = 3; // 连接池最大连接数,默认为3 + + public String getDriver() { + return driver; + } + + public void setDriver(String driver) { + this.driver = driver; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public Integer getMaxActive() { + return maxActive; + } + + public void setMaxActive(Integer maxActive) { + this.maxActive = maxActive; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java index fe7333e5..97204867 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java @@ -7,20 +7,43 @@ import java.util.Map; /** * DML操作转换对象 * - * @author machengyuan 2018-8-19 下午11:30:49 + * @author rewerma 2018-8-19 下午11:30:49 * @version 1.0.0 */ public class Dml implements Serializable { private static final long serialVersionUID = 2611556444074013268L; - private String database; - private String table; - private String type; - private Long ts; - private String sql; - private List> data; - private List> old; + private String destination; // 对应canal的实例或者MQ的topic + private String groupId; // 对应mq的group id + private String database; // 数据库或schema + private String table; // 表名 + private List pkNames; + private Boolean isDdl; + private String type; // 类型: INSERT UPDATE DELETE + // binlog executeTime + private Long es; // 执行耗时 + // dml build timeStamp + private Long ts; // 同步时间 + private String sql; // 执行的sql, dml sql为空 + private List> data; // 数据列表 + private List> old; // 旧数据列表, 用于update, size和data的size一一对应 + + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } public String getDatabase() { return database; @@ -38,6 +61,22 @@ public class Dml implements Serializable { this.table = table; } + public List getPkNames() { + return pkNames; + } + + public void setPkNames(List pkNames) { + this.pkNames = pkNames; + } + + public Boolean getIsDdl() { + return isDdl; + } + + public void setIsDdl(Boolean isDdl) { + this.isDdl = isDdl; + } + public String getType() { return type; } @@ -78,11 +117,20 @@ public class Dml implements Serializable { this.old = old; } + public Long getEs() { + return es; + } + + public void setEs(Long es) { + this.es = es; + } + public void clear() { database = null; table = null; type = null; ts = null; + es = null; data = null; old = null; sql = null; @@ -90,7 +138,8 @@ public class Dml implements Serializable { @Override public String toString() { - return "Dml{" + "database='" + database + '\'' + ", table='" + table + '\'' + ", type='" + type + '\'' + ", ts=" - + ts + ", sql='" + sql + '\'' + ", data=" + data + ", old=" + old + '}'; + return "Dml{" + "destination='" + destination + '\'' + ", database='" + database + '\'' + ", table='" + table + + '\'' + ", type='" + type + '\'' + ", es=" + es + ", ts=" + ts + ", sql='" + sql + '\'' + ", data=" + + data + ", old=" + old + '}'; } } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/EtlResult.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/EtlResult.java new file mode 100644 index 00000000..498af7a3 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/EtlResult.java @@ -0,0 +1,44 @@ +package com.alibaba.otter.canal.client.adapter.support; + +import java.io.Serializable; + +/** + * ETL的结果对象 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +public class EtlResult implements Serializable { + + private static final long serialVersionUID = 4250522736289866505L; + + private boolean succeeded = false; + + private String resultMessage; + + private String errorMessage; + + public boolean getSucceeded() { + return succeeded; + } + + public void setSucceeded(boolean succeeded) { + this.succeeded = succeeded; + } + + public String getResultMessage() { + return resultMessage; + } + + public void setResultMessage(String resultMessage) { + this.resultMessage = resultMessage; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java index 01b05054..41d1864e 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java @@ -1,28 +1,33 @@ package com.alibaba.otter.canal.client.adapter.support; -import java.io.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; import java.nio.file.Paths; -import java.util.*; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.regex.Pattern; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * SPI 类加载器 * - * @author machengyuan 2018-8-19 下午11:30:49 + * @author rewerma 2018-8-19 下午11:30:49 * @version 1.0.0 */ public class ExtensionLoader { - private static final Logger logger = LoggerFactory - .getLogger(ExtensionLoader.class); + private static final Logger logger = LoggerFactory.getLogger(ExtensionLoader.class); private static final String SERVICES_DIRECTORY = "META-INF/services/"; @@ -30,13 +35,14 @@ public class ExtensionLoader { private static final String DEFAULT_CLASSLOADER_POLICY = "internal"; - private static final Pattern NAME_SEPARATOR = Pattern - .compile("\\s*[,]+\\s*"); + private static final Pattern NAME_SEPARATOR = Pattern.compile("\\s*[,]+\\s*"); private static final ConcurrentMap, ExtensionLoader> EXTENSION_LOADERS = new ConcurrentHashMap<>(); private static final ConcurrentMap, Object> EXTENSION_INSTANCES = new ConcurrentHashMap<>(); + private static final ConcurrentMap EXTENSION_KEY_INSTANCE = new ConcurrentHashMap<>(); + private final Class type; private final String classLoaderPolicy; @@ -83,47 +89,6 @@ public class ExtensionLoader { this.classLoaderPolicy = classLoaderPolicy; } - public String getExtensionName(T extensionInstance) { - return getExtensionName(extensionInstance.getClass()); - } - - public String getExtensionName(Class extensionClass) { - return cachedNames.get(extensionClass); - } - - public ConcurrentHashMap getExceptions() { - return exceptions; - } - - /** - * 返回扩展点实例，如果没有指定的扩展点或是还没加载（即实例化）则返回null注意：此方法不会触发扩展点的加载 - *

    - * 一般应该调用{@link #getExtension(String)}方法获得扩展，这个方法会触发扩展点加载 - * - * @see #getExtension(String) - */ - @SuppressWarnings("unchecked") - public T getLoadedExtension(String name) { - if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null"); - Holder holder = cachedInstances.get(name); - if (holder == null) { - cachedInstances.putIfAbsent(name, new Holder<>()); - holder = cachedInstances.get(name); - } - return (T) holder.get(); - } - - /** - * 返回已经加载的扩展点的名字 - *

    - * 一般应该调用{@link #getSupportedExtensions()}方法获得扩展，这个方法会返回所有的扩展点 - * - * @see #getSupportedExtensions() - */ - public Set getLoadedExtensions() { - return Collections.unmodifiableSet(new TreeSet<>(cachedInstances.keySet())); - } - /** * 返回指定名字的扩展 * @@ -154,6 +119,31 @@ public class ExtensionLoader { return (T) instance; } + @SuppressWarnings("unchecked") + public T getExtension(String name, String key) { + if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null"); + if ("true".equals(name)) { + return getDefaultExtension(); + } + String extKey = name + "-" + StringUtils.trimToEmpty(key); + Holder holder = cachedInstances.get(extKey); + if (holder == null) { + cachedInstances.putIfAbsent(extKey, new Holder<>()); + holder = cachedInstances.get(extKey); + } + Object instance = holder.get(); + if (instance == null) { + synchronized (holder) { + instance = holder.get(); + if (instance == null) { + instance = createExtension(name, key); + holder.set(instance); + } + } + } + return (T) instance; + } + /** * 返回缺省的扩展，如果没有设置则返回null */ @@ -165,87 +155,6 @@ public class ExtensionLoader { return getExtension(cachedDefaultName); } - public boolean hasExtension(String name) { - if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null"); - try { - return getExtensionClass(name) != null; - } catch (Throwable t) { - return false; - } - } - - public Set getSupportedExtensions() { - Map> clazzes = getExtensionClasses(); - return Collections.unmodifiableSet(new TreeSet(clazzes.keySet())); - } - - /** - * 返回缺省的扩展点名，如果没有设置缺省则返回null - */ - public String getDefaultExtensionName() { - getExtensionClasses(); - return cachedDefaultName; - } - - /** - * 编程方式添加新扩展点 - * - * @param name 扩展点名 - * @param clazz 扩展点类 - * @throws IllegalStateException 要添加扩展点名已经存在 - */ - public void addExtension(String name, Class clazz) { - getExtensionClasses(); // load classes - - if (!type.isAssignableFrom(clazz)) { - throw new IllegalStateException("Input type " + clazz + "not implement Extension " + type); - } - if (clazz.isInterface()) { - throw new IllegalStateException("Input type " + clazz + "can not be interface!"); - } - - if (name == null || "".equals(name)) { - throw new IllegalStateException("Extension name is blank (Extension " + type + ")!"); - } - if (cachedClasses.get().containsKey(name)) { - throw new IllegalStateException("Extension name " + name + " already existed(Extension " + type + ")!"); - } - - cachedNames.put(clazz, name); - cachedClasses.get().put(name, clazz); - } - - /** - * 编程方式添加替换已有扩展点 - * - * @param name 扩展点名 - * @param clazz 扩展点类 - * @throws IllegalStateException 要添加扩展点名已经存在 - * @deprecated 不推荐应用使用，一般只在测试时可以使用 - */ - @Deprecated - public void replaceExtension(String name, Class clazz) { - getExtensionClasses(); // load classes - - if (!type.isAssignableFrom(clazz)) { - throw new IllegalStateException("Input type " + clazz + "not implement Extension " + type); - } - if (clazz.isInterface()) { - throw new IllegalStateException("Input type " + clazz + "can not be interface!"); - } - - if (name == null || "".equals(name)) { - throw new IllegalStateException("Extension name is blank (Extension " + type + ")!"); - } - if (!cachedClasses.get().containsKey(name)) { - throw new IllegalStateException("Extension name " + name + " not existed(Extension " + type + ")!"); - } - - cachedNames.put(clazz, name); - cachedClasses.get().put(name, clazz); - cachedInstances.remove(name); - } - @SuppressWarnings("unchecked") private T createExtension(String name) { Class clazz = getExtensionClasses().get(name); @@ -262,18 +171,28 @@ public class ExtensionLoader { return instance; } catch (Throwable t) { throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type - + ") could not be instantiated: " + t.getMessage(), - t); + + ") could not be instantiated: " + t.getMessage(), t); } } - private Class getExtensionClass(String name) { - if (type == null) throw new IllegalArgumentException("Extension type == null"); - if (name == null) throw new IllegalArgumentException("Extension name == null"); + @SuppressWarnings("unchecked") + private T createExtension(String name, String key) { Class clazz = getExtensionClasses().get(name); - if (clazz == null) - throw new IllegalStateException("No such extension \"" + name + "\" for " + type.getName() + "!"); - return clazz; + if (clazz == null) { + throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type + + ") could not be instantiated: class could not be found"); + } + try { + T instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key); + if (instance == null) { + EXTENSION_KEY_INSTANCE.putIfAbsent(name + "-" + key, clazz.newInstance()); + instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key); + } + return instance; + } catch (Throwable t) { + throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type + + ") could not be instantiated: " + t.getMessage(), t); + } } private Map> getExtensionClasses() { @@ -287,19 +206,30 @@ public class ExtensionLoader { } } } + return classes; } private String getJarDirectoryPath() { URL url = Thread.currentThread().getContextClassLoader().getResource(""); - if (url == null) { - throw new IllegalStateException("failed to get class loader resource"); + String dirtyPath; + if (url != null) { + dirtyPath = url.toString(); + } else { + File file = new File(""); + dirtyPath = file.getAbsolutePath(); } - String dirtyPath = url.toString(); - String jarPath = dirtyPath.replaceAll("^.*file:/", ""); // removes file:/ and everything before it - jarPath = jarPath.replaceAll("jar!.*", "jar"); // removes everything after .jar, if .jar exists in dirtyPath - jarPath = jarPath.replaceAll("%20", " "); // necessary if path has spaces within - if (!jarPath.endsWith(".jar")) { // this is needed if you plan to run the app using Spring Tools Suit play + String jarPath = dirtyPath.replaceAll("^.*file:/", ""); // removes + // file:/ and + // everything + // before it + jarPath = jarPath.replaceAll("jar!.*", "jar"); // removes everything + // after .jar, if .jar + // exists in dirtyPath + jarPath = jarPath.replaceAll("%20", " "); // necessary if path has + // spaces within + if (!jarPath.endsWith(".jar")) { // this is needed if you plan to run + // the app using Spring Tools Suit play // button. jarPath = jarPath.replaceAll("/classes/.*", "/classes/"); } @@ -322,25 +252,20 @@ public class ExtensionLoader { Map> extensionClasses = new HashMap>(); - // 1. lib folder，customized extension classLoader （jar_dir/lib） - String dir = File.separator + this.getJarDirectoryPath() + File.separator + "lib"; - logger.info("extension classpath dir: " + dir); + // 1. plugin folder，customized extension classLoader （jar_dir/plugin） + String dir = File.separator + this.getJarDirectoryPath() + File.separator + "plugin"; + File externalLibDir = new File(dir); if (!externalLibDir.exists()) { - externalLibDir = new File( - File.separator + this.getJarDirectoryPath() + File.separator + "canal_client" + File.separator + "lib"); + externalLibDir = new File(File.separator + this.getJarDirectoryPath() + File.separator + "canal-adapter" + + File.separator + "plugin"); } + logger.info("extension classpath dir: " + externalLibDir.getAbsolutePath()); if (externalLibDir.exists()) { - File[] files = externalLibDir.listFiles(new FilenameFilter() { - - @Override - public boolean accept(File dir, String name) { - return name.endsWith(".jar"); - } - }); + File[] files = externalLibDir.listFiles((dir1, name) -> name.endsWith(".jar")); if (files != null) { for (File f : files) { - URL url = null; + URL url; try { url = f.toURI().toURL(); } catch (MalformedURLException e) { @@ -351,47 +276,7 @@ public class ExtensionLoader { URLClassLoader localClassLoader; if (classLoaderPolicy == null || "".equals(classLoaderPolicy) || DEFAULT_CLASSLOADER_POLICY.equalsIgnoreCase(classLoaderPolicy)) { - localClassLoader = new URLClassLoader(new URL[] { url }, parent) { - - @Override - public Class loadClass(String name) throws ClassNotFoundException { - Class c = findLoadedClass(name); - if (c != null) { - return c; - } - - if (name.startsWith("java.") || name.startsWith("org.slf4j.") - || name.startsWith("org.apache.logging") - || name.startsWith("org.apache.commons.logging.")) { - // || name.startsWith("org.apache.hadoop.")) { - c = super.loadClass(name); - } - if (c != null) return c; - - try { - // 先加载jar内的class，可避免jar冲突 - c = findClass(name); - } catch (ClassNotFoundException e) { - c = null; - } - if (c != null) { - return c; - } - - return super.loadClass(name); - } - - @Override - public Enumeration getResources(String name) throws IOException { - @SuppressWarnings("unchecked") - Enumeration[] tmp = (Enumeration[]) new Enumeration[2]; - - tmp[0] = findResources(name); // local class path first - // tmp[1] = super.getResources(name); - - return new CompoundEnumeration<>(tmp); - } - }; + localClassLoader = new URLClassExtensionLoader(new URL[] { url }); } else { localClassLoader = new URLClassLoader(new URL[] { url }, parent); } @@ -401,48 +286,15 @@ public class ExtensionLoader { } } } + // 只加载外部spi, 不加载classpath // 2. load inner extension class with default classLoader - ClassLoader classLoader = findClassLoader(); - loadFile(extensionClasses, CANAL_DIRECTORY, classLoader); - loadFile(extensionClasses, SERVICES_DIRECTORY, classLoader); + // ClassLoader classLoader = findClassLoader(); + // loadFile(extensionClasses, CANAL_DIRECTORY, classLoader); + // loadFile(extensionClasses, SERVICES_DIRECTORY, classLoader); return extensionClasses; } - public static class CompoundEnumeration implements Enumeration { - - private Enumeration[] enums; - private int index = 0; - - public CompoundEnumeration(Enumeration[] enums){ - this.enums = enums; - } - - private boolean next() { - while (this.index < this.enums.length) { - if (this.enums[this.index] != null && this.enums[this.index].hasMoreElements()) { - return true; - } - - ++this.index; - } - - return false; - } - - public boolean hasMoreElements() { - return this.next(); - } - - public E nextElement() { - if (!this.next()) { - throw new NoSuchElementException(); - } else { - return this.enums[this.index].nextElement(); - } - } - } - private void loadFile(Map> extensionClasses, String dir, ClassLoader classLoader) { String fileName = dir + type.getName(); try { @@ -474,12 +326,16 @@ public class ExtensionLoader { } if (line.length() > 0) { Class clazz = classLoader.loadClass(line); - // Class clazz = Class.forName(line, true, classLoader); + // Class clazz = + // Class.forName(line, true, + // classLoader); if (!type.isAssignableFrom(clazz)) { - throw new IllegalStateException( - "Error when load extension class(interface: " + type - + ", class line: " + clazz.getName() - + "), class " + clazz.getName() + throw new IllegalStateException("Error when load extension class(interface: " + + type + + ", class line: " + + clazz.getName() + + "), class " + + clazz.getName() + "is not subtype of interface."); } else { try { @@ -497,9 +353,9 @@ public class ExtensionLoader { extensionClasses.put(n, clazz); } else if (c != clazz) { cachedNames.remove(clazz); - throw new IllegalStateException( - "Duplicate extension " + type.getName() + " name " - + n + " on " + throw new IllegalStateException("Duplicate extension " + + type.getName() + + " name " + n + " on " + c.getName() + " and " + clazz.getName()); } @@ -509,9 +365,12 @@ public class ExtensionLoader { } } } catch (Throwable t) { - IllegalStateException e = new IllegalStateException( - "Failed to load extension class(interface: " + type + ", class line: " - + line + ") in " + url + IllegalStateException e = new IllegalStateException("Failed to load extension class(interface: " + + type + + ", class line: " + + line + + ") in " + + url + ", cause: " + t.getMessage(), t); @@ -526,18 +385,17 @@ public class ExtensionLoader { } } catch (Throwable t) { logger.error("Exception when load extension class(interface: " + type + ", class file: " + url - + ") in " + url, - t); + + ") in " + url, t); } } // end of while urls } } catch (Throwable t) { - logger.error( - "Exception when load extension class(interface: " + type + ", description file: " + fileName + ").", - t); + logger.error("Exception when load extension class(interface: " + type + ", description file: " + fileName + + ").", t); } } + @SuppressWarnings("unused") private static ClassLoader findClassLoader() { return ExtensionLoader.class.getClassLoader(); } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java index 777770ef..c24ff7de 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java @@ -2,10 +2,7 @@ package com.alibaba.otter.canal.client.adapter.support; import java.math.BigDecimal; import java.math.BigInteger; -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; -import java.sql.Types; +import java.sql.*; import org.joda.time.DateTime; import org.slf4j.Logger; @@ -14,13 +11,62 @@ import org.slf4j.LoggerFactory; /** * 类型转换工具类 * - * @author machengyuan 2018-8-19 下午06:14:23 + * @author rewerma 2018-8-19 下午06:14:23 * @version 1.0.0 */ public class JdbcTypeUtil { private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class); + public static Object getRSData(ResultSet rs, String columnName, int jdbcType) throws SQLException { + if (jdbcType == Types.BIT || jdbcType == Types.BOOLEAN) { + return rs.getByte(columnName); + } else { + return rs.getObject(columnName); + } + } + + public static Class jdbcType2javaType(int jdbcType) { + switch (jdbcType) { + case Types.BIT: + case Types.BOOLEAN: + // return Boolean.class; + case Types.TINYINT: + return Byte.TYPE; + case Types.SMALLINT: + return Short.class; + case Types.INTEGER: + return Integer.class; + case Types.BIGINT: + return Long.class; + case Types.DECIMAL: + case Types.NUMERIC: + return BigDecimal.class; + case Types.REAL: + return Float.class; + case Types.FLOAT: + case Types.DOUBLE: + return Double.class; + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + return String.class; + case Types.BINARY: + case Types.VARBINARY: + case Types.LONGVARBINARY: + case Types.BLOB: + return byte[].class; + case Types.DATE: + return java.sql.Date.class; + case Types.TIME: + return Time.class; + case Types.TIMESTAMP: + return Timestamp.class; + default: + return String.class; + } + } + public static Object typeConvert(String columnName, String value, int sqlType, String mysqlType) { if (value == null || value.equals("")) { return null; diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MappingConfigsLoader.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MappingConfigsLoader.java new file mode 100644 index 00000000..a7c8d7d6 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MappingConfigsLoader.java @@ -0,0 +1,71 @@ +package com.alibaba.otter.canal.client.adapter.support; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +public class MappingConfigsLoader { + + public static Map loadConfigs(String name) { + Map configContentMap = new HashMap<>(); + + // 先取本地文件，再取类路径 + File configDir = new File(".." + File.separator + Constant.CONF_DIR + File.separator + name); + if (!configDir.exists()) { + URL url = MappingConfigsLoader.class.getClassLoader().getResource(""); + if (url != null) { + configDir = new File(url.getPath() + name + File.separator); + } + } + + File[] files = configDir.listFiles(); + if (files != null) { + for (File file : files) { + String fileName = file.getName(); + if (!fileName.endsWith(".yml")) { + continue; + } + try (InputStream in = new FileInputStream(file)) { + byte[] bytes = new byte[in.available()]; + in.read(bytes); + String configContent = new String(bytes, StandardCharsets.UTF_8); + configContentMap.put(fileName, configContent); + } catch (IOException e) { + throw new RuntimeException("Read " + name + "mapping config: " + fileName + " error. ", e); + } + } + } + + return configContentMap; + } + + public static String loadConfig(String name) { + // 先取本地文件，再取类路径 + File filePath = new File(".." + File.separator + Constant.CONF_DIR + File.separator + name); + if (!filePath.exists()) { + URL url = MappingConfigsLoader.class.getClassLoader().getResource(""); + if (url != null) { + filePath = new File(url.getPath() + name); + } + } + if (filePath.exists()) { + String fileName = filePath.getName(); + if (!fileName.endsWith(".yml")) { + return null; + } + try (InputStream in = new FileInputStream(filePath)) { + byte[] bytes = new byte[in.available()]; + in.read(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException("Read mapping config: " + filePath.getAbsolutePath() + " error. ", e); + } + } + return null; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java index 4765ec30..93ca987b 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java @@ -9,17 +9,17 @@ import com.alibaba.otter.canal.protocol.Message; /** * Message对象解析工具类 * - * @author machengyuan 2018-8-19 下午06:14:23 + * @author rewerma 2018-8-19 下午06:14:23 * @version 1.0.0 */ public class MessageUtil { - public static void parse4Dml(Message message, Consumer consumer) { + public static List parse4Dml(String destination, String groupId, Message message) { if (message == null) { - return; + return null; } List entries = message.getEntries(); - + List dmls = new ArrayList(entries.size()); for (CanalEntry.Entry entry : entries) { if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) { @@ -37,16 +37,23 @@ public class MessageUtil { CanalEntry.EventType eventType = rowChange.getEventType(); final Dml dml = new Dml(); + dml.setDestination(destination); + dml.setGroupId(groupId); dml.setDatabase(entry.getHeader().getSchemaName()); dml.setTable(entry.getHeader().getTableName()); dml.setType(eventType.toString()); + dml.setEs(entry.getHeader().getExecuteTime()); + dml.setIsDdl(rowChange.getIsDdl()); dml.setTs(System.currentTimeMillis()); dml.setSql(rowChange.getSql()); + dmls.add(dml); List> data = new ArrayList<>(); List> old = new ArrayList<>(); if (!rowChange.getIsDdl()) { Set updateSet = new HashSet<>(); + dml.setPkNames(new ArrayList<>()); + int i = 0; for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) { if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE && eventType != CanalEntry.EventType.DELETE) { @@ -63,6 +70,11 @@ public class MessageUtil { } for (CanalEntry.Column column : columns) { + if (i == 0) { + if (column.getIsKey()) { + dml.getPkNames().add(column.getName()); + } + } row.put(column.getName(), JdbcTypeUtil.typeConvert(column.getName(), column.getValue(), @@ -93,6 +105,8 @@ public class MessageUtil { old.add(rowOld); } } + + i++; } if (!data.isEmpty()) { dml.setData(data); @@ -100,24 +114,42 @@ public class MessageUtil { if (!old.isEmpty()) { dml.setOld(old); } - consumer.accept(dml); } } + + return dmls; } - public static Dml flatMessage2Dml(FlatMessage flatMessage) { + public static List flatMessage2Dml(String destination, String groupId, List flatMessages) { + List dmls = new ArrayList(flatMessages.size()); + for (FlatMessage flatMessage : flatMessages) { + Dml dml = flatMessage2Dml(destination, groupId, flatMessage); + if (dml != null) { + dmls.add(dml); + } + } + + return dmls; + } + + public static Dml flatMessage2Dml(String destination, String groupId, FlatMessage flatMessage) { if (flatMessage == null) { return null; } Dml dml = new Dml(); + dml.setDestination(destination); + dml.setGroupId(groupId); dml.setDatabase(flatMessage.getDatabase()); dml.setTable(flatMessage.getTable()); + dml.setPkNames(flatMessage.getPkNames()); + dml.setIsDdl(flatMessage.getIsDdl()); dml.setType(flatMessage.getType()); dml.setTs(flatMessage.getTs()); + dml.setEs(flatMessage.getEs()); dml.setSql(flatMessage.getSql()); - if (flatMessage.getSqlType() == null || flatMessage.getMysqlType() == null) { - throw new RuntimeException("SqlType or mysqlType is null"); - } + // if (flatMessage.getSqlType() == null || flatMessage.getMysqlType() == null) { + // throw new RuntimeException("SqlType or mysqlType is null"); + // } List> data = flatMessage.getData(); if (data != null) { dml.setData(changeRows(data, flatMessage.getSqlType(), flatMessage.getMysqlType())); @@ -155,9 +187,4 @@ public class MessageUtil { } return result; } - - public interface Consumer { - - void accept(T t); - } } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/OuterAdapterConfig.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/OuterAdapterConfig.java new file mode 100644 index 00000000..a767b203 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/OuterAdapterConfig.java @@ -0,0 +1,62 @@ +package com.alibaba.otter.canal.client.adapter.support; + +import java.util.Map; + +/** + * 外部适配器配置信息类 + * + * @author rewerma 2018-8-18 下午10:15:12 + * @version 1.0.0 + */ +public class OuterAdapterConfig { + + private String name; // 适配器名称, 如: logger, hbase, es + + private String key; // 适配器唯一键 + + private String hosts; // 适配器内部的地址, 比如对应es该参数可以填写es的server地址 + + private String zkHosts; // 适配器内部的ZK地址, 比如对应HBase该参数可以填写HBase对应的ZK地址 + + private Map properties; // 其余参数, 可填写适配器中的所需的配置信息 + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getHosts() { + return hosts; + } + + public void setHosts(String hosts) { + this.hosts = hosts; + } + + public Map getProperties() { + return properties; + } + + public void setProperties(Map properties) { + this.properties = properties; + } + + public String getZkHosts() { + return zkHosts; + } + + public void setZkHosts(String zkHosts) { + this.zkHosts = zkHosts; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java new file mode 100644 index 00000000..6e9a9954 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java @@ -0,0 +1,57 @@ +package com.alibaba.otter.canal.client.adapter.support; + +import java.io.Serializable; +import java.util.Date; + +/** + * 用于rest的结果返回对象 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +public class Result implements Serializable { + + private static final long serialVersionUID = -3276409502352405716L; + public Integer code = 20000; + public Object data; + public String message; + public Date sysTime; + + public static Result createSuccess(String message) { + Result result = new Result(); + result.setMessage(message); + return result; + } + + public Integer getCode() { + return code; + } + + public void setCode(Integer code) { + this.code = code; + } + + public Object getData() { + return data; + } + + public void setData(Object data) { + this.data = data; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public Date getSysTime() { + return sysTime; + } + + public void setSysTime(Date sysTime) { + this.sysTime = sysTime; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/SPI.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/SPI.java index 9c0116c6..b4f89b6a 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/SPI.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/SPI.java @@ -1,7 +1,17 @@ package com.alibaba.otter.canal.client.adapter.support; -import java.lang.annotation.*; +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +/** + * SPI装载器注解 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ @Documented @Retention(RetentionPolicy.RUNTIME) @Target({ ElementType.TYPE }) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/URLClassExtensionLoader.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/URLClassExtensionLoader.java new file mode 100644 index 00000000..725d23e8 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/URLClassExtensionLoader.java @@ -0,0 +1,88 @@ +package com.alibaba.otter.canal.client.adapter.support; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Enumeration; +import java.util.NoSuchElementException; + +public class URLClassExtensionLoader extends URLClassLoader { + public URLClassExtensionLoader(URL[] urls) { + super(urls); + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + Class c = findLoadedClass(name); + if (c != null) { + return c; + } + + if (name.startsWith("java.") || name.startsWith("org.slf4j.") + || name.startsWith("org.apache.logging") + || name.startsWith("org.apache.commons.logging.")) { + // || name.startsWith("org.apache.hadoop.")) + // { + c = super.loadClass(name); + } + if (c != null) return c; + + try { + // 先加载jar内的class，可避免jar冲突 + c = findClass(name); + } catch (ClassNotFoundException e) { + c = null; + } + if (c != null) { + return c; + } + + return super.loadClass(name); + } + + @Override + public Enumeration getResources(String name) throws IOException { + @SuppressWarnings("unchecked") + Enumeration[] tmp = (Enumeration[]) new Enumeration[2]; + + tmp[0] = findResources(name); // local class + // path first + // tmp[1] = super.getResources(name); + + return new CompoundEnumeration<>(tmp); + } + + private static class CompoundEnumeration implements Enumeration { + + private Enumeration[] enums; + private int index = 0; + + public CompoundEnumeration(Enumeration[] enums){ + this.enums = enums; + } + + private boolean next() { + while (this.index < this.enums.length) { + if (this.enums[this.index] != null && this.enums[this.index].hasMoreElements()) { + return true; + } + + ++this.index; + } + + return false; + } + + public boolean hasMoreElements() { + return this.next(); + } + + public E nextElement() { + if (!this.next()) { + throw new NoSuchElementException(); + } else { + return this.enums[this.index].nextElement(); + } + } + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java new file mode 100644 index 00000000..919974a1 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java @@ -0,0 +1,139 @@ +package com.alibaba.otter.canal.client.adapter.support; + +import java.io.File; +import java.net.URL; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.function.Consumer; +import java.util.function.Function; + +import javax.sql.DataSource; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Util { + + private static final Logger logger = LoggerFactory.getLogger(Util.class); + + /** + * 通过DS执行sql + */ + public static Object sqlRS(DataSource ds, String sql, Function fun) throws SQLException { + Connection conn = null; + Statement stmt = null; + ResultSet rs = null; + try { + conn = ds.getConnection(); + stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + stmt.setFetchSize(Integer.MIN_VALUE); + rs = stmt.executeQuery(sql); + return fun.apply(rs); + } finally { + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + } + } + + /** + * sql执行获取resultSet + * + * @param conn sql connection + * @param sql sql + * @param consumer 回调方法 + */ + public static void sqlRS(Connection conn, String sql, Consumer consumer) { + Statement stmt = null; + ResultSet rs = null; + try { + stmt = conn.createStatement(); + rs = stmt.executeQuery(sql); + consumer.accept(rs); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } finally { + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + } + } + + public static File getConfDirPath() { + return getConfDirPath(""); + } + + public static File getConfDirPath(String subConf) { + URL url = Util.class.getClassLoader().getResource(""); + String path; + if (url != null) { + path = url.getPath(); + } else { + path = new File("").getAbsolutePath(); + } + File file = null; + if (path != null) { + file = new File( + path + ".." + File.separator + Constant.CONF_DIR + File.separator + StringUtils.trimToEmpty(subConf)); + if (!file.exists()) { + file = new File(path + StringUtils.trimToEmpty(subConf)); + } + } + if (file == null || !file.exists()) { + throw new RuntimeException("Config dir not found."); + } + + return file; + } + + public static String cleanColumn(String column) { + if (column == null) { + return null; + } + if (column.contains("`")) { + column = column.replaceAll("`", ""); + } + + if (column.contains("'")) { + column = column.replaceAll("'", ""); + } + + if (column.contains("\"")) { + column = column.replaceAll("\"", ""); + } + + return column; + } +} diff --git a/client-adapter/elasticsearch/pom.xml b/client-adapter/elasticsearch/pom.xml new file mode 100644 index 00000000..90c64f80 --- /dev/null +++ b/client-adapter/elasticsearch/pom.xml @@ -0,0 +1,89 @@ + + + + canal.client-adapter + com.alibaba.otter + 1.1.3-SNAPSHOT + + 4.0.0 + com.alibaba.otter + client-adapter.elasticsearch + jar + canal client adapter elasticsearch module for otter ${project.version} + + + + com.alibaba.otter + client-adapter.common + ${project.version} + provided + + + com.alibaba.fastsql + fastsql + 2.0.0_preview_644 + + + org.elasticsearch + elasticsearch + 6.2.3 + + + org.elasticsearch.client + transport + 6.2.3 + + + + junit + junit + 4.12 + test + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + maven-antrun-plugin + + + package + + run + + + + + + + + + + + + + + + + + diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java new file mode 100644 index 00000000..edd651f9 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -0,0 +1,249 @@ +package com.alibaba.otter.canal.client.adapter.es; + +import java.net.InetAddress; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.sql.DataSource; + +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.transport.client.PreBuiltTransportClient; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SqlParser; +import com.alibaba.otter.canal.client.adapter.es.monitor.ESConfigMonitor; +import com.alibaba.otter.canal.client.adapter.es.service.ESEtlService; +import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService; +import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; +import com.alibaba.otter.canal.client.adapter.support.*; + +/** + * ES外部适配器 + * + * @author rewerma 2018-10-20 + * @version 1.0.0 + */ +@SPI("es") +public class ESAdapter implements OuterAdapter { + + private Map esSyncConfig = new ConcurrentHashMap<>(); // 文件名对应配置 + private Map> dbTableEsSyncConfig = new ConcurrentHashMap<>(); // schema-table对应配置 + + private TransportClient transportClient; + + private ESSyncService esSyncService; + + private ESConfigMonitor esConfigMonitor; + + private Properties envProperties; + + public TransportClient getTransportClient() { + return transportClient; + } + + public ESSyncService getEsSyncService() { + return esSyncService; + } + + public Map getEsSyncConfig() { + return esSyncConfig; + } + + public Map> getDbTableEsSyncConfig() { + return dbTableEsSyncConfig; + } + + @Override + public void init(OuterAdapterConfig configuration, Properties envProperties) { + try { + this.envProperties = envProperties; + Map esSyncConfigTmp = ESSyncConfigLoader.load(envProperties); + // 过滤不匹配的key的配置 + esSyncConfigTmp.forEach((key, config) -> { + if ((config.getOuterAdapterKey() == null && configuration.getKey() == null) + || (config.getOuterAdapterKey() != null + && config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) { + esSyncConfig.put(key, config); + } + }); + + for (Map.Entry entry : esSyncConfig.entrySet()) { + String configName = entry.getKey(); + ESSyncConfig config = entry.getValue(); + SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql()); + config.getEsMapping().setSchemaItem(schemaItem); + + DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (dataSource == null || dataSource.getUrl() == null) { + throw new RuntimeException("No data source found: " + config.getDataSourceKey()); + } + Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$"); + Matcher matcher = pattern.matcher(dataSource.getUrl()); + if (!matcher.find()) { + throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey()); + } + String schema = matcher.group(2); + + schemaItem.getAliasTableItems().values().forEach(tableItem -> { + Map esSyncConfigMap; + if (envProperties != null + && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + esSyncConfigMap = dbTableEsSyncConfig + .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "-" + + StringUtils.trimToEmpty(config.getGroupId()) + "_" + schema + "-" + + tableItem.getTableName(), + k -> new ConcurrentHashMap<>()); + } else { + esSyncConfigMap = dbTableEsSyncConfig + .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "_" + schema + "-" + + tableItem.getTableName(), + k -> new ConcurrentHashMap<>()); + } + + esSyncConfigMap.put(configName, config); + }); + } + + Map properties = configuration.getProperties(); + Settings.Builder settingBuilder = Settings.builder(); + properties.forEach(settingBuilder::put); + Settings settings = settingBuilder.build(); + transportClient = new PreBuiltTransportClient(settings); + String[] hostArray = configuration.getHosts().split(","); + for (String host : hostArray) { + int i = host.indexOf(":"); + transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName(host.substring(0, i)), + Integer.parseInt(host.substring(i + 1)))); + } + ESTemplate esTemplate = new ESTemplate(transportClient); + esSyncService = new ESSyncService(esTemplate); + + esConfigMonitor = new ESConfigMonitor(); + esConfigMonitor.init(this); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public void sync(List dmls) { + if (dmls == null || dmls.isEmpty()) { + return; + } + for (Dml dml : dmls) { + sync(dml); + } + } + + public void sync(Dml dml) { + String database = dml.getDatabase(); + String table = dml.getTable(); + Map configMap; + if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + configMap = dbTableEsSyncConfig + .get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId()) + + "_" + database + "-" + table); + } else { + configMap = dbTableEsSyncConfig + .get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-" + table); + } + + if (configMap != null && !configMap.values().isEmpty()) { + esSyncService.sync(configMap.values(), dml); + } + } + + @Override + public EtlResult etl(String task, List params) { + EtlResult etlResult = new EtlResult(); + ESSyncConfig config = esSyncConfig.get(task); + if (config != null) { + DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + ESEtlService esEtlService = new ESEtlService(transportClient, config); + if (dataSource != null) { + return esEtlService.importData(params, false); + } else { + etlResult.setSucceeded(false); + etlResult.setErrorMessage("DataSource not found"); + return etlResult; + } + } else { + StringBuilder resultMsg = new StringBuilder(); + boolean resSuccess = true; + // ds不为空说明传入的是datasourceKey + for (ESSyncConfig configTmp : esSyncConfig.values()) { + // 取所有的destination为task的配置 + if (configTmp.getDestination().equals(task)) { + ESEtlService esEtlService = new ESEtlService(transportClient, configTmp); + EtlResult etlRes = esEtlService.importData(params, false); + if (!etlRes.getSucceeded()) { + resSuccess = false; + resultMsg.append(etlRes.getErrorMessage()).append("\n"); + } else { + resultMsg.append(etlRes.getResultMessage()).append("\n"); + } + } + } + if (resultMsg.length() > 0) { + etlResult.setSucceeded(resSuccess); + if (resSuccess) { + etlResult.setResultMessage(resultMsg.toString()); + } else { + etlResult.setErrorMessage(resultMsg.toString()); + } + return etlResult; + } + } + etlResult.setSucceeded(false); + etlResult.setErrorMessage("Task not found"); + return etlResult; + } + + @Override + public Map count(String task) { + ESSyncConfig config = esSyncConfig.get(task); + ESMapping mapping = config.getEsMapping(); + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setSize(0) + .get(); + + long rowCount = response.getHits().getTotalHits(); + Map res = new LinkedHashMap<>(); + res.put("esIndex", mapping.get_index()); + res.put("count", rowCount); + return res; + } + + @Override + public void destroy() { + if (esConfigMonitor != null) { + esConfigMonitor.destroy(); + } + if (transportClient != null) { + transportClient.close(); + } + } + + @Override + public String getDestination(String task) { + ESSyncConfig config = esSyncConfig.get(task); + if (config != null) { + return config.getDestination(); + } + return null; + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java new file mode 100644 index 00000000..09c2f4e0 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java @@ -0,0 +1,204 @@ +package com.alibaba.otter.canal.client.adapter.es.config; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * ES 映射配置 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESSyncConfig { + + private String dataSourceKey; // 数据源key + + private String outerAdapterKey; // adapter key + + private String groupId; // group id + + private String destination; // canal destination + + private ESMapping esMapping; + + public void validate() { + if (esMapping._index == null) { + throw new NullPointerException("esMapping._index"); + } + if (esMapping._type == null) { + throw new NullPointerException("esMapping._type"); + } + if (esMapping._id == null && esMapping.pk == null) { + throw new NullPointerException("esMapping._id and esMapping.pk"); + } + if (esMapping.sql == null) { + throw new NullPointerException("esMapping.sql"); + } + } + + public String getDataSourceKey() { + return dataSourceKey; + } + + public void setDataSourceKey(String dataSourceKey) { + this.dataSourceKey = dataSourceKey; + } + + public String getOuterAdapterKey() { + return outerAdapterKey; + } + + public void setOuterAdapterKey(String outerAdapterKey) { + this.outerAdapterKey = outerAdapterKey; + } + + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public ESMapping getEsMapping() { + return esMapping; + } + + public void setEsMapping(ESMapping esMapping) { + this.esMapping = esMapping; + } + + public static class ESMapping { + + private String _index; + private String _type; + private String _id; + private String pk; + private String parent; + private String sql; + // 对象字段, 例: objFields: + // - _labels: array:; + private Map objFields = new LinkedHashMap<>(); + private List skips = new ArrayList<>(); + private int commitBatch = 1000; + private String etlCondition; + private boolean syncByTimestamp = false; // 是否按时间戳定时同步 + private Long syncInterval; // 同步时间间隔 + + private SchemaItem schemaItem; // sql解析结果模型 + + public String get_index() { + return _index; + } + + public void set_index(String _index) { + this._index = _index; + } + + public String get_type() { + return _type; + } + + public void set_type(String _type) { + this._type = _type; + } + + public String get_id() { + return _id; + } + + public void set_id(String _id) { + this._id = _id; + } + + public String getPk() { + return pk; + } + + public void setPk(String pk) { + this.pk = pk; + } + + public String getParent() { + return parent; + } + + public void setParent(String parent) { + this.parent = parent; + } + + public Map getObjFields() { + return objFields; + } + + public void setObjFields(Map objFields) { + this.objFields = objFields; + } + + public List getSkips() { + return skips; + } + + public void setSkips(List skips) { + this.skips = skips; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public int getCommitBatch() { + return commitBatch; + } + + public void setCommitBatch(int commitBatch) { + this.commitBatch = commitBatch; + } + + public String getEtlCondition() { + return etlCondition; + } + + public void setEtlCondition(String etlCondition) { + this.etlCondition = etlCondition; + } + + public Long getSyncInterval() { + return syncInterval; + } + + public void setSyncInterval(Long syncInterval) { + this.syncInterval = syncInterval; + } + + public boolean isSyncByTimestamp() { + return syncByTimestamp; + } + + public void setSyncByTimestamp(boolean syncByTimestamp) { + this.syncByTimestamp = syncByTimestamp; + } + + public SchemaItem getSchemaItem() { + return schemaItem; + } + + public void setSchemaItem(SchemaItem schemaItem) { + this.schemaItem = schemaItem; + } + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java new file mode 100644 index 00000000..51216665 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java @@ -0,0 +1,45 @@ +package com.alibaba.otter.canal.client.adapter.es.config; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; +import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; + +/** + * ES 配置装载器 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESSyncConfigLoader { + + private static Logger logger = LoggerFactory.getLogger(ESSyncConfigLoader.class); + + public static synchronized Map load(Properties envProperties) { + logger.info("## Start loading es mapping config ... "); + + Map esSyncConfig = new LinkedHashMap<>(); + + Map configContentMap = MappingConfigsLoader.loadConfigs("es"); + configContentMap.forEach((fileName, content) -> { + ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null, content, ESSyncConfig.class, null, envProperties); + if (config == null) { + return; + } + try { + config.validate(); + } catch (Exception e) { + throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e); + } + esSyncConfig.put(fileName, config); + }); + + logger.info("## ES mapping config loaded"); + return esSyncConfig; + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java new file mode 100644 index 00000000..5bb1eb41 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java @@ -0,0 +1,422 @@ +package com.alibaba.otter.canal.client.adapter.es.config; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; + +/** + * ES 映射配置视图 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class SchemaItem { + + private Map aliasTableItems = new LinkedHashMap<>(); // 别名对应表名 + private Map selectFields = new LinkedHashMap<>(); // 查询字段 + private String sql; + + private volatile Map> tableItemAliases; + private volatile Map> columnFields; + private volatile Boolean allFieldsSimple; + + public void init() { + this.getTableItemAliases(); + this.getColumnFields(); + this.isAllFieldsSimple(); + aliasTableItems.values().forEach(tableItem -> { + tableItem.getRelationTableFields(); + tableItem.getRelationSelectFieldItems(); + }); + } + + public Map getAliasTableItems() { + return aliasTableItems; + } + + public void setAliasTableItems(Map aliasTableItems) { + this.aliasTableItems = aliasTableItems; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public Map getSelectFields() { + return selectFields; + } + + public void setSelectFields(Map selectFields) { + this.selectFields = selectFields; + } + + public Map> getTableItemAliases() { + if (tableItemAliases == null) { + synchronized (SchemaItem.class) { + if (tableItemAliases == null) { + tableItemAliases = new LinkedHashMap<>(); + aliasTableItems.forEach((alias, tableItem) -> { + List aliases = tableItemAliases + .computeIfAbsent(tableItem.getTableName().toLowerCase(), k -> new ArrayList<>()); + aliases.add(tableItem); + }); + } + } + } + return tableItemAliases; + } + + public Map> getColumnFields() { + if (columnFields == null) { + synchronized (SchemaItem.class) { + if (columnFields == null) { + columnFields = new LinkedHashMap<>(); + getSelectFields() + .forEach((fieldName, fieldItem) -> fieldItem.getColumnItems().forEach(columnItem -> { + // TableItem tableItem = getAliasTableItems().get(columnItem.getOwner()); + // if (!tableItem.isSubQuery()) { + List fieldItems = columnFields.computeIfAbsent( + columnItem.getOwner() + "." + columnItem.getColumnName(), + k -> new ArrayList<>()); + fieldItems.add(fieldItem); + // } else { + // tableItem.getSubQueryFields().forEach(subQueryField -> { + // List fieldItems = columnFields.computeIfAbsent( + // columnItem.getOwner() + "." + subQueryField.getColumn().getColumnName(), + // k -> new ArrayList<>()); + // fieldItems.add(fieldItem); + // }); + // } + })); + } + } + } + return columnFields; + } + + public boolean isAllFieldsSimple() { + if (allFieldsSimple == null) { + synchronized (SchemaItem.class) { + if (allFieldsSimple == null) { + allFieldsSimple = true; + + for (FieldItem fieldItem : getSelectFields().values()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + } + } + } + + return allFieldsSimple; + } + + public TableItem getMainTable() { + if (!aliasTableItems.isEmpty()) { + return aliasTableItems.values().iterator().next(); + } else { + return null; + } + } + + public FieldItem getIdFieldItem(ESMapping mapping) { + if (mapping.get_id() != null) { + return getSelectFields().get(mapping.get_id()); + } else { + return getSelectFields().get(mapping.getPk()); + } + } + + public static class TableItem { + + private SchemaItem schemaItem; + + private String schema; + private String tableName; + private String alias; + private String subQuerySql; + private List subQueryFields = new ArrayList<>(); + private List relationFields = new ArrayList<>(); + + private boolean main; + private boolean subQuery; + + private volatile Map> relationTableFields; // 当前表关联条件字段对应主表查询字段 + private volatile List relationSelectFieldItems; // 子表所在主表的查询字段 + + public TableItem(SchemaItem schemaItem){ + this.schemaItem = schemaItem; + } + + public SchemaItem getSchemaItem() { + return schemaItem; + } + + public void setSchemaItem(SchemaItem schemaItem) { + this.schemaItem = schemaItem; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public String getSubQuerySql() { + return subQuerySql; + } + + public void setSubQuerySql(String subQuerySql) { + this.subQuerySql = subQuerySql; + } + + public boolean isMain() { + return main; + } + + public void setMain(boolean main) { + this.main = main; + } + + public boolean isSubQuery() { + return subQuery; + } + + public void setSubQuery(boolean subQuery) { + this.subQuery = subQuery; + } + + public List getSubQueryFields() { + return subQueryFields; + } + + public void setSubQueryFields(List subQueryFields) { + this.subQueryFields = subQueryFields; + } + + public List getRelationFields() { + return relationFields; + } + + public void setRelationFields(List relationFields) { + this.relationFields = relationFields; + } + + public Map> getRelationTableFields() { + if (relationTableFields == null) { + synchronized (SchemaItem.class) { + if (relationTableFields == null) { + relationTableFields = new LinkedHashMap<>(); + + getRelationFields().forEach(relationFieldsPair -> { + FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem(); + FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); + FieldItem currentTableRelField = null; + if (getAlias().equals(leftFieldItem.getOwner())) { + currentTableRelField = leftFieldItem; + } else if (getAlias().equals(rightFieldItem.getOwner())) { + currentTableRelField = rightFieldItem; + } + + if (currentTableRelField != null) { + List selectFieldItem = getSchemaItem().getColumnFields() + .get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName()); + if (selectFieldItem != null && !selectFieldItem.isEmpty()) { + relationTableFields.put(currentTableRelField, selectFieldItem); + } else { + selectFieldItem = getSchemaItem().getColumnFields() + .get(rightFieldItem.getOwner() + "." + + rightFieldItem.getColumn().getColumnName()); + if (selectFieldItem != null && !selectFieldItem.isEmpty()) { + relationTableFields.put(currentTableRelField, selectFieldItem); + } else { + throw new UnsupportedOperationException( + "Relation condition column must in select columns."); + } + } + } + }); + } + } + } + return relationTableFields; + } + + public List getRelationSelectFieldItems() { + if (relationSelectFieldItems == null) { + synchronized (SchemaItem.class) { + if (relationSelectFieldItems == null) { + relationSelectFieldItems = new ArrayList<>(); + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + if (fieldItem.getOwners().contains(getAlias())) { + relationSelectFieldItems.add(fieldItem); + } + } + } + } + } + return relationSelectFieldItems; + } + } + + public static class RelationFieldsPair { + + private FieldItem leftFieldItem; + private FieldItem rightFieldItem; + + public RelationFieldsPair(FieldItem leftFieldItem, FieldItem rightFieldItem){ + this.leftFieldItem = leftFieldItem; + this.rightFieldItem = rightFieldItem; + } + + public FieldItem getLeftFieldItem() { + return leftFieldItem; + } + + public void setLeftFieldItem(FieldItem leftFieldItem) { + this.leftFieldItem = leftFieldItem; + } + + public FieldItem getRightFieldItem() { + return rightFieldItem; + } + + public void setRightFieldItem(FieldItem rightFieldItem) { + this.rightFieldItem = rightFieldItem; + } + } + + public static class FieldItem { + + private String fieldName; + private List columnItems = new ArrayList<>(); + private List owners = new ArrayList<>(); + + private boolean method; + private boolean binaryOp; + + public String getFieldName() { + return fieldName; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public List getColumnItems() { + return columnItems; + } + + public void setColumnItems(List columnItems) { + this.columnItems = columnItems; + } + + public boolean isMethod() { + return method; + } + + public void setMethod(boolean method) { + this.method = method; + } + + public boolean isBinaryOp() { + return binaryOp; + } + + public void setBinaryOp(boolean binaryOp) { + this.binaryOp = binaryOp; + } + + public List getOwners() { + return owners; + } + + public void setOwners(List owners) { + this.owners = owners; + } + + public void addColumn(ColumnItem columnItem) { + columnItems.add(columnItem); + } + + public ColumnItem getColumn() { + if (!columnItems.isEmpty()) { + return columnItems.get(0); + } else { + return null; + } + } + + public String getOwner() { + if (!owners.isEmpty()) { + return owners.get(0); + } else { + return null; + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + FieldItem fieldItem = (FieldItem) o; + + return fieldName != null ? fieldName.equals(fieldItem.fieldName) : fieldItem.fieldName == null; + } + + @Override + public int hashCode() { + return fieldName != null ? fieldName.hashCode() : 0; + } + } + + public static class ColumnItem { + + private String owner; + private String columnName; + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } + + public String getColumnName() { + return columnName; + } + + public void setColumnName(String columnName) { + this.columnName = columnName; + } + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SqlParser.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SqlParser.java new file mode 100644 index 00000000..0c2d7b2c --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SqlParser.java @@ -0,0 +1,210 @@ +package com.alibaba.otter.canal.client.adapter.es.config; + +import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.BooleanAnd; +import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.Equality; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import com.alibaba.fastsql.sql.SQLUtils; +import com.alibaba.fastsql.sql.ast.SQLExpr; +import com.alibaba.fastsql.sql.ast.expr.SQLBinaryOpExpr; +import com.alibaba.fastsql.sql.ast.expr.SQLIdentifierExpr; +import com.alibaba.fastsql.sql.ast.expr.SQLMethodInvokeExpr; +import com.alibaba.fastsql.sql.ast.expr.SQLPropertyExpr; +import com.alibaba.fastsql.sql.ast.statement.*; +import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; +import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser; +import com.alibaba.fastsql.sql.parser.ParserException; +import com.alibaba.fastsql.sql.parser.SQLStatementParser; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.RelationFieldsPair; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; + +/** + * ES同步指定sql格式解析 + * + * @author rewerma 2018-10-26 下午03:45:49 + * @version 1.0.0 + */ +public class SqlParser { + + /** + * 解析sql + * + * @param sql sql + * @return 视图对象 + */ + public static SchemaItem parse(String sql) { + try { + SQLStatementParser parser = new MySqlStatementParser(sql); + SQLSelectStatement statement = (SQLSelectStatement) parser.parseStatement(); + MySqlSelectQueryBlock sqlSelectQueryBlock = (MySqlSelectQueryBlock) statement.getSelect().getQuery(); + + SchemaItem schemaItem = new SchemaItem(); + schemaItem.setSql(SQLUtils.toMySqlString(sqlSelectQueryBlock)); + SQLTableSource sqlTableSource = sqlSelectQueryBlock.getFrom(); + List tableItems = new ArrayList<>(); + SqlParser.visitSelectTable(schemaItem, sqlTableSource, tableItems, null); + tableItems.forEach(tableItem -> schemaItem.getAliasTableItems().put(tableItem.getAlias(), tableItem)); + + List fieldItems = collectSelectQueryFields(sqlSelectQueryBlock); + fieldItems.forEach(fieldItem -> schemaItem.getSelectFields().put(fieldItem.getFieldName(), fieldItem)); + + schemaItem.init(); + + if (schemaItem.getAliasTableItems().isEmpty() || schemaItem.getSelectFields().isEmpty()) { + throw new ParserException("Parse sql error"); + } + return schemaItem; + } catch (Exception e) { + throw new ParserException(); + } + } + + /** + * 归集字段 + * + * @param sqlSelectQueryBlock sqlSelectQueryBlock + * @return 字段属性列表 + */ + private static List collectSelectQueryFields(MySqlSelectQueryBlock sqlSelectQueryBlock) { + return sqlSelectQueryBlock.getSelectList().stream().map(selectItem -> { + FieldItem fieldItem = new FieldItem(); + fieldItem.setFieldName(selectItem.getAlias()); + visitColumn(selectItem.getExpr(), fieldItem); + return fieldItem; + }).collect(Collectors.toList()); + } + + /** + * 解析字段 + * + * @param expr sql expr + * @param fieldItem 字段属性 + */ + private static void visitColumn(SQLExpr expr, FieldItem fieldItem) { + if (expr instanceof SQLIdentifierExpr) { + // 无owner + SQLIdentifierExpr identifierExpr = (SQLIdentifierExpr) expr; + if (fieldItem.getFieldName() == null) { + fieldItem.setFieldName(identifierExpr.getName()); + } + ColumnItem columnItem = new ColumnItem(); + columnItem.setColumnName(identifierExpr.getName()); + fieldItem.getOwners().add(null); + fieldItem.addColumn(columnItem); + } else if (expr instanceof SQLPropertyExpr) { + // 有owner + SQLPropertyExpr sqlPropertyExpr = (SQLPropertyExpr) expr; + if (fieldItem.getFieldName() == null) { + fieldItem.setFieldName(sqlPropertyExpr.getName()); + } + fieldItem.getOwners().add(sqlPropertyExpr.getOwnernName()); + ColumnItem columnItem = new ColumnItem(); + columnItem.setColumnName(sqlPropertyExpr.getName()); + columnItem.setOwner(sqlPropertyExpr.getOwnernName()); + fieldItem.addColumn(columnItem); + } else if (expr instanceof SQLMethodInvokeExpr) { + SQLMethodInvokeExpr methodInvokeExpr = (SQLMethodInvokeExpr) expr; + fieldItem.setMethod(true); + for (SQLExpr sqlExpr : methodInvokeExpr.getArguments()) { + visitColumn(sqlExpr, fieldItem); + } + } else if (expr instanceof SQLBinaryOpExpr) { + SQLBinaryOpExpr sqlBinaryOpExpr = (SQLBinaryOpExpr) expr; + fieldItem.setBinaryOp(true); + visitColumn(sqlBinaryOpExpr.getLeft(), fieldItem); + visitColumn(sqlBinaryOpExpr.getRight(), fieldItem); + } + } + + /** + * 解析表 + * + * @param schemaItem 视图对象 + * @param sqlTableSource sqlTableSource + * @param tableItems 表对象列表 + * @param tableItemTmp 表对象(临时) + */ + private static void visitSelectTable(SchemaItem schemaItem, SQLTableSource sqlTableSource, + List tableItems, TableItem tableItemTmp) { + if (sqlTableSource instanceof SQLExprTableSource) { + SQLExprTableSource sqlExprTableSource = (SQLExprTableSource) sqlTableSource; + TableItem tableItem; + if (tableItemTmp != null) { + tableItem = tableItemTmp; + } else { + tableItem = new TableItem(schemaItem); + } + tableItem.setSchema(sqlExprTableSource.getSchema()); + tableItem.setTableName(sqlExprTableSource.getTableName()); + if (tableItem.getAlias() == null) { + tableItem.setAlias(sqlExprTableSource.getAlias()); + } + if (tableItems.isEmpty()) { + // 第一张表为主表 + tableItem.setMain(true); + } + tableItems.add(tableItem); + } else if (sqlTableSource instanceof SQLJoinTableSource) { + SQLJoinTableSource sqlJoinTableSource = (SQLJoinTableSource) sqlTableSource; + SQLTableSource leftTableSource = sqlJoinTableSource.getLeft(); + visitSelectTable(schemaItem, leftTableSource, tableItems, null); + SQLTableSource rightTableSource = sqlJoinTableSource.getRight(); + TableItem rightTableItem = new TableItem(schemaItem); + // 解析on条件字段 + visitOnCondition(sqlJoinTableSource.getCondition(), rightTableItem); + visitSelectTable(schemaItem, rightTableSource, tableItems, rightTableItem); + + } else if (sqlTableSource instanceof SQLSubqueryTableSource) { + SQLSubqueryTableSource subQueryTableSource = (SQLSubqueryTableSource) sqlTableSource; + MySqlSelectQueryBlock sqlSelectQuery = (MySqlSelectQueryBlock) subQueryTableSource.getSelect().getQuery(); + TableItem tableItem; + if (tableItemTmp != null) { + tableItem = tableItemTmp; + } else { + tableItem = new TableItem(schemaItem); + } + tableItem.setAlias(subQueryTableSource.getAlias()); + tableItem.setSubQuerySql(SQLUtils.toMySqlString(sqlSelectQuery)); + tableItem.setSubQuery(true); + tableItem.setSubQueryFields(collectSelectQueryFields(sqlSelectQuery)); + visitSelectTable(schemaItem, sqlSelectQuery.getFrom(), tableItems, tableItem); + } + } + + /** + * 解析on条件 + * + * @param expr sql expr + * @param tableItem 表对象 + */ + private static void visitOnCondition(SQLExpr expr, TableItem tableItem) { + if (!(expr instanceof SQLBinaryOpExpr)) { + throw new UnsupportedOperationException(); + } + SQLBinaryOpExpr sqlBinaryOpExpr = (SQLBinaryOpExpr) expr; + if (sqlBinaryOpExpr.getOperator() == BooleanAnd) { + visitOnCondition(sqlBinaryOpExpr.getLeft(), tableItem); + visitOnCondition(sqlBinaryOpExpr.getRight(), tableItem); + } else if (sqlBinaryOpExpr.getOperator() == Equality) { + FieldItem leftFieldItem = new FieldItem(); + visitColumn(sqlBinaryOpExpr.getLeft(), leftFieldItem); + if (leftFieldItem.getColumnItems().size() != 1 || leftFieldItem.isMethod() || leftFieldItem.isBinaryOp()) { + throw new UnsupportedOperationException("Unsupported for complex of on-condition"); + } + FieldItem rightFieldItem = new FieldItem(); + visitColumn(sqlBinaryOpExpr.getRight(), rightFieldItem); + if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod() + || rightFieldItem.isBinaryOp()) { + throw new UnsupportedOperationException("Unsupported for complex of on-condition"); + } + tableItem.getRelationFields().add(new RelationFieldsPair(leftFieldItem, rightFieldItem)); + } else { + throw new UnsupportedOperationException("Unsupported for complex of on-condition"); + } + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/monitor/ESConfigMonitor.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/monitor/ESConfigMonitor.java new file mode 100644 index 00000000..566a9fe0 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/monitor/ESConfigMonitor.java @@ -0,0 +1,155 @@ +package com.alibaba.otter.canal.client.adapter.es.monitor; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.io.monitor.FileAlterationListenerAdaptor; +import org.apache.commons.io.monitor.FileAlterationMonitor; +import org.apache.commons.io.monitor.FileAlterationObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SqlParser; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; +import com.alibaba.otter.canal.client.adapter.support.Util; + +public class ESConfigMonitor { + + private static final Logger logger = LoggerFactory.getLogger(ESConfigMonitor.class); + + private static final String adapterName = "es"; + + private ESAdapter esAdapter; + + private FileAlterationMonitor fileMonitor; + + public void init(ESAdapter esAdapter) { + this.esAdapter = esAdapter; + File confDir = Util.getConfDirPath(adapterName); + try { + FileAlterationObserver observer = new FileAlterationObserver(confDir, + FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml"))); + FileListener listener = new FileListener(); + observer.addListener(listener); + fileMonitor = new FileAlterationMonitor(3000, observer); + fileMonitor.start(); + + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + public void destroy() { + try { + fileMonitor.stop(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private class FileListener extends FileAlterationListenerAdaptor { + + @Override + public void onFileCreate(File file) { + super.onFileCreate(file); + try { + // 加载新增的配置文件 + String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); + ESSyncConfig config = new Yaml().loadAs(configContent, ESSyncConfig.class); + config.validate(); + addConfigToCache(file, config); + + logger.info("Add a new es mapping config: {} to canal adapter", file.getName()); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public void onFileChange(File file) { + super.onFileChange(file); + + try { + if (esAdapter.getEsSyncConfig().containsKey(file.getName())) { + // 加载配置文件 + String configContent = MappingConfigsLoader + .loadConfig(adapterName + File.separator + file.getName()); + if (configContent == null) { + onFileDelete(file); + return; + } + ESSyncConfig config = new Yaml().loadAs(configContent, ESSyncConfig.class); + config.validate(); + if (esAdapter.getEsSyncConfig().containsKey(file.getName())) { + deleteConfigFromCache(file); + } + addConfigToCache(file, config); + + logger.info("Change a es mapping config: {} of canal adapter", file.getName()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public void onFileDelete(File file) { + super.onFileDelete(file); + + try { + if (esAdapter.getEsSyncConfig().containsKey(file.getName())) { + deleteConfigFromCache(file); + + logger.info("Delete a es mapping config: {} of canal adapter", file.getName()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private void addConfigToCache(File file, ESSyncConfig config) { + esAdapter.getEsSyncConfig().put(file.getName(), config); + + SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql()); + config.getEsMapping().setSchemaItem(schemaItem); + + DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (dataSource == null || dataSource.getUrl() == null) { + throw new RuntimeException("No data source found: " + config.getDataSourceKey()); + } + Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$"); + Matcher matcher = pattern.matcher(dataSource.getUrl()); + if (!matcher.find()) { + throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey()); + } + String schema = matcher.group(2); + + schemaItem.getAliasTableItems().values().forEach(tableItem -> { + Map esSyncConfigMap = esAdapter.getDbTableEsSyncConfig() + .computeIfAbsent(schema + "-" + tableItem.getTableName(), k -> new HashMap<>()); + esSyncConfigMap.put(file.getName(), config); + }); + + } + + private void deleteConfigFromCache(File file) { + esAdapter.getEsSyncConfig().remove(file.getName()); + for (Map configMap : esAdapter.getDbTableEsSyncConfig().values()) { + if (configMap != null) { + configMap.remove(file.getName()); + } + } + + } + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java new file mode 100644 index 00000000..51f0a924 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java @@ -0,0 +1,286 @@ +package com.alibaba.otter.canal.client.adapter.es.service; + +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.sql.DataSource; + +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.support.ESSyncUtil; +import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.google.common.base.Joiner; + +/** + * ES ETL Service + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESEtlService { + + private static Logger logger = LoggerFactory.getLogger(ESEtlService.class); + + private TransportClient transportClient; + private ESTemplate esTemplate; + private ESSyncConfig config; + + public ESEtlService(TransportClient transportClient, ESSyncConfig config){ + this.transportClient = transportClient; + this.esTemplate = new ESTemplate(transportClient); + this.config = config; + } + + public EtlResult importData(List params, boolean bulk) { + EtlResult etlResult = new EtlResult(); + AtomicLong impCount = new AtomicLong(); + List errMsg = new ArrayList<>(); + String esIndex = ""; + if (config == null) { + logger.warn("esSycnCofnig is null, etl go end !"); + etlResult.setErrorMessage("esSycnCofnig is null, etl go end !"); + return etlResult; + } + + ESMapping mapping = config.getEsMapping(); + + esIndex = mapping.get_index(); + DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$"); + Matcher matcher = pattern.matcher(dataSource.getUrl()); + if (!matcher.find()) { + throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey()); + } + String schema = matcher.group(2); + + logger.info("etl from db: {}, to es index: {}", schema, esIndex); + long start = System.currentTimeMillis(); + try { + String sql = mapping.getSql(); + + // 拼接条件 + if (mapping.getEtlCondition() != null && params != null) { + String etlCondition = mapping.getEtlCondition(); + int size = params.size(); + for (int i = 0; i < size; i++) { + etlCondition = etlCondition.replace("{" + i + "}", params.get(i)); + } + + sql += " " + etlCondition; + } + + if (logger.isDebugEnabled()) { + logger.debug("etl sql : {}", mapping.getSql()); + } + + if (bulk) { + // 获取总数 + String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT "; + long cnt = (Long) ESSyncUtil.sqlRS(dataSource, countSql, rs -> { + Long count = null; + try { + if (rs.next()) { + count = ((Number) rs.getObject(1)).longValue(); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + return count == null ? 0L : count; + }); + + // 当大于1万条记录时开启多线程 + if (cnt >= 10000) { + int threadCount = 3; // TODO 从配置读取默认为3 + long perThreadCnt = cnt / threadCount; + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + List> futures = new ArrayList<>(threadCount); + for (int i = 0; i < threadCount; i++) { + long offset = i * perThreadCnt; + Long size = null; + if (i != threadCount - 1) { + size = perThreadCnt; + } + String sqlFinal; + if (size != null) { + sqlFinal = sql + " LIMIT " + offset + "," + size; + } else { + sqlFinal = sql + " LIMIT " + offset + "," + cnt; + } + Future future = executor + .submit(() -> executeSqlImport(dataSource, sqlFinal, mapping, impCount, errMsg)); + futures.add(future); + } + + for (Future future : futures) { + future.get(); + } + + executor.shutdown(); + } else { + executeSqlImport(dataSource, sql, mapping, impCount, errMsg); + } + } else { + logger.info("自动ETL，无需统计记录总条数，直接进行ETL, index: {}", esIndex); + executeSqlImport(dataSource, sql, mapping, impCount, errMsg); + } + + logger.info("数据全量导入完成,一共导入 {} 条数据, 耗时: {}", impCount.get(), System.currentTimeMillis() - start); + etlResult.setResultMessage("导入ES索引 " + esIndex + " 数据：" + impCount.get() + " 条"); + } catch (Exception e) { + logger.error(e.getMessage(), e); + errMsg.add(esIndex + " etl failed! ==>" + e.getMessage()); + } + if (errMsg.isEmpty()) { + etlResult.setSucceeded(true); + } else { + etlResult.setErrorMessage(Joiner.on("\n").join(errMsg)); + } + return etlResult; + } + + private void processFailBulkResponse(BulkResponse bulkResponse, boolean hasParent) { + for (BulkItemResponse response : bulkResponse.getItems()) { + if (!response.isFailed()) { + continue; + } + + if (response.getFailure().getStatus() == RestStatus.NOT_FOUND) { + logger.warn(response.getFailureMessage()); + } else { + logger.error("全量导入数据有误 {}", response.getFailureMessage()); + throw new RuntimeException("全量数据 etl 异常: " + response.getFailureMessage()); + } + } + } + + private boolean executeSqlImport(DataSource ds, String sql, ESMapping mapping, AtomicLong impCount, + List errMsg) { + try { + ESSyncUtil.sqlRS(ds, sql, rs -> { + int count = 0; + try { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + + long batchBegin = System.currentTimeMillis(); + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : mapping.getSchemaItem().getSelectFields().values()) { + + // 如果是主键字段则不插入 + if (fieldItem.getFieldName().equals(mapping.get_id())) { + continue; + } + + String fieldName = fieldItem.getFieldName(); + if (mapping.getSkips().contains(fieldName)) { + continue; + } + + Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName); + esFieldData.put(fieldName, val); + } + Object idVal = null; + if (mapping.get_id() != null) { + idVal = rs.getObject(mapping.get_id()); + } + + if (idVal != null) { + if (mapping.getParent() == null) { + bulkRequestBuilder.add(transportClient + .prepareIndex(mapping.get_index(), mapping.get_type(), idVal.toString()) + .setSource(esFieldData)); + } else { + // ignore + } + } else { + idVal = rs.getObject(mapping.getPk()); + if (mapping.getParent() == null) { + // 删除pk对应的数据 + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)) + .get(); + for (SearchHit hit : response.getHits()) { + bulkRequestBuilder.add(transportClient + .prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); + } + + bulkRequestBuilder + .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type()) + .setSource(esFieldData)); + } else { + // ignore + } + } + + if (bulkRequestBuilder.numberOfActions() % mapping.getCommitBatch() == 0 + && bulkRequestBuilder.numberOfActions() > 0) { + long esBatchBegin = System.currentTimeMillis(); + BulkResponse rp = bulkRequestBuilder.execute().actionGet(); + if (rp.hasFailures()) { + this.processFailBulkResponse(rp, Objects.nonNull(mapping.getParent())); + } + + if (logger.isDebugEnabled()) { + logger.debug("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}", + (System.currentTimeMillis() - batchBegin), + (System.currentTimeMillis() - esBatchBegin), + bulkRequestBuilder.numberOfActions(), + mapping.get_index()); + } + batchBegin = System.currentTimeMillis(); + bulkRequestBuilder = transportClient.prepareBulk(); + } + count++; + impCount.incrementAndGet(); + } + + if (bulkRequestBuilder.numberOfActions() > 0) { + long esBatchBegin = System.currentTimeMillis(); + BulkResponse rp = bulkRequestBuilder.execute().actionGet(); + if (rp.hasFailures()) { + this.processFailBulkResponse(rp, Objects.nonNull(mapping.getParent())); + } + if (logger.isDebugEnabled()) { + logger.debug("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}", + (System.currentTimeMillis() - batchBegin), + (System.currentTimeMillis() - esBatchBegin), + bulkRequestBuilder.numberOfActions(), + mapping.get_index()); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage()); + throw new RuntimeException(e); + } + return count; + }); + + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java new file mode 100644 index 00000000..dba99028 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -0,0 +1,862 @@ +package com.alibaba.otter.canal.client.adapter.es.service; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import javax.sql.DataSource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; +import com.alibaba.otter.canal.client.adapter.es.support.ESSyncUtil; +import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +/** + * ES 同步 Service + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESSyncService { + + private static Logger logger = LoggerFactory.getLogger(ESSyncService.class); + + private ESTemplate esTemplate; + + public ESSyncService(ESTemplate esTemplate){ + this.esTemplate = esTemplate; + } + + public void sync(Collection esSyncConfigs, Dml dml) { + long begin = System.currentTimeMillis(); + if (esSyncConfigs != null) { + if (logger.isTraceEnabled()) { + logger.trace("Destination: {}, database:{}, table:{}, type:{}, effect index count: {}", + dml.getDestination(), + dml.getDatabase(), + dml.getTable(), + dml.getType(), + esSyncConfigs.size()); + } + + for (ESSyncConfig config : esSyncConfigs) { + if (logger.isTraceEnabled()) { + logger.trace("Prepared to sync index: {}, destination: {}", + config.getEsMapping().get_index(), + dml.getDestination()); + } + this.sync(config, dml); + if (logger.isTraceEnabled()) { + logger.trace("Sync completed: {}, destination: {}", + config.getEsMapping().get_index(), + dml.getDestination()); + } + } + if (logger.isTraceEnabled()) { + logger.trace("Sync elapsed time: {} ms, effect index count：{}, destination: {}", + (System.currentTimeMillis() - begin), + esSyncConfigs.size(), + dml.getDestination()); + } + if (logger.isDebugEnabled()) { + logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue)); + } + } + } + + public void sync(ESSyncConfig config, Dml dml) { + try { + // 如果是按时间戳定时更新则返回 + if (config.getEsMapping().isSyncByTimestamp()) { + return; + } + + long begin = System.currentTimeMillis(); + + String type = dml.getType(); + if (type != null && type.equalsIgnoreCase("INSERT")) { + insert(config, dml); + } else if (type != null && type.equalsIgnoreCase("UPDATE")) { + update(config, dml); + } else if (type != null && type.equalsIgnoreCase("DELETE")) { + delete(config, dml); + } + + if (logger.isTraceEnabled()) { + logger.trace("Sync elapsed time: {} ms,destination: {}, es index: {}", + (System.currentTimeMillis() - begin), + dml.getDestination(), + config.getEsMapping().get_index()); + } + } catch (Throwable e) { + logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml); + throw new RuntimeException(e); + } + } + + /** + * 插入操作dml + * + * @param config es配置 + * @param dml dml数据 + */ + private void insert(ESSyncConfig config, Dml dml) { + List> dataList = dml.getData(); + if (dataList == null || dataList.isEmpty()) { + return; + } + SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); + for (Map data : dataList) { + if (data == null || data.isEmpty()) { + continue; + } + + if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { + // ------单表 & 所有字段都为简单字段------ + singleTableSimpleFiledInsert(config, dml, data); + } else { + // ------是主表 查询sql来插入------ + if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { + mainTableInsert(config, dml, data); + } + + // 从表的操作 + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + if (!tableItem.getTableName().equals(dml.getTable())) { + continue; + } + // 关联条件出现在主表查询条件是否为简单字段 + boolean allFieldsSimple = true; + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + // 所有查询字段均为简单字段 + if (allFieldsSimple) { + // 不是子查询 + if (!tableItem.isSubQuery()) { + // ------关联表简单字段插入------ + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + Object value = esTemplate.getValFromData(config.getEsMapping(), + data, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), value); + } + + joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); + } else { + // ------关联子表简单字段插入------ + subTableSimpleFieldOperation(config, dml, data, null, tableItem); + } + } else { + // ------关联子表复杂字段插入 执行全sql更新es------ + wholeSqlOperation(config, dml, data, null, tableItem); + } + } + } + } + } + + /** + * 更新操作dml + * + * @param config es配置 + * @param dml dml数据 + */ + private void update(ESSyncConfig config, Dml dml) { + List> dataList = dml.getData(); + List> oldList = dml.getOld(); + if (dataList == null || dataList.isEmpty() || oldList == null || oldList.isEmpty()) { + return; + } + SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); + int i = 0; + for (Map data : dataList) { + Map old = oldList.get(i); + if (data == null || data.isEmpty() || old == null || old.isEmpty()) { + continue; + } + + if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { + // ------单表 & 所有字段都为简单字段------ + singleTableSimpleFiledUpdate(config, dml, data, old); + } else { + // ------主表 查询sql来更新------ + if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { + ESMapping mapping = config.getEsMapping(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + FieldItem idFieldItem = schemaItem.getSelectFields().get(idFieldName); + + boolean idFieldSimple = true; + if (idFieldItem.isMethod() || idFieldItem.isBinaryOp()) { + idFieldSimple = false; + } + + boolean allUpdateFieldSimple = true; + out: for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + for (ColumnItem columnItem : fieldItem.getColumnItems()) { + if (old.containsKey(columnItem.getColumnName())) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allUpdateFieldSimple = false; + break out; + } + } + } + } + + // 不支持主键更新!! + + // 判断是否有外键更新 + boolean fkChanged = false; + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + boolean changed = false; + for (List fieldItems : tableItem.getRelationTableFields().values()) { + for (FieldItem fieldItem : fieldItems) { + if (old.containsKey(fieldItem.getColumn().getColumnName())) { + fkChanged = true; + changed = true; + break; + } + } + } + // 如果外键有修改,则更新所对应该表的所有查询条件数据 + if (changed) { + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + fieldItem.getColumnItems() + .forEach(columnItem -> old.put(columnItem.getColumnName(), null)); + } + } + } + + // 判断主键和所更新的字段是否全为简单字段 + if (idFieldSimple && allUpdateFieldSimple && !fkChanged) { + singleTableSimpleFiledUpdate(config, dml, data, old); + } else { + mainTableUpdate(config, dml, data, old); + } + } + + // 从表的操作 + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + if (!tableItem.getTableName().equals(dml.getTable())) { + continue; + } + + // 关联条件出现在主表查询条件是否为简单字段 + boolean allFieldsSimple = true; + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + + // 所有查询字段均为简单字段 + if (allFieldsSimple) { + // 不是子查询 + if (!tableItem.isSubQuery()) { + // ------关联表简单字段更新------ + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (old.containsKey(fieldItem.getColumn().getColumnName())) { + Object value = esTemplate.getValFromData(config.getEsMapping(), + data, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), value); + } + } + joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); + } else { + // ------关联子表简单字段更新------ + subTableSimpleFieldOperation(config, dml, data, old, tableItem); + } + } else { + // ------关联子表复杂字段更新 执行全sql更新es------ + wholeSqlOperation(config, dml, data, old, tableItem); + } + } + } + + i++; + } + } + + /** + * 删除操作dml + * + * @param config es配置 + * @param dml dml数据 + */ + private void delete(ESSyncConfig config, Dml dml) { + List> dataList = dml.getData(); + if (dataList == null || dataList.isEmpty()) { + return; + } + SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); + + for (Map data : dataList) { + if (data == null || data.isEmpty()) { + continue; + } + + ESMapping mapping = config.getEsMapping(); + + // ------是主表------ + if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { + FieldItem idFieldItem = schemaItem.getIdFieldItem(mapping); + // 主键为简单字段 + if (!idFieldItem.isMethod() && !idFieldItem.isBinaryOp()) { + Object idVal = esTemplate.getValFromData(mapping, + data, + idFieldItem.getFieldName(), + idFieldItem.getColumn().getColumnName()); + + if (logger.isTraceEnabled()) { + logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.delete(mapping, idVal); + if (!result) { + logger.error("Main table delete es index error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + } else { + // ------主键带函数, 查询sql获取主键删除------ + mainTableDelete(config, dml, data); + } + } + + // 从表的操作 + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + if (!tableItem.getTableName().equals(dml.getTable())) { + continue; + } + + // 关联条件出现在主表查询条件是否为简单字段 + boolean allFieldsSimple = true; + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + + // 所有查询字段均为简单字段 + if (allFieldsSimple) { + // 不是子查询 + if (!tableItem.isSubQuery()) { + // ------关联表简单字段更新为null------ + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + esFieldData.put(fieldItem.getFieldName(), null); + } + joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); + } else { + // ------关联子表简单字段更新------ + subTableSimpleFieldOperation(config, dml, data, null, tableItem); + } + } else { + // ------关联子表复杂字段更新 执行全sql更新es------ + wholeSqlOperation(config, dml, data, null, tableItem); + } + } + } + } + + /** + * 单表简单字段insert + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + */ + private void singleTableSimpleFiledInsert(ESSyncConfig config, Dml dml, Map data) { + ESMapping mapping = config.getEsMapping(); + Map esFieldData = new LinkedHashMap<>(); + Object idVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace("Single table insert ot es index, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.insert(mapping, idVal, esFieldData); + if (!result) { + logger.error("Single table insert to es index error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + } + + /** + * 主表(单表)复杂字段insert + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + */ + private void mainTableInsert(ESSyncConfig config, Dml dml, Map data) { + ESMapping mapping = config.getEsMapping(); + String sql = mapping.getSql(); + String condition = ESSyncUtil.pkConditionSql(mapping, data); + sql = ESSyncUtil.appendCondition(sql, condition); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Main table insert ot es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.replace("\n", " ")); + } + ESSyncUtil.sqlRS(ds, sql, rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + Object idVal = esTemplate.getESDataFromRS(mapping, rs, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace( + "Main table insert ot es index by query sql, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.insert(mapping, idVal, esFieldData); + if (!result) { + logger.error( + "Main table insert to es index by query sql error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } + + private void mainTableDelete(ESSyncConfig config, Dml dml, Map data) { + ESMapping mapping = config.getEsMapping(); + String sql = mapping.getSql(); + String condition = ESSyncUtil.pkConditionSql(mapping, data); + sql = ESSyncUtil.appendCondition(sql, condition); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Main table delete es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.replace("\n", " ")); + } + ESSyncUtil.sqlRS(ds, sql, rs -> { + try { + while (rs.next()) { + Object idVal = esTemplate.getIdValFromRS(mapping, rs); + + if (logger.isTraceEnabled()) { + logger.trace( + "Main table delete ot es index by query sql, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.delete(mapping, idVal); + if (!result) { + logger.error( + "Main table delete to es index by query sql error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } + + /** + * 关联表主表简单字段operation + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + * @param tableItem 当前表配置 + */ + private void joinTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map data, + TableItem tableItem, Map esFieldData) { + ESMapping mapping = config.getEsMapping(); + + Map paramsTmp = new LinkedHashMap<>(); + for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { + for (FieldItem fieldItem : entry.getValue()) { + if (fieldItem.getColumnItems().size() == 1) { + Object value = esTemplate.getValFromData(mapping, + data, + fieldItem.getFieldName(), + entry.getKey().getColumn().getColumnName()); + + String fieldName = fieldItem.getFieldName(); + // 判断是否是主键 + if (fieldName.equals(mapping.get_id())) { + fieldName = "_id"; + } + paramsTmp.put(fieldName, value); + } + } + } + + if (logger.isDebugEnabled()) { + logger.trace("Join table update es index by foreign key, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + boolean result = esTemplate.updateByQuery(config, paramsTmp, esFieldData); + if (!result) { + logger.error("Join table update es index by foreign key error, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + } + + /** + * 关联子查询, 主表简单字段operation + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + * @param old 单行old数据 + * @param tableItem 当前表配置 + */ + private void subTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map data, + Map old, TableItem tableItem) { + ESMapping mapping = config.getEsMapping(); + StringBuilder sql = new StringBuilder( + "SELECT * FROM (" + tableItem.getSubQuerySql() + ") " + tableItem.getAlias() + " WHERE "); + + for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) { + String columnName = fkFieldItem.getColumn().getColumnName(); + Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName); + ESSyncUtil.appendCondition(sql, value, tableItem.getAlias(), columnName); + } + int len = sql.length(); + sql.delete(len - 5, len); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.toString().replace("\n", " ")); + } + ESSyncUtil.sqlRS(ds, sql.toString(), rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (old != null) { + out: for (FieldItem fieldItem1 : tableItem.getSubQueryFields()) { + for (ColumnItem columnItem0 : fieldItem.getColumnItems()) { + if (fieldItem1.getFieldName().equals(columnItem0.getColumnName())) + for (ColumnItem columnItem : fieldItem1.getColumnItems()) { + if (old.containsKey(columnItem.getColumnName())) { + Object val = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), val); + break out; + } + } + } + } + } else { + Object val = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), val); + } + } + + Map paramsTmp = new LinkedHashMap<>(); + for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { + for (FieldItem fieldItem : entry.getValue()) { + if (fieldItem.getColumnItems().size() == 1) { + Object value = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + entry.getKey().getColumn().getColumnName()); + String fieldName = fieldItem.getFieldName(); + // 判断是否是主键 + if (fieldName.equals(mapping.get_id())) { + fieldName = "_id"; + } + paramsTmp.put(fieldName, value); + } + } + } + + if (logger.isDebugEnabled()) { + logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + boolean result = esTemplate.updateByQuery(config, paramsTmp, esFieldData); + if (!result) { + logger.error( + "Join table update es index by query sql error, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } + + /** + * 关联(子查询), 主表复杂字段operation, 全sql执行 + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + * @param tableItem 当前表配置 + */ + private void wholeSqlOperation(ESSyncConfig config, Dml dml, Map data, Map old, + TableItem tableItem) { + ESMapping mapping = config.getEsMapping(); + StringBuilder sql = new StringBuilder(mapping.getSql() + " WHERE "); + + for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) { + String columnName = fkFieldItem.getColumn().getColumnName(); + Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName); + ESSyncUtil.appendCondition(sql, value, tableItem.getAlias(), columnName); + } + int len = sql.length(); + sql.delete(len - 5, len); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Join table update es index by query whole sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.toString().replace("\n", " ")); + } + ESSyncUtil.sqlRS(ds, sql.toString(), rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (old != null) { + // 从表子查询 + out: for (FieldItem fieldItem1 : tableItem.getSubQueryFields()) { + for (ColumnItem columnItem0 : fieldItem.getColumnItems()) { + if (fieldItem1.getFieldName().equals(columnItem0.getColumnName())) + for (ColumnItem columnItem : fieldItem1.getColumnItems()) { + if (old.containsKey(columnItem.getColumnName())) { + Object val = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + fieldItem.getFieldName()); + esFieldData.put(fieldItem.getFieldName(), val); + break out; + } + } + } + } + // 从表非子查询 + for (FieldItem fieldItem1 : tableItem.getRelationSelectFieldItems()) { + if (fieldItem1.equals(fieldItem)) { + for (ColumnItem columnItem : fieldItem1.getColumnItems()) { + if (old.containsKey(columnItem.getColumnName())) { + Object val = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + fieldItem.getFieldName()); + esFieldData.put(fieldItem.getFieldName(), val); + break; + } + } + } + } + } else { + Object val = esTemplate + .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName()); + esFieldData.put(fieldItem.getFieldName(), val); + } + } + + Map paramsTmp = new LinkedHashMap<>(); + for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { + for (FieldItem fieldItem : entry.getValue()) { + Object value = esTemplate + .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName()); + String fieldName = fieldItem.getFieldName(); + // 判断是否是主键 + if (fieldName.equals(mapping.get_id())) { + fieldName = "_id"; + } + paramsTmp.put(fieldName, value); + } + } + + if (logger.isDebugEnabled()) { + logger.trace( + "Join table update es index by query whole sql, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + boolean result = esTemplate.updateByQuery(config, paramsTmp, esFieldData); + if (!result) { + logger.error( + "Join table update es index by query whole sql error, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } + + /** + * 单表简单字段update + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行data数据 + * @param old 单行old数据 + */ + private void singleTableSimpleFiledUpdate(ESSyncConfig config, Dml dml, Map data, + Map old) { + ESMapping mapping = config.getEsMapping(); + Map esFieldData = new LinkedHashMap<>(); + + Object idVal = esTemplate.getESDataFromDmlData(mapping, data, old, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace("Main table update ot es index, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.update(mapping, idVal, esFieldData); + if (!result) { + logger.error("Main table update to es index error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + } + + /** + * 主表(单表)复杂字段update + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + */ + private void mainTableUpdate(ESSyncConfig config, Dml dml, Map data, Map old) { + ESMapping mapping = config.getEsMapping(); + String sql = mapping.getSql(); + String condition = ESSyncUtil.pkConditionSql(mapping, data); + sql = ESSyncUtil.appendCondition(sql, condition); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Main table update ot es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.replace("\n", " ")); + } + ESSyncUtil.sqlRS(ds, sql, rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + Object idVal = esTemplate.getESDataFromRS(mapping, rs, old, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace( + "Main table update ot es index by query sql, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.update(mapping, idVal, esFieldData); + if (!result) { + logger.error( + "Main table update to es index by query sql error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java new file mode 100644 index 00000000..cf8f8fd4 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java @@ -0,0 +1,335 @@ +package com.alibaba.otter.canal.client.adapter.es.support; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.*; +import java.util.*; +import java.util.Date; +import java.util.function.Function; + +import javax.sql.DataSource; + +import org.apache.commons.codec.binary.Base64; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; + +/** + * ES 同步工具同类 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESSyncUtil { + + private static Logger logger = LoggerFactory.getLogger(ESSyncUtil.class); + + public static Object convertToEsObj(Object val, String fieldInfo) { + if (val == null) { + return null; + } + if (fieldInfo.startsWith("array:")) { + String separator = fieldInfo.substring("array:".length()).trim(); + String[] values = val.toString().split(separator); + return Arrays.asList(values); + } else if (fieldInfo.startsWith("object")) { + return JSON.parse(val.toString()); + } + return null; + } + + /** + * 类型转换为Mapping中对应的类型 + */ + public static Object typeConvert(Object val, String esType) { + if (val == null) { + return null; + } + if (esType == null) { + return val; + } + Object res = null; + if ("integer".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).intValue(); + } else { + res = Integer.parseInt(val.toString()); + } + } else if ("long".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).longValue(); + } else { + res = Long.parseLong(val.toString()); + } + } else if ("short".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).shortValue(); + } else { + res = Short.parseShort(val.toString()); + } + } else if ("byte".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).byteValue(); + } else { + res = Byte.parseByte(val.toString()); + } + } else if ("double".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).doubleValue(); + } else { + res = Double.parseDouble(val.toString()); + } + } else if ("float".equals(esType) || "half_float".equals(esType) || "scaled_float".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).floatValue(); + } else { + res = Float.parseFloat(val.toString()); + } + } else if ("boolean".equals(esType)) { + if (val instanceof Boolean) { + res = val; + } else if (val instanceof Number) { + int v = ((Number) val).intValue(); + res = v != 0; + } else { + res = Boolean.parseBoolean(val.toString()); + } + } else if ("date".equals(esType)) { + if (val instanceof java.sql.Time) { + DateTime dateTime = new DateTime(((java.sql.Time) val).getTime()); + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("HH:mm:ss.SSS"); + } else { + res = dateTime.toString("HH:mm:ss"); + } + } else if (val instanceof java.sql.Timestamp) { + DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime()); + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00"); + } + } else if (val instanceof java.sql.Date || val instanceof Date) { + DateTime dateTime; + if (val instanceof java.sql.Date) { + dateTime = new DateTime(((java.sql.Date) val).getTime()); + } else { + dateTime = new DateTime(((Date) val).getTime()); + } + if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0 + && dateTime.getMillisOfSecond() == 0) { + res = dateTime.toString("yyyy-MM-dd"); + } else { + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00"); + } + } + } else if (val instanceof Long) { + DateTime dateTime = new DateTime(((Long) val).longValue()); + if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0 + && dateTime.getMillisOfSecond() == 0) { + res = dateTime.toString("yyyy-MM-dd"); + } else if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00"); + } + } else if (val instanceof String) { + String v = ((String) val).trim(); + if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' ' + && v.charAt(13) == ':' && v.charAt(16) == ':') { + String dt = v.substring(0, 10) + "T" + v.substring(11); + DateTime dateTime = new DateTime(dt); + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00"); + } + } else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') { + DateTime dateTime = new DateTime(v); + res = dateTime.toString("yyyy-MM-dd"); + } + } + } else if ("binary".equals(esType)) { + if (val instanceof byte[]) { + Base64 base64 = new Base64(); + res = base64.encodeAsString((byte[]) val); + } else if (val instanceof Blob) { + byte[] b = blobToBytes((Blob) val); + Base64 base64 = new Base64(); + res = base64.encodeAsString(b); + } else if (val instanceof String) { + // 对应canal中的单字节编码 + byte[] b = ((String) val).getBytes(StandardCharsets.ISO_8859_1); + Base64 base64 = new Base64(); + res = base64.encodeAsString(b); + } + } else if ("geo_point".equals(esType)) { + if (!(val instanceof String)) { + logger.error("es type is geo_point, but source type is not String"); + return val; + } + + if (!((String) val).contains(",")) { + logger.error("es type is geo_point, source value not contains ',' separator"); + return val; + } + + String[] point = ((String) val).split(","); + Map location = new HashMap<>(); + location.put("lat", Double.valueOf(point[0].trim())); + location.put("lon", Double.valueOf(point[1].trim())); + return location; + } else if ("array".equals(esType)) { + if ("".equals(val.toString().trim())) { + res = new ArrayList<>(); + } else { + String value = val.toString(); + String separator = ","; + if (!value.contains(",")) { + if (value.contains(";")) { + separator = ";"; + } else if (value.contains("|")) { + separator = "|"; + } else if (value.contains("-")) { + separator = "-"; + } + } + String[] values = value.split(separator); + return Arrays.asList(values); + } + } else if ("object".equals(esType)) { + if ("".equals(val.toString().trim())) { + res = new HashMap<>(); + } else { + res = JSON.parseObject(val.toString(), Map.class); + } + } else { + // 其他类全以字符串处理 + res = val.toString(); + } + + return res; + } + + /** + * Blob转byte[] + */ + private static byte[] blobToBytes(Blob blob) { + try (InputStream is = blob.getBinaryStream()) { + byte[] b = new byte[(int) blob.length()]; + is.read(b); + return b; + } catch (IOException | SQLException e) { + logger.error(e.getMessage()); + return null; + } + } + + /** + * 拼接主键条件 + * + * @param mapping + * @param data + * @return + */ + public static String pkConditionSql(ESMapping mapping, Map data) { + Set idColumns = new LinkedHashSet<>(); + SchemaItem schemaItem = mapping.getSchemaItem(); + + TableItem mainTable = schemaItem.getMainTable(); + + for (ColumnItem idColumnItem : schemaItem.getIdFieldItem(mapping).getColumnItems()) { + if ((mainTable.getAlias() == null && idColumnItem.getOwner() == null) + || (mainTable.getAlias() != null && mainTable.getAlias().equals(idColumnItem.getOwner()))) { + idColumns.add(idColumnItem); + } + } + + if (idColumns.isEmpty()) { + throw new RuntimeException("Not found primary key field in main table"); + } + + // 拼接condition + StringBuilder condition = new StringBuilder(" "); + for (ColumnItem idColumn : idColumns) { + Object idVal = data.get(idColumn.getColumnName()); + if (mainTable.getAlias() != null) condition.append(mainTable.getAlias()).append("."); + condition.append(idColumn.getColumnName()).append("="); + if (idVal instanceof String) { + condition.append("'").append(idVal).append("' AND "); + } else { + condition.append(idVal).append(" AND "); + } + } + + if (condition.toString().endsWith("AND ")) { + int len2 = condition.length(); + condition.delete(len2 - 4, len2); + } + return condition.toString(); + } + + public static String appendCondition(String sql, String condition) { + return sql + " WHERE " + condition + " "; + } + + public static void appendCondition(StringBuilder sql, Object value, String owner, String columnName) { + if (value instanceof String) { + sql.append(owner).append(".").append(columnName).append("='").append(value).append("' AND "); + } else { + sql.append(owner).append(".").append(columnName).append("=").append(value).append(" AND "); + } + } + + /** + * 执行查询sql + */ + public static Object sqlRS(DataSource ds, String sql, Function fun) { + Connection conn = null; + Statement smt = null; + ResultSet rs = null; + try { + conn = ds.getConnection(); + smt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + smt.setFetchSize(Integer.MIN_VALUE); + rs = smt.executeQuery(sql); + + return fun.apply(rs); + } catch (SQLException e) { + logger.error("sqlRs has error, sql: {} ", sql); + throw new RuntimeException(e); + } finally { + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + logger.error("error to close result set"); + } + } + if (smt != null) { + try { + smt.close(); + } catch (SQLException e) { + logger.error("error to close statement"); + } + } + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + logger.error("error to close db connection"); + } + } + } + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java new file mode 100644 index 00000000..e6442fa1 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -0,0 +1,526 @@ +package com.alibaba.otter.canal.client.adapter.es.support; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +import javax.sql.DataSource; + +import com.alibaba.fastjson.JSON; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.UpdateByQueryAction; +import org.elasticsearch.index.reindex.UpdateByQueryRequestBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.SearchHit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.CollectionUtils; + +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; + +/** + * ES 操作模板 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESTemplate { + + private static final Logger logger = LoggerFactory.getLogger(ESTemplate.class); + + private static final int MAX_BATCH_SIZE = 1000; + + private TransportClient transportClient; + + public ESTemplate(TransportClient transportClient){ + this.transportClient = transportClient; + } + + /** + * 插入数据 + * + * @param mapping + * @param pkVal + * @param esFieldData + * @return + */ + public boolean insert(ESMapping mapping, Object pkVal, Map esFieldData) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + if (mapping.get_id() != null) { + bulkRequestBuilder + .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type(), pkVal.toString()) + .setSource(esFieldData)); + } else { + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .setSize(MAX_BATCH_SIZE) + .get(); + for (SearchHit hit : response.getHits()) { + bulkRequestBuilder + .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); + } + bulkRequestBuilder + .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type()).setSource(esFieldData)); + } + return commitBulkRequest(bulkRequestBuilder); + } + + /** + * 根据主键更新数据 + * + * @param mapping + * @param pkVal + * @param esFieldData + * @return + */ + public boolean update(ESMapping mapping, Object pkVal, Map esFieldData) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + append4Update(bulkRequestBuilder, mapping, pkVal, esFieldData); + return commitBulkRequest(bulkRequestBuilder); + } + + public void append4Update(BulkRequestBuilder bulkRequestBuilder, ESMapping mapping, Object pkVal, + Map esFieldData) { + if (mapping.get_id() != null) { + bulkRequestBuilder + .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString()) + .setDoc(esFieldData)); + } else { + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .setSize(MAX_BATCH_SIZE) + .get(); + for (SearchHit hit : response.getHits()) { + bulkRequestBuilder + .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId()) + .setDoc(esFieldData)); + } + } + } + + /** + * update by query + * + * @param config + * @param paramsTmp + * @param esFieldData + * @return + */ + public boolean updateByQuery(ESSyncConfig config, Map paramsTmp, Map esFieldData) { + if (paramsTmp.isEmpty()) { + return false; + } + ESMapping mapping = config.getEsMapping(); + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value))); + + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setSize(0) + .setQuery(queryBuilder) + .get(); + long count = response.getHits().getTotalHits(); + // 如果更新量大于Max, 查询sql批量更新 + if (count > MAX_BATCH_SIZE) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + // 查询sql更新 + StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE "); + paramsTmp.forEach( + (fieldName, value) -> sql.append("_v.").append(fieldName).append("=").append(value).append(" AND ")); + int len = sql.length(); + sql.delete(len - 4, len); + ESSyncUtil.sqlRS(ds, sql.toString(), rs -> { + int exeCount = 1; + try { + BulkRequestBuilder bulkRequestBuilderTmp = bulkRequestBuilder; + while (rs.next()) { + Object idVal = getIdValFromRS(mapping, rs); + append4Update(bulkRequestBuilderTmp, mapping, idVal, esFieldData); + + if (exeCount % mapping.getCommitBatch() == 0 && bulkRequestBuilderTmp.numberOfActions() > 0) { + commitBulkRequest(bulkRequestBuilderTmp); + bulkRequestBuilderTmp = transportClient.prepareBulk(); + } + exeCount++; + } + + if (bulkRequestBuilder.numberOfActions() > 0) { + commitBulkRequest(bulkRequestBuilderTmp); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + return true; + } else { + return updateByQuery(mapping, queryBuilder, esFieldData, 1); + } + } + + private boolean updateByQuery(ESMapping mapping, QueryBuilder queryBuilder, Map esFieldData, + int counter) { + if (CollectionUtils.isEmpty(esFieldData)) { + return true; + } + + StringBuilder sb = new StringBuilder(); + esFieldData.forEach((key, value) -> { + if (value instanceof Map) { + Map mapValue = (Map) value; + if (mapValue.containsKey("lon") && mapValue.containsKey("lat") && mapValue.size() == 2) { + sb.append("ctx._source") + .append("['") + .append(key) + .append("']") + .append(" = [") + .append(mapValue.get("lon")) + .append(", ") + .append(mapValue.get("lat")) + .append("];"); + } else { + sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = "); + sb.append(JSON.toJSONString(value)); + sb.append(";"); + } + } else if (value instanceof List) { + sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = "); + sb.append(JSON.toJSONString(value)); + sb.append(";"); + } else if (value instanceof String) { + sb.append("ctx._source") + .append("['") + .append(key) + .append("']") + .append(" = '") + .append(value) + .append("';"); + } else { + sb.append("ctx._source").append("['").append(key).append("']").append(" = ").append(value).append(";"); + } + }); + String scriptLine = sb.toString(); + if (logger.isTraceEnabled()) { + logger.trace(scriptLine); + } + + UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(transportClient); + updateByQuery.source(mapping.get_index()) + .abortOnVersionConflict(false) + .filter(queryBuilder) + .script(new Script(ScriptType.INLINE, "painless", scriptLine, Collections.emptyMap())); + + BulkByScrollResponse response = updateByQuery.get(); + if (logger.isTraceEnabled()) { + logger.trace("updateByQuery response: {}", response.getStatus()); + } + if (!CollectionUtils.isEmpty(response.getSearchFailures())) { + logger.error("script update_for_search has search error: " + response.getBulkFailures()); + return false; + } + + if (!CollectionUtils.isEmpty(response.getBulkFailures())) { + logger.error("script update_for_search has update error: " + response.getBulkFailures()); + return false; + } + + if (response.getStatus().getVersionConflicts() > 0) { + if (counter >= 3) { + logger.error("第 {} 次执行updateByQuery, 依旧存在分片版本冲突，不再继续重试。", counter); + return false; + } + logger.warn("本次updateByQuery存在分片版本冲突，准备重新执行..."); + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + // ignore + } + return updateByQuery(mapping, queryBuilder, esFieldData, ++counter); + } + + return true; + } + + /** + * 通过主键删除数据 + * + * @param mapping + * @param pkVal + * @return + */ + public boolean delete(ESMapping mapping, Object pkVal) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + if (mapping.get_id() != null) { + bulkRequestBuilder + .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), pkVal.toString())); + } else { + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .setSize(MAX_BATCH_SIZE) + .get(); + for (SearchHit hit : response.getHits()) { + bulkRequestBuilder + .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); + } + } + return commitBulkRequest(bulkRequestBuilder); + } + + /** + * 批量提交 + * + * @param bulkRequestBuilder + * @return + */ + private static boolean commitBulkRequest(BulkRequestBuilder bulkRequestBuilder) { + if (bulkRequestBuilder.numberOfActions() > 0) { + BulkResponse response = bulkRequestBuilder.execute().actionGet(); + if (response.hasFailures()) { + for (BulkItemResponse itemResponse : response.getItems()) { + if (!itemResponse.isFailed()) { + continue; + } + + if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { + logger.warn(itemResponse.getFailureMessage()); + } else { + logger.error("ES sync commit error: {}", itemResponse.getFailureMessage()); + } + } + } + + return !response.hasFailures(); + } + return true; + } + + public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName, + String columnName) throws SQLException { + String esType = getEsType(mapping, fieldName); + + Object value = resultSet.getObject(columnName); + if (value instanceof Boolean) { + if (!"boolean".equals(esType)) { + value = resultSet.getByte(columnName); + } + } + + // 如果是对象类型 + if (mapping.getObjFields().containsKey(fieldName)) { + return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); + } else { + return ESSyncUtil.typeConvert(value, esType); + } + } + + public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, + Map esFieldData) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + } + + if (!fieldItem.getFieldName().equals(mapping.get_id()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), value); + } + } + return resultIdVal; + } + + public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + break; + } + } + return resultIdVal; + } + + public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map dmlOld, + Map esFieldData) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + } + + for (ColumnItem columnItem : fieldItem.getColumnItems()) { + if (dmlOld.containsKey(columnItem.getColumnName()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), + getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName())); + break; + } + } + } + return resultIdVal; + } + + public Object getValFromData(ESMapping mapping, Map dmlData, String fieldName, String columnName) { + String esType = getEsType(mapping, fieldName); + Object value = dmlData.get(columnName); + if (value instanceof Byte) { + if ("boolean".equals(esType)) { + value = ((Byte) value).intValue() != 0; + } + } + + // 如果是对象类型 + if (mapping.getObjFields().containsKey(fieldName)) { + return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); + } else { + return ESSyncUtil.typeConvert(value, esType); + } + } + + /** + * 将dml的data转换为es的data + * + * @param mapping 配置mapping + * @param dmlData dml data + * @param esFieldData es data + * @return 返回 id 值 + */ + public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, + Map esFieldData) { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); + Object value = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + } + + if (!fieldItem.getFieldName().equals(mapping.get_id()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), value); + } + } + return resultIdVal; + } + + /** + * 将dml的data, old转换为es的data + * + * @param mapping 配置mapping + * @param dmlData dml data + * @param esFieldData es data + * @return 返回 id 值 + */ + public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, Map dmlOld, + Map esFieldData) { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); + } + + if (dmlOld.containsKey(columnName) && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), + getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName)); + } + } + return resultIdVal; + } + + /** + * es 字段类型本地缓存 + */ + private static ConcurrentMap> esFieldTypes = new ConcurrentHashMap<>(); + + /** + * 获取es mapping中的属性类型 + * + * @param mapping mapping配置 + * @param fieldName 属性名 + * @return 类型 + */ + @SuppressWarnings("unchecked") + private String getEsType(ESMapping mapping, String fieldName) { + String key = mapping.get_index() + "-" + mapping.get_type(); + Map fieldType = esFieldTypes.get(key); + if (fieldType == null) { + ImmutableOpenMap mappings; + try { + mappings = transportClient.admin() + .cluster() + .prepareState() + .execute() + .actionGet() + .getState() + .getMetaData() + .getIndices() + .get(mapping.get_index()) + .getMappings(); + } catch (NullPointerException e) { + throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); + } + MappingMetaData mappingMetaData = mappings.get(mapping.get_type()); + if (mappingMetaData == null) { + throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); + } + + fieldType = new LinkedHashMap<>(); + + Map sourceMap = mappingMetaData.getSourceAsMap(); + Map esMapping = (Map) sourceMap.get("properties"); + for (Map.Entry entry : esMapping.entrySet()) { + Map value = (Map) entry.getValue(); + if (value.containsKey("properties")) { + fieldType.put(entry.getKey(), "object"); + } else { + fieldType.put(entry.getKey(), (String) value.get("type")); + } + } + esFieldTypes.put(key, fieldType); + } + + return fieldType.get(fieldName); + } +} diff --git a/client-adapter/elasticsearch/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter b/client-adapter/elasticsearch/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter new file mode 100644 index 00000000..b30310ee --- /dev/null +++ b/client-adapter/elasticsearch/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter @@ -0,0 +1 @@ +es=com.alibaba.otter.canal.client.adapter.es.ESAdapter \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml new file mode 100644 index 00000000..9585227e --- /dev/null +++ b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml @@ -0,0 +1,15 @@ +dataSourceKey: defaultDS +destination: example +groupId: g1 +esMapping: + _index: mytest_user + _type: _doc + _id: _id +# pk: id + sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name, + a.c_time as _c_time from user a + left join role b on b.id=a.role_id" +# objFields: +# _labels: array:; + etlCondition: "where a.c_time>='{0}'" + commitBatch: 3000 diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java new file mode 100644 index 00000000..a66acbb9 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java @@ -0,0 +1,38 @@ +package com.alibaba.otter.canal.client.adapter.es.test; + +import java.util.Map; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; + +public class ConfigLoadTest { + + @Before + public void before() { + // AdapterConfigs.put("es", "mytest_user.yml"); + // 加载数据源连接池 + DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource); + } + + @Test + public void testLoad() { + Map configMap = ESSyncConfigLoader.load(null); + ESSyncConfig config = configMap.get("mytest_user.yml"); + Assert.assertNotNull(config); + Assert.assertEquals("defaultDS", config.getDataSourceKey()); + ESSyncConfig.ESMapping esMapping = config.getEsMapping(); + Assert.assertEquals("mytest_user", esMapping.get_index()); + Assert.assertEquals("_doc", esMapping.get_type()); + Assert.assertEquals("id", esMapping.get_id()); + Assert.assertNotNull(esMapping.getSql()); + + // Map> dbTableEsSyncConfig = + // ESSyncConfigLoader.getDbTableEsSyncConfig(); + // Assert.assertFalse(dbTableEsSyncConfig.isEmpty()); + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java new file mode 100644 index 00000000..d07defe9 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java @@ -0,0 +1,47 @@ +package com.alibaba.otter.canal.client.adapter.es.test; + +import java.util.List; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; +import com.alibaba.otter.canal.client.adapter.es.config.SqlParser; + +public class SqlParseTest { + + @Test + public void parseTest() { + String sql = "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a " + + "left join role b on a.role_id=b.id " + + "left join (select user_id, group_concat(label,',') as labels from user_label " + + "group by user_id) c on c.user_id=a.id"; + SchemaItem schemaItem = SqlParser.parse(sql); + + // 通过表名找 TableItem + List tableItems = schemaItem.getTableItemAliases().get("user_label".toLowerCase()); + tableItems.forEach(tableItem -> Assert.assertEquals("c", tableItem.getAlias())); + + TableItem tableItem = tableItems.get(0); + Assert.assertFalse(tableItem.isMain()); + Assert.assertTrue(tableItem.isSubQuery()); + // 通过字段名找 FieldItem + List fieldItems = schemaItem.getColumnFields().get(tableItem.getAlias() + ".label".toLowerCase()); + fieldItems.forEach( + fieldItem -> Assert.assertEquals("c.labels", fieldItem.getOwner() + "." + fieldItem.getFieldName())); + + // 获取当前表关联条件字段 + Map> relationTableFields = tableItem.getRelationTableFields(); + relationTableFields.keySet() + .forEach(fieldItem -> Assert.assertEquals("user_id", fieldItem.getColumn().getColumnName())); + + // 获取关联字段在select中的对应字段 + // List relationSelectFieldItem = + // tableItem.getRelationKeyFieldItems(); + // relationSelectFieldItem.forEach(fieldItem -> Assert.assertEquals("c.labels", + // fieldItem.getOwner() + "." + fieldItem.getColumn().getColumnName())); + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/TestConstant.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/TestConstant.java new file mode 100644 index 00000000..46b0f7a9 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/TestConstant.java @@ -0,0 +1,40 @@ +package com.alibaba.otter.canal.client.adapter.es.test; + +import java.sql.SQLException; + +import com.alibaba.druid.pool.DruidDataSource; + +public class TestConstant { + + public final static String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true"; + public final static String jdbcUser = "root"; + public final static String jdbcPassword = "121212"; + + public final static String esHosts = "127.0.0.1:9300"; + public final static String clusterNmae = "elasticsearch"; + + public static DruidDataSource dataSource; + + static { + dataSource = new DruidDataSource(); + dataSource.setDriverClassName("com.mysql.jdbc.Driver"); + dataSource.setUrl(jdbcUrl); + dataSource.setUsername(jdbcUser); + dataSource.setPassword(jdbcPassword); + dataSource.setInitialSize(1); + dataSource.setMinIdle(1); + dataSource.setMaxActive(1); + dataSource.setMaxWait(60000); + dataSource.setTimeBetweenEvictionRunsMillis(60000); + dataSource.setMinEvictableIdleTimeMillis(300000); + dataSource.setPoolPreparedStatements(false); + dataSource.setMaxPoolPreparedStatementPerConnectionSize(20); + dataSource.setValidationQuery("select 1"); + try { + dataSource.init(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java new file mode 100644 index 00000000..3f9c10d4 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java @@ -0,0 +1,68 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + +import javax.sql.DataSource; + +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.es.test.TestConstant; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; + +public class Common { + + public static ESAdapter init() { + DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource); + + OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig(); + outerAdapterConfig.setName("es"); + outerAdapterConfig.setHosts(TestConstant.esHosts); + Map properties = new HashMap<>(); + properties.put("cluster.name", TestConstant.clusterNmae); + outerAdapterConfig.setProperties(properties); + + ESAdapter esAdapter = new ESAdapter(); + esAdapter.init(outerAdapterConfig, null); + return esAdapter; + } + + public static void sqlExe(DataSource dataSource, String sql) { + Connection conn = null; + Statement stmt = null; + try { + conn = dataSource.getConnection(); + conn.setAutoCommit(false); + stmt = conn.createStatement(); + stmt.execute(sql); + conn.commit(); + } catch (Exception e) { + if (conn != null) { + try { + conn.rollback(); + } catch (SQLException e1) { + // ignore + } + } + e.printStackTrace(); + } finally { + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + // ignore + } + } + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + // ignore + } + } + } + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java new file mode 100644 index 00000000..966b115b --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java @@ -0,0 +1,130 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.util.*; + +import javax.sql.DataSource; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class LabelSyncJoinSub2Test { + + private ESAdapter esAdapter; + + @Before + public void init() { + // AdapterConfigs.put("es", "mytest_user_join_sub2.yml"); + esAdapter = Common.init(); + } + + /** + * 带函数子查询从表插入 + */ + @Test + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from label where id=1 or id=2"); + Common.sqlExe(ds, "insert into label (id,user_id,label) values (1,1,'a')"); + Common.sqlExe(ds, "insert into label (id,user_id,label) values (2,1,'b')"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 2L); + data.put("user_id", 1L); + data.put("label", "b"); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("b;a_", response.getSource().get("_labels")); + } + + /** + * 带函数子查询从表更新 + */ + @Test + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "update label set label='aa' where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("user_id", 1L); + data.put("label", "aa"); + dml.setData(dataList); + + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("label", "v"); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("b;aa_", response.getSource().get("_labels")); + } + + /** + * 带函数子查询从表删除 + */ + @Test + public void test03() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from label where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("DELETE"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("user_id", 1L); + data.put("label", "a"); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("b_", response.getSource().get("_labels")); + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java new file mode 100644 index 00000000..e8cbbe59 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java @@ -0,0 +1,130 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.util.*; + +import javax.sql.DataSource; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class LabelSyncJoinSubTest { + + private ESAdapter esAdapter; + + @Before + public void init() { + // AdapterConfigs.put("es", "mytest_user_join_sub.yml"); + esAdapter = Common.init(); + } + + /** + * 子查询从表插入 + */ + @Test + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from label where id=1 or id=2"); + Common.sqlExe(ds, "insert into label (id,user_id,label) values (1,1,'a')"); + Common.sqlExe(ds, "insert into label (id,user_id,label) values (2,1,'b')"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 2L); + data.put("user_id", 1L); + data.put("label", "b"); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("b;a", response.getSource().get("_labels")); + } + + /** + * 子查询从表更新 + */ + @Test + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "update label set label='aa' where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("user_id", 1L); + data.put("label", "aa"); + dml.setData(dataList); + + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("label", "a"); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("b;aa", response.getSource().get("_labels")); + } + + /** + * 子查询从表删除 + */ + @Test + public void test03() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from label where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("DELETE"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("user_id", 1L); + data.put("label", "a"); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("b", response.getSource().get("_labels")); + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java new file mode 100644 index 00000000..0fba73d3 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java @@ -0,0 +1,95 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.util.*; + +import javax.sql.DataSource; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class RoleSyncJoinOne2Test { + + private ESAdapter esAdapter; + + @Before + public void init() { + // AdapterConfigs.put("es", "mytest_user_join_one2.yml"); + esAdapter = Common.init(); + } + + /** + * 带函数非子查询从表插入 + */ + @Test + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from role where id=1"); + Common.sqlExe(ds, "insert into role (id,role_name) values (1,'admin')"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin"); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("admin_", response.getSource().get("_role_name")); + } + + /** + * 带函数非子查询从表更新 + */ + @Test + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "update role set role_name='admin3' where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin3"); + dml.setData(dataList); + + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("role_name", "admin"); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("admin3_", response.getSource().get("_role_name")); + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java new file mode 100644 index 00000000..9739a355 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java @@ -0,0 +1,191 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.util.*; + +import javax.sql.DataSource; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class RoleSyncJoinOneTest { + + private ESAdapter esAdapter; + + @Before + public void init() { + // AdapterConfigs.put("es", "mytest_user_join_one.yml"); + esAdapter = Common.init(); + } + + /** + * 非子查询从表插入 + */ + @Test + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from role where id=1"); + Common.sqlExe(ds, "insert into role (id,role_name) values (1,'admin')"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin"); + + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("admin", response.getSource().get("_role_name")); + } + + /** + * 非子查询从表更新 + */ + @Test + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "update role set role_name='admin2' where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin2"); + dml.setData(dataList); + + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("role_name", "admin"); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("admin2", response.getSource().get("_role_name")); + } + + /** + * 主表更新外键值 + */ + @Test + public void test03() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from role where id=2"); + Common.sqlExe(ds, "insert into role (id,role_name) values (2,'operator')"); + Common.sqlExe(ds, "update user set role_id=2 where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_id", 2L); + dml.setData(dataList); + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("role_id", 1L); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("operator", response.getSource().get("_role_name")); + + Common.sqlExe(ds, "update user set role_id=1 where id=1"); + + Dml dml2 = new Dml(); + dml2.setDestination("example"); + dml2.setTs(new Date().getTime()); + dml2.setType("UPDATE"); + dml2.setDatabase("mytest"); + dml2.setTable("user"); + List> dataList2 = new ArrayList<>(); + Map data2 = new LinkedHashMap<>(); + dataList2.add(data2); + data2.put("id", 1L); + data2.put("role_id", 1L); + dml2.setData(dataList2); + List> oldList2 = new ArrayList<>(); + Map old2 = new LinkedHashMap<>(); + oldList2.add(old2); + old2.put("role_id", 2L); + dml2.setOld(oldList2); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml2); + + GetResponse response2 = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("admin2", response2.getSource().get("_role_name")); + } + + /** + * 非子查询从表删除 + */ + @Test + public void test04() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from role where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("DELETE"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin"); + + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertNull(response.getSource().get("_role_name")); + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java new file mode 100644 index 00000000..c0e43b61 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java @@ -0,0 +1,96 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.util.*; + +import javax.sql.DataSource; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class UserSyncJoinOneTest { + + private ESAdapter esAdapter; + + @Before + public void init() { + // AdapterConfigs.put("es", "mytest_user_join_one.yml"); + esAdapter = Common.init(); + } + + /** + * 主表带函数插入 + */ + @Test + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from user where id=1"); + Common.sqlExe(ds, "insert into user (id,name,role_id) values (1,'Eric',1)"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + data.put("role_id", 1L); + data.put("c_time", new Date()); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("Eric_", response.getSource().get("_name")); + } + + /** + * 主表带函数更新 + */ + @Test + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "update user set name='Eric2' where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric2"); + dml.setData(dataList); + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("name", "Eric"); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("Eric2_", response.getSource().get("_name")); + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java new file mode 100644 index 00000000..6bf4b75b --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java @@ -0,0 +1,122 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.util.*; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class UserSyncSingleTest { + + private ESAdapter esAdapter; + + @Before + public void init() { + // AdapterConfigs.put("es", "mytest_user_single.yml"); + esAdapter = Common.init(); + } + + /** + * 单表插入 + */ + @Test + public void test01() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + data.put("role_id", 1L); + data.put("c_time", new Date()); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("Eric", response.getSource().get("_name")); + } + + /** + * 单表更新 + */ + @Test + public void test02() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric2"); + dml.setData(dataList); + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("name", "Eric"); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("Eric2", response.getSource().get("_name")); + } + + /** + * 单表删除 + */ + @Test + public void test03() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("DELETE"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + data.put("role_id", 1L); + data.put("c_time", new Date()); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertNull(response.getSource()); + } + + // @After + // public void after() { + // esAdapter.destroy(); + // DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); + // } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql new file mode 100644 index 00000000..bff2e118 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql @@ -0,0 +1,39 @@ +-- ---------------------------- +-- Table structure for label +-- ---------------------------- +DROP TABLE IF EXISTS `label`; +CREATE TABLE `label` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `user_id` bigint(20) NOT NULL, + `label` varchar(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for role +-- ---------------------------- +DROP TABLE IF EXISTS `role`; +CREATE TABLE `role` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `role_name` varchar(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for user +-- ---------------------------- +DROP TABLE IF EXISTS `user`; +CREATE TABLE `user` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(30) NOT NULL, + `c_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `role_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; + +insert into user (id,name,role_id) values (1,'Eric',1); +insert into role (id,role_name) values (1,'admin'); +insert into role (id,role_name) values (2,'operator'); +insert into label (id,user_id,label) values (1,1,'a'); +insert into label (id,user_id,label) values (2,1,'b'); +commit; \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/es_mapping.json b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/es_mapping.json new file mode 100644 index 00000000..55b0389a --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/es_mapping.json @@ -0,0 +1,21 @@ +{ + "_doc": { + "properties": { + "_name": { + "type": "text" + }, + "_role_id": { + "type": "long" + }, + "_role_name": { + "type": "text" + }, + "_labels": { + "type": "text" + }, + "_c_time": { + "type": "date" + } + } + } +} \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml_ b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml_ new file mode 100644 index 00000000..23e1b491 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml_ @@ -0,0 +1,8 @@ +dataSourceKey: defaultDS +destination: example +esMapping: + _index: mytest_user + _type: _doc + _id: _id + sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, a.c_time as _c_time from user a" + commitBatch: 3000 \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/resources/log4j2-test.xml b/client-adapter/elasticsearch/src/test/resources/log4j2-test.xml new file mode 100644 index 00000000..9c3d2ee6 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/resources/log4j2-test.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/resources/logback-test.xml b/client-adapter/elasticsearch/src/test/resources/logback-test.xml new file mode 100644 index 00000000..91f0de80 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/resources/logback-test.xml @@ -0,0 +1,13 @@ + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + + \ No newline at end of file diff --git a/client-adapter/hbase/pom.xml b/client-adapter/hbase/pom.xml index 1437ed5d..87e61c92 100644 --- a/client-adapter/hbase/pom.xml +++ b/client-adapter/hbase/pom.xml @@ -1,11 +1,9 @@ - + canal.client-adapter com.alibaba.otter - 1.1.1-SNAPSHOT + 1.1.3-SNAPSHOT 4.0.0 com.alibaba.otter @@ -19,11 +17,6 @@ ${project.version} provided - - org.yaml - snakeyaml - 1.17 - org.apache.hbase hbase-client @@ -44,38 +37,6 @@ - - maven-jar-plugin - - - true - - - **/hbase-mapping/** - - - - - maven-antrun-plugin - - - package - - run - - - - - - - - - - - - - - org.apache.maven.plugins maven-assembly-plugin @@ -95,6 +56,26 @@ + + maven-antrun-plugin + + + package + + run + + + + + + + + + + + + + diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index bbd8fb1c..c307aec9 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -1,25 +1,31 @@ package com.alibaba.otter.canal.client.adapter.hbase; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +import javax.sql.DataSource; -import com.alibaba.otter.canal.protocol.FlatMessage; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfigLoader; +import com.alibaba.otter.canal.client.adapter.hbase.monitor.HbaseConfigMonitor; +import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseEtlService; import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseSyncService; -import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.client.adapter.support.MessageUtil; -import com.alibaba.otter.canal.client.adapter.support.SPI; -import com.alibaba.otter.canal.protocol.Message; +import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate; +import com.alibaba.otter.canal.client.adapter.support.*; /** * HBase外部适配器 @@ -28,83 +34,202 @@ import com.alibaba.otter.canal.protocol.Message; * @version 1.0.0 */ @SPI("hbase") -public class HbaseAdapter implements CanalOuterAdapter { +public class HbaseAdapter implements OuterAdapter { - private static volatile Map mappingConfigCache = null; + private static Logger logger = LoggerFactory.getLogger(HbaseAdapter.class); - private Connection conn; - private HbaseSyncService hbaseSyncService; + private Map hbaseMapping = new ConcurrentHashMap<>(); // 文件名对应配置 + private Map> mappingConfigCache = new ConcurrentHashMap<>(); // 库名-表名对应配置 + + private HbaseSyncService hbaseSyncService; + private HbaseTemplate hbaseTemplate; + + private HbaseConfigMonitor configMonitor; + + private Properties envProperties; + + public Map getHbaseMapping() { + return hbaseMapping; + } + + public Map> getMappingConfigCache() { + return mappingConfigCache; + } @Override - public void init(CanalOuterAdapterConfiguration configuration) { + public void init(OuterAdapterConfig configuration, Properties envProperties) { try { - if (mappingConfigCache == null) { - synchronized (MappingConfig.class) { - if (mappingConfigCache == null) { - Map hbaseMapping = MappingConfigLoader.load(); - mappingConfigCache = new HashMap<>(); - for (MappingConfig mappingConfig : hbaseMapping.values()) { - mappingConfigCache.put(mappingConfig.getHbaseOrm().getDatabase() + "-" - + mappingConfig.getHbaseOrm().getTable(), - mappingConfig); - } - } + this.envProperties = envProperties; + Map hbaseMappingTmp = MappingConfigLoader.load(envProperties); + // 过滤不匹配的key的配置 + hbaseMappingTmp.forEach((key, mappingConfig) -> { + if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null) + || (mappingConfig.getOuterAdapterKey() != null + && mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) { + hbaseMapping.put(key, mappingConfig); } + }); + for (Map.Entry entry : hbaseMapping.entrySet()) { + String configName = entry.getKey(); + MappingConfig mappingConfig = entry.getValue(); + String k; + if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-" + + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_" + + mappingConfig.getHbaseMapping().getDatabase() + "-" + + mappingConfig.getHbaseMapping().getTable(); + } else { + k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_" + + mappingConfig.getHbaseMapping().getDatabase() + "-" + + mappingConfig.getHbaseMapping().getTable(); + } + Map configMap = mappingConfigCache.computeIfAbsent(k, + k1 -> new ConcurrentHashMap<>()); + configMap.put(configName, mappingConfig); } - String hosts = configuration.getZkHosts(); - if (StringUtils.isEmpty(hosts)) { - hosts = configuration.getHosts(); - } - if (StringUtils.isEmpty(hosts)) { - throw new RuntimeException("Empty zookeeper hosts"); - } - String[] zkHosts = StringUtils.split(hosts, ","); - int zkPort = 0; - StringBuilder hostsWithoutPort = new StringBuilder(); - for (String host : zkHosts) { - int i = host.indexOf(":"); - hostsWithoutPort.append(host, 0, i); - hostsWithoutPort.append(","); - if (zkPort == 0) zkPort = Integer.parseInt(host.substring(i + 1)); - } - hostsWithoutPort.deleteCharAt(hostsWithoutPort.length() - 1); - - String znode = configuration.getProperties().getProperty("znodeParent"); - if (StringUtils.isEmpty(znode)) { - znode = "/hbase"; - } + Map properties = configuration.getProperties(); Configuration hbaseConfig = HBaseConfiguration.create(); - hbaseConfig.set("hbase.zookeeper.quorum", hostsWithoutPort.toString()); - hbaseConfig.set("hbase.zookeeper.property.clientPort", Integer.toString(zkPort)); - hbaseConfig.set("zookeeper.znode.parent", znode); - conn = ConnectionFactory.createConnection(hbaseConfig); - hbaseSyncService = new HbaseSyncService(conn); + properties.forEach(hbaseConfig::set); + hbaseTemplate = new HbaseTemplate(hbaseConfig); + hbaseSyncService = new HbaseSyncService(hbaseTemplate); + + configMonitor = new HbaseConfigMonitor(); + configMonitor.init(this); } catch (Exception e) { throw new RuntimeException(e); } } - @Override - public void writeOut(Dml dml) { + public void sync(List dmls) { + if (dmls == null || dmls.isEmpty()) { + return; + } + for (Dml dml : dmls) { + sync(dml); + } + } + + public void sync(Dml dml) { if (dml == null) { return; } + String destination = StringUtils.trimToEmpty(dml.getDestination()); + String groupId = StringUtils.trimToEmpty(dml.getGroupId()); String database = dml.getDatabase(); String table = dml.getTable(); - MappingConfig config = mappingConfigCache.get(database + "-" + table); - hbaseSyncService.sync(config, dml); + Map configMap; + if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + configMap = mappingConfigCache.get(destination + "-" + groupId + "_" + database + "-" + table); + } else { + configMap = mappingConfigCache.get(destination + "_" + database + "-" + table); + } + if (configMap != null) { + List configs = new ArrayList<>(); + configMap.values().forEach(config -> { + if (StringUtils.isNotEmpty(config.getGroupId())) { + if (config.getGroupId().equals(dml.getGroupId())) { + configs.add(config); + } + } else { + configs.add(config); + } + }); + if (!configs.isEmpty()) { + configs.forEach(config -> hbaseSyncService.sync(config, dml)); + } + } + } + + @Override + public EtlResult etl(String task, List params) { + EtlResult etlResult = new EtlResult(); + MappingConfig config = hbaseMapping.get(task); + if (config != null) { + DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (dataSource != null) { + return HbaseEtlService.importData(dataSource, hbaseTemplate, config, params); + } else { + etlResult.setSucceeded(false); + etlResult.setErrorMessage("DataSource not found"); + return etlResult; + } + } else { + StringBuilder resultMsg = new StringBuilder(); + boolean resSucc = true; + // ds不为空说明传入的是datasourceKey + for (MappingConfig configTmp : hbaseMapping.values()) { + // 取所有的destination为task的配置 + if (configTmp.getDestination().equals(task)) { + DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(configTmp.getDataSourceKey()); + if (dataSource == null) { + continue; + } + EtlResult etlRes = HbaseEtlService.importData(dataSource, hbaseTemplate, configTmp, params); + if (!etlRes.getSucceeded()) { + resSucc = false; + resultMsg.append(etlRes.getErrorMessage()).append("\n"); + } else { + resultMsg.append(etlRes.getResultMessage()).append("\n"); + } + } + } + if (resultMsg.length() > 0) { + etlResult.setSucceeded(resSucc); + if (resSucc) { + etlResult.setResultMessage(resultMsg.toString()); + } else { + etlResult.setErrorMessage(resultMsg.toString()); + } + return etlResult; + } + } + etlResult.setSucceeded(false); + etlResult.setErrorMessage("Task not found"); + return etlResult; + } + + @Override + public Map count(String task) { + MappingConfig config = hbaseMapping.get(task); + String hbaseTable = config.getHbaseMapping().getHbaseTable(); + long rowCount = 0L; + try { + HTable table = (HTable) hbaseTemplate.getConnection().getTable(TableName.valueOf(hbaseTable)); + Scan scan = new Scan(); + scan.setFilter(new FirstKeyOnlyFilter()); + ResultScanner resultScanner = table.getScanner(scan); + for (Result result : resultScanner) { + rowCount += result.size(); + } + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + Map res = new LinkedHashMap<>(); + res.put("hbaseTable", hbaseTable); + res.put("count", rowCount); + return res; } @Override public void destroy() { - if (conn != null) { - try { - conn.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } + if (configMonitor != null) { + configMonitor.destroy(); + } + try { + hbaseTemplate.close(); + } catch (IOException e) { + throw new RuntimeException(e); } } + + @Override + public String getDestination(String task) { + MappingConfig config = hbaseMapping.get(task); + if (config != null && config.getHbaseMapping() != null) { + return config.getDestination(); + } + return null; + } } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java index 0e4cbfe6..3d1c38be 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java @@ -5,35 +5,75 @@ import java.util.*; /** * HBase表映射配置 * - * @author machengyuan 2018-8-21 下午06:45:49 + * @author rewerma 2018-8-21 下午06:45:49 * @version 1.0.0 */ public class MappingConfig { - private HbaseOrm hbaseOrm; + private String dataSourceKey; // 数据源key - public HbaseOrm getHbaseOrm() { - return hbaseOrm; + private String outerAdapterKey; // adapter key + + private String groupId; // groupId + + private String destination; // canal实例或MQ的topic + + private HbaseMapping hbaseMapping; // hbase映射配置 + + public String getDataSourceKey() { + return dataSourceKey; } - public void setHbaseOrm(HbaseOrm hbaseOrm) { - this.hbaseOrm = hbaseOrm; + public void setDataSourceKey(String dataSourceKey) { + this.dataSourceKey = dataSourceKey; + } + + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + + public String getOuterAdapterKey() { + return outerAdapterKey; + } + + public void setOuterAdapterKey(String outerAdapterKey) { + this.outerAdapterKey = outerAdapterKey; + } + + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public HbaseMapping getHbaseMapping() { + return hbaseMapping; + } + + public void setHbaseMapping(HbaseMapping hbaseMapping) { + this.hbaseMapping = hbaseMapping; } public void validate() { - if (hbaseOrm.database == null || hbaseOrm.database.isEmpty()) { - throw new NullPointerException("hbaseOrm.database"); + if (hbaseMapping.database == null || hbaseMapping.database.isEmpty()) { + throw new NullPointerException("hbaseMapping.database"); } - if (hbaseOrm.table == null || hbaseOrm.table.isEmpty()) { - throw new NullPointerException("hbaseOrm.table"); + if (hbaseMapping.table == null || hbaseMapping.table.isEmpty()) { + throw new NullPointerException("hbaseMapping.table"); } - if (hbaseOrm.hbaseTable == null || hbaseOrm.hbaseTable.isEmpty()) { - throw new NullPointerException("hbaseOrm.hbaseTable"); + if (hbaseMapping.hbaseTable == null || hbaseMapping.hbaseTable.isEmpty()) { + throw new NullPointerException("hbaseMapping.hbaseTable"); } - if (hbaseOrm.mode == null) { - throw new NullPointerException("hbaseOrm.mode"); + if (hbaseMapping.mode == null) { + throw new NullPointerException("hbaseMapping.mode"); } - if (hbaseOrm.rowKey != null && hbaseOrm.rowKeyColumn != null) { + if (hbaseMapping.rowKey != null && hbaseMapping.rowKeyColumn != null) { throw new RuntimeException("已配置了复合主键作为RowKey，无需再指定RowKey列"); } } @@ -45,17 +85,18 @@ public class MappingConfig { MappingConfig config = (MappingConfig) o; - return hbaseOrm != null ? hbaseOrm.equals(config.hbaseOrm) : config.hbaseOrm == null; + return hbaseMapping != null ? hbaseMapping.equals(config.hbaseMapping) : config.hbaseMapping == null; } @Override public int hashCode() { - return hbaseOrm != null ? hbaseOrm.hashCode() : 0; + return hbaseMapping != null ? hbaseMapping.hashCode() : 0; } public static class ColumnItem { private boolean isRowKey = false; + private Integer rowKeyLen; private String column; private String family; private String qualifier; @@ -69,6 +110,14 @@ public class MappingConfig { isRowKey = rowKey; } + public Integer getRowKeyLen() { + return rowKeyLen; + } + + public void setRowKeyLen(Integer rowKeyLen) { + this.rowKeyLen = rowKeyLen; + } + public String getColumn() { return column; } @@ -130,24 +179,25 @@ public class MappingConfig { } } - public static class HbaseOrm { + public static class HbaseMapping { - private Mode mode = Mode.STRING; - private String database; - private String table; - private String hbaseTable; - private String family = "CF"; - private boolean uppercaseQualifier = true; - private boolean autoCreateTable = false; // 同步时HBase中表不存在的情况下自动建表 - private String rowKey; // 指定复合主键为rowKey - private Map columns; - private ColumnItem rowKeyColumn; - private String etlCondition; + private Mode mode = Mode.STRING; // hbase默认转换格式 + private String database; // 数据库名或schema名 + private String table; // 表面名 + private String hbaseTable; // hbase表名 + private String family = "CF"; // 默认统一column family + private boolean uppercaseQualifier = true; // 是否转大写 + private boolean autoCreateTable = false; // 同步时HBase中表不存在的情况下自动建表 + private String rowKey; // 指定复合主键为rowKey + private Map columns; // 字段映射 + private List excludeColumns; // 不映射的字段 + private ColumnItem rowKeyColumn; // rowKey字段 + private String etlCondition; // etl条件sql - private Map columnItems = new LinkedHashMap<>(); - private Set families = new LinkedHashSet<>(); + private Map columnItems = new LinkedHashMap<>(); // 转换后的字段映射列表 + private Set families = new LinkedHashSet<>(); // column family列表 private int readBatch = 5000; - private int commitBatch = 5000; + private int commitBatch = 5000; // etl等批量提交大小 public Mode getMode() { return mode; @@ -243,7 +293,16 @@ public class MappingConfig { ColumnItem columnItem = new ColumnItem(); columnItem.setColumn(columnField.getKey()); columnItem.setType(type); - if ("rowKey".equalsIgnoreCase(field)) { + if (field != null && field.toUpperCase().startsWith("ROWKEY")) { + int idx = field.toUpperCase().indexOf("LEN:"); + if (idx > -1) { + String len = field.substring(idx + 4); + try { + columnItem.setRowKeyLen(Integer.parseInt(len)); + } catch (Exception e) { + // ignore + } + } columnItem.setRowKey(true); rowKeyColumn = columnItem; } else { @@ -273,6 +332,14 @@ public class MappingConfig { } } + public List getExcludeColumns() { + return excludeColumns; + } + + public void setExcludeColumns(List excludeColumns) { + this.excludeColumns = excludeColumns; + } + public String getFamily() { return family; } @@ -321,10 +388,10 @@ public class MappingConfig { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - HbaseOrm hbaseOrm = (HbaseOrm) o; + HbaseMapping hbaseMapping = (HbaseMapping) o; - if (table != null ? !table.equals(hbaseOrm.table) : hbaseOrm.table != null) return false; - return hbaseTable != null ? hbaseTable.equals(hbaseOrm.hbaseTable) : hbaseOrm.hbaseTable == null; + if (table != null ? !table.equals(hbaseMapping.table) : hbaseMapping.table != null) return false; + return hbaseTable != null ? hbaseTable.equals(hbaseMapping.hbaseTable) : hbaseMapping.hbaseTable == null; } @Override diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java index 929e9d6a..2b43492f 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java @@ -1,149 +1,51 @@ package com.alibaba.otter.canal.client.adapter.hbase.config; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Properties; -import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.Yaml; + +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; +import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; /** * HBase表映射配置加载器 - *

    - * 配置统一从hbase-mapping/configs.conf文件作为入口, 该文件包含所有表映射配置的名称或者文件名列表。 - * 每个对应的表配置可以yml配置文件或者以database.table为配置名的简化形式 - *

    * - * @author machengyuan 2018-8-21 下午06:45:49 + * @author rewerma 2018-8-21 下午06:45:49 * @version 1.0.0 */ public class MappingConfigLoader { - private static Logger logger = LoggerFactory.getLogger(MappingConfigLoader.class); - - private static final String BASE_PATH = "hbase-mapping/"; + private static Logger logger = LoggerFactory.getLogger(MappingConfigLoader.class); /** * 加载HBase表映射配置 - * + * * @return 配置名/配置文件名--对象 */ - public static Map load() { - logger.info("## Start loading mapping config ... "); - String mappingConfigContent = readConfigContent(BASE_PATH + "configs.conf"); + public static Map load(Properties envProperties) { + logger.info("## Start loading hbase mapping config ... "); Map result = new LinkedHashMap<>(); - String[] configLines = mappingConfigContent.split("\n"); - for (String c : configLines) { - if (c == null) { - continue; + Map configContentMap = MappingConfigsLoader.loadConfigs("hbase"); + configContentMap.forEach((fileName, content) -> { + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); + if (config == null) { + return; } - c = c.trim(); - if (c.equals("") || c.startsWith("#")) { - continue; - } - - MappingConfig config; - String configContent = null; - - if (c.endsWith(".yml")) { - configContent = readConfigContent(BASE_PATH + "/" + c); - } - - // 简单配置database.table@datasourcekey?rowKey=key1,key2 - if (StringUtils.isEmpty(configContent)) { - String[] mapping = c.split("\\?"); - String params = mapping.length == 2 ? mapping[1] : null; - String rowKey = null; - String srcMeta = mapping[0]; - // - if (params != null) { - for (String entry : params.split("&")) { - if ("rowKey".equals(entry.split("=")[0])) { - rowKey = entry.split("=")[1]; - } - } - } - String dsKey = srcMeta.split("@").length == 2 ? srcMeta.split("@")[1] : null; - String[] dbTable; - if (dsKey == null) { - dbTable = srcMeta.split("\\."); - } else { - dbTable = srcMeta.split("@")[0].split("\\."); - } - - if (dbTable.length == 2) { - config = new MappingConfig(); - - MappingConfig.HbaseOrm hbaseOrm = new MappingConfig.HbaseOrm(); - hbaseOrm.setHbaseTable(dbTable[0].toUpperCase() + "." + dbTable[1].toUpperCase()); - hbaseOrm.setAutoCreateTable(true); - hbaseOrm.setDatabase(dbTable[0]); - hbaseOrm.setTable(dbTable[1]); - hbaseOrm.setMode(MappingConfig.Mode.STRING); - hbaseOrm.setRowKey(rowKey); - // 有定义rowKey - if (rowKey != null) { - MappingConfig.ColumnItem columnItem = new MappingConfig.ColumnItem(); - columnItem.setRowKey(true); - columnItem.setColumn(rowKey); - hbaseOrm.setRowKeyColumn(columnItem); - } - config.setHbaseOrm(hbaseOrm); - - } else { - throw new RuntimeException(String.format("配置项[%s]内容为空, 或格式不符合database.table", c)); - } - - } else { // 配置文件配置 - config = new Yaml().loadAs(configContent, MappingConfig.class); - } - try { config.validate(); } catch (Exception e) { - throw new RuntimeException("ERROR Config: " + c + " " + e.getMessage(), e); + throw new RuntimeException("ERROR load Config: " + fileName + " " + e.getMessage(), e); } - result.put(c, config); - } + result.put(fileName, config); + }); - logger.info("## Mapping config loaded"); + logger.info("## Hbase mapping config loaded"); return result; } - - public static String readConfigContent(String config) { - InputStream in = null; - try { - // 先取本地文件，再取类路径 - File configFile = new File("config/" + config); - if (configFile.exists()) { - in = new FileInputStream(configFile); - } else { - in = MappingConfigLoader.class.getClassLoader().getResourceAsStream(config); - } - if (in == null) { - throw new RuntimeException("Config file not found."); - } - - byte[] bytes = new byte[in.available()]; - in.read(bytes); - return new String(bytes, "UTF-8"); - } catch (IOException e) { - throw new RuntimeException("Read ds-config.yml or hbase-mappings.conf error. ", e); - } finally { - try { - if (in != null) { - in.close(); - } - } catch (IOException e) { - // ignore - } - } - } } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/monitor/HbaseConfigMonitor.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/monitor/HbaseConfigMonitor.java new file mode 100644 index 00000000..803bbe1c --- /dev/null +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/monitor/HbaseConfigMonitor.java @@ -0,0 +1,133 @@ +package com.alibaba.otter.canal.client.adapter.hbase.monitor; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.io.monitor.FileAlterationListenerAdaptor; +import org.apache.commons.io.monitor.FileAlterationMonitor; +import org.apache.commons.io.monitor.FileAlterationObserver; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; + +import com.alibaba.otter.canal.client.adapter.hbase.HbaseAdapter; +import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; +import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; +import com.alibaba.otter.canal.client.adapter.support.Util; + +public class HbaseConfigMonitor { + + private static final Logger logger = LoggerFactory.getLogger(HbaseConfigMonitor.class); + + private static final String adapterName = "hbase"; + + private HbaseAdapter hbaseAdapter; + + private FileAlterationMonitor fileMonitor; + + public void init(HbaseAdapter hbaseAdapter) { + this.hbaseAdapter = hbaseAdapter; + File confDir = Util.getConfDirPath(adapterName); + try { + FileAlterationObserver observer = new FileAlterationObserver(confDir, + FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml"))); + FileListener listener = new FileListener(); + observer.addListener(listener); + fileMonitor = new FileAlterationMonitor(3000, observer); + fileMonitor.start(); + + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + public void destroy() { + try { + fileMonitor.stop(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private class FileListener extends FileAlterationListenerAdaptor { + + @Override + public void onFileCreate(File file) { + super.onFileCreate(file); + try { + // 加载新增的配置文件 + String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); + MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class); + config.validate(); + addConfigToCache(file, config); + + logger.info("Add a new hbase mapping config: {} to canal adapter", file.getName()); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public void onFileChange(File file) { + super.onFileChange(file); + + try { + if (hbaseAdapter.getHbaseMapping().containsKey(file.getName())) { + // 加载配置文件 + String configContent = MappingConfigsLoader + .loadConfig(adapterName + File.separator + file.getName()); + if (configContent == null) { + onFileDelete(file); + return; + } + MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class); + config.validate(); + if (hbaseAdapter.getHbaseMapping().containsKey(file.getName())) { + deleteConfigFromCache(file); + } + addConfigToCache(file, config); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public void onFileDelete(File file) { + super.onFileDelete(file); + + try { + if (hbaseAdapter.getHbaseMapping().containsKey(file.getName())) { + deleteConfigFromCache(file); + + logger.info("Delete a hbase mapping config: {} of canal adapter", file.getName()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private void addConfigToCache(File file, MappingConfig config) { + hbaseAdapter.getHbaseMapping().put(file.getName(), config); + Map configMap = hbaseAdapter.getMappingConfigCache() + .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "." + + config.getHbaseMapping().getDatabase() + "." + config.getHbaseMapping().getTable(), + k1 -> new HashMap<>()); + configMap.put(file.getName(), config); + } + + private void deleteConfigFromCache(File file) { + + hbaseAdapter.getHbaseMapping().remove(file.getName()); + for (Map configMap : hbaseAdapter.getMappingConfigCache().values()) { + if (configMap != null) { + configMap.remove(file.getName()); + } + } + + } + } +} diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java new file mode 100644 index 00000000..d5a2ef58 --- /dev/null +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java @@ -0,0 +1,397 @@ +package com.alibaba.otter.canal.client.adapter.hbase.service; + +import java.sql.ResultSetMetaData; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import javax.sql.DataSource; + +import org.apache.hadoop.hbase.util.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; +import com.alibaba.otter.canal.client.adapter.hbase.support.HRow; +import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate; +import com.alibaba.otter.canal.client.adapter.hbase.support.PhType; +import com.alibaba.otter.canal.client.adapter.hbase.support.PhTypeUtil; +import com.alibaba.otter.canal.client.adapter.hbase.support.Type; +import com.alibaba.otter.canal.client.adapter.hbase.support.TypeUtil; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil; +import com.alibaba.otter.canal.client.adapter.support.Util; +import com.google.common.base.Joiner; + +/** + * HBase ETL 操作业务类 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +public class HbaseEtlService { + + private static Logger logger = LoggerFactory.getLogger(HbaseEtlService.class); + + + /** + * 建表 + * + * @param hbaseTemplate + * @param config + */ + public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) { + try { + // 判断hbase表是否存在，不存在则建表 + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); + if (!hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) { + hbaseTemplate.createTable(hbaseMapping.getHbaseTable(), hbaseMapping.getFamily()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + /** + * 导入数据 + * + * @param ds 数据源 + * @param hbaseTemplate hbaseTemplate + * @param config 配置 + * @param params 筛选条件 + * @return 导入结果 + */ + public static EtlResult importData(DataSource ds, HbaseTemplate hbaseTemplate, MappingConfig config, + List params) { + EtlResult etlResult = new EtlResult(); + AtomicLong successCount = new AtomicLong(); + List errMsg = new ArrayList<>(); + String hbaseTable = ""; + try { + if (config == null) { + logger.error("Config is null!"); + etlResult.setSucceeded(false); + etlResult.setErrorMessage("Config is null!"); + return etlResult; + } + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); + hbaseTable = hbaseMapping.getHbaseTable(); + + long start = System.currentTimeMillis(); + + if (params != null && params.size() == 1 && "rebuild".equalsIgnoreCase(params.get(0))) { + logger.info(hbaseMapping.getHbaseTable() + " rebuild is starting!"); + // 如果表存在则删除 + if (hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) { + hbaseTemplate.disableTable(hbaseMapping.getHbaseTable()); + hbaseTemplate.deleteTable(hbaseMapping.getHbaseTable()); + } + params = null; + } else { + logger.info(hbaseMapping.getHbaseTable() + " etl is starting!"); + } + createTable(hbaseTemplate, config); + + // 拼接sql + String sql = "SELECT * FROM " + config.getHbaseMapping().getDatabase() + "." + hbaseMapping.getTable(); + + // 拼接条件 + if (params != null && params.size() == 1 && hbaseMapping.getEtlCondition() == null) { + AtomicBoolean stExists = new AtomicBoolean(false); + // 验证是否有SYS_TIME字段 + Util.sqlRS(ds, sql, rs -> { + try { + ResultSetMetaData rsmd = rs.getMetaData(); + int cnt = rsmd.getColumnCount(); + for (int i = 1; i <= cnt; i++) { + String columnName = rsmd.getColumnName(i); + if ("SYS_TIME".equalsIgnoreCase(columnName)) { + stExists.set(true); + break; + } + } + } catch (Exception e) { + // ignore + } + return null; + }); + if (stExists.get()) { + sql += " WHERE SYS_TIME >= '" + params.get(0) + "' "; + } + } else if (hbaseMapping.getEtlCondition() != null && params != null) { + String etlCondition = hbaseMapping.getEtlCondition(); + int size = params.size(); + for (int i = 0; i < size; i++) { + etlCondition = etlCondition.replace("{" + i + "}", params.get(i)); + } + + sql += " " + etlCondition; + } + + // 获取总数 + String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT "; + long cnt = (Long) Util.sqlRS(ds, countSql, rs -> { + Long count = null; + try { + if (rs.next()) { + count = ((Number) rs.getObject(1)).longValue(); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + return count == null ? 0 : count; + }); + + // 当大于1万条记录时开启多线程 + if (cnt >= 10000) { + int threadCount = 3; + long perThreadCnt = cnt / threadCount; + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + List> futures = new ArrayList<>(threadCount); + for (int i = 0; i < threadCount; i++) { + long offset = i * perThreadCnt; + Long size = null; + if (i != threadCount - 1) { + size = perThreadCnt; + } + String sqlFinal; + if (size != null) { + sqlFinal = sql + " LIMIT " + offset + "," + size; + } else { + sqlFinal = sql + " LIMIT " + offset + "," + cnt; + } + Future future = executor.submit( + () -> executeSqlImport(ds, sqlFinal, hbaseMapping, hbaseTemplate, successCount, errMsg)); + futures.add(future); + } + + for (Future future : futures) { + future.get(); + } + + executor.shutdown(); + } else { + executeSqlImport(ds, sql, hbaseMapping, hbaseTemplate, successCount, errMsg); + } + + logger.info(hbaseMapping.getHbaseTable() + " etl completed in: " + + (System.currentTimeMillis() - start) / 1000 + "s!"); + + etlResult.setResultMessage("导入HBase表 " + hbaseMapping.getHbaseTable() + " 数据：" + successCount.get() + " 条"); + } catch (Exception e) { + logger.error(e.getMessage(), e); + errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage()); + } + + if (errMsg.isEmpty()) { + etlResult.setSucceeded(true); + } else { + etlResult.setErrorMessage(Joiner.on("\n").join(errMsg)); + } + return etlResult; + } + + /** + * 执行导入 + * + * @param ds + * @param sql + * @param hbaseMapping + * @param hbaseTemplate + * @param successCount + * @param errMsg + * @return + */ + private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseMapping hbaseMapping, + HbaseTemplate hbaseTemplate, AtomicLong successCount, List errMsg) { + try { + Util.sqlRS(ds, sql, rs -> { + int i = 1; + + try { + boolean complete = false; + List rows = new ArrayList<>(); + String[] rowKeyColumns = null; + if (hbaseMapping.getRowKey() != null) { + rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); + } + while (rs.next()) { + int cc = rs.getMetaData().getColumnCount(); + int[] jdbcTypes = new int[cc]; + Class[] classes = new Class[cc]; + for (int j = 1; j <= cc; j++) { + int jdbcType = rs.getMetaData().getColumnType(j); + jdbcTypes[j - 1] = jdbcType; + classes[j - 1] = JdbcTypeUtil.jdbcType2javaType(jdbcType); + } + HRow row = new HRow(); + + if (rowKeyColumns != null) { + // 取rowKey字段拼接 + StringBuilder rowKeyVale = new StringBuilder(); + for (String rowKeyColumnName : rowKeyColumns) { + Object obj = rs.getObject(rowKeyColumnName); + if (obj != null) { + rowKeyVale.append(obj.toString()); + } + rowKeyVale.append("|"); + } + int len = rowKeyVale.length(); + if (len > 0) { + rowKeyVale.delete(len - 1, len); + } + row.setRowKey(Bytes.toBytes(rowKeyVale.toString())); + } + + for (int j = 1; j <= cc; j++) { + String columnName = rs.getMetaData().getColumnName(j); + + Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]); + if (val == null) { + continue; + } + + MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName); + // 没有配置映射 + if (columnItem == null) { + String family = hbaseMapping.getFamily(); + String qualifile = columnName; + if (hbaseMapping.isUppercaseQualifier()) { + qualifile = qualifile.toUpperCase(); + } + if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) { + if (hbaseMapping.getRowKey() == null && j == 1) { + row.setRowKey(Bytes.toBytes(val.toString())); + } else { + row.addCell(family, qualifile, Bytes.toBytes(val.toString())); + } + } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) { + Type type = Type.getType(classes[j - 1]); + if (hbaseMapping.getRowKey() == null && j == 1) { + row.setRowKey(TypeUtil.toBytes(val, type)); + } else { + row.addCell(family, qualifile, TypeUtil.toBytes(val, type)); + } + } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) { + PhType phType = PhType.getType(classes[j - 1]); + if (hbaseMapping.getRowKey() == null && j == 1) { + row.setRowKey(PhTypeUtil.toBytes(val, phType)); + } else { + row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType)); + } + } + } else { + // 如果不需要类型转换 + if (columnItem.getType() == null || "".equals(columnItem.getType())) { + if (val instanceof java.sql.Date) { + SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd"); + val = dateFmt.format((Date) val); + } else if (val instanceof Timestamp) { + SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + val = datetimeFmt.format((Date) val); + } + + byte[] valBytes = Bytes.toBytes(val.toString()); + if (columnItem.isRowKey()) { + if (columnItem.getRowKeyLen() != null) { + valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val)); + row.setRowKey(valBytes); + } else { + row.setRowKey(valBytes); + } + } else { + row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes); + } + } else { + if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) { + byte[] valBytes = Bytes.toBytes(val.toString()); + if (columnItem.isRowKey()) { + if (columnItem.getRowKeyLen() != null) { + valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val)); + } + row.setRowKey(valBytes); + } else { + row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes); + } + } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) { + Type type = Type.getType(columnItem.getType()); + if (columnItem.isRowKey()) { + if (columnItem.getRowKeyLen() != null) { + String v = limitLenNum(columnItem.getRowKeyLen(), val); + row.setRowKey(Bytes.toBytes(v)); + } else { + row.setRowKey(TypeUtil.toBytes(val, type)); + } + } else { + row.addCell(columnItem.getFamily(), + columnItem.getQualifier(), + TypeUtil.toBytes(val, type)); + } + } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) { + PhType phType = PhType.getType(columnItem.getType()); + if (columnItem.isRowKey()) { + row.setRowKey(PhTypeUtil.toBytes(val, phType)); + } else { + row.addCell(columnItem.getFamily(), + columnItem.getQualifier(), + PhTypeUtil.toBytes(val, phType)); + } + } + } + } + } + + if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空"); + + rows.add(row); + complete = false; + if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) { + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); + rows.clear(); + complete = true; + } + i++; + successCount.incrementAndGet(); + if (logger.isDebugEnabled()) { + logger.debug("successful import count:" + successCount.get()); + } + } + + if (!complete && !rows.isEmpty()) { + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); + } + + } catch (Exception e) { + logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e); + errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage()); + // throw new RuntimeException(e); + } + return i; + }); + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + } + + private static String limitLenNum(int len, Object val) { + if (val == null) { + return null; + } + if (val instanceof Number) { + return String.format("%0" + len + "d", (Number) ((Number) val).longValue()); + } else if (val instanceof String) { + return String.format("%0" + len + "d", Long.parseLong((String) val)); + } + return null; + } +} diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java index 13f6a3ac..cb6f1b61 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java @@ -2,11 +2,12 @@ package com.alibaba.otter.canal.client.adapter.hbase.service; import java.util.*; -import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.hbase.support.*; import com.alibaba.otter.canal.client.adapter.support.Dml; @@ -14,7 +15,7 @@ import com.alibaba.otter.canal.client.adapter.support.Dml; /** * HBase同步操作业务 * - * @author machengyuan 2018-8-21 下午06:45:49 + * @author rewerma 2018-8-21 下午06:45:49 * @version 1.0.0 */ public class HbaseSyncService { @@ -23,34 +24,29 @@ public class HbaseSyncService { private HbaseTemplate hbaseTemplate; // HBase操作模板 - public HbaseSyncService(Connection conn){ - hbaseTemplate = new HbaseTemplate(conn); + public HbaseSyncService(HbaseTemplate hbaseTemplate){ + this.hbaseTemplate = hbaseTemplate; } public void sync(MappingConfig config, Dml dml) { - try { - if (config != null) { - String type = dml.getType(); - if (type != null && type.equalsIgnoreCase("INSERT")) { - insert(config, dml); - } else if (type != null && type.equalsIgnoreCase("UPDATE")) { - update(config, dml); - } else if (type != null && type.equalsIgnoreCase("DELETE")) { - delete(config, dml); - } - if (logger.isDebugEnabled()) { - String res = dml.toString(); - logger.debug(res); - } + if (config != null) { + String type = dml.getType(); + if (type != null && type.equalsIgnoreCase("INSERT")) { + insert(config, dml); + } else if (type != null && type.equalsIgnoreCase("UPDATE")) { + update(config, dml); + } else if (type != null && type.equalsIgnoreCase("DELETE")) { + delete(config, dml); + } + if (logger.isDebugEnabled()) { + logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue)); } - } catch (Exception e) { - logger.error(e.getMessage(), e); } } /** * 插入操作 - * + * * @param config 配置项 * @param dml DML数据 */ @@ -60,10 +56,11 @@ public class HbaseSyncService { return; } - MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); // if (!validHTable(config)) { - // logger.error("HBase table '{}' not exists", hbaseOrm.getHbaseTable()); + // logger.error("HBase table '{}' not exists", + // hbaseMapping.getHbaseTable()); // return; // } int i = 1; @@ -73,63 +70,80 @@ public class HbaseSyncService { HRow hRow = new HRow(); // 拼接复合rowKey - if (hbaseOrm.getRowKey() != null) { - String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); + if (hbaseMapping.getRowKey() != null) { + String[] rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); String rowKeyVale = getRowKeys(rowKeyColumns, r); // params.put("rowKey", Bytes.toBytes(rowKeyVale)); hRow.setRowKey(Bytes.toBytes(rowKeyVale)); } - convertData2Row(hbaseOrm, hRow, r); + convertData2Row(hbaseMapping, hRow, r); if (hRow.getRowKey() == null) { throw new RuntimeException("empty rowKey"); } rows.add(hRow); complete = false; - if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rows.isEmpty()) { - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + if (i % config.getHbaseMapping().getCommitBatch() == 0 && !rows.isEmpty()) { + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); rows.clear(); complete = true; } i++; } if (!complete && !rows.isEmpty()) { - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); } } /** * 将Map数据转换为HRow行数据 - * - * @param hbaseOrm hbase映射配置 + * + * @param hbaseMapping hbase映射配置 * @param hRow 行对象 * @param data Map数据 */ - private static void convertData2Row(MappingConfig.HbaseOrm hbaseOrm, HRow hRow, Map data) { - Map columnItems = hbaseOrm.getColumnItems(); + private static void convertData2Row(MappingConfig.HbaseMapping hbaseMapping, HRow hRow, Map data) { + Map columnItems = hbaseMapping.getColumnItems(); int i = 0; for (Map.Entry entry : data.entrySet()) { + if (hbaseMapping.getExcludeColumns() != null && hbaseMapping.getExcludeColumns().contains(entry.getKey())) { + continue; + } if (entry.getValue() != null) { MappingConfig.ColumnItem columnItem = columnItems.get(entry.getKey()); - byte[] bytes = typeConvert(columnItem, hbaseOrm, entry.getValue()); + byte[] bytes = typeConvert(columnItem, hbaseMapping, entry.getValue()); if (columnItem == null) { - String familyName = hbaseOrm.getFamily(); + String familyName = hbaseMapping.getFamily(); String qualifier = entry.getKey(); - if (hbaseOrm.isUppercaseQualifier()) { + if (hbaseMapping.isUppercaseQualifier()) { qualifier = qualifier.toUpperCase(); } - if (hbaseOrm.getRowKey() == null && i == 0) { + if (hbaseMapping.getRowKey() == null && i == 0) { hRow.setRowKey(bytes); } else { hRow.addCell(familyName, qualifier, bytes); } } else { if (columnItem.isRowKey()) { - // row.put("rowKey", bytes); + if (columnItem.getRowKeyLen() != null && entry.getValue() != null) { + if (entry.getValue() instanceof Number) { + String v = String.format("%0" + columnItem.getRowKeyLen() + "d", + ((Number) entry.getValue()).longValue()); + bytes = Bytes.toBytes(v); + } else { + try { + String v = String.format("%0" + columnItem.getRowKeyLen() + "d", + Integer.parseInt((String) entry.getValue())); + bytes = Bytes.toBytes(v); + } catch (Exception e) { + // ignore + } + } + } hRow.setRowKey(bytes); } else { hRow.addCell(columnItem.getFamily(), columnItem.getQualifier(), bytes); @@ -142,9 +156,9 @@ public class HbaseSyncService { /** * 更新操作 - * - * @param config - * @param dml + * + * @param config 配置对象 + * @param dml dml对象 */ private void update(MappingConfig config, Dml dml) { List> data = dml.getData(); @@ -153,14 +167,15 @@ public class HbaseSyncService { return; } - MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); // if (!validHTable(config)) { - // logger.error("HBase table '{}' not exists", hbaseOrm.getHbaseTable()); + // logger.error("HBase table '{}' not exists", + // hbaseMapping.getHbaseTable()); // return; // } - MappingConfig.ColumnItem rowKeyColumn = hbaseOrm.getRowKeyColumn(); + MappingConfig.ColumnItem rowKeyColumn = hbaseMapping.getRowKeyColumn(); int index = 0; int i = 1; boolean complete = false; @@ -168,8 +183,8 @@ public class HbaseSyncService { out: for (Map r : data) { byte[] rowKeyBytes; - if (hbaseOrm.getRowKey() != null) { - String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); + if (hbaseMapping.getRowKey() != null) { + String[] rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); // 判断是否有复合主键修改 for (String updateColumn : old.get(index).keySet()) { @@ -186,20 +201,24 @@ public class HbaseSyncService { rowKeyBytes = Bytes.toBytes(rowKeyVale); } else if (rowKeyColumn == null) { Map rowKey = data.get(0); - rowKeyBytes = typeConvert(null, hbaseOrm, rowKey.values().iterator().next()); + rowKeyBytes = typeConvert(null, hbaseMapping, rowKey.values().iterator().next()); } else { - rowKeyBytes = typeConvert(rowKeyColumn, hbaseOrm, r.get(rowKeyColumn.getColumn())); + rowKeyBytes = getRowKeyBytes(hbaseMapping, rowKeyColumn, r); } if (rowKeyBytes == null) throw new RuntimeException("rowKey值为空"); - Map columnItems = hbaseOrm.getColumnItems(); + Map columnItems = hbaseMapping.getColumnItems(); HRow hRow = new HRow(rowKeyBytes); for (String updateColumn : old.get(index).keySet()) { + if (hbaseMapping.getExcludeColumns() != null + && hbaseMapping.getExcludeColumns().contains(updateColumn)) { + continue; + } MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn); if (columnItem == null) { - String family = hbaseOrm.getFamily(); + String family = hbaseMapping.getFamily(); String qualifier = updateColumn; - if (hbaseOrm.isUppercaseQualifier()) { + if (hbaseMapping.isUppercaseQualifier()) { qualifier = qualifier.toUpperCase(); } @@ -208,7 +227,7 @@ public class HbaseSyncService { if (newVal == null) { hRow.addCell(family, qualifier, null); } else { - hRow.addCell(family, qualifier, typeConvert(null, hbaseOrm, newVal)); + hRow.addCell(family, qualifier, typeConvert(null, hbaseMapping, newVal)); } } else { // 排除修改id的情况 @@ -220,14 +239,14 @@ public class HbaseSyncService { } else { hRow.addCell(columnItem.getFamily(), columnItem.getQualifier(), - typeConvert(columnItem, hbaseOrm, newVal)); + typeConvert(columnItem, hbaseMapping, newVal)); } } } rows.add(hRow); complete = false; - if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rows.isEmpty()) { - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + if (i % config.getHbaseMapping().getCommitBatch() == 0 && !rows.isEmpty()) { + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); rows.clear(); complete = true; } @@ -235,7 +254,7 @@ public class HbaseSyncService { index++; } if (!complete && !rows.isEmpty()) { - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); } } @@ -245,44 +264,44 @@ public class HbaseSyncService { return; } - MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); // if (!validHTable(config)) { - // logger.error("HBase table '{}' not exists", hbaseOrm.getHbaseTable()); + // logger.error("HBase table '{}' not exists", + // hbaseMapping.getHbaseTable()); // return; // } - MappingConfig.ColumnItem rowKeyColumn = hbaseOrm.getRowKeyColumn(); + MappingConfig.ColumnItem rowKeyColumn = hbaseMapping.getRowKeyColumn(); boolean complete = false; int i = 1; Set rowKeys = new HashSet<>(); for (Map r : data) { byte[] rowKeyBytes; - if (hbaseOrm.getRowKey() != null) { - String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); + if (hbaseMapping.getRowKey() != null) { + String[] rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); String rowKeyVale = getRowKeys(rowKeyColumns, r); rowKeyBytes = Bytes.toBytes(rowKeyVale); } else if (rowKeyColumn == null) { // 如果不需要类型转换 Map rowKey = data.get(0); - rowKeyBytes = typeConvert(null, hbaseOrm, rowKey.values().iterator().next()); + rowKeyBytes = typeConvert(null, hbaseMapping, rowKey.values().iterator().next()); } else { - Object val = r.get(rowKeyColumn.getColumn()); - rowKeyBytes = typeConvert(rowKeyColumn, hbaseOrm, val); + rowKeyBytes = getRowKeyBytes(hbaseMapping, rowKeyColumn, r); } if (rowKeyBytes == null) throw new RuntimeException("rowKey值为空"); rowKeys.add(rowKeyBytes); complete = false; - if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rowKeys.isEmpty()) { - hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys); + if (i % config.getHbaseMapping().getCommitBatch() == 0 && !rowKeys.isEmpty()) { + hbaseTemplate.deletes(hbaseMapping.getHbaseTable(), rowKeys); rowKeys.clear(); complete = true; } i++; } if (!complete && !rowKeys.isEmpty()) { - hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys); + hbaseTemplate.deletes(hbaseMapping.getHbaseTable(), rowKeys); } } @@ -292,9 +311,9 @@ public class HbaseSyncService { if (old == null || old.isEmpty() || data == null || data.isEmpty()) { return; } - MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); - String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); + String[] rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); int index = 0; int i = 1; @@ -335,13 +354,13 @@ public class HbaseSyncService { rowKeys.add(oldRowKeyBytes); HRow row = new HRow(newRowKeyBytes); - convertData2Row(hbaseOrm, row, r); + convertData2Row(hbaseMapping, row, r); rows.add(row); complete = false; - if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rows.isEmpty()) { - hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys); + if (i % config.getHbaseMapping().getCommitBatch() == 0 && !rows.isEmpty()) { + hbaseTemplate.deletes(hbaseMapping.getHbaseTable(), rowKeys); - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); rowKeys.clear(); rows.clear(); complete = true; @@ -350,41 +369,41 @@ public class HbaseSyncService { index++; } if (!complete && !rows.isEmpty()) { - hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys); - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + hbaseTemplate.deletes(hbaseMapping.getHbaseTable(), rowKeys); + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); } } /** * 根据对应的类型进行转换 - * + * * @param columnItem 列项配置 - * @param hbaseOrm hbase映射配置 + * @param hbaseMapping hbase映射配置 * @param value 值 * @return 复合字段rowKey */ - private static byte[] typeConvert(MappingConfig.ColumnItem columnItem, MappingConfig.HbaseOrm hbaseOrm, + private static byte[] typeConvert(MappingConfig.ColumnItem columnItem, MappingConfig.HbaseMapping hbaseMapping, Object value) { if (value == null) { return null; } byte[] bytes = null; if (columnItem == null || columnItem.getType() == null || "".equals(columnItem.getType())) { - if (MappingConfig.Mode.STRING == hbaseOrm.getMode()) { + if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) { bytes = Bytes.toBytes(value.toString()); - } else if (MappingConfig.Mode.NATIVE == hbaseOrm.getMode()) { + } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) { bytes = TypeUtil.toBytes(value); - } else if (MappingConfig.Mode.PHOENIX == hbaseOrm.getMode()) { + } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) { PhType phType = PhType.getType(value.getClass()); bytes = PhTypeUtil.toBytes(value, phType); } } else { - if (hbaseOrm.getMode() == MappingConfig.Mode.STRING) { + if (hbaseMapping.getMode() == MappingConfig.Mode.STRING) { bytes = Bytes.toBytes(value.toString()); - } else if (hbaseOrm.getMode() == MappingConfig.Mode.NATIVE) { + } else if (hbaseMapping.getMode() == MappingConfig.Mode.NATIVE) { Type type = Type.getType(columnItem.getType()); bytes = TypeUtil.toBytes(value, type); - } else if (hbaseOrm.getMode() == MappingConfig.Mode.PHOENIX) { + } else if (hbaseMapping.getMode() == MappingConfig.Mode.PHOENIX) { PhType phType = PhType.getType(columnItem.getType()); bytes = PhTypeUtil.toBytes(value, phType); } @@ -415,4 +434,22 @@ public class HbaseSyncService { return rowKeyValue.toString(); } + private static byte[] getRowKeyBytes(MappingConfig.HbaseMapping hbaseMapping, MappingConfig.ColumnItem rowKeyColumn, + Map rowData) { + Object val = rowData.get(rowKeyColumn.getColumn()); + String v = null; + if (rowKeyColumn.getRowKeyLen() != null) { + if (val instanceof Number) { + v = String.format("%0" + rowKeyColumn.getRowKeyLen() + "d", (Number) ((Number) val).longValue()); + } else if (val instanceof String) { + v = String.format("%0" + rowKeyColumn.getRowKeyLen() + "d", Long.parseLong((String) val)); + } + } + if (v != null) { + return Bytes.toBytes(v); + } else { + return typeConvert(rowKeyColumn, hbaseMapping, val); + } + } + } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/HbaseTemplate.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/HbaseTemplate.java index 62b7ca03..9481c795 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/HbaseTemplate.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/HbaseTemplate.java @@ -5,10 +5,16 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,16 +27,33 @@ import org.slf4j.LoggerFactory; */ public class HbaseTemplate { - private Logger logger = LoggerFactory.getLogger(this.getClass()); + private Logger logger = LoggerFactory.getLogger(this.getClass()); - private Connection conn; + private Configuration hbaseConfig; // hbase配置对象 + private Connection conn; // hbase连接 - public HbaseTemplate(Connection conn){ - this.conn = conn; + public HbaseTemplate(Configuration hbaseConfig){ + this.hbaseConfig = hbaseConfig; + initConn(); + } + + private void initConn() { + try { + this.conn = ConnectionFactory.createConnection(hbaseConfig); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public Connection getConnection() { + if (conn == null || conn.isAborted() || conn.isClosed()) { + initConn(); + } + return conn; } public boolean tableExists(String tableName) { - try (HBaseAdmin admin = (HBaseAdmin) conn.getAdmin()) { + try (HBaseAdmin admin = (HBaseAdmin) getConnection().getAdmin()) { return admin.tableExists(TableName.valueOf(tableName)); } catch (IOException e) { @@ -39,7 +62,7 @@ public class HbaseTemplate { } public void createTable(String tableName, String... familyNames) { - try (HBaseAdmin admin = (HBaseAdmin) conn.getAdmin()) { + try (HBaseAdmin admin = (HBaseAdmin) getConnection().getAdmin()) { HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); // 添加列簇 @@ -56,7 +79,7 @@ public class HbaseTemplate { } public void disableTable(String tableName) { - try (HBaseAdmin admin = (HBaseAdmin) conn.getAdmin()) { + try (HBaseAdmin admin = (HBaseAdmin) getConnection().getAdmin()) { admin.disableTable(tableName); } catch (IOException e) { logger.error(e.getMessage(), e); @@ -65,7 +88,7 @@ public class HbaseTemplate { } public void deleteTable(String tableName) { - try (HBaseAdmin admin = (HBaseAdmin) conn.getAdmin()) { + try (HBaseAdmin admin = (HBaseAdmin) getConnection().getAdmin()) { if (admin.isTableEnabled(tableName)) { disableTable(tableName); } @@ -78,7 +101,7 @@ public class HbaseTemplate { /** * 插入一行数据 - * + * * @param tableName 表名 * @param hRow 行数据对象 * @return 是否成功 @@ -86,7 +109,7 @@ public class HbaseTemplate { public Boolean put(String tableName, HRow hRow) { boolean flag = false; try { - HTable table = (HTable) conn.getTable(TableName.valueOf(tableName)); + HTable table = (HTable) getConnection().getTable(TableName.valueOf(tableName)); Put put = new Put(hRow.getRowKey()); for (HRow.HCell hCell : hRow.getCells()) { put.addColumn(Bytes.toBytes(hCell.getFamily()), Bytes.toBytes(hCell.getQualifier()), hCell.getValue()); @@ -95,6 +118,7 @@ public class HbaseTemplate { flag = true; } catch (Exception e) { logger.error(e.getMessage(), e); + throw new RuntimeException(e); } return flag; @@ -102,7 +126,7 @@ public class HbaseTemplate { /** * 批量插入 - * + * * @param tableName 表名 * @param rows 行数据对象集合 * @return 是否成功 @@ -110,7 +134,7 @@ public class HbaseTemplate { public Boolean puts(String tableName, List rows) { boolean flag = false; try { - HTable table = (HTable) conn.getTable(TableName.valueOf(tableName)); + HTable table = (HTable) getConnection().getTable(TableName.valueOf(tableName)); List puts = new ArrayList<>(); for (HRow hRow : rows) { Put put = new Put(hRow.getRowKey()); @@ -127,13 +151,14 @@ public class HbaseTemplate { flag = true; } catch (Exception e) { logger.error(e.getMessage(), e); + throw new RuntimeException(e); } return flag; } /** * 批量删除数据 - * + * * @param tableName 表名 * @param rowKeys rowKey集合 * @return 是否成功 @@ -141,7 +166,7 @@ public class HbaseTemplate { public Boolean deletes(String tableName, Set rowKeys) { boolean flag = false; try { - HTable table = (HTable) conn.getTable(TableName.valueOf(tableName)); + HTable table = (HTable) getConnection().getTable(TableName.valueOf(tableName)); List deletes = new ArrayList<>(); for (byte[] rowKey : rowKeys) { Delete delete = new Delete(rowKey); @@ -153,7 +178,14 @@ public class HbaseTemplate { flag = true; } catch (Exception e) { logger.error(e.getMessage(), e); + throw new RuntimeException(e); } return flag; } + + public void close() throws IOException { + if (conn != null) { + conn.close(); + } + } } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhType.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhType.java index 09c63a6c..0a6a74d5 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhType.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhType.java @@ -1,5 +1,8 @@ package com.alibaba.otter.canal.client.adapter.hbase.support; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Time; @@ -13,81 +16,44 @@ import java.util.Date; * @version 1.0.0 */ public enum PhType { - DEFAULT(-1, "VARCHAR"), - UNSIGNED_INT(4, "UNSIGNED_INT"), - UNSIGNED_LONG(8, "UNSIGNED_LONG"), - UNSIGNED_TINYINT(1, "UNSIGNED_TINYINT"), - UNSIGNED_SMALLINT(2, "UNSIGNED_SMALLINT"), - UNSIGNED_FLOAT(4, "UNSIGNED_FLOAT"), - UNSIGNED_DOUBLE(8, "UNSIGNED_DOUBLE"), - INTEGER(4, "INTEGER"), - BIGINT(8, "BIGINT"), - TINYINT(1, "TINYINT"), - SMALLINT(2, "SMALLINT"), - FLOAT(4, "FLOAT"), - DOUBLE(8, "DOUBLE"), - DECIMAL(-1, "DECIMAL"), - BOOLEAN(1, "BOOLEAN"), - UNSIGNED_TIME(8, "UNSIGNED_TIME"), - UNSIGNED_DATE(8, "UNSIGNED_DATE"), - UNSIGNED_TIMESTAMP(12, "UNSIGNED_TIMESTAMP"), - TIME(8, "TIME"), - DATE(8, "DATE"), - TIMESTAMP(12, "TIMESTAMP"), - VARCHAR(-1, "VARCHAR"), - VARBINARY(-1, "VARBINARY"); + DEFAULT, UNSIGNED_INT, UNSIGNED_LONG, UNSIGNED_TINYINT, UNSIGNED_SMALLINT, UNSIGNED_FLOAT, + UNSIGNED_DOUBLE, INTEGER, BIGINT, TINYINT, SMALLINT, FLOAT, DOUBLE, DECIMAL, BOOLEAN, UNSIGNED_TIME, + UNSIGNED_DATE, UNSIGNED_TIMESTAMP, TIME, DATE, TIMESTAMP, VARCHAR, VARBINARY; - /** - * -1：长度可变 - */ - private int len; - private String type; - - PhType(int len, String type) { - this.len = len; - this.type = type; - } - - public int getLen() { - return len; - } - - public String getType() { - return this.type; - } + private static Logger logger = LoggerFactory.getLogger(PhType.class); public static PhType getType(Class javaType) { if (javaType == null) return DEFAULT; PhType phType; - if (Integer.class.isAssignableFrom(javaType) || int.class.isAssignableFrom(javaType)) { + if (Integer.class == javaType || int.class == javaType) { phType = INTEGER; - } else if (Long.class.isAssignableFrom(javaType) || long.class.isAssignableFrom(javaType)) { + } else if (Long.class == javaType || long.class == javaType) { phType = BIGINT; - } else if (Byte.class.isAssignableFrom(javaType) || byte.class.isAssignableFrom(javaType)) { + } else if (Byte.class == javaType || byte.class == javaType) { phType = TINYINT; - } else if (Short.class.isAssignableFrom(javaType) || short.class.isAssignableFrom(javaType)) { + } else if (Short.class == javaType || short.class == javaType) { phType = SMALLINT; - } else if (Float.class.isAssignableFrom(javaType) || float.class.isAssignableFrom(javaType)) { + } else if (Float.class == javaType || float.class == javaType) { phType = FLOAT; - } else if (Double.class.isAssignableFrom(javaType) || double.class.isAssignableFrom(javaType)) { + } else if (Double.class == javaType || double.class == javaType) { phType = DOUBLE; - } else if (Boolean.class.isAssignableFrom(javaType) || boolean.class.isAssignableFrom(javaType)) { + } else if (Boolean.class == javaType || boolean.class == javaType) { phType = BOOLEAN; - } else if (java.sql.Date.class.isAssignableFrom(javaType)) { + } else if (java.sql.Date.class == javaType) { phType = DATE; - } else if (Time.class.isAssignableFrom(javaType)) { + } else if (Time.class == javaType) { phType = DATE; - } else if (Timestamp.class.isAssignableFrom(javaType)) { + } else if (Timestamp.class == javaType) { phType = TIMESTAMP; - } else if (Date.class.isAssignableFrom(javaType)) { + } else if (Date.class == javaType) { phType = DATE; - } else if (byte[].class.isAssignableFrom(javaType)) { + } else if (byte[].class == javaType) { phType = VARBINARY; - } else if (String.class.isAssignableFrom(javaType)) { + } else if (String.class == javaType) { phType = VARCHAR; - } else if (BigDecimal.class.isAssignableFrom(javaType)) { + } else if (BigDecimal.class == javaType) { phType = DECIMAL; - } else if (BigInteger.class.isAssignableFrom(javaType)) { + } else if (BigInteger.class == javaType) { phType = UNSIGNED_LONG; } else { phType = DEFAULT; @@ -97,54 +63,11 @@ public enum PhType { public static PhType getType(String type) { if (type == null) return DEFAULT; - PhType phType; - if (type.equalsIgnoreCase(UNSIGNED_INT.type)) { - phType = UNSIGNED_INT; - } else if (type.equalsIgnoreCase(UNSIGNED_LONG.type)) { - phType = UNSIGNED_LONG; - } else if (type.equalsIgnoreCase(UNSIGNED_TINYINT.type)) { - phType = UNSIGNED_TINYINT; - } else if (type.equalsIgnoreCase(UNSIGNED_SMALLINT.type)) { - phType = UNSIGNED_SMALLINT; - } else if (type.equalsIgnoreCase(UNSIGNED_FLOAT.type)) { - phType = UNSIGNED_FLOAT; - } else if (type.equalsIgnoreCase(UNSIGNED_DOUBLE.type)) { - phType = UNSIGNED_DOUBLE; - } else if (type.equalsIgnoreCase(INTEGER.type)) { - phType = INTEGER; - } else if (type.equalsIgnoreCase(BIGINT.type)) { - phType = BIGINT; - } else if (type.equalsIgnoreCase(TINYINT.type)) { - phType = TINYINT; - } else if (type.equalsIgnoreCase(SMALLINT.type)) { - phType = SMALLINT; - } else if (type.equalsIgnoreCase(FLOAT.type)) { - phType = FLOAT; - } else if (type.equalsIgnoreCase(DOUBLE.type)) { - phType = DOUBLE; - } else if (type.equalsIgnoreCase(BOOLEAN.type)) { - phType = BOOLEAN; - } else if (type.equalsIgnoreCase(UNSIGNED_TIME.type)) { - phType = UNSIGNED_TIME; - } else if (type.equalsIgnoreCase(UNSIGNED_DATE.type)) { - phType = UNSIGNED_DATE; - } else if (type.equalsIgnoreCase(UNSIGNED_TIMESTAMP.type)) { - phType = UNSIGNED_TIMESTAMP; - } else if (type.equalsIgnoreCase(TIME.type)) { - phType = TIME; - } else if (type.equalsIgnoreCase(DATE.type)) { - phType = DATE; - } else if (type.equalsIgnoreCase(TIMESTAMP.type)) { - phType = TIMESTAMP; - } else if (type.equalsIgnoreCase(VARCHAR.type)) { - phType = VARCHAR; - } else if (type.equalsIgnoreCase(VARBINARY.type)) { - phType = VARBINARY; - } else if (type.equalsIgnoreCase(DECIMAL.type)) { - phType = DECIMAL; - } else { - phType = DEFAULT; + try { + return PhType.valueOf(type.toUpperCase()); + } catch (Exception e) { + logger.error(e.getMessage(), e); + return DEFAULT; } - return phType; } } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java index 25473681..a3a8e0b9 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java @@ -169,7 +169,8 @@ public class PhTypeUtil { private static int encodeInt(int v, byte[] b, int o) { checkForSufficientLength(b, o, Bytes.SIZEOF_INT); - b[o + 0] = (byte) ((v >> 24) ^ 0x80); // Flip sign bit so that INTEGER is binary comparable + b[o + 0] = (byte) ((v >> 24) ^ 0x80); // Flip sign bit so that INTEGER + // is binary comparable b[o + 1] = (byte) (v >> 16); b[o + 2] = (byte) (v >> 8); b[o + 3] = (byte) v; @@ -209,7 +210,8 @@ public class PhTypeUtil { private static int encodeLong(long v, byte[] b, int o) { checkForSufficientLength(b, o, Bytes.SIZEOF_LONG); - b[o + 0] = (byte) ((v >> 56) ^ 0x80); // Flip sign bit so that INTEGER is binary comparable + b[o + 0] = (byte) ((v >> 56) ^ 0x80); // Flip sign bit so that INTEGER + // is binary comparable b[o + 1] = (byte) (v >> 48); b[o + 2] = (byte) (v >> 40); b[o + 3] = (byte) (v >> 32); @@ -255,7 +257,8 @@ public class PhTypeUtil { private static int encodeShort(short v, byte[] b, int o) { checkForSufficientLength(b, o, Bytes.SIZEOF_SHORT); - b[o + 0] = (byte) ((v >> 8) ^ 0x80); // Flip sign bit so that Short is binary comparable + b[o + 0] = (byte) ((v >> 8) ^ 0x80); // Flip sign bit so that Short is + // binary comparable b[o + 1] = (byte) v; return Bytes.SIZEOF_SHORT; } @@ -287,7 +290,8 @@ public class PhTypeUtil { private static int encodeByte(byte v, byte[] b, int o) { checkForSufficientLength(b, o, Bytes.SIZEOF_BYTE); - b[o] = (byte) (v ^ 0x80); // Flip sign bit so that Short is binary comparable + b[o] = (byte) (v ^ 0x80); // Flip sign bit so that Short is binary + // comparable return Bytes.SIZEOF_BYTE; } @@ -546,7 +550,8 @@ public class PhTypeUtil { multiplyBy = 10; divideBy = BigInteger.TEN; } - // Normalize the scale based on what is necessary to end up with a base 100 + // Normalize the scale based on what is necessary to end up with a base + // 100 // decimal (i.e. 10.123e3) int digitOffset; BigInteger compareAgainst; @@ -564,7 +569,8 @@ public class PhTypeUtil { if (length <= MAX_BIG_DECIMAL_BYTES) { result[--index] = NEG_TERMINAL_BYTE; } else { - // Adjust length and offset down because we don't have enough room + // Adjust length and offset down because we don't have enough + // room length = MAX_BIG_DECIMAL_BYTES; index = offset + length; } @@ -593,8 +599,8 @@ public class PhTypeUtil { private static void checkForSufficientLength(byte[] b, int offset, int requiredLength) { if (b.length < offset + requiredLength) { - throw new RuntimeException( - "Expected length of at least " + requiredLength + " bytes, but had " + (b.length - offset)); + throw new RuntimeException("Expected length of at least " + requiredLength + " bytes, but had " + + (b.length - offset)); } } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/Type.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/Type.java index 9a567191..bbe3f879 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/Type.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/Type.java @@ -1,5 +1,8 @@ package com.alibaba.otter.canal.client.adapter.hbase.support; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.math.BigDecimal; import java.math.BigInteger; import java.util.Date; @@ -11,60 +14,20 @@ import java.util.Date; * @version 1.0.0 */ public enum Type { - DEFAULT("STRING"), - STRING("STRING"), - INTEGER("INTEGER"), - LONG("LONG"), - SHORT("SHORT"), - BOOLEAN("BOOLEAN"), - FLOAT("FLOAT"), - DOUBLE("DOUBLE"), - BIGDECIMAL("BIGDECIMAL"), - DATE("DATE"), - BYTE("BYTE"), - BYTES("BYTES"); + DEFAULT, STRING, INTEGER, LONG, SHORT, BOOLEAN, FLOAT, DOUBLE, BIGDECIMAL, DATE, BYTE, BYTES; - private String type; - - Type(String type) { - this.type = type; - } - - public String getType() { - return type; - } + private static Logger logger = LoggerFactory.getLogger(Type.class); public static Type getType(String type) { if (type == null) { return DEFAULT; } - Type res; - if (type.equalsIgnoreCase("STRING")) { - res = STRING; - } else if (type.equalsIgnoreCase("INTEGER")) { - res = INTEGER; - } else if (type.equalsIgnoreCase("LONG")) { - res = LONG; - } else if (type.equalsIgnoreCase("SHORT")) { - res = SHORT; - } else if (type.equalsIgnoreCase("BOOLEAN")) { - res = BOOLEAN; - } else if (type.equalsIgnoreCase("FLOAT")) { - res = FLOAT; - } else if (type.equalsIgnoreCase("DOUBLE")) { - res = DOUBLE; - } else if (type.equalsIgnoreCase("BIGDECIMAL")) { - res = BIGDECIMAL; - } else if (type.equalsIgnoreCase("DATE")) { - res = DATE; - } else if (type.equalsIgnoreCase("BYTE")) { - res = BYTE; - } else if (type.equalsIgnoreCase("BYTES")) { - res = BYTES; - } else { - res = DEFAULT; + try { + return Type.valueOf(type); + } catch (Exception e) { + logger.error(e.getMessage(), e); + return DEFAULT; } - return res; } public static Type getType(Class javaType) { @@ -72,29 +35,29 @@ public enum Type { return DEFAULT; } Type type; - if (Integer.class.isAssignableFrom(javaType) || int.class.isAssignableFrom(javaType)) { + if (Integer.class == javaType || int.class == javaType) { type = INTEGER; - } else if (Long.class.isAssignableFrom(javaType) || long.class.isAssignableFrom(javaType)) { + } else if (Long.class == javaType || long.class == javaType) { type = LONG; - } else if (Byte.class.isAssignableFrom(javaType) || byte.class.isAssignableFrom(javaType)) { + } else if (Byte.class == javaType || byte.class == javaType) { type = BYTE; - } else if (Short.class.isAssignableFrom(javaType) || short.class.isAssignableFrom(javaType)) { + } else if (Short.class == javaType || short.class == javaType) { type = SHORT; - } else if (Float.class.isAssignableFrom(javaType) || float.class.isAssignableFrom(javaType)) { + } else if (Float.class == javaType || float.class == javaType) { type = FLOAT; - } else if (Double.class.isAssignableFrom(javaType) || double.class.isAssignableFrom(javaType)) { + } else if (Double.class == javaType || double.class == javaType) { type = DOUBLE; - } else if (Boolean.class.isAssignableFrom(javaType) || boolean.class.isAssignableFrom(javaType)) { + } else if (Boolean.class == javaType || boolean.class == javaType) { type = BOOLEAN; - } else if (Date.class.isAssignableFrom(javaType)) { + } else if (Date.class == javaType) { type = DATE; - } else if (byte[].class.isAssignableFrom(javaType)) { + } else if (byte[].class == javaType) { type = BYTES; - } else if (String.class.isAssignableFrom(javaType)) { + } else if (String.class == javaType) { type = STRING; - } else if (BigDecimal.class.isAssignableFrom(javaType)) { + } else if (BigDecimal.class == javaType) { type = BIGDECIMAL; - } else if (BigInteger.class.isAssignableFrom(javaType)) { + } else if (BigInteger.class == javaType) { type = LONG; } else { type = DEFAULT; diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/TypeUtil.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/TypeUtil.java index 07aad00f..22500a47 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/TypeUtil.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/TypeUtil.java @@ -21,29 +21,30 @@ public class TypeUtil { return null; } byte[] bytes; - if (obj instanceof String) { + Class clazz = obj.getClass(); + if (clazz == String.class) { bytes = Bytes.toBytes((String) obj); - } else if (obj instanceof Integer) { + } else if (clazz == Integer.class || clazz == int.class) { bytes = Bytes.toBytes((Integer) obj); - } else if (obj instanceof Long) { + } else if (clazz == Long.class || clazz == long.class) { bytes = Bytes.toBytes((Long) obj); - } else if (obj instanceof Short) { + } else if (clazz == Short.class || clazz == short.class) { bytes = Bytes.toBytes((Short) obj); - } else if (obj instanceof Boolean) { + } else if (clazz == Boolean.class || clazz == boolean.class) { bytes = Bytes.toBytes((Boolean) obj); - } else if (obj instanceof Float) { + } else if (clazz == Float.class || clazz == float.class) { bytes = Bytes.toBytes((Float) obj); - } else if (obj instanceof Double) { + } else if (clazz == Double.class || clazz == double.class) { bytes = Bytes.toBytes((Double) obj); - } else if (obj instanceof BigDecimal) { - bytes = Bytes.toBytes((BigDecimal) obj); - } else if (obj instanceof BigInteger) { - bytes = Bytes.toBytes(((BigInteger) obj).longValue()); - } else if (obj instanceof Date) { - bytes = Bytes.toBytes(((Date) obj).getTime()); - } else if (obj instanceof Byte) { + } else if (clazz == Byte.class || clazz == byte.class) { bytes = new byte[] { (byte) obj }; - } else if (obj instanceof byte[]) { + } else if (clazz == BigDecimal.class) { + bytes = Bytes.toBytes((BigDecimal) obj); + } else if (clazz == BigInteger.class) { + bytes = Bytes.toBytes(((BigInteger) obj).longValue()); + } else if (clazz == Date.class) { + bytes = Bytes.toBytes(((Date) obj).getTime()); + } else if (clazz == byte[].class) { bytes = (byte[]) obj; } else { // 其余类型统一转换为string @@ -92,50 +93,51 @@ public class TypeUtil { return b; } + @SuppressWarnings("unchecked") public static T toObject(byte[] bytes, Class clazz) { if (bytes == null) { return null; } Object res; - if (String.class.isAssignableFrom(clazz)) { + if (String.class == clazz) { res = Bytes.toString(bytes); - } else if (Integer.class.isAssignableFrom(clazz)) { + } else if (Integer.class == clazz || int.class == clazz) { res = Bytes.toInt(bytes); - } else if (Long.class.isAssignableFrom(clazz)) { + } else if (Long.class == clazz || long.class == clazz) { res = Bytes.toLong(bytes); - } else if (Short.class.isAssignableFrom(clazz)) { + } else if (Short.class == clazz || short.class == clazz) { res = Bytes.toShort(bytes); - } else if (Boolean.class.isAssignableFrom(clazz)) { + } else if (Boolean.class == clazz || boolean.class == clazz) { res = Bytes.toBoolean(bytes); - } else if (Float.class.isAssignableFrom(clazz)) { + } else if (Float.class == clazz || float.class == clazz) { res = Bytes.toFloat(bytes); - } else if (Double.class.isAssignableFrom(clazz)) { + } else if (Double.class == clazz || double.class == clazz) { res = Bytes.toDouble(bytes); - } else if (BigDecimal.class.isAssignableFrom(clazz)) { + } else if (Byte.class == clazz || byte.class == clazz) { + res = bytes[0]; + } else if (BigDecimal.class == clazz) { res = Bytes.toBigDecimal(bytes); - } else if (BigInteger.class.isAssignableFrom(clazz)) { + } else if (BigInteger.class == clazz) { res = Bytes.toLong(bytes); - } else if (java.sql.Date.class.isAssignableFrom(clazz)) { + } else if (java.sql.Date.class == clazz) { long ts = Bytes.toLong(bytes); res = new java.sql.Date(ts); - } else if (Time.class.isAssignableFrom(clazz)) { + } else if (Time.class == clazz) { long ts = Bytes.toLong(bytes); res = new Time(ts); - } else if (Timestamp.class.isAssignableFrom(clazz)) { + } else if (Timestamp.class == clazz) { long ts = Bytes.toLong(bytes); res = new Timestamp(ts); - } else if (Date.class.isAssignableFrom(clazz)) { + } else if (Date.class == clazz) { long ts = Bytes.toLong(bytes); res = new Date(ts); - } else if (Byte.class.isAssignableFrom(clazz)) { - res = bytes[0]; } else { throw new IllegalArgumentException("mismatch class type"); } - // noinspection unchecked return (T) res; } + @SuppressWarnings("unchecked") public static T toObject(byte[] bytes, Type type) { if (bytes == null) { return null; @@ -182,7 +184,6 @@ public class TypeUtil { } else { throw new IllegalArgumentException("mismatch class type"); } - // noinspection unchecked return (T) res; } } diff --git a/client-adapter/hbase/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.CanalOuterAdapter b/client-adapter/hbase/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter similarity index 100% rename from client-adapter/hbase/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.CanalOuterAdapter rename to client-adapter/hbase/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter diff --git a/client-adapter/hbase/src/main/resources/hbase-mapping/configs.conf b/client-adapter/hbase/src/main/resources/hbase-mapping/configs.conf deleted file mode 100644 index ff5b292e..00000000 --- a/client-adapter/hbase/src/main/resources/hbase-mapping/configs.conf +++ /dev/null @@ -1,5 +0,0 @@ -# 详细映射配置 -mytest_person2.yml - -# 简易配置, 只用指定数据库名.表名, 详细配置全部使用默认 -mytest.person \ No newline at end of file diff --git a/client-adapter/hbase/src/main/resources/hbase-mapping/mytest_person2.yml b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml similarity index 87% rename from client-adapter/hbase/src/main/resources/hbase-mapping/mytest_person2.yml rename to client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml index 575a1bf5..703894b3 100644 --- a/client-adapter/hbase/src/main/resources/hbase-mapping/mytest_person2.yml +++ b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml @@ -1,5 +1,8 @@ -hbaseOrm: - mode: PHOENIX #NATIVE #STRING +dataSourceKey: defaultDS +destination: example +groupId: g1 +hbaseMapping: + mode: STRING #NATIVE #PHOENIX database: mytest # 数据库名 table: person2 # 数据库表名 hbaseTable: MYTEST.PERSON2 # HBase表名 @@ -9,12 +12,14 @@ hbaseOrm: #rowKey: id,type # 复合字段rowKey不能和columns中的rowKey重复 columns: # 数据库字段:HBase对应字段 - id: ROWKEY$UNSIGNED_LONG + id: ROWKEY LEN:15 name: NAME email: EMAIL - type: $DECIMAL - c_time: C_TIME$UNSIGNED_TIMESTAMP - birthday: BIRTHDAY$DATE + type: + c_time: C_TIME + birthday: BIRTHDAY +# excludeColumns: +# - lat # 忽略字段 # -- NATIVE类型 # $DEFAULT @@ -43,7 +48,7 @@ hbaseOrm: # $TINYINT 对应PHOENIX里的TINYINT 1字节 # $SMALLINT 对应PHOENIX里的SMALLINT 2字节 # $FLOAT 对应PHOENIX里的FLOAT 4字节 -# DOUBLE 对应PHOENIX里的DOUBLE 8字节 +# $DOUBLE 对应PHOENIX里的DOUBLE 8字节 # $BOOLEAN 对应PHOENIX里的BOOLEAN 1字节 # $TIME 对应PHOENIX里的TIME 8字节 # $DATE 对应PHOENIX里的DATE 8字节 @@ -53,4 +58,4 @@ hbaseOrm: # $UNSIGNED_TIMESTAMP 对应PHOENIX里的UNSIGNED_TIMESTAMP 12字节 # $VARCHAR 对应PHOENIX里的VARCHAR 动态长度 # $VARBINARY 对应PHOENIX里的VARBINARY 动态长度 -# $DECIMAL 对应PHOENIX里的DECIMAL 动态长度 \ No newline at end of file +# $DECIMAL 对应PHOENIX里的DECIMAL 动态长度 diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml new file mode 100644 index 00000000..110bc97b --- /dev/null +++ b/client-adapter/launcher/pom.xml @@ -0,0 +1,266 @@ + + + + canal.client-adapter + com.alibaba.otter + 1.1.3-SNAPSHOT + + 4.0.0 + com.alibaba.otter + client-adapter.launcher + jar + canal client adapter launcher module for otter ${project.version} + + + + + org.springframework.boot + spring-boot-dependencies + 2.0.1.RELEASE + pom + import + + + + + + + com.alibaba.otter + client-adapter.common + ${project.version} + + + com.alibaba.otter + canal.client + 1.1.3-SNAPSHOT + + + org.yaml + snakeyaml + 1.19 + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + org.springframework.cloud + spring-cloud-context + 2.0.0.RELEASE + + + org.apache.curator + curator-recipes + 2.10.0 + + + + org.apache.rocketmq + rocketmq-client + 4.3.0 + + + + org.apache.kafka + kafka-clients + 1.1.1 + + + + + mysql + mysql-connector-java + 5.1.40 + + + org.postgresql + postgresql + 42.1.4 + + + com.oracle + ojdbc6 + 11.2.0.3 + + + com.microsoft.sqlserver + mssql-jdbc + 7.0.0.jre8 + + + + + com.alibaba.otter + client-adapter.logger + ${project.version} + + + * + * + + + jar-with-dependencies + provided + + + com.alibaba.otter + client-adapter.hbase + ${project.version} + + + * + * + + + jar-with-dependencies + provided + + + com.alibaba.otter + client-adapter.elasticsearch + ${project.version} + + + * + * + + + jar-with-dependencies + provided + + + com.alibaba.otter + client-adapter.rdb + ${project.version} + + + * + * + + + jar-with-dependencies + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + copy-dependencies-to-canal-client-service + package + + copy-dependencies + + + jar-with-dependencies + ${project.basedir}/target/canal-adapter/plugin + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.2.1 + + + assemble + + single + + package + + + + false + false + + + + org.apache.maven.plugins + maven-jar-plugin + 3.0.2 + + + + true + lib/ + com.alibaba.otter.canal.adapter.launcher.CanalAdapterApplication + + + + **/*.properties + **/*.xml + **/*.yml + static/** + templates/** + + + + + + + + + dev + + true + + env + !release + + + + + + + maven-assembly-plugin + + + ${basedir}/src/main/assembly/dev.xml + + canal-adapter + ${project.build.directory} + + + + + + + + + release + + + env + release + + + + + + + maven-assembly-plugin + + + ${basedir}/src/main/assembly/release.xml + + canal.adapter-${project.version} + ${project.basedir}/../../target + + + + + + + diff --git a/client-adapter/launcher/src/main/assembly/dev.xml b/client-adapter/launcher/src/main/assembly/dev.xml new file mode 100644 index 00000000..7beaf269 --- /dev/null +++ b/client-adapter/launcher/src/main/assembly/dev.xml @@ -0,0 +1,69 @@ + + dist + + dir + + false + + + . + / + + README* + + + + ./src/main/bin + bin + + **/* + + 0755 + + + ./src/main/resources + /conf + + **/* + + + + + ../elasticsearch/src/main/resources/es + /conf/es + + **/* + + + + ../hbase/src/main/resources/hbase + /conf/hbase + + **/* + + + + ../rdb/src/main/resources/ + /conf + + META-INF/** + + + + target + logs + + **/* + + + + + + lib + + junit:junit + + + + diff --git a/client-adapter/launcher/src/main/assembly/release.xml b/client-adapter/launcher/src/main/assembly/release.xml new file mode 100644 index 00000000..456b4a6a --- /dev/null +++ b/client-adapter/launcher/src/main/assembly/release.xml @@ -0,0 +1,74 @@ + + dist + + tar.gz + + false + + + . + / + + README* + + + + ./src/main/bin + bin + + **/* + + 0755 + + + ./src/main/resources + /conf + + **/* + + + + + ../elasticsearch/src/main/resources/es + /conf/es + + **/* + + + + ../hbase/src/main/resources/hbase + /conf/hbase + + **/* + + + + ../rdb/src/main/resources/ + /conf + + META-INF/** + + + + target + logs + + **/* + + + + ${project.basedir}/target/canal-adapter/plugin + /plugin/ + + + + + lib + + junit:junit + + + + \ No newline at end of file diff --git a/client-adapter/launcher/src/main/bin/restart.sh b/client-adapter/launcher/src/main/bin/restart.sh new file mode 100644 index 00000000..63749ce5 --- /dev/null +++ b/client-adapter/launcher/src/main/bin/restart.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +sh stop.sh + +sh startup.sh diff --git a/client-adapter/launcher/src/main/bin/startup.bat b/client-adapter/launcher/src/main/bin/startup.bat new file mode 100755 index 00000000..0c464dff --- /dev/null +++ b/client-adapter/launcher/src/main/bin/startup.bat @@ -0,0 +1,22 @@ +@echo off +@if not "%ECHO%" == "" echo %ECHO% +@if "%OS%" == "Windows_NT" setlocal + +set ENV_PATH=.\ +if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0% + +set conf_dir=%ENV_PATH%\..\conf + +set CLASSPATH=%conf_dir% +set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH% + +set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m +set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 +set ADAPTER_OPTS= -DappName=canal-adapter + +set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %ADAPTER_OPTS% + +set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.adapter.launcher.CanalAdapterApplication +echo start cmd : %CMD_STR% + +java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.adapter.launcher.CanalAdapterApplication \ No newline at end of file diff --git a/client-adapter/launcher/src/main/bin/startup.sh b/client-adapter/launcher/src/main/bin/startup.sh new file mode 100644 index 00000000..650fba7d --- /dev/null +++ b/client-adapter/launcher/src/main/bin/startup.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +current_path=`pwd` +case "`uname`" in + Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + ;; + *) + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +base=${bin_abs_path}/.. +export LANG=en_US.UTF-8 +export BASE=$base + +if [ -f $base/bin/adapter.pid ] ; then + echo "found adapter.pid , Please run stop.sh first ,then startup.sh" 2>&2 + exit 1 +fi + +if [ ! -d $base/logs ] ; then + mkdir -p $base/logs +fi + +## set java path +if [ -z "$JAVA" ] ; then + JAVA=$(which java) +fi + +ALIBABA_JAVA="/usr/alibaba/java/bin/java" +TAOBAO_JAVA="/opt/taobao/java/bin/java" +if [ -z "$JAVA" ]; then + if [ -f $ALIBABA_JAVA ] ; then + JAVA=$ALIBABA_JAVA + elif [ -f $TAOBAO_JAVA ] ; then + JAVA=$TAOBAO_JAVA + else + echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2 + exit 1 + fi +fi + +str=`file -L $JAVA | grep 64-bit` +if [ -n "$str" ]; then + JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -Xss256k -XX:-UseAdaptiveSizePolicy -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError" +else + JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m " +fi + +JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8" +ADAPTER_OPTS="-DappName=canal-adapter" + +for i in $base/lib/*; + do CLASSPATH=$i:"$CLASSPATH"; +done + +CLASSPATH="$base/conf:$CLASSPATH"; + +echo "cd to $bin_abs_path for workaround relative path" +cd $bin_abs_path + +echo CLASSPATH :$CLASSPATH +$JAVA $JAVA_OPTS $ADAPTER_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.adapter.launcher.CanalAdapterApplication 1>>$base/logs/adapter.log 2>&1 & +echo $! > $base/bin/adapter.pid + +echo "cd to $current_path for continue" +cd $current_path \ No newline at end of file diff --git a/client-launcher/src/main/bin/stop.sh b/client-adapter/launcher/src/main/bin/stop.sh similarity index 87% rename from client-launcher/src/main/bin/stop.sh rename to client-adapter/launcher/src/main/bin/stop.sh index c96f2807..fbdd5317 100644 --- a/client-launcher/src/main/bin/stop.sh +++ b/client-adapter/launcher/src/main/bin/stop.sh @@ -37,15 +37,15 @@ get_pid() { } base=`dirname $0`/.. -pidfile=$base/bin/canal_client.pid +pidfile=$base/bin/adapter.pid if [ ! -f "$pidfile" ];then - echo "canal client is not running. exists" + echo "canal-adapter is not running. exists" exit fi pid=`cat $pidfile` if [ "$pid" == "" ] ; then - pid=`get_pid "appName=otter-canal-client"` + pid=`get_pid "appName=canal-adapter"` fi echo -e "`hostname`: stopping canal $pid ... " @@ -54,7 +54,7 @@ kill $pid LOOPS=0 while (true); do - gpid=`get_pid "appName=otter-canal-client" "$pid"` + gpid=`get_pid "appName=canal-adapter" "$pid"` if [ "$gpid" == "" ] ; then echo "Oook! cost:$LOOPS" `rm $pidfile` diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java new file mode 100644 index 00000000..3d2ef585 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java @@ -0,0 +1,21 @@ +package com.alibaba.otter.canal.adapter.launcher; + +import org.springframework.boot.Banner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +/** + * 启动入口 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +@SpringBootApplication +public class CanalAdapterApplication { + + public static void main(String[] args) { + SpringApplication application = new SpringApplication(CanalAdapterApplication.class); + application.setBannerMode(Banner.Mode.OFF); + application.run(args); + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java new file mode 100644 index 00000000..5fd5f3fe --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java @@ -0,0 +1,120 @@ +package com.alibaba.otter.canal.adapter.launcher.common; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; +import org.springframework.stereotype.Component; + +import com.alibaba.otter.canal.adapter.launcher.config.CuratorClient; + +/** + * Etl 同步锁 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +@Component +public class EtlLock { + + private static final Map LOCAL_LOCK = new ConcurrentHashMap<>(); + + private static final Map DISTRIBUTED_LOCK = new ConcurrentHashMap<>(); + + private static Mode mode = Mode.LOCAL; + + @Resource + private CuratorClient curatorClient; + + @PostConstruct + public void init() { + CuratorFramework curator = curatorClient.getCurator(); + if (curator != null) { + mode = Mode.DISTRIBUTED; + } else { + mode = Mode.LOCAL; + } + } + + private ReentrantLock getLock(String key) { + ReentrantLock lock = LOCAL_LOCK.get(key); + if (lock == null) { + synchronized (EtlLock.class) { + lock = LOCAL_LOCK.get(key); + if (lock == null) { + lock = new ReentrantLock(); + LOCAL_LOCK.put(key, lock); + } + } + } + return lock; + } + + private InterProcessMutex getRemoteLock(String key) { + InterProcessMutex lock = DISTRIBUTED_LOCK.get(key); + if (lock == null) { + synchronized (EtlLock.class) { + lock = DISTRIBUTED_LOCK.get(key); + if (lock == null) { + lock = new InterProcessMutex(curatorClient.getCurator(), key); + DISTRIBUTED_LOCK.put(key, lock); + } + } + } + return lock; + } + + public void lock(String key) throws Exception { + if (mode == Mode.LOCAL) { + getLock(key).lock(); + } else { + InterProcessMutex lock = getRemoteLock(key); + lock.acquire(); + } + } + + public boolean tryLock(String key, long timeout, TimeUnit unit) { + try { + if (mode == Mode.LOCAL) { + return getLock(key).tryLock(timeout, unit); + } else { + InterProcessMutex lock = getRemoteLock(key); + return lock.acquire(timeout, unit); + } + } catch (Exception e) { + return false; + } + } + + public boolean tryLock(String key) { + try { + if (mode == Mode.LOCAL) { + return getLock(key).tryLock(); + } else { + InterProcessMutex lock = getRemoteLock(key); + return lock.acquire(500, TimeUnit.MILLISECONDS); + } + } catch (Exception e) { + return false; + } + } + + public void unlock(String key) { + if (mode == Mode.LOCAL) { + getLock(key).unlock(); + } else { + InterProcessMutex lock = getRemoteLock(key); + try { + lock.release(); + } catch (Exception e) { + // ignore + } + } + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/Mode.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/Mode.java new file mode 100644 index 00000000..ac73b85e --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/Mode.java @@ -0,0 +1,6 @@ +package com.alibaba.otter.canal.adapter.launcher.common; + +public enum Mode { + LOCAL, // 本地模式 + DISTRIBUTED // 分布式 +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java new file mode 100644 index 00000000..c0e5d394 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java @@ -0,0 +1,214 @@ +package com.alibaba.otter.canal.adapter.launcher.common; + +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.NodeCache; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.data.Stat; +import org.springframework.stereotype.Component; + +import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig; +import com.alibaba.otter.canal.adapter.launcher.config.CuratorClient; +import com.alibaba.otter.canal.common.utils.BooleanMutex; + +/** + * 同步开关 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +@Component +public class SyncSwitch { + + private static final String SYN_SWITCH_ZK_NODE = "/sync-switch/"; + + private static final Map LOCAL_LOCK = new ConcurrentHashMap<>(); + + private static final Map DISTRIBUTED_LOCK = new ConcurrentHashMap<>(); + + private static Mode mode = Mode.LOCAL; + + @Resource + private AdapterCanalConfig adapterCanalConfig; + @Resource + private CuratorClient curatorClient; + + @PostConstruct + public void init() { + CuratorFramework curator = curatorClient.getCurator(); + if (curator != null) { + mode = Mode.DISTRIBUTED; + DISTRIBUTED_LOCK.clear(); + for (String destination : adapterCanalConfig.DESTINATIONS) { + // 对应每个destination注册锁 + BooleanMutex mutex = new BooleanMutex(true); + initMutex(curator, destination, mutex); + DISTRIBUTED_LOCK.put(destination, mutex); + startListen(destination, mutex); + } + } else { + mode = Mode.LOCAL; + LOCAL_LOCK.clear(); + for (String destination : adapterCanalConfig.DESTINATIONS) { + // 对应每个destination注册锁 + LOCAL_LOCK.put(destination, new BooleanMutex(true)); + } + } + } + + @SuppressWarnings("resource") + private synchronized void startListen(String destination, BooleanMutex mutex) { + try { + String path = SYN_SWITCH_ZK_NODE + destination; + CuratorFramework curator = curatorClient.getCurator(); + NodeCache nodeCache = new NodeCache(curator, path); + nodeCache.start(); + nodeCache.getListenable().addListener(() -> initMutex(curator, destination, mutex)); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + private synchronized void initMutex(CuratorFramework curator, String destination, BooleanMutex mutex) { + try { + String path = SYN_SWITCH_ZK_NODE + destination; + Stat stat = curator.checkExists().forPath(path); + if (stat == null) { + if (!mutex.state()) { + mutex.set(true); + } + } else { + String data = new String(curator.getData().forPath(path), StandardCharsets.UTF_8); + if ("on".equals(data)) { + if (!mutex.state()) { + mutex.set(true); + } + } else { + if (mutex.state()) { + mutex.set(false); + } + } + } + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + public synchronized void off(String destination) { + if (mode == Mode.LOCAL) { + BooleanMutex mutex = LOCAL_LOCK.get(destination); + if (mutex != null && mutex.state()) { + mutex.set(false); + } + } else { + try { + String path = SYN_SWITCH_ZK_NODE + destination; + try { + curatorClient.getCurator() + .create() + .creatingParentContainersIfNeeded() + .withMode(CreateMode.PERSISTENT) + .forPath(path, "off".getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + curatorClient.getCurator().setData().forPath(path, "off".getBytes(StandardCharsets.UTF_8)); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + public synchronized void on(String destination) { + if (mode == Mode.LOCAL) { + BooleanMutex mutex = LOCAL_LOCK.get(destination); + if (mutex != null && !mutex.state()) { + mutex.set(true); + } + } else { + try { + String path = SYN_SWITCH_ZK_NODE + destination; + try { + curatorClient.getCurator() + .create() + .creatingParentContainersIfNeeded() + .withMode(CreateMode.PERSISTENT) + .forPath(path, "on".getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + curatorClient.getCurator().setData().forPath(path, "on".getBytes(StandardCharsets.UTF_8)); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + public synchronized void release(String destination) { + if (mode == Mode.LOCAL) { + BooleanMutex mutex = LOCAL_LOCK.get(destination); + if (mutex != null && !mutex.state()) { + mutex.set(true); + } + } + if (mode == Mode.DISTRIBUTED) { + BooleanMutex mutex = DISTRIBUTED_LOCK.get(destination); + if (mutex != null && !mutex.state()) { + mutex.set(true); + } + } + } + + public Boolean status(String destination) { + if (mode == Mode.LOCAL) { + BooleanMutex mutex = LOCAL_LOCK.get(destination); + if (mutex != null) { + return mutex.state(); + } else { + return null; + } + } else { + BooleanMutex mutex = DISTRIBUTED_LOCK.get(destination); + if (mutex != null) { + return mutex.state(); + } else { + return null; + } + } + } + + public void get(String destination) throws InterruptedException { + if (mode == Mode.LOCAL) { + BooleanMutex mutex = LOCAL_LOCK.get(destination); + if (mutex != null) { + mutex.get(); + } + } else { + BooleanMutex mutex = DISTRIBUTED_LOCK.get(destination); + if (mutex != null) { + mutex.get(); + } + } + } + + public void get(String destination, long timeout, TimeUnit unit) throws InterruptedException, TimeoutException { + if (mode == Mode.LOCAL) { + BooleanMutex mutex = LOCAL_LOCK.get(destination); + if (mutex != null) { + mutex.get(timeout, unit); + } + } else { + BooleanMutex mutex = DISTRIBUTED_LOCK.get(destination); + if (mutex != null) { + mutex.get(timeout, unit); + } + } + } + +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java new file mode 100644 index 00000000..3ad0d5f7 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java @@ -0,0 +1,79 @@ +package com.alibaba.otter.canal.adapter.launcher.config; + +import java.sql.SQLException; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; + +/** + * canal 的相关配置类 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +@Component +@ConfigurationProperties(prefix = "canal.conf") +public class AdapterCanalConfig extends CanalClientConfig { + + public final Set DESTINATIONS = new LinkedHashSet<>(); + + private Map srcDataSources; + + @Override + public void setCanalAdapters(List canalAdapters) { + super.setCanalAdapters(canalAdapters); + + if (canalAdapters != null) { + synchronized (DESTINATIONS) { + DESTINATIONS.clear(); + for (CanalAdapter canalAdapter : canalAdapters) { + if (canalAdapter.getInstance() != null) { + DESTINATIONS.add(canalAdapter.getInstance()); + } + } + } + } + } + + public Map getSrcDataSources() { + return srcDataSources; + } + + @SuppressWarnings("resource") + public void setSrcDataSources(Map srcDataSources) { + this.srcDataSources = srcDataSources; + + if (srcDataSources != null) { + for (Map.Entry entry : srcDataSources.entrySet()) { + DatasourceConfig datasourceConfig = entry.getValue(); + // 加载数据源连接池 + DruidDataSource ds = new DruidDataSource(); + ds.setDriverClassName(datasourceConfig.getDriver()); + ds.setUrl(datasourceConfig.getUrl()); + ds.setUsername(datasourceConfig.getUsername()); + ds.setPassword(datasourceConfig.getPassword()); + ds.setInitialSize(1); + ds.setMinIdle(1); + ds.setMaxActive(datasourceConfig.getMaxActive()); + ds.setMaxWait(60000); + ds.setTimeBetweenEvictionRunsMillis(60000); + ds.setMinEvictableIdleTimeMillis(300000); + ds.setValidationQuery("select 1"); + try { + ds.init(); + } catch (SQLException e) { + throw new RuntimeException(e.getMessage(), e); + } + DatasourceConfig.DATA_SOURCES.put(entry.getKey(), ds); + } + } + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/BootstrapConfiguration.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/BootstrapConfiguration.java new file mode 100644 index 00000000..ffae1b14 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/BootstrapConfiguration.java @@ -0,0 +1,41 @@ +package com.alibaba.otter.canal.adapter.launcher.config; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; + +import com.alibaba.otter.canal.adapter.launcher.monitor.remote.RemoteConfigLoader; +import com.alibaba.otter.canal.adapter.launcher.monitor.remote.RemoteConfigLoaderFactory; + +/** + * Bootstrap级别配置加载 + * + * @author rewerma @ 2019-01-05 + * @version 1.0.0 + */ +public class BootstrapConfiguration { + + @Autowired + private Environment env; + + private RemoteConfigLoader remoteConfigLoader = null; + + @PostConstruct + public void loadRemoteConfig() { + remoteConfigLoader = RemoteConfigLoaderFactory.getRemoteConfigLoader(env); + if (remoteConfigLoader != null) { + remoteConfigLoader.loadRemoteConfig(); + remoteConfigLoader.loadRemoteAdapterConfigs(); + remoteConfigLoader.startMonitor(); // 启动监听 + } + } + + @PreDestroy + public synchronized void destroy() { + if (remoteConfigLoader != null) { + remoteConfigLoader.destroy(); + } + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java new file mode 100644 index 00000000..0ec60392 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java @@ -0,0 +1,42 @@ +package com.alibaba.otter.canal.adapter.launcher.config; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.springframework.stereotype.Component; + +/** + * curator 配置类 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +@Component +public class CuratorClient { + + @Resource + private AdapterCanalConfig adapterCanalConfig; + + private CuratorFramework curator = null; + + @PostConstruct + public void init() { + if (adapterCanalConfig.getZookeeperHosts() != null) { + curator = CuratorFrameworkFactory.builder() + .connectString(adapterCanalConfig.getZookeeperHosts()) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .sessionTimeoutMs(6000) + .connectionTimeoutMs(3000) + .namespace("canal-adapter") + .build(); + curator.start(); + } + } + + public CuratorFramework getCurator() { + return curator; + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/SpringContext.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/SpringContext.java new file mode 100644 index 00000000..987371e2 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/SpringContext.java @@ -0,0 +1,34 @@ +package com.alibaba.otter.canal.adapter.launcher.config; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +/** + * spring util配置类 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +@Component +public class SpringContext implements ApplicationContextAware { + + private static ApplicationContext context; + + /* + * 注入ApplicationContext + */ + public void setApplicationContext(final ApplicationContext context) throws BeansException { + // 在加载Spring时自动获得context + SpringContext.context = context; + } + + public static Object getBean(final String beanName) { + return SpringContext.context.getBean(beanName); + } + + public static Object getBean(final Class clz) { + return context.getBean(clz); + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java new file mode 100644 index 00000000..6f21ebd5 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java @@ -0,0 +1,255 @@ +package com.alibaba.otter.canal.adapter.launcher.loader; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch; +import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; +import com.alibaba.otter.canal.client.CanalMQConnector; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; +import com.alibaba.otter.canal.client.adapter.support.MessageUtil; +import com.alibaba.otter.canal.protocol.FlatMessage; +import com.alibaba.otter.canal.protocol.Message; + +/** + * 适配器工作线程抽象类 + * + * @author rewerma 2018-8-19 下午11:30:49 + * @version 1.0.0 + */ +public abstract class AbstractCanalAdapterWorker { + + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + + protected String canalDestination; // canal实例 + protected String groupId = null; // groupId + protected List> canalOuterAdapters; // 外部适配器 + protected CanalClientConfig canalClientConfig; // 配置 + protected ExecutorService groupInnerExecutorService; // 组内工作线程池 + protected volatile boolean running = false; // 是否运行中 + protected Thread thread = null; + protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e); + + protected SyncSwitch syncSwitch; + + public AbstractCanalAdapterWorker(List> canalOuterAdapters){ + this.canalOuterAdapters = canalOuterAdapters; + this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); + syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class); + } + + protected void writeOut(final Message message) { + List> futures = new ArrayList<>(); + // 组间适配器并行运行 + canalOuterAdapters.forEach(outerAdapters -> { + final List adapters = outerAdapters; + futures.add(groupInnerExecutorService.submit(() -> { + try { + // 组内适配器穿行运行，尽量不要配置组内适配器 + adapters.forEach(adapter -> { + long begin = System.currentTimeMillis(); + List dmls = MessageUtil.parse4Dml(canalDestination, groupId, message); + if (dmls != null) { + batchSync(dmls, adapter); + + if (logger.isDebugEnabled()) { + logger.debug("{} elapsed time: {}", + adapter.getClass().getName(), + (System.currentTimeMillis() - begin)); + } + } + }); + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + })); + + // 等待所有适配器写入完成 + // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 + RuntimeException exception = null; + for (Future future : futures) { + try { + if (!future.get()) { + exception = new RuntimeException("Outer adapter sync failed! "); + } + } catch (Exception e) { + exception = new RuntimeException(e); + } + } + if (exception != null) { + throw exception; + } + }); + } + + protected void writeOut(final List flatMessages) { + List> futures = new ArrayList<>(); + // 组间适配器并行运行 + canalOuterAdapters.forEach(outerAdapters -> { + futures.add(groupInnerExecutorService.submit(() -> { + try { + // 组内适配器穿行运行，尽量不要配置组内适配器 + outerAdapters.forEach(adapter -> { + long begin = System.currentTimeMillis(); + List dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, flatMessages); + batchSync(dmls, adapter); + + if (logger.isDebugEnabled()) { + logger.debug("{} elapsed time: {}", + adapter.getClass().getName(), + (System.currentTimeMillis() - begin)); + } + }); + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + })); + + // 等待所有适配器写入完成 + // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 + RuntimeException exception = null; + for (Future future : futures) { + try { + if (!future.get()) { + exception = new RuntimeException("Outer adapter sync failed! "); + } + } catch (Exception e) { + exception = new RuntimeException(e); + } + } + if (exception != null) { + throw exception; + } + }); + } + + @SuppressWarnings("unchecked") + protected boolean mqWriteOutData(int retry, long timeout, int i, final boolean flatMessage, + CanalMQConnector connector, ExecutorService workerExecutor) { + try { + List messages; + if (!flatMessage) { + messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS); + } else { + messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS); + } + if (messages != null && !messages.isEmpty()) { + Future future = workerExecutor.submit(() -> { + if (flatMessage) { + // batch write + writeOut((List) messages); + } else { + for (final Object message : messages) { + writeOut((Message) message); + } + } + return true; + }); + + try { + future.get(timeout, TimeUnit.MILLISECONDS); + } catch (Exception e) { + future.cancel(true); + throw e; + } + connector.ack(); + } + return true; + } catch (Throwable e) { + if (i == retry - 1) { + connector.ack(); + logger.error(e.getMessage() + " Error sync but ACK!"); + return true; + } else { + connector.rollback(); + logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1)); + } + try { + Thread.sleep(500); + } catch (InterruptedException e1) { + // ignore + } + } + return false; + } + + /** + * 分批同步 + * + * @param dmls + * @param adapter + */ + private void batchSync(List dmls, OuterAdapter adapter) { + // 分批同步 + if (dmls.size() <= canalClientConfig.getSyncBatchSize()) { + adapter.sync(dmls); + } else { + int len = 0; + List dmlsBatch = new ArrayList<>(); + for (Dml dml : dmls) { + dmlsBatch.add(dml); + len += dml.getData().size(); + if (len >= canalClientConfig.getSyncBatchSize()) { + adapter.sync(dmlsBatch); + dmlsBatch.clear(); + len = 0; + } + } + if (!dmlsBatch.isEmpty()) { + adapter.sync(dmlsBatch); + } + } + } + + public void start() { + if (!running) { + thread = new Thread(this::process); + thread.setUncaughtExceptionHandler(handler); + thread.start(); + running = true; + } + } + + protected abstract void process(); + + public void stop() { + try { + if (!running) { + return; + } + + running = false; + + syncSwitch.release(canalDestination); + + logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination); + if (thread != null) { + try { + thread.join(); + } catch (InterruptedException e) { + // ignore + } + } + groupInnerExecutorService.shutdown(); + logger.info("destination {} adapters worker thread dead!", canalDestination); + canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy)); + logger.info("destination {} all adapters destroyed!", canalDestination); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java new file mode 100644 index 00000000..72291ccb --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java @@ -0,0 +1,96 @@ +package com.alibaba.otter.canal.adapter.launcher.loader; + +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.common.errors.WakeupException; + +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; +import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector; + +/** + * kafka对应的client适配器工作线程 + * + * @author rewerma 2018-8-19 下午11:30:49 + * @version 1.0.0 + */ +public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { + + private KafkaCanalConnector connector; + private String topic; + private boolean flatMessage; + + public CanalAdapterKafkaWorker(CanalClientConfig canalClientConfig, String bootstrapServers, String topic, + String groupId, List> canalOuterAdapters, boolean flatMessage){ + super(canalOuterAdapters); + this.canalClientConfig = canalClientConfig; + this.topic = topic; + super.canalDestination = topic; + super.groupId = groupId; + this.flatMessage = flatMessage; + this.connector = new KafkaCanalConnector(bootstrapServers, + topic, + null, + groupId, + canalClientConfig.getBatchSize(), + flatMessage); + connector.setSessionTimeout(30L, TimeUnit.SECONDS); + } + + @Override + protected void process() { + while (!running) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // ignore + } + } + ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); + int retry = canalClientConfig.getRetries() == null + || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries(); + long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 + + while (running) { + try { + syncSwitch.get(canalDestination); + logger.info("=============> Start to connect topic: {} <=============", this.topic); + connector.connect(); + logger.info("=============> Start to subscribe topic: {} <=============", this.topic); + connector.subscribe(); + logger.info("=============> Subscribe topic: {} succeed <=============", this.topic); + while (running) { + Boolean status = syncSwitch.status(canalDestination); + if (status != null && !status) { + connector.disconnect(); + break; + } + if (retry == -1) { + retry = Integer.MAX_VALUE; + } + for (int i = 0; i < retry; i++) { + if (!running) { + break; + } + if (mqWriteOutData(retry, timeout, i, flatMessage, connector, workerExecutor)) { + break; + } + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + try { + connector.unsubscribe(); + } catch (WakeupException e) { + // No-op. Continue process + } + connector.disconnect(); + logger.info("=============> Disconnect topic: {} <=============", this.topic); + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java new file mode 100644 index 00000000..12d5a040 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java @@ -0,0 +1,216 @@ +package com.alibaba.otter.canal.adapter.launcher.loader; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.env.EnumerablePropertySource; +import org.springframework.core.env.Environment; +import org.springframework.core.env.PropertySource; +import org.springframework.core.env.StandardEnvironment; + +import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; +import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; + +/** + * 外部适配器的加载器 + * + * @version 1.0.0 + */ +public class CanalAdapterLoader { + + private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class); + + private CanalClientConfig canalClientConfig; + + private Map canalWorkers = new HashMap<>(); + + private Map canalMQWorker = new HashMap<>(); + + private ExtensionLoader loader; + + public CanalAdapterLoader(CanalClientConfig canalClientConfig){ + this.canalClientConfig = canalClientConfig; + } + + /** + * 初始化canal-client + */ + public void init() { + loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class); + + String canalServerHost = this.canalClientConfig.getCanalServerHost(); + SocketAddress sa = null; + if (canalServerHost != null) { + String[] ipPort = canalServerHost.split(":"); + sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1])); + } + String zkHosts = this.canalClientConfig.getZookeeperHosts(); + + if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) { + // 初始化canal-client的适配器 + for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { + List> canalOuterAdapterGroups = new ArrayList<>(); + + for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) { + List canalOutConnectors = new ArrayList<>(); + for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) { + loadAdapter(c, canalOutConnectors); + } + canalOuterAdapterGroups.add(canalOutConnectors); + } + CanalAdapterWorker worker; + if (sa != null) { + worker = new CanalAdapterWorker(canalClientConfig, + canalAdapter.getInstance(), + sa, + canalOuterAdapterGroups); + } else if (zkHosts != null) { + worker = new CanalAdapterWorker(canalClientConfig, + canalAdapter.getInstance(), + zkHosts, + canalOuterAdapterGroups); + } else { + throw new RuntimeException("No canal server connector found"); + } + canalWorkers.put(canalAdapter.getInstance(), worker); + worker.start(); + logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance()); + } + } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) { + // 初始化canal-client-kafka的适配器 + for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { + for (CanalClientConfig.Group group : canalAdapter.getGroups()) { + List> canalOuterAdapterGroups = new ArrayList<>(); + List canalOuterAdapters = new ArrayList<>(); + for (OuterAdapterConfig config : group.getOuterAdapters()) { + loadAdapter(config, canalOuterAdapters); + } + canalOuterAdapterGroups.add(canalOuterAdapters); + + CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig, + canalClientConfig.getMqServers(), + canalAdapter.getInstance(), + group.getGroupId(), + canalOuterAdapterGroups, + canalClientConfig.getFlatMessage()); + canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker); + canalKafkaWorker.start(); + logger.info("Start adapter for canal-client mq topic: {} succeed", + canalAdapter.getInstance() + "-" + group.getGroupId()); + } + } + } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) { + // 初始化canal-client-rocketMQ的适配器 + for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { + for (CanalClientConfig.Group group : canalAdapter.getGroups()) { + List> canalOuterAdapterGroups = new ArrayList<>(); + List canalOuterAdapters = new ArrayList<>(); + for (OuterAdapterConfig config : group.getOuterAdapters()) { + loadAdapter(config, canalOuterAdapters); + } + canalOuterAdapterGroups.add(canalOuterAdapters); + CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig, + canalClientConfig.getMqServers(), + canalAdapter.getInstance(), + group.getGroupId(), + canalOuterAdapterGroups, + canalClientConfig.getAccessKey(), + canalClientConfig.getSecretKey(), + canalClientConfig.getFlatMessage()); + canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); + rocketMQWorker.start(); + + logger.info("Start adapter for canal-client mq topic: {} succeed", + canalAdapter.getInstance() + "-" + group.getGroupId()); + } + } + } + } + + private void loadAdapter(OuterAdapterConfig config, List canalOutConnectors) { + try { + OuterAdapter adapter; + adapter = loader.getExtension(config.getName(), StringUtils.trimToEmpty(config.getKey())); + + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + // 替换ClassLoader + Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader()); + Environment env = (Environment) SpringContext.getBean(Environment.class); + Properties evnProperties = null; + if (env instanceof StandardEnvironment) { + evnProperties = new Properties(); + for (PropertySource propertySource : ((StandardEnvironment) env).getPropertySources()) { + if (propertySource instanceof EnumerablePropertySource) { + String[] names = ((EnumerablePropertySource) propertySource).getPropertyNames(); + for (String name : names) { + Object val = propertySource.getProperty(name); + if (val != null) { + evnProperties.put(name, val); + } + } + } + } + } + adapter.init(config, evnProperties); + Thread.currentThread().setContextClassLoader(cl); + canalOutConnectors.add(adapter); + logger.info("Load canal adapter: {} succeed", config.getName()); + } catch (Exception e) { + logger.error("Load canal adapter: {} failed", config.getName(), e); + } + } + + /** + * 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁 + */ + public void destroy() { + if (!canalWorkers.isEmpty()) { + ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size()); + List> futures = new ArrayList<>(); + for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) { + futures.add(stopExecutorService.submit(() -> { + canalAdapterWorker.stop(); + return true; + })); + } + futures.forEach(future -> { + try { + future.get(); + } catch (Exception e) { + // ignore + } + }); + stopExecutorService.shutdown(); + } + + if (!canalMQWorker.isEmpty()) { + ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size()); + List> futures = new ArrayList<>(); + for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) { + futures.add(stopMQWorkerService.submit(() -> { + canalAdapterMQWorker.stop(); + return true; + })); + } + futures.forEach(future -> { + try { + future.get(); + } catch (Exception e) { + // ignore + } + }); + stopMQWorkerService.shutdown(); + } + logger.info("All canal adapters destroyed"); + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java new file mode 100644 index 00000000..369e8965 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java @@ -0,0 +1,95 @@ +package com.alibaba.otter.canal.adapter.launcher.loader; + +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.kafka.common.errors.WakeupException; + +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; +import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector; + +/** + * rocketmq对应的client适配器工作线程 + * + * @version 1.0.0 + */ +public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker { + + private RocketMQCanalConnector connector; + private String topic; + private boolean flatMessage; + + public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic, + String groupId, List> canalOuterAdapters, String accessKey, + String secretKey, boolean flatMessage){ + super(canalOuterAdapters); + this.canalClientConfig = canalClientConfig; + this.topic = topic; + this.flatMessage = flatMessage; + super.canalDestination = topic; + super.groupId = groupId; + this.connector = new RocketMQCanalConnector(nameServers, + topic, + groupId, + accessKey, + secretKey, + canalClientConfig.getBatchSize(), + flatMessage); + logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId); + } + + @Override + protected void process() { + while (!running) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + + ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); + int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries(); + long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 + + while (running) { + try { + syncSwitch.get(canalDestination); + logger.info("=============> Start to connect topic: {} <=============", this.topic); + connector.connect(); + logger.info("=============> Start to subscribe topic: {}<=============", this.topic); + connector.subscribe(); + logger.info("=============> Subscribe topic: {} succeed<=============", this.topic); + while (running) { + Boolean status = syncSwitch.status(canalDestination); + if (status != null && !status) { + connector.disconnect(); + break; + } + if (retry == -1) { + retry = Integer.MAX_VALUE; + } + for (int i = 0; i < retry; i++) { + if (!running) { + break; + } + if (mqWriteOutData(retry, timeout, i, flatMessage, connector, workerExecutor)) { + break; + } + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + try { + connector.unsubscribe(); + } catch (WakeupException e) { + // No-op. Continue process + } + connector.disconnect(); + logger.info("=============> Disconnect topic: {} <=============", this.topic); + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java new file mode 100644 index 00000000..48cad990 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java @@ -0,0 +1,93 @@ +package com.alibaba.otter.canal.adapter.launcher.loader; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.annotation.Resource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.cloud.context.config.annotation.RefreshScope; +import org.springframework.cloud.context.refresh.ContextRefresher; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch; +import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig; +import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; + +/** + * 适配器启动业务类 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +@Component +@RefreshScope +public class CanalAdapterService { + + private static final Logger logger = LoggerFactory.getLogger(CanalAdapterService.class); + + private CanalAdapterLoader adapterLoader; + + @Resource + private ContextRefresher contextRefresher; + + @Resource + private AdapterCanalConfig adapterCanalConfig; + @Resource + private Environment env; + + // 注入bean保证优先注册 + @Resource + private SpringContext springContext; + @Resource + private SyncSwitch syncSwitch; + + private volatile boolean running = false; + + @PostConstruct + public synchronized void init() { + if (running) { + return; + } + try { + logger.info("## start the canal client adapters."); + adapterLoader = new CanalAdapterLoader(adapterCanalConfig); + adapterLoader.init(); + running = true; + logger.info("## the canal client adapters are running now ......"); + } catch (Exception e) { + logger.error("## something goes wrong when starting up the canal client adapters:", e); + } + } + + @PreDestroy + public synchronized void destroy() { + if (!running) { + return; + } + try { + running = false; + logger.info("## stop the canal client adapters"); + + if (adapterLoader != null) { + adapterLoader.destroy(); + adapterLoader = null; + } + for (DruidDataSource druidDataSource : DatasourceConfig.DATA_SOURCES.values()) { + try { + druidDataSource.close(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + DatasourceConfig.DATA_SOURCES.clear(); + } catch (Throwable e) { + logger.warn("## something goes wrong when stopping canal client adapters:", e); + } finally { + logger.info("## canal client adapters are down."); + } + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java new file mode 100644 index 00000000..5ed1063f --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java @@ -0,0 +1,193 @@ +package com.alibaba.otter.canal.adapter.launcher.loader; + +import java.net.SocketAddress; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import com.alibaba.otter.canal.client.CanalConnector; +import com.alibaba.otter.canal.client.CanalConnectors; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; +import com.alibaba.otter.canal.client.impl.ClusterCanalConnector; +import com.alibaba.otter.canal.client.impl.SimpleCanalConnector; +import com.alibaba.otter.canal.protocol.Message; + +/** + * 原生canal-server对应的client适配器工作线程 + * + * @author rewrema 2018-8-19 下午11:30:49 + * @version 1.0.0 + */ +public class CanalAdapterWorker extends AbstractCanalAdapterWorker { + + private static final int BATCH_SIZE = 50; + private static final int SO_TIMEOUT = 0; + + private CanalConnector connector; + + /** + * 单台client适配器worker的构造方法 + * + * @param canalDestination canal实例名 + * @param address canal-server地址 + * @param canalOuterAdapters 外部适配器组 + */ + public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, SocketAddress address, + List> canalOuterAdapters){ + super(canalOuterAdapters); + this.canalClientConfig = canalClientConfig; + this.canalDestination = canalDestination; + connector = CanalConnectors.newSingleConnector(address, canalDestination, "", ""); + } + + /** + * HA模式下client适配器worker的构造方法 + * + * @param canalDestination canal实例名 + * @param zookeeperHosts zookeeper地址 + * @param canalOuterAdapters 外部适配器组 + */ + public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, String zookeeperHosts, + List> canalOuterAdapters){ + super(canalOuterAdapters); + this.canalDestination = canalDestination; + this.canalClientConfig = canalClientConfig; + connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", ""); + ((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT); + } + + @Override + protected void process() { + while (!running) { // waiting until running == true + while (!running) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + } + + int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries(); + if (retry == -1) { + // 重试次数-1代表异常时一直阻塞重试 + retry = Integer.MAX_VALUE; + } + // long timeout = canalClientConfig.getTimeout() == null ? 300000 : + // canalClientConfig.getTimeout(); // 默认超时5分钟 + Integer batchSize = canalClientConfig.getBatchSize(); + if (batchSize == null) { + batchSize = BATCH_SIZE; + } + + while (running) { + try { + syncSwitch.get(canalDestination); + + logger.info("=============> Start to connect destination: {} <=============", this.canalDestination); + connector.connect(); + logger.info("=============> Start to subscribe destination: {} <=============", this.canalDestination); + connector.subscribe(); + logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination); + while (running) { + try { + syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES); + } catch (TimeoutException e) { + break; + } + if (!running) { + break; + } + + for (int i = 0; i < retry; i++) { + if (!running) { + break; + } + Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据 + long batchId = message.getId(); + try { + int size = message.getEntries().size(); + if (batchId == -1 || size == 0) { + Thread.sleep(500); + } else { + if (logger.isDebugEnabled()) { + logger.debug("destination: {} batchId: {} batchSize: {} ", + canalDestination, + batchId, + size); + } + long begin = System.currentTimeMillis(); + writeOut(message); + if (logger.isDebugEnabled()) { + logger.debug("destination: {} batchId: {} elapsed time: {} ms", + canalDestination, + batchId, + System.currentTimeMillis() - begin); + } + } + connector.ack(batchId); // 提交确认 + break; + } catch (Exception e) { + if (i != retry - 1) { + connector.rollback(batchId); // 处理失败, 回滚数据 + logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1)); + } else { + connector.ack(batchId); + logger.error(e.getMessage() + " Error sync but ACK!"); + } + Thread.sleep(500); + } + } + } + + } catch (Exception e) { + logger.error("process error!", e); + } finally { + connector.disconnect(); + logger.info("=============> Disconnect destination: {} <=============", this.canalDestination); + } + + if (running) { // is reconnect + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // ignore + } + } + } + } + + @Override + public void stop() { + try { + if (!running) { + return; + } + + if (connector instanceof ClusterCanalConnector) { + ((ClusterCanalConnector) connector).stopRunning(); + } else if (connector instanceof SimpleCanalConnector) { + ((SimpleCanalConnector) connector).stopRunning(); + } + + running = false; + + syncSwitch.release(canalDestination); + + logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination); + if (thread != null) { + try { + thread.join(); + } catch (InterruptedException e) { + // ignore + } + } + groupInnerExecutorService.shutdown(); + logger.info("destination {} adapters worker thread dead!", canalDestination); + canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy)); + logger.info("destination {} all adapters destroyed!", canalDestination); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/ApplicationConfigMonitor.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/ApplicationConfigMonitor.java new file mode 100644 index 00000000..8d519184 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/ApplicationConfigMonitor.java @@ -0,0 +1,90 @@ +package com.alibaba.otter.canal.adapter.launcher.monitor; + +import java.io.File; +import java.io.FileReader; +import java.util.Map; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.annotation.Resource; + +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.io.monitor.FileAlterationListenerAdaptor; +import org.apache.commons.io.monitor.FileAlterationMonitor; +import org.apache.commons.io.monitor.FileAlterationObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.cloud.context.refresh.ContextRefresher; +import org.springframework.stereotype.Component; +import org.yaml.snakeyaml.Yaml; + +import com.alibaba.otter.canal.adapter.launcher.loader.CanalAdapterService; +import com.alibaba.otter.canal.client.adapter.support.Util; + +@Component +public class ApplicationConfigMonitor { + + private static final Logger logger = LoggerFactory.getLogger(ApplicationConfigMonitor.class); + + @Resource + private ContextRefresher contextRefresher; + + @Resource + private CanalAdapterService canalAdapterService; + + private FileAlterationMonitor fileMonitor; + + @PostConstruct + public void init() { + File confDir = Util.getConfDirPath(); + try { + FileAlterationObserver observer = new FileAlterationObserver(confDir, + FileFilterUtils.and(FileFilterUtils.fileFileFilter(), + FileFilterUtils.prefixFileFilter("application"), + FileFilterUtils.suffixFileFilter("yml"))); + FileListener listener = new FileListener(); + observer.addListener(listener); + fileMonitor = new FileAlterationMonitor(3000, observer); + fileMonitor.start(); + + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @PreDestroy + public void destroy() { + try { + fileMonitor.stop(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private class FileListener extends FileAlterationListenerAdaptor { + + @Override + public void onFileChange(File file) { + super.onFileChange(file); + try { + // 检查yml格式 + new Yaml().loadAs(new FileReader(file), Map.class); + + canalAdapterService.destroy(); + + // refresh context + contextRefresher.refresh(); + + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + // ignore + } + canalAdapterService.init(); + logger.info("## adapter application config reloaded."); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/ConfigItem.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/ConfigItem.java new file mode 100644 index 00000000..0074fbbe --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/ConfigItem.java @@ -0,0 +1,56 @@ +package com.alibaba.otter.canal.adapter.launcher.monitor.remote; + +/** + * 配置对应对象 + * + * @author rewerma 2019-01-25 下午05:20:16 + * @version 1.0.0 + */ +public class ConfigItem { + + private Long id; + private String category; + private String name; + private String content; + private long modifiedTime; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public long getModifiedTime() { + return modifiedTime; + } + + public void setModifiedTime(long modifiedTime) { + this.modifiedTime = modifiedTime; + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java new file mode 100644 index 00000000..400e0463 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java @@ -0,0 +1,246 @@ +package com.alibaba.otter.canal.adapter.launcher.monitor.remote; + +import java.io.FileWriter; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.common.utils.CommonUtils; +import com.alibaba.otter.canal.common.utils.NamedThreadFactory; +import com.google.common.base.Joiner; +import com.google.common.collect.MapMaker; + +/** + * 基于数据库的远程配置装载器 + * + * @author rewerma 2019-01-25 下午05:20:16 + * @version 1.0.0 + */ +public class DbRemoteConfigLoader implements RemoteConfigLoader { + + private static final Logger logger = LoggerFactory.getLogger(DbRemoteConfigLoader.class); + + private DruidDataSource dataSource; + + private static volatile long currentConfigTimestamp = 0; + private Map remoteAdapterConfigs = new MapMaker().makeMap(); + + private ScheduledExecutorService executor = Executors.newScheduledThreadPool(2, + new NamedThreadFactory("remote-adapter-config-scan")); + + private RemoteAdapterMonitor remoteAdapterMonitor = new RemoteAdapterMonitorImpl(); + + public DbRemoteConfigLoader(String driverName, String jdbcUrl, String jdbcUsername, String jdbcPassword){ + dataSource = new DruidDataSource(); + if (StringUtils.isEmpty(driverName)) { + driverName = "com.mysql.jdbc.Driver"; + } + dataSource.setDriverClassName(driverName); + dataSource.setUrl(jdbcUrl); + dataSource.setUsername(jdbcUsername); + dataSource.setPassword(jdbcPassword); + dataSource.setInitialSize(1); + dataSource.setMinIdle(1); + dataSource.setMaxActive(1); + dataSource.setMaxWait(60000); + dataSource.setTimeBetweenEvictionRunsMillis(60000); + dataSource.setMinEvictableIdleTimeMillis(300000); + try { + dataSource.init(); + } catch (SQLException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + /** + * 加载远程application.yml配置 + */ + @Override + public void loadRemoteConfig() { + try { + // 加载远程adapter配置 + ConfigItem configItem = getRemoteAdapterConfig(); + if (configItem != null) { + if (configItem.getModifiedTime() != currentConfigTimestamp) { + currentConfigTimestamp = configItem.getModifiedTime(); + overrideLocalCanalConfig(configItem.getContent()); + logger.info("## Loaded remote adapter config: application.yml"); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + /** + * 获取远程application.yml配置 + * + * @return 配置对象 + */ + private ConfigItem getRemoteAdapterConfig() { + String sql = "select name, content, modified_time from canal_config where id=2"; + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(sql)) { + if (rs.next()) { + ConfigItem configItem = new ConfigItem(); + configItem.setId(2L); + configItem.setName(rs.getString("name")); + configItem.setContent(rs.getString("content")); + configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime()); + return configItem; + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + return null; + } + + /** + * 覆盖本地application.yml文件 + * + * @param content 文件内容 + */ + private void overrideLocalCanalConfig(String content) { + try (FileWriter writer = new FileWriter(CommonUtils.getConfPath() + "application.yml")) { + writer.write(content); + writer.flush(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + /** + * 加载adapter配置 + */ + @Override + public void loadRemoteAdapterConfigs() { + try { + // 加载远程adapter配置 + loadModifiedAdapterConfigs(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + /** + * 加载有变动的adapter配置 + */ + private void loadModifiedAdapterConfigs() { + Map remoteConfigStatus = new HashMap<>(); + String sql = "select id, category, name, modified_time from canal_adapter_config"; + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(sql)) { + while (rs.next()) { + ConfigItem configItem = new ConfigItem(); + configItem.setId(rs.getLong("id")); + configItem.setCategory(rs.getString("category")); + configItem.setName(rs.getString("name")); + configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime()); + remoteConfigStatus.put(configItem.getCategory() + "/" + configItem.getName(), configItem); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + + if (!remoteConfigStatus.isEmpty()) { + List changedIds = new ArrayList<>(); + + for (ConfigItem remoteConfigStat : remoteConfigStatus.values()) { + ConfigItem currentConfig = remoteAdapterConfigs + .get(remoteConfigStat.getCategory() + "/" + remoteConfigStat.getName()); + if (currentConfig == null) { + // 新增 + changedIds.add(remoteConfigStat.getId()); + } else { + // 修改 + if (currentConfig.getModifiedTime() != remoteConfigStat.getModifiedTime()) { + changedIds.add(remoteConfigStat.getId()); + } + } + } + if (!changedIds.isEmpty()) { + String contentsSql = "select id, category, name, content, modified_time from canal_adapter_config where id in (" + + Joiner.on(",").join(changedIds) + ")"; + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(contentsSql)) { + while (rs.next()) { + ConfigItem configItemNew = new ConfigItem(); + configItemNew.setId(rs.getLong("id")); + configItemNew.setCategory(rs.getString("category")); + configItemNew.setName(rs.getString("name")); + configItemNew.setContent(rs.getString("content")); + configItemNew.setModifiedTime(rs.getTimestamp("modified_time").getTime()); + + remoteAdapterConfigs.put(configItemNew.getCategory() + "/" + configItemNew.getName(), + configItemNew); + remoteAdapterMonitor.onModify(configItemNew); + } + + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + } + + for (ConfigItem configItem : remoteAdapterConfigs.values()) { + if (!remoteConfigStatus.containsKey(configItem.getCategory() + "/" + configItem.getName())) { + // 删除 + remoteAdapterConfigs.remove(configItem.getCategory() + "/" + configItem.getName()); + remoteAdapterMonitor.onDelete(configItem.getCategory() + "/" + configItem.getName()); + } + } + } + + /** + * 启动监听数据库变化 + */ + @Override + public void startMonitor() { + // 监听application.yml变化 + executor.scheduleWithFixedDelay(() -> { + try { + loadRemoteConfig(); + } catch (Throwable e) { + logger.error("scan remote application.yml failed", e); + } + }, 10, 3, TimeUnit.SECONDS); + + // 监听adapter变化 + executor.scheduleWithFixedDelay(() -> { + try { + loadRemoteAdapterConfigs(); + } catch (Throwable e) { + logger.error("scan remote adapter configs failed", e); + } + }, 10, 3, TimeUnit.SECONDS); + } + + /** + * 销毁 + */ + @Override + public void destroy() { + executor.shutdownNow(); + try { + dataSource.close(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitor.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitor.java new file mode 100644 index 00000000..b0907314 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitor.java @@ -0,0 +1,31 @@ +package com.alibaba.otter.canal.adapter.launcher.monitor.remote; + +/** + * 远程配置监听器接口 + * + * @author rewerma 2019-01-25 下午05:20:16 + * @version 1.0.0 + */ +public interface RemoteAdapterMonitor { + + /** + * 新增配置事件 + * + * @param configItem 配置项 + */ + void onAdd(ConfigItem configItem); + + /** + * 修改配置事件 + * + * @param configItem 配置项 + */ + void onModify(ConfigItem configItem); + + /** + * 删除配置事件 + * + * @param name 配置名 + */ + void onDelete(String name); +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java new file mode 100644 index 00000000..6b4c2b81 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java @@ -0,0 +1,56 @@ +package com.alibaba.otter.canal.adapter.launcher.monitor.remote; + +import com.alibaba.otter.canal.common.utils.CommonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileWriter; + +/** + * 远程配置监听器实现 + * + * @author rewerma 2019-01-25 下午05:20:16 + * @version 1.0.0 + */ +public class RemoteAdapterMonitorImpl implements RemoteAdapterMonitor { + + private static final Logger logger = LoggerFactory.getLogger(RemoteAdapterMonitorImpl.class); + + @Override + public void onAdd(ConfigItem configItem) { + this.onModify(configItem); + } + + @Override + public void onModify(ConfigItem configItem) { + String confPath = CommonUtils.getConfPath(); + String category = configItem.getCategory(); + File categoryDir = new File(confPath + category); + if (!categoryDir.isDirectory()) { + boolean mkDirs = categoryDir.mkdirs(); + if (!mkDirs) { + logger.info("## Create adapter category dir error: {}", category); + return; + } + } + String name = configItem.getName(); + try (FileWriter writer = new FileWriter(confPath + category + "/" + configItem.getName())) { + writer.write(configItem.getContent()); + writer.flush(); + logger.info("## Loaded remote adapter config: {}/{}", category, name); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public void onDelete(String name) { + File file = new File(CommonUtils.getConfPath() + name); + if (file.exists()) { + CommonUtils.deleteDir(file); + logger.info("## Deleted and reloaded remote adapter config: {}", name); + } + } + +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteConfigLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteConfigLoader.java new file mode 100644 index 00000000..36af05bd --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteConfigLoader.java @@ -0,0 +1,30 @@ +package com.alibaba.otter.canal.adapter.launcher.monitor.remote; + +/** + * 远程配置装载器接口 + * + * @author rewerma 2019-01-25 下午05:20:16 + * @version 1.0.0 + */ +public interface RemoteConfigLoader { + + /** + * 加载远程application.yml配置到本地 + */ + void loadRemoteConfig(); + + /** + * 加载adapter配置 + */ + void loadRemoteAdapterConfigs(); + + /** + * 启动监听数据库变化 + */ + void startMonitor(); + + /** + * 销毁 + */ + void destroy(); +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteConfigLoaderFactory.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteConfigLoaderFactory.java new file mode 100644 index 00000000..355d3df5 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteConfigLoaderFactory.java @@ -0,0 +1,34 @@ +package com.alibaba.otter.canal.adapter.launcher.monitor.remote; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.env.Environment; + +/** + * 远程配置装载器工厂类 + * + * @author rewerma 2019-01-25 下午05:20:16 + * @version 1.0.0 + */ +public class RemoteConfigLoaderFactory { + + private static final Logger logger = LoggerFactory.getLogger(RemoteConfigLoaderFactory.class); + + public static RemoteConfigLoader getRemoteConfigLoader(Environment env) { + try { + String jdbcUrl = env.getProperty("canal.manager.jdbc.url"); + if (!StringUtils.isEmpty(jdbcUrl)) { + // load remote config + String driverName = env.getProperty("canal.manager.jdbc.driverName"); + String jdbcUsername = env.getProperty("canal.manager.jdbc.username"); + String jdbcPassword = env.getProperty("canal.manager.jdbc.password"); + return new DbRemoteConfigLoader(driverName, jdbcUrl, jdbcUsername, jdbcPassword); + } + // 可扩展其它远程配置加载器 + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + return null; + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java new file mode 100644 index 00000000..302206be --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java @@ -0,0 +1,209 @@ +package com.alibaba.otter.canal.adapter.launcher.rest; + +import java.util.*; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.*; + +import com.alibaba.otter.canal.adapter.launcher.common.EtlLock; +import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch; +import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; +import com.alibaba.otter.canal.client.adapter.support.Result; + +/** + * 适配器操作Rest + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +@RestController +public class CommonRest { + + private static Logger logger = LoggerFactory.getLogger(CommonRest.class); + + private static final String ETL_LOCK_ZK_NODE = "/sync-etl/"; + + private ExtensionLoader loader; + + @Resource + private SyncSwitch syncSwitch; + @Resource + private EtlLock etlLock; + + @Resource + private AdapterCanalConfig adapterCanalConfig; + + @PostConstruct + public void init() { + loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class); + } + + /** + * ETL curl http://127.0.0.1:8081/etl/rdb/oracle1/mytest_user.yml -X POST + * + * @param type 类型 hbase, es + * @param key adapter key + * @param task 任务名对应配置文件名 mytest_user.yml + * @param params etl where条件参数, 为空全部导入 + */ + @PostMapping("/etl/{type}/{key}/{task}") + public EtlResult etl(@PathVariable String type, @PathVariable String key, @PathVariable String task, + @RequestParam(name = "params", required = false) String params) { + OuterAdapter adapter = loader.getExtension(type, key); + String destination = adapter.getDestination(task); + String lockKey = destination == null ? task : destination; + + boolean locked = etlLock.tryLock(ETL_LOCK_ZK_NODE + type + "-" + lockKey); + if (!locked) { + EtlResult result = new EtlResult(); + result.setSucceeded(false); + result.setErrorMessage(task + " 有其他进程正在导入中, 请稍后再试"); + return result; + } + try { + + Boolean oriSwitchStatus; + if (destination != null) { + oriSwitchStatus = syncSwitch.status(destination); + if (oriSwitchStatus != null && oriSwitchStatus) { + syncSwitch.off(destination); + } + } else { + // task可能为destination，直接锁task + oriSwitchStatus = syncSwitch.status(task); + if (oriSwitchStatus != null && oriSwitchStatus) { + syncSwitch.off(task); + } + } + try { + List paramArray = null; + if (params != null) { + paramArray = Arrays.asList(params.trim().split(";")); + } + return adapter.etl(task, paramArray); + } finally { + if (destination != null && oriSwitchStatus != null && oriSwitchStatus) { + syncSwitch.on(destination); + } else if (destination == null && oriSwitchStatus != null && oriSwitchStatus) { + syncSwitch.on(task); + } + } + } finally { + etlLock.unlock(ETL_LOCK_ZK_NODE + type + "-" + lockKey); + } + } + + /** + * ETL curl http://127.0.0.1:8081/etl/hbase/mytest_person2.yml -X POST + * + * @param type 类型 hbase, es + * @param task 任务名对应配置文件名 mytest_person2.yml + * @param params etl where条件参数, 为空全部导入 + */ + @PostMapping("/etl/{type}/{task}") + public EtlResult etl(@PathVariable String type, @PathVariable String task, + @RequestParam(name = "params", required = false) String params) { + return etl(type, null, task, params); + } + + /** + * 统计总数 curl http://127.0.0.1:8081/count/rdb/oracle1/mytest_user.yml + * + * @param type 类型 hbase, es + * @param key adapter key + * @param task 任务名对应配置文件名 mytest_person2.yml + * @return + */ + @GetMapping("/count/{type}/{key}/{task}") + public Map count(@PathVariable String type, @PathVariable String key, @PathVariable String task) { + OuterAdapter adapter = loader.getExtension(type, key); + return adapter.count(task); + } + + /** + * 统计总数 curl http://127.0.0.1:8081/count/hbase/mytest_person2.yml + * + * @param type 类型 hbase, es + * @param task 任务名对应配置文件名 mytest_person2.yml + * @return + */ + @GetMapping("/count/{type}/{task}") + public Map count(@PathVariable String type, @PathVariable String task) { + return count(type, null, task); + } + + /** + * 返回所有实例 curl http://127.0.0.1:8081/destinations + */ + @GetMapping("/destinations") + public List> destinations() { + List> result = new ArrayList<>(); + Set destinations = adapterCanalConfig.DESTINATIONS; + for (String destination : destinations) { + Map resMap = new LinkedHashMap<>(); + Boolean status = syncSwitch.status(destination); + String resStatus = "none"; + if (status != null && status) { + resStatus = "on"; + } else if (status != null && !status) { + resStatus = "off"; + } + resMap.put("destination", destination); + resMap.put("status", resStatus); + result.add(resMap); + } + return result; + } + + /** + * 实例同步开关 curl http://127.0.0.1:8081/syncSwitch/example/off -X PUT + * + * @param destination 实例名称 + * @param status 开关状态: off on + * @return + */ + @PutMapping("/syncSwitch/{destination}/{status}") + public Result etl(@PathVariable String destination, @PathVariable String status) { + if (status.equals("on")) { + syncSwitch.on(destination); + logger.info("#Destination: {} sync on", destination); + return Result.createSuccess("实例: " + destination + " 开启同步成功"); + } else if (status.equals("off")) { + syncSwitch.off(destination); + logger.info("#Destination: {} sync off", destination); + return Result.createSuccess("实例: " + destination + " 关闭同步成功"); + } else { + Result result = new Result(); + result.setCode(50000); + result.setMessage("实例: " + destination + " 操作失败"); + return result; + } + } + + /** + * 获取实例开关状态 curl http://127.0.0.1:8081/syncSwitch/example + * + * @param destination 实例名称 + * @return + */ + @GetMapping("/syncSwitch/{destination}") + public Map etl(@PathVariable String destination) { + Boolean status = syncSwitch.status(destination); + String resStatus = "none"; + if (status != null && status) { + resStatus = "on"; + } else if (status != null && !status) { + resStatus = "off"; + } + Map res = new LinkedHashMap<>(); + res.put("stauts", resStatus); + return res; + } +} diff --git a/client-adapter/launcher/src/main/resources/META-INF/spring.factories b/client-adapter/launcher/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000..29018217 --- /dev/null +++ b/client-adapter/launcher/src/main/resources/META-INF/spring.factories @@ -0,0 +1,3 @@ +# Bootstrap Configuration +org.springframework.cloud.bootstrap.BootstrapConfiguration=\ + com.alibaba.otter.canal.adapter.launcher.config.BootstrapConfiguration \ No newline at end of file diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml new file mode 100644 index 00000000..f08b1df9 --- /dev/null +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -0,0 +1,69 @@ +server: + port: 8081 +logging: + level: + org.springframework: WARN + com.alibaba.otter.canal.client.adapter.hbase: DEBUG + com.alibaba.otter.canal.client.adapter.es: DEBUG + com.alibaba.otter.canal.client.adapter.rdb: DEBUG +spring: + jackson: + date-format: yyyy-MM-dd HH:mm:ss + time-zone: GMT+8 + default-property-inclusion: non_null + +canal.conf: + canalServerHost: 127.0.0.1:11111 +# zookeeperHosts: slave1:2181 +# mqServers: 127.0.0.1:9092 #or rocketmq +# flatMessage: true + batchSize: 500 + syncBatchSize: 1000 + retries: 0 + timeout: + accessKey: + secretKey: + mode: tcp # kafka rocketMQ +# srcDataSources: +# defaultDS: +# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true +# username: root +# password: 121212 + canalAdapters: + - instance: example # canal instance Name or mq topic name + groups: + - groupId: g1 + outerAdapters: + - name: logger +# - name: rdb +# key: mysql1 +# properties: +# jdbc.driverClassName: com.mysql.jdbc.Driver +# jdbc.url: jdbc:mysql://127.0.0.1:3306/mytest2?useUnicode=true +# jdbc.username: root +# jdbc.password: 121212 +# - name: rdb +# key: oracle1 +# properties: +# jdbc.driverClassName: oracle.jdbc.OracleDriver +# jdbc.url: jdbc:oracle:thin:@localhost:49161:XE +# jdbc.username: mytest +# jdbc.password: m121212 +# - name: rdb +# key: postgres1 +# properties: +# jdbc.driverClassName: org.postgresql.Driver +# jdbc.url: jdbc:postgresql://localhost:5432/postgres +# jdbc.username: postgres +# jdbc.password: 121212 +# threads: 1 +# commitSize: 3000 +# - name: hbase +# properties: +# hbase.zookeeper.quorum: 127.0.0.1 +# hbase.zookeeper.property.clientPort: 2181 +# zookeeper.znode.parent: /hbase +# - name: es +# hosts: 127.0.0.1:9300 +# properties: +# cluster.name: elasticsearch diff --git a/client-adapter/launcher/src/main/resources/bootstrap.yml b/client-adapter/launcher/src/main/resources/bootstrap.yml new file mode 100644 index 00000000..c36f305f --- /dev/null +++ b/client-adapter/launcher/src/main/resources/bootstrap.yml @@ -0,0 +1,6 @@ +#canal: +# manager: +# jdbc: +# url: jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8 +# username: root +# password: 121212 \ No newline at end of file diff --git a/client-adapter/logger/pom.xml b/client-adapter/logger/pom.xml index 8f2b4edb..11c8d53e 100644 --- a/client-adapter/logger/pom.xml +++ b/client-adapter/logger/pom.xml @@ -1,11 +1,9 @@ - + canal.client-adapter com.alibaba.otter - 1.1.1-SNAPSHOT + 1.1.3-SNAPSHOT 4.0.0 com.alibaba.otter @@ -21,6 +19,7 @@     + diff --git a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java index c72a0ee0..08fd171f 100644 --- a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java +++ b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java @@ -1,11 +1,16 @@ package com.alibaba.otter.canal.client.adapter.logger; +import java.util.List; +import java.util.Properties; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; -import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.support.Dml; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; import com.alibaba.otter.canal.client.adapter.support.SPI; /** @@ -14,19 +19,25 @@ import com.alibaba.otter.canal.client.adapter.support.SPI; * @author machengyuan 2018-8-19 下午11:45:38 * @version 1.0.0 */ -@SPI("logger") // logger参数对应CanalOuterAdapterConfiguration配置中的name -public class LoggerAdapterExample implements CanalOuterAdapter { +@SPI("logger") +// logger参数对应CanalOuterAdapterConfiguration配置中的name +public class LoggerAdapterExample implements OuterAdapter { private Logger logger = LoggerFactory.getLogger(this.getClass()); @Override - public void init(CanalOuterAdapterConfiguration configuration) { + public void init(OuterAdapterConfig configuration, Properties envProperties) { } - @Override - public void writeOut(Dml dml) { - logger.info(dml.toString()); + public void sync(List dmls) { + for (Dml dml : dmls) { + sync(dml); + } + } + + public void sync(Dml dml) { + logger.info("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue)); } @Override diff --git a/client-adapter/logger/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.CanalOuterAdapter b/client-adapter/logger/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter similarity index 100% rename from client-adapter/logger/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.CanalOuterAdapter rename to client-adapter/logger/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter diff --git a/client-adapter/pom.xml b/client-adapter/pom.xml index 792ce283..84bc7332 100644 --- a/client-adapter/pom.xml +++ b/client-adapter/pom.xml @@ -1,21 +1,123 @@ - - - canal - com.alibaba.otter - 1.1.1-SNAPSHOT - + 4.0.0 com.alibaba.otter canal.client-adapter + 1.1.3-SNAPSHOT pom canal client adapter module for otter ${project.version} + + + UTF-8 + true + true + 1.8 + 1.8 + UTF-8 + + common logger hbase + elasticsearch + launcher + rdb - + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0 + + + + + git@github.com:alibaba/canal.git + scm:git:git@github.com:alibaba/canal.git + scm:git:git@github.com:alibaba/canal.git + + + + + central + http://repo1.maven.org/maven2 + + true + + + false + + + + java.net + http://download.java.net/maven/2/ + + true + + + false + + + + aliyun + http://maven.aliyun.com/nexus/content/groups/public/ + + true + + + false + + + + sonatype + sonatype + https://oss.sonatype.org/content/repositories/snapshots + + false + + + true + + + + sonatype-release + sonatype-release + https://oss.sonatype.org/service/local/repositories/releases/content + + false + + + true + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.7.0 + + ${java_source_version} + ${java_target_version} + ${file_encoding} + + + + + + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + + sonatype-nexus-staging + Nexus Release Repository + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + \ No newline at end of file diff --git a/client-adapter/rdb/pom.xml b/client-adapter/rdb/pom.xml new file mode 100644 index 00000000..0fb2c0ea --- /dev/null +++ b/client-adapter/rdb/pom.xml @@ -0,0 +1,96 @@ + + + + canal.client-adapter + com.alibaba.otter + 1.1.3-SNAPSHOT + + 4.0.0 + com.alibaba.otter + client-adapter.rdb + jar + canal client adapter rdb module for otter ${project.version} + + + + com.alibaba.otter + client-adapter.common + ${project.version} + provided + + + mysql + mysql-connector-java + 5.1.40 + test + + + org.postgresql + postgresql + 42.1.4 + test + + + com.oracle + ojdbc6 + 11.2.0.3 + test + + + com.microsoft.sqlserver + mssql-jdbc + 7.0.0.jre8 + test + + + junit + junit + 4.12 + test + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + maven-antrun-plugin + + + package + + run + + + + + + + + + + + + + + + + diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java new file mode 100644 index 00000000..8d1cab75 --- /dev/null +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java @@ -0,0 +1,298 @@ +package com.alibaba.otter.canal.client.adapter.rdb; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; + +import javax.sql.DataSource; + +import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.rdb.config.ConfigLoader; +import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; +import com.alibaba.otter.canal.client.adapter.rdb.config.MirrorDbConfig; +import com.alibaba.otter.canal.client.adapter.rdb.monitor.RdbConfigMonitor; +import com.alibaba.otter.canal.client.adapter.rdb.service.RdbEtlService; +import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService; +import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService; +import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil; +import com.alibaba.otter.canal.client.adapter.support.*; + +/** + * RDB适配器实现类 + * + * @author rewerma 2018-11-7 下午06:45:49 + * @version 1.0.0 + */ +@SPI("rdb") +public class RdbAdapter implements OuterAdapter { + + private static Logger logger = LoggerFactory.getLogger(RdbAdapter.class); + + private Map rdbMapping = new ConcurrentHashMap<>(); // 文件名对应配置 + private Map> mappingConfigCache = new ConcurrentHashMap<>(); // 库名-表名对应配置 + private Map mirrorDbConfigCache = new ConcurrentHashMap<>(); // 镜像库配置 + + private DruidDataSource dataSource; + + private RdbSyncService rdbSyncService; + private RdbMirrorDbSyncService rdbMirrorDbSyncService; + + private RdbConfigMonitor rdbConfigMonitor; + + private Properties envProperties; + + public Map getRdbMapping() { + return rdbMapping; + } + + public Map> getMappingConfigCache() { + return mappingConfigCache; + } + + public Map getMirrorDbConfigCache() { + return mirrorDbConfigCache; + } + + /** + * 初始化方法 + * + * @param configuration 外部适配器配置信息 + */ + @Override + public void init(OuterAdapterConfig configuration, Properties envProperties) { + this.envProperties = envProperties; + Map rdbMappingTmp = ConfigLoader.load(envProperties); + // 过滤不匹配的key的配置 + rdbMappingTmp.forEach((key, mappingConfig) -> { + if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null) + || (mappingConfig.getOuterAdapterKey() != null + && mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) { + rdbMapping.put(key, mappingConfig); + } + }); + + if (rdbMapping.isEmpty()) { + throw new RuntimeException("No rdb adapter found for config key: " + configuration.getKey()); + } + + for (Map.Entry entry : rdbMapping.entrySet()) { + String configName = entry.getKey(); + MappingConfig mappingConfig = entry.getValue(); + if (!mappingConfig.getDbMapping().getMirrorDb()) { + String key; + if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-" + + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_" + + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable(); + } else { + key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_" + + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable(); + } + Map configMap = mappingConfigCache.computeIfAbsent(key, + k1 -> new ConcurrentHashMap<>()); + configMap.put(configName, mappingConfig); + } else { + // mirrorDB + String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + + mappingConfig.getDbMapping().getDatabase(); + mirrorDbConfigCache.put(key, MirrorDbConfig.create(configName, mappingConfig)); + } + } + + // 初始化连接池 + Map properties = configuration.getProperties(); + dataSource = new DruidDataSource(); + dataSource.setDriverClassName(properties.get("jdbc.driverClassName")); + dataSource.setUrl(properties.get("jdbc.url")); + dataSource.setUsername(properties.get("jdbc.username")); + dataSource.setPassword(properties.get("jdbc.password")); + dataSource.setInitialSize(1); + dataSource.setMinIdle(1); + dataSource.setMaxActive(30); + dataSource.setMaxWait(60000); + dataSource.setTimeBetweenEvictionRunsMillis(60000); + dataSource.setMinEvictableIdleTimeMillis(300000); + dataSource.setUseUnfairLock(true); + + try { + dataSource.init(); + } catch (SQLException e) { + logger.error("ERROR ## failed to initial datasource: " + properties.get("jdbc.url"), e); + } + + String threads = properties.get("threads"); + // String commitSize = properties.get("commitSize"); + + boolean skipDupException = BooleanUtils + .toBoolean(configuration.getProperties().getOrDefault("skipDupException", "true")); + rdbSyncService = new RdbSyncService(dataSource, + threads != null ? Integer.valueOf(threads) : null, + skipDupException); + + rdbMirrorDbSyncService = new RdbMirrorDbSyncService(mirrorDbConfigCache, + dataSource, + threads != null ? Integer.valueOf(threads) : null, + rdbSyncService.getColumnsTypeCache(), + skipDupException); + + rdbConfigMonitor = new RdbConfigMonitor(); + rdbConfigMonitor.init(configuration.getKey(), this); + } + + /** + * 同步方法 + * + * @param dmls 数据包 + */ + @Override + public void sync(List dmls) { + if (dmls == null || dmls.isEmpty()) { + return; + } + try { + rdbSyncService.sync(mappingConfigCache, dmls, envProperties); + rdbMirrorDbSyncService.sync(dmls); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * ETL方法 + * + * @param task 任务名, 对应配置名 + * @param params etl筛选条件 + * @return ETL结果 + */ + @Override + public EtlResult etl(String task, List params) { + EtlResult etlResult = new EtlResult(); + MappingConfig config = rdbMapping.get(task); + if (config != null) { + DataSource srcDataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (srcDataSource != null) { + return RdbEtlService.importData(srcDataSource, dataSource, config, params); + } else { + etlResult.setSucceeded(false); + etlResult.setErrorMessage("DataSource not found"); + return etlResult; + } + } else { + StringBuilder resultMsg = new StringBuilder(); + boolean resSucc = true; + // ds不为空说明传入的是destination + for (MappingConfig configTmp : rdbMapping.values()) { + // 取所有的destination为task的配置 + if (configTmp.getDestination().equals(task)) { + DataSource srcDataSource = DatasourceConfig.DATA_SOURCES.get(configTmp.getDataSourceKey()); + if (srcDataSource == null) { + continue; + } + EtlResult etlRes = RdbEtlService.importData(srcDataSource, dataSource, configTmp, params); + if (!etlRes.getSucceeded()) { + resSucc = false; + resultMsg.append(etlRes.getErrorMessage()).append("\n"); + } else { + resultMsg.append(etlRes.getResultMessage()).append("\n"); + } + } + } + if (resultMsg.length() > 0) { + etlResult.setSucceeded(resSucc); + if (resSucc) { + etlResult.setResultMessage(resultMsg.toString()); + } else { + etlResult.setErrorMessage(resultMsg.toString()); + } + return etlResult; + } + } + etlResult.setSucceeded(false); + etlResult.setErrorMessage("Task not found"); + return etlResult; + } + + /** + * 获取总数方法 + * + * @param task 任务名, 对应配置名 + * @return 总数 + */ + @Override + public Map count(String task) { + MappingConfig config = rdbMapping.get(task); + MappingConfig.DbMapping dbMapping = config.getDbMapping(); + String sql = "SELECT COUNT(1) AS cnt FROM " + SyncUtil.getDbTableName(dbMapping); + Connection conn = null; + Map res = new LinkedHashMap<>(); + try { + conn = dataSource.getConnection(); + Util.sqlRS(conn, sql, rs -> { + try { + if (rs.next()) { + Long rowCount = rs.getLong("cnt"); + res.put("count", rowCount); + } + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + }); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } finally { + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + } + res.put("targetTable", SyncUtil.getDbTableName(dbMapping)); + + return res; + } + + /** + * 获取对应canal instance name 或 mq topic + * + * @param task 任务名, 对应配置名 + * @return destination + */ + @Override + public String getDestination(String task) { + MappingConfig config = rdbMapping.get(task); + if (config != null) { + return config.getDestination(); + } + return null; + } + + /** + * 销毁方法 + */ + @Override + public void destroy() { + if (rdbConfigMonitor != null) { + rdbConfigMonitor.destroy(); + } + + if (rdbSyncService != null) { + rdbSyncService.close(); + } + + if (dataSource != null) { + dataSource.close(); + } + } +} diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java new file mode 100644 index 00000000..d65e0ec5 --- /dev/null +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java @@ -0,0 +1,51 @@ +package com.alibaba.otter.canal.client.adapter.rdb.config; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; +import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; + +/** + * RDB表映射配置加载器 + * + * @author rewerma 2018-11-07 下午02:41:34 + * @version 1.0.0 + */ +public class ConfigLoader { + + private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class); + + /** + * 加载RDB表映射配置 + * + * @return 配置名/配置文件名--对象 + */ + public static Map load(Properties envProperties) { + logger.info("## Start loading rdb mapping config ... "); + + Map result = new LinkedHashMap<>(); + + Map configContentMap = MappingConfigsLoader.loadConfigs("rdb"); + configContentMap.forEach((fileName, content) -> { + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); + if (config == null) { + return; + } + try { + config.validate(); + } catch (Exception e) { + throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e); + } + result.put(fileName, config); + }); + + logger.info("## Rdb mapping config loaded"); + return result; + } +} diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java new file mode 100644 index 00000000..d2421020 --- /dev/null +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java @@ -0,0 +1,209 @@ +package com.alibaba.otter.canal.client.adapter.rdb.config; + +import org.apache.commons.lang.StringUtils; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * RDB表映射配置 + * + * @author rewerma 2018-11-07 下午02:41:34 + * @version 1.0.0 + */ +public class MappingConfig { + + private String dataSourceKey; // 数据源key + + private String destination; // canal实例或MQ的topic + + private String groupId; // groupId + + private String outerAdapterKey; // 对应适配器的key + + private Boolean concurrent; // 是否并行同步 + + private DbMapping dbMapping; // db映射配置 + + public String getDataSourceKey() { + return dataSourceKey; + } + + public void setDataSourceKey(String dataSourceKey) { + this.dataSourceKey = dataSourceKey; + } + + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + + public String getOuterAdapterKey() { + return outerAdapterKey; + } + + public void setOuterAdapterKey(String outerAdapterKey) { + this.outerAdapterKey = outerAdapterKey; + } + + public Boolean getConcurrent() { + return concurrent == null ? false : concurrent; + } + + public void setConcurrent(Boolean concurrent) { + this.concurrent = concurrent; + } + + public DbMapping getDbMapping() { + return dbMapping; + } + + public void setDbMapping(DbMapping dbMapping) { + this.dbMapping = dbMapping; + } + + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public void validate() { + if (dbMapping.database == null || dbMapping.database.isEmpty()) { + throw new NullPointerException("dbMapping.database"); + } + if (!dbMapping.getMirrorDb() && (dbMapping.table == null || dbMapping.table.isEmpty())) { + throw new NullPointerException("dbMapping.table"); + } + if (!dbMapping.getMirrorDb() && (dbMapping.targetTable == null || dbMapping.targetTable.isEmpty())) { + throw new NullPointerException("dbMapping.targetTable"); + } + } + + public static class DbMapping { + + private Boolean mirrorDb = false; // 是否镜像库 + private String database; // 数据库名或schema名 + private String table; // 表名 + private Map targetPk = new LinkedHashMap<>(); // 目标表主键字段 + private Boolean mapAll = false; // 映射所有字段 + private String targetDb; // 目标库名 + private String targetTable; // 目标表名 + private Map targetColumns; // 目标表字段映射 + + private String etlCondition; // etl条件sql + + private int readBatch = 5000; + private int commitBatch = 5000; // etl等批量提交大小 + + private Map allMapColumns; + + public Boolean getMirrorDb() { + return mirrorDb == null ? false : mirrorDb; + } + + public void setMirrorDb(Boolean mirrorDb) { + this.mirrorDb = mirrorDb; + } + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public Map getTargetPk() { + return targetPk; + } + + public void setTargetPk(Map targetPk) { + this.targetPk = targetPk; + } + + public Boolean getMapAll() { + return mapAll; + } + + public void setMapAll(Boolean mapAll) { + this.mapAll = mapAll; + } + + public String getTargetDb() { + return targetDb; + } + + public void setTargetDb(String targetDb) { + this.targetDb = targetDb; + } + + public String getTargetTable() { + return targetTable; + } + + public void setTargetTable(String targetTable) { + this.targetTable = targetTable; + } + + public Map getTargetColumns() { + if (targetColumns != null) { + targetColumns.forEach((key, value) -> { + if (StringUtils.isEmpty(value)) { + targetColumns.put(key, key); + } + }); + } + return targetColumns; + } + + public void setTargetColumns(Map targetColumns) { + this.targetColumns = targetColumns; + } + + public String getEtlCondition() { + return etlCondition; + } + + public void setEtlCondition(String etlCondition) { + this.etlCondition = etlCondition; + } + + public int getReadBatch() { + return readBatch; + } + + public void setReadBatch(int readBatch) { + this.readBatch = readBatch; + } + + public int getCommitBatch() { + return commitBatch; + } + + public void setCommitBatch(int commitBatch) { + this.commitBatch = commitBatch; + } + + public Map getAllMapColumns() { + return allMapColumns; + } + + public void setAllMapColumns(Map allMapColumns) { + this.allMapColumns = allMapColumns; + } + } +} diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MirrorDbConfig.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MirrorDbConfig.java new file mode 100644 index 00000000..dbead72e --- /dev/null +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MirrorDbConfig.java @@ -0,0 +1,44 @@ +package com.alibaba.otter.canal.client.adapter.rdb.config; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class MirrorDbConfig { + + private String fileName; + private MappingConfig mappingConfig; + private Map tableConfig = new ConcurrentHashMap<>(); + + public static MirrorDbConfig create(String fileName, MappingConfig mappingConfig) { + return new MirrorDbConfig(fileName, mappingConfig); + } + + public MirrorDbConfig(String fileName, MappingConfig mappingConfig){ + this.fileName = fileName; + this.mappingConfig = mappingConfig; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public MappingConfig getMappingConfig() { + return mappingConfig; + } + + public void setMappingConfig(MappingConfig mappingConfig) { + this.mappingConfig = mappingConfig; + } + + public Map getTableConfig() { + return tableConfig; + } + + public void setTableConfig(Map tableConfig) { + this.tableConfig = tableConfig; + } +} diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java new file mode 100644 index 00000000..72529a37 --- /dev/null +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java @@ -0,0 +1,169 @@ +package com.alibaba.otter.canal.client.adapter.rdb.monitor; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.io.monitor.FileAlterationListenerAdaptor; +import org.apache.commons.io.monitor.FileAlterationMonitor; +import org.apache.commons.io.monitor.FileAlterationObserver; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; + +import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter; +import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; +import com.alibaba.otter.canal.client.adapter.rdb.config.MirrorDbConfig; +import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; +import com.alibaba.otter.canal.client.adapter.support.Util; + +public class RdbConfigMonitor { + + private static final Logger logger = LoggerFactory.getLogger(RdbConfigMonitor.class); + + private static final String adapterName = "rdb"; + + private String key; + + private RdbAdapter rdbAdapter; + + private FileAlterationMonitor fileMonitor; + + public void init(String key, RdbAdapter rdbAdapter) { + this.key = key; + this.rdbAdapter = rdbAdapter; + File confDir = Util.getConfDirPath(adapterName); + try { + FileAlterationObserver observer = new FileAlterationObserver(confDir, + FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml"))); + FileListener listener = new FileListener(); + observer.addListener(listener); + fileMonitor = new FileAlterationMonitor(3000, observer); + fileMonitor.start(); + + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + public void destroy() { + try { + fileMonitor.stop(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private class FileListener extends FileAlterationListenerAdaptor { + + @Override + public void onFileCreate(File file) { + super.onFileCreate(file); + try { + // 加载新增的配置文件 + String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); + MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class); + config.validate(); + if ((key == null && config.getOuterAdapterKey() == null) + || (key != null && key.equals(config.getOuterAdapterKey()))) { + addConfigToCache(file, config); + + logger.info("Add a new rdb mapping config: {} to canal adapter", file.getName()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public void onFileChange(File file) { + super.onFileChange(file); + + try { + if (rdbAdapter.getRdbMapping().containsKey(file.getName())) { + // 加载配置文件 + String configContent = MappingConfigsLoader + .loadConfig(adapterName + File.separator + file.getName()); + if (configContent == null) { + onFileDelete(file); + return; + } + MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class); + config.validate(); + if ((key == null && config.getOuterAdapterKey() == null) + || (key != null && key.equals(config.getOuterAdapterKey()))) { + if (rdbAdapter.getRdbMapping().containsKey(file.getName())) { + deleteConfigFromCache(file); + } + addConfigToCache(file, config); + } else { + // 不能修改outerAdapterKey + throw new RuntimeException("Outer adapter key not allowed modify"); + } + logger.info("Change a rdb mapping config: {} of canal adapter", file.getName()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public void onFileDelete(File file) { + super.onFileDelete(file); + + try { + if (rdbAdapter.getRdbMapping().containsKey(file.getName())) { + deleteConfigFromCache(file); + + logger.info("Delete a rdb mapping config: {} of canal adapter", file.getName()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private void addConfigToCache(File file, MappingConfig mappingConfig) { + if (mappingConfig == null || mappingConfig.getDbMapping() == null) { + return; + } + rdbAdapter.getRdbMapping().put(file.getName(), mappingConfig); + if (!mappingConfig.getDbMapping().getMirrorDb()) { + Map configMap = rdbAdapter.getMappingConfigCache() + .computeIfAbsent(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + + mappingConfig.getDbMapping().getDatabase() + "." + + mappingConfig.getDbMapping().getTable(), + k1 -> new HashMap<>()); + configMap.put(file.getName(), mappingConfig); + } else { + Map mirrorDbConfigCache = rdbAdapter.getMirrorDbConfigCache(); + mirrorDbConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + + mappingConfig.getDbMapping().getDatabase(), + MirrorDbConfig.create(file.getName(), mappingConfig)); + } + } + + private void deleteConfigFromCache(File file) { + MappingConfig mappingConfig = rdbAdapter.getRdbMapping().remove(file.getName()); + + if (mappingConfig == null || mappingConfig.getDbMapping() == null) { + return; + } + if (!mappingConfig.getDbMapping().getMirrorDb()) { + for (Map configMap : rdbAdapter.getMappingConfigCache().values()) { + if (configMap != null) { + configMap.remove(file.getName()); + } + } + } else { + rdbAdapter.getMirrorDbConfigCache().forEach((key, mirrorDbConfig) -> { + if (mirrorDbConfig.getFileName().equals(file.getName())) { + rdbAdapter.getMirrorDbConfigCache().remove(key); + } + }); + } + + } + } +} diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java new file mode 100644 index 00000000..fbb188da --- /dev/null +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java @@ -0,0 +1,306 @@ +package com.alibaba.otter.canal.client.adapter.rdb.service; + +import java.sql.*; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import javax.sql.DataSource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; +import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig.DbMapping; +import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.alibaba.otter.canal.client.adapter.support.Util; +import com.google.common.base.Joiner; + +/** + * RDB ETL 操作业务类 + * + * @author rewerma @ 2018-11-7 + * @version 1.0.0 + */ +public class RdbEtlService { + + private static final Logger logger = LoggerFactory.getLogger(RdbEtlService.class); + + /** + * 导入数据 + */ + public static EtlResult importData(DataSource srcDS, DataSource targetDS, MappingConfig config, + List params) { + EtlResult etlResult = new EtlResult(); + AtomicLong successCount = new AtomicLong(); + List errMsg = new ArrayList<>(); + String hbaseTable = ""; + try { + if (config == null) { + logger.error("Config is null!"); + etlResult.setSucceeded(false); + etlResult.setErrorMessage("Config is null!"); + return etlResult; + } + DbMapping dbMapping = config.getDbMapping(); + + long start = System.currentTimeMillis(); + + // 拼接sql + StringBuilder sql = new StringBuilder( + "SELECT * FROM " + dbMapping.getDatabase() + "." + dbMapping.getTable()); + + // 拼接条件 + appendCondition(params, dbMapping, srcDS, sql); + + // 获取总数 + String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT "; + long cnt = (Long) Util.sqlRS(srcDS, countSql, rs -> { + Long count = null; + try { + if (rs.next()) { + count = ((Number) rs.getObject(1)).longValue(); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + return count == null ? 0 : count; + }); + + // 当大于1万条记录时开启多线程 + if (cnt >= 10000) { + int threadCount = 3; + long perThreadCnt = cnt / threadCount; + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + List> futures = new ArrayList<>(threadCount); + for (int i = 0; i < threadCount; i++) { + long offset = i * perThreadCnt; + Long size = null; + if (i != threadCount - 1) { + size = perThreadCnt; + } + String sqlFinal; + if (size != null) { + sqlFinal = sql + " LIMIT " + offset + "," + size; + } else { + sqlFinal = sql + " LIMIT " + offset + "," + cnt; + } + Future future = executor + .submit(() -> executeSqlImport(srcDS, targetDS, sqlFinal, dbMapping, successCount, errMsg)); + futures.add(future); + } + + for (Future future : futures) { + future.get(); + } + + executor.shutdown(); + } else { + executeSqlImport(srcDS, targetDS, sql.toString(), dbMapping, successCount, errMsg); + } + + logger.info( + dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!"); + + etlResult + .setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get() + " 条"); + } catch (Exception e) { + logger.error(e.getMessage(), e); + errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage()); + } + + if (errMsg.isEmpty()) { + etlResult.setSucceeded(true); + } else { + etlResult.setErrorMessage(Joiner.on("\n").join(errMsg)); + } + return etlResult; + } + + private static void appendCondition(List params, DbMapping dbMapping, DataSource ds, + StringBuilder sql) throws SQLException { + if (params != null && params.size() == 1 && dbMapping.getEtlCondition() == null) { + AtomicBoolean stExists = new AtomicBoolean(false); + // 验证是否有SYS_TIME字段 + Util.sqlRS(ds, sql.toString(), rs -> { + try { + ResultSetMetaData rsmd = rs.getMetaData(); + int cnt = rsmd.getColumnCount(); + for (int i = 1; i <= cnt; i++) { + String columnName = rsmd.getColumnName(i); + if ("SYS_TIME".equalsIgnoreCase(columnName)) { + stExists.set(true); + break; + } + } + } catch (Exception e) { + // ignore + } + return null; + }); + if (stExists.get()) { + sql.append(" WHERE SYS_TIME >= '").append(params.get(0)).append("' "); + } + } else if (dbMapping.getEtlCondition() != null && params != null) { + String etlCondition = dbMapping.getEtlCondition(); + int size = params.size(); + for (int i = 0; i < size; i++) { + etlCondition = etlCondition.replace("{" + i + "}", params.get(i)); + } + + sql.append(" ").append(etlCondition); + } + } + + /** + * 执行导入 + */ + private static boolean executeSqlImport(DataSource srcDS, DataSource targetDS, String sql, DbMapping dbMapping, + AtomicLong successCount, List errMsg) { + try { + Map columnsMap = new LinkedHashMap<>(); + Map columnType = new LinkedHashMap<>(); + + Util.sqlRS(targetDS, "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping) + " LIMIT 1 ", rs -> { + try { + + ResultSetMetaData rsd = rs.getMetaData(); + int columnCount = rsd.getColumnCount(); + List columns = new ArrayList<>(); + for (int i = 1; i <= columnCount; i++) { + columnType.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i)); + columns.add(rsd.getColumnName(i)); + } + + columnsMap.putAll(SyncUtil.getColumnsMap(dbMapping, columns)); + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + }); + + Util.sqlRS(srcDS, sql, rs -> { + int idx = 1; + + try { + boolean completed = false; + + // if (dbMapping.isMapAll()) { + // columnsMap = dbMapping.getAllColumns(); + // } else { + // columnsMap = dbMapping.getTargetColumns(); + // } + + StringBuilder insertSql = new StringBuilder(); + insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" ("); + columnsMap + .forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(",")); + + int len = insertSql.length(); + insertSql.delete(len - 1, len).append(") VALUES ("); + int mapLen = columnsMap.size(); + for (int i = 0; i < mapLen; i++) { + insertSql.append("?,"); + } + len = insertSql.length(); + insertSql.delete(len - 1, len).append(")"); + try (Connection connTarget = targetDS.getConnection(); + PreparedStatement pstmt = connTarget.prepareStatement(insertSql.toString())) { + connTarget.setAutoCommit(false); + + while (rs.next()) { + pstmt.clearParameters(); + + // 删除数据 + Map values = new LinkedHashMap<>(); + StringBuilder deleteSql = new StringBuilder( + "DELETE FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE "); + appendCondition(dbMapping, deleteSql, values, rs); + try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) { + int k = 1; + for (Object val : values.values()) { + pstmt2.setObject(k++, val); + } + pstmt2.execute(); + } + + int i = 1; + for (Map.Entry entry : columnsMap.entrySet()) { + String targetClolumnName = entry.getKey(); + String srcColumnName = entry.getValue(); + if (srcColumnName == null) { + srcColumnName = targetClolumnName; + } + + Integer type = columnType.get(targetClolumnName.toLowerCase()); + + Object value = rs.getObject(srcColumnName); + if (value != null) { + SyncUtil.setPStmt(type, pstmt, value, i); + } else { + pstmt.setNull(i, type); + } + + i++; + } + + pstmt.execute(); + if (logger.isTraceEnabled()) { + logger.trace("Insert into target table, sql: {}", insertSql); + } + + if (idx % dbMapping.getCommitBatch() == 0) { + connTarget.commit(); + completed = true; + } + idx++; + successCount.incrementAndGet(); + if (logger.isDebugEnabled()) { + logger.debug("successful import count:" + successCount.get()); + } + } + if (!completed) { + connTarget.commit(); + } + } + + } catch (Exception e) { + logger.error(dbMapping.getTable() + " etl failed! ==>" + e.getMessage(), e); + errMsg.add(dbMapping.getTable() + " etl failed! ==>" + e.getMessage()); + } + return idx; + }); + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + } + + /** + * 拼接目标表主键where条件 + */ + private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map values, + ResultSet rs) throws SQLException { + // 拼接主键 + for (Map.Entry entry : dbMapping.getTargetPk().entrySet()) { + String targetColumnName = entry.getKey(); + String srcColumnName = entry.getValue(); + if (srcColumnName == null) { + srcColumnName = targetColumnName; + } + sql.append(targetColumnName).append("=? AND "); + values.put(targetColumnName, rs.getObject(srcColumnName)); + } + int len = sql.length(); + sql.delete(len - 4, len); + } +} diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java new file mode 100644 index 00000000..9d574fc6 --- /dev/null +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java @@ -0,0 +1,165 @@ +package com.alibaba.otter.canal.client.adapter.rdb.service; + +import java.sql.Connection; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import javax.sql.DataSource; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; +import com.alibaba.otter.canal.client.adapter.rdb.config.MirrorDbConfig; +import com.alibaba.otter.canal.client.adapter.rdb.support.SingleDml; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +/** + * RDB镜像库同步操作业务 + * + * @author rewerma 2018-12-12 下午011:23 + * @version 1.0.0 + */ +public class RdbMirrorDbSyncService { + + private static final Logger logger = LoggerFactory.getLogger(RdbMirrorDbSyncService.class); + + private Map mirrorDbConfigCache; // 镜像库配置 + private DataSource dataSource; + private RdbSyncService rdbSyncService; // rdbSyncService代理 + + public RdbMirrorDbSyncService(Map mirrorDbConfigCache, DataSource dataSource, + Integer threads, Map> columnsTypeCache, + boolean skipDupException){ + this.mirrorDbConfigCache = mirrorDbConfigCache; + this.dataSource = dataSource; + this.rdbSyncService = new RdbSyncService(dataSource, threads, columnsTypeCache, skipDupException); + } + + /** + * 批量同步方法 + * + * @param dmls 批量 DML + */ + public void sync(List dmls) { + List dmlList = new ArrayList<>(); + for (Dml dml : dmls) { + String destination = StringUtils.trimToEmpty(dml.getDestination()); + String database = dml.getDatabase(); + MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(destination + "." + database); + if (mirrorDbConfig == null) { + continue; + } + if (mirrorDbConfig.getMappingConfig() == null) { + continue; + } + if (dml.getGroupId() != null && StringUtils.isNotEmpty(mirrorDbConfig.getMappingConfig().getGroupId())) { + if (!mirrorDbConfig.getMappingConfig().getGroupId().equals(dml.getGroupId())) { + continue; // 如果groupId不匹配则过滤 + } + } + + if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) { + // DDL + if (logger.isDebugEnabled()) { + logger.debug("DDL: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue)); + } + executeDdl(mirrorDbConfig, dml); + rdbSyncService.getColumnsTypeCache().remove(destination + "." + database + "." + dml.getTable()); + mirrorDbConfig.getTableConfig().remove(dml.getTable()); // 删除对应库表配置 + } else { + // DML + initMappingConfig(dml.getTable(), mirrorDbConfig.getMappingConfig(), mirrorDbConfig, dml); + dmlList.add(dml); + } + } + if (!dmlList.isEmpty()) { + rdbSyncService.sync(dmlList, dml -> { + MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "." + dml.getDatabase()); + if (mirrorDbConfig == null) { + return false; + } + String table = dml.getTable(); + MappingConfig config = mirrorDbConfig.getTableConfig().get(table); + + if (config == null) { + return false; + } + + if (config.getConcurrent()) { + List singleDmls = SingleDml.dml2SingleDmls(dml); + singleDmls.forEach(singleDml -> { + int hash = rdbSyncService.pkHash(config.getDbMapping(), singleDml.getData()); + RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml); + rdbSyncService.getDmlsPartition()[hash].add(syncItem); + }); + } else { + int hash = 0; + List singleDmls = SingleDml.dml2SingleDmls(dml); + singleDmls.forEach(singleDml -> { + RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml); + rdbSyncService.getDmlsPartition()[hash].add(syncItem); + }); + } + return true; + }); + } + } + + /** + * 初始化表配置 + * + * @param key 配置key: destination.database.table + * @param baseConfigMap db sync config + * @param dml DML + */ + private void initMappingConfig(String key, MappingConfig baseConfigMap, MirrorDbConfig mirrorDbConfig, Dml dml) { + MappingConfig mappingConfig = mirrorDbConfig.getTableConfig().get(key); + if (mappingConfig == null) { + // 构造表配置 + mappingConfig = new MappingConfig(); + mappingConfig.setDataSourceKey(baseConfigMap.getDataSourceKey()); + mappingConfig.setDestination(baseConfigMap.getDestination()); + mappingConfig.setGroupId(baseConfigMap.getGroupId()); + mappingConfig.setOuterAdapterKey(baseConfigMap.getOuterAdapterKey()); + mappingConfig.setConcurrent(baseConfigMap.getConcurrent()); + MappingConfig.DbMapping dbMapping = new MappingConfig.DbMapping(); + mappingConfig.setDbMapping(dbMapping); + dbMapping.setDatabase(dml.getDatabase()); + dbMapping.setTable(dml.getTable()); + dbMapping.setTargetDb(dml.getDatabase()); + dbMapping.setTargetTable(dml.getTable()); + dbMapping.setMapAll(true); + List pkNames = dml.getPkNames(); + Map pkMapping = new LinkedHashMap<>(); + pkNames.forEach(pkName -> pkMapping.put(pkName, pkName)); + dbMapping.setTargetPk(pkMapping); + + mirrorDbConfig.getTableConfig().put(key, mappingConfig); + } + } + + /** + * DDL 操作 + * + * @param ddl DDL + */ + private void executeDdl(MirrorDbConfig mirrorDbConfig, Dml ddl) { + try (Connection conn = dataSource.getConnection(); Statement statement = conn.createStatement()) { + statement.execute(ddl.getSql()); + // 移除对应配置 + mirrorDbConfig.getTableConfig().remove(ddl.getTable()); + if (logger.isTraceEnabled()) { + logger.trace("Execute DDL sql: {} for database: {}", ddl.getSql(), ddl.getDatabase()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java new file mode 100644 index 00000000..2c3ecfa9 --- /dev/null +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java @@ -0,0 +1,506 @@ +package com.alibaba.otter.canal.client.adapter.rdb.service; + +import java.sql.Connection; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.function.Function; + +import javax.sql.DataSource; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; +import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig.DbMapping; +import com.alibaba.otter.canal.client.adapter.rdb.support.BatchExecutor; +import com.alibaba.otter.canal.client.adapter.rdb.support.SingleDml; +import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil; +import com.alibaba.otter.canal.client.adapter.support.Dml; +import com.alibaba.otter.canal.client.adapter.support.Util; + +/** + * RDB同步操作业务 + * + * @author rewerma 2018-11-7 下午06:45:49 + * @version 1.0.0 + */ +public class RdbSyncService { + + private static final Logger logger = LoggerFactory.getLogger(RdbSyncService.class); + + // 源库表字段类型缓存: instance.schema.table -> + private Map> columnsTypeCache; + + private int threads = 3; + private boolean skipDupException; + + private List[] dmlsPartition; + private BatchExecutor[] batchExecutors; + private ExecutorService[] executorThreads; + + public List[] getDmlsPartition() { + return dmlsPartition; + } + + public Map> getColumnsTypeCache() { + return columnsTypeCache; + } + + public RdbSyncService(DataSource dataSource, Integer threads, boolean skipDupException){ + this(dataSource, threads, new ConcurrentHashMap<>(), skipDupException); + } + + @SuppressWarnings("unchecked") + public RdbSyncService(DataSource dataSource, Integer threads, Map> columnsTypeCache, + boolean skipDupException){ + this.columnsTypeCache = columnsTypeCache; + this.skipDupException = skipDupException; + try { + if (threads != null) { + this.threads = threads; + } + this.dmlsPartition = new List[this.threads]; + this.batchExecutors = new BatchExecutor[this.threads]; + this.executorThreads = new ExecutorService[this.threads]; + for (int i = 0; i < this.threads; i++) { + dmlsPartition[i] = new ArrayList<>(); + batchExecutors[i] = new BatchExecutor(dataSource); + executorThreads[i] = Executors.newSingleThreadExecutor(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * 批量同步回调 + * + * @param dmls 批量 DML + * @param function 回调方法 + */ + public void sync(List dmls, Function function) { + try { + boolean toExecute = false; + for (Dml dml : dmls) { + if (!toExecute) { + toExecute = function.apply(dml); + } else { + function.apply(dml); + } + } + if (toExecute) { + List> futures = new ArrayList<>(); + for (int i = 0; i < threads; i++) { + int j = i; + futures.add(executorThreads[i].submit(() -> { + try { + dmlsPartition[j] + .forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml)); + dmlsPartition[j].clear(); + batchExecutors[j].commit(); + return true; + } catch (Throwable e) { + batchExecutors[j].rollback(); + throw new RuntimeException(e); + } + })); + } + + futures.forEach(future -> { + try { + future.get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + }); + } + } finally { + for (BatchExecutor batchExecutor : batchExecutors) { + if (batchExecutor != null) { + batchExecutor.close(); + } + } + } + } + + /** + * 批量同步 + * + * @param mappingConfig 配置集合 + * @param dmls 批量 DML + */ + public void sync(Map> mappingConfig, List dmls, Properties envProperties) { + sync(dmls, dml -> { + if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) { + // DDL + columnsTypeCache.remove(dml.getDestination() + "." + dml.getDatabase() + "." + dml.getTable()); + return false; + } else { + // DML + String destination = StringUtils.trimToEmpty(dml.getDestination()); + String groupId = StringUtils.trimToEmpty(dml.getGroupId()); + String database = dml.getDatabase(); + String table = dml.getTable(); + Map configMap; + if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + configMap = mappingConfig.get(destination + "-" + groupId + "_" + database + "-" + table); + } else { + configMap = mappingConfig.get(destination + "_" + database + "-" + table); + } + + if (configMap == null) { + return false; + } + + if (configMap.values().isEmpty()) { + return false; + } + + for (MappingConfig config : configMap.values()) { + if (config.getConcurrent()) { + List singleDmls = SingleDml.dml2SingleDmls(dml); + singleDmls.forEach(singleDml -> { + int hash = pkHash(config.getDbMapping(), singleDml.getData()); + SyncItem syncItem = new SyncItem(config, singleDml); + dmlsPartition[hash].add(syncItem); + }); + } else { + int hash = 0; + List singleDmls = SingleDml.dml2SingleDmls(dml); + singleDmls.forEach(singleDml -> { + SyncItem syncItem = new SyncItem(config, singleDml); + dmlsPartition[hash].add(syncItem); + }); + } + } + return true; + } + }); + } + + /** + * 单条 dml 同步 + * + * @param batchExecutor 批量事务执行器 + * @param config 对应配置对象 + * @param dml DML + */ + public void sync(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) { + if (config != null) { + try { + String type = dml.getType(); + if (type != null && type.equalsIgnoreCase("INSERT")) { + insert(batchExecutor, config, dml); + } else if (type != null && type.equalsIgnoreCase("UPDATE")) { + update(batchExecutor, config, dml); + } else if (type != null && type.equalsIgnoreCase("DELETE")) { + delete(batchExecutor, config, dml); + } else if (type != null && type.equalsIgnoreCase("TRUNCATE")) { + truncate(batchExecutor, config); + } + if (logger.isDebugEnabled()) { + logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue)); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + } + + /** + * 插入操作 + * + * @param config 配置项 + * @param dml DML数据 + */ + private void insert(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException { + Map data = dml.getData(); + if (data == null || data.isEmpty()) { + return; + } + + DbMapping dbMapping = config.getDbMapping(); + + Map columnsMap = SyncUtil.getColumnsMap(dbMapping, data); + + StringBuilder insertSql = new StringBuilder(); + insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" ("); + + columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(",")); + int len = insertSql.length(); + insertSql.delete(len - 1, len).append(") VALUES ("); + int mapLen = columnsMap.size(); + for (int i = 0; i < mapLen; i++) { + insertSql.append("?,"); + } + len = insertSql.length(); + insertSql.delete(len - 1, len).append(")"); + + Map ctype = getTargetColumnType(batchExecutor.getConn(), config); + + List> values = new ArrayList<>(); + for (Map.Entry entry : columnsMap.entrySet()) { + String targetColumnName = entry.getKey(); + String srcColumnName = entry.getValue(); + if (srcColumnName == null) { + srcColumnName = Util.cleanColumn(targetColumnName); + } + + Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase()); + if (type == null) { + throw new RuntimeException("Target column: " + targetColumnName + " not matched"); + } + Object value = data.get(srcColumnName); + BatchExecutor.setValue(values, type, value); + } + + try { + batchExecutor.execute(insertSql.toString(), values); + } catch (SQLException e) { + if (skipDupException + && (e.getMessage().contains("Duplicate entry") || e.getMessage().startsWith("ORA-00001: 违反唯一约束条件"))) { + // ignore + // TODO 增加更多关系数据库的主键冲突的错误码 + } else { + throw e; + } + } + if (logger.isTraceEnabled()) { + logger.trace("Insert into target table, sql: {}", insertSql); + } + + } + + /** + * 更新操作 + * + * @param config 配置项 + * @param dml DML数据 + */ + private void update(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException { + Map data = dml.getData(); + if (data == null || data.isEmpty()) { + return; + } + + Map old = dml.getOld(); + if (old == null || old.isEmpty()) { + return; + } + + DbMapping dbMapping = config.getDbMapping(); + + Map columnsMap = SyncUtil.getColumnsMap(dbMapping, data); + + Map ctype = getTargetColumnType(batchExecutor.getConn(), config); + + StringBuilder updateSql = new StringBuilder(); + updateSql.append("UPDATE ").append(SyncUtil.getDbTableName(dbMapping)).append(" SET "); + List> values = new ArrayList<>(); + boolean hasMatched = false; + for (String srcColumnName : old.keySet()) { + List targetColumnNames = new ArrayList<>(); + columnsMap.forEach((targetColumn, srcColumn) -> { + if (srcColumnName.toLowerCase().equals(srcColumn.toLowerCase())) { + targetColumnNames.add(targetColumn); + } + }); + if (!targetColumnNames.isEmpty()) { + hasMatched = true; + for (String targetColumnName : targetColumnNames) { + updateSql.append(targetColumnName).append("=?, "); + Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase()); + if (type == null) { + throw new RuntimeException("Target column: " + targetColumnName + " not matched"); + } + BatchExecutor.setValue(values, type, data.get(srcColumnName)); + } + } + } + if (!hasMatched) { + logger.warn("Did not matched any columns to update "); + return; + } + int len = updateSql.length(); + updateSql.delete(len - 2, len).append(" WHERE "); + + // 拼接主键 + appendCondition(dbMapping, updateSql, ctype, values, data, old); + batchExecutor.execute(updateSql.toString(), values); + if (logger.isTraceEnabled()) { + logger.trace("Update target table, sql: {}", updateSql); + } + } + + /** + * 删除操作 + * + * @param config + * @param dml + */ + private void delete(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException { + Map data = dml.getData(); + if (data == null || data.isEmpty()) { + return; + } + + DbMapping dbMapping = config.getDbMapping(); + + Map ctype = getTargetColumnType(batchExecutor.getConn(), config); + + StringBuilder sql = new StringBuilder(); + sql.append("DELETE FROM ").append(SyncUtil.getDbTableName(dbMapping)).append(" WHERE "); + + List> values = new ArrayList<>(); + // 拼接主键 + appendCondition(dbMapping, sql, ctype, values, data); + batchExecutor.execute(sql.toString(), values); + if (logger.isTraceEnabled()) { + logger.trace("Delete from target table, sql: {}", sql); + } + } + + /** + * truncate操作 + * + * @param config + */ + private void truncate(BatchExecutor batchExecutor, MappingConfig config) throws SQLException { + DbMapping dbMapping = config.getDbMapping(); + StringBuilder sql = new StringBuilder(); + sql.append("TRUNCATE TABLE ").append(SyncUtil.getDbTableName(dbMapping)); + batchExecutor.execute(sql.toString(), new ArrayList<>()); + if (logger.isTraceEnabled()) { + logger.trace("Truncate target table, sql: {}", sql); + } + } + + /** + * 获取目标字段类型 + * + * @param conn sql connection + * @param config 映射配置 + * @return 字段sqlType + */ + private Map getTargetColumnType(Connection conn, MappingConfig config) { + DbMapping dbMapping = config.getDbMapping(); + String cacheKey = config.getDestination() + "." + dbMapping.getDatabase() + "." + dbMapping.getTable(); + Map columnType = columnsTypeCache.get(cacheKey); + if (columnType == null) { + synchronized (RdbSyncService.class) { + columnType = columnsTypeCache.get(cacheKey); + if (columnType == null) { + columnType = new LinkedHashMap<>(); + final Map columnTypeTmp = columnType; + String sql = "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE 1=2"; + Util.sqlRS(conn, sql, rs -> { + try { + ResultSetMetaData rsd = rs.getMetaData(); + int columnCount = rsd.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + columnTypeTmp.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i)); + } + columnsTypeCache.put(cacheKey, columnTypeTmp); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + }); + } + } + } + return columnType; + } + + /** + * 拼接主键 where条件 + */ + private void appendCondition(MappingConfig.DbMapping dbMapping, StringBuilder sql, Map ctype, + List> values, Map d) { + appendCondition(dbMapping, sql, ctype, values, d, null); + } + + private void appendCondition(MappingConfig.DbMapping dbMapping, StringBuilder sql, Map ctype, + List> values, Map d, Map o) { + // 拼接主键 + for (Map.Entry entry : dbMapping.getTargetPk().entrySet()) { + String targetColumnName = entry.getKey(); + String srcColumnName = entry.getValue(); + if (srcColumnName == null) { + srcColumnName = Util.cleanColumn(targetColumnName); + } + sql.append(targetColumnName).append("=? AND "); + Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase()); + if (type == null) { + throw new RuntimeException("Target column: " + targetColumnName + " not matched"); + } + // 如果有修改主键的情况 + if (o != null && o.containsKey(srcColumnName)) { + BatchExecutor.setValue(values, type, o.get(srcColumnName)); + } else { + BatchExecutor.setValue(values, type, d.get(srcColumnName)); + } + } + int len = sql.length(); + sql.delete(len - 4, len); + } + + public static class SyncItem { + + private MappingConfig config; + private SingleDml singleDml; + + public SyncItem(MappingConfig config, SingleDml singleDml){ + this.config = config; + this.singleDml = singleDml; + } + } + + /** + * 取主键hash + */ + public int pkHash(DbMapping dbMapping, Map d) { + return pkHash(dbMapping, d, null); + } + + public int pkHash(DbMapping dbMapping, Map d, Map o) { + int hash = 0; + // 取主键 + for (Map.Entry entry : dbMapping.getTargetPk().entrySet()) { + String targetColumnName = entry.getKey(); + String srcColumnName = entry.getValue(); + if (srcColumnName == null) { + srcColumnName = Util.cleanColumn(targetColumnName); + } + Object value = null; + if (o != null && o.containsKey(srcColumnName)) { + value = o.get(srcColumnName); + } else if (d != null) { + value = d.get(srcColumnName); + } + if (value != null) { + hash += value.hashCode(); + } + } + hash = Math.abs(hash) % threads; + return Math.abs(hash); + } + + public void close() { + for (int i = 0; i < threads; i++) { + executorThreads[i].shutdown(); + } + } +} diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/BatchExecutor.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/BatchExecutor.java new file mode 100644 index 00000000..e9cf905f --- /dev/null +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/BatchExecutor.java @@ -0,0 +1,95 @@ +package com.alibaba.otter.canal.client.adapter.rdb.support; + +import java.io.Closeable; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.sql.DataSource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * sql批量执行器 + * + * @author rewerma 2018-11-7 下午06:45:49 + * @version 1.0.0 + */ +public class BatchExecutor implements Closeable { + + private static final Logger logger = LoggerFactory.getLogger(BatchExecutor.class); + + private DataSource dataSource; + private Connection conn; + private AtomicInteger idx = new AtomicInteger(0); + + public BatchExecutor(DataSource dataSource){ + this.dataSource = dataSource; + } + + public Connection getConn() { + if (conn == null) { + try { + conn = dataSource.getConnection(); + this.conn.setAutoCommit(false); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + return conn; + } + + public static void setValue(List> values, int type, Object value) { + Map valueItem = new HashMap<>(); + valueItem.put("type", type); + valueItem.put("value", value); + values.add(valueItem); + } + + public void execute(String sql, List> values) throws SQLException { + PreparedStatement pstmt = getConn().prepareStatement(sql); + int len = values.size(); + for (int i = 0; i < len; i++) { + int type = (Integer) values.get(i).get("type"); + Object value = values.get(i).get("value"); + SyncUtil.setPStmt(type, pstmt, value, i + 1); + } + + pstmt.execute(); + idx.incrementAndGet(); + } + + public void commit() throws SQLException { + getConn().commit(); + if (logger.isTraceEnabled()) { + logger.trace("Batch executor commit " + idx.get() + " rows"); + } + idx.set(0); + } + + public void rollback() throws SQLException { + getConn().rollback(); + if (logger.isTraceEnabled()) { + logger.trace("Batch executor rollback " + idx.get() + " rows"); + } + idx.set(0); + } + + @Override + public void close() { + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } finally { + conn = null; + } + } + } +} diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SingleDml.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SingleDml.java new file mode 100644 index 00000000..bad12249 --- /dev/null +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SingleDml.java @@ -0,0 +1,92 @@ +package com.alibaba.otter.canal.client.adapter.rdb.support; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class SingleDml { + + private String destination; + private String database; + private String table; + private String type; + private Map data; + private Map old; + + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Map getData() { + return data; + } + + public void setData(Map data) { + this.data = data; + } + + public Map getOld() { + return old; + } + + public void setOld(Map old) { + this.old = old; + } + + public static List dml2SingleDmls(Dml dml) { + List singleDmls = new ArrayList<>(); + if (dml.getData() != null) { + int size = dml.getData().size(); + for (int i = 0; i < size; i++) { + SingleDml singleDml = new SingleDml(); + singleDml.setDestination(dml.getDestination()); + singleDml.setDatabase(dml.getDatabase()); + singleDml.setTable(dml.getTable()); + singleDml.setType(dml.getType()); + singleDml.setData(dml.getData().get(i)); + if (dml.getOld() != null) { + singleDml.setOld(dml.getOld().get(i)); + } + singleDmls.add(singleDml); + } + } else if ("TRUNCATE".equalsIgnoreCase(dml.getType())) { + SingleDml singleDml = new SingleDml(); + singleDml.setDestination(dml.getDestination()); + singleDml.setDatabase(dml.getDatabase()); + singleDml.setTable(dml.getTable()); + singleDml.setType(dml.getType()); + singleDmls.add(singleDml); + } + return singleDmls; + } +} diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java new file mode 100644 index 00000000..6be006d2 --- /dev/null +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java @@ -0,0 +1,254 @@ +package com.alibaba.otter.canal.client.adapter.rdb.support; + +import java.io.Reader; +import java.io.StringReader; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.sql.*; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.joda.time.DateTime; + +import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; + +public class SyncUtil { + + public static Map getColumnsMap(MappingConfig.DbMapping dbMapping, Map data) { + return getColumnsMap(dbMapping, data.keySet()); + } + + public static Map getColumnsMap(MappingConfig.DbMapping dbMapping, Collection columns) { + Map columnsMap; + if (dbMapping.getMapAll()) { + if (dbMapping.getAllMapColumns() != null) { + return dbMapping.getAllMapColumns(); + } + columnsMap = new LinkedHashMap<>(); + for (String srcColumn : columns) { + boolean flag = true; + if (dbMapping.getTargetColumns() != null) { + for (Map.Entry entry : dbMapping.getTargetColumns().entrySet()) { + if (srcColumn.equals(entry.getValue())) { + columnsMap.put(entry.getKey(), srcColumn); + flag = false; + break; + } + } + } + if (flag) { + columnsMap.put(srcColumn, srcColumn); + } + } + dbMapping.setAllMapColumns(columnsMap); + } else { + columnsMap = dbMapping.getTargetColumns(); + } + return columnsMap; + } + + /** + * 设置 preparedStatement + * + * @param type sqlType + * @param pstmt 需要设置的preparedStatement + * @param value 值 + * @param i 索引号 + */ + public static void setPStmt(int type, PreparedStatement pstmt, Object value, int i) throws SQLException { + switch (type) { + case Types.BIT: + case Types.BOOLEAN: + if (value instanceof Boolean) { + pstmt.setBoolean(i, (Boolean) value); + } else if (value instanceof String) { + boolean v = !value.equals("0"); + pstmt.setBoolean(i, v); + } else if (value instanceof Number) { + boolean v = ((Number) value).intValue() != 0; + pstmt.setBoolean(i, v); + } else { + pstmt.setNull(i, type); + } + break; + case Types.CHAR: + case Types.NCHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + if (value instanceof String) { + pstmt.setString(i, (String) value); + } else if (value == null) { + pstmt.setNull(i, type); + } else { + pstmt.setString(i, value.toString()); + } + break; + case Types.TINYINT: + if (value instanceof Number) { + pstmt.setByte(i, ((Number) value).byteValue()); + } else if (value instanceof String) { + pstmt.setByte(i, Byte.parseByte((String) value)); + } else { + pstmt.setNull(i, type); + } + break; + case Types.SMALLINT: + if (value instanceof Number) { + pstmt.setShort(i, ((Number) value).shortValue()); + } else if (value instanceof String) { + pstmt.setShort(i, Short.parseShort((String) value)); + } else { + pstmt.setNull(i, type); + } + break; + case Types.INTEGER: + if (value instanceof Number) { + pstmt.setInt(i, ((Number) value).intValue()); + } else if (value instanceof String) { + pstmt.setInt(i, Integer.parseInt((String) value)); + } else { + pstmt.setNull(i, type); + } + break; + case Types.BIGINT: + if (value instanceof Number) { + pstmt.setLong(i, ((Number) value).longValue()); + } else if (value instanceof String) { + pstmt.setLong(i, Long.parseLong((String) value)); + } else { + pstmt.setNull(i, type); + } + break; + case Types.DECIMAL: + case Types.NUMERIC: + if (value instanceof BigDecimal) { + pstmt.setBigDecimal(i, (BigDecimal) value); + } else if (value instanceof Byte) { + pstmt.setInt(i, ((Byte) value).intValue()); + } else if (value instanceof Short) { + pstmt.setInt(i, ((Short) value).intValue()); + } else if (value instanceof Integer) { + pstmt.setInt(i, (Integer) value); + } else if (value instanceof Long) { + pstmt.setLong(i, (Long) value); + } else if (value instanceof Float) { + pstmt.setBigDecimal(i, new BigDecimal((float) value)); + } else if (value instanceof Double) { + pstmt.setBigDecimal(i, new BigDecimal((double) value)); + } else if (value != null) { + pstmt.setBigDecimal(i, new BigDecimal(value.toString())); + } else { + pstmt.setNull(i, type); + } + break; + case Types.REAL: + if (value instanceof Number) { + pstmt.setFloat(i, ((Number) value).floatValue()); + } else if (value instanceof String) { + pstmt.setFloat(i, Float.parseFloat((String) value)); + } else { + pstmt.setNull(i, type); + } + break; + case Types.FLOAT: + case Types.DOUBLE: + if (value instanceof Number) { + pstmt.setDouble(i, ((Number) value).doubleValue()); + } else if (value instanceof String) { + pstmt.setDouble(i, Double.parseDouble((String) value)); + } else { + pstmt.setNull(i, type); + } + break; + case Types.BINARY: + case Types.VARBINARY: + case Types.LONGVARBINARY: + case Types.BLOB: + if (value instanceof Blob) { + pstmt.setBlob(i, (Blob) value); + } else if (value instanceof byte[]) { + pstmt.setBytes(i, (byte[]) value); + } else if (value instanceof String) { + pstmt.setBytes(i, ((String) value).getBytes(StandardCharsets.ISO_8859_1)); + } else { + pstmt.setNull(i, type); + } + break; + case Types.CLOB: + if (value instanceof Clob) { + pstmt.setClob(i, (Clob) value); + } else if (value instanceof byte[]) { + pstmt.setBytes(i, (byte[]) value); + } else if (value instanceof String) { + Reader clobReader = new StringReader((String) value); + pstmt.setCharacterStream(i, clobReader); + } else { + pstmt.setNull(i, type); + } + break; + case Types.DATE: + if (value instanceof java.sql.Date) { + pstmt.setDate(i, (java.sql.Date) value); + } else if (value instanceof java.util.Date) { + pstmt.setDate(i, new java.sql.Date(((java.util.Date) value).getTime())); + } else if (value instanceof String) { + String v = (String) value; + if (!v.startsWith("0000-00-00")) { + v = v.trim().replace(" ", "T"); + DateTime dt = new DateTime(v); + pstmt.setDate(i, new Date(dt.toDate().getTime())); + } else { + pstmt.setNull(i, type); + } + } else { + pstmt.setNull(i, type); + } + break; + case Types.TIME: + if (value instanceof java.sql.Time) { + pstmt.setTime(i, (java.sql.Time) value); + } else if (value instanceof java.util.Date) { + pstmt.setTime(i, new java.sql.Time(((java.util.Date) value).getTime())); + } else if (value instanceof String) { + String v = (String) value; + v = "T" + v; + DateTime dt = new DateTime(v); + pstmt.setTime(i, new Time(dt.toDate().getTime())); + } else { + pstmt.setNull(i, type); + } + break; + case Types.TIMESTAMP: + if (value instanceof java.sql.Timestamp) { + pstmt.setTimestamp(i, (java.sql.Timestamp) value); + } else if (value instanceof java.util.Date) { + pstmt.setTimestamp(i, new java.sql.Timestamp(((java.util.Date) value).getTime())); + } else if (value instanceof String) { + String v = (String) value; + if (!v.startsWith("0000-00-00")) { + v = v.trim().replace(" ", "T"); + DateTime dt = new DateTime(v); + pstmt.setTimestamp(i, new Timestamp(dt.toDate().getTime())); + } else { + pstmt.setNull(i, type); + } + } else { + pstmt.setNull(i, type); + } + break; + default: + pstmt.setObject(i, value, type); + } + } + + public static String getDbTableName(MappingConfig.DbMapping dbMapping) { + String result = ""; + if (StringUtils.isNotEmpty(dbMapping.getTargetDb())) { + result += dbMapping.getTargetDb() + "."; + } + result += dbMapping.getTargetTable(); + return result; + } +} diff --git a/client-adapter/rdb/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter b/client-adapter/rdb/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter new file mode 100644 index 00000000..0a840275 --- /dev/null +++ b/client-adapter/rdb/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter @@ -0,0 +1 @@ +rdb=com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter \ No newline at end of file diff --git a/client-adapter/rdb/src/main/resources/rdb/mytest_user.yml b/client-adapter/rdb/src/main/resources/rdb/mytest_user.yml new file mode 100644 index 00000000..74fb2fd0 --- /dev/null +++ b/client-adapter/rdb/src/main/resources/rdb/mytest_user.yml @@ -0,0 +1,29 @@ +dataSourceKey: defaultDS +destination: example +groupId: g1 +outerAdapterKey: mysql1 +concurrent: true +dbMapping: + database: mytest + table: user + targetTable: mytest2.user + targetPk: + id: id +# mapAll: true + targetColumns: + id: + name: + role_id: + c_time: + test1: + + +## Mirror schema synchronize config +#dataSourceKey: defaultDS +#destination: example +#groupId: g1 +#outerAdapterKey: mysql1 +#concurrent: true +#dbMapping: +# mirrorDb: true +# database: mytest diff --git a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/ConfigLoadTest.java b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/ConfigLoadTest.java new file mode 100644 index 00000000..6fbf3fd3 --- /dev/null +++ b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/ConfigLoadTest.java @@ -0,0 +1,27 @@ +package com.alibaba.otter.canal.client.adapter.rdb.test; + +import java.util.Map; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.rdb.config.ConfigLoader; +import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; + +public class ConfigLoadTest { + + @Before + public void before() { + // 加载数据源连接池 + DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource); + } + + @Test + public void testLoad() { + Map configMap = ConfigLoader.load(null); + + Assert.assertFalse(configMap.isEmpty()); + } +} diff --git a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/DBTest.java b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/DBTest.java new file mode 100644 index 00000000..c983d7d8 --- /dev/null +++ b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/DBTest.java @@ -0,0 +1,109 @@ +package com.alibaba.otter.canal.client.adapter.rdb.test; + +import java.io.BufferedReader; +import java.io.Reader; +import java.sql.Clob; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +import org.junit.Test; + +import com.alibaba.druid.pool.DruidDataSource; + +public class DBTest { + + @Test + public void test01() throws SQLException { + DruidDataSource dataSource = new DruidDataSource(); + // dataSource.setDriverClassName("oracle.jdbc.OracleDriver"); + // dataSource.setUrl("jdbc:oracle:thin:@127.0.0.1:49161:XE"); + // dataSource.setUsername("mytest"); + // dataSource.setPassword("m121212"); + + dataSource.setDriverClassName("com.mysql.jdbc.Driver"); + dataSource.setUrl("jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true"); + dataSource.setUsername("root"); + dataSource.setPassword("121212"); + + dataSource.setInitialSize(1); + dataSource.setMinIdle(1); + dataSource.setMaxActive(2); + dataSource.setMaxWait(60000); + dataSource.setTimeBetweenEvictionRunsMillis(60000); + dataSource.setMinEvictableIdleTimeMillis(300000); + + dataSource.init(); + + Connection conn = dataSource.getConnection(); + + conn.setAutoCommit(false); + PreparedStatement pstmt = conn + .prepareStatement("insert into user (id,name,role_id,c_time,test1,test2) values (?,?,?,?,?,?)"); + + java.util.Date now = new java.util.Date(); + for (int i = 1; i <= 10000; i++) { + pstmt.clearParameters(); + pstmt.setLong(1, (long) i); + pstmt.setString(2, "test_" + i); + pstmt.setLong(3, (long) i % 4 + 1); + pstmt.setDate(4, new java.sql.Date(now.getTime())); + pstmt.setString(5, null); + pstmt.setBytes(6, null); + + pstmt.execute(); + if (i % 5000 == 0) { + conn.commit(); + } + } + conn.commit(); + + pstmt.close(); + + // Statement stmt = conn.createStatement(); + // ResultSet rs = stmt.executeQuery("select * from user t where 1=2"); + // + // ResultSetMetaData rsm = rs.getMetaData(); + // int cnt = rsm.getColumnCount(); + // for (int i = 1; i <= cnt; i++) { + // System.out.println(rsm.getColumnName(i) + " " + rsm.getColumnType(i)); + // } + + // rs.close(); + // stmt.close(); + + // PreparedStatement pstmt = conn + // .prepareStatement("insert into tb_user (id,name,role_id,c_time,test1,test2) + // values (?,?,?,?,?,?)"); + // pstmt.setBigDecimal(1, new BigDecimal("5")); + // pstmt.setString(2, "test"); + // pstmt.setBigDecimal(3, new BigDecimal("1")); + // pstmt.setDate(4, new Date(new java.util.Date().getTime())); + // byte[] a = { (byte) 1, (byte) 2 }; + // pstmt.setBytes(5, a); + // pstmt.setBytes(6, a); + // pstmt.execute(); + // + // pstmt.close(); + + conn.close(); + dataSource.close(); + } + + private String clob2Str(Clob clob) { + String content = ""; + try { + Reader is = clob.getCharacterStream(); + BufferedReader buff = new BufferedReader(is); + String line = buff.readLine(); + StringBuffer sb = new StringBuffer(); + while (line != null) { + sb.append(line); + line = buff.readLine(); + } + content = sb.toString(); + } catch (Exception e) { + } + return content; + } +} diff --git a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/SqlParserTest.java b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/SqlParserTest.java new file mode 100644 index 00000000..698f5059 --- /dev/null +++ b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/SqlParserTest.java @@ -0,0 +1,86 @@ +//wpackage com.alibaba.otter.canal.client.adapter.rdb.test; +// +//import com.alibaba.fastsql.sql.ast.SQLName; +//import com.alibaba.fastsql.sql.ast.SQLStatement; +//import com.alibaba.fastsql.sql.ast.statement.SQLCreateTableStatement; +//import com.alibaba.fastsql.sql.ast.statement.SQLExprTableSource; +//import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlCreateTableParser; +//import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser; +//import com.alibaba.fastsql.sql.dialect.mysql.visitor.MySqlOutputVisitor; +//import com.alibaba.fastsql.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor; +//import com.alibaba.fastsql.sql.parser.SQLStatementParser; +// +//import java.io.StringWriter; +// +//public class SqlParserTest { +// +// public static class TableNameVisitor extends MySqlOutputVisitor { +// +// public TableNameVisitor(Appendable appender){ +// super(appender); +// } +// +// @Override +// public boolean visit(SQLExprTableSource x) { +// SQLName table = (SQLName) x.getExpr(); +// String tableName = table.getSimpleName(); +// +// // 改写tableName +// print0("new_" + tableName.toUpperCase()); +// +// return true; +// } +// +// } +// +// public static void main(String[] args) { +// // String sql = "select * from `mytest`.`t` where id=1 and name=ming group by +// // uid limit 1,200 order by ctime"; +// +// String sql = "CREATE TABLE `mytest`.`user` (\n" + " `id` bigint(20) NOT NULL AUTO_INCREMENT,\n" +// + " `name` varchar(30) NOT NULL,\n" + " `c_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,\n" +// + " `role_id` bigint(20) DEFAULT NULL,\n" + " `test1` text,\n" + " `test2` blob,\n" +// + " `key` varchar(30) DEFAULT NULL,\n" + " PRIMARY KEY (`id`)\n" +// + ") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8;"; +// +// // // 新建 MySQL Parser +// // SQLStatementParser parser = new MySqlStatementParser(sql); +// // +// // // 使用Parser解析生成AST，这里SQLStatement就是AST +// // SQLStatement sqlStatement = parser.parseStatement(); +// // +// // MySqlSchemaStatVisitor visitor = new MySqlSchemaStatVisitor(); +// // sqlStatement.accept(visitor); +// // +// // System.out.println("getTables:" + visitor.getTables()); +// // System.out.println("getParameters:" + visitor.getParameters()); +// // System.out.println("getOrderByColumns:" + visitor.getOrderByColumns()); +// // System.out.println("getGroupByColumns:" + visitor.getGroupByColumns()); +// // System.out.println("---------------------------------------------------------------------------"); +// // +// // // 使用select访问者进行select的关键信息打印 +// // // SelectPrintVisitor selectPrintVisitor = new SelectPrintVisitor(); +// // // sqlStatement.accept(selectPrintVisitor); +// // +// // System.out.println("---------------------------------------------------------------------------"); +// // // 最终sql输出 +// // StringWriter out = new StringWriter(); +// // TableNameVisitor outputVisitor = new TableNameVisitor(out); +// // sqlStatement.accept(outputVisitor); +// // System.out.println(out.toString()); +// +// MySqlCreateTableParser parser1 = new MySqlCreateTableParser(sql); +// SQLCreateTableStatement createTableStatement = parser1.parseCreateTable(); +//// MySqlSchemaStatVisitor visitor1 = new MySqlSchemaStatVisitor(); +//// createTableStatement.accept(visitor1); +// // visitor1.getTables().forEach((k, v) -> { +// // System.out.println(k.); +// // System.out.println(v); +// // }); +// // 最终sql输出 +// StringWriter out = new StringWriter(); +// TableNameVisitor outputVisitor = new TableNameVisitor(out); +// createTableStatement.accept(outputVisitor); +// System.out.println(out.toString()); +// } +//} diff --git a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/TestConstant.java b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/TestConstant.java new file mode 100644 index 00000000..813f0e85 --- /dev/null +++ b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/TestConstant.java @@ -0,0 +1,37 @@ +package com.alibaba.otter.canal.client.adapter.rdb.test; + +import java.sql.SQLException; + +import com.alibaba.druid.pool.DruidDataSource; + +public class TestConstant { + + public final static String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true"; + public final static String jdbcUser = "root"; + public final static String jdbcPassword = "121212"; + + public static DruidDataSource dataSource; + + static { + dataSource = new DruidDataSource(); + dataSource.setDriverClassName("com.mysql.jdbc.Driver"); + dataSource.setUrl(jdbcUrl); + dataSource.setUsername(jdbcUser); + dataSource.setPassword(jdbcPassword); + dataSource.setInitialSize(1); + dataSource.setMinIdle(1); + dataSource.setMaxActive(1); + dataSource.setMaxWait(60000); + dataSource.setTimeBetweenEvictionRunsMillis(60000); + dataSource.setMinEvictableIdleTimeMillis(300000); + dataSource.setPoolPreparedStatements(false); + dataSource.setMaxPoolPreparedStatementPerConnectionSize(20); + dataSource.setValidationQuery("select 1"); + try { + dataSource.init(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} diff --git a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java new file mode 100644 index 00000000..de949147 --- /dev/null +++ b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java @@ -0,0 +1,30 @@ +package com.alibaba.otter.canal.client.adapter.rdb.test.sync; + +import java.util.HashMap; +import java.util.Map; + +import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter; +import com.alibaba.otter.canal.client.adapter.rdb.test.TestConstant; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; + +public class Common { + + public static RdbAdapter init() { + DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource); + + OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig(); + outerAdapterConfig.setName("rdb"); + outerAdapterConfig.setKey("oracle1"); + Map properties = new HashMap<>(); + properties.put("jdbc.driveClassName", "oracle.jdbc.OracleDriver"); + properties.put("jdbc.url", "jdbc:oracle:thin:@127.0.0.1:49161:XE"); + properties.put("jdbc.username", "mytest"); + properties.put("jdbc.password", "m121212"); + outerAdapterConfig.setProperties(properties); + + RdbAdapter adapter = new RdbAdapter(); + adapter.init(outerAdapterConfig, null); + return adapter; + } +} diff --git a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/OracleSyncTest.java b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/OracleSyncTest.java new file mode 100644 index 00000000..5ea374e2 --- /dev/null +++ b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/OracleSyncTest.java @@ -0,0 +1,64 @@ +package com.alibaba.otter.canal.client.adapter.rdb.test.sync; + +import java.util.*; + +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class OracleSyncTest { + + private RdbAdapter rdbAdapter; + + @Before + public void init() { + rdbAdapter = Common.init(); + } + + @Test + public void test01() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + data.put("role_id", 1L); + data.put("c_time", new Date()); + data.put("test1", "sdfasdfawe中国asfwef"); + dml.setData(dataList); + + rdbAdapter.sync(Collections.singletonList(dml)); + } + + @Test + public void test02() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric2"); + dml.setData(dataList); + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("name", "Eric"); + dml.setOld(oldList); + + rdbAdapter.sync(Collections.singletonList(dml)); + } + +} diff --git a/client-adapter/rdb/src/test/resources/log4j2-test.xml b/client-adapter/rdb/src/test/resources/log4j2-test.xml new file mode 100644 index 00000000..9c3d2ee6 --- /dev/null +++ b/client-adapter/rdb/src/test/resources/log4j2-test.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/client-adapter/rdb/src/test/resources/logback-test.xml b/client-adapter/rdb/src/test/resources/logback-test.xml new file mode 100644 index 00000000..91f0de80 --- /dev/null +++ b/client-adapter/rdb/src/test/resources/logback-test.xml @@ -0,0 +1,13 @@ + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + + \ No newline at end of file diff --git a/client-adapter/rdb/src/test/resources/rdb/mytest_user.yml b/client-adapter/rdb/src/test/resources/rdb/mytest_user.yml new file mode 100644 index 00000000..9b40dd75 --- /dev/null +++ b/client-adapter/rdb/src/test/resources/rdb/mytest_user.yml @@ -0,0 +1,17 @@ +dataSourceKey: defaultDS +destination: example +outerAdapterKey: oracle1 +concurrent: true +dbMapping: + database: mytest + table: user + targetTable: mytest.tb_user + targetPk: + id: id + mapAll: true +# targetColumns: +# id: +# name: +# role_id: +# c_time: +# test1: \ No newline at end of file diff --git a/client-launcher/pom.xml b/client-launcher/pom.xml deleted file mode 100644 index 623f2a97..00000000 --- a/client-launcher/pom.xml +++ /dev/null @@ -1,149 +0,0 @@ - - - - canal - com.alibaba.otter - 1.1.1-SNAPSHOT - - 4.0.0 - com.alibaba.otter - canal.client-launcher - jar - canal client launcher module for otter ${project.version} - - - com.alibaba.otter - client-adapter.common - ${project.version} - - - com.alibaba.otter - canal.client - ${project.version} - - - - org.apache.kafka - kafka-clients - 1.1.1 - - - org.yaml - snakeyaml - 1.17 - - - - - com.alibaba.otter - client-adapter.logger - ${project.version} - jar-with-dependencies - - - com.alibaba.otter - client-adapter.hbase - ${project.version} - jar-with-dependencies - - - - - - - - maven-jar-plugin - - - true - - - **/logback.xml - **/canal-client.yml - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - 2.2.1 - - - assemble - - single - - package - - - - false - false - - - - - - - - dev - - true - - env - !release - - - - - - - maven-assembly-plugin - - - - ${basedir}/src/main/assembly/dev.xml - - canal_client - ${project.build.directory} - - - - - - - - - release - - - env - release - - - - - - - maven-assembly-plugin - - - - ${basedir}/src/main/assembly/release.xml - - - ${project.artifactId}-${project.version} - - ${project.parent.build.directory} - - - - - - - - diff --git a/client-launcher/src/main/assembly/dev.xml b/client-launcher/src/main/assembly/dev.xml deleted file mode 100644 index a3cbc20c..00000000 --- a/client-launcher/src/main/assembly/dev.xml +++ /dev/null @@ -1,57 +0,0 @@ - - dist - - dir - - false - - - . - / - - README* - - - - ./src/main/bin - bin - - **/* - - 0755 - - - ./src/main/conf - /conf - - **/* - - - - ./src/main/resources - /conf - - **/* - - - META-INF/** - - - - target - logs - - **/* - - - - - - lib - - junit:junit - - - - diff --git a/client-launcher/src/main/assembly/release.xml b/client-launcher/src/main/assembly/release.xml deleted file mode 100644 index 9350da6e..00000000 --- a/client-launcher/src/main/assembly/release.xml +++ /dev/null @@ -1,57 +0,0 @@ - - dist - - tar.gz - - false - - - . - / - - README* - - - - ./src/main/bin - bin - - **/* - - 0755 - - - ./src/main/conf - /conf - - **/* - - - - ./src/main/resources - /conf - - **/* - - - META-INF/** - - - - target - logs - - **/* - - - - - - lib - - junit:junit - - - - diff --git a/client-launcher/src/main/bin/startup.bat b/client-launcher/src/main/bin/startup.bat deleted file mode 100644 index 33fd7945..00000000 --- a/client-launcher/src/main/bin/startup.bat +++ /dev/null @@ -1,25 +0,0 @@ -@echo off -@if not "%ECHO%" == "" echo %ECHO% -@if "%OS%" == "Windows_NT" setlocal - -set ENV_PATH=.\ -if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0% - -set conf_dir=%ENV_PATH%\..\conf -set canal_conf=%conf_dir%\canal-client.yml -set logback_configurationFile=%conf_dir%\logback.xml - -set CLASSPATH=%conf_dir% -set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH% - -set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m -set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 -set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9099,server=y,suspend=n -set CANAL_OPTS= -DappName=otter-canal -Dlogback.configurationFile="%logback_configurationFile%" -Dcanal.conf="%canal_conf%" - -set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS% - -set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.client.ClientLauncher -echo start cmd : %CMD_STR% - -java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.client.ClientLauncher \ No newline at end of file diff --git a/client-launcher/src/main/bin/startup.sh b/client-launcher/src/main/bin/startup.sh deleted file mode 100644 index 0e09ddff..00000000 --- a/client-launcher/src/main/bin/startup.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - -current_path=`pwd` -case "`uname`" in - Linux) - bin_abs_path=$(readlink -f $(dirname $0)) - ;; - *) - bin_abs_path=`cd $(dirname $0); pwd` - ;; -esac -base=${bin_abs_path}/.. -canal_conf=$base/conf/canal-client.yml -logback_configurationFile=$base/conf/logback.xml -export LANG=en_US.UTF-8 -export BASE=$base - -if [ -f $base/bin/canal_client.pid ] ; then - echo "found canal_client.pid , Please run stop.sh first ,then startup.sh" 2>&2 - exit 1 -fi - -## set java path -if [ -z "$JAVA" ] ; then - JAVA=$(which java) -fi - -ALIBABA_JAVA="/usr/alibaba/java/bin/java" -TAOBAO_JAVA="/opt/taobao/java/bin/java" -if [ -z "$JAVA" ]; then - if [ -f $ALIBABA_JAVA ] ; then - JAVA=$ALIBABA_JAVA - elif [ -f $TAOBAO_JAVA ] ; then - JAVA=$TAOBAO_JAVA - else - echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2 - exit 1 - fi -fi - -case "$#" -in -0 ) - ;; -1 ) - var=$* - if [ -f $var ] ; then - canal_conf=$var - else - echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN." - exit - fi;; -2 ) - var=$1 - if [ -f $var ] ; then - canal_conf=$var - else - if [ "$1" = "debug" ]; then - DEBUG_PORT=$2 - DEBUG_SUSPEND="n" - JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND" - fi - fi;; -* ) - echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN." - exit;; -esac - -str=`file -L $JAVA | grep 64-bit` -if [ -n "$str" ]; then - JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -Xss256k -XX:-UseAdaptiveSizePolicy -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError" -else - JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m " -fi - -JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8" -CANAL_OPTS="-DappName=otter-canal-client -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf" - -if [ -e $canal_conf -a -e $logback_configurationFile ] -then - - for i in $base/lib/*; - do CLASSPATH=$i:"$CLASSPATH"; - done - CLASSPATH="$base/conf:$CLASSPATH"; - - echo "cd to $bin_abs_path for workaround relative path" - cd $bin_abs_path - - echo LOG CONFIGURATION : $logback_configurationFile - echo canal conf : $canal_conf - echo CLASSPATH :$CLASSPATH - $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.client.ClientLauncher 1>>$base/logs/canal_client.log 2>&1 & - echo $! > $base/bin/canal_client.pid - - echo "cd to $current_path for continue" - cd $current_path -else - echo "canal client conf("$canal_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!" -fi diff --git a/client-launcher/src/main/java/com/alibaba/otter/canal/client/ClientLauncher.java b/client-launcher/src/main/java/com/alibaba/otter/canal/client/ClientLauncher.java deleted file mode 100644 index c94fcd28..00000000 --- a/client-launcher/src/main/java/com/alibaba/otter/canal/client/ClientLauncher.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.alibaba.otter.canal.client; - -import com.alibaba.otter.canal.client.adapter.loader.CanalAdapterLoader; -import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.Yaml; - -import java.io.FileInputStream; - -public class ClientLauncher { - - private static final String CLASSPATH_URL_PREFIX = "classpath:"; - private static final Logger logger = LoggerFactory.getLogger(ClientLauncher.class); - - public static void main(String[] args) { - try { - logger.info("## set default uncaught exception handler"); - setGlobalUncaughtExceptionHandler(); - - logger.info("## load canal client configurations"); - String conf = System.getProperty("client.conf", "classpath:canal-client.yml"); - CanalClientConfig canalClientConfig; - if (conf.startsWith(CLASSPATH_URL_PREFIX)) { - conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX); - canalClientConfig = new Yaml().loadAs(ClientLauncher.class.getClassLoader().getResourceAsStream(conf), - CanalClientConfig.class); - } else { - canalClientConfig = new Yaml().loadAs(new FileInputStream(conf), CanalClientConfig.class); - } - logger.info("## start the canal client adapters."); - final CanalAdapterLoader adapterLoader = new CanalAdapterLoader(canalClientConfig); - adapterLoader.init(); - logger.info("## the canal client adapters are running now ......"); - Runtime.getRuntime().addShutdownHook(new Thread() { - - public void run() { - try { - logger.info("## stop the canal client adapters"); - adapterLoader.destroy(); - } catch (Throwable e) { - logger.warn("## something goes wrong when stopping canal client adapters:", e); - } finally { - logger.info("## canal client adapters are down."); - } - } - - }); - } catch (Throwable e) { - logger.error("## something goes wrong when starting up the canal client adapters:", e); - System.exit(0); - } - } - - private static void setGlobalUncaughtExceptionHandler() { - Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { - - @Override - public void uncaughtException(Thread t, Throwable e) { - logger.error("UnCaughtException", e); - } - }); - } -} diff --git a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java b/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java deleted file mode 100644 index 46e5922f..00000000 --- a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java +++ /dev/null @@ -1,135 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.loader; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.client.adapter.support.MessageUtil; -import com.alibaba.otter.canal.protocol.FlatMessage; -import com.alibaba.otter.canal.protocol.Message; - -/** - * 适配器工作线程抽象类 - * - * @author machengyuan 2018-8-19 下午11:30:49 - * @version 1.0.0 - */ -public abstract class AbstractCanalAdapterWorker { - - protected final Logger logger = LoggerFactory.getLogger(this.getClass()); - - protected String canalDestination; // canal实例 - protected List> canalOuterAdapters; // 外部适配器 - protected ExecutorService groupInnerExecutorService; // 组内工作线程池 - protected volatile boolean running = false; // 是否运行中 - protected Thread thread = null; - protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { - - @Override - public void uncaughtException(Thread t, Throwable e) { - logger.error("parse events has an error", e); - } - }; - - protected void writeOut(final Message message) { - List> futures = new ArrayList<>(); - // 组间适配器并行运行 - for (List outerAdapters : canalOuterAdapters) { - final List adapters = outerAdapters; - futures.add(groupInnerExecutorService.submit(new Callable() { - - @Override - public Boolean call() { - try { - // 组内适配器穿行运行，尽量不要配置组内适配器 - for (final CanalOuterAdapter c : adapters) { - long begin = System.currentTimeMillis(); - MessageUtil.parse4Dml(message, new MessageUtil.Consumer() { - - @Override - public void accept(Dml dml) { - c.writeOut(dml); - } - }); - - if (logger.isDebugEnabled()) { - logger.debug("{} elapsed time: {}", - c.getClass().getName(), - (System.currentTimeMillis() - begin)); - } - } - return true; - } catch (Exception e) { - return false; - } - } - })); - - // 等待所有适配器写入完成 - // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 - for (Future f : futures) { - try { - if (!f.get()) { - logger.error("Outer adapter write failed"); - } - } catch (InterruptedException | ExecutionException e) { - // ignore - } - } - } - } - - protected void writeOut(final FlatMessage flatMessage) { - List> futures = new ArrayList<>(); - // 组间适配器并行运行 - for (List outerAdapters : canalOuterAdapters) { - final List adapters = outerAdapters; - futures.add(groupInnerExecutorService.submit(new Callable() { - - @Override - public Boolean call() { - try { - // 组内适配器穿行运行，尽量不要配置组内适配器 - for (CanalOuterAdapter c : adapters) { - long begin = System.currentTimeMillis(); - Dml dml = MessageUtil.flatMessage2Dml(flatMessage); - c.writeOut(dml); - if (logger.isDebugEnabled()) { - logger.debug("{} elapsed time: {}", - c.getClass().getName(), - (System.currentTimeMillis() - begin)); - } - } - return true; - } catch (Exception e) { - return false; - } - } - })); - - // 等待所有适配器写入完成 - // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 - for (Future f : futures) { - try { - if (!f.get()) { - logger.error("Outer adapter write failed"); - } - } catch (InterruptedException | ExecutionException e) { - // ignore - } - } - } - } - - public abstract void start(); - - public abstract void stop(); -} diff --git a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java b/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java deleted file mode 100644 index 72320bb5..00000000 --- a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java +++ /dev/null @@ -1,174 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.loader; - -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.kafka.clients.consumer.CommitFailedException; -import org.apache.kafka.common.errors.WakeupException; - -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; -import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector; -import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors; -import com.alibaba.otter.canal.protocol.FlatMessage; -import com.alibaba.otter.canal.protocol.Message; - -/** - * kafka对应的client适配器工作线程 - * - * @author machengyuan 2018-8-19 下午11:30:49 - * @version 1.0.0 - */ -public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { - - private KafkaCanalConnector connector; - - private String topic; - - private boolean flatMessage; - - public CanalAdapterKafkaWorker(String bootstrapServers, String topic, String groupId, - List> canalOuterAdapters, boolean flatMessage){ - this.canalOuterAdapters = canalOuterAdapters; - this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); - this.topic = topic; - this.canalDestination = topic; - this.flatMessage = flatMessage; - connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage); - // connector.setSessionTimeout(1L, TimeUnit.MINUTES); - - // super.initSwitcher(topic); - } - - @Override - public void start() { - if (!running) { - thread = new Thread(new Runnable() { - - @Override - public void run() { - process(); - } - }); - thread.setUncaughtExceptionHandler(handler); - running = true; - thread.start(); - } - } - - @Override - public void stop() { - try { - if (!running) { - return; - } - - connector.stopRunning(); - running = false; - - // if (switcher != null && !switcher.state()) { - // switcher.set(true); - // } - - if (thread != null) { - try { - thread.join(); - } catch (InterruptedException e) { - // ignore - } - } - groupInnerExecutorService.shutdown(); - logger.info("topic {} connectors' worker thread dead!", this.topic); - for (List outerAdapters : canalOuterAdapters) { - for (CanalOuterAdapter adapter : outerAdapters) { - adapter.destroy(); - } - } - logger.info("topic {} all connectors destroyed!", this.topic); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - private void process() { - while (!running) - ; - ExecutorService executor = Executors.newSingleThreadExecutor(); - final AtomicBoolean executing = new AtomicBoolean(true); - while (running) { - try { - logger.info("=============> Start to connect topic: {} <=============", this.topic); - connector.connect(); - logger.info("=============> Start to subscribe topic: {} <=============", this.topic); - connector.subscribe(); - logger.info("=============> Subscribe topic: {} succeed <=============", this.topic); - while (running) { - try { - // switcher.get(); //等待开关开启 - - List messages; - if (!flatMessage) { - messages = connector.getWithoutAck(); - } else { - messages = connector.getFlatMessageWithoutAck(100L, TimeUnit.MILLISECONDS); - } - if (messages != null) { - for (final Object message : messages) { - executing.set(true); - if (message != null) { - executor.submit(new Runnable() { - - @Override - public void run() { - try { - if (message instanceof FlatMessage) { - writeOut((FlatMessage) message); - } else { - writeOut((Message) message); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } finally { - executing.compareAndSet(true, false); - } - } - }); - - // 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端 - long currentTS = System.currentTimeMillis(); - while (executing.get()) { - // 大于10秒未消费完ack一次keep alive - if (System.currentTimeMillis() - currentTS > 10000) { - connector.ack(); - currentTS = System.currentTimeMillis(); - } - } - } - } - } - connector.ack(); - } catch (CommitFailedException e) { - logger.warn(e.getMessage()); - } catch (Exception e) { - logger.error(e.getMessage(), e); - TimeUnit.SECONDS.sleep(1L); - } - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - executor.shutdown(); - - try { - connector.unsubscribe(); - } catch (WakeupException e) { - // No-op. Continue process - } - connector.disconnect(); - logger.info("=============> Disconnect topic: {} <=============", this.topic); - } -} diff --git a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java b/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java deleted file mode 100644 index 15945b6c..00000000 --- a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java +++ /dev/null @@ -1,173 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.loader; - -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; -import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; -import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; -import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; - -/** - * 外部适配器的加载器 - * - * @author machengyuan 2018-8-19 下午11:45:49 - * @version 1.0.0 - */ -public class CanalAdapterLoader { - - private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class); - - private CanalClientConfig canalClientConfig; - - private Map canalWorkers = new HashMap<>(); - - private Map canalKafkaWorkers = new HashMap<>(); - - private ExtensionLoader loader; - - public CanalAdapterLoader(CanalClientConfig canalClientConfig){ - this.canalClientConfig = canalClientConfig; - } - - /** - * 初始化canal-client、 canal-client-kafka的适配器 - */ - public void init() { - // canal instances 和 kafka topics 配置不能同时为空 - if (canalClientConfig.getCanalInstances() == null && canalClientConfig.getKafkaTopics() == null) { - throw new RuntimeException("Blank config property: canalInstances or canalKafkaTopics"); - } - - loader = ExtensionLoader.getExtensionLoader(CanalOuterAdapter.class, - "" /* TODO canalClientConfig.getClassloaderPolicy() */); - - String canalServerHost = this.canalClientConfig.getCanalServerHost(); - SocketAddress sa = null; - if (canalServerHost != null) { - String[] ipPort = canalServerHost.split(":"); - sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1])); - } - String zkHosts = this.canalClientConfig.getZookeeperHosts(); - - boolean flatMessage = this.canalClientConfig.getFlatMessage(); - - // if (zkHosts == null && sa == null) { - // throw new RuntimeException("Blank config property: canalServerHost or - // zookeeperHosts"); - // } - - // 初始化canal-client的适配器 - if (canalClientConfig.getCanalInstances() != null) { - for (CanalClientConfig.CanalInstance instance : canalClientConfig.getCanalInstances()) { - List> canalOuterAdapterGroups = new ArrayList<>(); - - for (CanalClientConfig.AdapterGroup connectorGroup : instance.getAdapterGroups()) { - List canalOutConnectors = new ArrayList<>(); - for (CanalOuterAdapterConfiguration c : connectorGroup.getOutAdapters()) { - loadConnector(c, canalOutConnectors); - } - canalOuterAdapterGroups.add(canalOutConnectors); - } - CanalAdapterWorker worker; - if (zkHosts != null) { - worker = new CanalAdapterWorker(instance.getInstance(), zkHosts, canalOuterAdapterGroups); - } else { - worker = new CanalAdapterWorker(instance.getInstance(), sa, canalOuterAdapterGroups); - } - canalWorkers.put(instance.getInstance(), worker); - worker.start(); - logger.info("Start adapter for canal instance: {} succeed", instance.getInstance()); - } - } - - // 初始化canal-client-kafka的适配器 - if (canalClientConfig.getKafkaTopics() != null) { - for (CanalClientConfig.KafkaTopic kafkaTopic : canalClientConfig.getKafkaTopics()) { - for (CanalClientConfig.Group group : kafkaTopic.getGroups()) { - List> canalOuterAdapterGroups = new ArrayList<>(); - - List canalOuterAdapters = new ArrayList<>(); - - for (CanalOuterAdapterConfiguration config : group.getOutAdapters()) { - // for (CanalOuterAdapterConfiguration config : adaptor.getOutAdapters()) { - loadConnector(config, canalOuterAdapters); - // } - } - canalOuterAdapterGroups.add(canalOuterAdapters); - - // String zkServers = canalClientConfig.getZookeeperHosts(); - CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker( - canalClientConfig.getBootstrapServers(), - kafkaTopic.getTopic(), - group.getGroupId(), - canalOuterAdapterGroups, - flatMessage); - canalKafkaWorkers.put(kafkaTopic.getTopic() + "-" + group.getGroupId(), canalKafkaWorker); - canalKafkaWorker.start(); - logger.info("Start adapter for canal-client kafka topic: {} succeed", - kafkaTopic.getTopic() + "-" + group.getGroupId()); - } - } - } - } - - private void loadConnector(CanalOuterAdapterConfiguration config, List canalOutConnectors) { - try { - CanalOuterAdapter adapter = loader.getExtension(config.getName()); - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - // 替换ClassLoader - Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader()); - adapter.init(config); - Thread.currentThread().setContextClassLoader(cl); - canalOutConnectors.add(adapter); - logger.info("Load canal adapter: {} succeed", config.getName()); - } catch (Exception e) { - logger.error("Load canal adapter: {} failed", config.getName(), e); - } - } - - /** - * 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁 - */ - public void destroy() { - if (canalWorkers.size() > 0) { - ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size()); - for (CanalAdapterWorker v : canalWorkers.values()) { - final CanalAdapterWorker caw = v; - stopExecutorService.submit(new Runnable() { - - @Override - public void run() { - caw.stop(); - } - }); - } - stopExecutorService.shutdown(); - } - if (canalKafkaWorkers.size() > 0) { - ExecutorService stopKafkaExecutorService = Executors.newFixedThreadPool(canalKafkaWorkers.size()); - for (CanalAdapterKafkaWorker v : canalKafkaWorkers.values()) { - final CanalAdapterKafkaWorker cakw = v; - stopKafkaExecutorService.submit(new Runnable() { - - @Override - public void run() { - cakw.stop(); - } - }); - } - stopKafkaExecutorService.shutdown(); - } - logger.info("All canal adapters destroyed"); - } -} diff --git a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java b/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java deleted file mode 100644 index e341c6a3..00000000 --- a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java +++ /dev/null @@ -1,186 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.loader; - -import java.net.SocketAddress; -import java.util.List; -import java.util.concurrent.Executors; - -import com.alibaba.otter.canal.client.CanalConnector; -import com.alibaba.otter.canal.client.CanalConnectors; -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; -import com.alibaba.otter.canal.client.impl.ClusterCanalConnector; -import com.alibaba.otter.canal.protocol.Message; - -/** - * 原生canal-server对应的client适配器工作线程 - * - * @author machengyuan 2018-8-19 下午11:30:49 - * @version 1.0.0 - */ -public class CanalAdapterWorker extends AbstractCanalAdapterWorker { - - private static final int BATCH_SIZE = 50; - private static final int SO_TIMEOUT = 0; - - private CanalConnector connector; - - /** - * 单台client适配器worker的构造方法 - * - * @param canalDestination canal实例名 - * @param address canal-server地址 - * @param canalOuterAdapters 外部适配器组 - */ - public CanalAdapterWorker(String canalDestination, SocketAddress address, - List> canalOuterAdapters){ - this.canalOuterAdapters = canalOuterAdapters; - this.canalDestination = canalDestination; - groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); - connector = CanalConnectors.newSingleConnector(address, canalDestination, "", ""); - } - - /** - * HA模式下client适配器worker的构造方法 - * - * @param canalDestination canal实例名 - * @param zookeeperHosts zookeeper地址 - * @param canalOuterAdapters 外部适配器组 - */ - public CanalAdapterWorker(String canalDestination, String zookeeperHosts, - List> canalOuterAdapters){ - this.canalOuterAdapters = canalOuterAdapters; - this.canalDestination = canalDestination; - groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); - connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", ""); - ((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT); - - // super.initSwitcher(canalDestination); - } - - @Override - public void start() { - if (!running) { - thread = new Thread(new Runnable() { - - @Override - public void run() { - process(); - } - }); - thread.setUncaughtExceptionHandler(handler); - thread.start(); - running = true; - } - } - - @Override - public void stop() { - try { - if (!running) { - return; - } - - // if (switcher != null && !switcher.state()) { - // switcher.set(true); - // } - - connector.stopRunning(); - running = false; - - logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination); - if (thread != null) { - try { - thread.join(); - } catch (InterruptedException e) { - // ignore - } - } - groupInnerExecutorService.shutdown(); - logger.info("destination {} adapters' worker thread dead!", canalDestination); - for (List outerAdapters : canalOuterAdapters) { - for (CanalOuterAdapter adapter : outerAdapters) { - adapter.destroy(); - } - } - logger.info("destination {} all adapters destroyed!", canalDestination); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - private void process() { - while (!running) - ; // waiting until running == true - while (running) { - try { - // if (switcher != null) { - // switcher.get(); - // } - logger.info("=============> Start to connect destination: {} <=============", this.canalDestination); - connector.connect(); - logger.info("=============> Start to subscribe destination: {} <=============", this.canalDestination); - connector.subscribe(); - logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination); - while (running) { - // try { - // if (switcher != null) { - // switcher.get(); - // } - // } catch (TimeoutException e) { - // break; - // } - - // server配置canal.instance.network.soTimeout(默认: 30s) - // 范围内未与server交互，server将关闭本次socket连接 - Message message = connector.getWithoutAck(BATCH_SIZE); // 获取指定数量的数据 - long batchId = message.getId(); - try { - int size = message.getEntries().size(); - - if (batchId == -1 || size == 0) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // ignore - } - } else { - if (logger.isDebugEnabled()) { - logger.debug("destination: {} batchId: {} batchSize: {} ", - this.canalDestination, - batchId, - size); - } - long begin = System.currentTimeMillis(); - writeOut(message); - long now = System.currentTimeMillis(); - if (logger.isDebugEnabled()) { - logger.debug("destination: {} batchId: {} elapsed time: {} ms", - this.canalDestination, - batchId, - now - begin); - } - } - connector.ack(batchId); // 提交确认 - } catch (Exception e) { - connector.rollback(batchId); // 处理失败, 回滚数据 - throw e; - } - } - - } catch (Exception e) { - logger.error("process error!", e); - } finally { - connector.disconnect(); - logger.info("=============> Disconnect destination: {} <=============", this.canalDestination); - } - - if (running) { // is reconnect - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // ignore - } - } - - } - } -} diff --git a/client-launcher/src/main/resources/canal-client.yml b/client-launcher/src/main/resources/canal-client.yml deleted file mode 100644 index 6c8874c3..00000000 --- a/client-launcher/src/main/resources/canal-client.yml +++ /dev/null @@ -1,22 +0,0 @@ -#canalServerHost: 127.0.0.1:11111 -#zookeeperHosts: slave1:2181 -bootstrapServers: slave1:6667,slave2:6667 -flatMessage: true - -#canalInstances: -#- instance: example -# adapterGroups: -# - outAdapters: -# - name: logger -# - name: hbase -# hosts: slave1:2181 -# properties: {znodeParent: "/hbase-unsecure"} -kafkaTopics: -- topic: example - groups: - - groupId: egroup - outAdapters: - - name: logger -# - name: hbase -# hosts: slave1:2181 -# properties: {znodeParent: "/hbase-unsecure"} diff --git a/client-launcher/src/main/resources/logback.xml b/client-launcher/src/main/resources/logback.xml deleted file mode 100644 index 665954f7..00000000 --- a/client-launcher/src/main/resources/logback.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n - - - - - - - destination - canal_client - - - - ../logs/${destination}.log - - - ../logs/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz - - - 512MB - - 60 - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/client/pom.xml b/client/pom.xml index 02b0f66a..293febe2 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.1.1-SNAPSHOT + 1.1.3-SNAPSHOT ../pom.xml com.alibaba.otter @@ -101,8 +101,15 @@ ${spring_version} test - - + + org.apache.rocketmq + rocketmq-client + + + com.aliyun.openservices + aliware-apache-rocketmq-cloud + + org.apache.kafka kafka-clients @@ -127,25 +134,13 @@ false - - org.apache.zookeeper:zookeeper - com.101tec:zkclient - commons-io:commons-io - commons-lang:commons-lang - commons-codec:commons-codec - com.alibaba:fastjson - ch.qos.logback:logback-core - ch.qos.logback:logback-classic - org.slf4j:jcl-over-slf4j - org.slf4j:slf4j-api - org.springframework:* - org.jboss.netty:netty - commons-logging:commons-logging - aopalliance:aopalliance - com.google.protobuf:protobuf-java - io.netty:* - junit:junit - + + com.alibaba.otter:* + com.google.guava:* + org.joda:* + com.google.common:* + com.google.thirdparty:* + @@ -174,8 +169,7 @@ - + diff --git a/client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java index 8e52333d..7f3eeb80 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java @@ -149,11 +149,4 @@ public interface CanalConnector { * @throws CanalClientException */ void rollback() throws CanalClientException; - - /** - * 中断的阻塞，用于优雅停止client - * - * @throws CanalClientException - */ - void stopRunning() throws CanalClientException; } diff --git a/client/src/main/java/com/alibaba/otter/canal/client/CanalMQConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/CanalMQConnector.java new file mode 100644 index 00000000..8ad86022 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/CanalMQConnector.java @@ -0,0 +1,91 @@ +package com.alibaba.otter.canal.client; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +import com.alibaba.otter.canal.protocol.FlatMessage; +import com.alibaba.otter.canal.protocol.Message; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; + +/** + * canal MQ数据操作客户端 + * + * 
    + * 1. canal server写入MQ消息，考虑性能会合并多条数据写入为一个MQ消息，一个Message对应一个MQ消息
+ * 2. canal client消费MQ消息，因为client性能会弱于server的写入，MQ数据获取时会拿到堆积的多条MQ消息，会拿到List
+ * 3. client的ack/rollback，都是和MQ直接交互，不存在对应的batchId概念
+ *
    + * + * @author agapple 2018年10月28日 下午6:42:27 + * @since 1.1.1 + */ +public interface CanalMQConnector extends CanalConnector { + + /** + * 获取数据，自动进行确认，设置timeout时间直到拿到数据为止 + * + * 
    +     * 该方法返回的条件：
+     *  a. 如果timeout=0，有多少取多少，不会阻塞等待
+     *  b. 如果timeout不为0，尝试阻塞对应的超时时间，直到拿到数据就返回
+     *
    + * + * @return + * @throws CanalClientException + */ + List getList(Long timeout, TimeUnit unit) throws CanalClientException; + + /** + * 获取数据，设置timeout时间直到拿到数据为止 + * + * 
    +     * 该方法返回的条件：
+     *  a. 如果timeout=0，有多少取多少，不会阻塞等待
+     *  b. 如果timeout不为0，尝试阻塞对应的超时时间，直到拿到数据就返回
+     *
    + * + * @throws CanalClientException + */ + List getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException; + + /** + * 获取数据，自动进行确认，设置timeout时间直到拿到数据为止 + * + * 
    +     * 该方法返回的条件：
+     *  a. 如果timeout=0，有多少取多少，不会阻塞等待
+     *  b. 如果timeout不为0，尝试阻塞对应的超时时间，直到拿到数据就返回
+     *
    + * + * @return + * @throws CanalClientException + */ + List getFlatList(Long timeout, TimeUnit unit) throws CanalClientException; + + /** + * 获取数据，设置timeout时间直到拿到数据为止 + * + * 
    +     * 该方法返回的条件：
+     *  a. 如果timeout=0，有多少取多少，不会阻塞等待
+     *  b. 如果timeout不为0，尝试阻塞对应的超时时间，直到拿到数据就返回
+     *
    + * + * @throws CanalClientException + */ + List getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException; + + /** + * 消费确认。 + * + * @throws CanalClientException + */ + void ack() throws CanalClientException; + + /** + * 回滚到未进行 {@link #ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link #ack} 的地方开始拿 + * + * @throws CanalClientException + */ + void rollback() throws CanalClientException; +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/CanalMessageDeserializer.java b/client/src/main/java/com/alibaba/otter/canal/client/CanalMessageDeserializer.java new file mode 100644 index 00000000..9d24a23b --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/CanalMessageDeserializer.java @@ -0,0 +1,57 @@ +package com.alibaba.otter.canal.client; + +import com.alibaba.otter.canal.protocol.CanalEntry; +import com.alibaba.otter.canal.protocol.CanalPacket; +import com.alibaba.otter.canal.protocol.CanalPacket.Ack; +import com.alibaba.otter.canal.protocol.CanalPacket.Compression; +import com.alibaba.otter.canal.protocol.Message; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; +import com.google.protobuf.ByteString; + +public class CanalMessageDeserializer { + + public static Message deserializer(byte[] data) { + return deserializer(data, false); + } + + public static Message deserializer(byte[] data, boolean lazyParseEntry) { + try { + if (data == null) { + return null; + } else { + CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data); + switch (p.getType()) { + case MESSAGES: { + if (!p.getCompression().equals(Compression.NONE) + && !p.getCompression().equals(Compression.COMPRESSIONCOMPATIBLEPROTO2)) { + throw new CanalClientException("compression is not supported in this connector"); + } + + CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody()); + Message result = new Message(messages.getBatchId()); + if (lazyParseEntry) { + // byteString + result.setRawEntries(messages.getMessagesList()); + result.setRaw(true); + } else { + for (ByteString byteString : messages.getMessagesList()) { + result.addEntry(CanalEntry.Entry.parseFrom(byteString)); + } + result.setRaw(false); + } + return result; + } + case ACK: { + Ack ack = Ack.parseFrom(p.getBody()); + throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage()); + } + default: { + throw new CanalClientException("unexpected packet type: " + p.getType()); + } + } + } + } catch (Exception e) { + throw new CanalClientException("deserializer failed", e); + } + } +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterCanalConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterCanalConnector.java index 7ec1781a..3fd9c2c5 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterCanalConnector.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterCanalConnector.java @@ -218,8 +218,7 @@ public class ClusterCanalConnector implements CanalConnector { return; } catch (Throwable t) { logger.warn(String.format("something goes wrong when rollbacking data from server:%s", - currentConnector != null ? currentConnector.getAddress() : "null"), - t); + currentConnector != null ? currentConnector.getAddress() : "null"), t); times++; restart(); logger.info("restart the connector for next round retry."); @@ -236,8 +235,7 @@ public class ClusterCanalConnector implements CanalConnector { return; } catch (Throwable t) { logger.warn(String.format("something goes wrong when rollbacking data from server:%s", - currentConnector != null ? currentConnector.getAddress() : "null"), - t); + currentConnector != null ? currentConnector.getAddress() : "null"), t); times++; restart(); logger.info("restart the connector for next round retry."); @@ -255,8 +253,7 @@ public class ClusterCanalConnector implements CanalConnector { return; } catch (Throwable t) { logger.warn(String.format("something goes wrong when acking data from server:%s", - currentConnector != null ? currentConnector.getAddress() : "null"), - t); + currentConnector != null ? currentConnector.getAddress() : "null"), t); times++; restart(); logger.info("restart the connector for next round retry."); diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java index 6461a974..269d99dc 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java @@ -15,7 +15,6 @@ import com.alibaba.otter.canal.common.utils.JsonUtils; import com.alibaba.otter.canal.common.zookeeper.ZkClientx; import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData; -import com.alibaba.otter.canal.protocol.exception.CanalClientException; /** * 集群模式的调度策略 @@ -25,6 +24,7 @@ import com.alibaba.otter.canal.protocol.exception.CanalClientException; */ public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy { + private String destination; private IZkChildListener childListener; // 监听所有的服务器列表 private IZkDataListener dataListener; // 监听当前的工作节点 private ZkClientx zkClient; @@ -32,6 +32,7 @@ public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy { private volatile InetSocketAddress runningAddress = null; public ClusterNodeAccessStrategy(String destination, ZkClientx zkClient){ + this.destination = destination; this.zkClient = zkClient; childListener = new IZkChildListener() { @@ -73,7 +74,7 @@ public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy { } else if (!currentAddress.isEmpty()) { // 如果不存在已经启动的服务，可能服务是一种lazy启动，随机选择一台触发服务器进行启动 return currentAddress.get(0);// 默认返回第一个节点，之前已经做过shuffle } else { - throw new CanalClientException("no alive canal server"); + throw new ServerNotFoundException("no alive canal server for " + destination); } } diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/ServerNotFoundException.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/ServerNotFoundException.java new file mode 100644 index 00000000..22f95046 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/ServerNotFoundException.java @@ -0,0 +1,29 @@ +package com.alibaba.otter.canal.client.impl; + +import com.alibaba.otter.canal.protocol.exception.CanalClientException; + +public class ServerNotFoundException extends CanalClientException { + + private static final long serialVersionUID = -3471518241911601774L; + + public ServerNotFoundException(String errorCode, String errorDesc, Throwable cause){ + super(errorCode, errorDesc, cause); + } + + public ServerNotFoundException(String errorCode, String errorDesc){ + super(errorCode, errorDesc); + } + + public ServerNotFoundException(String errorCode, Throwable cause){ + super(errorCode, cause); + } + + public ServerNotFoundException(String errorCode){ + super(errorCode); + } + + public ServerNotFoundException(Throwable cause){ + super(cause); + } + +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java index 599c678c..4c2211d2 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java @@ -18,13 +18,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.CanalConnector; +import com.alibaba.otter.canal.client.CanalMessageDeserializer; import com.alibaba.otter.canal.client.impl.running.ClientRunningData; import com.alibaba.otter.canal.client.impl.running.ClientRunningListener; import com.alibaba.otter.canal.client.impl.running.ClientRunningMonitor; import com.alibaba.otter.canal.common.utils.AddressUtils; import com.alibaba.otter.canal.common.utils.BooleanMutex; import com.alibaba.otter.canal.common.zookeeper.ZkClientx; -import com.alibaba.otter.canal.protocol.CanalEntry.Entry; import com.alibaba.otter.canal.protocol.CanalPacket.Ack; import com.alibaba.otter.canal.protocol.CanalPacket.ClientAck; import com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth; @@ -32,7 +32,6 @@ import com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback; import com.alibaba.otter.canal.protocol.CanalPacket.Compression; import com.alibaba.otter.canal.protocol.CanalPacket.Get; import com.alibaba.otter.canal.protocol.CanalPacket.Handshake; -import com.alibaba.otter.canal.protocol.CanalPacket.Messages; import com.alibaba.otter.canal.protocol.CanalPacket.Packet; import com.alibaba.otter.canal.protocol.CanalPacket.PacketType; import com.alibaba.otter.canal.protocol.CanalPacket.Sub; @@ -134,7 +133,7 @@ public class SimpleCanalConnector implements CanalConnector { runningMonitor.stop(); } } else { - doDisconnnect(); + doDisconnect(); } } @@ -180,8 +179,8 @@ public class SimpleCanalConnector implements CanalConnector { Ack ackBody = Ack.parseFrom(ack.getBody()); if (ackBody.getErrorCode() > 0) { - throw new CanalClientException( - "something goes wrong when doing authentication: " + ackBody.getErrorMessage()); + throw new CanalClientException("something goes wrong when doing authentication: " + + ackBody.getErrorMessage()); } connected = true; @@ -191,7 +190,7 @@ public class SimpleCanalConnector implements CanalConnector { } } - private void doDisconnnect() throws CanalClientException { + private void doDisconnect() throws CanalClientException { if (readableChannel != null) { quietlyClose(readableChannel); readableChannel = null; @@ -320,34 +319,7 @@ public class SimpleCanalConnector implements CanalConnector { private Message receiveMessages() throws IOException { byte[] data = readNextPacket(); - Packet p = Packet.parseFrom(data); - switch (p.getType()) { - case MESSAGES: { - if (!p.getCompression().equals(Compression.NONE) - && !p.getCompression().equals(Compression.COMPRESSIONCOMPATIBLEPROTO2)) { - throw new CanalClientException("compression is not supported in this connector"); - } - - Messages messages = Messages.parseFrom(p.getBody()); - Message result = new Message(messages.getBatchId()); - if (lazyParseEntry) { - // byteString - result.setRawEntries(messages.getMessagesList()); - } else { - for (ByteString byteString : messages.getMessagesList()) { - result.addEntry(Entry.parseFrom(byteString)); - } - } - return result; - } - case ACK: { - Ack ack = Ack.parseFrom(p.getBody()); - throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage()); - } - default: { - throw new CanalClientException("unexpected packet type: " + p.getType()); - } - } + return CanalMessageDeserializer.deserializer(data, lazyParseEntry); } public void ack(long batchId) throws CanalClientException { @@ -361,8 +333,11 @@ public class SimpleCanalConnector implements CanalConnector { .setBatchId(batchId) .build(); try { - writeWithHeader( - Packet.newBuilder().setType(PacketType.CLIENTACK).setBody(ca.toByteString()).build().toByteArray()); + writeWithHeader(Packet.newBuilder() + .setType(PacketType.CLIENTACK) + .setBody(ca.toByteString()) + .build() + .toByteArray()); } catch (IOException e) { throw new CanalClientException(e); } @@ -459,7 +434,7 @@ public class SimpleCanalConnector implements CanalConnector { public void processActiveExit() { mutex.set(false); - doDisconnnect(); + doDisconnect(); } }); diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java index 32202dc1..9de7f4ee 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java @@ -16,6 +16,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; +import com.alibaba.otter.canal.client.impl.ServerNotFoundException; import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; import com.alibaba.otter.canal.common.utils.BooleanMutex; import com.alibaba.otter.canal.common.utils.JsonUtils; @@ -98,7 +99,7 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId()); zkClient.unsubscribeDataChanges(path, dataListener); releaseRunning(); // 尝试一下release - //Fix issue #697 + // Fix issue #697 if (delayExector != null) { delayExector.shutdown(); } @@ -141,6 +142,15 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { logger.error(MessageFormat.format("There is an error when execute initRunning method, with destination [{0}].", destination), t); + + // fixed issue 1220, 针对server节点不工作避免死循环 + if (t instanceof ServerNotFoundException) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + // 出现任何异常尝试release releaseRunning(); throw new CanalClientException("something goes wrong in initRunning method. ", t); diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java index ad94ba98..199e04ab 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java @@ -1,249 +1,325 @@ -package com.alibaba.otter.canal.client.kafka; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.serialization.StringDeserializer; - -import com.alibaba.fastjson.JSON; -import com.alibaba.otter.canal.protocol.FlatMessage; -import com.alibaba.otter.canal.protocol.Message; - -/** - * canal kafka 数据操作客户端 - * - * @author machengyuan @ 2018-6-12 - * @version 1.0.0 - */ -public class KafkaCanalConnector { - - private KafkaConsumer kafkaConsumer; - private KafkaConsumer kafkaConsumer2; // 用于扁平message的数据消费 - private String topic; - private Integer partition; - private Properties properties; - private volatile boolean connected = false; - private volatile boolean running = false; - private boolean flatMessage; - - public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, - boolean flatMessage){ - this.topic = topic; - this.partition = partition; - this.flatMessage = flatMessage; - - properties = new Properties(); - properties.put("bootstrap.servers", servers); - properties.put("group.id", groupId); - properties.put("enable.auto.commit", false); - properties.put("auto.commit.interval.ms", "1000"); - properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读 - properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置 - properties.put("session.timeout.ms", "30000"); // 默认为30秒 - properties.put("max.poll.records", "100"); - properties.put("key.deserializer", StringDeserializer.class.getName()); - if (!flatMessage) { - properties.put("value.deserializer", MessageDeserializer.class.getName()); - } else { - properties.put("value.deserializer", StringDeserializer.class.getName()); - } - } - - /** - * 重新设置sessionTime - * - * @param timeout - * @param unit - */ - public void setSessionTimeout(Long timeout, TimeUnit unit) { - long t = unit.toMillis(timeout); - properties.put("request.timeout.ms", String.valueOf(t + 60000)); - properties.put("session.timeout.ms", String.valueOf(t)); - } - - /** - * 打开连接 - */ - public void connect() { - if (connected) { - return; - } - - connected = true; - - if (kafkaConsumer == null && !flatMessage) { - kafkaConsumer = new KafkaConsumer(properties); - } - if (kafkaConsumer2 == null && flatMessage) { - kafkaConsumer2 = new KafkaConsumer(properties); - } - } - - /** - * 关闭链接 - */ - public void disconnect() { - if (kafkaConsumer != null) { - kafkaConsumer.close(); - } - if (kafkaConsumer2 != null) { - kafkaConsumer2.close(); - } - - connected = false; - } - - private void waitClientRunning() { - running = true; - } - - public boolean checkValid() { - return true;// 默认都放过 - } - - /** - * 订阅topic - */ - public void subscribe() { - waitClientRunning(); - if (!running) { - return; - } - - if (partition == null) { - if (kafkaConsumer != null) { - kafkaConsumer.subscribe(Collections.singletonList(topic)); - } - if (kafkaConsumer2 != null) { - kafkaConsumer2.subscribe(Collections.singletonList(topic)); - } - } else { - TopicPartition topicPartition = new TopicPartition(topic, partition); - if (kafkaConsumer != null) { - kafkaConsumer.assign(Collections.singletonList(topicPartition)); - } - if (kafkaConsumer2 != null) { - kafkaConsumer2.assign(Collections.singletonList(topicPartition)); - } - } - } - - /** - * 取消订阅 - */ - public void unsubscribe() { - waitClientRunning(); - if (!running) { - return; - } - - if (kafkaConsumer != null) { - kafkaConsumer.unsubscribe(); - } - if (kafkaConsumer2 != null) { - kafkaConsumer2.unsubscribe(); - } - } - - /** - * 获取数据，自动进行确认 - * - * @return - */ - public List get() { - return get(100L, TimeUnit.MILLISECONDS); - } - - public List get(Long timeout, TimeUnit unit) { - waitClientRunning(); - if (!running) { - return null; - } - - List messages = getWithoutAck(timeout, unit); - this.ack(); - return messages; - } - - public List getWithoutAck() { - return getWithoutAck(100L, TimeUnit.MILLISECONDS); - } - - /** - * 获取数据，不进行确认，等待处理完成手工确认 - * - * @return - */ - public List getWithoutAck(Long timeout, TimeUnit unit) { - waitClientRunning(); - if (!running) { - return null; - } - - ConsumerRecords records = kafkaConsumer.poll(unit.toMillis(timeout)); // 基于配置，最多只能poll到一条数据 - - if (!records.isEmpty()) { - // return records.iterator().next().value(); - List messages = new ArrayList<>(); - for (ConsumerRecord record : records) { - messages.add(record.value()); - } - return messages; - } - return null; - } - - public List getFlatMessageWithoutAck(Long timeout, TimeUnit unit) { - waitClientRunning(); - if (!running) { - return null; - } - - ConsumerRecords records = kafkaConsumer2.poll(unit.toMillis(timeout)); - - if (!records.isEmpty()) { - List flatMessages = new ArrayList<>(); - for (ConsumerRecord record : records) { - String flatMessageJson = record.value(); - FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class); - flatMessages.add(flatMessage); - } - - return flatMessages; - } - return null; - } - - /** - * 提交offset，如果超过 session.timeout.ms 设置的时间没有ack则会抛出异常，ack失败 - */ - public void ack() { - waitClientRunning(); - if (!running) { - return; - } - - if (kafkaConsumer != null) { - kafkaConsumer.commitSync(); - } - if (kafkaConsumer2 != null) { - kafkaConsumer2.commitAsync(); - } - } - - public void stopRunning() { - if (running) { - running = false; // 设置为非running状态 - // if (!mutex.state()) { - // mutex.set(true); // 中断阻塞 - // } - } - } -} +package com.alibaba.otter.canal.client.kafka; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.StringDeserializer; + +import com.alibaba.fastjson.JSON; +import com.alibaba.otter.canal.client.CanalMQConnector; +import com.alibaba.otter.canal.client.impl.SimpleCanalConnector; +import com.alibaba.otter.canal.protocol.FlatMessage; +import com.alibaba.otter.canal.protocol.Message; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; +import com.google.common.collect.Lists; + +/** + * canal kafka 数据操作客户端 + * + * 
    + * 注意点：
+ * 1. 相比于canal {@linkplain SimpleCanalConnector}, 这里get和ack操作不能有并发, 必须是一个线程执行get后，内存里执行完毕ack后再取下一个get
+ *
    + * + * @author machengyuan @ 2018-6-12 + * @version 1.1.1 + */ +public class KafkaCanalConnector implements CanalMQConnector { + + protected KafkaConsumer kafkaConsumer; + protected KafkaConsumer kafkaConsumer2; // 用于扁平message的数据消费 + protected String topic; + protected Integer partition; + protected Properties properties; + protected volatile boolean connected = false; + protected volatile boolean running = false; + protected boolean flatMessage; + + private Map currentOffsets = new ConcurrentHashMap<>(); + + public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize, + boolean flatMessage){ + this.topic = topic; + this.partition = partition; + this.flatMessage = flatMessage; + + properties = new Properties(); + properties.put("bootstrap.servers", servers); + properties.put("group.id", groupId); + properties.put("enable.auto.commit", false); + properties.put("auto.commit.interval.ms", "1000"); + properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读 + properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置 + properties.put("session.timeout.ms", "30000"); // 默认为30秒 + if (batchSize == null) { + batchSize = 100; + } + properties.put("max.poll.records", batchSize.toString()); + properties.put("key.deserializer", StringDeserializer.class.getName()); + if (!flatMessage) { + properties.put("value.deserializer", MessageDeserializer.class.getName()); + } else { + properties.put("value.deserializer", StringDeserializer.class.getName()); + } + } + + /** + * 打开连接 + */ + @Override + public void connect() { + if (connected) { + return; + } + + connected = true; + if (kafkaConsumer == null && !flatMessage) { + kafkaConsumer = new KafkaConsumer(properties); + + } + if (kafkaConsumer2 == null && flatMessage) { + kafkaConsumer2 = new KafkaConsumer(properties); + } + } + + /** + * 关闭链接 + */ + @Override + public void disconnect() { + if (kafkaConsumer != null) { + kafkaConsumer.close(); + kafkaConsumer = null; + } + if (kafkaConsumer2 != null) { + kafkaConsumer2.close(); + kafkaConsumer2 = null; + } + + connected = false; + } + + protected void waitClientRunning() { + running = true; + } + + @Override + public boolean checkValid() { + return true;// 默认都放过 + } + + /** + * 订阅topic + */ + @Override + public void subscribe() { + waitClientRunning(); + if (!running) { + return; + } + + if (partition == null) { + if (kafkaConsumer != null) { + kafkaConsumer.subscribe(Collections.singletonList(topic)); + } + if (kafkaConsumer2 != null) { + kafkaConsumer2.subscribe(Collections.singletonList(topic)); + } + } else { + TopicPartition topicPartition = new TopicPartition(topic, partition); + if (kafkaConsumer != null) { + kafkaConsumer.assign(Collections.singletonList(topicPartition)); + } + if (kafkaConsumer2 != null) { + kafkaConsumer2.assign(Collections.singletonList(topicPartition)); + } + } + } + + /** + * 取消订阅 + */ + @Override + public void unsubscribe() { + waitClientRunning(); + if (!running) { + return; + } + + if (kafkaConsumer != null) { + kafkaConsumer.unsubscribe(); + } + if (kafkaConsumer2 != null) { + kafkaConsumer2.unsubscribe(); + } + } + + @Override + public List getList(Long timeout, TimeUnit unit) throws CanalClientException { + waitClientRunning(); + if (!running) { + return Lists.newArrayList(); + } + + List messages = getListWithoutAck(timeout, unit); + if (messages != null && !messages.isEmpty()) { + this.ack(); + } + return messages; + } + + @Override + public List getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException { + waitClientRunning(); + if (!running) { + return Lists.newArrayList(); + } + + ConsumerRecords records = kafkaConsumer.poll(unit.toMillis(timeout)); + + currentOffsets.clear(); + for (TopicPartition topicPartition : records.partitions()) { + currentOffsets.put(topicPartition.partition(), kafkaConsumer.position(topicPartition)); + } + + if (!records.isEmpty()) { + List messages = new ArrayList<>(); + for (ConsumerRecord record : records) { + messages.add(record.value()); + } + return messages; + } + return Lists.newArrayList(); + } + + @Override + public List getFlatList(Long timeout, TimeUnit unit) throws CanalClientException { + waitClientRunning(); + if (!running) { + return Lists.newArrayList(); + } + + List messages = getFlatListWithoutAck(timeout, unit); + if (messages != null && !messages.isEmpty()) { + this.ack(); + } + return messages; + } + + @Override + public List getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException { + waitClientRunning(); + if (!running) { + return Lists.newArrayList(); + } + + ConsumerRecords records = kafkaConsumer2.poll(unit.toMillis(timeout)); + + currentOffsets.clear(); + for (TopicPartition topicPartition : records.partitions()) { + currentOffsets.put(topicPartition.partition(), kafkaConsumer2.position(topicPartition)); + } + + if (!records.isEmpty()) { + List flatMessages = new ArrayList<>(); + for (ConsumerRecord record : records) { + String flatMessageJson = record.value(); + FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class); + flatMessages.add(flatMessage); + } + + return flatMessages; + } + return Lists.newArrayList(); + } + + @Override + public void rollback() { + waitClientRunning(); + if (!running) { + return; + } + // 回滚所有分区 + if (kafkaConsumer != null) { + for (Map.Entry entry : currentOffsets.entrySet()) { + kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1); + } + } + if (kafkaConsumer2 != null) { + for (Map.Entry entry : currentOffsets.entrySet()) { + kafkaConsumer2.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1); + } + } + } + + /** + * 提交offset，如果超过 session.timeout.ms 设置的时间没有ack则会抛出异常，ack失败 + */ + @Override + public void ack() { + waitClientRunning(); + if (!running) { + return; + } + + if (kafkaConsumer != null) { + kafkaConsumer.commitSync(); + } + if (kafkaConsumer2 != null) { + kafkaConsumer2.commitSync(); + } + } + + @Override + public void subscribe(String filter) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public Message get(int batchSize) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public Message getWithoutAck(int batchSize) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public void ack(long batchId) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public void rollback(long batchId) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + /** + * 重新设置sessionTime + * + * @param timeout + * @param unit + */ + public void setSessionTimeout(Long timeout, TimeUnit unit) { + long t = unit.toMillis(timeout); + properties.put("request.timeout.ms", String.valueOf(t + 60000)); + properties.put("session.timeout.ms", String.valueOf(t)); + } + +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnectors.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnectors.java deleted file mode 100644 index 436fde8f..00000000 --- a/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnectors.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.alibaba.otter.canal.client.kafka; - -/** - * canal kafka connectors创建工具类 - * - * @author machengyuan @ 2018-6-12 - * @version 1.0.0 - */ -public class KafkaCanalConnectors { - - /** - * 创建kafka客户端链接，独立运行不注册zk信息 - * - * @param servers - * @param topic - * @param partition - * @param groupId - * @return - */ - public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition, - String groupId) { - return new KafkaCanalConnector(servers, topic, partition, groupId, false); - } - - /** - * 创建kafka客户端链接，独立运行不注册zk信息 - * - * @param servers - * @param topic - * @param groupId - * @return - */ - public static KafkaCanalConnector newKafkaConnector(String servers, String topic, String groupId) { - return new KafkaCanalConnector(servers, topic, null, groupId, false); - } - - /** - * 创建kafka客户端链接 - * - * @param servers - * @param topic - * @param partition - * @param groupId - * @param flatMessage - * @return - */ - public static KafkaCanalConnector newKafkaConnector( String servers, String topic, - Integer partition, String groupId,boolean flatMessage) { - return new KafkaCanalConnector(servers, topic, partition, groupId, flatMessage); - } -} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaOffsetCanalConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaOffsetCanalConnector.java new file mode 100644 index 00000000..2bfa35f9 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaOffsetCanalConnector.java @@ -0,0 +1,112 @@ +package com.alibaba.otter.canal.client.kafka; + +import com.alibaba.fastjson.JSON; +import com.alibaba.otter.canal.client.kafka.protocol.KafkaFlatMessage; +import com.alibaba.otter.canal.client.kafka.protocol.KafkaMessage; +import com.alibaba.otter.canal.protocol.FlatMessage; +import com.alibaba.otter.canal.protocol.Message; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.common.TopicPartition; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * kafka带消息offset的连接器 + * + * @Author panjianping + * @Email ipanjianping@qq.com + * @Date 2018/12/17 + */ +public class KafkaOffsetCanalConnector extends KafkaCanalConnector { + + public KafkaOffsetCanalConnector(String servers, String topic, Integer partition, String groupId, boolean flatMessage) { + super(servers, topic, partition, groupId, 100, flatMessage); + // 启动时从未消费的消息位置开始 + properties.put("auto.offset.reset", "earliest"); + } + + /** + * 获取Kafka消息，不确认 + * + * @param timeout + * @param unit + * @param offset 消息偏移地址（-1为不偏移） + * @return + * @throws CanalClientException + */ + public List getListWithoutAck(Long timeout, TimeUnit unit, long offset) throws CanalClientException { + waitClientRunning(); + if (!running) { + return Lists.newArrayList(); + } + + if (offset > -1) { + TopicPartition tp = new TopicPartition(topic, partition == null ? 0 : partition); + kafkaConsumer.seek(tp, offset); + } + + ConsumerRecords records = kafkaConsumer.poll(unit.toMillis(timeout)); + + if (!records.isEmpty()) { + List messages = new ArrayList<>(); + for (ConsumerRecord record : records) { + KafkaMessage message = new KafkaMessage(record.value(), record.offset()); + messages.add(message); + } + return messages; + } + return Lists.newArrayList(); + } + + /** + * 获取Kafka消息，不确认 + * + * @param timeout + * @param unit + * @param offset 消息偏移地址（-1为不偏移） + * @return + * @throws CanalClientException + */ + public List getFlatListWithoutAck(Long timeout, TimeUnit unit, long offset) throws CanalClientException { + waitClientRunning(); + if (!running) { + return Lists.newArrayList(); + } + + if (offset > -1) { + TopicPartition tp = new TopicPartition(topic, partition == null ? 0 : partition); + kafkaConsumer2.seek(tp, offset); + } + + ConsumerRecords records = kafkaConsumer2.poll(unit.toMillis(timeout)); + if (!records.isEmpty()) { + List flatMessages = new ArrayList<>(); + for (ConsumerRecord record : records) { + String flatMessageJson = record.value(); + FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class); + KafkaFlatMessage message = new KafkaFlatMessage(flatMessage, record.offset()); + flatMessages.add(message); + } + + return flatMessages; + } + return Lists.newArrayList(); + } + + /** + * 重新设置AutoOffsetReset（默认 earliest ） + * + * @param value + */ + public void setAutoOffsetReset(String value) { + if (StringUtils.isNotBlank(value)) { + properties.put("auto.offset.reset", value); + } + } +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/MessageDeserializer.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/MessageDeserializer.java index b5640cfd..6b78e0e4 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/kafka/MessageDeserializer.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/kafka/MessageDeserializer.java @@ -3,14 +3,9 @@ package com.alibaba.otter.canal.client.kafka; import java.util.Map; import org.apache.kafka.common.serialization.Deserializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import com.alibaba.otter.canal.protocol.CanalEntry; -import com.alibaba.otter.canal.protocol.CanalPacket; +import com.alibaba.otter.canal.client.CanalMessageDeserializer; import com.alibaba.otter.canal.protocol.Message; -import com.alibaba.otter.canal.protocol.exception.CanalClientException; -import com.google.protobuf.ByteString; /** * Kafka Message类的反序列化 @@ -20,42 +15,13 @@ import com.google.protobuf.ByteString; */ public class MessageDeserializer implements Deserializer { - private static Logger logger = LoggerFactory.getLogger(MessageDeserializer.class); - @Override public void configure(Map configs, boolean isKey) { } @Override - public Message deserialize(String topic, byte[] data) { - try { - if (data == null) { - return null; - } - else { - CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data); - switch (p.getType()) { - case MESSAGES: { - if (!p.getCompression().equals(CanalPacket.Compression.NONE) - && !p.getCompression().equals(CanalPacket.Compression.COMPRESSIONCOMPATIBLEPROTO2)) { - throw new CanalClientException("compression is not supported in this connector"); - } - - CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody()); - Message result = new Message(messages.getBatchId()); - for (ByteString byteString : messages.getMessagesList()) { - result.addEntry(CanalEntry.Entry.parseFrom(byteString)); - } - return result; - } - default: - break; - } - } - } catch (Exception e) { - logger.error("Error when deserializing byte[] to message "); - } - return null; + public Message deserialize(String topic1, byte[] data) { + return CanalMessageDeserializer.deserializer(data); } @Override diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaFlatMessage.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaFlatMessage.java new file mode 100644 index 00000000..62898a1d --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaFlatMessage.java @@ -0,0 +1,35 @@ +package com.alibaba.otter.canal.client.kafka.protocol; + +import com.alibaba.otter.canal.protocol.FlatMessage; +import org.springframework.beans.BeanUtils; + +/** + * 消息对象（Kafka） + * + * @Author panjianping + * @Email ipanjianping@qq.com + * @Date 2018/12/17 + */ +public class KafkaFlatMessage extends FlatMessage { + + private static final long serialVersionUID = 5748024400508080710L; + + /** + * Kafka 消息 offset + */ + private long offset; + + public KafkaFlatMessage(FlatMessage message, long offset){ + super(message.getId()); + BeanUtils.copyProperties(message, this); + this.offset = offset; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaMessage.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaMessage.java new file mode 100644 index 00000000..4cfafc69 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaMessage.java @@ -0,0 +1,35 @@ +package com.alibaba.otter.canal.client.kafka.protocol; + +import com.alibaba.otter.canal.protocol.Message; +import org.springframework.beans.BeanUtils; + +/** + * 消息对象（Kafka） + * + * @Author panjianping + * @Email ipanjianping@qq.com + * @Date 2018/12/17 + */ +public class KafkaMessage extends Message { + + private static final long serialVersionUID = -293120358490119447L; + + /** + * Kafka 消息 offset + */ + private long offset; + + public KafkaMessage(Message message, long offset){ + super(message.getId()); + BeanUtils.copyProperties(message, this); + this.offset = offset; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningData.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningData.java deleted file mode 100644 index e528f9ee..00000000 --- a/client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningData.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.alibaba.otter.canal.client.kafka.running; - -/** - * client running状态信息 - * - * @author machengyuan 2018-06-20 下午04:10:12 - * @version 1.0.0 - */ -public class ClientRunningData { - - private String groupId; - private String address; - private boolean active = true; - - public String getGroupId() { - return groupId; - } - - public void setGroupId(String groupId) { - this.groupId = groupId; - } - - public String getAddress() { - return address; - } - - public void setAddress(String address) { - this.address = address; - } - - public boolean isActive() { - return active; - } - - public void setActive(boolean active) { - this.active = active; - } - -} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningListener.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningListener.java deleted file mode 100644 index 016f7f10..00000000 --- a/client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningListener.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.alibaba.otter.canal.client.kafka.running; - -/** - * client running状态信息 - * - * @author machengyuan 2018-06-20 下午04:10:12 - * @version 1.0.0 - */ -public interface ClientRunningListener { - - /** - * 触发现在轮到自己做为active - */ - public void processActiveEnter(); - - /** - * 触发一下当前active模式失败 - */ - public void processActiveExit(); - -} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningMonitor.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningMonitor.java deleted file mode 100644 index 9a84d81e..00000000 --- a/client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningMonitor.java +++ /dev/null @@ -1,281 +0,0 @@ -package com.alibaba.otter.canal.client.kafka.running; - -import java.text.MessageFormat; -import java.util.Random; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import org.I0Itec.zkclient.IZkDataListener; -import org.I0Itec.zkclient.exception.ZkException; -import org.I0Itec.zkclient.exception.ZkInterruptedException; -import org.I0Itec.zkclient.exception.ZkNoNodeException; -import org.I0Itec.zkclient.exception.ZkNodeExistsException; -import org.apache.zookeeper.CreateMode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - -import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; -import com.alibaba.otter.canal.common.utils.AddressUtils; -import com.alibaba.otter.canal.common.utils.BooleanMutex; -import com.alibaba.otter.canal.common.utils.JsonUtils; -import com.alibaba.otter.canal.common.zookeeper.ZkClientx; -import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; -import com.alibaba.otter.canal.protocol.exception.CanalClientException; - - -/** - * kafka client running状态信息 - * - * @author machengyuan 2018-06-20 下午04:10:12 - * @version 1.0.0 - */ -public class ClientRunningMonitor extends AbstractCanalLifeCycle { - - private static final String TOPIC_ROOT_NODE = ZookeeperPathUtils.CANAL_ROOT_NODE - + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "topics"; - - private static final String TOPIC_NODE = TOPIC_ROOT_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR - + "{0}"; - - private static final String TOPIC_CLIENTID_NODE = TOPIC_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR - + "{1}"; - - private static final String TOPIC_CLIENTID_RUNNING_NODE = TOPIC_CLIENTID_NODE - + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR - + ZookeeperPathUtils.RUNNING_NODE; - - private static String getTopicClientRunning(String topic, String groupId) { - return MessageFormat.format(TOPIC_CLIENTID_RUNNING_NODE, topic, groupId); - } - - private static String getClientIdNodePath(String topic, String groupId) { - return MessageFormat.format(TOPIC_CLIENTID_NODE, topic, groupId); - } - - private static final Logger logger = LoggerFactory.getLogger(ClientRunningMonitor.class); - private ZkClientx zkClient; - private String topic; - private ClientRunningData clientData; - private IZkDataListener dataListener; - private BooleanMutex mutex = new BooleanMutex(false); - private volatile boolean release = false; - private volatile ClientRunningData activeData; - private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1); - private ClientRunningListener listener; - private int delayTime = 5; - - private static Integer virtualPort; - - public ClientRunningMonitor(){ - if (virtualPort == null) { - Random rand = new Random(); - virtualPort = rand.nextInt(9000) + 1000; - } - - dataListener = new IZkDataListener() { - - public void handleDataChange(String dataPath, Object data) throws Exception { - MDC.put("kafkaTopic", topic); - ClientRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ClientRunningData.class); - if (!isMine(runningData.getAddress())) { - mutex.set(false); - } - - if (!runningData.isActive() && isMine(runningData.getAddress())) { // 说明出现了主动释放的操作，并且本机之前是active - release = true; - releaseRunning();// 彻底释放mainstem - } - - activeData = (ClientRunningData) runningData; - } - - public void handleDataDeleted(String dataPath) throws Exception { - MDC.put("kafkaTopic", topic); - - mutex.set(false); - // 触发一下退出,可能是人为干预的释放操作或者网络闪断引起的session expired timeout - processActiveExit(); - if (!release && activeData != null && isMine(activeData.getAddress())) { - // 如果上一次active的状态就是本机，则即时触发一下active抢占 - initRunning(); - } else { - // 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作 - delayExector.schedule(new Runnable() { - - public void run() { - initRunning(); - } - }, delayTime, TimeUnit.SECONDS); - } - } - - }; - - } - - public void start() { - super.start(); - - String path = getTopicClientRunning(this.topic, clientData.getGroupId()); - - zkClient.subscribeDataChanges(path, dataListener); - initRunning(); - } - - public void stop() { - super.stop(); - String path = getTopicClientRunning(this.topic, clientData.getGroupId()); - zkClient.unsubscribeDataChanges(path, dataListener); - releaseRunning(); // 尝试一下release - // Fix issue #697 - if (delayExector != null) { - delayExector.shutdown(); - } - } - - // 改动记录： - // 1,在方法上加synchronized关键字,保证同步顺序执行; - // 2,判断Zk上已经存在的activeData是否是本机，是的话把mutex重置为true，否则会导致死锁 - // 3,增加异常处理，保证出现异常时，running节点能被删除,否则会导致死锁 - public synchronized void initRunning() { - if (!isStart()) { - return; - } - - String path = getTopicClientRunning(this.topic, clientData.getGroupId()); - // 序列化 - byte[] bytes = JsonUtils.marshalToByte(clientData); - try { - mutex.set(false); - zkClient.create(path, bytes, CreateMode.EPHEMERAL); - processActiveEnter();// 触发一下事件 - activeData = clientData; - mutex.set(true); - } catch (ZkNodeExistsException e) { - bytes = zkClient.readData(path, true); - if (bytes == null) {// 如果不存在节点，立即尝试一次 - initRunning(); - } else { - activeData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class); - // 如果发现已经存在,判断一下是否自己,避免活锁 - if (activeData.getAddress().contains(":") && isMine(activeData.getAddress())) { - mutex.set(true); - } - } - } catch (ZkNoNodeException e) { - zkClient.createPersistent(getClientIdNodePath(this.topic, clientData.getGroupId()), true); // 尝试创建父节点 - initRunning(); - } catch (Throwable t) { - logger.error(MessageFormat.format("There is an error when execute initRunning method, with destination [{0}].", - topic), - t); - // 出现任何异常尝试release - releaseRunning(); - throw new CanalClientException("something goes wrong in initRunning method. ", t); - } - } - - /** - * 阻塞等待自己成为active，如果自己成为active，立马返回 - * - * @throws InterruptedException - */ - public void waitForActive() throws InterruptedException { - initRunning(); - mutex.get(); - } - - /** - * 检查当前的状态 - */ - public boolean check() { - String path = getTopicClientRunning(this.topic, clientData.getGroupId()); - // ZookeeperPathUtils.getDestinationClientRunning(this.destination, - // clientData.getClientId()); - try { - byte[] bytes = zkClient.readData(path); - ClientRunningData eventData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class); - activeData = eventData;// 更新下为最新值 - // 检查下nid是否为自己 - boolean result = isMine(activeData.getAddress()); - if (!result) { - logger.warn("canal is running in [{}] , but not in [{}]", - activeData.getAddress(), - clientData.getAddress()); - } - return result; - } catch (ZkNoNodeException e) { - logger.warn("canal is not run any in node"); - return false; - } catch (ZkInterruptedException e) { - logger.warn("canal check is interrupt"); - Thread.interrupted();// 清除interrupt标记 - return check(); - } catch (ZkException e) { - logger.warn("canal check is failed"); - return false; - } - } - - public boolean releaseRunning() { - if (check()) { - String path = getTopicClientRunning(this.topic, clientData.getGroupId()); - zkClient.delete(path); - mutex.set(false); - processActiveExit(); - return true; - } - - return false; - } - - // ====================== helper method ====================== - - private boolean isMine(String address) { - return address.equals(clientData.getAddress()); - } - - private void processActiveEnter() { - if (listener != null) { - // 触发回调 - listener.processActiveEnter(); - this.clientData.setAddress(/* address */AddressUtils.getHostIp() + ":" + virtualPort); - - String path = getTopicClientRunning(this.topic, clientData.getGroupId()); - // 序列化 - byte[] bytes = JsonUtils.marshalToByte(clientData); - zkClient.writeData(path, bytes); - } - } - - private void processActiveExit() { - if (listener != null) { - listener.processActiveExit(); - } - } - - public void setListener(ClientRunningListener listener) { - this.listener = listener; - } - - // ===================== setter / getter ======================= - - public void setTopic(String topic) { - this.topic = topic; - } - - public void setClientData(ClientRunningData clientData) { - this.clientData = clientData; - } - - public void setDelayTime(int delayTime) { - this.delayTime = delayTime; - } - - public void setZkClient(ZkClientx zkClient) { - this.zkClient = zkClient; - } - -} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/rocketmq/ConsumerBatchMessage.java b/client/src/main/java/com/alibaba/otter/canal/client/rocketmq/ConsumerBatchMessage.java new file mode 100644 index 00000000..1def9163 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/rocketmq/ConsumerBatchMessage.java @@ -0,0 +1,48 @@ +package com.alibaba.otter.canal.client.rocketmq; + +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class ConsumerBatchMessage { + + private final List data; + private CountDownLatch latch; + private boolean hasFailure = false; + + public ConsumerBatchMessage(List data){ + this.data = data; + latch = new CountDownLatch(1); + } + + public boolean waitFinish(long timeout) throws InterruptedException { + return latch.await(timeout, TimeUnit.MILLISECONDS); + } + + public boolean isSuccess() { + return !hasFailure; + } + + public List getData() { + return data; + } + + /** + * Countdown if the sub message is successful. + */ + public void ack() { + latch.countDown(); + } + + /** + * Countdown and fail-fast if the sub message is failed. + */ + public void fail() { + hasFailure = true; + // fail fast + long count = latch.getCount(); + for (int i = 0; i < count; i++) { + latch.countDown(); + } + } +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnector.java new file mode 100644 index 00000000..a39c1ba5 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnector.java @@ -0,0 +1,297 @@ +package com.alibaba.otter.canal.client.rocketmq; + +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang.StringUtils; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.RPCHook; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.otter.canal.client.CanalMQConnector; +import com.alibaba.otter.canal.client.CanalMessageDeserializer; +import com.alibaba.otter.canal.client.impl.SimpleCanalConnector; +import com.alibaba.otter.canal.protocol.FlatMessage; +import com.alibaba.otter.canal.protocol.Message; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; +import com.aliyun.openservices.apache.api.impl.authority.SessionCredentials; +import com.aliyun.openservices.apache.api.impl.rocketmq.ClientRPCHook; +import com.google.common.collect.Lists; + +/** + * RocketMQ的连接 + * + * 
    + * 注意点:
+ * 1. 相比于canal {@linkplain SimpleCanalConnector}, 这里get和ack操作不能有并发, 必须是一个线程执行get后，内存里执行完毕ack后再取下一个get
+ *
    + * + * @since 1.1.1 + */ +public class RocketMQCanalConnector implements CanalMQConnector { + + private static final Logger logger = LoggerFactory.getLogger(RocketMQCanalConnector.class); + + private String nameServer; + private String topic; + private String groupName; + private volatile boolean connected = false; + private DefaultMQPushConsumer rocketMQConsumer; + private BlockingQueue messageBlockingQueue; + private int batchSize = -1; + private long batchProcessTimeout = 60 * 1000; + private boolean flatMessage; + private volatile ConsumerBatchMessage lastGetBatchMessage = null; + private String accessKey; + private String secretKey; + + public RocketMQCanalConnector(String nameServer, String topic, String groupName, Integer batchSize, + boolean flatMessage){ + this.nameServer = nameServer; + this.topic = topic; + this.groupName = groupName; + this.flatMessage = flatMessage; + this.messageBlockingQueue = new LinkedBlockingQueue<>(1024); + this.batchSize = batchSize; + } + + public RocketMQCanalConnector(String nameServer, String topic, String groupName, String accessKey, + String secretKey, Integer batchSize, boolean flatMessage){ + this(nameServer, topic, groupName, batchSize, flatMessage); + this.accessKey = accessKey; + this.secretKey = secretKey; + } + + public void connect() throws CanalClientException { + + RPCHook rpcHook = null; + if (null != accessKey && accessKey.length() > 0 && null != secretKey && secretKey.length() > 0) { + SessionCredentials sessionCredentials = new SessionCredentials(); + sessionCredentials.setAccessKey(accessKey); + sessionCredentials.setSecretKey(secretKey); + rpcHook = new ClientRPCHook(sessionCredentials); + } + rocketMQConsumer = new DefaultMQPushConsumer(groupName, rpcHook, new AllocateMessageQueueAveragely()); + rocketMQConsumer.setVipChannelEnabled(false); + if (!StringUtils.isBlank(nameServer)) { + rocketMQConsumer.setNamesrvAddr(nameServer); + } + if (batchSize != -1) { + rocketMQConsumer.setConsumeMessageBatchMaxSize(batchSize); + } + } + + public void disconnect() throws CanalClientException { + rocketMQConsumer.shutdown(); + connected = false; + } + + public boolean checkValid() throws CanalClientException { + return connected; + } + + public synchronized void subscribe(String filter) throws CanalClientException { + if (connected) { + return; + } + try { + if (rocketMQConsumer == null) { + this.connect(); + } + rocketMQConsumer.subscribe(this.topic, "*"); + rocketMQConsumer.registerMessageListener(new MessageListenerOrderly() { + + @Override + public ConsumeOrderlyStatus consumeMessage(List messageExts, ConsumeOrderlyContext context) { + context.setAutoCommit(true); + boolean isSuccess = process(messageExts); + if (isSuccess) { + return ConsumeOrderlyStatus.SUCCESS; + } else { + return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT; + } + } + }); + rocketMQConsumer.start(); + } catch (MQClientException ex) { + connected = false; + logger.error("Start RocketMQ consumer error", ex); + } + connected = true; + } + + private boolean process(List messageExts) { + logger.info("Get Message:{}", messageExts); + List messageList = Lists.newArrayList(); + for (MessageExt messageExt : messageExts) { + byte[] data = messageExt.getBody(); + if (data != null) { + try { + if (!flatMessage) { + Message message = CanalMessageDeserializer.deserializer(data); + messageList.add(message); + } else { + FlatMessage flatMessage = JSON.parseObject(data, FlatMessage.class); + messageList.add(flatMessage); + } + } catch (Exception ex) { + logger.error("Add message error", ex); + throw new CanalClientException(ex); + } + } else { + logger.warn("Received message data is null"); + } + } + ConsumerBatchMessage batchMessage; + if (!flatMessage) { + batchMessage = new ConsumerBatchMessage(messageList); + } else { + batchMessage = new ConsumerBatchMessage(messageList); + } + try { + messageBlockingQueue.put(batchMessage); + } catch (InterruptedException e) { + logger.error("Put message to queue error", e); + throw new RuntimeException(e); + } + boolean isCompleted; + try { + isCompleted = batchMessage.waitFinish(batchProcessTimeout); + } catch (InterruptedException e) { + logger.error("Interrupted when waiting messages to be finished.", e); + throw new RuntimeException(e); + } + boolean isSuccess = batchMessage.isSuccess(); + return isCompleted && isSuccess; + } + + public void subscribe() throws CanalClientException { + this.subscribe(null); + } + + public void unsubscribe() throws CanalClientException { + this.rocketMQConsumer.unsubscribe(this.topic); + } + + @Override + public List getList(Long timeout, TimeUnit unit) throws CanalClientException { + List messages = getListWithoutAck(timeout, unit); + if (messages != null && !messages.isEmpty()) { + ack(); + } + return messages; + } + + @Override + public List getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException { + try { + if (this.lastGetBatchMessage != null) { + throw new CanalClientException("mq get/ack not support concurrent & async ack"); + } + + ConsumerBatchMessage batchMessage = messageBlockingQueue.poll(timeout, unit); + if (batchMessage != null) { + this.lastGetBatchMessage = batchMessage; + return batchMessage.getData(); + } + } catch (InterruptedException ex) { + logger.warn("Get message timeout", ex); + throw new CanalClientException("Failed to fetch the data after: " + timeout); + } + return Lists.newArrayList(); + } + + @Override + public List getFlatList(Long timeout, TimeUnit unit) throws CanalClientException { + List messages = getFlatListWithoutAck(timeout, unit); + if (messages != null && !messages.isEmpty()) { + ack(); + } + return messages; + } + + @Override + public List getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException { + try { + if (this.lastGetBatchMessage != null) { + throw new CanalClientException("mq get/ack not support concurrent & async ack"); + } + + ConsumerBatchMessage batchMessage = messageBlockingQueue.poll(timeout, unit); + if (batchMessage != null) { + this.lastGetBatchMessage = batchMessage; + return batchMessage.getData(); + } + } catch (InterruptedException ex) { + logger.warn("Get message timeout", ex); + throw new CanalClientException("Failed to fetch the data after: " + timeout); + } + return Lists.newArrayList(); + } + + @Override + public void ack() throws CanalClientException { + try { + if (this.lastGetBatchMessage != null) { + this.lastGetBatchMessage.ack(); + } + } catch (Throwable e) { + if (this.lastGetBatchMessage != null) { + this.lastGetBatchMessage.fail(); + } + } finally { + this.lastGetBatchMessage = null; + } + } + + @Override + public void rollback() throws CanalClientException { + try { + if (this.lastGetBatchMessage != null) { + this.lastGetBatchMessage.fail(); + } + } finally { + this.lastGetBatchMessage = null; + } + } + + public Message get(int batchSize) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public Message getWithoutAck(int batchSize) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public void ack(long batchId) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public void rollback(long batchId) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + +} diff --git a/client/src/test/java/com/alibaba/otter/canal/client/running/ClientRunningTest.java b/client/src/test/java/com/alibaba/otter/canal/client/running/ClientRunningTest.java index 40755671..cbdfd0ea 100644 --- a/client/src/test/java/com/alibaba/otter/canal/client/running/ClientRunningTest.java +++ b/client/src/test/java/com/alibaba/otter/canal/client/running/ClientRunningTest.java @@ -20,117 +20,117 @@ import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; public class ClientRunningTest extends AbstractZkTest { - private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2); - private short clientId = 1001; + private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2); + private short clientId = 1001; - @Before - public void setUp() { - String path = ZookeeperPathUtils.getDestinationPath(destination); - zkclientx.deleteRecursive(path); + @Before + public void setUp() { + String path = ZookeeperPathUtils.getDestinationPath(destination); + zkclientx.deleteRecursive(path); - zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true); - } + zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true); + } - @After - public void tearDown() { - String path = ZookeeperPathUtils.getDestinationPath(destination); - zkclientx.deleteRecursive(path); - } + @After + public void tearDown() { + String path = ZookeeperPathUtils.getDestinationPath(destination); + zkclientx.deleteRecursive(path); + } - @Test - public void testOneServer() { - final CountDownLatch countLatch = new CountDownLatch(2); - ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088); - runningMonitor.start(); - sleep(2000L); - runningMonitor.stop(); - sleep(2000L); + @Test + public void testOneServer() { + final CountDownLatch countLatch = new CountDownLatch(2); + ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088); + runningMonitor.start(); + sleep(2000L); + runningMonitor.stop(); + sleep(2000L); - if (countLatch.getCount() != 0) { - Assert.fail(); - } - } + if (countLatch.getCount() != 0) { + Assert.fail(); + } + } - @Test - public void testMultiServer() { - final CountDownLatch countLatch = new CountDownLatch(30); - final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088); - final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089); - final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090); - final ExecutorService executor = Executors.newFixedThreadPool(3); - executor.submit(new Runnable() { + @Test + public void testMultiServer() { + final CountDownLatch countLatch = new CountDownLatch(30); + final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088); + final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089); + final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090); + final ExecutorService executor = Executors.newFixedThreadPool(3); + executor.submit(new Runnable() { - public void run() { - for (int i = 0; i < 10; i++) { - if (!runningMonitor1.isStart()) { - runningMonitor1.start(); - } - sleep(2000L + RandomUtils.nextInt(500)); - runningMonitor1.stop(); - sleep(2000L + RandomUtils.nextInt(500)); - } - } + public void run() { + for (int i = 0; i < 10; i++) { + if (!runningMonitor1.isStart()) { + runningMonitor1.start(); + } + sleep(2000L + RandomUtils.nextInt(500)); + runningMonitor1.stop(); + sleep(2000L + RandomUtils.nextInt(500)); + } + } - }); + }); - executor.submit(new Runnable() { + executor.submit(new Runnable() { - public void run() { - for (int i = 0; i < 10; i++) { - if (!runningMonitor2.isStart()) { - runningMonitor2.start(); - } - sleep(2000L + RandomUtils.nextInt(500)); - runningMonitor2.stop(); - sleep(2000L + RandomUtils.nextInt(500)); - } - } + public void run() { + for (int i = 0; i < 10; i++) { + if (!runningMonitor2.isStart()) { + runningMonitor2.start(); + } + sleep(2000L + RandomUtils.nextInt(500)); + runningMonitor2.stop(); + sleep(2000L + RandomUtils.nextInt(500)); + } + } - }); + }); - executor.submit(new Runnable() { + executor.submit(new Runnable() { - public void run() { - for (int i = 0; i < 10; i++) { - if (!runningMonitor3.isStart()) { - runningMonitor3.start(); - } - sleep(2000L + RandomUtils.nextInt(500)); - runningMonitor3.stop(); - sleep(2000L + RandomUtils.nextInt(500)); - } - } + public void run() { + for (int i = 0; i < 10; i++) { + if (!runningMonitor3.isStart()) { + runningMonitor3.start(); + } + sleep(2000L + RandomUtils.nextInt(500)); + runningMonitor3.stop(); + sleep(2000L + RandomUtils.nextInt(500)); + } + } - }); + }); - sleep(30000L); - } + sleep(30000L); + } - private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId, - final int port) { - ClientRunningData clientData = new ClientRunningData(); - clientData.setClientId(clientId); - clientData.setAddress(AddressUtils.getHostIp()); + private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId, + final int port) { + ClientRunningData clientData = new ClientRunningData(); + clientData.setClientId(clientId); + clientData.setAddress(AddressUtils.getHostIp()); - ClientRunningMonitor runningMonitor = new ClientRunningMonitor(); - runningMonitor.setDestination(destination); - runningMonitor.setZkClient(zkclientx); - runningMonitor.setClientData(clientData); - runningMonitor.setListener(new ClientRunningListener() { + ClientRunningMonitor runningMonitor = new ClientRunningMonitor(); + runningMonitor.setDestination(destination); + runningMonitor.setZkClient(zkclientx); + runningMonitor.setClientData(clientData); + runningMonitor.setListener(new ClientRunningListener() { - public InetSocketAddress processActiveEnter() { - System.out.println(String.format("clientId:%s port:%s has start", clientId, port)); - countLatch.countDown(); - return new InetSocketAddress(AddressUtils.getHostIp(), port); - } + public InetSocketAddress processActiveEnter() { + System.out.println(String.format("clientId:%s port:%s has start", clientId, port)); + countLatch.countDown(); + return new InetSocketAddress(AddressUtils.getHostIp(), port); + } - public void processActiveExit() { - countLatch.countDown(); - System.out.println(String.format("clientId:%s port:%s has stop", clientId, port)); - } + public void processActiveExit() { + countLatch.countDown(); + System.out.println(String.format("clientId:%s port:%s has stop", clientId, port)); + } - }); - runningMonitor.setDelayTime(1); - return runningMonitor; - } -} \ No newline at end of file + }); + runningMonitor.setDelayTime(1); + return runningMonitor; + } +} diff --git a/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java b/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java index 93f443cd..bf739a06 100644 --- a/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java +++ b/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java @@ -9,7 +9,6 @@ import org.slf4j.LoggerFactory; import org.springframework.util.Assert; import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector; -import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors; import com.alibaba.otter.canal.protocol.Message; /** @@ -36,7 +35,7 @@ public class CanalKafkaClientExample { }; public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId){ - connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId, false); + connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false); } public static void main(String[] args) { @@ -89,7 +88,6 @@ public class CanalKafkaClientExample { if (!running) { return; } - connector.stopRunning(); running = false; if (thread != null) { try { @@ -101,16 +99,20 @@ public class CanalKafkaClientExample { } private void process() { - while (!running) - ; + while (!running) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + while (running) { try { connector.connect(); connector.subscribe(); while (running) { try { - List messages = connector.getWithoutAck(1L, TimeUnit.SECONDS); // 获取message - + List messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS); // 获取message if (messages == null) { continue; } diff --git a/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaOffsetClientExample.java b/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaOffsetClientExample.java new file mode 100644 index 00000000..ff45e88f --- /dev/null +++ b/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaOffsetClientExample.java @@ -0,0 +1,174 @@ +package com.alibaba.otter.canal.client.running.kafka; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.common.errors.WakeupException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.Assert; + +import com.alibaba.otter.canal.client.kafka.KafkaOffsetCanalConnector; +import com.alibaba.otter.canal.client.kafka.protocol.KafkaMessage; + +/** + * KafkaOffsetCanalConnector 使用示例 + *

    KafkaOffsetCanalConnector 与 KafkaCanalConnector 的另一区别是 auto.offset.reset 默认值不同；

    + *

    KafkaOffsetCanalConnector 默认为 earliest；canal-kafka-client重启后从未被消费的记录开始拉取消息，同时提供了修改 auto.offset.reset 的方法 setAutoOffsetReset

    + * + * @author panjianping @ 2018-12-18 + * @version 1.1.3 + */ +public class CanalKafkaOffsetClientExample { + + protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaOffsetClientExample.class); + + private KafkaOffsetCanalConnector connector; + + private static volatile boolean running = false; + + private Thread thread = null; + + private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { + + public void uncaughtException(Thread t, Throwable e) { + logger.error("parse events has an error", e); + } + }; + + public CanalKafkaOffsetClientExample(String servers, String topic, Integer partition, String groupId) { + connector = new KafkaOffsetCanalConnector(servers, topic, partition, groupId, false); + } + + public static void main(String[] args) { + try { + final CanalKafkaOffsetClientExample kafkaCanalClientExample = new CanalKafkaOffsetClientExample( + AbstractKafkaTest.servers, + AbstractKafkaTest.topic, + AbstractKafkaTest.partition, + AbstractKafkaTest.groupId); + logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId); + kafkaCanalClientExample.start(); + logger.info("## the canal kafka consumer is running now ......"); + Runtime.getRuntime().addShutdownHook(new Thread() { + + public void run() { + try { + logger.info("## stop the kafka consumer"); + kafkaCanalClientExample.stop(); + } catch (Throwable e) { + logger.warn("##something goes wrong when stopping kafka consumer:", e); + } finally { + logger.info("## kafka consumer is down."); + } + } + + }); + while (running) + ; + } catch (Throwable e) { + logger.error("## Something goes wrong when starting up the kafka consumer:", e); + System.exit(0); + } + } + + public void start() { + Assert.notNull(connector, "connector is null"); + thread = new Thread(new Runnable() { + + public void run() { + process(); + } + }); + thread.setUncaughtExceptionHandler(handler); + thread.start(); + running = true; + } + + public void stop() { + if (!running) { + return; + } + running = false; + if (thread != null) { + try { + thread.join(); + } catch (InterruptedException e) { + // ignore + } + } + } + + private void process() { + while (!running) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + + while (running) { + try { + // 修改 AutoOffsetReset 的值，默认（earliest） + //connector.setAutoOffsetReset(null); + connector.connect(); + connector.subscribe(); + // 消息起始偏移地址 + long offset = -1; + // 错误次数 + int errorCount = 0; + while (running) { + try { + // 错误重试次数超过3次后，每30秒递增重试 + if (errorCount > 2) { + Thread.sleep((errorCount - 2) * 1000 * 30); + } + + List messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS, offset); // 获取message + if (messages == null) { + continue; + } + for (KafkaMessage message : messages) { + long batchId = message.getId(); + int size = message.getEntries().size(); + + if (batchId == -1 || size == 0) { + continue; + } + + // 记录第一条消息的offset，用于处理数据异常时重新从此位置获取消息 + if (offset < 0) { + offset = message.getOffset(); + } + + // printSummary(message, batchId, size); + // printEntry(message.getEntries()); + logger.info(message.toString()); + } + + connector.ack(); // 提交确认 + // 还原offset + offset = -1; + errorCount = 0; + } catch (Exception e) { + errorCount++; + logger.error(e.getMessage(), e); + if (errorCount == 3) { + // 重试3次后发送邮件提醒异常 + // mailService.sendMail("同步数据异常，请及时处理", "错误消息"); + } + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + try { + connector.unsubscribe(); + } catch (WakeupException e) { + // No-op. Continue process + } + connector.disconnect(); + } +} diff --git a/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java b/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java index fcdc0bf6..9133f74c 100644 --- a/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java +++ b/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java @@ -11,7 +11,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector; -import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors; import com.alibaba.otter.canal.protocol.Message; /** @@ -30,7 +29,7 @@ public class KafkaClientRunningTest extends AbstractKafkaTest { public void testKafkaConsumer() { final ExecutorService executor = Executors.newFixedThreadPool(1); - final KafkaCanalConnector connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId); + final KafkaCanalConnector connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false); executor.submit(new Runnable() { @@ -40,7 +39,7 @@ public class KafkaClientRunningTest extends AbstractKafkaTest { connector.subscribe(); while (running) { try { - List messages = connector.getWithoutAck(3L, TimeUnit.SECONDS); + List messages = connector.getList(3L, TimeUnit.SECONDS); if (messages != null) { System.out.println(messages); } diff --git a/client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/AbstractRocektMQTest.java b/client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/AbstractRocektMQTest.java new file mode 100644 index 00000000..1b4e35d5 --- /dev/null +++ b/client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/AbstractRocektMQTest.java @@ -0,0 +1,9 @@ +package com.alibaba.otter.canal.client.running.rocketmq; + +public class AbstractRocektMQTest { + + public static String topic = "example"; + public static String groupId = "group"; + public static String nameServers = "localhost:9876"; + +} diff --git a/client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/CanalRocketMQClientExample.java b/client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/CanalRocketMQClientExample.java new file mode 100644 index 00000000..2eaa398c --- /dev/null +++ b/client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/CanalRocketMQClientExample.java @@ -0,0 +1,142 @@ +package com.alibaba.otter.canal.client.running.rocketmq; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.common.errors.WakeupException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.Assert; + +import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector; +import com.alibaba.otter.canal.client.running.kafka.AbstractKafkaTest; +import com.alibaba.otter.canal.protocol.Message; + +/** + * Kafka client example + * + * @author machengyuan @ 2018-6-12 + * @version 1.0.0 + */ +public class CanalRocketMQClientExample extends AbstractRocektMQTest { + + protected final static Logger logger = LoggerFactory.getLogger(CanalRocketMQClientExample.class); + + private RocketMQCanalConnector connector; + + private static volatile boolean running = false; + + private Thread thread = null; + + private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { + + public void uncaughtException(Thread t, Throwable e) { + logger.error("parse events has an error", e); + } + }; + + public CanalRocketMQClientExample(String nameServers, String topic, String groupId){ + connector = new RocketMQCanalConnector(nameServers, topic, groupId, 500, false); + } + + public static void main(String[] args) { + try { + final CanalRocketMQClientExample rocketMQClientExample = new CanalRocketMQClientExample(nameServers, + topic, + groupId); + logger.info("## Start the rocketmq consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId); + rocketMQClientExample.start(); + logger.info("## The canal rocketmq consumer is running now ......"); + Runtime.getRuntime().addShutdownHook(new Thread() { + + public void run() { + try { + logger.info("## Stop the rocketmq consumer"); + rocketMQClientExample.stop(); + } catch (Throwable e) { + logger.warn("## Something goes wrong when stopping rocketmq consumer:", e); + } finally { + logger.info("## Rocketmq consumer is down."); + } + } + + }); + while (running) + ; + } catch (Throwable e) { + logger.error("## Something going wrong when starting up the rocketmq consumer:", e); + System.exit(0); + } + } + + public void start() { + Assert.notNull(connector, "connector is null"); + thread = new Thread(new Runnable() { + + public void run() { + process(); + } + }); + thread.setUncaughtExceptionHandler(handler); + thread.start(); + running = true; + } + + public void stop() { + if (!running) { + return; + } + running = false; + if (thread != null) { + try { + thread.join(); + } catch (InterruptedException e) { + // ignore + } + } + } + + private void process() { + while (!running) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + + while (running) { + try { + connector.connect(); + connector.subscribe(); + while (running) { + List messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS); // 获取message + for (Message message : messages) { + long batchId = message.getId(); + int size = message.getEntries().size(); + if (batchId == -1 || size == 0) { + // try { + // Thread.sleep(1000); + // } catch (InterruptedException e) { + // } + } else { + // printSummary(message, batchId, size); + // printEntry(message.getEntries()); + logger.info(message.toString()); + } + } + + connector.ack(); // 提交确认 + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + try { + connector.unsubscribe(); + } catch (WakeupException e) { + // No-op. Continue process + } +// connector.stopRunning(); + } +} diff --git a/common/pom.xml b/common/pom.xml index 1f2ece8b..9d6777b3 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.1.1-SNAPSHOT + 1.1.3-SNAPSHOT ../pom.xml canal.common @@ -83,6 +83,10 @@ org.springframework spring-orm     + + com.google.code.findbugs + jsr305 + org.springframework spring-test diff --git a/common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java b/common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java index 14d1831a..e380cf3b 100644 --- a/common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java +++ b/common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java @@ -54,15 +54,6 @@ public class AddressUtils { public static InetAddress getHostAddress() { InetAddress localAddress = null; - try { - localAddress = InetAddress.getLocalHost(); - if (isValidHostAddress(localAddress)) { - return localAddress; - } - } catch (Throwable e) { - logger.warn("Failed to retriving local host ip address, try scan network card ip address. cause: " - + e.getMessage()); - } try { Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); if (interfaces != null) { diff --git a/common/src/main/java/com/alibaba/otter/canal/common/utils/BooleanMutex.java b/common/src/main/java/com/alibaba/otter/canal/common/utils/BooleanMutex.java index d729ad01..3a2ee258 100644 --- a/common/src/main/java/com/alibaba/otter/canal/common/utils/BooleanMutex.java +++ b/common/src/main/java/com/alibaba/otter/canal/common/utils/BooleanMutex.java @@ -5,7 +5,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.locks.AbstractQueuedSynchronizer; /** - * 实现一个互斥实现，基于Cocurrent中的{@linkplain AbstractQueuedSynchronizer}实现了自己的sync
    + * 实现一个互斥实现，基于Cocurrent中的AQS实现了自己的sync
    * 应用场景：系统初始化/授权控制，没权限时阻塞等待。有权限时所有线程都可以快速通过 * * 
    diff --git a/common/src/main/java/com/alibaba/otter/canal/common/utils/CommonUtils.java b/common/src/main/java/com/alibaba/otter/canal/common/utils/CommonUtils.java
new file mode 100644
index 00000000..52b9621a
--- /dev/null
+++ b/common/src/main/java/com/alibaba/otter/canal/common/utils/CommonUtils.java
@@ -0,0 +1,53 @@
+package com.alibaba.otter.canal.common.utils;
+
+import java.io.File;
+
+/**
+ * 通用工具类
+ *
+ * @author rewerma 2019-01-25 下午05:20:16
+ * @version 1.0.0
+ */
+public class CommonUtils {
+
+    /**
+     * 获取conf文件夹所在路径
+     *
+     * @return 路径地址
+     */
+    public static String getConfPath() {
+        String classpath = CommonUtils.class.getResource("/").getPath();
+        String confPath = classpath + "../conf/";
+        if (new File(confPath).exists()) {
+            return confPath;
+        } else {
+            return classpath;
+        }
+    }
+
+    /**
+     * 删除文件夹
+     *
+     * @param dirFile 文件夹对象
+     * @return 是否删除成功
+     */
+    public static boolean deleteDir(File dirFile) {
+        if (!dirFile.exists()) {
+            return false;
+        }
+
+        if (dirFile.isFile()) {
+            return dirFile.delete();
+        } else {
+            File[] files = dirFile.listFiles();
+            if (files == null || files.length == 0) {
+                return dirFile.delete();
+            }
+            for (File file : files) {
+                deleteDir(file);
+            }
+        }
+
+        return dirFile.delete();
+    }
+}
diff --git a/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java
index 76255bd1..268d32d2 100644
--- a/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java
+++ b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java
@@ -16,7 +16,7 @@ import com.google.common.collect.MigrateMap;
 
 /**
  * 使用自定义的ZooKeeperx for zk connection
- * 
+ *
  * @author jianghang 2012-7-10 下午02:31:15
  * @version 1.0.0
  */
@@ -34,6 +34,10 @@ public class ZkClientx extends ZkClient {
         return clients.get(servers);
     }
 
+    public static void clearClients() {
+        clients.clear();
+    }
+
     public ZkClientx(String serverstring){
         this(serverstring, Integer.MAX_VALUE);
     }
@@ -60,7 +64,7 @@ public class ZkClientx extends ZkClient {
 
     /**
      * Create a persistent Sequential node.
-     * 
+     *
      * @param path
      * @param createParents if true all parent dirs are created as well and no
      * {@link ZkNodeExistsException} is thrown in case the path already exists
@@ -88,7 +92,7 @@ public class ZkClientx extends ZkClient {
 
     /**
      * Create a persistent Sequential node.
-     * 
+     *
      * @param path
      * @param data
      * @param createParents if true all parent dirs are created as well and no
@@ -119,7 +123,7 @@ public class ZkClientx extends ZkClient {
 
     /**
      * Create a persistent Sequential node.
-     * 
+     *
      * @param path
      * @param data
      * @param createParents if true all parent dirs are created as well and no
diff --git a/dbsync/pom.xml b/dbsync/pom.xml
index ac606af4..4b027523 100644
--- a/dbsync/pom.xml
+++ b/dbsync/pom.xml
@@ -3,7 +3,7 @@
 	
 		com.alibaba.otter
 		canal
-		1.1.1-SNAPSHOT
+		1.1.3-SNAPSHOT
 		../pom.xml
 	
 	com.alibaba.otter
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java
index 41156123..725abe60 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java
@@ -32,7 +32,7 @@ public final class CharsetConversion {
     }
 
     // Character set data used in lookups. The array will be sparse.
-    static final Entry[] entries = new Entry[0xff];
+    static final Entry[] entries = new Entry[2048];
 
     static Entry getEntry(final int id) {
         if (id >= 0 && id < entries.length) {
@@ -95,7 +95,7 @@ public final class CharsetConversion {
         putEntry(34, "cp1250", "cp1250_czech_cs", "Cp1250");
         putEntry(35, "ucs2", "ucs2_general_ci", "UnicodeBig");
         putEntry(36, "cp866", "cp866_general_ci", "Cp866");
-        putEntry(37, "keybcs2", "keybcs2_general_ci", "Cp895");
+        putEntry(37, "keybcs2", "keybcs2_general_ci", "Cp852");
         putEntry(38, "macce", "macce_general_ci", "MacCentralEurope");
         putEntry(39, "macroman", "macroman_general_ci", "MacRoman");
         putEntry(40, "cp852", "cp852_general_ci", "Cp852");
@@ -129,7 +129,7 @@ public final class CharsetConversion {
         putEntry(70, "greek", "greek_bin", "ISO8859_7");
         putEntry(71, "hebrew", "hebrew_bin", "ISO8859_8");
         putEntry(72, "hp8", "hp8_bin", "US-ASCII");
-        putEntry(73, "keybcs2", "keybcs2_bin", "Cp895");
+        putEntry(73, "keybcs2", "keybcs2_bin", "Cp852");
         putEntry(74, "koi8r", "koi8r_bin", "KOI8_R");
         putEntry(75, "koi8u", "koi8u_bin", "KOI8_U");
         putEntry(77, "latin2", "latin2_bin", "ISO8859_2");
@@ -260,8 +260,86 @@ public final class CharsetConversion {
         putEntry(241, "utf8mb4", "utf8mb4_esperanto_ci", "UTF-8");
         putEntry(242, "utf8mb4", "utf8mb4_hungarian_ci", "UTF-8");
         putEntry(243, "utf8mb4", "utf8mb4_sinhala_ci", "UTF-8");
+        putEntry(244, "utf8mb4", "utf8mb4_german2_ci", "UTF-8");
+        putEntry(245, "utf8mb4", "utf8mb4_croatian_ci", "UTF-8");
+        putEntry(246, "utf8mb4", "utf8mb4_unicode_520_ci", "UTF-8");
+        putEntry(247, "utf8mb4", "utf8mb4_vietnamese_ci", "UTF-8");
+        putEntry(248, "gb18030", "gb18030_chinese_ci", "GB18030");
+        putEntry(249, "gb18030", "gb18030_bin", "GB18030");
+        putEntry(250, "gb18030", "gb18030_unicode_520_ci", "GB18030");
 
         putEntry(254, "utf8", "utf8_general_cs", "UTF-8");
+        putEntry(255, "utf8mb4", "utf8mb4_0900_ai_ci", "UTF-8");
+        putEntry(256, "utf8mb4", "utf8mb4_de_pb_0900_ai_ci", "UTF-8");
+        putEntry(257, "utf8mb4", "utf8mb4_is_0900_ai_ci", "UTF-8");
+        putEntry(258, "utf8mb4", "utf8mb4_lv_0900_ai_ci", "UTF-8");
+        putEntry(259, "utf8mb4", "utf8mb4_ro_0900_ai_ci", "UTF-8");
+        putEntry(260, "utf8mb4", "utf8mb4_sl_0900_ai_ci", "UTF-8");
+        putEntry(261, "utf8mb4", "utf8mb4_pl_0900_ai_ci", "UTF-8");
+        putEntry(262, "utf8mb4", "utf8mb4_et_0900_ai_ci", "UTF-8");
+        putEntry(263, "utf8mb4", "utf8mb4_es_0900_ai_ci", "UTF-8");
+        putEntry(264, "utf8mb4", "utf8mb4_sv_0900_ai_ci", "UTF-8");
+        putEntry(265, "utf8mb4", "utf8mb4_tr_0900_ai_ci", "UTF-8");
+        putEntry(266, "utf8mb4", "utf8mb4_cs_0900_ai_ci", "UTF-8");
+        putEntry(267, "utf8mb4", "utf8mb4_da_0900_ai_ci", "UTF-8");
+        putEntry(268, "utf8mb4", "utf8mb4_lt_0900_ai_ci", "UTF-8");
+        putEntry(269, "utf8mb4", "utf8mb4_sk_0900_ai_ci", "UTF-8");
+        putEntry(270, "utf8mb4", "utf8mb4_es_trad_0900_ai_ci", "UTF-8");
+        putEntry(271, "utf8mb4", "utf8mb4_la_0900_ai_ci", "UTF-8");
+
+        putEntry(273, "utf8mb4", "utf8mb4_eo_0900_ai_ci", "UTF-8");
+        putEntry(274, "utf8mb4", "utf8mb4_hu_0900_ai_ci", "UTF-8");
+        putEntry(275, "utf8mb4", "utf8mb4_hr_0900_ai_ci", "UTF-8");
+
+        putEntry(277, "utf8mb4", "utf8mb4_vi_0900_ai_ci", "UTF-8");
+        putEntry(278, "utf8mb4", "utf8mb4_0900_as_cs", "UTF-8");
+        putEntry(279, "utf8mb4", "utf8mb4_de_pb_0900_as_cs", "UTF-8");
+        putEntry(280, "utf8mb4", "utf8mb4_is_0900_as_cs", "UTF-8");
+        putEntry(281, "utf8mb4", "utf8mb4_lv_0900_as_cs", "UTF-8");
+        putEntry(282, "utf8mb4", "utf8mb4_ro_0900_as_cs", "UTF-8");
+        putEntry(283, "utf8mb4", "utf8mb4_sl_0900_as_cs", "UTF-8");
+        putEntry(284, "utf8mb4", "utf8mb4_pl_0900_as_cs", "UTF-8");
+        putEntry(285, "utf8mb4", "utf8mb4_et_0900_as_cs", "UTF-8");
+        putEntry(286, "utf8mb4", "utf8mb4_es_0900_as_cs", "UTF-8");
+        putEntry(287, "utf8mb4", "utf8mb4_sv_0900_as_cs", "UTF-8");
+        putEntry(288, "utf8mb4", "utf8mb4_tr_0900_as_cs", "UTF-8");
+        putEntry(289, "utf8mb4", "utf8mb4_cs_0900_as_cs", "UTF-8");
+        putEntry(290, "utf8mb4", "utf8mb4_da_0900_as_cs", "UTF-8");
+        putEntry(291, "utf8mb4", "utf8mb4_lt_0900_as_cs", "UTF-8");
+        putEntry(292, "utf8mb4", "utf8mb4_sk_0900_as_cs", "UTF-8");
+        putEntry(293, "utf8mb4", "utf8mb4_es_trad_0900_as_cs", "UTF-8");
+        putEntry(294, "utf8mb4", "utf8mb4_la_0900_as_cs", "UTF-8");
+
+        putEntry(296, "utf8mb4", "utf8mb4_eo_0900_as_cs", "UTF-8");
+        putEntry(297, "utf8mb4", "utf8mb4_hu_0900_as_cs", "UTF-8");
+        putEntry(298, "utf8mb4", "utf8mb4_hr_0900_as_cs", "UTF-8");
+
+        putEntry(300, "utf8mb4", "utf8mb4_vi_0900_as_cs", "UTF-8");
+        putEntry(303, "utf8mb4", "utf8mb4_ja_0900_as_cs", "UTF-8");
+        putEntry(304, "utf8mb4", "utf8mb4_ja_0900_as_cs_ks", "UTF-8");
+        putEntry(305, "utf8mb4", "utf8mb4_0900_as_ci", "UTF-8");
+        putEntry(306, "utf8mb4", "utf8mb4_ru_0900_ai_ci", "UTF-8");
+        putEntry(307, "utf8mb4", "utf8mb4_ru_0900_as_cs", "UTF-8");
+
+        putEntry(326, "utf8mb4", "utf8mb4_test_ci", "UTF-8");
+        putEntry(327, "utf16", "utf16_test_ci", "UTF-16");
+        putEntry(328, "utf8mb4", "utf8mb4_test_400_ci", "UTF-8");
+
+        putEntry(336, "utf8", "utf8_bengali_standard_ci", "UTF-8");
+        putEntry(337, "utf8", "utf8_bengali_standard_ci", "UTF-8");
+        putEntry(352, "utf8", "utf8_phone_ci", "UTF-8");
+        putEntry(353, "utf8", "utf8_test_ci", "UTF-8");
+        putEntry(354, "utf8", "utf8_5624_1", "UTF-8");
+        putEntry(355, "utf8", "utf8_5624_2", "UTF-8");
+        putEntry(356, "utf8", "utf8_5624_3", "UTF-8");
+        putEntry(357, "utf8", "utf8_5624_4", "UTF-8");
+        putEntry(358, "ucs2", "ucs2_test_ci", "UnicodeBig");
+        putEntry(359, "ucs2", "ucs2_vn_ci", "UnicodeBig");
+        putEntry(360, "ucs2", "ucs2_5624_1", "UnicodeBig");
+
+        putEntry(368, "utf8", "utf8_5624_5", "UTF-8");
+        putEntry(391, "utf32", "utf32_test_ci", "UTF-32");
+        putEntry(2047, "utf8", "utf8_maxuserid_ci", "UTF-8");
     }
 
     /**
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java
index 020591f7..e61b6645 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java
@@ -1,5 +1,9 @@
 package com.taobao.tddl.dbsync.binlog;
 
+import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber2;
+import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber4;
+import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.usecondsToStr;
+
 /**
  * 处理下MySQL json二进制转化为可读的字符串
  * 
@@ -330,12 +334,30 @@ public class JsonConversion {
                             long ultime = Math.abs(packed_value);
                             long intpart = ultime >> 24;
                             int frac = (int) (ultime % (1L << 24));
-                            text = String.format("%s%02d:%02d:%02d",
-                                packed_value >= 0 ? "" : "-",
-                                (int) ((intpart >> 12) % (1 << 10)),
-                                (int) ((intpart >> 6) % (1 << 6)),
-                                (int) (intpart % (1 << 6)));
-                            text = text + "." + usecondsToStr(frac, 6);
+                            // text = String.format("%s%02d:%02d:%02d",
+                            // packed_value >= 0 ? "" : "-",
+                            // (int) ((intpart >> 12) % (1 << 10)),
+                            // (int) ((intpart >> 6) % (1 << 6)),
+                            // (int) (intpart % (1 << 6)));
+                            // text = text + "." + usecondsToStr(frac, 6);
+                            StringBuilder builder = new StringBuilder(17);
+                            if (packed_value < 0) {
+                                builder.append('-');
+                            }
+
+                            int d = (int) ((intpart >> 12) % (1 << 10));
+                            if (d > 100) {
+                                builder.append(String.valueOf(d));
+                            } else {
+                                appendNumber2(builder, d);
+                            }
+                            builder.append(':');
+                            appendNumber2(builder, (int) ((intpart >> 6) % (1 << 6)));
+                            builder.append(':');
+                            appendNumber2(builder, (int) (intpart % (1 << 6)));
+
+                            builder.append('.').append(usecondsToStr(frac, 6));
+                            text = builder.toString();
                         }
                         buf.append('"').append(text).append('"');
                     } else if (m_field_type == LogEvent.MYSQL_TYPE_DATE || m_field_type == LogEvent.MYSQL_TYPE_DATETIME
@@ -351,14 +373,28 @@ public class JsonConversion {
                             long ymd = intpart >> 17;
                             long ym = ymd >> 5;
                             long hms = intpart % (1 << 17);
-                            text = String.format("%04d-%02d-%02d %02d:%02d:%02d",
-                                (int) (ym / 13),
-                                (int) (ym % 13),
-                                (int) (ymd % (1 << 5)),
-                                (int) (hms >> 12),
-                                (int) ((hms >> 6) % (1 << 6)),
-                                (int) (hms % (1 << 6)));
-                            text = text + "." + usecondsToStr(frac, 6);
+                            // text =
+                            // String.format("%04d-%02d-%02d %02d:%02d:%02d",
+                            // (int) (ym / 13),
+                            // (int) (ym % 13),
+                            // (int) (ymd % (1 << 5)),
+                            // (int) (hms >> 12),
+                            // (int) ((hms >> 6) % (1 << 6)),
+                            // (int) (hms % (1 << 6)));
+                            StringBuilder builder = new StringBuilder(26);
+                            appendNumber4(builder, (int) (ym / 13));
+                            builder.append('-');
+                            appendNumber2(builder, (int) (ym % 13));
+                            builder.append('-');
+                            appendNumber2(builder, (int) (ymd % (1 << 5)));
+                            builder.append(' ');
+                            appendNumber2(builder, (int) (hms >> 12));
+                            builder.append(':');
+                            appendNumber2(builder, (int) ((hms >> 6) % (1 << 6)));
+                            builder.append(':');
+                            appendNumber2(builder, (int) (hms % (1 << 6)));
+                            builder.append('.').append(usecondsToStr(frac, 6));
+                            text = builder.toString();
                         }
                         buf.append('"').append(text).append('"');
                     } else {
@@ -397,22 +433,4 @@ public class JsonConversion {
         OBJECT, ARRAY, STRING, INT, UINT, DOUBLE, LITERAL_NULL, LITERAL_TRUE, LITERAL_FALSE, OPAQUE, ERROR
     }
 
-    private static String usecondsToStr(int frac, int meta) {
-        String sec = String.valueOf(frac);
-        if (meta > 6) {
-            throw new IllegalArgumentException("unknow useconds meta : " + meta);
-        }
-
-        if (sec.length() < 6) {
-            StringBuilder result = new StringBuilder(6);
-            int len = 6 - sec.length();
-            for (; len > 0; len--) {
-                result.append('0');
-            }
-            result.append(sec);
-            sec = result.toString();
-        }
-
-        return sec.substring(0, meta);
-    }
 }
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java
new file mode 100644
index 00000000..223b5d9c
--- /dev/null
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java
@@ -0,0 +1,154 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_Value;
+import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_enum_type;
+
+/**
+ * 处理mysql8.0 parital json diff解析
+ * 
+ * @author agapple 2018年11月4日 下午3:53:46
+ * @since 1.1.2
+ */
+public class JsonDiffConversion {
+
+    /**
+     * The JSON value in the given path is replaced with a new value. It has the
+     * same effect as `JSON_REPLACE(col, path, value)`.
+     */
+    public static final int DIFF_OPERATION_REPLACE    = 0;
+    /**
+     * Add a new element at the given path. If the path specifies an array
+     * element, it has the same effect as `JSON_ARRAY_INSERT(col, path, value)`.
+     * If the path specifies an object member, it has the same effect as
+     * `JSON_INSERT(col, path, value)`.
+     */
+    public static final int DIFF_OPERATION_INSERT     = 1;
+    /**
+     * The JSON value at the given path is removed from an array or object. It
+     * has the same effect as `JSON_REMOVE(col, path)`.
+     */
+    public static final int DIFF_OPERATION_REMOVE     = 2;
+
+    public static final int JSON_DIFF_OPERATION_COUNT = 3;
+
+    public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex,
+                                                String charsetName) {
+        int position = buffer.position();
+        List operation_names = new ArrayList();
+        while (buffer.hasRemaining()) {
+            int operation_int = buffer.getUint8();
+            if (operation_int >= JSON_DIFF_OPERATION_COUNT) {
+                throw new IllegalArgumentException("reading operation type (invalid operation code)");
+            }
+
+            // skip path
+            long path_length = buffer.getPackedLong();
+            if (path_length > len) {
+                throw new IllegalArgumentException("skipping path");
+            }
+
+            // compute operation name
+            byte[] lastP = buffer.getData(buffer.position() + (int) path_length - 1, 1);
+            String operation_name = json_diff_operation_name(operation_int, lastP[0]);
+            operation_names.add(operation_name);
+
+            buffer.forward((int) path_length);
+            // skip value
+            if (operation_int != DIFF_OPERATION_REMOVE) {
+                long value_length = buffer.getPackedLong();
+                if (value_length > len) {
+                    throw new IllegalArgumentException("skipping path");
+                }
+
+                buffer.forward((int) value_length);
+            }
+        }
+
+        // Print function names in reverse order.
+        StringBuilder builder = new StringBuilder();
+        for (int i = operation_names.size() - 1; i >= 0; i--) {
+            if (i == 0 || operation_names.get(i - 1) != operation_names.get(i)) {
+                builder.append(operation_names.get(i)).append("(");
+            }
+        }
+
+        // Print column id
+        if (columnName != null) {
+            builder.append(columnName);
+        } else {
+            builder.append("@").append(columnIndex);
+        }
+
+        // In case this vector is empty (a no-op), make an early return
+        // after printing only the column name
+        if (operation_names.size() == 0) {
+            return builder;
+        }
+
+        // Print comma between column name and next function argument
+        builder.append(", ");
+        // Print paths and values.
+        buffer.position(position);
+        int diff_i = 0;
+        while (buffer.hasRemaining()) {
+            // Read operation
+            int operation_int = buffer.getUint8();
+
+            // Read path length
+            long path_length = buffer.getPackedLong();
+            // Print path
+            builder.append('\'').append(buffer.getFixString((int) path_length)).append('\'');
+
+            if (operation_int != DIFF_OPERATION_REMOVE) {
+                // Print comma between path and value
+                builder.append(", ");
+                // Read value length
+                long value_length = buffer.getPackedLong();
+
+                Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(),
+                    buffer,
+                    value_length - 1,
+                    charsetName);
+                buffer.forward((int) value_length - 1);
+                // Read value
+                if (jsonValue.m_type == Json_enum_type.ERROR) {
+                    throw new IllegalArgumentException("parsing json value");
+                }
+                StringBuilder jsonBuilder = new StringBuilder();
+                jsonValue.toJsonString(jsonBuilder, charsetName);
+                builder.append(jsonBuilder);
+            }
+
+            // Print closing parenthesis
+            if (!buffer.hasRemaining() || operation_names.get(diff_i + 1) != operation_names.get(diff_i)) {
+                builder.append(")");
+            }
+
+            if (buffer.hasRemaining()) {
+                builder.append(", ");
+            }
+            diff_i++;
+        }
+
+        return builder;
+    }
+
+    private static String json_diff_operation_name(int operationType, int last_path_char) {
+        switch (operationType) {
+            case DIFF_OPERATION_REPLACE:
+                return "JSON_REPLACE";
+            case DIFF_OPERATION_INSERT:
+                if (last_path_char == ']') {
+                    return "JSON_ARRAY_INSERT";
+                } else {
+                    return "JSON_INSERT";
+                }
+            case DIFF_OPERATION_REMOVE:
+                return "JSON_REMOVE";
+        }
+        throw new IllegalArgumentException("illeagal operationType : " + operationType);
+    }
+}
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java
index be61c99f..408eaa4e 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java
@@ -1484,7 +1484,9 @@ public class LogBuffer {
 
         for (int bit = 0; bit < len; bit += 8) {
             int flag = ((int) buf[pos++]) & 0xff;
-            if (flag == 0) continue;
+            if (flag == 0) {
+                continue;
+            }
             if ((flag & 0x01) != 0) bitmap.set(bit);
             if ((flag & 0x02) != 0) bitmap.set(bit + 1);
             if ((flag & 0x04) != 0) bitmap.set(bit + 2);
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java
index 6670f259..3c1bff17 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java
@@ -109,11 +109,6 @@ public final class LogDecoder {
                     try {
                         /* Decoding binary-log to event */
                         event = decode(buffer, header, context);
-                        if (event != null) {
-                            // set logFileName
-                            event.getHeader().setLogFileName(context.getLogPosition().getFileName());
-                            event.setSemival(buffer.semival);
-                        }
                     } catch (IOException e) {
                         if (logger.isWarnEnabled()) {
                             logger.warn("Decoding " + LogEvent.getTypeName(header.getType()) + " failed from: "
@@ -128,6 +123,12 @@ public final class LogDecoder {
                     event = new UnknownLogEvent(header);
                 }
 
+                if (event != null) {
+                    // set logFileName
+                    event.getHeader().setLogFileName(context.getLogPosition().getFileName());
+                    event.setSemival(buffer.semival);
+                }
+
                 /* consume this binary-log. */
                 buffer.consume(len);
                 return event;
@@ -189,6 +190,7 @@ public final class LogDecoder {
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
             case LogEvent.UPDATE_ROWS_EVENT_V1: {
@@ -196,6 +198,7 @@ public final class LogDecoder {
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
             case LogEvent.DELETE_ROWS_EVENT_V1: {
@@ -203,6 +206,7 @@ public final class LogDecoder {
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
             case LogEvent.ROTATE_EVENT: {
@@ -369,6 +373,14 @@ public final class LogDecoder {
                 header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
+            case LogEvent.PARTIAL_UPDATE_ROWS_EVENT: {
+                RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent, true);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
+                return event;
+            }
             case LogEvent.GTID_LOG_EVENT:
             case LogEvent.ANONYMOUS_GTID_LOG_EVENT: {
                 GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent);
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java
index 8a5957d0..41208516 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java
@@ -169,9 +169,15 @@ public abstract class LogEvent {
     /* Prepared XA transaction terminal event similar to Xid */
     public static final int    XA_PREPARE_LOG_EVENT                     = 38;
 
+    /**
+     * Extension of UPDATE_ROWS_EVENT, allowing partial values according to
+     * binlog_row_value_options.
+     */
+    public static final int    PARTIAL_UPDATE_ROWS_EVENT                = 39;
+
     // mariaDb 5.5.34
     /* New MySQL/Sun events are to be added right above this comment */
-    public static final int    MYSQL_EVENTS_END                         = 39;
+    public static final int    MYSQL_EVENTS_END                         = 49;
 
     public static final int    MARIA_EVENTS_BEGIN                       = 160;
     /* New Maria event numbers start from here */
@@ -361,6 +367,8 @@ public abstract class LogEvent {
                 return "Anonymous_Gtid";
             case PREVIOUS_GTIDS_LOG_EVENT:
                 return "Previous_gtids";
+            case PARTIAL_UPDATE_ROWS_EVENT:
+                return "Update_rows_partial";
             default:
                 return "Unknown"; /* impossible */
         }
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java
index 8492e2d7..a14b9111 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java
@@ -211,6 +211,7 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
                 postHeaderLen[TRANSACTION_CONTEXT_EVENT - 1] = TRANSACTION_CONTEXT_HEADER_LEN;
                 postHeaderLen[VIEW_CHANGE_EVENT - 1] = VIEW_CHANGE_HEADER_LEN;
                 postHeaderLen[XA_PREPARE_LOG_EVENT - 1] = XA_PREPARE_HEADER_LEN;
+                postHeaderLen[PARTIAL_UPDATE_ROWS_EVENT - 1] = ROWS_HEADER_LEN_V2;
 
                 // mariadb 10
                 postHeaderLen[ANNOTATE_ROWS_EVENT - 1] = ANNOTATE_ROWS_HEADER_LEN;
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java
index f6e0459c..c732da99 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java
@@ -1,6 +1,7 @@
 package com.taobao.tddl.dbsync.binlog.event;
 
 import java.io.IOException;
+import java.math.BigInteger;
 import java.nio.charset.Charset;
 
 import com.taobao.tddl.dbsync.binlog.CharsetConversion;
@@ -285,12 +286,12 @@ public class QueryLogEvent extends LogEvent {
                                                                   * type,
                                                                   * sql_mode
                                                                   */
-                                                         + 1 + 1 + 255 /*
-                                                                        * type,
-                                                                        * length
-                                                                        * ,
-                                                                        * catalog
-                                                                        */
+                                                         + 1 + 1 + 255/*
+                                                                       * type,
+                                                                       * length
+                                                                       * ,
+                                                                       * catalog
+                                                                       */
                                                          + 1 + 4 /*
                                                                   * type,
                                                                   * auto_increment
@@ -330,14 +331,32 @@ public class QueryLogEvent extends LogEvent {
                                                           * MariaDb type,
                                                           * sec_part of NOW()
                                                           */
-                                                         + 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3 + 1 + 32 * 3
-                                                         + 1 + 60/*
-                                                                  * type ,
-                                                                  * user_len ,
-                                                                  * user ,
-                                                                  * host_len ,
-                                                                  * host
-                                                                  */);
+                                                         + 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3 /*
+                                                                                                            * type
+                                                                                                            * ,
+                                                                                                            * microseconds
+                                                                                                            */+ 1 + 32
+                                                         * 3 + 1 + 60/*
+                                                                      * type ,
+                                                                      * user_len
+                                                                      * , user ,
+                                                                      * host_len
+                                                                      * , host
+                                                                      */)
+                                                        + 1 + 1 /*
+                                                                 * type,
+                                                                 * explicit_def
+                                                                 * ..ts
+                                                                 */+ 1 + 8 /*
+                                                                            * type,
+                                                                            * xid
+                                                                            * of
+                                                                            * DDL
+                                                                            */+ 1 + 2 /*
+                                                                                       * type
+                                                                                       * ,
+                                                                                       * default_collation_for_utf8mb4_number
+                                                                                       */+ 1 /* sql_require_primary_key */;
     /**
      * Fixed data part:
      * 
      
@@ -394,7 +413,7 @@ public class QueryLogEvent extends LogEvent {
     // inspection by the DBA
     private final long      execTime;
     private final int       errorCode;
-    private final long      sessionId;                                                            /* thread_id */
+    private final long      sessionId;                                                                                     /* thread_id */
 
     /**
      * 'flags2' is a second set of flags (on top of those in Log_event), for
@@ -412,6 +431,8 @@ public class QueryLogEvent extends LogEvent {
     private int             clientCharset             = -1;
     private int             clientCollation           = -1;
     private int             serverCollation           = -1;
+    private int             tvSec                     = -1;
+    private BigInteger      ddlXid                    = BigInteger.valueOf(-1L);
     private String          charsetName;
 
     private String          timezone;
@@ -562,6 +583,21 @@ public class QueryLogEvent extends LogEvent {
      */
     public static final int Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP = 16;
 
+    /**
+     * The variable carries xid info of 2pc-aware (recoverable) DDL queries.
+     */
+    public static final int Q_DDL_LOGGED_WITH_XID             = 17;
+    /**
+     * This variable stores the default collation for the utf8mb4 character set.
+     * Used to support cross-version replication.
+     */
+    public static final int Q_DEFAULT_COLLATION_FOR_UTF8MB4   = 18;
+
+    /**
+     * Replicate sql_require_primary_key.
+     */
+    public static final int Q_SQL_REQUIRE_PRIMARY_KEY         = 19;
+
     /**
      * FROM MariaDB 5.5.34
      */
@@ -625,7 +661,7 @@ public class QueryLogEvent extends LogEvent {
                         break;
                     case Q_MICROSECONDS:
                         // when.tv_usec= uint3korr(pos);
-                        buffer.forward(3);
+                        tvSec = buffer.getInt24();
                         break;
                     case Q_UPDATED_DB_NAMES:
                         int mtsAccessedDbs = buffer.getUint8();
@@ -646,6 +682,19 @@ public class QueryLogEvent extends LogEvent {
                         }
                         break;
                     case Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP:
+                        // thd->variables.explicit_defaults_for_timestamp
+                        buffer.forward(1);
+                        break;
+                    case Q_DDL_LOGGED_WITH_XID:
+                        ddlXid = buffer.getUlong64();
+                        break;
+                    case Q_DEFAULT_COLLATION_FOR_UTF8MB4:
+                        // int2store(start,
+                        // default_collation_for_utf8mb4_number);
+                        buffer.forward(2);
+                        break;
+                    case Q_SQL_REQUIRE_PRIMARY_KEY:
+                        // *start++ = thd->variables.sql_require_primary_key;
                         buffer.forward(1);
                         break;
                     case Q_HRNOW:
@@ -657,8 +706,10 @@ public class QueryLogEvent extends LogEvent {
                          * That's why you must write status vars in growing
                          * order of code
                          */
-                        if (logger.isDebugEnabled()) logger.debug("Query_log_event has unknown status vars (first has code: "
-                                                                  + code + "), skipping the rest of them");
+                        if (logger.isDebugEnabled()) {
+                            logger.debug("Query_log_event has unknown status vars (first has code: " + code
+                                         + "), skipping the rest of them");
+                        }
                         break; // Break loop
                 }
             }
@@ -695,6 +746,12 @@ public class QueryLogEvent extends LogEvent {
                 return "Q_UPDATED_DB_NAMES";
             case Q_MICROSECONDS:
                 return "Q_MICROSECONDS";
+            case Q_DDL_LOGGED_WITH_XID:
+                return "Q_DDL_LOGGED_WITH_XID";
+            case Q_DEFAULT_COLLATION_FOR_UTF8MB4:
+                return "Q_DEFAULT_COLLATION_FOR_UTF8MB4";
+            case Q_SQL_REQUIRE_PRIMARY_KEY:
+                return "Q_SQL_REQUIRE_PRIMARY_KEY";
         }
         return "CODE#" + code;
     }
@@ -777,6 +834,14 @@ public class QueryLogEvent extends LogEvent {
         return serverCollation;
     }
 
+    public int getTvSec() {
+        return tvSec;
+    }
+
+    public BigInteger getDdlXid() {
+        return ddlXid;
+    }
+
     /**
      * Returns the sql_mode value.
      * 
      
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java
index bc0f3dde..013494d5 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java
@@ -10,6 +10,7 @@ import org.apache.commons.logging.LogFactory;
 
 import com.taobao.tddl.dbsync.binlog.JsonConversion;
 import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_Value;
+import com.taobao.tddl.dbsync.binlog.JsonDiffConversion;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
@@ -31,22 +32,33 @@ public final class RowsLogBuffer {
 
     private final LogBuffer    buffer;
     private final int          columnLen;
+    private final int          jsonColumnCount;
     private final String       charsetName;
-    // private Calendar cal;
 
     private final BitSet       nullBits;
     private int                nullBitIndex;
 
+    // Read value_options if this is AI for PARTIAL_UPDATE_ROWS_EVENT
+    private final boolean      partial;
+    private final BitSet       partialBits;
+
     private boolean            fNull;
     private int                javaType;
     private int                length;
     private Serializable       value;
 
-    public RowsLogBuffer(LogBuffer buffer, final int columnLen, String charsetName){
+    public RowsLogBuffer(LogBuffer buffer, final int columnLen, String charsetName, int jsonColumnCount, boolean partial){
         this.buffer = buffer;
         this.columnLen = columnLen;
         this.charsetName = charsetName;
+        this.partial = partial;
+        this.jsonColumnCount = jsonColumnCount;
         this.nullBits = new BitSet(columnLen);
+        this.partialBits = new BitSet(1);
+    }
+
+    public final boolean nextOneRow(BitSet columns) {
+        return nextOneRow(columns, false);
     }
 
     /**
@@ -55,18 +67,30 @@ public final class RowsLogBuffer {
      * @see mysql-5.1.60/sql/log_event.cc -
      * Rows_log_event::print_verbose_one_row
      */
-    public final boolean nextOneRow(BitSet columns) {
+    public final boolean nextOneRow(BitSet columns, boolean after) {
         final boolean hasOneRow = buffer.hasRemaining();
 
         if (hasOneRow) {
             int column = 0;
 
             for (int i = 0; i < columnLen; i++)
-                if (columns.get(i)) column++;
+                if (columns.get(i)) {
+                    column++;
+                }
 
+            if (after && partial) {
+                partialBits.clear();
+                long valueOptions = buffer.getPackedLong();
+                int PARTIAL_JSON_UPDATES = 1;
+                if ((valueOptions & PARTIAL_JSON_UPDATES) != 0) {
+                    partialBits.set(1);
+                    buffer.forward((jsonColumnCount + 7) / 8);
+                }
+            }
             nullBitIndex = 0;
             nullBits.clear();
             buffer.fillBitmap(nullBits, column);
+
         }
         return hasOneRow;
     }
@@ -77,8 +101,8 @@ public final class RowsLogBuffer {
      * @see mysql-5.1.60/sql/log_event.cc -
      * Rows_log_event::print_verbose_one_row
      */
-    public final Serializable nextValue(final int type, final int meta) {
-        return nextValue(type, meta, false);
+    public final Serializable nextValue(final String columName, final int columnIndex, final int type, final int meta) {
+        return nextValue(columName, columnIndex, type, meta, false);
     }
 
     /**
@@ -87,7 +111,8 @@ public final class RowsLogBuffer {
      * @see mysql-5.1.60/sql/log_event.cc -
      * Rows_log_event::print_verbose_one_row
      */
-    public final Serializable nextValue(final int type, final int meta, boolean isBinary) {
+    public final Serializable nextValue(final String columName, final int columnIndex, final int type, final int meta,
+                                        boolean isBinary) {
         fNull = nullBits.get(nullBitIndex++);
 
         if (fNull) {
@@ -97,7 +122,7 @@ public final class RowsLogBuffer {
             return null;
         } else {
             // Extracting field value from packed buffer.
-            return fetchValue(type, meta, isBinary);
+            return fetchValue(columName, columnIndex, type, meta, isBinary);
         }
     }
 
@@ -248,7 +273,7 @@ public final class RowsLogBuffer {
      * 
      * @see mysql-5.1.60/sql/log_event.cc - log_event_print_value
      */
-    final Serializable fetchValue(int type, final int meta, boolean isBinary) {
+    final Serializable fetchValue(String columnName, int columnIndex, int type, final int meta, boolean isBinary) {
         int len = 0;
 
         if (type == LogEvent.MYSQL_TYPE_STRING) {
@@ -367,7 +392,7 @@ public final class RowsLogBuffer {
                     // 转化为unsign long
                     switch (len) {
                         case 1:
-                            value = buffer.getInt8();
+                            value = buffer.getUint8();
                             break;
                         case 2:
                             value = buffer.getBeUint16();
@@ -610,7 +635,7 @@ public final class RowsLogBuffer {
                     // (u32 % 10000) / 100,
                     // u32 % 100);
 
-                    StringBuilder builder = new StringBuilder(12);
+                    StringBuilder builder = new StringBuilder(17);
                     if (i32 < 0) {
                         builder.append('-');
                     }
@@ -859,7 +884,7 @@ public final class RowsLogBuffer {
                     // 转化为unsign long
                     switch (len) {
                         case 1:
-                            value = buffer.getInt8();
+                            value = buffer.getUint8();
                             break;
                         case 2:
                             value = buffer.getUint16();
@@ -1041,17 +1066,34 @@ public final class RowsLogBuffer {
                     default:
                         throw new IllegalArgumentException("!! Unknown JSON packlen = " + meta);
                 }
-                if (0 == len) {
-                    // fixed issue #1 by lava, json column of zero length has no
-                    // value, value parsing should be skipped
-                    value = "";
-                } else {
+
+                if (partialBits.get(1)) {
+                    // print_json_diff
                     int position = buffer.position();
-                    Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(), buffer, len - 1, charsetName);
-                    StringBuilder builder = new StringBuilder();
-                    jsonValue.toJsonString(builder, charsetName);
+                    StringBuilder builder = JsonDiffConversion.print_json_diff(buffer,
+                        len,
+                        columnName,
+                        columnIndex,
+                        charsetName);
                     value = builder.toString();
                     buffer.position(position + len);
+                } else {
+                    if (0 == len) {
+                        // fixed issue #1 by lava, json column of zero length
+                        // has no
+                        // value, value parsing should be skipped
+                        value = "";
+                    } else {
+                        int position = buffer.position();
+                        Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(),
+                            buffer,
+                            len - 1,
+                            charsetName);
+                        StringBuilder builder = new StringBuilder();
+                        jsonValue.toJsonString(builder, charsetName);
+                        value = builder.toString();
+                        buffer.position(position + len);
+                    }
                 }
                 javaType = Types.VARCHAR;
                 length = len;
@@ -1082,10 +1124,10 @@ public final class RowsLogBuffer {
                 buffer.fillBytes(binary, 0, len);
 
                 /* Warning unsupport cloumn type */
-                logger.warn(String.format("!! Unsupport column type MYSQL_TYPE_GEOMETRY: meta=%d (%04X), len = %d",
-                    meta,
-                    meta,
-                    len));
+                // logger.warn(String.format("!! Unsupport column type MYSQL_TYPE_GEOMETRY: meta=%d (%04X), len = %d",
+                // meta,
+                // meta,
+                // len));
                 javaType = Types.BINARY;
                 value = binary;
                 length = len;
@@ -1120,7 +1162,7 @@ public final class RowsLogBuffer {
         return length;
     }
 
-    private String usecondsToStr(int frac, int meta) {
+    public static String usecondsToStr(int frac, int meta) {
         String sec = String.valueOf(frac);
         if (meta > 6) {
             throw new IllegalArgumentException("unknow useconds meta : " + meta);
@@ -1139,7 +1181,7 @@ public final class RowsLogBuffer {
         return sec.substring(0, meta);
     }
 
-    private void appendNumber4(StringBuilder builder, int d) {
+    public static void appendNumber4(StringBuilder builder, int d) {
         if (d >= 1000) {
             builder.append(digits[d / 1000])
                 .append(digits[(d / 100) % 10])
@@ -1151,7 +1193,7 @@ public final class RowsLogBuffer {
         }
     }
 
-    private void appendNumber3(StringBuilder builder, int d) {
+    public static void appendNumber3(StringBuilder builder, int d) {
         if (d >= 100) {
             builder.append(digits[d / 100]).append(digits[(d / 10) % 10]).append(digits[d % 10]);
         } else {
@@ -1160,7 +1202,7 @@ public final class RowsLogBuffer {
         }
     }
 
-    private void appendNumber2(StringBuilder builder, int d) {
+    public static void appendNumber2(StringBuilder builder, int d) {
         if (d >= 10) {
             builder.append(digits[(d / 10) % 10]).append(digits[d % 10]);
         } else {
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java
index f2ce79b0..f11b0734 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java
@@ -5,6 +5,7 @@ import java.util.BitSet;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogContext;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
 
 /**
  * Common base class for all row-containing log events.
@@ -61,6 +62,7 @@ public abstract class RowsLogEvent extends LogEvent {
 
     /** Bitmap denoting columns available */
     protected final int      columnLen;
+    protected final boolean  partial;
     protected final BitSet   columns;
 
     /**
@@ -71,6 +73,8 @@ public abstract class RowsLogEvent extends LogEvent {
      */
     protected final BitSet   changeColumns;
 
+    protected int            jsonColumnCount         = 0;
+
     /** XXX: Don't handle buffer in another thread. */
     private final LogBuffer  rowsBuf;                           /*
                                                                   * The rows in
@@ -109,6 +113,10 @@ public abstract class RowsLogEvent extends LogEvent {
     public static final int  RW_V_EXTRAINFO_TAG      = 0;
 
     public RowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
+        this(header, buffer, descriptionEvent, false);
+    }
+
+    public RowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent, boolean partial){
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -153,9 +161,11 @@ public abstract class RowsLogEvent extends LogEvent {
 
         buffer.position(commonHeaderLen + postHeaderLen + headerLen);
         columnLen = (int) buffer.getPackedLong();
+        this.partial = partial;
         columns = buffer.getBitmap(columnLen);
 
-        if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT) {
+        if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT
+            || header.type == PARTIAL_UPDATE_ROWS_EVENT) {
             changeColumns = buffer.getBitmap(columnLen);
         } else {
             changeColumns = columns;
@@ -175,6 +185,17 @@ public abstract class RowsLogEvent extends LogEvent {
             // delete original table map events stored in the map).
             context.clearAllTables();
         }
+
+        int jsonColumnCount = 0;
+        int columnCnt = table.getColumnCnt();
+        ColumnInfo[] columnInfo = table.getColumnInfo();
+        for (int i = 0; i < columnCnt; i++) {
+            ColumnInfo info = columnInfo[i];
+            if (info.type == LogEvent.MYSQL_TYPE_JSON) {
+                jsonColumnCount++;
+            }
+        }
+        this.jsonColumnCount = jsonColumnCount;
     }
 
     public final long getTableId() {
@@ -194,7 +215,7 @@ public abstract class RowsLogEvent extends LogEvent {
     }
 
     public final RowsLogBuffer getRowsBuf(String charsetName) {
-        return new RowsLogBuffer(rowsBuf, columnLen, charsetName);
+        return new RowsLogBuffer(rowsBuf, columnLen, charsetName, jsonColumnCount, partial);
     }
 
     public final int getFlags(final int flags) {
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java
index e6550b27..18201ea9 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java
@@ -1,6 +1,8 @@
 package com.taobao.tddl.dbsync.binlog.event;
 
+import java.util.ArrayList;
 import java.util.BitSet;
+import java.util.List;
 
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
@@ -88,6 +90,17 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * first byte, the ninth is in the least significant bit of the second byte, and
  * so on.
  * 
+ * 
+ * optional metadata fields
+ * optional metadata fields are stored in Type, Length, Value(TLV) format.
+ * Type takes 1 byte. Length is a packed integer value. Values takes Length
+ * bytes.
+ * There are some optional metadata defined. They are listed in the table
+ * 
+ * @ref Table_table_map_event_optional_metadata. Optional metadata fields follow
+ * null_bits. Whether binlogging an optional metadata is decided by the server.
+ * The order is not defined, so they can be binlogged in any order. 
+ * 
  * 
  * The table below lists all column types, along with the numerical identifier
  * for it and the size and interpretation of meta-data used to describe the
@@ -284,7 +297,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * of the geometry: 1, 2, 3, or 4.
  * 
  * 
- * 
  * @author Changyuan.lh
  * @version 1.0
  */
@@ -322,20 +334,63 @@ public final class TableMapLogEvent extends LogEvent {
      */
     public static final class ColumnInfo {
 
-        public int type;
-        public int meta;
+        public int          type;
+        public int          meta;
+        public String       name;
+        public boolean      unsigned;
+        public boolean      pk;
+        public List set_enum_values;
+        public int          charset;        // 可以通过CharsetUtil进行转化
+        public int          geoType;
+        public boolean      nullable;
+
+        @Override
+        public String toString() {
+            return "ColumnInfo [type=" + type + ", meta=" + meta + ", name=" + name + ", unsigned=" + unsigned
+                   + ", pk=" + pk + ", set_enum_values=" + set_enum_values + ", charset=" + charset + ", geoType="
+                   + geoType + ", nullable=" + nullable + "]";
+        }
     }
 
     protected final int          columnCnt;
-    protected final ColumnInfo[] columnInfo;         // buffer for field
-                                                      // metadata
+    protected final ColumnInfo[] columnInfo;                     // buffer
+                                                                  // for
+                                                                  // field
+                                                                  // metadata
 
     protected final long         tableId;
     protected BitSet             nullBits;
 
     /** TM = "Table Map" */
-    public static final int      TM_MAPID_OFFSET = 0;
-    public static final int      TM_FLAGS_OFFSET = 6;
+    public static final int      TM_MAPID_OFFSET         = 0;
+    public static final int      TM_FLAGS_OFFSET         = 6;
+
+    // UNSIGNED flag of numeric columns
+    public static final int      SIGNEDNESS              = 1;
+    // Default character set of string columns
+    public static final int      DEFAULT_CHARSET         = 2;
+    // Character set of string columns
+    public static final int      COLUMN_CHARSET          = 3;
+    public static final int      COLUMN_NAME             = 4;
+    // String value of SET columns
+    public static final int      SET_STR_VALUE           = 5;
+    // String value of ENUM columns
+    public static final int      ENUM_STR_VALUE          = 6;
+    // Real type of geometry columns
+    public static final int      GEOMETRY_TYPE           = 7;
+    // Primary key without prefix
+    public static final int      SIMPLE_PRIMARY_KEY      = 8;
+    // Primary key with prefix
+    public static final int      PRIMARY_KEY_WITH_PREFIX = 9;
+
+    private int                  default_charset;
+    private boolean              existOptionalMetaData   = false;
+
+    private static final class Pair {
+
+        public int col_index;
+        public int col_charset;
+    }
 
     /**
      * Constructor used by slave to read the event from the binary log.
@@ -379,7 +434,91 @@ public final class TableMapLogEvent extends LogEvent {
             final int fieldSize = (int) buffer.getPackedLong();
             decodeFields(buffer, fieldSize);
             nullBits = buffer.getBitmap(columnCnt);
+
+            for (int i = 0; i < columnCnt; i++) {
+                if (nullBits.get(i)) {
+                    columnInfo[i].nullable = true;
+                }
+            }
+            /*
+             * After null_bits field, there are some new fields for extra
+             * metadata.
+             */
+            existOptionalMetaData = false;
+            List defaultCharsetPairs = null;
+            List columnCharsets = null;
+            while (buffer.hasRemaining()) {
+                // optional metadata fields
+                int type = buffer.getUint8();
+                int len = (int) buffer.getPackedLong();
+
+                switch (type) {
+                    case SIGNEDNESS:
+                        parse_signedness(buffer, len);
+                        break;
+                    case DEFAULT_CHARSET:
+                        defaultCharsetPairs = parse_default_charset(buffer, len);
+                        break;
+                    case COLUMN_CHARSET:
+                        columnCharsets = parse_column_charset(buffer, len);
+                        break;
+                    case COLUMN_NAME:
+                        // set @@global.binlog_row_metadata='FULL'
+                        // 主要是补充列名相关信息
+                        existOptionalMetaData = true;
+                        parse_column_name(buffer, len);
+                        break;
+                    case SET_STR_VALUE:
+                        parse_set_str_value(buffer, len, true);
+                        break;
+                    case ENUM_STR_VALUE:
+                        parse_set_str_value(buffer, len, false);
+                        break;
+                    case GEOMETRY_TYPE:
+                        parse_geometry_type(buffer, len);
+                        break;
+                    case SIMPLE_PRIMARY_KEY:
+                        parse_simple_pk(buffer, len);
+                        break;
+                    case PRIMARY_KEY_WITH_PREFIX:
+                        parse_pk_with_prefix(buffer, len);
+                        break;
+                    default:
+                        throw new IllegalArgumentException("unknow type : " + type);
+                }
+            }
+
+            if (existOptionalMetaData) {
+                int index = 0;
+                int char_col_index = 0;
+                for (int i = 0; i < columnCnt; i++) {
+                    int cs = -1;
+                    int type = getRealType(columnInfo[i].type, columnInfo[i].meta);
+                    if (is_character_type(type)) {
+                        if (defaultCharsetPairs != null && !defaultCharsetPairs.isEmpty()) {
+                            if (index < defaultCharsetPairs.size()
+                                && char_col_index == defaultCharsetPairs.get(index).col_index) {
+                                cs = defaultCharsetPairs.get(index).col_charset;
+                                index++;
+                            } else {
+                                cs = default_charset;
+                            }
+
+                            char_col_index++;
+                        } else if (columnCharsets != null) {
+                            cs = columnCharsets.get(index);
+                            index++;
+                        }
+
+                        columnInfo[i].charset = cs;
+                    }
+                }
+            }
         }
+
+        // for (int i = 0; i < columnCnt; i++) {
+        // System.out.println(columnInfo[i]);
+        // }
     }
 
     /**
@@ -459,6 +598,192 @@ public final class TableMapLogEvent extends LogEvent {
         buffer.limit(limit);
     }
 
+    private void parse_signedness(LogBuffer buffer, int length) {
+        // stores the signedness flags extracted from field
+        List datas = new ArrayList();
+        for (int i = 0; i < length; i++) {
+            int ut = buffer.getUint8();
+            for (int c = 0x80; c != 0; c >>= 1) {
+                datas.add((ut & c) > 0);
+            }
+        }
+
+        int index = 0;
+        for (int i = 0; i < columnCnt; i++) {
+            if (is_numeric_type(columnInfo[i].type)) {
+                columnInfo[i].unsigned = datas.get(index);
+                index++;
+            }
+        }
+    }
+
+    private List parse_default_charset(LogBuffer buffer, int length) {
+        // stores collation numbers extracted from field.
+        int limit = buffer.position() + length;
+        this.default_charset = (int) buffer.getPackedLong();
+        List datas = new ArrayList();
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int col_index = (int) buffer.getPackedLong();
+            int col_charset = (int) buffer.getPackedLong();
+
+            Pair pair = new Pair();
+            pair.col_index = col_index;
+            pair.col_charset = col_charset;
+            datas.add(pair);
+        }
+
+        return datas;
+    }
+
+    private List parse_column_charset(LogBuffer buffer, int length) {
+        // stores collation numbers extracted from field.
+        int limit = buffer.position() + length;
+        List datas = new ArrayList();
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int col_charset = (int) buffer.getPackedLong();
+            datas.add(col_charset);
+        }
+
+        return datas;
+    }
+
+    private void parse_column_name(LogBuffer buffer, int length) {
+        // stores column names extracted from field
+        int limit = buffer.position() + length;
+        int index = 0;
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int len = (int) buffer.getPackedLong();
+            columnInfo[index++].name = buffer.getFixString(len);
+        }
+    }
+
+    private void parse_set_str_value(LogBuffer buffer, int length, boolean set) {
+        // stores SET/ENUM column's string values extracted from
+        // field. Each SET/ENUM column's string values are stored
+        // into a string separate vector. All of them are stored
+        // in 'vec'.
+        int limit = buffer.position() + length;
+        List> datas = new ArrayList>();
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int count = (int) buffer.getPackedLong();
+            List data = new ArrayList(count);
+            for (int i = 0; i < count; i++) {
+                int len1 = (int) buffer.getPackedLong();
+                data.add(buffer.getFixString(len1));
+            }
+
+            datas.add(data);
+        }
+
+        int index = 0;
+        for (int i = 0; i < columnCnt; i++) {
+            if (set && getRealType(columnInfo[i].type, columnInfo[i].meta) == LogEvent.MYSQL_TYPE_SET) {
+                columnInfo[i].set_enum_values = datas.get(index);
+                index++;
+            }
+
+            if (!set && getRealType(columnInfo[i].type, columnInfo[i].meta) == LogEvent.MYSQL_TYPE_ENUM) {
+                columnInfo[i].set_enum_values = datas.get(index);
+                index++;
+            }
+        }
+    }
+
+    private void parse_geometry_type(LogBuffer buffer, int length) {
+        // stores geometry column's types extracted from field.
+        int limit = buffer.position() + length;
+
+        List datas = new ArrayList();
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int col_type = (int) buffer.getPackedLong();
+            datas.add(col_type);
+        }
+
+        int index = 0;
+        for (int i = 0; i < columnCnt; i++) {
+            if (columnInfo[i].type == LogEvent.MYSQL_TYPE_GEOMETRY) {
+                columnInfo[i].geoType = datas.get(index);
+                index++;
+            }
+        }
+    }
+
+    private void parse_simple_pk(LogBuffer buffer, int length) {
+        // stores primary key's column information extracted from
+        // field. Each column has an index and a prefix which are
+        // stored as a unit_pair. prefix is always 0 for
+        // SIMPLE_PRIMARY_KEY field.
+
+        int limit = buffer.position() + length;
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int col_index = (int) buffer.getPackedLong();
+            columnInfo[col_index].pk = true;
+        }
+    }
+
+    private void parse_pk_with_prefix(LogBuffer buffer, int length) {
+        // stores primary key's column information extracted from
+        // field. Each column has an index and a prefix which are
+        // stored as a unit_pair.
+        int limit = buffer.position() + length;
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int col_index = (int) buffer.getPackedLong();
+            // prefix length, 比如 char(32)
+            @SuppressWarnings("unused")
+            int col_prefix = (int) buffer.getPackedLong();
+            columnInfo[col_index].pk = true;
+        }
+    }
+
+    private boolean is_numeric_type(int type) {
+        switch (type) {
+            case MYSQL_TYPE_TINY:
+            case MYSQL_TYPE_SHORT:
+            case MYSQL_TYPE_INT24:
+            case MYSQL_TYPE_LONG:
+            case MYSQL_TYPE_LONGLONG:
+            case MYSQL_TYPE_NEWDECIMAL:
+            case MYSQL_TYPE_FLOAT:
+            case MYSQL_TYPE_DOUBLE:
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    private boolean is_character_type(int type) {
+        switch (type) {
+            case MYSQL_TYPE_STRING:
+            case MYSQL_TYPE_VAR_STRING:
+            case MYSQL_TYPE_VARCHAR:
+            case MYSQL_TYPE_BLOB:
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    private int getRealType(int type, int meta) {
+        if (type == LogEvent.MYSQL_TYPE_STRING) {
+            if (meta >= 256) {
+                int byte0 = meta >> 8;
+                if ((byte0 & 0x30) != 0x30) {
+                    /* a long CHAR() field: see #37426 */
+                    type = byte0 | 0x30;
+                } else {
+                    switch (byte0) {
+                        case LogEvent.MYSQL_TYPE_SET:
+                        case LogEvent.MYSQL_TYPE_ENUM:
+                        case LogEvent.MYSQL_TYPE_STRING:
+                            type = byte0;
+                    }
+                }
+            }
+        }
+
+        return type;
+    }
+
     public final String getDbName() {
         return dbname;
     }
@@ -478,4 +803,13 @@ public final class TableMapLogEvent extends LogEvent {
     public final long getTableId() {
         return tableId;
     }
+
+    public boolean isExistOptionalMetaData() {
+        return existOptionalMetaData;
+    }
+
+    public void setExistOptionalMetaData(boolean existOptional) {
+        this.existOptionalMetaData = existOptional;
+    }
+
 }
diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java
index f5e5866e..b604e04d 100644
--- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java
+++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java
@@ -14,6 +14,11 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 public final class UpdateRowsLogEvent extends RowsLogEvent {
 
     public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-        super(header, buffer, descriptionEvent);
+        super(header, buffer, descriptionEvent, false);
+    }
+
+    public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent,
+                              boolean partial){
+        super(header, buffer, descriptionEvent, partial);
     }
 }
diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java
index d0c6746d..27ecf3db 100644
--- a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java
+++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java
@@ -68,7 +68,7 @@ public class BaseLogFetcherTest {
                     // update需要处理before/after
                     System.out.println("-------> before");
                     parseOneRow(event, buffer, columns, false);
-                    if (!buffer.nextOneRow(changeColumns)) {
+                    if (!buffer.nextOneRow(changeColumns, true)) {
                         break;
                     }
                     System.out.println("-------> after");
@@ -97,7 +97,7 @@ public class BaseLogFetcherTest {
             }
 
             ColumnInfo info = columnInfo[i];
-            buffer.nextValue(info.type, info.meta);
+            buffer.nextValue(null , i ,info.type, info.meta);
 
             if (buffer.isNull()) {
                 //
diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java
index 78d5b586..3648da68 100644
--- a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java
+++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java
@@ -8,6 +8,15 @@ import java.sql.Statement;
 import org.junit.Assert;
 import org.junit.Test;
 
+import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
+
 public class DirectLogFetcherTest extends BaseLogFetcherTest {
 
     @Test
@@ -15,54 +24,50 @@ public class DirectLogFetcherTest extends BaseLogFetcherTest {
         DirectLogFetcher fecther = new DirectLogFetcher();
         try {
             Class.forName("com.mysql.jdbc.Driver");
-            Connection connection = DriverManager.getConnection("jdbc:mysql://100.81.154.142:3306", "root", "hello");
+            Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "root", "hello");
             Statement statement = connection.createStatement();
             statement.execute("SET @master_binlog_checksum='@@global.binlog_checksum'");
             statement.execute("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'");
 
-            fecther.open(connection, "mysql-bin.000006", 120L, 2);
+            fecther.open(connection, "mysql-bin.000007", 89797036L, 2);
 
-            LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.UNKNOWN_EVENT);
+            LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
             LogContext context = new LogContext();
             while (fecther.fetch()) {
-                decoder.decode(fecther, context);
-                continue;
-                // if (event == null) {
-                // continue;
-                // }
-                //
-                // int eventType = event.getHeader().getType();
-                // switch (eventType) {
-                // case LogEvent.ROTATE_EVENT:
-                // binlogFileName = ((RotateLogEvent) event).getFilename();
-                // break;
-                // case LogEvent.WRITE_ROWS_EVENT_V1:
-                // case LogEvent.WRITE_ROWS_EVENT:
-                // parseRowsEvent((WriteRowsLogEvent) event);
-                // break;
-                // case LogEvent.UPDATE_ROWS_EVENT_V1:
-                // case LogEvent.UPDATE_ROWS_EVENT:
-                // parseRowsEvent((UpdateRowsLogEvent) event);
-                // break;
-                // case LogEvent.DELETE_ROWS_EVENT_V1:
-                // case LogEvent.DELETE_ROWS_EVENT:
-                // parseRowsEvent((DeleteRowsLogEvent) event);
-                // break;
-                // case LogEvent.QUERY_EVENT:
-                // parseQueryEvent((QueryLogEvent) event);
-                // break;
-                // case LogEvent.ROWS_QUERY_LOG_EVENT:
-                // parseRowsQueryEvent((RowsQueryLogEvent) event);
-                // break;
-                // case LogEvent.ANNOTATE_ROWS_EVENT:
-                // parseAnnotateRowsEvent((AnnotateRowsEvent) event);
-                // break;
-                // case LogEvent.XID_EVENT:
-                // parseXidEvent((XidLogEvent) event);
-                // break;
-                // default:
-                // break;
-                // }
+                LogEvent event = decoder.decode(fecther, context);
+                int eventType = event.getHeader().getType();
+                switch (eventType) {
+                    case LogEvent.ROTATE_EVENT:
+                        binlogFileName = ((RotateLogEvent) event).getFilename();
+                        break;
+                    case LogEvent.WRITE_ROWS_EVENT_V1:
+                    case LogEvent.WRITE_ROWS_EVENT:
+                        parseRowsEvent((WriteRowsLogEvent) event);
+                        break;
+                    case LogEvent.UPDATE_ROWS_EVENT_V1:
+                    case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
+                    case LogEvent.UPDATE_ROWS_EVENT:
+                        parseRowsEvent((UpdateRowsLogEvent) event);
+                        break;
+                    case LogEvent.DELETE_ROWS_EVENT_V1:
+                    case LogEvent.DELETE_ROWS_EVENT:
+                        parseRowsEvent((DeleteRowsLogEvent) event);
+                        break;
+                    case LogEvent.QUERY_EVENT:
+                        parseQueryEvent((QueryLogEvent) event);
+                        break;
+                    case LogEvent.ROWS_QUERY_LOG_EVENT:
+                        parseRowsQueryEvent((RowsQueryLogEvent) event);
+                        break;
+                    case LogEvent.ANNOTATE_ROWS_EVENT:
+                        parseAnnotateRowsEvent((AnnotateRowsEvent) event);
+                        break;
+                    case LogEvent.XID_EVENT:
+                        parseXidEvent((XidLogEvent) event);
+                        break;
+                    default:
+                        break;
+                }
             }
         } catch (Exception e) {
             e.printStackTrace();
diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java
index 7673f71b..aada3de8 100644
--- a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java
+++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java
@@ -53,6 +53,7 @@ public class FileLogFetcherTest extends BaseLogFetcherTest {
                             parseRowsEvent((WriteRowsLogEvent) event);
                             break;
                         case LogEvent.UPDATE_ROWS_EVENT_V1:
+                        case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
                         case LogEvent.UPDATE_ROWS_EVENT:
                             parseRowsEvent((UpdateRowsLogEvent) event);
                             break;
diff --git a/deployer/manager_ddl.sql b/deployer/manager_ddl.sql
new file mode 100644
index 00000000..c175c24e
--- /dev/null
+++ b/deployer/manager_ddl.sql
@@ -0,0 +1,37 @@
+CREATE SCHEMA `canal_manager` DEFAULT CHARACTER SET utf8mb4 ;
+
+CREATE TABLE `canal_config` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `name` varchar(45) NOT NULL,
+  `content` text NOT NULL,
+  `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `name_UNIQUE` (`name`)
+) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 ;
+
+CREATE TABLE `canal_instance_config` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `name` varchar(45) NOT NULL,
+  `content` text NOT NULL,
+  `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `name_UNIQUE` (`name`)
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 ;
+
+CREATE TABLE `canal_adapter_config` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `category` varchar(45) NOT NULL,
+  `name` varchar(45) NOT NULL,
+  `content` text NOT NULL,
+  `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 ;
+
+INSERT INTO `canal_config` VALUES ('1', 'canal.properties', '#################################################\r\n######### 		common argument		############# \r\n#################################################\r\n#canal.manager.jdbc.url=jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8\r\n#canal.manager.jdbc.username=root\r\n#canal.manager.jdbc.password=121212\r\ncanal.id = 1\r\ncanal.ip =\r\ncanal.port = 11111\r\ncanal.metrics.pull.port = 11112\r\ncanal.zkServers =\r\n# flush data to zk\r\ncanal.zookeeper.flush.period = 1000\r\ncanal.withoutNetty = false\r\n# tcp, kafka, RocketMQ\r\ncanal.serverMode = tcp\r\n# flush meta cursor/parse position to file\r\ncanal.file.data.dir = ${canal.conf.dir}\r\ncanal.file.flush.period = 1000\r\n## memory store RingBuffer size, should be Math.pow(2,n)\r\ncanal.instance.memory.buffer.size = 16384\r\n## memory store RingBuffer used memory unit size , default 1kb\r\ncanal.instance.memory.buffer.memunit = 1024 \r\n## meory store gets mode used MEMSIZE or ITEMSIZE\r\ncanal.instance.memory.batch.mode = MEMSIZE\r\ncanal.instance.memory.rawEntry = true\r\n\r\n## detecing config\r\ncanal.instance.detecting.enable = false\r\n#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now()\r\ncanal.instance.detecting.sql = select 1\r\ncanal.instance.detecting.interval.time = 3\r\ncanal.instance.detecting.retry.threshold = 3\r\ncanal.instance.detecting.heartbeatHaEnable = false\r\n\r\n# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery\r\ncanal.instance.transaction.size =  1024\r\n# mysql fallback connected to new master should fallback times\r\ncanal.instance.fallbackIntervalInSeconds = 60\r\n\r\n# network config\r\ncanal.instance.network.receiveBufferSize = 16384\r\ncanal.instance.network.sendBufferSize = 16384\r\ncanal.instance.network.soTimeout = 30\r\n\r\n# binlog filter config\r\ncanal.instance.filter.druid.ddl = true\r\ncanal.instance.filter.query.dcl = false\r\ncanal.instance.filter.query.dml = false\r\ncanal.instance.filter.query.ddl = false\r\ncanal.instance.filter.table.error = false\r\ncanal.instance.filter.rows = false\r\ncanal.instance.filter.transaction.entry = false\r\n\r\n# binlog format/image check\r\ncanal.instance.binlog.format = ROW,STATEMENT,MIXED \r\ncanal.instance.binlog.image = FULL,MINIMAL,NOBLOB\r\n\r\n# binlog ddl isolation\r\ncanal.instance.get.ddl.isolation = false\r\n\r\n# parallel parser config\r\ncanal.instance.parser.parallel = true\r\n## concurrent thread number, default 60% available processors, suggest not to exceed Runtime.getRuntime().availableProcessors()\r\n#canal.instance.parser.parallelThreadSize = 16\r\n## disruptor ringbuffer size, must be power of 2\r\ncanal.instance.parser.parallelBufferSize = 256\r\n\r\n# table meta tsdb info\r\ncanal.instance.tsdb.enable = true\r\ncanal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}\r\ncanal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;\r\ncanal.instance.tsdb.dbUsername = canal\r\ncanal.instance.tsdb.dbPassword = canal\r\n# dump snapshot interval, default 24 hour\r\ncanal.instance.tsdb.snapshot.interval = 24\r\n# purge snapshot expire , default 360 hour(15 days)\r\ncanal.instance.tsdb.snapshot.expire = 360\r\n\r\n# aliyun ak/sk , support rds/mq\r\ncanal.aliyun.accesskey =\r\ncanal.aliyun.secretkey =\r\n\r\n#################################################\r\n######### 		destinations		############# \r\n#################################################\r\ncanal.destinations = example\r\n# conf root dir\r\ncanal.conf.dir = ../conf\r\n# auto scan instance dir add/remove and start/stop instance\r\ncanal.auto.scan = true\r\ncanal.auto.scan.interval = 5\r\n\r\ncanal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml\r\n#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml\r\n\r\ncanal.instance.global.mode = spring\r\ncanal.instance.global.lazy = false\r\n#canal.instance.global.manager.address = 127.0.0.1:1099\r\n#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml\r\ncanal.instance.global.spring.xml = classpath:spring/file-instance.xml\r\n#canal.instance.global.spring.xml = classpath:spring/default-instance.xml\r\n\r\n##################################################\r\n######### 		     MQ 		     #############\r\n##################################################\r\ncanal.mq.servers = 127.0.0.1:6667\r\ncanal.mq.retries = 0\r\ncanal.mq.batchSize = 16384\r\ncanal.mq.maxRequestSize = 1048576\r\ncanal.mq.lingerMs = 1\r\ncanal.mq.bufferMemory = 33554432\r\ncanal.mq.canalBatchSize = 50\r\ncanal.mq.canalGetTimeout = 100\r\ncanal.mq.flatMessage = true\r\ncanal.mq.compressionType = none\r\ncanal.mq.acks = all\r\n', '2018-12-30 16:46:00');
+INSERT INTO `canal_config` VALUES ('2', 'application.yml', 'server:\n  port: 8081\nlogging:\n  level:\n    org.springframework: WARN\n    com.alibaba.otter.canal.client.adapter.hbase: DEBUG\n    com.alibaba.otter.canal.client.adapter.es: DEBUG\n    com.alibaba.otter.canal.client.adapter.rdb: DEBUG\nspring:\n  jackson:\n    date-format: yyyy-MM-dd HH:mm:ss\n    time-zone: GMT+8\n    default-property-inclusion: non_null\n\ncanal.conf:\n  canalServerHost: 127.0.0.1:11111\n#  zookeeperHosts: slave1:2181\n#  mqServers: 127.0.0.1:9092 #or rocketmq\n#  flatMessage: true\n  batchSize: 500\n  syncBatchSize: 1000\n  retries: 0\n  timeout:\n  accessKey:\n  secretKey:\n  mode: tcp # kafka rocketMQ\n#  srcDataSources:\n#    defaultDS:\n#      url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true\n#      username: root\n#      password: 121212\n  canalAdapters:\n  - instance: example # canal instance Name or mq topic name\n    groups:\n    - groupId: g1\n      outerAdapters:\n      - name: logger\n#      - name: rdb\n#        key: mysql1\n#        properties:\n#          jdbc.driverClassName: com.mysql.jdbc.Driver\n#          jdbc.url: jdbc:mysql://127.0.0.1:3306/mytest2?useUnicode=true\n#          jdbc.username: root\n#          jdbc.password: 121212\n#      - name: rdb\n#        key: oracle1\n#        properties:\n#          jdbc.driverClassName: oracle.jdbc.OracleDriver\n#          jdbc.url: jdbc:oracle:thin:@localhost:49161:XE\n#          jdbc.username: mytest\n#          jdbc.password: m121212\n#      - name: rdb\n#        key: postgres1\n#        properties:\n#          jdbc.driverClassName: org.postgresql.Driver\n#          jdbc.url: jdbc:postgresql://localhost:5432/postgres\n#          jdbc.username: postgres\n#          jdbc.password: 121212\n#          threads: 1\n#          commitSize: 3000\n#      - name: hbase\n#        properties:\n#          hbase.zookeeper.quorum: 127.0.0.1\n#          hbase.zookeeper.property.clientPort: 2181\n#          zookeeper.znode.parent: /hbase\n#      - name: es\n#        hosts: 127.0.0.1:9300\n#        properties:\n#          cluster.name: elasticsearch\n', '2019-01-05 01:37:43');
+
+INSERT INTO `canal_instance_config` VALUES ('1', 'example', '#################################################\r\n## mysql serverId , v1.0.26+ will autoGen\r\n# canal.instance.mysql.slaveId=0\r\n\r\n# enable gtid use true/false\r\ncanal.instance.gtidon=false\r\n\r\n# position info\r\ncanal.instance.master.address=127.0.0.1:3306\r\ncanal.instance.master.journal.name=\r\ncanal.instance.master.position=\r\ncanal.instance.master.timestamp=\r\ncanal.instance.master.gtid=\r\n\r\n# rds oss binlog\r\ncanal.instance.rds.accesskey=\r\ncanal.instance.rds.secretkey=\r\ncanal.instance.rds.instanceId=\r\n\r\n# table meta tsdb info\r\ncanal.instance.tsdb.enable=true\r\n#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb\r\n#canal.instance.tsdb.dbUsername=canal\r\n#canal.instance.tsdb.dbPassword=canal\r\n\r\n#canal.instance.standby.address =\r\n#canal.instance.standby.journal.name =\r\n#canal.instance.standby.position =\r\n#canal.instance.standby.timestamp =\r\n#canal.instance.standby.gtid=\r\n\r\n# username/password\r\ncanal.instance.dbUsername=canal\r\ncanal.instance.dbPassword=canal\r\ncanal.instance.connectionCharset = UTF-8\r\ncanal.instance.defaultDatabaseName =test\r\n# enable druid Decrypt database password\r\ncanal.instance.enableDruid=false\r\n#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==\r\n\r\n# table regex\r\ncanal.instance.filter.regex=.*\\\\..*\r\n# table black regex\r\ncanal.instance.filter.black.regex=\r\n\r\n# mq config\r\ncanal.mq.topic=example\r\n# 动态topic, 需mq支持动态创建topic\r\n#canal.mq.dynamicTopic=.*,mytest\\\\..*,mytest2.user\r\ncanal.mq.partition=0\r\n# hash partition config\r\n#canal.mq.partitionsNum=3\r\n#canal.mq.partitionHash=test.table:id^name,.*\\\\..*\r\n#################################################\r\n', '2018-12-30 16:46:16');
+
+INSERT INTO `canal_adapter_config` VALUES ('1', 'es', 'mytest_user.yml', 'dataSourceKey: defaultDS\ndestination: example\nesMapping:\n  _index: mytest_user\n  _type: _doc\n  _id: _id\n#  pk: id\n  sql: \"select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name,\n        a.c_time as _c_time, c.labels as _labels from user a\n        left join role b on b.id=a.role_id\n        left join (select user_id, group_concat(label order by id desc separator \';\') as labels from label\n        group by user_id) c on c.user_id=a.id\"\n#  objFields:\n#    _labels: array:;\n  etlCondition: \"where a.c_time>=\'{0}\'\"\n  commitBatch: 3000\n', '2019-01-01 17:05:43');
+INSERT INTO `canal_adapter_config` VALUES ('2', 'hbase', 'mytest_person2.yml', 'dataSourceKey: defaultDS\ndestination: example\nhbaseMapping:\n  mode: STRING  #NATIVE   #PHOENIX\n  database: mytest  # 数据库名\n  table: person2     # 数据库表名\n  hbaseTable: MYTEST.PERSON2   # HBase表名\n  family: CF  # 默认统一Family名称\n  uppercaseQualifier: true  # 字段名转大写, 默认为true\n  commitBatch: 3000 # 批量提交的大小\n  #rowKey: id,type  # 复合字段rowKey不能和columns中的rowKey重复\n  columns:\n    # 数据库字段:HBase对应字段\n    id: ROWKEY LEN:15\n    name: NAME\n    email: EMAIL\n    type:\n    c_time: C_TIME\n    birthday: BIRTHDAY\n#  excludeColumns:\n#    - lat   # 忽略字段\n\n# -- NATIVE类型\n# $DEFAULT\n# $STRING\n# $INTEGER\n# $LONG\n# $SHORT\n# $BOOLEAN\n# $FLOAT\n# $DOUBLE\n# $BIGDECIMAL\n# $DATE\n# $BYTE\n# $BYTES\n\n# -- PHOENIX类型\n# $DEFAULT                  对应PHOENIX里的VARCHAR\n# $UNSIGNED_INT             对应PHOENIX里的UNSIGNED_INT           4字节\n# $UNSIGNED_LONG            对应PHOENIX里的UNSIGNED_LONG          8字节\n# $UNSIGNED_TINYINT         对应PHOENIX里的UNSIGNED_TINYINT       1字节\n# $UNSIGNED_SMALLINT        对应PHOENIX里的UNSIGNED_SMALLINT      2字节\n# $UNSIGNED_FLOAT           对应PHOENIX里的UNSIGNED_FLOAT         4字节\n# $UNSIGNED_DOUBLE          对应PHOENIX里的UNSIGNED_DOUBLE        8字节\n# $INTEGER                  对应PHOENIX里的INTEGER                4字节\n# $BIGINT                   对应PHOENIX里的BIGINT                 8字节\n# $TINYINT                  对应PHOENIX里的TINYINT                1字节\n# $SMALLINT                 对应PHOENIX里的SMALLINT               2字节\n# $FLOAT                    对应PHOENIX里的FLOAT                  4字节\n# $DOUBLE                    对应PHOENIX里的DOUBLE                 8字节\n# $BOOLEAN                  对应PHOENIX里的BOOLEAN                1字节\n# $TIME                     对应PHOENIX里的TIME                   8字节\n# $DATE                     对应PHOENIX里的DATE                   8字节\n# $TIMESTAMP                对应PHOENIX里的TIMESTAMP              12字节\n# $UNSIGNED_TIME            对应PHOENIX里的UNSIGNED_TIME          8字节\n# $UNSIGNED_DATE            对应PHOENIX里的UNSIGNED_DATE          8字节\n# $UNSIGNED_TIMESTAMP       对应PHOENIX里的UNSIGNED_TIMESTAMP     12字节\n# $VARCHAR                  对应PHOENIX里的VARCHAR                动态长度\n# $VARBINARY                对应PHOENIX里的VARBINARY              动态长度\n# $DECIMAL                  对应PHOENIX里的DECIMAL                动态长度', '2019-01-01 17:08:08');
+INSERT INTO `canal_adapter_config` VALUES ('3', 'rdb', 'mytest_user.yml', 'dataSourceKey: defaultDS\ndestination: example\nouterAdapterKey: oracle1\nconcurrent: true\ndbMapping:\n  database: mytest\n  table: user\n  targetTable: mytest.tb_user\n  targetPk:\n    id: id\n#  mapAll: true\n  targetColumns:\n    id:\n    name:\n    role_id:\n    c_time:\n    test1:\n\n\n# Mirror schema synchronize config\n#dataSourceKey: defaultDS\n#destination: example\n#outerAdapterKey: mysql1\n#concurrent: true\n#dbMapping:\n#  mirrorDb: true\n#  database: mytest\n', '2019-01-01 17:08:50');
diff --git a/deployer/pom.xml b/deployer/pom.xml
index 3580869b..d7de9395 100644
--- a/deployer/pom.xml
+++ b/deployer/pom.xml
@@ -3,7 +3,7 @@
 	
 		com.alibaba.otter
 		canal
-		1.1.1-SNAPSHOT
+		1.1.3-SNAPSHOT
 		../pom.xml
 	
 	com.alibaba.otter
@@ -40,7 +40,7 @@
 						**/canal.properties
 						**/spring/**
 						**/example/**
-						**/kafka.yml
+						**/mq.yml
 					
 				
 			
diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java
index a2a2d532..f387e1b4 100644
--- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java
@@ -37,6 +37,22 @@ public class CanalConstants {
 
     public static final String CANAL_SOCKETCHANNEL               = ROOT + "." + "socketChannel";
 
+    public static final String CANAL_MQ_SERVERS                  = ROOT + "." + "mq.servers";
+    public static final String CANAL_MQ_RETRIES                  = ROOT + "." + "mq.retries";
+    public static final String CANAL_MQ_BATCHSIZE                = ROOT + "." + "mq.batchSize";
+    public static final String CANAL_MQ_LINGERMS                 = ROOT + "." + "mq.lingerMs";
+    public static final String CANAL_MQ_MAXREQUESTSIZE           = ROOT + "." + "mq.maxRequestSize";
+    public static final String CANAL_MQ_BUFFERMEMORY             = ROOT + "." + "mq.bufferMemory";
+    public static final String CANAL_MQ_CANALBATCHSIZE           = ROOT + "." + "mq.canalBatchSize";
+    public static final String CANAL_MQ_CANALGETTIMEOUT          = ROOT + "." + "mq.canalGetTimeout";
+    public static final String CANAL_MQ_FLATMESSAGE              = ROOT + "." + "mq.flatMessage";
+    public static final String CANAL_MQ_COMPRESSION_TYPE         = ROOT + "." + "mq.compressionType";
+    public static final String CANAL_MQ_ACKS                     = ROOT + "." + "mq.acks";
+    public static final String CANAL_MQ_TRANSACTION              = ROOT + "." + "mq.transaction";
+    public static final String CANAL_ALIYUN_ACCESSKEY            = ROOT + "." + "aliyun.accessKey";
+    public static final String CANAL_ALIYUN_SECRETKEY            = ROOT + "." + "aliyun.secretKey";
+    public static final String CANAL_MQ_PROPERTIES               = ROOT + "." + "mq.properties";
+
     public static String getInstanceModeKey(String destination) {
         return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);
     }
diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
index 42821405..d8ca17fd 100644
--- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
@@ -35,6 +35,7 @@ import com.alibaba.otter.canal.instance.manager.CanalConfigClient;
 import com.alibaba.otter.canal.instance.manager.ManagerCanalInstanceGenerator;
 import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator;
 import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.server.CanalMQStarter;
 import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
 import com.alibaba.otter.canal.server.exception.CanalServerException;
 import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
@@ -68,6 +69,8 @@ public class CanalController {
     private CanalInstanceGenerator                   instanceGenerator;
     private ZkClientx                                zkclientx;
 
+    private CanalMQStarter                           canalMQStarter;
+
     public CanalController(){
         this(System.getProperties());
     }
@@ -92,6 +95,16 @@ public class CanalController {
             System.setProperty(CanalConstants.CANAL_SOCKETCHANNEL, socketChannel);
         }
 
+        // 兼容1.1.0版本的ak/sk参数名
+        String accesskey = getProperty(properties, "canal.instance.rds.accesskey");
+        String secretkey = getProperty(properties, "canal.instance.rds.secretkey");
+        if (StringUtils.isNotEmpty(accesskey)) {
+            System.setProperty(CanalConstants.CANAL_ALIYUN_ACCESSKEY, accesskey);
+        }
+        if (StringUtils.isNotEmpty(secretkey)) {
+            System.setProperty(CanalConstants.CANAL_ALIYUN_SECRETKEY, secretkey);
+        }
+
         // 准备canal server
         cid = Long.valueOf(getProperty(properties, CanalConstants.CANAL_ID));
         ip = getProperty(properties, CanalConstants.CANAL_IP);
@@ -105,7 +118,7 @@ public class CanalController {
             logger.info("No valid metrics server port found, use default 11112.");
             embededCanalServer.setMetricsPort(11112);
         }
-        
+
         String canalWithoutNetty = getProperty(properties, CanalConstants.CANAL_WITHOUT_NETTY);
         if (canalWithoutNetty == null || "false".equals(canalWithoutNetty)) {
             canalServer = CanalServerWithNetty.instance();
@@ -127,80 +140,81 @@ public class CanalController {
 
         final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
         ServerRunningMonitors.setServerData(serverData);
-        ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function() {
+        ServerRunningMonitors
+            .setRunningMonitors(MigrateMap.makeComputingMap(new Function() {
 
-            public ServerRunningMonitor apply(final String destination) {
-                ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
-                runningMonitor.setDestination(destination);
-                runningMonitor.setListener(new ServerRunningListener() {
+                public ServerRunningMonitor apply(final String destination) {
+                    ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
+                    runningMonitor.setDestination(destination);
+                    runningMonitor.setListener(new ServerRunningListener() {
 
-                    public void processActiveEnter() {
-                        try {
-                            MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
-                            embededCanalServer.start(destination);
-                        } finally {
-                            MDC.remove(CanalConstants.MDC_DESTINATION);
-                        }
-                    }
-
-                    public void processActiveExit() {
-                        try {
-                            MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
-                            embededCanalServer.stop(destination);
-                        } finally {
-                            MDC.remove(CanalConstants.MDC_DESTINATION);
-                        }
-                    }
-
-                    public void processStart() {
-                        try {
-                            if (zkclientx != null) {
-                                final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
-                                                                                                              + port);
-                                initCid(path);
-                                zkclientx.subscribeStateChanges(new IZkStateListener() {
-
-                                    public void handleStateChanged(KeeperState state) throws Exception {
-
-                                    }
-
-                                    public void handleNewSession() throws Exception {
-                                        initCid(path);
-                                    }
-
-                                    @Override
-                                    public void handleSessionEstablishmentError(Throwable error) throws Exception {
-                                        logger.error("failed to connect to zookeeper", error);
-                                    }
-                                });
+                        public void processActiveEnter() {
+                            try {
+                                MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
+                                embededCanalServer.start(destination);
+                            } finally {
+                                MDC.remove(CanalConstants.MDC_DESTINATION);
                             }
-                        } finally {
-                            MDC.remove(CanalConstants.MDC_DESTINATION);
                         }
-                    }
 
-                    public void processStop() {
-                        try {
-                            MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
-                            if (zkclientx != null) {
-                                final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
-                                                                                                              + port);
-                                releaseCid(path);
+                        public void processActiveExit() {
+                            try {
+                                MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
+                                embededCanalServer.stop(destination);
+                            } finally {
+                                MDC.remove(CanalConstants.MDC_DESTINATION);
                             }
-                        } finally {
-                            MDC.remove(CanalConstants.MDC_DESTINATION);
                         }
-                    }
 
-                });
-                if (zkclientx != null) {
-                    runningMonitor.setZkClient(zkclientx);
+                        public void processStart() {
+                            try {
+                                if (zkclientx != null) {
+                                    final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
+                                        ip + ":" + port);
+                                    initCid(path);
+                                    zkclientx.subscribeStateChanges(new IZkStateListener() {
+
+                                        public void handleStateChanged(KeeperState state) throws Exception {
+
+                                        }
+
+                                        public void handleNewSession() throws Exception {
+                                            initCid(path);
+                                        }
+
+                                        @Override
+                                        public void handleSessionEstablishmentError(Throwable error) throws Exception {
+                                            logger.error("failed to connect to zookeeper", error);
+                                        }
+                                    });
+                                }
+                            } finally {
+                                MDC.remove(CanalConstants.MDC_DESTINATION);
+                            }
+                        }
+
+                        public void processStop() {
+                            try {
+                                MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
+                                if (zkclientx != null) {
+                                    final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
+                                        ip + ":" + port);
+                                    releaseCid(path);
+                                }
+                            } finally {
+                                MDC.remove(CanalConstants.MDC_DESTINATION);
+                            }
+                        }
+
+                    });
+                    if (zkclientx != null) {
+                        runningMonitor.setZkClient(zkclientx);
+                    }
+                    // 触发创建一下cid节点
+                    runningMonitor.init();
+                    return runningMonitor;
                 }
-                // 触发创建一下cid节点
-                runningMonitor.init();
-                return runningMonitor;
-            }
-        }));
+            }));
 
         // 初始化monitor机制
         autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN));
@@ -220,6 +234,9 @@ public class CanalController {
                         ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
                         if (!config.getLazy() && !runningMonitor.isStart()) {
                             runningMonitor.start();
+                            if (canalMQStarter != null) {
+                                canalMQStarter.startDestination(destination);
+                            }
                         }
                     }
                 }
@@ -228,6 +245,9 @@ public class CanalController {
                     // 此处的stop，代表强制退出，非HA机制，所以需要退出HA的monitor和配置信息
                     InstanceConfig config = instanceConfigs.remove(destination);
                     if (config != null) {
+                        if (canalMQStarter != null) {
+                            canalMQStarter.stopDestination(destination);
+                        }
                         embededCanalServer.stop(destination);
                         ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
                         if (runningMonitor.isStart()) {
@@ -246,7 +266,8 @@ public class CanalController {
             instanceConfigMonitors = MigrateMap.makeComputingMap(new Function() {
 
                 public InstanceConfigMonitor apply(InstanceMode mode) {
-                    int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
+                    int scanInterval = Integer
+                        .valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
 
                     if (mode.isSpring()) {
                         SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
@@ -354,8 +375,8 @@ public class CanalController {
             InstanceConfig oldConfig = instanceConfigs.put(destination, config);
 
             if (oldConfig != null) {
-                logger.warn("destination:{} old config:{} has replace by new config:{}", new Object[] { destination,
-                        oldConfig, config });
+                logger
+                    .warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
             }
         }
     }
@@ -387,7 +408,7 @@ public class CanalController {
         return config;
     }
 
-    public String getProperty(Properties properties, String key) {
+    public static String getProperty(Properties properties, String key) {
         key = StringUtils.trim(key);
         String value = System.getProperty(key);
 
@@ -460,6 +481,7 @@ public class CanalController {
     }
 
     public void stop() throws Throwable {
+
         if (canalServer != null) {
             canalServer.stop();
         }
@@ -485,6 +507,19 @@ public class CanalController {
         if (zkclientx != null) {
             zkclientx.close();
         }
+
+        //关闭时清理缓存
+        if (instanceConfigs != null) {
+            instanceConfigs.clear();
+        }
+        if (managerClients != null) {
+            managerClients.clear();
+        }
+        if (instanceConfigMonitors != null) {
+            instanceConfigMonitors.clear();
+        }
+
+        ZkClientx.clearClients();
     }
 
     private void initCid(String path) {
@@ -514,4 +549,11 @@ public class CanalController {
         }
     }
 
+    public CanalMQStarter getCanalMQStarter() {
+        return canalMQStarter;
+    }
+
+    public void setCanalMQStarter(CanalMQStarter canalMQStarter) {
+        this.canalMQStarter = canalMQStarter;
+    }
 }
diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
index 62b73b1d..aab5cda6 100644
--- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
@@ -7,28 +7,32 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.otter.canal.kafka.CanalKafkaStarter;
-import com.alibaba.otter.canal.server.CanalServerStarter;
+import com.alibaba.otter.canal.deployer.monitor.remote.RemoteConfigLoader;
+import com.alibaba.otter.canal.deployer.monitor.remote.RemoteConfigLoaderFactory;
+import com.alibaba.otter.canal.deployer.monitor.remote.RemoteCanalConfigMonitor;
 
 /**
  * canal独立版本启动的入口类
- * 
+ *
  * @author jianghang 2012-11-6 下午05:20:49
  * @version 1.0.0
  */
 public class CanalLauncher {
 
-    private static final String CLASSPATH_URL_PREFIX = "classpath:";
-    private static final Logger logger               = LoggerFactory.getLogger(CanalLauncher.class);
+    private static final String    CLASSPATH_URL_PREFIX = "classpath:";
+    private static final Logger    logger               = LoggerFactory.getLogger(CanalLauncher.class);
+    public static volatile boolean running              = false;
 
-    public static void main(String[] args) throws Throwable {
+    public static void main(String[] args) {
         try {
+            running = true;
             logger.info("## set default uncaught exception handler");
             setGlobalUncaughtExceptionHandler();
 
             logger.info("## load canal configurations");
             String conf = System.getProperty("canal.conf", "classpath:canal.properties");
             Properties properties = new Properties();
+            RemoteConfigLoader remoteConfigLoader = null;
             if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
                 conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
                 properties.load(CanalLauncher.class.getClassLoader().getResourceAsStream(conf));
@@ -36,39 +40,47 @@ public class CanalLauncher {
                 properties.load(new FileInputStream(conf));
             }
 
-            logger.info("## start the canal server.");
-            final CanalController controller = new CanalController(properties);
-            controller.start();
-            logger.info("## the canal server is running now ......");
-            Runtime.getRuntime().addShutdownHook(new Thread() {
-
-                public void run() {
-                    try {
-                        logger.info("## stop the canal server");
-                        controller.stop();
-                    } catch (Throwable e) {
-                        logger.warn("##something goes wrong when stopping canal Server:", e);
-                    } finally {
-                        logger.info("## canal server is down.");
-                    }
+            remoteConfigLoader = RemoteConfigLoaderFactory.getRemoteConfigLoader(properties);
+            if (remoteConfigLoader != null) {
+                // 加载远程canal.properties
+                Properties remoteConfig = remoteConfigLoader.loadRemoteConfig();
+                // 加载remote instance配置
+                remoteConfigLoader.loadRemoteInstanceConfigs();
+                if (remoteConfig != null) {
+                    properties = remoteConfig;
+                } else {
+                    remoteConfigLoader = null;
                 }
-
-            });
-
-            CanalServerStarter canalServerStarter = null;
-            String serverMode = controller.getProperty(properties, CanalConstants.CANAL_SERVER_MODE);
-            if (serverMode.equalsIgnoreCase("kafka")) {
-                canalServerStarter = new CanalKafkaStarter();
-            } else if (serverMode.equalsIgnoreCase("rocketMQ")) {
-                // 预留rocketMQ启动
             }
 
-            if (canalServerStarter != null) {
-                canalServerStarter.init();
+            final CanalStater canalStater = new CanalStater();
+            canalStater.start(properties);
+
+            if (remoteConfigLoader != null) {
+                remoteConfigLoader.startMonitor(new RemoteCanalConfigMonitor() {
+
+                    @Override
+                    public void onChange(Properties properties) {
+                        try {
+                            // 远程配置canal.properties修改重新加载整个应用
+                            canalStater.destroy();
+                            canalStater.start(properties);
+                        } catch (Throwable throwable) {
+                            logger.error(throwable.getMessage(), throwable);
+                        }
+                    }
+                });
+            }
+
+            while (running) {
+                Thread.sleep(1000);
+            }
+
+            if (remoteConfigLoader != null) {
+                remoteConfigLoader.destroy();
             }
         } catch (Throwable e) {
             logger.error("## Something goes wrong when starting up the canal Server:", e);
-            System.exit(0);
         }
     }
 
@@ -81,4 +93,5 @@ public class CanalLauncher {
             }
         });
     }
+
 }
diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStater.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStater.java
new file mode 100644
index 00000000..05c8c6f0
--- /dev/null
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStater.java
@@ -0,0 +1,178 @@
+package com.alibaba.otter.canal.deployer;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.common.MQProperties;
+import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
+import com.alibaba.otter.canal.rocketmq.CanalRocketMQProducer;
+import com.alibaba.otter.canal.server.CanalMQStarter;
+import com.alibaba.otter.canal.spi.CanalMQProducer;
+
+/**
+ * Canal server 启动类
+ *
+ * @author rewerma 2018-12-30 下午05:12:16
+ * @version 1.0.1
+ */
+public class CanalStater {
+
+    private static final Logger logger          = LoggerFactory.getLogger(CanalStater.class);
+
+    private CanalController     controller      = null;
+    private CanalMQProducer     canalMQProducer = null;
+    private Thread              shutdownThread  = null;
+    private CanalMQStarter      canalMQStarter  = null;
+
+    /**
+     * 启动方法
+     * 
+     * @param properties canal.properties 配置
+     * @throws Throwable
+     */
+    synchronized void start(Properties properties) throws Throwable {
+        String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE);
+        if (serverMode.equalsIgnoreCase("kafka")) {
+            canalMQProducer = new CanalKafkaProducer();
+        } else if (serverMode.equalsIgnoreCase("rocketmq")) {
+            canalMQProducer = new CanalRocketMQProducer();
+        }
+
+        if (canalMQProducer != null) {
+            // disable netty
+            System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true");
+            System.setProperty(CanalConstants.CANAL_DESTINATIONS,
+                properties.getProperty(CanalConstants.CANAL_DESTINATIONS));
+        }
+
+        logger.info("## start the canal server.");
+        controller = new CanalController(properties);
+        controller.start();
+        logger.info("## the canal server is running now ......");
+        shutdownThread = new Thread() {
+
+            public void run() {
+                try {
+                    logger.info("## stop the canal server");
+                    controller.stop();
+                    CanalLauncher.running = false;
+                } catch (Throwable e) {
+                    logger.warn("##something goes wrong when stopping canal Server:", e);
+                } finally {
+                    logger.info("## canal server is down.");
+                }
+            }
+
+        };
+        Runtime.getRuntime().addShutdownHook(shutdownThread);
+
+        if (canalMQProducer != null) {
+            canalMQStarter = new CanalMQStarter(canalMQProducer);
+            MQProperties mqProperties = buildMQProperties(properties);
+            canalMQStarter.start(mqProperties);
+            controller.setCanalMQStarter(canalMQStarter);
+        }
+    }
+
+    /**
+     * 销毁方法，远程配置变更时调用
+     * 
+     * @throws Throwable
+     */
+    synchronized void destroy() throws Throwable {
+        if (controller != null) {
+            controller.stop();
+            controller = null;
+        }
+        if (shutdownThread != null) {
+            Runtime.getRuntime().removeShutdownHook(shutdownThread);
+            shutdownThread = null;
+        }
+        if (canalMQProducer != null && canalMQStarter != null) {
+            canalMQStarter.destroy();
+            canalMQStarter = null;
+            canalMQProducer = null;
+        }
+    }
+
+    /**
+     * 构造MQ对应的配置
+     * 
+     * @param properties canal.properties 配置
+     * @return
+     */
+    private static MQProperties buildMQProperties(Properties properties) {
+        MQProperties mqProperties = new MQProperties();
+        String servers = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_SERVERS);
+        if (!StringUtils.isEmpty(servers)) {
+            mqProperties.setServers(servers);
+        }
+        String retires = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_RETRIES);
+        if (!StringUtils.isEmpty(retires)) {
+            mqProperties.setRetries(Integer.valueOf(retires));
+        }
+        String batchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BATCHSIZE);
+        if (!StringUtils.isEmpty(batchSize)) {
+            mqProperties.setBatchSize(Integer.valueOf(batchSize));
+        }
+        String lingerMs = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_LINGERMS);
+        if (!StringUtils.isEmpty(lingerMs)) {
+            mqProperties.setLingerMs(Integer.valueOf(lingerMs));
+        }
+        String maxRequestSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_MAXREQUESTSIZE);
+        if (!StringUtils.isEmpty(maxRequestSize)) {
+            mqProperties.setMaxRequestSize(Integer.valueOf(maxRequestSize));
+        }
+        String bufferMemory = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BUFFERMEMORY);
+        if (!StringUtils.isEmpty(bufferMemory)) {
+            mqProperties.setBufferMemory(Long.valueOf(bufferMemory));
+        }
+        String canalBatchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALBATCHSIZE);
+        if (!StringUtils.isEmpty(canalBatchSize)) {
+            mqProperties.setCanalBatchSize(Integer.valueOf(canalBatchSize));
+        }
+        String canalGetTimeout = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALGETTIMEOUT);
+        if (!StringUtils.isEmpty(canalGetTimeout)) {
+            mqProperties.setCanalGetTimeout(Long.valueOf(canalGetTimeout));
+        }
+        String flatMessage = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_FLATMESSAGE);
+        if (!StringUtils.isEmpty(flatMessage)) {
+            mqProperties.setFlatMessage(Boolean.valueOf(flatMessage));
+        }
+        String compressionType = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_COMPRESSION_TYPE);
+        if (!StringUtils.isEmpty(compressionType)) {
+            mqProperties.setCompressionType(compressionType);
+        }
+        String acks = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACKS);
+        if (!StringUtils.isEmpty(acks)) {
+            mqProperties.setAcks(acks);
+        }
+        String aliyunAccessKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_ACCESSKEY);
+        if (!StringUtils.isEmpty(aliyunAccessKey)) {
+            mqProperties.setAliyunAccessKey(aliyunAccessKey);
+        }
+        String aliyunSecretKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_SECRETKEY);
+        if (!StringUtils.isEmpty(aliyunSecretKey)) {
+            mqProperties.setAliyunSecretKey(aliyunSecretKey);
+        }
+        String transaction = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_TRANSACTION);
+        if (!StringUtils.isEmpty(transaction)) {
+            mqProperties.setTransaction(Boolean.valueOf(transaction));
+        }
+
+        for (Object key : properties.keySet()) {
+            key = StringUtils.trim(key.toString());
+            if (((String) key).startsWith(CanalConstants.CANAL_MQ_PROPERTIES)) {
+                String value = CanalController.getProperty(properties, (String) key);
+                String subKey = ((String) key).substring(CanalConstants.CANAL_MQ_PROPERTIES.length() + 1);
+                mqProperties.getProperties().put(subKey, value);
+            }
+        }
+
+        return mqProperties;
+    }
+}
diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/ConfigItem.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/ConfigItem.java
new file mode 100644
index 00000000..5fb9b4c6
--- /dev/null
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/ConfigItem.java
@@ -0,0 +1,47 @@
+package com.alibaba.otter.canal.deployer.monitor.remote;
+
+/**
+ * 配置对应对象
+ *
+ * @author rewerma 2019-01-25 下午05:20:16
+ * @version 1.0.0
+ */
+public class ConfigItem {
+
+    private Long   id;
+    private String name;
+    private String content;
+    private long   modifiedTime;
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getContent() {
+        return content;
+    }
+
+    public void setContent(String content) {
+        this.content = content;
+    }
+
+    public long getModifiedTime() {
+        return modifiedTime;
+    }
+
+    public void setModifiedTime(long modifiedTime) {
+        this.modifiedTime = modifiedTime;
+    }
+}
diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/DbRemoteConfigLoader.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/DbRemoteConfigLoader.java
new file mode 100644
index 00000000..6cec0e46
--- /dev/null
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/DbRemoteConfigLoader.java
@@ -0,0 +1,264 @@
+package com.alibaba.otter.canal.deployer.monitor.remote;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileWriter;
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.*;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.otter.canal.common.utils.CommonUtils;
+import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
+import com.alibaba.otter.canal.deployer.CanalConstants;
+import com.google.common.base.Joiner;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 基于数据库的远程配置装载器
+ *
+ * @author rewerma 2019-01-25 下午05:20:16
+ * @version 1.0.0
+ */
+public class DbRemoteConfigLoader implements RemoteConfigLoader {
+
+    private static final Logger      logger                 = LoggerFactory.getLogger(DbRemoteConfigLoader.class);
+
+    private Map  remoteInstanceConfigs  = new MapMaker().makeMap();
+
+    private DruidDataSource          dataSource;
+
+    private long                     currentConfigTimestamp = 0;
+
+    private long                     scanIntervalInSecond   = 5;
+    private ScheduledExecutorService executor               = Executors.newScheduledThreadPool(2,
+        new NamedThreadFactory("remote-canal-config-scan"));
+
+    private RemoteInstanceMonitor    remoteInstanceMonitor  = new RemoteInstanceMonitorImpl();
+
+    public DbRemoteConfigLoader(String driverName, String jdbcUrl, String jdbcUsername, String jdbcPassword){
+        dataSource = new DruidDataSource();
+        if (StringUtils.isEmpty(driverName)) {
+            driverName = "com.mysql.jdbc.Driver";
+        }
+        dataSource.setDriverClassName(driverName);
+        dataSource.setUrl(jdbcUrl);
+        dataSource.setUsername(jdbcUsername);
+        dataSource.setPassword(jdbcPassword);
+        dataSource.setInitialSize(1);
+        dataSource.setMinIdle(1);
+        dataSource.setMaxActive(1);
+        dataSource.setMaxWait(60000);
+        dataSource.setTimeBetweenEvictionRunsMillis(60000);
+        dataSource.setMinEvictableIdleTimeMillis(300000);
+        try {
+            dataSource.init();
+        } catch (SQLException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 加载远程 canal.properties文件
+     *
+     * @return 远程配置的properties
+     */
+    @Override
+    public Properties loadRemoteConfig() {
+        Properties properties = null;
+        try {
+            // 加载远程canal配置
+            ConfigItem configItem = getRemoteCanalConfig();
+            if (configItem != null) {
+                if (configItem.getModifiedTime() != currentConfigTimestamp) {
+                    currentConfigTimestamp = configItem.getModifiedTime();
+                    overrideLocalCanalConfig(configItem.getContent());
+                    properties = new Properties();
+                    properties.load(new ByteArrayInputStream(configItem.getContent().getBytes(StandardCharsets.UTF_8)));
+                    scanIntervalInSecond = Integer
+                        .valueOf(properties.getProperty(CanalConstants.CANAL_AUTO_SCAN_INTERVAL, "5"));
+                    logger.info("## Loaded remote canal config: canal.properties ");
+                }
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+        return properties;
+    }
+
+    /**
+     * 覆盖本地 canal.properties
+     *
+     * @param content 远程配置内容文本
+     */
+    private void overrideLocalCanalConfig(String content) {
+        try (FileWriter writer = new FileWriter(CommonUtils.getConfPath() + "canal.properties")) {
+            writer.write(content);
+            writer.flush();
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 获取远程canal.properties配置内容
+     *
+     * @return 内容对象
+     */
+    private ConfigItem getRemoteCanalConfig() {
+        String sql = "select name, content, modified_time from canal_config where id=1";
+        try (Connection conn = dataSource.getConnection();
+                Statement stmt = conn.createStatement();
+                ResultSet rs = stmt.executeQuery(sql)) {
+            if (rs.next()) {
+                ConfigItem configItem = new ConfigItem();
+                configItem.setId(1L);
+                configItem.setName(rs.getString("name"));
+                configItem.setContent(rs.getString("content"));
+                configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
+                return configItem;
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+        return null;
+    }
+
+    /**
+     * 加载远程的instance配置
+     */
+    @Override
+    public void loadRemoteInstanceConfigs() {
+        try {
+            // 加载远程instance配置
+            loadModifiedInstanceConfigs();
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 加载远程instance新增、修改、删除配置
+     */
+    private void loadModifiedInstanceConfigs() {
+        Map remoteConfigStatus = new HashMap<>();
+        String sql = "select id, name, modified_time from canal_instance_config";
+        try (Connection conn = dataSource.getConnection();
+                Statement stmt = conn.createStatement();
+                ResultSet rs = stmt.executeQuery(sql)) {
+            while (rs.next()) {
+                ConfigItem configItem = new ConfigItem();
+                configItem.setId(rs.getLong("id"));
+                configItem.setName(rs.getString("name"));
+                configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
+                remoteConfigStatus.put(configItem.getName(), configItem);
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+
+        if (!remoteConfigStatus.isEmpty()) {
+            List changedIds = new ArrayList<>();
+
+            for (ConfigItem remoteConfigStat : remoteConfigStatus.values()) {
+                ConfigItem currentConfig = remoteInstanceConfigs.get(remoteConfigStat.getName());
+                if (currentConfig == null) {
+                    // 新增
+                    changedIds.add(remoteConfigStat.getId());
+                } else {
+                    // 修改
+                    if (currentConfig.getModifiedTime() != remoteConfigStat.getModifiedTime()) {
+                        changedIds.add(remoteConfigStat.getId());
+                    }
+                }
+            }
+            if (!changedIds.isEmpty()) {
+                String contentsSql = "select id, name, content, modified_time from canal_instance_config  where id in ("
+                                     + Joiner.on(",").join(changedIds) + ")";
+                try (Connection conn = dataSource.getConnection();
+                        Statement stmt = conn.createStatement();
+                        ResultSet rs = stmt.executeQuery(contentsSql)) {
+                    while (rs.next()) {
+                        ConfigItem configItemNew = new ConfigItem();
+                        configItemNew.setId(rs.getLong("id"));
+                        configItemNew.setName(rs.getString("name"));
+                        configItemNew.setContent(rs.getString("content"));
+                        configItemNew.setModifiedTime(rs.getTimestamp("modified_time").getTime());
+
+                        remoteInstanceConfigs.put(configItemNew.getName(), configItemNew);
+                        remoteInstanceMonitor.onModify(configItemNew);
+                    }
+
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+        }
+
+        for (String name : remoteInstanceConfigs.keySet()) {
+            if (!remoteConfigStatus.containsKey(name)) {
+                // 删除
+                remoteInstanceConfigs.remove(name);
+                remoteInstanceMonitor.onDelete(name);
+            }
+        }
+    }
+
+    /**
+     * 监听 canal 主配置和 instance 配置变化
+     *
+     * @param remoteCanalConfigMonitor 监听回调方法
+     */
+    public void startMonitor(final RemoteCanalConfigMonitor remoteCanalConfigMonitor) {
+        // 监听canal.properties变化
+        executor.scheduleWithFixedDelay(new Runnable() {
+
+            public void run() {
+                try {
+                    Properties properties = loadRemoteConfig();
+                    if (properties != null) {
+                        remoteCanalConfigMonitor.onChange(properties);
+                    }
+                } catch (Throwable e) {
+                    logger.error("Scan remote canal config failed", e);
+                }
+            }
+
+        }, 10, scanIntervalInSecond, TimeUnit.SECONDS);
+
+        // 监听instance变化
+        executor.scheduleWithFixedDelay(new Runnable() {
+
+            public void run() {
+                try {
+                    loadRemoteInstanceConfigs();
+                } catch (Throwable e) {
+                    logger.error("Scan remote instance config failed", e);
+                }
+            }
+
+        }, 10, 3, TimeUnit.SECONDS);
+    }
+
+    /**
+     * 销毁
+     */
+    public void destroy() {
+        executor.shutdownNow();
+        try {
+            dataSource.close();
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+}
diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteCanalConfigMonitor.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteCanalConfigMonitor.java
new file mode 100644
index 00000000..4c0ab955
--- /dev/null
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteCanalConfigMonitor.java
@@ -0,0 +1,13 @@
+package com.alibaba.otter.canal.deployer.monitor.remote;
+
+import java.util.Properties;
+
+/**
+ * 远程canal.properties配置监听器接口
+ *
+ * @author rewerma 2019-01-25 下午05:20:16
+ * @version 1.0.0
+ */
+public interface RemoteCanalConfigMonitor {
+    void onChange(Properties properties);
+}
diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteConfigLoader.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteConfigLoader.java
new file mode 100644
index 00000000..6b7444c0
--- /dev/null
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteConfigLoader.java
@@ -0,0 +1,36 @@
+package com.alibaba.otter.canal.deployer.monitor.remote;
+
+import java.util.Properties;
+
+/**
+ * 远程配置装载器接口
+ *
+ * @author rewerma 2019-01-25 下午05:20:16
+ * @version 1.0.0
+ */
+public interface RemoteConfigLoader {
+
+    /**
+     * 加载远程 canal.properties文件
+     *
+     * @return 远程配置的properties
+     */
+    Properties loadRemoteConfig();
+
+    /**
+     * 加载远程的instance配置
+     */
+    void loadRemoteInstanceConfigs();
+
+    /**
+     * 启动监听 canal 主配置和 instance 配置变化
+     *
+     * @param remoteCanalConfigMonitor 监听回调方法
+     */
+    void startMonitor(final RemoteCanalConfigMonitor remoteCanalConfigMonitor);
+
+    /**
+     * 销毁
+     */
+    void destroy();
+}
diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteConfigLoaderFactory.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteConfigLoaderFactory.java
new file mode 100644
index 00000000..9256a38a
--- /dev/null
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteConfigLoaderFactory.java
@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.deployer.monitor.remote;
+
+import java.util.Properties;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 远程配置装载器工厂类
+ *
+ * @author rewerma 2019-01-25 下午05:20:16
+ * @version 1.0.0
+ */
+public class RemoteConfigLoaderFactory {
+
+    private static final Logger logger = LoggerFactory.getLogger(RemoteConfigLoaderFactory.class);
+
+    public static RemoteConfigLoader getRemoteConfigLoader(Properties localProperties) {
+        String jdbcUrl = localProperties.getProperty("canal.manager.jdbc.url");
+        if (!StringUtils.isEmpty(jdbcUrl)) {
+            logger.info("## load remote canal configurations");
+            // load remote config
+            String driverName = localProperties.getProperty("canal.manager.jdbc.driverName");
+            String jdbcUsername = localProperties.getProperty("canal.manager.jdbc.username");
+            String jdbcPassword = localProperties.getProperty("canal.manager.jdbc.password");
+            return new DbRemoteConfigLoader(driverName, jdbcUrl, jdbcUsername, jdbcPassword);
+        }
+        // 可扩展其它远程配置加载器
+
+        logger.info("## load local canal configurations");
+
+        return null;
+    }
+}
diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteInstanceMonitor.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteInstanceMonitor.java
new file mode 100644
index 00000000..28c97788
--- /dev/null
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteInstanceMonitor.java
@@ -0,0 +1,31 @@
+package com.alibaba.otter.canal.deployer.monitor.remote;
+
+/**
+ * 远程xxx/instance.properties配置监听器接口
+ *
+ * @author rewerma 2019-01-25 下午05:20:16
+ * @version 1.0.0
+ */
+public interface RemoteInstanceMonitor {
+
+    /**
+     * 新增配置事件
+     *
+     * @param configItem 配置项
+     */
+    void onAdd(ConfigItem configItem);
+
+    /**
+     * 修改配置事件
+     *
+     * @param configItem 配置项
+     */
+    void onModify(ConfigItem configItem);
+
+    /**
+     * 删除配置事件
+     *
+     * @param instanceName 实例名
+     */
+    void onDelete(String instanceName);
+}
diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteInstanceMonitorImpl.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteInstanceMonitorImpl.java
new file mode 100644
index 00000000..8eded2b1
--- /dev/null
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteInstanceMonitorImpl.java
@@ -0,0 +1,55 @@
+package com.alibaba.otter.canal.deployer.monitor.remote;
+
+import java.io.File;
+import java.io.FileWriter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.common.utils.CommonUtils;
+
+/**
+ * 远程xxx/instance.properties配置监听器实现
+ *
+ * @author rewerma 2019-01-25 下午05:20:16
+ * @version 1.0.0
+ */
+public class RemoteInstanceMonitorImpl implements RemoteInstanceMonitor {
+
+    private static final Logger logger = LoggerFactory.getLogger(RemoteInstanceMonitorImpl.class);
+
+    @Override
+    public void onAdd(ConfigItem configItem) {
+        this.onModify(configItem);
+    }
+
+    @Override
+    public void onModify(ConfigItem configItem) {
+        String confDir = CommonUtils.getConfPath() + configItem.getName();
+        File instanceDir = new File(confDir);
+        if (!instanceDir.exists()) {
+            boolean mkDirs = instanceDir.mkdirs();
+            if (!mkDirs) {
+                logger.info("## Error to create instance config dir: {}", configItem.getName());
+                return;
+            }
+        }
+        try (FileWriter writer = new FileWriter(confDir + "/instance.properties")) {
+            writer.write(configItem.getContent());
+            writer.flush();
+            logger.info("## Loaded remote instance config: {}/instance.properties ", configItem.getName());
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public void onDelete(String instanceName) {
+        File file = new File(CommonUtils.getConfPath() + instanceName + "/");
+        if (file.exists()) {
+            CommonUtils.deleteDir(file);
+            logger.info("## Deleted and loaded remote instance config: {} ", instanceName);
+        }
+    }
+
+}
diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties
index 3d580bae..8b5c09fc 100644
--- a/deployer/src/main/resources/canal.properties
+++ b/deployer/src/main/resources/canal.properties
@@ -1,15 +1,18 @@
 #################################################
 ######### 		common argument		############# 
 #################################################
-canal.id= 1
-canal.ip=
-canal.port=11111
-canal.metrics.pull.port=11112
-canal.zkServers=
+#canal.manager.jdbc.url=jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8
+#canal.manager.jdbc.username=root
+#canal.manager.jdbc.password=121212
+canal.id = 1
+canal.ip =
+canal.port = 11111
+canal.metrics.pull.port = 11112
+canal.zkServers =
 # flush data to zk
 canal.zookeeper.flush.period = 1000
 canal.withoutNetty = false
-# tcp, kafka, rocketMQ
+# tcp, kafka, RocketMQ
 canal.serverMode = tcp
 # flush meta cursor/parse position to file
 canal.file.data.dir = ${canal.conf.dir}
@@ -20,6 +23,7 @@ canal.instance.memory.buffer.size = 16384
 canal.instance.memory.buffer.memunit = 1024 
 ## meory store gets mode used MEMSIZE or ITEMSIZE
 canal.instance.memory.batch.mode = MEMSIZE
+canal.instance.memory.rawEntry = true
 
 ## detecing config
 canal.instance.detecting.enable = false
@@ -63,32 +67,54 @@ canal.instance.parser.parallel = true
 canal.instance.parser.parallelBufferSize = 256
 
 # table meta tsdb info
-canal.instance.tsdb.enable=true
-canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:}
-canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
-canal.instance.tsdb.dbUsername=canal
-canal.instance.tsdb.dbPassword=canal
+canal.instance.tsdb.enable = true
+canal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}
+canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
+canal.instance.tsdb.dbUsername = canal
+canal.instance.tsdb.dbPassword = canal
+# dump snapshot interval, default 24 hour
+canal.instance.tsdb.snapshot.interval = 24
+# purge snapshot expire , default 360 hour(15 days)
+canal.instance.tsdb.snapshot.expire = 360
 
-# rds oss binlog account
-canal.instance.rds.accesskey =
-canal.instance.rds.secretkey =
+# aliyun ak/sk , support rds/mq
+canal.aliyun.accesskey =
+canal.aliyun.secretkey =
 
 #################################################
 ######### 		destinations		############# 
 #################################################
-canal.destinations= example
+canal.destinations = example
 # conf root dir
 canal.conf.dir = ../conf
 # auto scan instance dir add/remove and start/stop instance
 canal.auto.scan = true
 canal.auto.scan.interval = 5
 
-canal.instance.tsdb.spring.xml=classpath:spring/tsdb/h2-tsdb.xml
-#canal.instance.tsdb.spring.xml=classpath:spring/tsdb/mysql-tsdb.xml
+canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
+#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml
 
-canal.instance.global.mode = spring 
+canal.instance.global.mode = spring
 canal.instance.global.lazy = false
 #canal.instance.global.manager.address = 127.0.0.1:1099
 #canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
 canal.instance.global.spring.xml = classpath:spring/file-instance.xml
 #canal.instance.global.spring.xml = classpath:spring/default-instance.xml
+
+##################################################
+######### 		     MQ 		     #############
+##################################################
+canal.mq.servers = 127.0.0.1:6667
+canal.mq.retries = 0
+canal.mq.batchSize = 16384
+canal.mq.maxRequestSize = 1048576
+canal.mq.lingerMs = 1
+canal.mq.bufferMemory = 33554432
+canal.mq.canalBatchSize = 50
+canal.mq.canalGetTimeout = 100
+canal.mq.flatMessage = true
+canal.mq.compressionType = none
+canal.mq.acks = all
+# use transaction for kafka flatMessage batch produce
+canal.mq.transaction = false
+#canal.mq.properties. =
\ No newline at end of file
diff --git a/deployer/src/main/resources/example/instance.properties b/deployer/src/main/resources/example/instance.properties
index f72b65fe..03929772 100644
--- a/deployer/src/main/resources/example/instance.properties
+++ b/deployer/src/main/resources/example/instance.properties
@@ -1,5 +1,5 @@
 #################################################
-## mysql serverId , v1.0.26+ will autoGen 
+## mysql serverId , v1.0.26+ will autoGen
 # canal.instance.mysql.slaveId=0
 
 # enable gtid use true/false
@@ -25,17 +25,30 @@ canal.instance.tsdb.enable=true
 
 #canal.instance.standby.address =
 #canal.instance.standby.journal.name =
-#canal.instance.standby.position = 
+#canal.instance.standby.position =
 #canal.instance.standby.timestamp =
 #canal.instance.standby.gtid=
 
 # username/password
 canal.instance.dbUsername=canal
 canal.instance.dbPassword=canal
-canal.instance.connectionCharset=UTF-8
+canal.instance.connectionCharset = UTF-8
+canal.instance.defaultDatabaseName =test
+# enable druid Decrypt database password
+canal.instance.enableDruid=false
+#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==
 
 # table regex
 canal.instance.filter.regex=.*\\..*
 # table black regex
 canal.instance.filter.black.regex=
-#################################################
\ No newline at end of file
+
+# mq config
+canal.mq.topic=example
+# dynamic topic route by schema or table regex
+#canal.mq.dynamicTopic=.*,mytest,mytest\\..*,mytest2.user
+canal.mq.partition=0
+# hash partition config
+#canal.mq.partitionsNum=3
+#canal.mq.partitionHash=test.table:id^name,.*\\..*
+#################################################
diff --git a/deployer/src/main/resources/kafka.yml b/deployer/src/main/resources/kafka.yml
deleted file mode 100644
index 735f723d..00000000
--- a/deployer/src/main/resources/kafka.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-servers: slave1:6667,slave2:6667,slave3:6667
-retries: 0
-batchSize: 16384
-lingerMs: 1
-bufferMemory: 33554432
-
-# Canal的batch size, 默认50K, 由于kafka最大消息体限制请勿超过1M(900K以下)
-canalBatchSize: 50
-# Canal get数据的超时时间, 单位: 毫秒, 空为不限超时
-canalGetTimeout: 100
-flatMessage: true
-
-canalDestinations:
-  - canalDestination: example
-    topic: example
-    partition:
-
-
diff --git a/deployer/src/main/resources/logback.xml b/deployer/src/main/resources/logback.xml
index d27a1d78..aaaefac7 100644
--- a/deployer/src/main/resources/logback.xml
+++ b/deployer/src/main/resources/logback.xml
@@ -6,7 +6,7 @@
 			
 		
 	
-	
+
 	
 		
 			destination
@@ -16,7 +16,7 @@
 			
 				../logs/${destination}/${destination}.log
 				
+						class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
 					
 					../logs/${destination}/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz
 					
@@ -33,7 +33,7 @@
 			
 		
 	
-	
+
 	
 		
 			destination
@@ -43,7 +43,7 @@
 			
 				../logs/${destination}/meta.log
 				
+						class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
 					
 					../logs/${destination}/%d{yyyy-MM-dd}/meta-%d{yyyy-MM-dd}-%i.log.gz
 					
@@ -60,26 +60,26 @@
 			
 		
 	
-	
-      
-          
-        
-    
-      
-          
-        
-    
-      
-        
-        
-    
+
+	
+		
+		
+	
+	
+		
+		
+	
+	
+		
+		
+	
 	
 		
 		
 	
-    
+
 	
-		
+		
 		
 	
 
\ No newline at end of file
diff --git a/deployer/src/main/resources/metrics/Canal_instances_tmpl.json b/deployer/src/main/resources/metrics/Canal_instances_tmpl.json
index 141810c0..89fd3329 100644
--- a/deployer/src/main/resources/metrics/Canal_instances_tmpl.json
+++ b/deployer/src/main/resources/metrics/Canal_instances_tmpl.json
@@ -826,7 +826,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "$datasource",
       "description": "client 请求的GET与ACK包的QPS。",
       "fill": 1,
       "gridPos": {
diff --git a/deployer/src/main/resources/spring/base-instance.xml b/deployer/src/main/resources/spring/base-instance.xml
index e6c500b5..a89b7400 100644
--- a/deployer/src/main/resources/spring/base-instance.xml
+++ b/deployer/src/main/resources/spring/base-instance.xml
@@ -32,8 +32,8 @@
 	
 	
 	
-		
-		
+		
+		
 		
 	
 
\ No newline at end of file
diff --git a/deployer/src/main/resources/spring/default-instance.xml b/deployer/src/main/resources/spring/default-instance.xml
index 07b9109d..019c3031 100644
--- a/deployer/src/main/resources/spring/default-instance.xml
+++ b/deployer/src/main/resources/spring/default-instance.xml
@@ -11,7 +11,7 @@
 	default-autowire="byName">
 
 	
-	
+
 	
 		
 		
@@ -29,11 +29,14 @@
 		
 			
 		
+        
+            
+        
 	
-	
+
 	
 	
-	
+
 	
 		
 		
@@ -43,7 +46,7 @@
 			
 		
 	
-	
+
 	
 		
 			
@@ -52,14 +55,15 @@
 		
 		
 	
-	
+
 	
 		
 		
 		
 		
+		
 	
-	
+
 	
 		
 		
@@ -78,35 +82,35 @@
 				
 			
 		
-		
+
 		
-		
+
 		
 		
 			
 				
 			
 		
-		
+
 		
 			
 				
 				
 			
 		
-		
+
 		
 		
-		
+
 		
 		
 		
 		
-		
+
 		
 		
 		
-	
+
 		
 		
 			
@@ -120,28 +124,32 @@
 				
 			
 		
-		
+
 		
 		
-		
+
 		
 		
-			
+			
 				
 				
 				
+				
+				
 				
 			
 		
 		
-			
+			
 				
 				
 				
+				
+				
 				
 			
 		
-		
+
 		
 		
 			
@@ -167,17 +175,27 @@
 		
 		
 		
-		
+
 		
 		
 		
-		
+		
+		
+
 		
 		
-		
+
 		
 		
 		
 		
 	
+
+	
+		
+		
+		
+		
+		
+	
 
diff --git a/deployer/src/main/resources/spring/file-instance.xml b/deployer/src/main/resources/spring/file-instance.xml
index 58dd32c9..636dc4e5 100644
--- a/deployer/src/main/resources/spring/file-instance.xml
+++ b/deployer/src/main/resources/spring/file-instance.xml
@@ -11,7 +11,7 @@
 	default-autowire="byName">
 
 	
-	
+
 	
 		
 		
@@ -29,23 +29,27 @@
 		
 			
 		
+        
+            
+        
 	
-	
+
 	
 	
-	
+
 	
 		
 		
 	
-	
+
 	
 		
 		
 		
 		
+		
 	
-	
+
 	
 		
 		
@@ -64,16 +68,16 @@
 				
 			
 		
-		
+
 		
-		
+
 		
 		
 			
 				
 			
 		
-		
+
 		
 			
 				
@@ -82,16 +86,16 @@
 		
 		
 		
-		
+
 		
 		
 		
 		
-		
+
 		
 		
 		
-	
+
 		
 		
 			
@@ -105,28 +109,32 @@
 				
 			
 		
-		
+
 		
 		
-		
+
 		
 		
-			
+			
 				
 				
 				
+				
+				
 				
 			
 		
 		
-			
+			
 				
 				
 				
+				
+				
 				
 			
 		
-		
+
 		
 		
 			
@@ -156,13 +164,23 @@
 		
 		
 		
+		
+		
 
 		
 		
-		
+
 		
 		
 		
 		
 	
+
+	
+        
+		
+        
+        
+        
+	
 
diff --git a/deployer/src/main/resources/spring/group-instance.xml b/deployer/src/main/resources/spring/group-instance.xml
index 47d3ba33..d0c94057 100644
--- a/deployer/src/main/resources/spring/group-instance.xml
+++ b/deployer/src/main/resources/spring/group-instance.xml
@@ -9,7 +9,7 @@
            http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
            http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
 	default-autowire="byName">
-	
+
 	
 
 	
@@ -29,25 +29,29 @@
 		
 			
 		
+		
+			
+		
 	
-	
+
 	
 	
-	
+
 	
-	
+
 	
 		
 		
 		
 		
+		
 	
-	
+
 	
 		
 		
 	
-	
+
 	
 		
 			
@@ -70,16 +74,16 @@
 				
 			
 		
-		
+
 		
-		
+
 		
 		
 			
 				
 			
 		
-		
+
 		
 			
 				
@@ -88,42 +92,46 @@
 		
 		
 		
-		
+
 		
 		
 		
 		
-		
+
 		
 		
 		
-	
+
 		
 		
 			
 		
-		
+
 		
 		
-		
+
 		
 		
-			
+			
 				
 				
 				
+				
+				
 				
 			
 		
 		
-			
+			
 				
 				
 				
+				
+				
 				
 			
 		
-		
+
 		
 		
 			
@@ -148,13 +156,13 @@
 		
 		
 		
-		
+
 		
 		
 		
 		
 	
-	
+
 	
 		
 		
@@ -168,16 +176,16 @@
 				
 			
 		
-		
+
 		
-		
+
 		
 		
 			
 				
 			
 		
-		
+
 		
 			
 				
@@ -186,42 +194,46 @@
 		
 		
 		
-		
+
 		
 		
 		
 		
-		
+
 		
 		
 		
-	
+
 		
 		
 			
 		
-		
+
 		
 		
-		
+
 		
 		
-			
+			
 				
 				
 				
+				
+				
 				
 			
 		
 		
-			
+			
 				
 				
 				
+				
+				
 				
 			
 		
-		
+
 		
 		
 			
@@ -247,10 +259,18 @@
 		
 		
 		
-		
+
 		
 		
 		
 		
 	
+
+    
+        
+		
+        
+        
+        
+    
 
diff --git a/deployer/src/main/resources/spring/memory-instance.xml b/deployer/src/main/resources/spring/memory-instance.xml
index 428454db..68f90f0c 100644
--- a/deployer/src/main/resources/spring/memory-instance.xml
+++ b/deployer/src/main/resources/spring/memory-instance.xml
@@ -9,7 +9,7 @@
            http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
            http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
 	default-autowire="byName">
-	
+
 	
 
 	
@@ -29,20 +29,24 @@
 		
 			
 		
+        
+            
+        
 	
-	
+
 	
 	
-	
+
 	
-	
+
 	
 		
 		
 		
 		
+		
 	
-	
+
 	
 		
 		
@@ -61,16 +65,16 @@
 				
 			
 		
-		
+
 		
-		
+
 		
 		
 			
 				
 			
 		
-		
+
 		
 			
 				
@@ -79,42 +83,46 @@
 		
 		
 		
-		
+
 		
 		
 		
 		
-		
+
 		
 		
 		
-	
+
 		
 		
 			
 		
-		
+
 		
 		
-		
+
 		
 		
-			
+			
 				
 				
 				
+				
+				
 				
 			
 		
 		
-			
+			
 				
 				
 				
+				
+				
 				
 			
 		
-		
+
 		
 		
 			
@@ -140,17 +148,27 @@
 		
 		
 		
-		
+
 		
 		
 		
-		
+		
+		
+
 		
 		
-		
+
 		
 		
 		
 		
 	
+
+	
+		
+		
+		
+		
+		
+	
 
diff --git a/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml b/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml
index beb61236..c05b5200 100644
--- a/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml
+++ b/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml
@@ -23,7 +23,7 @@
 	
 	
 	
-	
+	
 		
 		
 	
diff --git a/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml b/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml
index 58cab638..87f407b3 100644
--- a/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml
+++ b/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml
@@ -23,7 +23,7 @@
 	
 	
 	
-	
+	
 		
 		
 	
diff --git a/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml
index 7445bf69..58957c28 100644
--- a/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml
+++ b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml
@@ -36,10 +36,10 @@
     
 
 
-    
+    
         
     
 
\ No newline at end of file
diff --git a/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml
index 79aa25fe..2ee0f8f4 100644
--- a/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml
+++ b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml
@@ -42,10 +42,10 @@
         where destination=#destination#
     
 
-    
+    
          0
         ]]>
     
 
\ No newline at end of file
diff --git a/deployer/src/main/resources/spring/tsdb/sql/create_table.sql b/deployer/src/main/resources/spring/tsdb/sql/create_table.sql
index e5d6dbf4..322bc2b6 100644
--- a/deployer/src/main/resources/spring/tsdb/sql/create_table.sql
+++ b/deployer/src/main/resources/spring/tsdb/sql/create_table.sql
@@ -26,10 +26,10 @@ CREATE TABLE IF NOT EXISTS `meta_history` (
   `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
   `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
   `use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema',
-  `schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema',
-  `table` varchar(1024) DEFAULT NULL COMMENT '对应的table',
-  `sql` longtext DEFAULT NULL COMMENT '执行的sql',
-  `type` varchar(256) DEFAULT NULL COMMENT 'sql类型',
+  `sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema',
+  `sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table',
+  `sql_text` longtext DEFAULT NULL COMMENT '执行的sql',
+  `sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型',
   `extra` text DEFAULT NULL COMMENT '额外的扩展信息',
   PRIMARY KEY (`id`),
   UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
diff --git a/docker/build.sh b/docker/build.sh
index 6b42e3a4..4d838620 100644
--- a/docker/build.sh
+++ b/docker/build.sh
@@ -19,11 +19,6 @@ if [ "$1" == "base" ] ; then
 else 
     rm -rf $BASE/canal.*.tar.gz ; 
     cd $BASE/../ && mvn clean package -Dmaven.test.skip -Denv=release && cd $current_path ;
-    if [ "$1" == "kafka" ] ; then
-	   cp $BASE/../target/canal-kafka-*.tar.gz $BASE/
-	   docker build --no-cache -t canal/canal-server $BASE/
-    else 
-	   cp $BASE/../target/canal.deployer-*.tar.gz $BASE/
-	   docker build --no-cache -t canal/canal-server $BASE/
-    fi
+    cp $BASE/../target/canal.deployer-*.tar.gz $BASE/
+    docker build --no-cache -t canal/canal-server $BASE/
 fi
diff --git a/docker/image/admin/app.sh b/docker/image/admin/app.sh
index 66928ac3..413378c8 100755
--- a/docker/image/admin/app.sh
+++ b/docker/image/admin/app.sh
@@ -76,9 +76,9 @@ function checkStart() {
 
 function start_canal() {
     echo "start canal ..."
-    serverPort=`perl -le 'print $ENV{"canal.port"}'`
-    if [ -z "$serverPort" ] ; then
-        serverPort=11111
+    metricsPort=`perl -le 'print $ENV{"canal.metrics.pull.port"}'`
+    if [ -z "$metricsPort" ] ; then
+        metricsPort=11112
     fi
 
     destination=`perl -le 'print $ENV{"canal.destinations"}'`
@@ -87,13 +87,15 @@ function start_canal() {
         exit 1;
     else
         if [ "$destination" != "" ] && [ "$destination" != "example" ] ; then
-            mv /home/admin/canal-server/conf/example /home/admin/canal-server/conf/$destination
+            if [ -d /home/admin/canal-server/conf/example ]; then
+                mv /home/admin/canal-server/conf/example /home/admin/canal-server/conf/$destination
+            fi
         fi 
     fi
     su admin -c 'cd /home/admin/canal-server/bin/ && sh restart.sh 1>>/tmp/start.log 2>&1'
     sleep 5
     #check start
-    checkStart "canal" "nc 127.0.0.1 $serverPort -w 1 -z | wc -l" 30
+    checkStart "canal" "nc 127.0.0.1 $metricsPort -w 1 -z | wc -l" 30
 }
 
 function stop_canal() {
diff --git a/docker/run.sh b/docker/run.sh
index 28f32411..7dbdb682 100644
--- a/docker/run.sh
+++ b/docker/run.sh
@@ -87,6 +87,6 @@ done
 
 MEMORY="-m 4096m"
 LOCALHOST=`getMyIp`
-cmd="docker run -it -h $LOCALHOST $CONFIG --name=canal-server $VOLUMNS $NET_MODE $PORTS $MEMORY canal/canal-server"
+cmd="docker run -d -it -h $LOCALHOST $CONFIG --name=canal-server $VOLUMNS $NET_MODE $PORTS $MEMORY canal/canal-server"
 echo $cmd
 eval $cmd
diff --git a/driver/pom.xml b/driver/pom.xml
index 692475db..05baa5ac 100644
--- a/driver/pom.xml
+++ b/driver/pom.xml
@@ -3,7 +3,7 @@
 	
 		com.alibaba.otter
 		canal
-		1.1.1-SNAPSHOT
+		1.1.3-SNAPSHOT
 		../pom.xml
 	
 	com.alibaba.otter
diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/MysqlGTIDSet.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/MysqlGTIDSet.java
index 35872370..f5128e23 100644
--- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/MysqlGTIDSet.java
+++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/MysqlGTIDSet.java
@@ -86,7 +86,8 @@ public class MysqlGTIDSet implements GTIDSet {
         if (gtidData == null || gtidData.length() < 1) {
             m = new HashMap();
         } else {
-            String[] uuidStrs = gtidData.split(",");
+            // 存在多个GTID时会有回车符
+            String[] uuidStrs = gtidData.replaceAll("\n", "").split(",");
             m = new HashMap(uuidStrs.length);
             for (int i = 0; i < uuidStrs.length; i++) {
                 UUIDSet uuidSet = UUIDSet.parse(uuidStrs[i]);
diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java
index 8de2acfd..dfa2783f 100644
--- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java
+++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java
@@ -88,8 +88,9 @@ public class BioSocketChannel implements SocketChannel {
             }
         }
         if (remain > 0 && accTimeout >= timeout) {
-            throw new SocketTimeoutException("Timeout occurred, failed to read " + readSize + " bytes in " + timeout
-                                             + " milliseconds.");
+            throw new SocketTimeoutException("Timeout occurred, failed to read total " + readSize + " bytes in "
+                                             + timeout + " milliseconds, actual read only " + (readSize - remain)
+                                             + " bytes");
         }
         return data;
     }
@@ -120,8 +121,8 @@ public class BioSocketChannel implements SocketChannel {
         }
 
         if (n < len && accTimeout >= timeout) {
-            throw new SocketTimeoutException("Timeout occurred, failed to read " + len + " bytes in " + timeout
-                                             + " milliseconds.");
+            throw new SocketTimeoutException("Timeout occurred, failed to read total " + len + " bytes in " + timeout
+                                             + " milliseconds, actual read only " + n + " bytes");
         }
     }
 
diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/CharsetUtil.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/CharsetUtil.java
new file mode 100644
index 00000000..ba98a87f
--- /dev/null
+++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/CharsetUtil.java
@@ -0,0 +1,258 @@
+package com.alibaba.otter.canal.parse.driver.mysql.utils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * mysql collation转换mapping关系表
+ * 
+ * @author agapple 2018年11月5日 下午1:01:15
+ * @since 1.1.2
+ */
+public class CharsetUtil {
+
+    private static final String[]             INDEX_TO_CHARSET = new String[2048];
+    private static final Map CHARSET_TO_INDEX = new HashMap();
+    static {
+        INDEX_TO_CHARSET[1] = "big5";
+        INDEX_TO_CHARSET[84] = "big5";
+
+        INDEX_TO_CHARSET[3] = "dec8";
+        INDEX_TO_CHARSET[69] = "dec8";
+
+        INDEX_TO_CHARSET[4] = "cp850";
+        INDEX_TO_CHARSET[80] = "cp850";
+
+        INDEX_TO_CHARSET[6] = "hp8";
+        INDEX_TO_CHARSET[72] = "hp8";
+
+        INDEX_TO_CHARSET[7] = "koi8r";
+        INDEX_TO_CHARSET[74] = "koi8r";
+
+        INDEX_TO_CHARSET[5] = "latin1";
+        INDEX_TO_CHARSET[8] = "latin1";
+        INDEX_TO_CHARSET[15] = "latin1";
+        INDEX_TO_CHARSET[31] = "latin1";
+        INDEX_TO_CHARSET[47] = "latin1";
+        INDEX_TO_CHARSET[48] = "latin1";
+        INDEX_TO_CHARSET[49] = "latin1";
+        INDEX_TO_CHARSET[94] = "latin1";
+
+        INDEX_TO_CHARSET[9] = "latin2";
+        INDEX_TO_CHARSET[21] = "latin2";
+        INDEX_TO_CHARSET[27] = "latin2";
+        INDEX_TO_CHARSET[77] = "latin2";
+
+        INDEX_TO_CHARSET[10] = "swe7";
+        INDEX_TO_CHARSET[82] = "swe7";
+
+        INDEX_TO_CHARSET[11] = "ascii";
+        INDEX_TO_CHARSET[65] = "ascii";
+
+        INDEX_TO_CHARSET[12] = "ujis";
+        INDEX_TO_CHARSET[91] = "ujis";
+
+        INDEX_TO_CHARSET[13] = "sjis";
+        INDEX_TO_CHARSET[88] = "sjis";
+
+        INDEX_TO_CHARSET[16] = "hebrew";
+        INDEX_TO_CHARSET[71] = "hebrew";
+
+        INDEX_TO_CHARSET[18] = "tis620";
+        INDEX_TO_CHARSET[69] = "tis620";
+
+        INDEX_TO_CHARSET[19] = "euckr";
+        INDEX_TO_CHARSET[85] = "euckr";
+
+        INDEX_TO_CHARSET[22] = "koi8u";
+        INDEX_TO_CHARSET[75] = "koi8u";
+
+        INDEX_TO_CHARSET[24] = "gb2312";
+        INDEX_TO_CHARSET[86] = "gb2312";
+
+        INDEX_TO_CHARSET[25] = "greek";
+        INDEX_TO_CHARSET[70] = "greek";
+
+        INDEX_TO_CHARSET[26] = "cp1250";
+        INDEX_TO_CHARSET[34] = "cp1250";
+        INDEX_TO_CHARSET[44] = "cp1250";
+        INDEX_TO_CHARSET[66] = "cp1250";
+        INDEX_TO_CHARSET[99] = "cp1250";
+
+        INDEX_TO_CHARSET[28] = "gbk";
+        INDEX_TO_CHARSET[87] = "gbk";
+
+        INDEX_TO_CHARSET[30] = "latin5";
+        INDEX_TO_CHARSET[78] = "latin5";
+
+        INDEX_TO_CHARSET[32] = "armscii8";
+        INDEX_TO_CHARSET[64] = "armscii8";
+
+        INDEX_TO_CHARSET[33] = "utf8";
+        INDEX_TO_CHARSET[83] = "utf8";
+        for (int i = 192; i <= 223; i++) {
+            INDEX_TO_CHARSET[i] = "utf8";
+        }
+        for (int i = 336; i <= 337; i++) {
+            INDEX_TO_CHARSET[i] = "utf8";
+        }
+        for (int i = 352; i <= 357; i++) {
+            INDEX_TO_CHARSET[i] = "utf8";
+        }
+        INDEX_TO_CHARSET[368] = "utf8";
+        INDEX_TO_CHARSET[2047] = "utf8";
+
+        INDEX_TO_CHARSET[35] = "ucs2";
+        INDEX_TO_CHARSET[90] = "ucs2";
+        for (int i = 128; i <= 151; i++) {
+            INDEX_TO_CHARSET[i] = "ucs2";
+        }
+        INDEX_TO_CHARSET[159] = "ucs2";
+        for (int i = 358; i <= 360; i++) {
+            INDEX_TO_CHARSET[i] = "ucs2";
+        }
+
+        INDEX_TO_CHARSET[36] = "cp866";
+        INDEX_TO_CHARSET[68] = "cp866";
+
+        INDEX_TO_CHARSET[37] = "keybcs2";
+        INDEX_TO_CHARSET[73] = "keybcs2";
+
+        INDEX_TO_CHARSET[38] = "macce";
+        INDEX_TO_CHARSET[43] = "macce";
+
+        INDEX_TO_CHARSET[39] = "macroman";
+        INDEX_TO_CHARSET[53] = "macroman";
+
+        INDEX_TO_CHARSET[40] = "cp852";
+        INDEX_TO_CHARSET[81] = "cp852";
+
+        INDEX_TO_CHARSET[20] = "latin7";
+        INDEX_TO_CHARSET[41] = "latin7";
+        INDEX_TO_CHARSET[42] = "latin7";
+        INDEX_TO_CHARSET[79] = "latin7";
+
+        INDEX_TO_CHARSET[45] = "utf8mb4";
+        INDEX_TO_CHARSET[46] = "utf8mb4";
+        for (int i = 224; i <= 247; i++) {
+            INDEX_TO_CHARSET[i] = "utf8mb4";
+        }
+        for (int i = 255; i <= 271; i++) {
+            INDEX_TO_CHARSET[i] = "utf8mb4";
+        }
+        for (int i = 273; i <= 275; i++) {
+            INDEX_TO_CHARSET[i] = "utf8mb4";
+        }
+        for (int i = 277; i <= 294; i++) {
+            INDEX_TO_CHARSET[i] = "utf8mb4";
+        }
+        for (int i = 296; i <= 298; i++) {
+            INDEX_TO_CHARSET[i] = "utf8mb4";
+        }
+        INDEX_TO_CHARSET[300] = "utf8mb4";
+        for (int i = 303; i <= 307; i++) {
+            INDEX_TO_CHARSET[i] = "utf8mb4";
+        }
+        INDEX_TO_CHARSET[326] = "utf8mb4";
+        INDEX_TO_CHARSET[328] = "utf8mb4";
+
+        INDEX_TO_CHARSET[14] = "cp1251";
+        INDEX_TO_CHARSET[23] = "cp1251";
+        INDEX_TO_CHARSET[50] = "cp1251";
+        INDEX_TO_CHARSET[51] = "cp1251";
+        INDEX_TO_CHARSET[52] = "cp1251";
+
+        INDEX_TO_CHARSET[54] = "utf16";
+        INDEX_TO_CHARSET[55] = "utf16";
+        for (int i = 101; i <= 124; i++) {
+            INDEX_TO_CHARSET[i] = "utf16";
+        }
+        INDEX_TO_CHARSET[327] = "utf16";
+
+        INDEX_TO_CHARSET[56] = "utf16le";
+        INDEX_TO_CHARSET[62] = "utf16le";
+
+        INDEX_TO_CHARSET[57] = "cp1256";
+        INDEX_TO_CHARSET[67] = "cp1256";
+
+        INDEX_TO_CHARSET[29] = "cp1257";
+        INDEX_TO_CHARSET[58] = "cp1257";
+        INDEX_TO_CHARSET[59] = "cp1257";
+
+        INDEX_TO_CHARSET[60] = "utf32";
+        INDEX_TO_CHARSET[61] = "utf32";
+        for (int i = 160; i <= 183; i++) {
+            INDEX_TO_CHARSET[i] = "utf32";
+        }
+        INDEX_TO_CHARSET[391] = "utf32";
+
+        INDEX_TO_CHARSET[63] = "binary";
+
+        INDEX_TO_CHARSET[92] = "geostd8";
+        INDEX_TO_CHARSET[93] = "geostd8";
+
+        INDEX_TO_CHARSET[95] = "cp932";
+        INDEX_TO_CHARSET[96] = "cp932";
+
+        INDEX_TO_CHARSET[97] = "eucjpms";
+        INDEX_TO_CHARSET[98] = "eucjpms";
+
+        for (int i = 248; i <= 250; i++) {
+            INDEX_TO_CHARSET[i] = "gb18030";
+        }
+
+        // charset --> index
+        for (int i = 0; i < 2048; i++) {
+            String charset = INDEX_TO_CHARSET[i];
+            if (charset != null && CHARSET_TO_INDEX.get(charset) == null) {
+                CHARSET_TO_INDEX.put(charset, i);
+            }
+        }
+        CHARSET_TO_INDEX.put("iso-8859-1", 14);
+        CHARSET_TO_INDEX.put("iso_8859_1", 14);
+        CHARSET_TO_INDEX.put("utf-8", 33);
+        CHARSET_TO_INDEX.put("utf8mb4", 45);
+    }
+
+    public static final String getCharset(int index) {
+        return INDEX_TO_CHARSET[index];
+    }
+
+    public static final int getIndex(String charset) {
+        if (charset == null || charset.length() == 0) {
+            return 0;
+        } else {
+            Integer i = CHARSET_TO_INDEX.get(charset.toLowerCase());
+            return (i == null) ? 0 : i.intValue();
+        }
+    }
+
+    /**
+     * 'utf8' COLLATE 'utf8_general_ci'
+     * 
+     * @param charset
+     * @return
+     */
+    public static final String collateCharset(String charset) {
+        String[] output = StringUtils.split(charset, "COLLATE");
+        return output[0].replace('\'', ' ').trim();
+    }
+
+    public static String getJavaCharset(String charset) {
+        if ("utf8".equals(charset)) {
+            return charset;
+        }
+
+        if (StringUtils.endsWithIgnoreCase(charset, "utf8mb4")) {
+            return "utf-8";
+        }
+
+        if (StringUtils.endsWithIgnoreCase(charset, "binary")) {
+            return "iso_8859_1";
+        }
+
+        return charset;
+    }
+}
diff --git a/driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/CharsetUtilTest.java b/driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/CharsetUtilTest.java
new file mode 100644
index 00000000..09837d09
--- /dev/null
+++ b/driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/CharsetUtilTest.java
@@ -0,0 +1,51 @@
+package com.alibaba.otter.canal.parse.driver.mysql;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.driver.mysql.utils.CharsetUtil;
+
+public class CharsetUtilTest {
+
+    @Test
+    public void testLatin1() {
+        int charsetIndex = 5;
+        String charset = "latin1";
+        Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex)));
+    }
+
+    @Test
+    public void testGbk() {
+        int charsetIndex = 87;
+        String charset = "gbk";
+        Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex)));
+    }
+
+    @Test
+    public void testGb2312() {
+        int charsetIndex = 24;
+        String charset = "gb2312";
+        Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex)));
+    }
+
+    @Test
+    public void testUtf8() {
+        int charsetIndex = 213;
+        String charset = "utf8";
+        Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex)));
+    }
+
+    @Test
+    public void testUtf8mb4() {
+        int charsetIndex = 235;
+        String charset = "utf8mb4";
+        Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex)));
+    }
+
+    @Test
+    public void testBinary() {
+        int charsetIndex = 63;
+        String charset = "binary";
+        Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex)));
+    }
+}
diff --git a/example/pom.xml b/example/pom.xml
index 679bfb1d..819b4b12 100644
--- a/example/pom.xml
+++ b/example/pom.xml
@@ -3,7 +3,7 @@
 	
 		com.alibaba.otter
 		canal
-		1.1.1-SNAPSHOT
+		1.1.3-SNAPSHOT
 		../pom.xml
 	
 	com.alibaba.otter
diff --git a/example/src/main/bin/startup.bat b/example/src/main/bin/startup.bat
index 94330f09..bd7ebce6 100755
--- a/example/src/main/bin/startup.bat
+++ b/example/src/main/bin/startup.bat
@@ -8,7 +8,7 @@ if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0%
 set conf_dir=%ENV_PATH%\..\conf
 set logback_configurationFile=%conf_dir%\logback.xml
 set client_mode=Simple
-if "%1%" != "" set client_mode=%1%
+if not "%1" == "" set client_mode=%1
 
 set CLASSPATH=%conf_dir%
 set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH%
@@ -20,7 +20,8 @@ set CANAL_OPTS= -DappName=otter-canal-example -Dlogback.configurationFile="%logb
 
 set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS%
 
-if "%client_mode%" == "Cluster"  
+if "%client_mode%" == "Cluster" (
 	java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.example.ClusterCanalClientTest
-else 
+) else (
 	java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.example.SimpleCanalClientTest
+)
diff --git a/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java b/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java
index 9cb5eb00..552f19da 100644
--- a/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java
+++ b/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java
@@ -96,7 +96,6 @@ public class AbstractCanalClientTest {
         if (!running) {
             return;
         }
-        connector.stopRunning();
         running = false;
         if (thread != null) {
             try {
diff --git a/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientPermanceTest.java b/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientPermanceTest.java
index c58eb869..6b1f1800 100644
--- a/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientPermanceTest.java
+++ b/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientPermanceTest.java
@@ -1,6 +1,8 @@
 package com.alibaba.otter.canal.example;
+
 import java.net.InetSocketAddress;
 import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.TimeUnit;
 
 import com.alibaba.otter.canal.client.CanalConnector;
 import com.alibaba.otter.canal.client.CanalConnectors;
@@ -44,7 +46,7 @@ public class SimpleCanalClientPermanceTest {
             connector.connect();
             connector.subscribe();
             while (true) {
-                Message message = connector.getWithoutAck(batchSize);
+                Message message = connector.getWithoutAck(batchSize, 100L, TimeUnit.MILLISECONDS);
                 long batchId = message.getId();
                 int size = message.getRawEntries().size();
                 sum += size;
@@ -53,11 +55,13 @@ public class SimpleCanalClientPermanceTest {
                 queue.add(batchId);
                 if (count % 10 == 0) {
                     end = System.currentTimeMillis();
-                    long tps = (perSum * 1000) / (end - start);
-                    System.out.println(" total : " + sum + " , current : " + perSum + " , cost : " + (end - start)
-                                       + " , tps : " + tps);
-                    start = end;
-                    perSum = 0;
+                    if (end - start != 0) {
+                        long tps = (perSum * 1000) / (end - start);
+                        System.out.println(" total : " + sum + " , current : " + perSum + " , cost : " + (end - start)
+                                           + " , tps : " + tps);
+                        start = end;
+                        perSum = 0;
+                    }
                 }
             }
         } catch (Throwable e) {
diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/CanalConnectorClient.java b/example/src/main/java/com/alibaba/otter/canal/example/db/CanalConnectorClient.java
index 844f60db..247f9e1f 100644
--- a/example/src/main/java/com/alibaba/otter/canal/example/db/CanalConnectorClient.java
+++ b/example/src/main/java/com/alibaba/otter/canal/example/db/CanalConnectorClient.java
@@ -359,7 +359,7 @@ public abstract class CanalConnectorClient extends AbstractCanalLifeCycle implem
                 logger.warn("parse delete index event, table: {}, sql: {}", table, sql);
                 return;
             default:
-                logger.warn("parse unknown event: {}, table: {}, sql: {}", new String[]{eventType.name(), table, sql});
+                logger.warn("parse unknown event: {}, table: {}, sql: {}", eventType.name(), table, sql);
                 break;
         }
     }
@@ -375,7 +375,7 @@ public abstract class CanalConnectorClient extends AbstractCanalLifeCycle implem
     public void whenOthers(CanalEntry.Header header, String sql) {
         String schema = header.getSchemaName();
         String table = header.getTableName();
-        logger.error("ignore event,schema: {},table: {},SQL: {}", new String[]{schema, table, sql});
+        logger.error("ignore event,schema: {},table: {},SQL: {}", schema, table, sql);
     }
 
     public enum ExceptionStrategy {
diff --git a/filter/pom.xml b/filter/pom.xml
index 95c77f8f..6fc6ba88 100644
--- a/filter/pom.xml
+++ b/filter/pom.xml
@@ -3,7 +3,7 @@
 	
 		com.alibaba.otter
 		canal
-		1.1.1-SNAPSHOT
+		1.1.3-SNAPSHOT
 		../pom.xml
 	
 	com.alibaba.otter
diff --git a/filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java b/filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java
index d7f27503..52b192d0 100644
--- a/filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java
+++ b/filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java
@@ -7,17 +7,10 @@ import org.apache.oro.text.regex.Pattern;
 import org.apache.oro.text.regex.PatternCompiler;
 import org.apache.oro.text.regex.Perl5Compiler;
 
-import com.alibaba.otter.canal.filter.exception.CanalFilterException;
 import com.google.common.base.Function;
 import com.google.common.collect.MapMaker;
 import com.google.common.collect.MigrateMap;
 
-/**
- * 提供{@linkplain Pattern}的lazy get处理
- *
- * @author jianghang 2013-1-22 下午09:36:44
- * @version 1.0.0
- */
 public class PatternUtils {
 
     @SuppressWarnings("deprecation")
@@ -32,7 +25,7 @@ public class PatternUtils {
                                                                              | Perl5Compiler.READ_ONLY_MASK
                                                                              | Perl5Compiler.SINGLELINE_MASK);
                                                              } catch (MalformedPatternException e) {
-                                                                 throw new CanalFilterException(e);
+                                                                 throw new RuntimeException(e);
                                                              }
                                                          }
                                                      });
diff --git a/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java
index b634c40a..4eb0dffa 100644
--- a/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java
+++ b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java
@@ -125,4 +125,9 @@ public class AviaterRegexFilter implements CanalEventFilter {
         return result;
     }
 
+    @Override
+    public String toString() {
+        return pattern;
+    }
+
 }
diff --git a/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/RegexFunction.java b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/RegexFunction.java
index ec389416..72f2136d 100644
--- a/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/RegexFunction.java
+++ b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/RegexFunction.java
@@ -12,7 +12,7 @@ import com.googlecode.aviator.runtime.type.AviatorObject;
 
 /**
  * 提供aviator regex的代码扩展
- * 
+ *
  * @author jianghang 2012-7-23 上午10:29:23
  */
 public class RegexFunction extends AbstractFunction {
diff --git a/instance/core/pom.xml b/instance/core/pom.xml
index 1a73fc8a..a7346fc8 100644
--- a/instance/core/pom.xml
+++ b/instance/core/pom.xml
@@ -3,7 +3,7 @@
 	
 		com.alibaba.otter
 		canal
-		1.1.1-SNAPSHOT
+		1.1.3-SNAPSHOT
 		../../pom.xml
 	
 	canal.instance.core
diff --git a/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/AbstractCanalInstance.java b/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/AbstractCanalInstance.java
index 9143a626..b1b64d5b 100644
--- a/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/AbstractCanalInstance.java
+++ b/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/AbstractCanalInstance.java
@@ -38,6 +38,9 @@ public class AbstractCanalInstance extends AbstractCanalLifeCycle implements Can
     protected CanalEventSink> eventSink;                                                    // 链接parse和store的桥接器
     protected CanalMetaManager                       metaManager;                                                  // 消费信息管理器
     protected CanalAlarmHandler                      alarmHandler;                                                 // alarm报警机制
+    protected CanalMQConfig                          mqConfig;                                                     // mq的配置
+
+
 
     @Override
     public boolean subscribeChange(ClientIdentity identity) {
@@ -241,4 +244,9 @@ public class AbstractCanalInstance extends AbstractCanalLifeCycle implements Can
     public CanalAlarmHandler getAlarmHandler() {
         return alarmHandler;
     }
+
+    @Override
+    public CanalMQConfig getMqConfig() {
+        return mqConfig;
+    }
 }
diff --git a/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstance.java b/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstance.java
index 3bd0464b..3d1a08c2 100644
--- a/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstance.java
+++ b/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstance.java
@@ -32,4 +32,6 @@ public interface CanalInstance extends CanalLifeCycle {
      * 客户端发生订阅/取消订阅行为
      */
     boolean subscribeChange(ClientIdentity identity);
+
+    CanalMQConfig getMqConfig();
 }
diff --git a/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalMQConfig.java b/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalMQConfig.java
new file mode 100644
index 00000000..3d49f0fa
--- /dev/null
+++ b/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalMQConfig.java
@@ -0,0 +1,50 @@
+package com.alibaba.otter.canal.instance.core;
+
+public class CanalMQConfig {
+
+    private String  topic;
+    private Integer partition;
+    private Integer partitionsNum;
+    private String  partitionHash;
+    private String  dynamicTopic;
+
+    public String getTopic() {
+        return topic;
+    }
+
+    public void setTopic(String topic) {
+        this.topic = topic;
+    }
+
+    public Integer getPartition() {
+        return partition;
+    }
+
+    public void setPartition(Integer partition) {
+        this.partition = partition;
+    }
+
+    public Integer getPartitionsNum() {
+        return partitionsNum;
+    }
+
+    public void setPartitionsNum(Integer partitionsNum) {
+        this.partitionsNum = partitionsNum;
+    }
+
+    public String getPartitionHash() {
+        return partitionHash;
+    }
+
+    public void setPartitionHash(String partitionHash) {
+        this.partitionHash = partitionHash;
+    }
+
+    public String getDynamicTopic() {
+        return dynamicTopic;
+    }
+
+    public void setDynamicTopic(String dynamicTopic) {
+        this.dynamicTopic = dynamicTopic;
+    }
+}
diff --git a/instance/manager/pom.xml b/instance/manager/pom.xml
index ec85fa2c..ef5c987e 100644
--- a/instance/manager/pom.xml
+++ b/instance/manager/pom.xml
@@ -3,7 +3,7 @@
 	
 		com.alibaba.otter
 		canal
-		1.1.1-SNAPSHOT
+		1.1.3-SNAPSHOT
 		../../pom.xml
 	
 	com.alibaba.otter
diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
index 778a40aa..11091695 100644
--- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
+++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
@@ -6,7 +6,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-import com.alibaba.otter.canal.meta.FileMixedMetaManager;
+import org.apache.commons.lang.BooleanUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -39,7 +39,16 @@ import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
 import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
 import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser;
 import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
-import com.alibaba.otter.canal.parse.index.*;
+import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy;
+import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DefaultTableMetaTSDBFactory;
+import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB;
+import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDBBuilder;
+import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
+import com.alibaba.otter.canal.parse.index.FailbackLogPositionManager;
+import com.alibaba.otter.canal.parse.index.MemoryLogPositionManager;
+import com.alibaba.otter.canal.parse.index.MetaLogPositionManager;
+import com.alibaba.otter.canal.parse.index.PeriodMixedLogPositionManager;
+import com.alibaba.otter.canal.parse.index.ZooKeeperLogPositionManager;
 import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
 import com.alibaba.otter.canal.protocol.position.EntryPosition;
 import com.alibaba.otter.canal.sink.entry.EntryEventSink;
@@ -138,6 +147,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
             memoryEventStore.setBufferMemUnit(parameters.getMemoryStorageBufferMemUnit());
             memoryEventStore.setBatchMode(BatchMode.valueOf(parameters.getStorageBatchMode().name()));
             memoryEventStore.setDdlIsolation(parameters.getDdlIsolation());
+            memoryEventStore.setRaw(parameters.getMemoryStorageRawEntry());
             eventStore = memoryEventStore;
         } else if (mode.isFile()) {
             // 后续版本支持
@@ -229,7 +239,18 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
     private CanalEventParser doInitEventParser(SourcingType type, List dbAddresses) {
         CanalEventParser eventParser;
         if (type.isMysql()) {
-            MysqlEventParser mysqlEventParser = new MysqlEventParser();
+            MysqlEventParser mysqlEventParser = null;
+            if (StringUtils.isNotEmpty(parameters.getRdsAccesskey())
+                && StringUtils.isNotEmpty(parameters.getRdsSecretkey())
+                && StringUtils.isNotEmpty(parameters.getRdsInstanceId())) {
+
+                mysqlEventParser = new RdsBinlogEventParserProxy();
+                ((RdsBinlogEventParserProxy) mysqlEventParser).setAccesskey(parameters.getRdsAccesskey());
+                ((RdsBinlogEventParserProxy) mysqlEventParser).setSecretkey(parameters.getRdsSecretkey());
+                ((RdsBinlogEventParserProxy) mysqlEventParser).setInstanceId(parameters.getRdsInstanceId());
+            } else {
+                mysqlEventParser = new MysqlEventParser();
+            }
             mysqlEventParser.setDestination(destination);
             // 编码参数
             mysqlEventParser.setConnectionCharset(Charset.forName(parameters.getConnectionCharset()));
@@ -273,6 +294,40 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
             mysqlEventParser.setFallbackIntervalInSeconds(parameters.getFallbackIntervalInSeconds());
             mysqlEventParser.setProfilingEnabled(false);
             mysqlEventParser.setFilterTableError(parameters.getFilterTableError());
+            mysqlEventParser.setIsGTIDMode(BooleanUtils.toBoolean(parameters.getGtidEnable()));
+            // tsdb
+            if (parameters.getTsdbSnapshotInterval() != null) {
+                mysqlEventParser.setTsdbSnapshotInterval(parameters.getTsdbSnapshotInterval());
+            }
+            if (parameters.getTsdbSnapshotExpire() != null) {
+                mysqlEventParser.setTsdbSnapshotExpire(parameters.getTsdbSnapshotExpire());
+            }
+            boolean tsdbEnable = BooleanUtils.toBoolean(parameters.getTsdbEnable());
+            if (tsdbEnable) {
+                mysqlEventParser.setTableMetaTSDBFactory(new DefaultTableMetaTSDBFactory() {
+
+                    @Override
+                    public void destory(String destination) {
+                        TableMetaTSDBBuilder.destory(destination);
+                    }
+
+                    @Override
+                    public TableMetaTSDB build(String destination, String springXml) {
+                        try {
+                            System.setProperty("canal.instance.tsdb.url", parameters.getTsdbJdbcUrl());
+                            System.setProperty("canal.instance.tsdb.dbUsername", parameters.getTsdbJdbcUserName());
+                            System.setProperty("canal.instance.tsdb.dbPassword", parameters.getTsdbJdbcPassword());
+
+                            return TableMetaTSDBBuilder.build(destination, "classpath:spring/tsdb/mysql-tsdb.xml");
+                        } finally {
+                            System.setProperty("canal.instance.tsdb.url", "");
+                            System.setProperty("canal.instance.tsdb.dbUsername", "");
+                            System.setProperty("canal.instance.tsdb.dbPassword", "");
+                        }
+                    }
+                });
+                mysqlEventParser.setEnableTsdb(tsdbEnable);
+            }
             eventParser = mysqlEventParser;
         } else if (type.isLocalBinlog()) {
             LocalBinlogEventParser localBinlogEventParser = new LocalBinlogEventParser();
@@ -291,8 +346,8 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
                     parameters.getDbUsername(),
                     parameters.getDbPassword(),
                     parameters.getDefaultDatabaseName()));
-
             }
+
             eventParser = localBinlogEventParser;
         } else if (type.isOracle()) {
             throw new CanalException("unsupport SourcingType for " + type);
diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
index 40df1485..af3841dd 100644
--- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
+++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
@@ -39,6 +39,7 @@ public class CanalParameter implements Serializable {
     private BatchMode                storageBatchMode                   = BatchMode.MEMSIZE;         // 基于大小返回结果
     private Integer                  memoryStorageBufferSize            = 16 * 1024;                 // 内存存储的buffer大小
     private Integer                  memoryStorageBufferMemUnit         = 1024;                      // 内存存储的buffer内存占用单位，默认为1kb
+    private Boolean                  memoryStorageRawEntry              = Boolean.TRUE;              // 内存存储的对象是否启用raw的ByteString模式
     private String                   fileStorageDirectory;                                           // 文件存储的目录位置
     private Integer                  fileStorageStoreCount;                                          // 每个文件store存储的记录数
     private Integer                  fileStorageRollverCount;                                        // store文件的个数
@@ -97,6 +98,12 @@ public class CanalParameter implements Serializable {
     private String                   tsdbJdbcUrl;
     private String                   tsdbJdbcUserName;
     private String                   tsdbJdbcPassword;
+    private Integer                  tsdbSnapshotInterval               = 24;
+    private Integer                  tsdbSnapshotExpire                 = 360;
+    private String                   rdsAccesskey;
+    private String                   rdsSecretkey;
+    private String                   rdsInstanceId;
+    private Boolean                  gtidEnable                         = Boolean.FALSE;             // 是否开启gtid
     // ================================== 兼容字段处理
     private InetSocketAddress        masterAddress;                                                  // 主库信息
     private String                   masterUsername;                                                 // 帐号
@@ -919,6 +926,62 @@ public class CanalParameter implements Serializable {
         this.tsdbJdbcPassword = tsdbJdbcPassword;
     }
 
+    public String getRdsAccesskey() {
+        return rdsAccesskey;
+    }
+
+    public void setRdsAccesskey(String rdsAccesskey) {
+        this.rdsAccesskey = rdsAccesskey;
+    }
+
+    public String getRdsSecretkey() {
+        return rdsSecretkey;
+    }
+
+    public void setRdsSecretkey(String rdsSecretkey) {
+        this.rdsSecretkey = rdsSecretkey;
+    }
+
+    public String getRdsInstanceId() {
+        return rdsInstanceId;
+    }
+
+    public void setRdsInstanceId(String rdsInstanceId) {
+        this.rdsInstanceId = rdsInstanceId;
+    }
+
+    public Boolean getGtidEnable() {
+        return gtidEnable;
+    }
+
+    public void setGtidEnable(Boolean gtidEnable) {
+        this.gtidEnable = gtidEnable;
+    }
+
+    public Boolean getMemoryStorageRawEntry() {
+        return memoryStorageRawEntry;
+    }
+
+    public void setMemoryStorageRawEntry(Boolean memoryStorageRawEntry) {
+        this.memoryStorageRawEntry = memoryStorageRawEntry;
+    }
+
+    public Integer getTsdbSnapshotInterval() {
+        return tsdbSnapshotInterval;
+    }
+
+    public void setTsdbSnapshotInterval(Integer tsdbSnapshotInterval) {
+        this.tsdbSnapshotInterval = tsdbSnapshotInterval;
+    }
+
+    public Integer getTsdbSnapshotExpire() {
+        return tsdbSnapshotExpire;
+    }
+
+    public void setTsdbSnapshotExpire(Integer tsdbSnapshotExpire) {
+        this.tsdbSnapshotExpire = tsdbSnapshotExpire;
+    }
+
     public String toString() {
         return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
     }
diff --git a/instance/pom.xml b/instance/pom.xml
index a73b43fe..2a4da45c 100644
--- a/instance/pom.xml
+++ b/instance/pom.xml
@@ -3,7 +3,7 @@
 	
 		com.alibaba.otter
 		canal
-		1.1.1-SNAPSHOT
+		1.1.3-SNAPSHOT
 		../pom.xml
 	
 	com.alibaba.otter
diff --git a/instance/spring/pom.xml b/instance/spring/pom.xml
index 929fc273..bea5d37d 100644
--- a/instance/spring/pom.xml
+++ b/instance/spring/pom.xml
@@ -3,7 +3,7 @@
 	
 		com.alibaba.otter
 		canal
-		1.1.1-SNAPSHOT
+		1.1.3-SNAPSHOT
 		../../pom.xml
 	
 	com.alibaba.otter
diff --git a/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/CanalInstanceWithSpring.java b/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/CanalInstanceWithSpring.java
index 79693c4c..69672a6c 100644
--- a/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/CanalInstanceWithSpring.java
+++ b/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/CanalInstanceWithSpring.java
@@ -2,6 +2,8 @@ package com.alibaba.otter.canal.instance.spring;
 
 import java.util.List;
 
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.core.CanalMQConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -56,4 +58,8 @@ public class CanalInstanceWithSpring extends AbstractCanalInstance {
         this.alarmHandler = alarmHandler;
     }
 
+    public void setMqConfig(CanalMQConfig mqConfig){
+        this.mqConfig = mqConfig;
+    }
+
 }
diff --git a/instance/spring/src/test/resources/retl/instance.properties b/instance/spring/src/test/resources/retl/instance.properties
index 62ba5c4c..afc34305 100644
--- a/instance/spring/src/test/resources/retl/instance.properties
+++ b/instance/spring/src/test/resources/retl/instance.properties
@@ -27,9 +27,12 @@ canal.instance.master2.timestamp =
 
 # username/password
 canal.instance.dbUsername = xxxxx
-canal.instance.dbPassword = xxxxx
+canal.instance.dbPassword=cZozNf1mzW6EQLGO2q9u99619xbZLO0fbua3EX08r4BWNXb8lAt1aHrTEOBttd6UY8Vnuc0easlVXZDdLtt8BQ==
+canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==
 canal.instance.defaultDatabaseName =
 canal.instance.connectionCharset = UTF-8
+# enable druid Decrypt database password
+canal.instance.enableDruid=true
 
 # table regex
 canal.instance.filter.regex = .*\\..*
diff --git a/meta/pom.xml b/meta/pom.xml
index 98788116..002f6231 100644
--- a/meta/pom.xml
+++ b/meta/pom.xml
@@ -3,7 +3,7 @@
 	
 		com.alibaba.otter
 		canal
-		1.1.1-SNAPSHOT
+		1.1.3-SNAPSHOT
 		../pom.xml
 	
 	com.alibaba.otter
diff --git a/meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java b/meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java
index 38b459c9..2e3fdde5 100644
--- a/meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java
+++ b/meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java
@@ -1,329 +1,334 @@
-package com.alibaba.otter.canal.meta;
-
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.I0Itec.zkclient.exception.ZkNoNodeException;
-import org.I0Itec.zkclient.exception.ZkNodeExistsException;
-import org.apache.commons.lang.StringUtils;
-import org.springframework.util.Assert;
-import org.springframework.util.CollectionUtils;
-
-import com.alibaba.fastjson.serializer.SerializerFeature;
-import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
-import com.alibaba.otter.canal.common.utils.JsonUtils;
-import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
-import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
-import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
-import com.alibaba.otter.canal.protocol.ClientIdentity;
-import com.alibaba.otter.canal.protocol.position.Position;
-import com.alibaba.otter.canal.protocol.position.PositionRange;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- * zk 版本的 canal manager， 存储结构：
- * 
- * 
      - * /otter
- *    canal
- *      destinations
- *        dest1 
- *          client1
- *            filter
- *            batch_mark
- *              1
- *              2
- *              3
- * 
      
- * 
- * @author zebin.xuzb @ 2012-6-21
- * @author jianghang
- * @version 1.0.0
- */
-public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements CanalMetaManager {
-
-    private static final String ENCODE = "UTF-8";
-    private ZkClientx           zkClientx;
-
-    public void start() {
-        super.start();
-
-        Assert.notNull(zkClientx);
-    }
-
-    public void stop() {
-        super.stop();
-    }
-
-    public void subscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
-        String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
-            clientIdentity.getClientId());
-
-        try {
-            zkClientx.createPersistent(path, true);
-        } catch (ZkNodeExistsException e) {
-            // ignore
-        }
-        if (clientIdentity.hasFilter()) {
-            String filterPath = ZookeeperPathUtils.getFilterPath(clientIdentity.getDestination(),
-                clientIdentity.getClientId());
-
-            byte[] bytes = null;
-            try {
-                bytes = clientIdentity.getFilter().getBytes(ENCODE);
-            } catch (UnsupportedEncodingException e) {
-                throw new CanalMetaManagerException(e);
-            }
-
-            try {
-                zkClientx.createPersistent(filterPath, bytes);
-            } catch (ZkNodeExistsException e) {
-                // ignore
-                zkClientx.writeData(filterPath, bytes);
-            }
-        }
-    }
-
-    public boolean hasSubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
-        String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
-            clientIdentity.getClientId());
-        return zkClientx.exists(path);
-    }
-
-    public void unsubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
-        String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
-            clientIdentity.getClientId());
-        zkClientx.deleteRecursive(path); // 递归删除所有信息
-    }
-
-    public List listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
-        String path = ZookeeperPathUtils.getDestinationPath(destination);
-        List childs = null;
-        try {
-            childs = zkClientx.getChildren(path);
-        } catch (ZkNoNodeException e) {
-            // ignore
-        }
-
-        if (CollectionUtils.isEmpty(childs)) {
-            return new ArrayList();
-        }
-        List clientIds = new ArrayList();
-        for (String child : childs) {
-            if (StringUtils.isNumeric(child)) {
-                clientIds.add(ZookeeperPathUtils.getClientId(child));
-            }
-        }
-
-        Collections.sort(clientIds); // 进行一个排序
-        List clientIdentities = Lists.newArrayList();
-        for (Short clientId : clientIds) {
-            path = ZookeeperPathUtils.getFilterPath(destination, clientId);
-            byte[] bytes = zkClientx.readData(path, true);
-            String filter = null;
-            if (bytes != null) {
-                try {
-                    filter = new String(bytes, ENCODE);
-                } catch (UnsupportedEncodingException e) {
-                    throw new CanalMetaManagerException(e);
-                }
-            }
-            clientIdentities.add(new ClientIdentity(destination, clientId, filter));
-        }
-
-        return clientIdentities;
-    }
-
-    public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
-        String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
-
-        byte[] data = zkClientx.readData(path, true);
-        if (data == null || data.length == 0) {
-            return null;
-        }
-
-        return JsonUtils.unmarshalFromByte(data, Position.class);
-    }
-
-    public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
-        String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
-        byte[] data = JsonUtils.marshalToByte(position, SerializerFeature.WriteClassName);
-        try {
-            zkClientx.writeData(path, data);
-        } catch (ZkNoNodeException e) {
-            zkClientx.createPersistent(path, data, true);// 第一次节点不存在，则尝试重建
-        }
-    }
-
-    public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
-        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
-        byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
-        String batchPath = zkClientx.createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR,
-            data,
-            true);
-        String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR);
-        return ZookeeperPathUtils.getBatchMarkId(batchIdString);
-    }
-
-    public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange, Long batchId)
-                                                                                                  throws CanalMetaManagerException {
-        String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(),
-            clientIdentity.getClientId(),
-            batchId);
-        byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
-        zkClientx.createPersistent(path, data, true);
-    }
-
-    public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
-        String batchsPath = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
-            clientIdentity.getClientId());
-        List nodes = zkClientx.getChildren(batchsPath);
-        if (CollectionUtils.isEmpty(nodes)) {
-            // 没有batch记录
-            return null;
-        }
-
-        // 找到最小的Id
-        ArrayList batchIds = new ArrayList(nodes.size());
-        for (String batchIdString : nodes) {
-            batchIds.add(Long.valueOf(batchIdString));
-        }
-        Long minBatchId = Collections.min(batchIds);
-        if (!minBatchId.equals(batchId)) {
-            // 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
-            throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId, minBatchId));
-        }
-
-        if (!batchIds.contains(batchId)) {
-            // 不存在对应的batchId
-            return null;
-        }
-        PositionRange positionRange = getBatch(clientIdentity, batchId);
-        if (positionRange != null) {
-            String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(),
-                clientIdentity.getClientId(),
-                batchId);
-            zkClientx.delete(path);
-        }
-
-        return positionRange;
-    }
-
-    public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
-        String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(),
-            clientIdentity.getClientId(),
-            batchId);
-        byte[] data = zkClientx.readData(path, true);
-        if (data == null) {
-            return null;
-        }
-
-        PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
-        return positionRange;
-    }
-
-    public void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
-        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
-        List batchChilds = zkClientx.getChildren(path);
-
-        for (String batchChild : batchChilds) {
-            String batchPath = path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + batchChild;
-            zkClientx.delete(batchPath);
-        }
-    }
-
-    public PositionRange getLastestBatch(ClientIdentity clientIdentity) {
-        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
-        List nodes = null;
-        try {
-            nodes = zkClientx.getChildren(path);
-        } catch (ZkNoNodeException e) {
-            // ignore
-        }
-
-        if (CollectionUtils.isEmpty(nodes)) {
-            return null;
-        }
-        // 找到最大的Id
-        ArrayList batchIds = new ArrayList(nodes.size());
-        for (String batchIdString : nodes) {
-            batchIds.add(Long.valueOf(batchIdString));
-        }
-        Long maxBatchId = Collections.max(batchIds);
-        PositionRange result = getBatch(clientIdentity, maxBatchId);
-        if (result == null) { // 出现为null，说明zk节点有变化，重新获取
-            return getLastestBatch(clientIdentity);
-        } else {
-            return result;
-        }
-    }
-
-    public PositionRange getFirstBatch(ClientIdentity clientIdentity) {
-        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
-        List nodes = null;
-        try {
-            nodes = zkClientx.getChildren(path);
-        } catch (ZkNoNodeException e) {
-            // ignore
-        }
-
-        if (CollectionUtils.isEmpty(nodes)) {
-            return null;
-        }
-        // 找到最小的Id
-        ArrayList batchIds = new ArrayList(nodes.size());
-        for (String batchIdString : nodes) {
-            batchIds.add(Long.valueOf(batchIdString));
-        }
-        Long minBatchId = Collections.min(batchIds);
-        PositionRange result = getBatch(clientIdentity, minBatchId);
-        if (result == null) { // 出现为null，说明zk节点有变化，重新获取
-            return getFirstBatch(clientIdentity);
-        } else {
-            return result;
-        }
-    }
-
-    public Map listAllBatchs(ClientIdentity clientIdentity) {
-        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
-        List nodes = null;
-        try {
-            nodes = zkClientx.getChildren(path);
-        } catch (ZkNoNodeException e) {
-            // ignore
-        }
-
-        if (CollectionUtils.isEmpty(nodes)) {
-            return Maps.newHashMap();
-        }
-        // 找到最大的Id
-        ArrayList batchIds = new ArrayList(nodes.size());
-        for (String batchIdString : nodes) {
-            batchIds.add(Long.valueOf(batchIdString));
-        }
-
-        Collections.sort(batchIds); // 从小到大排序
-        Map positionRanges = Maps.newLinkedHashMap();
-        for (Long batchId : batchIds) {
-            PositionRange result = getBatch(clientIdentity, batchId);
-            if (result == null) {// 出现为null，说明zk节点有变化，重新获取
-                return listAllBatchs(clientIdentity);
-            } else {
-                positionRanges.put(batchId, result);
-            }
-        }
-
-        return positionRanges;
-    }
-
-    // =========== setter ==========
-
-    public void setZkClientx(ZkClientx zkClientx) {
-        this.zkClientx = zkClientx;
-    }
-
-}
+package com.alibaba.otter.canal.meta;
+
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.I0Itec.zkclient.exception.ZkNodeExistsException;
+import org.apache.commons.lang.StringUtils;
+import org.springframework.util.Assert;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.common.utils.JsonUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+/**
+ * zk 版本的 canal manager， 存储结构：
+ *
+ * 
      + * /otter
+ *    canal
+ *      destinations
+ *        dest1
+ *          client1
+ *            filter
+ *            batch_mark
+ *              1
+ *              2
+ *              3
+ * 
      
+ *
+ * @author zebin.xuzb @ 2012-6-21
+ * @author jianghang
+ * @version 1.0.0
+ */
+public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements CanalMetaManager {
+
+    private static final String ENCODE = "UTF-8";
+    private ZkClientx           zkClientx;
+
+    public void start() {
+        super.start();
+
+        Assert.notNull(zkClientx);
+    }
+
+    public void stop() {
+        zkClientx = null; //关闭时置空
+        super.stop();
+    }
+
+    public void subscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
+            clientIdentity.getClientId());
+
+        try {
+            zkClientx.createPersistent(path, true);
+        } catch (ZkNodeExistsException e) {
+            // ignore
+        }
+        if (clientIdentity.hasFilter()) {
+            String filterPath = ZookeeperPathUtils.getFilterPath(clientIdentity.getDestination(),
+                clientIdentity.getClientId());
+
+            byte[] bytes = null;
+            try {
+                bytes = clientIdentity.getFilter().getBytes(ENCODE);
+            } catch (UnsupportedEncodingException e) {
+                throw new CanalMetaManagerException(e);
+            }
+
+            try {
+                zkClientx.createPersistent(filterPath, bytes);
+            } catch (ZkNodeExistsException e) {
+                // ignore
+                zkClientx.writeData(filterPath, bytes);
+            }
+        }
+    }
+
+    public boolean hasSubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
+            clientIdentity.getClientId());
+        return zkClientx.exists(path);
+    }
+
+    public void unsubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
+            clientIdentity.getClientId());
+        zkClientx.deleteRecursive(path); // 递归删除所有信息
+    }
+
+    public List listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
+        if (zkClientx == null) { //重新加载时可能为空
+            return new ArrayList();
+        }
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        List childs = null;
+        try {
+            childs = zkClientx.getChildren(path);
+        } catch (ZkNoNodeException e) {
+            // ignore
+        }
+
+        if (CollectionUtils.isEmpty(childs)) {
+            return new ArrayList();
+        }
+        List clientIds = new ArrayList();
+        for (String child : childs) {
+            if (StringUtils.isNumeric(child)) {
+                clientIds.add(ZookeeperPathUtils.getClientId(child));
+            }
+        }
+
+        Collections.sort(clientIds); // 进行一个排序
+        List clientIdentities = Lists.newArrayList();
+        for (Short clientId : clientIds) {
+            path = ZookeeperPathUtils.getFilterPath(destination, clientId);
+            byte[] bytes = zkClientx.readData(path, true);
+            String filter = null;
+            if (bytes != null) {
+                try {
+                    filter = new String(bytes, ENCODE);
+                } catch (UnsupportedEncodingException e) {
+                    throw new CanalMetaManagerException(e);
+                }
+            }
+            clientIdentities.add(new ClientIdentity(destination, clientId, filter));
+        }
+
+        return clientIdentities;
+    }
+
+    public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
+
+        byte[] data = zkClientx.readData(path, true);
+        if (data == null || data.length == 0) {
+            return null;
+        }
+
+        return JsonUtils.unmarshalFromByte(data, Position.class);
+    }
+
+    public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
+        byte[] data = JsonUtils.marshalToByte(position, SerializerFeature.WriteClassName);
+        try {
+            zkClientx.writeData(path, data);
+        } catch (ZkNoNodeException e) {
+            zkClientx.createPersistent(path, data, true);// 第一次节点不存在，则尝试重建
+        }
+    }
+
+    public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
+            clientIdentity.getClientId());
+        byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
+        String batchPath = zkClientx
+            .createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR, data, true);
+        String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR);
+        return ZookeeperPathUtils.getBatchMarkId(batchIdString);
+    }
+
+    public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange,
+                         Long batchId) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils
+            .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
+        byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
+        zkClientx.createPersistent(path, data, true);
+    }
+
+    public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
+        String batchsPath = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
+            clientIdentity.getClientId());
+        List nodes = zkClientx.getChildren(batchsPath);
+        if (CollectionUtils.isEmpty(nodes)) {
+            // 没有batch记录
+            return null;
+        }
+
+        // 找到最小的Id
+        ArrayList batchIds = new ArrayList(nodes.size());
+        for (String batchIdString : nodes) {
+            batchIds.add(Long.valueOf(batchIdString));
+        }
+        Long minBatchId = Collections.min(batchIds);
+        if (!minBatchId.equals(batchId)) {
+            // 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
+            throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId, minBatchId));
+        }
+
+        if (!batchIds.contains(batchId)) {
+            // 不存在对应的batchId
+            return null;
+        }
+        PositionRange positionRange = getBatch(clientIdentity, batchId);
+        if (positionRange != null) {
+            String path = ZookeeperPathUtils
+                .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
+            zkClientx.delete(path);
+        }
+
+        return positionRange;
+    }
+
+    public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils
+            .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
+        byte[] data = zkClientx.readData(path, true);
+        if (data == null) {
+            return null;
+        }
+
+        PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
+        return positionRange;
+    }
+
+    public void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
+            clientIdentity.getClientId());
+        List batchChilds = zkClientx.getChildren(path);
+
+        for (String batchChild : batchChilds) {
+            String batchPath = path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + batchChild;
+            zkClientx.delete(batchPath);
+        }
+    }
+
+    public PositionRange getLastestBatch(ClientIdentity clientIdentity) {
+        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
+            clientIdentity.getClientId());
+        List nodes = null;
+        try {
+            nodes = zkClientx.getChildren(path);
+        } catch (ZkNoNodeException e) {
+            // ignore
+        }
+
+        if (CollectionUtils.isEmpty(nodes)) {
+            return null;
+        }
+        // 找到最大的Id
+        ArrayList batchIds = new ArrayList(nodes.size());
+        for (String batchIdString : nodes) {
+            batchIds.add(Long.valueOf(batchIdString));
+        }
+        Long maxBatchId = Collections.max(batchIds);
+        PositionRange result = getBatch(clientIdentity, maxBatchId);
+        if (result == null) { // 出现为null，说明zk节点有变化，重新获取
+            return getLastestBatch(clientIdentity);
+        } else {
+            return result;
+        }
+    }
+
+    public PositionRange getFirstBatch(ClientIdentity clientIdentity) {
+        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
+            clientIdentity.getClientId());
+        List nodes = null;
+        try {
+            nodes = zkClientx.getChildren(path);
+        } catch (ZkNoNodeException e) {
+            // ignore
+        }
+
+        if (CollectionUtils.isEmpty(nodes)) {
+            return null;
+        }
+        // 找到最小的Id
+        ArrayList batchIds = new ArrayList(nodes.size());
+        for (String batchIdString : nodes) {
+            batchIds.add(Long.valueOf(batchIdString));
+        }
+        Long minBatchId = Collections.min(batchIds);
+        PositionRange result = getBatch(clientIdentity, minBatchId);
+        if (result == null) { // 出现为null，说明zk节点有变化，重新获取
+            return getFirstBatch(clientIdentity);
+        } else {
+            return result;
+        }
+    }
+
+    public Map listAllBatchs(ClientIdentity clientIdentity) {
+        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
+            clientIdentity.getClientId());
+        List nodes = null;
+        try {
+            nodes = zkClientx.getChildren(path);
+        } catch (ZkNoNodeException e) {
+            // ignore
+        }
+
+        if (CollectionUtils.isEmpty(nodes)) {
+            return Maps.newHashMap();
+        }
+        // 找到最大的Id
+        ArrayList batchIds = new ArrayList(nodes.size());
+        for (String batchIdString : nodes) {
+            batchIds.add(Long.valueOf(batchIdString));
+        }
+
+        Collections.sort(batchIds); // 从小到大排序
+        Map positionRanges = Maps.newLinkedHashMap();
+        for (Long batchId : batchIds) {
+            PositionRange result = getBatch(clientIdentity, batchId);
+            if (result == null) {// 出现为null，说明zk节点有变化，重新获取
+                return listAllBatchs(clientIdentity);
+            } else {
+                positionRanges.put(batchId, result);
+            }
+        }
+
+        return positionRanges;
+    }
+
+    // =========== setter ==========
+
+    public void setZkClientx(ZkClientx zkClientx) {
+        this.zkClientx = zkClientx;
+    }
+
+}
diff --git a/parse/pom.xml b/parse/pom.xml
index 8e47215b..985d5de9 100644
--- a/parse/pom.xml
+++ b/parse/pom.xml
@@ -3,7 +3,7 @@
 	
 		com.alibaba.otter
 		canal
-		1.1.1-SNAPSHOT
+		1.1.3-SNAPSHOT
 		../pom.xml
 	
 	canal.parse
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
index e1f80d76..cd6e25be 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
@@ -181,6 +181,8 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
                             serverId = queryServerId;
                         }
                         // 4. 获取最后的位置信息
+                        long start = System.currentTimeMillis();
+                        logger.warn("---> begin to find start position, it will be long time for reset or first position");
                         EntryPosition position = findStartPosition(erosaConnection);
                         final EntryPosition startPosition = position;
                         if (startPosition == null) {
@@ -191,7 +193,10 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
                             throw new CanalParseException("can't find init table meta for " + destination
                                                           + " with position : " + startPosition);
                         }
-                        logger.warn("find start position : {}", startPosition.toString());
+                        long end = System.currentTimeMillis();
+                        logger.warn("---> find start position successfully, {}", startPosition.toString() + " cost : "
+                                                                                 + (end - start)
+                                                                                 + "ms , the next step is binlog dump");
                         // 重新链接，因为在找position过程中可能有状态，需要断开后重建
                         erosaConnection.reconnect();
 
@@ -317,10 +322,10 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
                     eventSink.interrupt();
                     transactionBuffer.reset();// 重置一下缓冲队列，重新记录数据
                     binlogParser.reset();// 重新置位
-                    if (multiStageCoprocessor != null) {
+                    if (multiStageCoprocessor != null && multiStageCoprocessor.isStart()) {
                         // 处理 RejectedExecutionException
                         try {
-                            multiStageCoprocessor.reset();
+                            multiStageCoprocessor.stop();
                         } catch (Throwable t) {
                             logger.debug("multi processor rejected:", t);
                         }
@@ -353,7 +358,11 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
         eventSink.interrupt();
 
         if (multiStageCoprocessor != null && multiStageCoprocessor.isStart()) {
-            multiStageCoprocessor.stop();
+            try {
+                multiStageCoprocessor.stop();
+            } catch (Throwable t) {
+                logger.debug("multi processor rejected:", t);
+            }
         }
 
         try {
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java
index 5355e2e1..758e8173 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java
@@ -20,7 +20,7 @@ public interface ErosaConnection {
     /**
      * 用于快速数据查找,和dump的区别在于，seek会只给出部分的数据
      */
-    public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException;
+    public void seek(String binlogfilename, Long binlogPosition, String gtid, SinkFunction func) throws IOException;
 
     public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException;
 
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java
index 4ab68e39..364230d5 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java
@@ -25,6 +25,4 @@ public interface MultiStageCoprocessor extends CanalLifeCycle {
     public boolean publish(LogBuffer buffer);
 
     public boolean publish(LogEvent event);
-
-    public void reset();
 }
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java
index 00e302e6..29a66fd2 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java
@@ -3,9 +3,6 @@ package com.alibaba.otter.canal.parse.inbound.mysql;
 import java.nio.charset.Charset;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.alibaba.otter.canal.filter.CanalEventFilter;
 import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
 import com.alibaba.otter.canal.parse.CanalEventParser;
@@ -14,6 +11,7 @@ import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
 import com.alibaba.otter.canal.parse.inbound.BinlogParser;
 import com.alibaba.otter.canal.parse.inbound.MultiStageCoprocessor;
 import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert;
+import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta;
 import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DefaultTableMetaTSDBFactory;
 import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB;
 import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDBFactory;
@@ -21,11 +19,12 @@ import com.alibaba.otter.canal.protocol.position.EntryPosition;
 
 public abstract class AbstractMysqlEventParser extends AbstractEventParser {
 
-    protected final Logger         logger                    = LoggerFactory.getLogger(this.getClass());
     protected static final long    BINLOG_START_OFFEST       = 4L;
 
     protected TableMetaTSDBFactory tableMetaTSDBFactory      = new DefaultTableMetaTSDBFactory();
     protected boolean              enableTsdb                = false;
+    protected int                  tsdbSnapshotInterval      = 24;
+    protected int                  tsdbSnapshotExpire        = 360;
     protected String               tsdbSpringXml;
     protected TableMetaTSDB        tableMetaTSDB;
 
@@ -58,6 +57,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
         convert.setFilterQueryDdl(filterQueryDdl);
         convert.setFilterRows(filterRows);
         convert.setFilterTableError(filterTableError);
+        convert.setUseDruidDdlFilter(useDruidDdlFilter);
         return convert;
     }
 
@@ -65,8 +65,29 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
         super.setEventFilter(eventFilter);
 
         // 触发一下filter变更
-        if (eventFilter != null && eventFilter instanceof AviaterRegexFilter && binlogParser instanceof LogEventConvert) {
-            ((LogEventConvert) binlogParser).setNameFilter((AviaterRegexFilter) eventFilter);
+        if (eventFilter != null && eventFilter instanceof AviaterRegexFilter) {
+            if (binlogParser instanceof LogEventConvert) {
+                ((LogEventConvert) binlogParser).setNameFilter((AviaterRegexFilter) eventFilter);
+            }
+
+            if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
+                ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
+            }
+        }
+    }
+
+    public void setEventBlackFilter(CanalEventFilter eventBlackFilter) {
+        super.setEventBlackFilter(eventBlackFilter);
+
+        // 触发一下filter变更
+        if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter) {
+            if (binlogParser instanceof LogEventConvert) {
+                ((LogEventConvert) binlogParser).setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
+            }
+
+            if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
+                ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
+            }
         }
     }
 
@@ -116,16 +137,6 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
         super.stop();
     }
 
-    public void setEventBlackFilter(CanalEventFilter eventBlackFilter) {
-        super.setEventBlackFilter(eventBlackFilter);
-
-        // 触发一下filter变更
-        if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter
-            && binlogParser instanceof LogEventConvert) {
-            ((LogEventConvert) binlogParser).setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
-        }
-    }
-
     protected MultiStageCoprocessor buildMultiStageCoprocessor() {
         MysqlMultiStageCoprocessor mysqlMultiStageCoprocessor = new MysqlMultiStageCoprocessor(parallelBufferSize,
             parallelThreadSize,
@@ -210,4 +221,20 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
         return this.receivedBinlogBytes;
     }
 
+    public int getTsdbSnapshotInterval() {
+        return tsdbSnapshotInterval;
+    }
+
+    public void setTsdbSnapshotInterval(int tsdbSnapshotInterval) {
+        this.tsdbSnapshotInterval = tsdbSnapshotInterval;
+    }
+
+    public int getTsdbSnapshotExpire() {
+        return tsdbSnapshotExpire;
+    }
+
+    public void setTsdbSnapshotExpire(int tsdbSnapshotExpire) {
+        this.tsdbSnapshotExpire = tsdbSnapshotExpire;
+    }
+
 }
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
index 5cf5bb9d..411a19d6 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
@@ -75,7 +75,7 @@ public class LocalBinLogConnection implements ErosaConnection {
         return running;
     }
 
-    public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
+    public void seek(String binlogfilename, Long binlogPosition, String gtid, SinkFunction func) throws IOException {
     }
 
     public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java
index ea6b0365..db235d9b 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java
@@ -55,6 +55,9 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements
             ((DatabaseTableMeta) tableMetaTSDB).setConnection(metaConnection);
             ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
             ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
+            ((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval);
+            ((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire);
+            ((DatabaseTableMeta) tableMetaTSDB).init(destination);
         }
 
         tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB);
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
index 4348dc29..338bd962 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
@@ -10,6 +10,7 @@ import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
+import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.math.NumberUtils;
 import org.slf4j.Logger;
@@ -37,21 +38,23 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogContext;
 import com.taobao.tddl.dbsync.binlog.LogDecoder;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
+import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
 
 public class MysqlConnection implements ErosaConnection {
 
-    private static final Logger logger      = LoggerFactory.getLogger(MysqlConnection.class);
+    private static final Logger logger         = LoggerFactory.getLogger(MysqlConnection.class);
 
     private MysqlConnector      connector;
     private long                slaveId;
-    private Charset             charset     = Charset.forName("UTF-8");
+    private Charset             charset        = Charset.forName("UTF-8");
     private BinlogFormat        binlogFormat;
     private BinlogImage         binlogImage;
 
     // tsdb releated
     private AuthenticationInfo  authInfo;
-    protected int               connTimeout = 5 * 1000;                                      // 5秒
-    protected int               soTimeout   = 60 * 60 * 1000;                                // 1小时
+    protected int               connTimeout    = 5 * 1000;                                      // 5秒
+    protected int               soTimeout      = 60 * 60 * 1000;                                // 1小时
+    private int                 binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
     // dump binlog bytes, 暂不包括meta与TSDB
     private AtomicLong          receivedBinlogBytes;
 
@@ -116,9 +119,9 @@ public class MysqlConnection implements ErosaConnection {
     /**
      * 加速主备切换时的查找速度，做一些特殊优化，比如只解析事务头或者尾
      */
-    public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
+    public void seek(String binlogfilename, Long binlogPosition, String gtid, SinkFunction func) throws IOException {
         updateSettings();
-
+        loadBinlogChecksum();
         sendBinlogDump(binlogfilename, binlogPosition);
         DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
         fetcher.start(connector.getChannel());
@@ -128,6 +131,14 @@ public class MysqlConnection implements ErosaConnection {
         decoder.handle(LogEvent.QUERY_EVENT);
         decoder.handle(LogEvent.XID_EVENT);
         LogContext context = new LogContext();
+        // 若entry position存在gtid，则使用传入的gtid作为gtidSet 拼接的标准,否则同时开启gtid和tsdb时，会导致丢失gtid
+        // 而当源端数据库gtid 有purged时会有如下类似报错
+        // 'errno = 1236, sqlstate = HY000 errmsg = The slave is connecting using CHANGE MASTER TO MASTER_AUTO_POSITION = 1 ...
+        if (StringUtils.isNotEmpty(gtid)) {
+            decoder.handle(LogEvent.GTID_LOG_EVENT);
+            context.setGtidSet(MysqlGTIDSet.parse(gtid));
+        }
+        context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
         while (fetcher.fetch()) {
             accumulateReceivedBytes(fetcher.limit());
             LogEvent event = null;
@@ -145,12 +156,14 @@ public class MysqlConnection implements ErosaConnection {
 
     public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
         updateSettings();
+        loadBinlogChecksum();
         sendRegisterSlave();
         sendBinlogDump(binlogfilename, binlogPosition);
         DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
         fetcher.start(connector.getChannel());
         LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
         LogContext context = new LogContext();
+        context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
         while (fetcher.fetch()) {
             accumulateReceivedBytes(fetcher.limit());
             LogEvent event = null;
@@ -173,6 +186,7 @@ public class MysqlConnection implements ErosaConnection {
     @Override
     public void dump(GTIDSet gtidSet, SinkFunction func) throws IOException {
         updateSettings();
+        loadBinlogChecksum();
         sendBinlogDumpGTID(gtidSet);
 
         DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
@@ -180,6 +194,7 @@ public class MysqlConnection implements ErosaConnection {
             fetcher.start(connector.getChannel());
             LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
             LogContext context = new LogContext();
+            context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
             // fix bug: #890 将gtid传输至context中，供decode使用
             context.setGtidSet(gtidSet);
             while (fetcher.fetch()) {
@@ -207,9 +222,11 @@ public class MysqlConnection implements ErosaConnection {
     @Override
     public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException {
         updateSettings();
+        loadBinlogChecksum();
         sendRegisterSlave();
         sendBinlogDump(binlogfilename, binlogPosition);
         ((MysqlMultiStageCoprocessor) coprocessor).setConnection(this);
+        ((MysqlMultiStageCoprocessor) coprocessor).setBinlogChecksum(binlogChecksum);
         DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
         try {
             fetcher.start(connector.getChannel());
@@ -234,9 +251,10 @@ public class MysqlConnection implements ErosaConnection {
     @Override
     public void dump(GTIDSet gtidSet, MultiStageCoprocessor coprocessor) throws IOException {
         updateSettings();
+        loadBinlogChecksum();
         sendBinlogDumpGTID(gtidSet);
-
         ((MysqlMultiStageCoprocessor) coprocessor).setConnection(this);
+        ((MysqlMultiStageCoprocessor) coprocessor).setBinlogChecksum(binlogChecksum);
         DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
         try {
             fetcher.start(connector.getChannel());
@@ -396,7 +414,8 @@ public class MysqlConnection implements ErosaConnection {
             // 如果不设置会出现错误： Slave can not handle replication events with the
             // checksum that master is configured to log
             // 但也不能乱设置，需要和mysql server的checksum配置一致，不然RotateLogEvent会出现乱码
-            update("set @master_binlog_checksum= '@@global.binlog_checksum'");
+            // '@@global.binlog_checksum'需要去掉单引号,在mysql 5.6.29下导致master退出
+            update("set @master_binlog_checksum= @@global.binlog_checksum");
         } catch (Exception e) {
             logger.warn("update master_binlog_checksum failed", e);
         }
@@ -482,6 +501,31 @@ public class MysqlConnection implements ErosaConnection {
         }
     }
 
+    /**
+     * 获取主库checksum信息
+     * 
+     * 
      +     * mariadb区别于mysql会在binlog的第一个事件Rotate_Event里也会采用checksum逻辑,而mysql是在第二个binlog事件之后才感知是否需要处理checksum
+     * 导致maraidb只要是开启checksum就会出现binlog文件名解析乱码
+     * fixed issue : https://github.com/alibaba/canal/issues/1081
+     * 
      
+     */
+    private void loadBinlogChecksum() {
+        ResultSetPacket rs = null;
+        try {
+            rs = query("select @@global.binlog_checksum");
+            List columnValues = rs.getFieldValues();
+            if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0).toUpperCase().equals("CRC32")) {
+                binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
+            } else {
+                binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
+            }
+        } catch (Throwable e) {
+            logger.error("", e);
+            binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
+        }
+    }
+
     private void accumulateReceivedBytes(long x) {
         if (receivedBinlogBytes != null) {
             receivedBinlogBytes.addAndGet(x);
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
index a5c8d870..d2ce8356 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
@@ -8,7 +8,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TimerTask;
-import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.lang.StringUtils;
 import org.springframework.util.CollectionUtils;
@@ -36,12 +35,12 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
  * 基于向mysql server复制binlog实现
- * 
+ *
  * 
        * 1. 自身不控制mysql主备切换，由ha机制来控制. 比如接入tddl/cobar/自身心跳包成功率
  * 2. 切换机制
  * 
      
- * 
+ *
  * @author jianghang 2012-6-21 下午04:06:32
  * @version 1.0.0
  */
@@ -49,27 +48,28 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
 
     private CanalHAController    haController                      = null;
 
-    private int                  defaultConnectionTimeoutInSeconds = 30;                // sotimeout
+    private int                  defaultConnectionTimeoutInSeconds = 30;       // sotimeout
     private int                  receiveBufferSize                 = 64 * 1024;
     private int                  sendBufferSize                    = 64 * 1024;
     // 数据库信息
-    protected AuthenticationInfo masterInfo;                                            // 主库
-    protected AuthenticationInfo standbyInfo;                                           // 备库
+    protected AuthenticationInfo masterInfo;                                   // 主库
+    protected AuthenticationInfo standbyInfo;                                  // 备库
     // binlog信息
     protected EntryPosition      masterPosition;
     protected EntryPosition      standbyPosition;
-    private long                 slaveId;                                               // 链接到mysql的slave
+    private long                 slaveId;                                      // 链接到mysql的slave
     // 心跳检查信息
-    private String               detectingSQL;                                          // 心跳sql
-    private MysqlConnection      metaConnection;                                        // 查询meta信息的链接
-    private TableMetaCache       tableMetaCache;                                        // 对应meta
-    private int                  fallbackIntervalInSeconds         = 60;                // 切换回退时间
-    private BinlogFormat[]       supportBinlogFormats;                                  // 支持的binlogFormat,如果设置会执行强校验
-    private BinlogImage[]        supportBinlogImages;                                   // 支持的binlogImage,如果设置会执行强校验
+    private String               detectingSQL;                                 // 心跳sql
+    private MysqlConnection      metaConnection;                               // 查询meta信息的链接
+    private TableMetaCache       tableMetaCache;                               // 对应meta
+    private int                  fallbackIntervalInSeconds         = 60;       // 切换回退时间
+    private BinlogFormat[]       supportBinlogFormats;                         // 支持的binlogFormat,如果设置会执行强校验
+    private BinlogImage[]        supportBinlogImages;                          // 支持的binlogImage,如果设置会执行强校验
 
     // update by yishun.chen,特殊异常处理参数
-    private int                  dumpErrorCount                    = 0;                 // binlogDump失败异常计数
-    private int                  dumpErrorCountThreshold           = 2;                 // binlogDump失败异常计数阀值
+    private int                  dumpErrorCount                    = 0;        // binlogDump失败异常计数
+    private int                  dumpErrorCountThreshold           = 2;        // binlogDump失败异常计数阀值
+    private boolean              rdsOssMode                        = false;
 
     protected ErosaConnection buildErosaConnection() {
         return buildMysqlConnection(this.runningInfo);
@@ -122,6 +122,9 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                 ((DatabaseTableMeta) tableMetaTSDB).setConnection(metaConnection);
                 ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
                 ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
+                ((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval);
+                ((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire);
+                ((DatabaseTableMeta) tableMetaTSDB).init(destination);
             }
 
             tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB);
@@ -132,6 +135,10 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
     protected void afterDump(ErosaConnection connection) {
         super.afterDump(connection);
 
+        if (connection == null) {
+            throw new CanalParseException("illegal connection is null");
+        }
+
         if (!(connection instanceof MysqlConnection)) {
             throw new CanalParseException("Unsupported connection type : " + connection.getClass().getSimpleName());
         }
@@ -201,7 +208,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
 
     /**
      * 心跳信息
-     * 
+     *
      * @author jianghang 2012-7-6 下午02:50:15
      * @version 1.0.0
      */
@@ -351,7 +358,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                 return logPosition.getPostion();
             }
 
-            if (StringUtils.isNotEmpty(masterPosition.getGtid())) {
+            if (masterPosition != null && StringUtils.isNotEmpty(masterPosition.getGtid())) {
                 return masterPosition;
             }
         }
@@ -492,6 +499,12 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                         dumpErrorCount = 0;
                         return findPosition;
                     }
+
+                    Long timestamp = logPosition.getPostion().getTimestamp();
+                    if (isRdsOssMode() && (timestamp != null && timestamp > 0)) {
+                        // 如果binlog位点不存在，并且属于timestamp不为空,可以返回null走到oss binlog处理
+                        return null;
+                    }
                 }
                 // 其余情况
                 logger.warn("prepare to find start position just last position\n {}",
@@ -514,7 +527,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
         // 针对开始的第一条为非Begin记录，需要从该binlog扫描
         final java.util.concurrent.atomic.AtomicLong preTransactionStartPosition = new java.util.concurrent.atomic.AtomicLong(0L);
         mysqlConnection.reconnect();
-        mysqlConnection.seek(entryPosition.getJournalName(), 4L, new SinkFunction() {
+        mysqlConnection.seek(entryPosition.getJournalName(), 4L, entryPosition.getGtid(), new SinkFunction() {
 
             private LogPosition lastPosition;
 
@@ -643,7 +656,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                 throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
             }
             EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
-            if (isGTIDMode && fields.size() > 4) {
+            if (isGTIDMode() && fields.size() > 4) {
                 endPosition.setGtid(fields.get(4));
             }
             return endPosition;
@@ -727,7 +740,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
         try {
             mysqlConnection.reconnect();
             // 开始遍历文件
-            mysqlConnection.seek(searchBinlogFile, 4L, new SinkFunction() {
+            mysqlConnection.seek(searchBinlogFile, 4L, endPosition.getGtid(), new SinkFunction() {
 
                 private LogPosition lastPosition;
 
@@ -738,20 +751,20 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                         if (justForPositionTimestamp && logPosition.getPostion() == null && event.getWhen() > 0) {
                             // 初始位点
                             entryPosition = new EntryPosition(searchBinlogFile,
-                                event.getLogPos(),
+                                event.getLogPos() - event.getEventLen(),
                                 event.getWhen() * 1000,
                                 event.getServerId());
+                            entryPosition.setGtid(event.getHeader().getGtidSetStr());
                             logPosition.setPostion(entryPosition);
                         }
 
-                        if (entry == null) {
-                            return true;
-                        }
-
-                        String logfilename = entry.getHeader().getLogfileName();
-                        Long logfileoffset = entry.getHeader().getLogfileOffset();
-                        Long logposTimestamp = entry.getHeader().getExecuteTime();
-                        Long serverId = entry.getHeader().getServerId();
+                        // 直接用event的位点来处理,解决一个binlog文件里没有任何事件导致死循环无法退出的问题
+                        String logfilename = event.getHeader().getLogFileName();
+                        // 记录的是binlog end offest,
+                        // 因为与其对比的offest是show master status里的end offest
+                        Long logfileoffset = event.getHeader().getLogPos();
+                        Long logposTimestamp = event.getHeader().getWhen() * 1000;
+                        Long serverId = event.getHeader().getServerId();
 
                         // 如果最小的一条记录都不满足条件，可直接退出
                         if (logposTimestamp >= startTimestamp) {
@@ -763,6 +776,10 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                             return false;
                         }
 
+                        if (entry == null) {
+                            return true;
+                        }
+
                         // 记录一下上一个事务结束的位置，即下一个事务的position
                         // position = current +
                         // data.length，代表该事务的下一条offest，避免多余的事务重复
@@ -773,6 +790,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                                     entryPosition);
                             }
                             logPosition.setPostion(entryPosition);
+                            entryPosition.setGtid(entry.getHeader().getGtid());
                         } else if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) {
                             // 当前事务开始位点
                             entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp, serverId);
@@ -780,6 +798,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                                 logger.debug("set {} to be pending start position before finding another proper one...",
                                     entryPosition);
                             }
+                            entryPosition.setGtid(entry.getHeader().getGtid());
                             logPosition.setPostion(entryPosition);
                         }
 
@@ -904,4 +923,12 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
         this.dumpErrorCountThreshold = dumpErrorCountThreshold;
     }
 
+    public boolean isRdsOssMode() {
+        return rdsOssMode;
+    }
+
+    public void setRdsOssMode(boolean rdsOssMode) {
+        this.rdsOssMode = rdsOssMode;
+    }
+
 }
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
index 99bb94f5..703de818 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.parse.inbound.mysql;
 
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.LockSupport;
 
@@ -32,6 +33,7 @@ import com.taobao.tddl.dbsync.binlog.LogContext;
 import com.taobao.tddl.dbsync.binlog.LogDecoder;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
@@ -51,20 +53,24 @@ import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
  */
 public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implements MultiStageCoprocessor {
 
-    private static final int             maxFullTimes = 10;
-    private LogEventConvert              logEventConvert;
-    private EventTransactionBuffer       transactionBuffer;
-    private ErosaConnection              connection;
+    private static final int                  maxFullTimes = 10;
+    private LogEventConvert                   logEventConvert;
+    private EventTransactionBuffer            transactionBuffer;
+    private ErosaConnection                   connection;
 
-    private int                          parserThreadCount;
-    private int                          ringBufferSize;
-    private RingBuffer     disruptorMsgBuffer;
-    private ExecutorService              parserExecutor;
-    private ExecutorService              stageExecutor;
-    private String                       destination;
-    private volatile CanalParseException exception;
-    private AtomicLong                   eventsPublishBlockingTime;
-    private GTIDSet                      gtidSet;
+    private int                               parserThreadCount;
+    private int                               ringBufferSize;
+    private RingBuffer          disruptorMsgBuffer;
+    private ExecutorService                   parserExecutor;
+    private ExecutorService                   stageExecutor;
+    private String                            destination;
+    private volatile CanalParseException      exception;
+    private AtomicLong                        eventsPublishBlockingTime;
+    private GTIDSet                           gtidSet;
+    private WorkerPool          workerPool;
+    private BatchEventProcessor simpleParserStage;
+    private BatchEventProcessor sinkStoreStage;
+    private LogContext                        logContext;
 
     public MysqlMultiStageCoprocessor(int ringBufferSize, int parserThreadCount, LogEventConvert logEventConvert,
                                       EventTransactionBuffer transactionBuffer, String destination){
@@ -91,9 +97,10 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
         SequenceBarrier sequenceBarrier = disruptorMsgBuffer.newBarrier();
         ExceptionHandler exceptionHandler = new SimpleFatalExceptionHandler();
         // stage 2
-        BatchEventProcessor simpleParserStage = new BatchEventProcessor(disruptorMsgBuffer,
+        this.logContext = new LogContext();
+        simpleParserStage = new BatchEventProcessor(disruptorMsgBuffer,
             sequenceBarrier,
-            new SimpleParserStage());
+            new SimpleParserStage(logContext));
         simpleParserStage.setExceptionHandler(exceptionHandler);
         disruptorMsgBuffer.addGatingSequences(simpleParserStage.getSequence());
 
@@ -103,7 +110,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
         for (int i = 0; i < parserThreadCount; i++) {
             workHandlers[i] = new DmlParserStage();
         }
-        WorkerPool workerPool = new WorkerPool(disruptorMsgBuffer,
+        workerPool = new WorkerPool(disruptorMsgBuffer,
             dmlParserSequenceBarrier,
             exceptionHandler,
             workHandlers);
@@ -112,7 +119,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
 
         // stage 4
         SequenceBarrier sinkSequenceBarrier = disruptorMsgBuffer.newBarrier(sequence);
-        BatchEventProcessor sinkStoreStage = new BatchEventProcessor(disruptorMsgBuffer,
+        sinkStoreStage = new BatchEventProcessor(disruptorMsgBuffer,
             sinkSequenceBarrier,
             new SinkStoreStage());
         sinkStoreStage.setExceptionHandler(exceptionHandler);
@@ -124,16 +131,40 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
         workerPool.start(parserExecutor);
     }
 
+    public void setBinlogChecksum(int binlogChecksum) {
+        if (binlogChecksum != LogEvent.BINLOG_CHECKSUM_ALG_OFF) {
+            logContext.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
+        }
+    }
+
     @Override
     public void stop() {
+        // fix bug #968，对于pool与
+        workerPool.halt();
+        simpleParserStage.halt();
+        sinkStoreStage.halt();
         try {
             parserExecutor.shutdownNow();
+            while (!parserExecutor.awaitTermination(1, TimeUnit.SECONDS)) {
+                if (parserExecutor.isShutdown() || parserExecutor.isTerminated()) {
+                    break;
+                }
+
+                parserExecutor.shutdownNow();
+            }
         } catch (Throwable e) {
             // ignore
         }
 
         try {
             stageExecutor.shutdownNow();
+            while (!stageExecutor.awaitTermination(1, TimeUnit.SECONDS)) {
+                if (stageExecutor.isShutdown() || stageExecutor.isTerminated()) {
+                    break;
+                }
+
+                stageExecutor.shutdownNow();
+            }
         } catch (Throwable e) {
             // ignore
         }
@@ -212,23 +243,14 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
 
     }
 
-    @Override
-    public void reset() {
-        if (isStart()) {
-            stop();
-        }
-
-        start();
-    }
-
     private class SimpleParserStage implements EventHandler, LifecycleAware {
 
         private LogDecoder decoder;
         private LogContext context;
 
-        public SimpleParserStage(){
+        public SimpleParserStage(LogContext context){
             decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
-            context = new LogContext();
+            this.context = context;
             if (gtidSet != null) {
                 context.setGtidSet(gtidSet);
             }
@@ -253,6 +275,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
                         needDmlParse = true;
                         break;
                     case LogEvent.UPDATE_ROWS_EVENT_V1:
+                    case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
                     case LogEvent.UPDATE_ROWS_EVENT:
                         tableMeta = logEventConvert.parseRowsEventForTableMeta((UpdateRowsLogEvent) logEvent);
                         needDmlParse = true;
@@ -455,4 +478,5 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
     public void setGtidSet(GTIDSet gtidSet) {
         this.gtidSet = gtidSet;
     }
+
 }
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
index 7fcd1a1f..5a18b9eb 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
@@ -120,6 +120,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
             case LogEvent.WRITE_ROWS_EVENT:
                 return parseRowsEvent((WriteRowsLogEvent) logEvent);
             case LogEvent.UPDATE_ROWS_EVENT_V1:
+            case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
             case LogEvent.UPDATE_ROWS_EVENT:
                 return parseRowsEvent((UpdateRowsLogEvent) logEvent);
             case LogEvent.DELETE_ROWS_EVENT_V1:
@@ -260,7 +261,9 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                 return null;
             }
 
-            if (!isSeek) {
+            boolean isDml = (type == EventType.INSERT || type == EventType.UPDATE || type == EventType.DELETE);
+
+            if (!isSeek && !isDml) {
                 // 使用新的表结构元数据管理方式
                 EntryPosition position = createPosition(event.getHeader());
                 tableMetaCache.apply(position, event.getDbName(), queryString, null);
@@ -268,8 +271,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
 
             Header header = createHeader(event.getHeader(), schemaName, tableName, type);
             RowChange.Builder rowChangeBuider = RowChange.newBuilder();
-            if (type != EventType.QUERY && type != EventType.INSERT && type != EventType.UPDATE
-                && type != EventType.DELETE) {
+            if (type != EventType.QUERY && !isDml) {
                 rowChangeBuider.setIsDdl(true);
             }
             rowChangeBuider.setSql(queryString);
@@ -503,7 +505,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
             int type = event.getHeader().getType();
             if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
                 eventType = EventType.INSERT;
-            } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
+            } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type
+                       || LogEvent.PARTIAL_UPDATE_ROWS_EVENT == type) {
                 eventType = EventType.UPDATE;
             } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
                 eventType = EventType.DELETE;
@@ -522,7 +525,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
 
             boolean tableError = false;
             int rowsCount = 0;
-            while (buffer.nextOneRow(columns)) {
+            while (buffer.nextOneRow(columns, false)) {
                 // 处理row记录
                 RowData.Builder rowDataBuilder = RowData.newBuilder();
                 if (EventType.INSERT == eventType) {
@@ -534,7 +537,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                 } else {
                     // update需要处理before/after
                     tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
-                    if (!buffer.nextOneRow(changeColumns)) {
+                    if (!buffer.nextOneRow(changeColumns, true)) {
                         rowChangeBuider.addRowDatas(rowDataBuilder.build());
                         break;
                     }
@@ -567,8 +570,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
     }
 
     private EntryPosition createPosition(LogHeader logHeader) {
-        return new EntryPosition(logHeader.getLogFileName(),
-            logHeader.getLogPos(),
+        return new EntryPosition(logHeader.getLogFileName(), logHeader.getLogPos() - logHeader.getEventLen(), // startPos
             logHeader.getWhen() * 1000L,
             logHeader.getServerId()); // 记录到秒
     }
@@ -577,7 +579,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                                 boolean isAfter, TableMeta tableMeta) throws UnsupportedEncodingException {
         int columnCnt = event.getTable().getColumnCnt();
         ColumnInfo[] columnInfo = event.getTable().getColumnInfo();
-
+        // mysql8.0针对set @@global.binlog_row_metadata='FULL' 可以记录部分的metadata信息
+        boolean existOptionalMetaData = event.getTable().isExistOptionalMetaData();
         boolean tableError = false;
         // check table fileds count，只能处理加字段
         boolean existRDSNoPrimaryKey = false;
@@ -618,9 +621,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                                                       + "," + columnInfo.length + " vs " + tableMeta.getFields().size());
                     }
                 }
-            } else {
-                logger.warn("[" + event.getTable().getDbName() + "." + event.getTable().getTableName()
-                            + "] is no primary key , skip alibaba_rds_row_id column");
+                // } else {
+                // logger.warn("[" + event.getTable().getDbName() + "." +
+                // event.getTable().getTableName()
+                // + "] is no primary key , skip alibaba_rds_row_id column");
             }
         }
 
@@ -633,20 +637,59 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
 
             if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
                 // 不解析最后一列
-                buffer.nextValue(info.type, info.meta, false);
+                String rdsRowIdColumnName = "#alibaba_rds_row_id#";
+                buffer.nextValue(rdsRowIdColumnName, i, info.type, info.meta, false);
+                Column.Builder columnBuilder = Column.newBuilder();
+                columnBuilder.setName(rdsRowIdColumnName);
+                columnBuilder.setIsKey(true);
+                columnBuilder.setMysqlType("bigint");
+                columnBuilder.setIndex(i);
+                columnBuilder.setIsNull(false);
+                Serializable value = buffer.getValue();
+                columnBuilder.setValue(value.toString());
+                columnBuilder.setSqlType(Types.BIGINT);
+                columnBuilder.setUpdated(false);
+
+                if (isAfter) {
+                    rowDataBuilder.addAfterColumns(columnBuilder.build());
+                } else {
+                    rowDataBuilder.addBeforeColumns(columnBuilder.build());
+                }
                 continue;
             }
 
-            Column.Builder columnBuilder = Column.newBuilder();
-
             FieldMeta fieldMeta = null;
             if (tableMeta != null && !tableError) {
                 // 处理file meta
                 fieldMeta = tableMeta.getFields().get(i);
+            }
+
+            if (fieldMeta != null && existOptionalMetaData && tableMetaCache.isOnTSDB()) {
+                // check column info
+                boolean check = StringUtils.equalsIgnoreCase(fieldMeta.getColumnName(), info.name);
+                check &= (fieldMeta.isUnsigned() == info.unsigned);
+                check &= (fieldMeta.isNullable() == info.nullable);
+
+                if (!check) {
+                    throw new CanalParseException("MySQL8.0 unmatch column metadata & pls submit issue , table : "
+                                                  + tableMeta.getFullName() + ", db fieldMeta : "
+                                                  + fieldMeta.toString() + " , binlog fieldMeta : " + info.toString()
+                                                  + " , on : " + event.getHeader().getLogFileName() + ":"
+                                                  + (event.getHeader().getLogPos() - event.getHeader().getEventLen()));
+                }
+            }
+
+            Column.Builder columnBuilder = Column.newBuilder();
+            if (fieldMeta != null) {
                 columnBuilder.setName(fieldMeta.getColumnName());
                 columnBuilder.setIsKey(fieldMeta.isKey());
                 // 增加mysql type类型,issue 73
                 columnBuilder.setMysqlType(fieldMeta.getColumnType());
+            } else if (existOptionalMetaData) {
+                columnBuilder.setName(info.name);
+                columnBuilder.setIsKey(info.pk);
+                // mysql8.0里没有mysql type类型
+                // columnBuilder.setMysqlType(fieldMeta.getColumnType());
             }
             columnBuilder.setIndex(i);
             columnBuilder.setIsNull(false);
@@ -664,16 +707,9 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                     isSingleBit = true;
                 }
             }
-            buffer.nextValue(info.type, info.meta, isBinary);
-            if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
-                // 不解析最后一列
-                continue;
-            }
 
+            buffer.nextValue(columnBuilder.getName(), i, info.type, info.meta, isBinary);
             int javaType = buffer.getJavaType();
-            if (isSingleBit && javaType == Types.TINYINT) {
-                javaType = Types.BIT;
-            }
             if (buffer.isNull()) {
                 columnBuilder.setIsNull(true);
             } else {
@@ -686,7 +722,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                     case Types.BIGINT:
                         // 处理unsigned类型
                         Number number = (Number) value;
-                        if (fieldMeta != null && fieldMeta.isUnsigned() && number.longValue() < 0) {
+                        boolean isUnsigned = (fieldMeta != null ? fieldMeta.isUnsigned() : (existOptionalMetaData ? info.unsigned : false));
+                        if (isUnsigned && number.longValue() < 0) {
                             switch (buffer.getLength()) {
                                 case 1: /* MYSQL_TYPE_TINY */
                                     columnBuilder.setValue(String.valueOf(Integer.valueOf(TINYINT_MAX_VALUE
@@ -722,6 +759,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                             // 对象为number类型，直接valueof即可
                             columnBuilder.setValue(String.valueOf(value));
                         }
+
+                        if (isSingleBit && javaType == Types.TINYINT) {
+                            javaType = Types.BIT;
+                        }
                         break;
                     case Types.REAL: // float
                     case Types.DOUBLE: // double
@@ -771,7 +812,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                     default:
                         columnBuilder.setValue(value.toString());
                 }
-
             }
 
             columnBuilder.setSqlType(javaType);
@@ -817,6 +857,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
         Header.Builder headerBuilder = Header.newBuilder();
         headerBuilder.setVersion(version);
         headerBuilder.setLogfileName(logHeader.getLogFileName());
+        // 记录的是该binlog的start offest
         headerBuilder.setLogfileOffset(logHeader.getLogPos() - logHeader.getEventLen());
         headerBuilder.setServerId(logHeader.getServerId());
         headerBuilder.setServerenCode(UTF_8);// 经过java输出后所有的编码为unicode
@@ -951,10 +992,12 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
 
     public void setNameFilter(AviaterRegexFilter nameFilter) {
         this.nameFilter = nameFilter;
+        logger.warn("--> init table filter : " + nameFilter.toString());
     }
 
     public void setNameBlackFilter(AviaterRegexFilter nameBlackFilter) {
         this.nameBlackFilter = nameBlackFilter;
+        logger.warn("--> init table black filter : " + nameBlackFilter.toString());
     }
 
     public void setTableMetaCache(TableMetaCache tableMetaCache) {
@@ -980,4 +1023,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
     public void setFilterRows(boolean filterRows) {
         this.filterRows = filterRows;
     }
+
+    public void setUseDruidDdlFilter(boolean useDruidDdlFilter) {
+        this.useDruidDdlFilter = useDruidDdlFilter;
+    }
 }
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
index 4c1cef3c..25554c29 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
@@ -39,6 +39,7 @@ public class TableMetaCache {
     public static final String              EXTRA          = "EXTRA";
     private MysqlConnection                 connection;
     private boolean                         isOnRDS        = false;
+    private boolean                         isOnTSDB       = false;
 
     private TableMetaTSDB                   tableMetaTSDB;
     // 第一层tableId,第二层schema.table,解决tableId重复，对应多张表
@@ -67,6 +68,8 @@ public class TableMetaCache {
                 }
 
             });
+        } else {
+            isOnTSDB = true;
         }
 
         try {
@@ -163,9 +166,16 @@ public class TableMetaCache {
             if (tableMeta == null) {
                 // 因为条件变化，可能第一次的tableMeta没取到，需要从db获取一次，并记录到snapshot中
                 String fullName = getFullName(schema, table);
+                ResultSetPacket packet = null;
+                String createDDL = null;
                 try {
-                    ResultSetPacket packet = connection.query("show create table " + fullName);
-                    String createDDL = null;
+                    try {
+                        packet = connection.query("show create table " + fullName);
+                    } catch (Exception e) {
+                        // 尝试做一次retry操作
+                        connection.reconnect();
+                        packet = connection.query("show create table " + fullName);
+                    }
                     if (packet.getFieldValues().size() > 0) {
                         createDDL = packet.getFieldValues().get(1);
                     }
@@ -244,6 +254,15 @@ public class TableMetaCache {
             .toString();
     }
 
+
+    public boolean isOnTSDB() {
+        return isOnTSDB;
+    }
+
+    public void setOnTSDB(boolean isOnTSDB) {
+        this.isOnTSDB = isOnTSDB;
+    }
+
     public boolean isOnRDS() {
         return isOnRDS;
     }
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java
index d82e5eff..68f1ff5a 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java
@@ -71,6 +71,7 @@ public class DruidDdlParser {
                         DdlResult ddlResult = new DdlResult();
                         processName(ddlResult, schmeaName, alterTable.getName(), true);
                         processName(ddlResult, schmeaName, ((SQLAlterTableRename) item).getToName(), false);
+                        ddlResult.setType(EventType.RENAME);
                         ddlResults.add(ddlResult);
                     } else if (item instanceof SQLAlterTableAddIndex) {
                         DdlResult ddlResult = new DdlResult();
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java
index e132514b..b90b89d5 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java
@@ -41,6 +41,7 @@ import org.apache.http.ssl.TrustStrategy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
 import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
 
 /**
@@ -103,6 +104,9 @@ public class BinlogDownloadQueue {
 
     public BinlogFile tryOne() throws Throwable {
         BinlogFile binlogFile = binlogList.poll();
+        if (binlogFile == null) {
+            throw new CanalParseException("download binlog is null");
+        }
         download(binlogFile);
         hostId = binlogFile.getHostInstanceID();
         this.currentSize++;
@@ -131,7 +135,7 @@ public class BinlogDownloadQueue {
         if (StringUtils.isNotEmpty(needCompareName) && StringUtils.endsWith(needCompareName, "tar")) {
             needCompareName = needCompareName.substring(0, needCompareName.indexOf("."));
         }
-        return fileName.equalsIgnoreCase(needCompareName) && binlogList.isEmpty();
+        return (needCompareName == null || fileName.equalsIgnoreCase(needCompareName)) && binlogList.isEmpty();
     }
 
     public void prepare() throws InterruptedException {
@@ -162,6 +166,14 @@ public class BinlogDownloadQueue {
         this.currentSize = 0;
         binlogList.clear();
         downloadQueue.clear();
+        try {
+            downloadThread.interrupt();
+            downloadThread.join();// 等待其结束
+        } catch (InterruptedException e) {
+            // ignore
+        } finally {
+            downloadThread = null;
+        }
     }
 
     private void download(BinlogFile binlogFile) throws Throwable {
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java
index 3c38ed0c..f3a220b0 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java
@@ -22,14 +22,14 @@ import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
  */
 public class RdsBinlogEventParserProxy extends MysqlEventParser {
 
-    private String                    rdsOpenApiUrl             = "https://rds.aliyuncs.com/";    // openapi地址
-    private String                    accesskey;                                                  // 云账号的ak
-    private String                    secretkey;                                                  // 云账号sk
-    private String                    instanceId;                                                 // rds实例id
-    private String                    directory;                                                  // binlog目录
-    private int                       batchFileSize             = 4;                              // 最多下载的binlog文件数量
+    private String                    rdsOpenApiUrl             = "https://rds.aliyuncs.com/"; // openapi地址
+    private String                    accesskey;                                              // 云账号的ak
+    private String                    secretkey;                                              // 云账号sk
+    private String                    instanceId;                                             // rds实例id
+    private String                    directory;                                              // binlog目录
+    private int                       batchFileSize             = 4;                          // 最多下载的binlog文件数量
 
-    private RdsLocalBinlogEventParser rdsLocalBinlogEventParser = new RdsLocalBinlogEventParser();
+    private RdsLocalBinlogEventParser rdsLocalBinlogEventParser = null;
     private ExecutorService           executorService           = Executors.newSingleThreadExecutor(new ThreadFactory() {
 
                                                                     @Override
@@ -43,8 +43,11 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
 
     @Override
     public void start() {
-        if (StringUtils.isNotEmpty(accesskey) && StringUtils.isNotEmpty(secretkey)
+        if (rdsLocalBinlogEventParser == null && StringUtils.isNotEmpty(accesskey) && StringUtils.isNotEmpty(secretkey)
             && StringUtils.isNotEmpty(instanceId)) {
+            rdsLocalBinlogEventParser = new RdsLocalBinlogEventParser();
+            // rds oss mode
+            setRdsOssMode(true);
             final ParserExceptionHandler targetHandler = this.getParserExceptionHandler();
             if (directory == null) {
                 directory = System.getProperty("java.io.tmpdir", "/tmp") + "/" + destination;
@@ -119,10 +122,18 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
                         long serverId = rdsBinlogEventParserProxy.getServerId();
                         rdsLocalBinlogEventParser.setServerId(serverId);
                         rdsBinlogEventParserProxy.stop();
+                    } catch (Throwable e) {
+                        logger.info("handle exception failed", e);
+                    }
+
+                    try {
                         logger.info("start rds mysql binlog parser!");
                         rdsLocalBinlogEventParser.start();
                     } catch (Throwable e) {
                         logger.info("handle exception failed", e);
+                        rdsLocalBinlogEventParser.stop();
+                        RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this;
+                        rdsBinlogEventParserProxy.start();// 继续重试
                     }
                 }
             });
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java
index aac14d99..2e0960fe 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java
@@ -57,7 +57,11 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
             if (entryPosition == null) {
                 throw new PositionNotFoundException("position not found!");
             }
-            long startTimeInMill = entryPosition.getTimestamp();
+            Long startTimeInMill = entryPosition.getTimestamp();
+            if (startTimeInMill == null || startTimeInMill <= 0) {
+                throw new PositionNotFoundException("position timestamp is empty!");
+            }
+
             startTime = startTimeInMill;
             List binlogFiles = RdsBinlogOpenApi.listBinlogFiles(url,
                 accesskey,
@@ -65,6 +69,10 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
                 instanceId,
                 new Date(startTime),
                 new Date(endTime));
+            if (binlogFiles.isEmpty()) {
+                throw new CanalParseException("start timestamp : " + startTimeInMill + " binlog files is empty");
+            }
+
             binlogDownloadQueue = new BinlogDownloadQueue(binlogFiles, batchFileSize, directory);
             binlogDownloadQueue.silenceDownload();
             needWait = true;
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java
index f43523b5..66d7591a 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java
@@ -160,7 +160,7 @@ public abstract class AbstractRequest {
     }
 
     public final String formatUTCTZ(Date date) {
-        SimpleDateFormat sdf = new SimpleDateFormat("YYYY-MM-dd'T'HH:mm:ss'Z'");
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
         sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
         return sdf.format(date);
     }
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
index bb2af45f..953cca5d 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
@@ -7,8 +7,12 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Pattern;
 
 import org.apache.commons.beanutils.BeanUtils;
@@ -44,19 +48,34 @@ import com.alibaba.otter.canal.protocol.position.EntryPosition;
  */
 public class DatabaseTableMeta implements TableMetaTSDB {
 
-    public static final EntryPosition INIT_POSITION = new EntryPosition("0", 0L, -2L, -1L);
-    private static Logger             logger        = LoggerFactory.getLogger(DatabaseTableMeta.class);
-    private static Pattern            pattern       = Pattern.compile("Duplicate entry '.*' for key '*'");
-    private static Pattern            h2Pattern     = Pattern.compile("Unique index or primary key violation");
-    private String                    destination;
-    private MemoryTableMeta           memoryTableMeta;
-    private MysqlConnection           connection;                                                              // 查询meta信息的链接
-    private CanalEventFilter          filter;
-    private CanalEventFilter          blackFilter;
-    private EntryPosition             lastPosition;
-    private ScheduledExecutorService  scheduler;
-    private MetaHistoryDAO            metaHistoryDAO;
-    private MetaSnapshotDAO           metaSnapshotDAO;
+    public static final EntryPosition       INIT_POSITION    = new EntryPosition("0", 0L, -2L, -1L);
+    private static Logger                   logger           = LoggerFactory.getLogger(DatabaseTableMeta.class);
+    private static Pattern                  pattern          = Pattern.compile("Duplicate entry '.*' for key '*'");
+    private static Pattern                  h2Pattern        = Pattern.compile("Unique index or primary key violation");
+    private static ScheduledExecutorService scheduler        = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
+
+                                                                 @Override
+                                                                 public Thread newThread(Runnable r) {
+                                                                     Thread thread = new Thread(r,
+                                                                         "[scheduler-table-meta-snapshot]");
+                                                                     thread.setDaemon(true);
+                                                                     return thread;
+                                                                 }
+                                                             });
+    private ReadWriteLock                   lock             = new ReentrantReadWriteLock();
+    private AtomicBoolean                   initialized      = new AtomicBoolean(false);
+    private String                          destination;
+    private MemoryTableMeta                 memoryTableMeta;
+    private volatile MysqlConnection        connection;                                                                 // 查询meta信息的链接
+    private CanalEventFilter                filter;
+    private CanalEventFilter                blackFilter;
+    private EntryPosition                   lastPosition;
+    private boolean                         hasNewDdl;
+    private MetaHistoryDAO                  metaHistoryDAO;
+    private MetaSnapshotDAO                 metaSnapshotDAO;
+    private int                             snapshotInterval = 24;
+    private int                             snapshotExpire   = 360;
+    private ScheduledFuture              scheduleSnapshotFuture;
 
     public DatabaseTableMeta(){
 
@@ -64,52 +83,84 @@ public class DatabaseTableMeta implements TableMetaTSDB {
 
     @Override
     public boolean init(final String destination) {
-        this.destination = destination;
-        this.memoryTableMeta = new MemoryTableMeta();
-        this.scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
+        if (initialized.compareAndSet(false, true)) {
+            this.destination = destination;
+            this.memoryTableMeta = new MemoryTableMeta();
 
-            @Override
-            public Thread newThread(Runnable r) {
-                Thread thread = new Thread(r, "[scheduler-table-meta-snapshot]");
-                thread.setDaemon(true);
-                return thread;
+            // 24小时生成一份snapshot
+            if (snapshotInterval > 0) {
+                scheduleSnapshotFuture = scheduler.scheduleWithFixedDelay(new Runnable() {
+
+                    @Override
+                    public void run() {
+                        boolean applyResult = false;
+                        try {
+                            MDC.put("destination", destination);
+                            applyResult = applySnapshotToDB(lastPosition, false);
+                        } catch (Throwable e) {
+                            logger.error("scheudle applySnapshotToDB faield", e);
+                        }
+
+                        try {
+                            MDC.put("destination", destination);
+                            if (applyResult) {
+                                snapshotExpire((int) TimeUnit.HOURS.toSeconds(snapshotExpire));
+                            }
+                        } catch (Throwable e) {
+                            logger.error("scheudle snapshotExpire faield", e);
+                        }
+                    }
+                }, snapshotInterval, snapshotInterval, TimeUnit.HOURS);
             }
-        });
-
-        // 24小时生成一份snapshot
-        scheduler.scheduleWithFixedDelay(new Runnable() {
-
-            @Override
-            public void run() {
-                try {
-                    MDC.put("destination", destination);
-                    applySnapshotToDB(lastPosition, false);
-                } catch (Throwable e) {
-                    logger.error("scheudle applySnapshotToDB faield", e);
-                }
-            }
-        }, 24, 24, TimeUnit.HOURS);
+        }
         return true;
     }
 
+    @Override
+    public void destory() {
+        if (memoryTableMeta != null) {
+            memoryTableMeta.destory();
+        }
+
+        if (connection != null) {
+            try {
+                connection.disconnect();
+            } catch (IOException e) {
+                logger.error("ERROR # disconnect meta connection for address:{}", connection.getConnector()
+                    .getAddress(), e);
+            }
+        }
+
+        if (scheduleSnapshotFuture != null) {
+            scheduleSnapshotFuture.cancel(false);
+        }
+    }
+
     @Override
     public TableMeta find(String schema, String table) {
-        synchronized (memoryTableMeta) {
+        lock.readLock().lock();
+        try {
             return memoryTableMeta.find(schema, table);
+        } finally {
+            lock.readLock().unlock();
         }
     }
 
     @Override
     public boolean apply(EntryPosition position, String schema, String ddl, String extra) {
         // 首先记录到内存结构
-        synchronized (memoryTableMeta) {
+        lock.writeLock().lock();
+        try {
             if (memoryTableMeta.apply(position, schema, ddl, extra)) {
                 this.lastPosition = position;
+                this.hasNewDdl = true;
                 // 同步每次变更给远程做历史记录
                 return applyHistoryToDB(position, schema, ddl, extra);
             } else {
                 throw new RuntimeException("apply to memory is failed");
             }
+        } finally {
+            lock.writeLock().unlock();
         }
     }
 
@@ -152,9 +203,13 @@ public class DatabaseTableMeta implements TableMetaTSDB {
             }
 
             for (String schema : schemas) {
-                packet = connection.query("show tables from `" + schema + "`");
+                // filter views
+                packet = connection.query("show full tables from `" + schema + "` where Table_type = 'BASE TABLE'");
                 List tables = new ArrayList();
                 for (String table : packet.getFieldValues()) {
+                    if ("BASE TABLE".equalsIgnoreCase(table)) {
+                        continue;
+                    }
                     String fullName = schema + "." + table;
                     if (blackFilter == null || !blackFilter.filter(fullName)) {
                         if (filter == null || filter.filter(fullName)) {
@@ -233,28 +288,41 @@ public class DatabaseTableMeta implements TableMetaTSDB {
      */
     private boolean applySnapshotToDB(EntryPosition position, boolean init) {
         // 获取一份快照
-        MemoryTableMeta tmpMemoryTableMeta = new MemoryTableMeta();
         Map schemaDdls = null;
-        synchronized (memoryTableMeta) {
-            if (!init && position == null) {
+        lock.readLock().lock();
+        try {
+            if (!init && !hasNewDdl) {
                 // 如果是持续构建,则识别一下是否有DDL变更过,如果没有就忽略了
                 return false;
             }
+            this.hasNewDdl = false;
             schemaDdls = memoryTableMeta.snapshot();
-            for (Map.Entry entry : schemaDdls.entrySet()) {
-                tmpMemoryTableMeta.apply(position, entry.getKey(), entry.getValue(), null);
-            }
+        } finally {
+            lock.readLock().unlock();
+        }
+
+        MemoryTableMeta tmpMemoryTableMeta = new MemoryTableMeta();
+        for (Map.Entry entry : schemaDdls.entrySet()) {
+            tmpMemoryTableMeta.apply(position, entry.getKey(), entry.getValue(), null);
         }
 
         // 基于临时内存对象进行对比
         boolean compareAll = true;
         for (Schema schema : tmpMemoryTableMeta.getRepository().getSchemas()) {
             for (String table : schema.showTables()) {
-                if (!compareTableMetaDbAndMemory(connection, tmpMemoryTableMeta, schema.getName(), table)) {
-                    compareAll = false;
+                String fullName = schema + "." + table;
+                if (blackFilter == null || !blackFilter.filter(fullName)) {
+                    if (filter == null || filter.filter(fullName)) {
+                        // issue : https://github.com/alibaba/canal/issues/1168
+                        // 在生成snapshot时重新过滤一遍
+                        if (!compareTableMetaDbAndMemory(connection, tmpMemoryTableMeta, schema.getName(), table)) {
+                            compareAll = false;
+                        }
+                    }
                 }
             }
         }
+
         if (compareAll) {
             Map content = new HashMap();
             content.put("destination", destination);
@@ -438,14 +506,34 @@ public class DatabaseTableMeta implements TableMetaTSDB {
                 return false;
             }
 
-            if (!StringUtils.equalsIgnoreCase(sourceField.getColumnType(), targetField.getColumnType())) {
+            // if (!StringUtils.equalsIgnoreCase(sourceField.getColumnType(),
+            // targetField.getColumnType())) {
+            // return false;
+            // }
+
+            // https://github.com/alibaba/canal/issues/1100
+            // 支持一下 int vs int(10)
+            if ((sourceField.isUnsigned() && !targetField.isUnsigned())
+                || (!sourceField.isUnsigned() && targetField.isUnsigned())) {
                 return false;
             }
 
-            if (!StringUtils.equalsIgnoreCase(sourceField.getDefaultValue(), targetField.getDefaultValue())) {
+            String sign = sourceField.isUnsigned() ? "unsigned" : "signed";
+            String sourceColumnType = StringUtils.removeEndIgnoreCase(sourceField.getColumnType(), sign).trim();
+            String targetColumnType = StringUtils.removeEndIgnoreCase(targetField.getColumnType(), sign).trim();
+
+            boolean columnTypeCompare = false;
+            columnTypeCompare |= StringUtils.containsIgnoreCase(sourceColumnType, targetColumnType);
+            columnTypeCompare |= StringUtils.containsIgnoreCase(targetColumnType, sourceColumnType);
+            if (!columnTypeCompare) {
                 return false;
             }
 
+            // if (!StringUtils.equalsIgnoreCase(sourceField.getDefaultValue(),
+            // targetField.getDefaultValue())) {
+            // return false;
+            // }
+
             if (sourceField.isNullable() != targetField.isNullable()) {
                 return false;
             }
@@ -461,6 +549,10 @@ public class DatabaseTableMeta implements TableMetaTSDB {
         return true;
     }
 
+    private int snapshotExpire(int expireTimestamp) {
+        return metaSnapshotDAO.deleteByTimestamp(destination, expireTimestamp);
+    }
+
     public void setConnection(MysqlConnection connection) {
         this.connection = connection;
     }
@@ -489,6 +581,22 @@ public class DatabaseTableMeta implements TableMetaTSDB {
         this.blackFilter = blackFilter;
     }
 
+    public int getSnapshotInterval() {
+        return snapshotInterval;
+    }
+
+    public void setSnapshotInterval(int snapshotInterval) {
+        this.snapshotInterval = snapshotInterval;
+    }
+
+    public int getSnapshotExpire() {
+        return snapshotExpire;
+    }
+
+    public void setSnapshotExpire(int snapshotExpire) {
+        this.snapshotExpire = snapshotExpire;
+    }
+
     public MysqlConnection getConnection() {
         return connection;
     }
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
index 904b55a9..f4a23dc8 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
@@ -30,6 +30,7 @@ import com.alibaba.fastsql.sql.ast.statement.SQLSelectOrderByItem;
 import com.alibaba.fastsql.sql.ast.statement.SQLTableElement;
 import com.alibaba.fastsql.sql.dialect.mysql.ast.MySqlPrimaryKey;
 import com.alibaba.fastsql.sql.dialect.mysql.ast.MySqlUnique;
+import com.alibaba.fastsql.sql.dialect.mysql.ast.expr.MySqlOrderingExpr;
 import com.alibaba.fastsql.sql.repository.Schema;
 import com.alibaba.fastsql.sql.repository.SchemaObject;
 import com.alibaba.fastsql.sql.repository.SchemaRepository;
@@ -59,6 +60,11 @@ public class MemoryTableMeta implements TableMetaTSDB {
         return true;
     }
 
+    @Override
+    public void destory() {
+        tableMetas.clear();
+    }
+
     public boolean apply(EntryPosition position, String schema, String ddl, String extra) {
         tableMetas.clear();
         synchronized (this) {
@@ -69,7 +75,11 @@ public class MemoryTableMeta implements TableMetaTSDB {
             try {
                 // druid暂时flush privileges语法解析有问题
                 if (!StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "flush")
-                    && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "grant")) {
+                    && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "grant")
+                    && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create user")
+                    && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "drop user")
+                    && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create database")
+                    && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "drop database")) {
                     repository.console(ddl);
                 }
             } catch (Throwable e) {
@@ -208,6 +218,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
                     fieldMeta.setNullable(true);
                 } else if (constraint instanceof SQLColumnPrimaryKey) {
                     fieldMeta.setKey(true);
+                    fieldMeta.setNullable(false);
                 } else if (constraint instanceof SQLColumnUniqueKey) {
                     fieldMeta.setUnique(true);
                 }
@@ -220,6 +231,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
                 String name = getSqlName(pk.getExpr());
                 FieldMeta field = tableMeta.getFieldMetaByName(name);
                 field.setKey(true);
+                field.setNullable(false);
             }
         } else if (element instanceof MySqlUnique) {
             MySqlUnique column = (MySqlUnique) element;
@@ -247,6 +259,8 @@ public class MemoryTableMeta implements TableMetaTSDB {
             return ((SQLCharExpr) sqlName).getText();
         } else if (sqlName instanceof SQLMethodInvokeExpr) {
             return DruidDdlParser.unescapeName(((SQLMethodInvokeExpr) sqlName).getMethodName());
+        } else if (sqlName instanceof MySqlOrderingExpr) {
+            return getSqlName(((MySqlOrderingExpr) sqlName).getExpr());
         } else {
             return sqlName.toString();
         }
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java
index d712adbe..f968ae12 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java
@@ -18,6 +18,11 @@ public interface TableMetaTSDB {
      */
     public boolean init(String destination);
 
+    /**
+     * 销毁资源
+     */
+    public void destory();
+
     /**
      * 获取当前的表结构
      */
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java
index 107e1013..75b97f14 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java
@@ -36,7 +36,6 @@ public class TableMetaTSDBBuilder {
                 }
             }
             TableMetaTSDB tableMetaTSDB = (TableMetaTSDB) applicationContext.getBean("tableMetaTSDB");
-            tableMetaTSDB.init(destination);
             logger.info("{} init TableMetaTSDB with {}", destination, springXml);
             return tableMetaTSDB;
         } else {
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java
index 6ec51614..c214d88f 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java
@@ -1,7 +1,5 @@
 package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao;
 
-import java.text.SimpleDateFormat;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 
@@ -37,13 +35,12 @@ public class MetaHistoryDAO extends MetaBaseDAO {
     /**
      * 删除interval秒之前的数据
      */
-    public Integer deleteByGmtModified(int interval) {
+    public Integer deleteByTimestamp(String destination, int interval) {
         HashMap params = Maps.newHashMapWithExpectedSize(2);
         long timestamp = System.currentTimeMillis() - interval * 1000;
-        Date date = new Date(timestamp);
-        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-        params.put("timestamp", format.format(date));
-        return getSqlMapClientTemplate().delete("meta_history.deleteByGmtModified", params);
+        params.put("timestamp", timestamp);
+        params.put("destination", destination);
+        return getSqlMapClientTemplate().delete("meta_history.deleteByTimestamp", params);
     }
 
     protected void initDao() throws Exception {
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java
index f525031a..ff79b107 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java
@@ -1,7 +1,5 @@
 package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao;
 
-import java.text.SimpleDateFormat;
-import java.util.Date;
 import java.util.HashMap;
 
 import com.google.common.collect.Maps;
@@ -40,13 +38,12 @@ public class MetaSnapshotDAO extends MetaBaseDAO {
     /**
      * 删除interval秒之前的数据
      */
-    public Integer deleteByGmtModified(int interval) {
+    public Integer deleteByTimestamp(String destination, int interval) {
         HashMap params = Maps.newHashMapWithExpectedSize(2);
         long timestamp = System.currentTimeMillis() - interval * 1000;
-        Date date = new Date(timestamp);
-        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-        params.put("timestamp", format.format(date));
-        return getSqlMapClientTemplate().delete("meta_snapshot.deleteByGmtModified", params);
+        params.put("timestamp", timestamp);
+        params.put("destination", destination);
+        return getSqlMapClientTemplate().delete("meta_snapshot.deleteByTimestamp", params);
     }
 
     protected void initDao() throws Exception {
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/support/AuthenticationInfo.java b/parse/src/main/java/com/alibaba/otter/canal/parse/support/AuthenticationInfo.java
index 5446d068..6b12ef0d 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/support/AuthenticationInfo.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/support/AuthenticationInfo.java
@@ -2,21 +2,32 @@ package com.alibaba.otter.canal.parse.support;
 
 import java.net.InetSocketAddress;
 
+import com.alibaba.druid.filter.config.ConfigTools;
 import org.apache.commons.lang.builder.ToStringBuilder;
 import org.apache.commons.lang.builder.ToStringStyle;
 
 /**
  * 数据库认证信息
- * 
+ *
  * @author jianghang 2012-7-11 上午11:22:19
  * @version 1.0.0
  */
 public class AuthenticationInfo {
 
+
+
     private InetSocketAddress address;            // 主库信息
     private String            username;           // 帐号
     private String            password;           // 密码
-    private String            defaultDatabaseName; // 默认链接的数据库
+    private String            defaultDatabaseName;// 默认链接的数据库
+    private String            pwdPublicKey;       //公钥
+    private boolean           enableDruid;        //是否使用druid加密解密数据库密码
+
+    public void initPwd() throws Exception{
+        if (enableDruid) {
+            this.password = ConfigTools.decrypt(pwdPublicKey, password);
+        }
+    }
 
     public AuthenticationInfo(){
         super();
@@ -65,6 +76,22 @@ public class AuthenticationInfo {
         this.defaultDatabaseName = defaultDatabaseName;
     }
 
+    public String getPwdPublicKey() {
+        return pwdPublicKey;
+    }
+
+    public void setPwdPublicKey(String pwdPublicKey) {
+        this.pwdPublicKey = pwdPublicKey;
+    }
+
+    public boolean isEnableDruid() {
+        return enableDruid;
+    }
+
+    public void setEnableDruid(boolean enableDruid) {
+        this.enableDruid = enableDruid;
+    }
+
     @Override
     public String toString() {
         return ToStringBuilder.reflectionToString(this, ToStringStyle.DEFAULT_STYLE);
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java
index aa48f9a3..089163d3 100644
--- a/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java
@@ -1,7 +1,15 @@
 package com.alibaba.otter.canal.parse;
 
+import static com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher.MASTER_HEARTBEAT_PERIOD_SECONDS;
+
 import java.io.IOException;
+import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
 import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.util.BitSet;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.lang.StringUtils;
 import org.junit.Assert;
@@ -10,29 +18,48 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlQueryExecutor;
 import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.client.RegisterSlaveCommandPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
 import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
 import com.taobao.tddl.dbsync.binlog.LogContext;
 import com.taobao.tddl.dbsync.binlog.LogDecoder;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
+import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
+import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
 
 public class DirectLogFetcherTest {
 
-    protected final Logger logger = LoggerFactory.getLogger(this.getClass());
+    protected final Logger logger         = LoggerFactory.getLogger(this.getClass());
+    protected String       binlogFileName = "mysql-bin.000001";
+    protected Charset      charset        = Charset.forName("utf-8");
+    private int            binlogChecksum;
 
     @Test
     public void testSimple() {
         DirectLogFetcher fetcher = new DirectLogFetcher();
         try {
-            MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxx", "xxxx");
+            MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
             connector.connect();
             updateSettings(connector);
+            loadBinlogChecksum(connector);
             sendRegisterSlave(connector, 3);
             sendBinlogDump(connector, "mysql-bin.000001", 4L, 3);
 
@@ -40,6 +67,8 @@ public class DirectLogFetcherTest {
 
             LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
             LogContext context = new LogContext();
+            context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
+
             while (fetcher.fetch()) {
                 LogEvent event = null;
                 event = decoder.decode(fetcher, context);
@@ -57,21 +86,22 @@ public class DirectLogFetcherTest {
                         break;
                     case LogEvent.WRITE_ROWS_EVENT_V1:
                     case LogEvent.WRITE_ROWS_EVENT:
-                        // parseRowsEvent((WriteRowsLogEvent) event);
+                        parseRowsEvent((WriteRowsLogEvent) event);
                         break;
                     case LogEvent.UPDATE_ROWS_EVENT_V1:
+                    case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
                     case LogEvent.UPDATE_ROWS_EVENT:
-                        // parseRowsEvent((UpdateRowsLogEvent) event);
+                        parseRowsEvent((UpdateRowsLogEvent) event);
                         break;
                     case LogEvent.DELETE_ROWS_EVENT_V1:
                     case LogEvent.DELETE_ROWS_EVENT:
-                        // parseRowsEvent((DeleteRowsLogEvent) event);
+                        parseRowsEvent((DeleteRowsLogEvent) event);
                         break;
                     case LogEvent.QUERY_EVENT:
-                        // parseQueryEvent((QueryLogEvent) event);
+                        parseQueryEvent((QueryLogEvent) event);
                         break;
                     case LogEvent.ROWS_QUERY_LOG_EVENT:
-                        // parseRowsQueryEvent((RowsQueryLogEvent) event);
+                        parseRowsQueryEvent((RowsQueryLogEvent) event);
                         break;
                     case LogEvent.ANNOTATE_ROWS_EVENT:
                         break;
@@ -165,7 +195,7 @@ public class DirectLogFetcherTest {
             // 如果不设置会出现错误： Slave can not handle replication events with the
             // checksum that master is configured to log
             // 但也不能乱设置，需要和mysql server的checksum配置一致，不然RotateLogEvent会出现乱码
-            update("set @master_binlog_checksum= '@@global.binlog_checksum'", connector);
+            update("set @master_binlog_checksum= @@global.binlog_checksum", connector);
         } catch (Exception e) {
             logger.warn("update master_binlog_checksum failed", e);
         }
@@ -186,6 +216,34 @@ public class DirectLogFetcherTest {
         } catch (Exception e) {
             logger.warn("update mariadb_slave_capability failed", e);
         }
+
+        try {
+            long period = TimeUnit.SECONDS.toNanos(MASTER_HEARTBEAT_PERIOD_SECONDS);
+            update("SET @master_heartbeat_period=" + period, connector);
+        } catch (Exception e) {
+            logger.warn("update master_heartbeat_period failed", e);
+        }
+    }
+
+    private void loadBinlogChecksum(MysqlConnector connector) {
+        ResultSetPacket rs = null;
+        try {
+            rs = query("select @@global.binlog_checksum", connector);
+        } catch (IOException e) {
+            throw new CanalParseException(e);
+        }
+
+        List columnValues = rs.getFieldValues();
+        if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0).toUpperCase().equals("CRC32")) {
+            binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
+        } else {
+            binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
+        }
+    }
+
+    public ResultSetPacket query(String cmd, MysqlConnector connector) throws IOException {
+        MysqlQueryExecutor exector = new MysqlQueryExecutor(connector);
+        return exector.query(cmd);
     }
 
     public void update(String cmd, MysqlConnector connector) throws IOException {
@@ -193,4 +251,98 @@ public class DirectLogFetcherTest {
         exector.update(cmd);
     }
 
+    protected void parseQueryEvent(QueryLogEvent event) {
+        System.out.println(String.format("================> binlog[%s:%s] , name[%s]",
+            binlogFileName,
+            event.getHeader().getLogPos() - event.getHeader().getEventLen(),
+            event.getCatalog()));
+        System.out.println("sql : " + event.getQuery());
+    }
+
+    protected void parseRowsQueryEvent(RowsQueryLogEvent event) throws Exception {
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
+            .getLogPos() - event.getHeader().getEventLen()));
+        System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
+    }
+
+    protected void parseAnnotateRowsEvent(AnnotateRowsEvent event) throws Exception {
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
+            .getLogPos() - event.getHeader().getEventLen()));
+        System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
+    }
+
+    protected void parseXidEvent(XidLogEvent event) throws Exception {
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
+            .getLogPos() - event.getHeader().getEventLen()));
+        System.out.println("xid : " + event.getXid());
+    }
+
+    protected void parseRowsEvent(RowsLogEvent event) {
+        try {
+            System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]",
+                binlogFileName,
+                event.getHeader().getLogPos() - event.getHeader().getEventLen(),
+                event.getTable().getDbName(),
+                event.getTable().getTableName()));
+            RowsLogBuffer buffer = event.getRowsBuf(charset.name());
+            BitSet columns = event.getColumns();
+            BitSet changeColumns = event.getChangeColumns();
+            while (buffer.nextOneRow(columns)) {
+                // 处理row记录
+                int type = event.getHeader().getType();
+                if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
+                    // insert的记录放在before字段中
+                    parseOneRow(event, buffer, columns, true);
+                } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
+                    // delete的记录放在before字段中
+                    parseOneRow(event, buffer, columns, false);
+                } else {
+                    // update需要处理before/after
+                    System.out.println("-------> before");
+                    parseOneRow(event, buffer, columns, false);
+                    if (!buffer.nextOneRow(changeColumns, true)) {
+                        break;
+                    }
+                    System.out.println("-------> after");
+                    parseOneRow(event, buffer, changeColumns, true);
+                }
+
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("parse row data failed.", e);
+        }
+    }
+
+    protected void parseOneRow(RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter)
+                                                                                                      throws UnsupportedEncodingException {
+        TableMapLogEvent map = event.getTable();
+        if (map == null) {
+            throw new RuntimeException("not found TableMap with tid=" + event.getTableId());
+        }
+
+        final int columnCnt = map.getColumnCnt();
+        final ColumnInfo[] columnInfo = map.getColumnInfo();
+
+        for (int i = 0; i < columnCnt; i++) {
+            if (!cols.get(i)) {
+                continue;
+            }
+
+            ColumnInfo info = columnInfo[i];
+            buffer.nextValue(null, i, info.type, info.meta);
+
+            if (buffer.isNull()) {
+                //
+            } else {
+                final Serializable value = buffer.getValue();
+                if (value instanceof byte[]) {
+                    System.out.println(new String((byte[]) value));
+                } else {
+                    System.out.println(value);
+                }
+            }
+        }
+
+    }
+
 }
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java
index 79b88375..70b2dbd7 100644
--- a/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java
@@ -89,6 +89,7 @@ public class MysqlBinlogParsePerformanceTest {
                     parseRowsEvent((WriteRowsLogEvent) event, sum);
                     break;
                 case LogEvent.UPDATE_ROWS_EVENT_V1:
+                case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
                 case LogEvent.UPDATE_ROWS_EVENT:
                     parseRowsEvent((UpdateRowsLogEvent) event, sum);
                     break;
@@ -154,7 +155,7 @@ public class MysqlBinlogParsePerformanceTest {
                     parseOneRow(event, buffer, columns, false);
                 } else {
                     parseOneRow(event, buffer, columns, false);
-                    if (!buffer.nextOneRow(changeColumns)) {
+                    if (!buffer.nextOneRow(changeColumns, true)) {
                         break;
                     }
                     parseOneRow(event, buffer, changeColumns, true);
@@ -182,7 +183,7 @@ public class MysqlBinlogParsePerformanceTest {
             }
 
             ColumnInfo info = columnInfo[i];
-            buffer.nextValue(info.type, info.meta);
+            buffer.nextValue(null, i, info.type, info.meta);
             if (buffer.isNull()) {
             } else {
                 buffer.getValue();
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java
index 6c11c637..8c749891 100644
--- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java
@@ -29,12 +29,12 @@ public class MysqlDumpTest {
     @Test
     public void testSimple() {
         final MysqlEventParser controller = new MysqlEventParser();
-        final EntryPosition startPosition = new EntryPosition("mysql-bin.000012", 34051L, 100L);
+        final EntryPosition startPosition = new EntryPosition("mysql-bin.000001", 4L);
         // startPosition.setGtid("f1ceb61a-a5d5-11e7-bdee-107c3dbcf8a7:1-17");
         controller.setConnectionCharset(Charset.forName("UTF-8"));
         controller.setSlaveId(3344L);
         controller.setDetectingEnable(false);
-        controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "canal", "canal"));
+        controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "root", "hello"));
         controller.setMasterPosition(startPosition);
         controller.setEnableTsdb(true);
         controller.setDestination("example");
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java
index c7e25607..bf0deafe 100644
--- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java
@@ -18,13 +18,19 @@ public class FastsqlSchemaTest {
     @Test
     public void testSimple() throws FileNotFoundException, IOException {
         SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
-        String sql = "create table yushitai_test.card_record ( id bigint auto_increment) auto_increment=256 "
-                     + "; alter table yushitai_test.card_record add column customization_id bigint unsigned NOT NULL COMMENT 'TEST' ;"
-                     + "; rename table yushitai_test.card_record to yushitai_test._card_record_del;";
+        String sql = "create table yushitai_test.card_record ( id bigint auto_increment, name varchar(32) DEFAULT NULL) auto_increment=256 "
+                     + "ALTER TABLE yushitai_test.card_record ADD COLUMN remark2 varchar(255) DEFAULT NULL , ALGORITHM=inplace,LOCK=NONE;"
+                     + "ALTER TABLE yushitai_test.card_record modify COLUMN name varchar(64) DEFAULT NULL , ALGORITHM=copy,LOCK=SHARED; "
+                     + "alter table yushitai_test.card_record add index index_name(name) ;"
+                     + "alter table yushitai_test.card_record add index index_name(name) ;"
+                     + "alter table yushitai_test.card_record add Constraint pk_id PRIMARY KEY (id);"
+                     + "alter table yushitai_test.card_record add Constraint pk_id PRIMARY KEY (id);"
+                     + "alter table yushitai_test.card_record add Constraint UNIQUE index uk_name(name);"
+                     + "alter table yushitai_test.card_record add Constraint UNIQUE index uk_name(name);";
         repository.console(sql);
 
         repository.setDefaultSchema("yushitai_test");
-        SchemaObject table = repository.findTable("_card_record_del");
+        SchemaObject table = repository.findTable("card_record");
         System.out.println(table.getStatement().toString());
     }
 }
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java
index e3005b62..fa0999fe 100644
--- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java
@@ -37,4 +37,19 @@ public class MemoryTableMeta_DDL_Test {
         meta = memoryTableMeta.find("yushitai_test", "_card_record_gho");
         Assert.assertNull(meta);
     }
+
+    @Test
+    public void test2() throws Throwable {
+        MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
+        URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
+        File dummyFile = new File(url.getFile());
+        File create = new File(dummyFile.getParent() + "/ddl", "ddl_test2.sql");
+        String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
+        memoryTableMeta.apply(null, "test", sql, null);
+
+        TableMeta meta = memoryTableMeta.find("yushitai_test", "card_record");
+        System.out.println(meta);
+        Assert.assertEquals(meta.getFieldMetaByName("id").isKey(), true);
+        Assert.assertEquals(meta.getFieldMetaByName("name").isUnique(), true);
+    }
 }
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java
index b199a326..cea6c087 100644
--- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java
@@ -16,7 +16,7 @@ import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDO;
  * Created by wanshao Date: 2017/9/20 Time: 下午5:00
  **/
 @RunWith(SpringJUnit4ClassRunner.class)
-@ContextConfiguration(locations = { "/tsdb/mysql-tsdb.xml" })
+@ContextConfiguration(locations = { "/tsdb/h2-tsdb.xml" })
 public class MetaHistoryDAOTest {
 
     @Resource
@@ -24,7 +24,22 @@ public class MetaHistoryDAOTest {
 
     @Test
     public void testSimple() {
-        List metaHistoryDOList = metaHistoryDAO.findByTimestamp("test", 0L, 0L);
+        MetaHistoryDO historyDO = new MetaHistoryDO();
+        historyDO.setDestination("test");
+        historyDO.setBinlogFile("000001");
+        historyDO.setBinlogOffest(4L);
+        historyDO.setBinlogMasterId("1");
+        historyDO.setBinlogTimestamp(System.currentTimeMillis() - 7300 * 1000);
+        historyDO.setSqlSchema("test");
+        historyDO.setUseSchema("test");
+        historyDO.setSqlTable("testTable");
+        historyDO.setSqlTable("drop table testTable");
+        metaHistoryDAO.insert(historyDO);
+
+        int count = metaHistoryDAO.deleteByTimestamp("test", 7200);
+        System.out.println(count);
+
+        List metaHistoryDOList = metaHistoryDAO.findByTimestamp("test", 0L, System.currentTimeMillis());
         for (MetaHistoryDO metaHistoryDO : metaHistoryDOList) {
             System.out.println(metaHistoryDO.getId());
         }
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaSnapshotDAOTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaSnapshotDAOTest.java
new file mode 100644
index 00000000..f503e4b2
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaSnapshotDAOTest.java
@@ -0,0 +1,41 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
+
+import javax.annotation.Resource;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO;
+import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO;
+
+/**
+ * Created by wanshao Date: 2017/9/20 Time: 下午5:00
+ **/
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(locations = { "/tsdb/h2-tsdb.xml" })
+public class MetaSnapshotDAOTest {
+
+    @Resource
+    MetaSnapshotDAO metaSnapshotDAO;
+
+    @Test
+    public void testSimple() {
+        MetaSnapshotDO metaSnapshotDO = new MetaSnapshotDO();
+        metaSnapshotDO.setDestination("test");
+        metaSnapshotDO.setBinlogFile("000001");
+        metaSnapshotDO.setBinlogOffest(4L);
+        metaSnapshotDO.setBinlogMasterId("1");
+        metaSnapshotDO.setBinlogTimestamp(System.currentTimeMillis() - 7300 * 1000);
+        metaSnapshotDO.setData("test");
+        metaSnapshotDAO.insert(metaSnapshotDO);
+
+        MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp("test", System.currentTimeMillis());
+        System.out.println(snapshotDO);
+
+        int count = metaSnapshotDAO.deleteByTimestamp("test", 7200);
+        System.out.println(count);
+    }
+
+}
diff --git a/parse/src/test/resources/ddl/ddl_test2.sql b/parse/src/test/resources/ddl/ddl_test2.sql
new file mode 100644
index 00000000..aaf7d286
--- /dev/null
+++ b/parse/src/test/resources/ddl/ddl_test2.sql
@@ -0,0 +1,8 @@
+CREATE TABLE yushitai_test.card_record (
+	id bigint AUTO_INCREMENT,
+	name varchar(32) DEFAULT NULL,
+	alias varchar(32) DEFAULT NULL,
+	INDEX index_name(name),
+	CONSTRAINT pk_id PRIMARY KEY (id),
+	UNIQUE uk_name (name,alias)
+) AUTO_INCREMENT = 256
diff --git a/parse/src/test/resources/tsdb/h2-tsdb.xml b/parse/src/test/resources/tsdb/h2-tsdb.xml
index c0a3d385..dcb6bc47 100644
--- a/parse/src/test/resources/tsdb/h2-tsdb.xml
+++ b/parse/src/test/resources/tsdb/h2-tsdb.xml
@@ -7,7 +7,7 @@
        default-autowire="byName">
 	
 	
-	
+	
 		
 		
 	
diff --git a/parse/src/test/resources/tsdb/mysql-tsdb.xml b/parse/src/test/resources/tsdb/mysql-tsdb.xml
index 099c034d..44f97203 100644
--- a/parse/src/test/resources/tsdb/mysql-tsdb.xml
+++ b/parse/src/test/resources/tsdb/mysql-tsdb.xml
@@ -7,7 +7,7 @@
        default-autowire="byName">
 	
 	
-	
+	
 		
 		
 	
diff --git a/parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml b/parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml
index 7445bf69..58957c28 100644
--- a/parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml
+++ b/parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml
@@ -36,10 +36,10 @@
     
 
 
-    
+    
         
     
 
\ No newline at end of file
diff --git a/parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml b/parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml
index 7ba0a435..60827c6c 100644
--- a/parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml
+++ b/parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml
@@ -54,10 +54,10 @@
         where destination=#destination#
     
 
-    
+    
          0
         ]]>
     
 
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index ea1e393c..266b1921 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
     canal
     pom
     canal module for otter ${project.version}
-    1.1.1-SNAPSHOT
+    1.1.3-SNAPSHOT
     https://github.com/alibaba/canal
     
         org.sonatype.oss
@@ -99,7 +99,7 @@
         1.7
         1.7
         UTF-8
-        3.2.9.RELEASE
+        3.2.18.RELEASE
     
 
     
@@ -119,7 +119,6 @@
         example
         prometheus
         client-adapter
-        client-launcher
     
 
     
@@ -247,7 +246,7 @@
             
                 com.alibaba.fastsql
                 fastsql
-                2.0.0_preview_540
+                2.0.0_preview_644
             
             
                 com.alibaba
@@ -298,7 +297,21 @@
                 5.1.40
                 
             
-
+            
+                com.google.code.findbugs
+                jsr305
+                3.0.2
+            
+            
+                com.aliyun.openservices
+                aliware-apache-rocketmq-cloud
+                1.0
+            
+            
+                org.apache.rocketmq
+                rocketmq-client
+                4.3.0
+            
         
     
 
@@ -370,59 +383,24 @@
             
             
+                org.apache.maven.plugins
+                maven-javadoc-plugin
+                2.9.1
+                
+                    
+                            attach-javadocs
+                            
+                                jar
+                            
+                        
+                    
+                
+                  ${file_encoding}
+                  ${file_encoding}
+                  -Xdoclint:none
+                
+            
+            -->
         
         src/main/java
         src/test/java
diff --git a/prometheus/pom.xml b/prometheus/pom.xml
index 29bf8c1f..f6357c20 100644
--- a/prometheus/pom.xml
+++ b/prometheus/pom.xml
@@ -3,13 +3,13 @@
     
         canal
         com.alibaba.otter
-        1.1.1-SNAPSHOT
+        1.1.3-SNAPSHOT
     
     4.0.0
 
     com.alibaba.otter
     canal.prometheus
-    1.1.1-SNAPSHOT
+    1.1.3-SNAPSHOT
     canal prometheus module for otter ${project.version}
     
         
diff --git a/protocol/pom.xml b/protocol/pom.xml
index f8211e56..1b776b7e 100644
--- a/protocol/pom.xml
+++ b/protocol/pom.xml
@@ -3,7 +3,7 @@
 	
 		com.alibaba.otter
 		canal
-		1.1.1-SNAPSHOT
+		1.1.3-SNAPSHOT
 		../pom.xml
 	
 	com.alibaba.otter
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalEntry.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalEntry.java
index 997f76c5..0ff7d1ab 100644
--- a/protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalEntry.java
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalEntry.java
@@ -5,59 +5,44 @@ package com.alibaba.otter.canal.protocol;
 
 public final class CanalEntry {
   private CanalEntry() {}
-  public static void registerAllExtensions(
-      com.google.protobuf.ExtensionRegistryLite registry) {
-  }
-
   public static void registerAllExtensions(
       com.google.protobuf.ExtensionRegistry registry) {
-    registerAllExtensions(
-        (com.google.protobuf.ExtensionRegistryLite) registry);
   }
   /**
+   * Protobuf enum {@code com.alibaba.otter.canal.protocol.EntryType}
+   *
    * 
          **打散后的事件类型，主要用于标识事务的开始，变更数据，结束*
    * 
      
-   *
-   * Protobuf enum {@code com.alibaba.otter.canal.protocol.EntryType}
    */
   public enum EntryType
-          implements com.google.protobuf.ProtocolMessageEnum {
-    /**
-     * ENTRYTYPECOMPATIBLEPROTO2 = 0;
-     */
-    ENTRYTYPECOMPATIBLEPROTO2(0),
+      implements com.google.protobuf.ProtocolMessageEnum {
     /**
      * TRANSACTIONBEGIN = 1;
      */
-    TRANSACTIONBEGIN(1),
+    TRANSACTIONBEGIN(0, 1),
     /**
      * ROWDATA = 2;
      */
-    ROWDATA(2),
+    ROWDATA(1, 2),
     /**
      * TRANSACTIONEND = 3;
      */
-    TRANSACTIONEND(3),
+    TRANSACTIONEND(2, 3),
     /**
+     * HEARTBEAT = 4;
+     *
      * 
            ** 心跳类型，内部使用，外部暂不可见，可忽略 *
      * 
      
-     *
-     * HEARTBEAT = 4;
      */
-    HEARTBEAT(4),
+    HEARTBEAT(3, 4),
     /**
      * GTIDLOG = 5;
      */
-    GTIDLOG(5),
-    UNRECOGNIZED(-1),
+    GTIDLOG(4, 5),
     ;
 
-    /**
-     * ENTRYTYPECOMPATIBLEPROTO2 = 0;
-     */
-    public static final int ENTRYTYPECOMPATIBLEPROTO2_VALUE = 0;
     /**
      * TRANSACTIONBEGIN = 1;
      */
@@ -71,11 +56,11 @@ public final class CanalEntry {
      */
     public static final int TRANSACTIONEND_VALUE = 3;
     /**
+     * HEARTBEAT = 4;
+     *
      * 
            ** 心跳类型，内部使用，外部暂不可见，可忽略 *
      * 
      
-     *
-     * HEARTBEAT = 4;
      */
     public static final int HEARTBEAT_VALUE = 4;
     /**
@@ -84,25 +69,10 @@ public final class CanalEntry {
     public static final int GTIDLOG_VALUE = 5;
 
 
-    public final int getNumber() {
-      if (this == UNRECOGNIZED) {
-        throw new java.lang.IllegalArgumentException(
-                "Can't get the number of an unknown enum value.");
-      }
-      return value;
-    }
+    public final int getNumber() { return value; }
 
-    /**
-     * @deprecated Use {@link #forNumber(int)} instead.
-     */
-    @java.lang.Deprecated
     public static EntryType valueOf(int value) {
-      return forNumber(value);
-    }
-
-    public static EntryType forNumber(int value) {
       switch (value) {
-        case 0: return ENTRYTYPECOMPATIBLEPROTO2;
         case 1: return TRANSACTIONBEGIN;
         case 2: return ROWDATA;
         case 3: return TRANSACTIONEND;
@@ -113,47 +83,46 @@ public final class CanalEntry {
     }
 
     public static com.google.protobuf.Internal.EnumLiteMap
-    internalGetValueMap() {
+        internalGetValueMap() {
       return internalValueMap;
     }
-    private static final com.google.protobuf.Internal.EnumLiteMap<
-            EntryType> internalValueMap =
-            new com.google.protobuf.Internal.EnumLiteMap() {
-              public EntryType findValueByNumber(int number) {
-                return EntryType.forNumber(number);
-              }
-            };
+    private static com.google.protobuf.Internal.EnumLiteMap
+        internalValueMap =
+          new com.google.protobuf.Internal.EnumLiteMap() {
+            public EntryType findValueByNumber(int number) {
+              return EntryType.valueOf(number);
+            }
+          };
 
     public final com.google.protobuf.Descriptors.EnumValueDescriptor
-    getValueDescriptor() {
-      return getDescriptor().getValues().get(ordinal());
+        getValueDescriptor() {
+      return getDescriptor().getValues().get(index);
     }
     public final com.google.protobuf.Descriptors.EnumDescriptor
-    getDescriptorForType() {
+        getDescriptorForType() {
       return getDescriptor();
     }
     public static final com.google.protobuf.Descriptors.EnumDescriptor
-    getDescriptor() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.getDescriptor().getEnumTypes().get(0);
+        getDescriptor() {
+      return CanalEntry.getDescriptor().getEnumTypes().get(0);
     }
 
     private static final EntryType[] VALUES = values();
 
     public static EntryType valueOf(
-            com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+        com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
       if (desc.getType() != getDescriptor()) {
-        throw new java.lang.IllegalArgumentException(
-                "EnumValueDescriptor is not for this type.");
-      }
-      if (desc.getIndex() == -1) {
-        return UNRECOGNIZED;
+        throw new IllegalArgumentException(
+          "EnumValueDescriptor is not for this type.");
       }
       return VALUES[desc.getIndex()];
     }
 
+    private final int index;
     private final int value;
 
-    private EntryType(int value) {
+    private EntryType(int index, int value) {
+      this.index = index;
       this.value = value;
     }
 
@@ -161,97 +130,88 @@ public final class CanalEntry {
   }
 
   /**
+   * Protobuf enum {@code com.alibaba.otter.canal.protocol.EventType}
+   *
    * 
          ** 事件类型 *
    * 
      
-   *
-   * Protobuf enum {@code com.alibaba.otter.canal.protocol.EventType}
    */
   public enum EventType
       implements com.google.protobuf.ProtocolMessageEnum {
-    /**
-     * EVENTTYPECOMPATIBLEPROTO2 = 0;
-     */
-    EVENTTYPECOMPATIBLEPROTO2(0),
     /**
      * INSERT = 1;
      */
-    INSERT(1),
+    INSERT(0, 1),
     /**
      * UPDATE = 2;
      */
-    UPDATE(2),
+    UPDATE(1, 2),
     /**
      * DELETE = 3;
      */
-    DELETE(3),
+    DELETE(2, 3),
     /**
      * CREATE = 4;
      */
-    CREATE(4),
+    CREATE(3, 4),
     /**
      * ALTER = 5;
      */
-    ALTER(5),
+    ALTER(4, 5),
     /**
      * ERASE = 6;
      */
-    ERASE(6),
+    ERASE(5, 6),
     /**
      * QUERY = 7;
      */
-    QUERY(7),
+    QUERY(6, 7),
     /**
      * TRUNCATE = 8;
      */
-    TRUNCATE(8),
+    TRUNCATE(7, 8),
     /**
      * RENAME = 9;
      */
-    RENAME(9),
+    RENAME(8, 9),
     /**
+     * CINDEX = 10;
+     *
      * 
            **CREATE INDEX*
      * 
      
-     *
-     * CINDEX = 10;
      */
-    CINDEX(10),
+    CINDEX(9, 10),
     /**
      * DINDEX = 11;
      */
-    DINDEX(11),
+    DINDEX(10, 11),
     /**
      * GTID = 12;
      */
-    GTID(12),
+    GTID(11, 12),
     /**
+     * XACOMMIT = 13;
+     *
      * 
            ** XA *
      * 
      
-     *
-     * XACOMMIT = 13;
      */
-    XACOMMIT(13),
+    XACOMMIT(12, 13),
     /**
      * XAROLLBACK = 14;
      */
-    XAROLLBACK(14),
+    XAROLLBACK(13, 14),
     /**
+     * MHEARTBEAT = 15;
+     *
      * 
            ** MASTER HEARTBEAT *
      * 
      
-     *
-     * MHEARTBEAT = 15;
      */
-    MHEARTBEAT(15),
-    UNRECOGNIZED(-1),
+    MHEARTBEAT(14, 15),
     ;
 
-    /**
-     * EVENTTYPECOMPATIBLEPROTO2 = 0;
-     */
-    public static final int EVENTTYPECOMPATIBLEPROTO2_VALUE = 0;
     /**
      * INSERT = 1;
      */
@@ -289,11 +249,11 @@ public final class CanalEntry {
      */
     public static final int RENAME_VALUE = 9;
     /**
+     * CINDEX = 10;
+     *
      * 
            **CREATE INDEX*
      * 
      
-     *
-     * CINDEX = 10;
      */
     public static final int CINDEX_VALUE = 10;
     /**
@@ -305,11 +265,11 @@ public final class CanalEntry {
      */
     public static final int GTID_VALUE = 12;
     /**
+     * XACOMMIT = 13;
+     *
      * 
            ** XA *
      * 
      
-     *
-     * XACOMMIT = 13;
      */
     public static final int XACOMMIT_VALUE = 13;
     /**
@@ -317,34 +277,19 @@ public final class CanalEntry {
      */
     public static final int XAROLLBACK_VALUE = 14;
     /**
+     * MHEARTBEAT = 15;
+     *
      * 
            ** MASTER HEARTBEAT *
      * 
      
-     *
-     * MHEARTBEAT = 15;
      */
     public static final int MHEARTBEAT_VALUE = 15;
 
 
-    public final int getNumber() {
-      if (this == UNRECOGNIZED) {
-        throw new java.lang.IllegalArgumentException(
-            "Can't get the number of an unknown enum value.");
-      }
-      return value;
-    }
+    public final int getNumber() { return value; }
 
-    /**
-     * @deprecated Use {@link #forNumber(int)} instead.
-     */
-    @java.lang.Deprecated
     public static EventType valueOf(int value) {
-      return forNumber(value);
-    }
-
-    public static EventType forNumber(int value) {
       switch (value) {
-        case 0: return EVENTTYPECOMPATIBLEPROTO2;
         case 1: return INSERT;
         case 2: return UPDATE;
         case 3: return DELETE;
@@ -368,17 +313,17 @@ public final class CanalEntry {
         internalGetValueMap() {
       return internalValueMap;
     }
-    private static final com.google.protobuf.Internal.EnumLiteMap<
-        EventType> internalValueMap =
+    private static com.google.protobuf.Internal.EnumLiteMap
+        internalValueMap =
           new com.google.protobuf.Internal.EnumLiteMap() {
             public EventType findValueByNumber(int number) {
-              return EventType.forNumber(number);
+              return EventType.valueOf(number);
             }
           };
 
     public final com.google.protobuf.Descriptors.EnumValueDescriptor
         getValueDescriptor() {
-      return getDescriptor().getValues().get(ordinal());
+      return getDescriptor().getValues().get(index);
     }
     public final com.google.protobuf.Descriptors.EnumDescriptor
         getDescriptorForType() {
@@ -386,7 +331,7 @@ public final class CanalEntry {
     }
     public static final com.google.protobuf.Descriptors.EnumDescriptor
         getDescriptor() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.getDescriptor().getEnumTypes().get(1);
+      return CanalEntry.getDescriptor().getEnumTypes().get(1);
     }
 
     private static final EventType[] VALUES = values();
@@ -394,18 +339,17 @@ public final class CanalEntry {
     public static EventType valueOf(
         com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
       if (desc.getType() != getDescriptor()) {
-        throw new java.lang.IllegalArgumentException(
+        throw new IllegalArgumentException(
           "EnumValueDescriptor is not for this type.");
       }
-      if (desc.getIndex() == -1) {
-        return UNRECOGNIZED;
-      }
       return VALUES[desc.getIndex()];
     }
 
+    private final int index;
     private final int value;
 
-    private EventType(int value) {
+    private EventType(int index, int value) {
+      this.index = index;
       this.value = value;
     }
 
@@ -413,37 +357,28 @@ public final class CanalEntry {
   }
 
   /**
+   * Protobuf enum {@code com.alibaba.otter.canal.protocol.Type}
+   *
    * 
          **数据库类型*
    * 
      
-   *
-   * Protobuf enum {@code com.alibaba.otter.canal.protocol.Type}
    */
   public enum Type
       implements com.google.protobuf.ProtocolMessageEnum {
-    /**
-     * TYPECOMPATIBLEPROTO2 = 0;
-     */
-    TYPECOMPATIBLEPROTO2(0),
     /**
      * ORACLE = 1;
      */
-    ORACLE(1),
+    ORACLE(0, 1),
     /**
      * MYSQL = 2;
      */
-    MYSQL(2),
+    MYSQL(1, 2),
     /**
      * PGSQL = 3;
      */
-    PGSQL(3),
-    UNRECOGNIZED(-1),
+    PGSQL(2, 3),
     ;
 
-    /**
-     * TYPECOMPATIBLEPROTO2 = 0;
-     */
-    public static final int TYPECOMPATIBLEPROTO2_VALUE = 0;
     /**
      * ORACLE = 1;
      */
@@ -458,25 +393,10 @@ public final class CanalEntry {
     public static final int PGSQL_VALUE = 3;
 
 
-    public final int getNumber() {
-      if (this == UNRECOGNIZED) {
-        throw new java.lang.IllegalArgumentException(
-            "Can't get the number of an unknown enum value.");
-      }
-      return value;
-    }
+    public final int getNumber() { return value; }
 
-    /**
-     * @deprecated Use {@link #forNumber(int)} instead.
-     */
-    @java.lang.Deprecated
     public static Type valueOf(int value) {
-      return forNumber(value);
-    }
-
-    public static Type forNumber(int value) {
       switch (value) {
-        case 0: return TYPECOMPATIBLEPROTO2;
         case 1: return ORACLE;
         case 2: return MYSQL;
         case 3: return PGSQL;
@@ -488,17 +408,17 @@ public final class CanalEntry {
         internalGetValueMap() {
       return internalValueMap;
     }
-    private static final com.google.protobuf.Internal.EnumLiteMap<
-        Type> internalValueMap =
+    private static com.google.protobuf.Internal.EnumLiteMap
+        internalValueMap =
           new com.google.protobuf.Internal.EnumLiteMap() {
             public Type findValueByNumber(int number) {
-              return Type.forNumber(number);
+              return Type.valueOf(number);
             }
           };
 
     public final com.google.protobuf.Descriptors.EnumValueDescriptor
         getValueDescriptor() {
-      return getDescriptor().getValues().get(ordinal());
+      return getDescriptor().getValues().get(index);
     }
     public final com.google.protobuf.Descriptors.EnumDescriptor
         getDescriptorForType() {
@@ -506,7 +426,7 @@ public final class CanalEntry {
     }
     public static final com.google.protobuf.Descriptors.EnumDescriptor
         getDescriptor() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.getDescriptor().getEnumTypes().get(2);
+      return CanalEntry.getDescriptor().getEnumTypes().get(2);
     }
 
     private static final Type[] VALUES = values();
@@ -514,18 +434,17 @@ public final class CanalEntry {
     public static Type valueOf(
         com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
       if (desc.getType() != getDescriptor()) {
-        throw new java.lang.IllegalArgumentException(
+        throw new IllegalArgumentException(
           "EnumValueDescriptor is not for this type.");
       }
-      if (desc.getIndex() == -1) {
-        return UNRECOGNIZED;
-      }
       return VALUES[desc.getIndex()];
     }
 
+    private final int index;
     private final int value;
 
-    private Type(int value) {
+    private Type(int index, int value) {
+      this.index = index;
       this.value = value;
     }
 
@@ -537,86 +456,105 @@ public final class CanalEntry {
       com.google.protobuf.MessageOrBuilder {
 
     /**
+     * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+     *
      * 
            **协议头部信息*
      * 
      
-     *
-     * .com.alibaba.otter.canal.protocol.Header header = 1;
      */
     boolean hasHeader();
     /**
+     * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+     *
      * 
            **协议头部信息*
      * 
      
-     *
-     * .com.alibaba.otter.canal.protocol.Header header = 1;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.Header getHeader();
+    Header getHeader();
     /**
+     * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+     *
      * 
            **协议头部信息*
      * 
      
+     */
+    HeaderOrBuilder getHeaderOrBuilder();
+
+    /**
+     * optional .com.alibaba.otter.canal.protocol.EntryType entryType = 2 [default = ROWDATA];
      *
-     * .com.alibaba.otter.canal.protocol.Header header = 1;
+     * 
      +     **打散后的事件类型*
+     * 
      
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder getHeaderOrBuilder();
-
-    /**
-     * .com.alibaba.otter.canal.protocol.EntryType entryType = 2;
-     */
-    int getEntryTypeValue();
-    /**
-     * .com.alibaba.otter.canal.protocol.EntryType entryType = 2;
-     */
-    com.alibaba.otter.canal.protocol.CanalEntry.EntryType getEntryType();
+    boolean hasEntryType();
+    /**
+     * optional .com.alibaba.otter.canal.protocol.EntryType entryType = 2 [default = ROWDATA];
+     *
+     * 
      +     **打散后的事件类型*
+     * 
      
+     */
+    EntryType getEntryType();
 
     /**
+     * optional bytes storeValue = 3;
+     *
      * 
            **传输的二进制数组*
      * 
      
+     */
+    boolean hasStoreValue();
+    /**
+     * optional bytes storeValue = 3;
      *
-     * bytes storeValue = 3;
+     * 
      +     **传输的二进制数组*
+     * 
      
      */
     com.google.protobuf.ByteString getStoreValue();
-
-    public com.alibaba.otter.canal.protocol.CanalEntry.Entry.EntryTypePresentCase getEntryTypePresentCase();
   }
   /**
+   * Protobuf type {@code com.alibaba.otter.canal.protocol.Entry}
+   *
    * 
          ****************************************************************
    * message model
    *如果要在Enum中新增类型，确保以前的类型的下标值不变.
    ***************************************************************
    * 
      
-   *
-   * Protobuf type {@code com.alibaba.otter.canal.protocol.Entry}
    */
-  public  static final class Entry extends
-      com.google.protobuf.GeneratedMessageV3 implements
+  public static final class Entry extends
+      com.google.protobuf.GeneratedMessage implements
       // @@protoc_insertion_point(message_implements:com.alibaba.otter.canal.protocol.Entry)
       EntryOrBuilder {
-  private static final long serialVersionUID = 0L;
     // Use Entry.newBuilder() to construct.
-    private Entry(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+    private Entry(com.google.protobuf.GeneratedMessage.Builder builder) {
       super(builder);
+      this.unknownFields = builder.getUnknownFields();
     }
-    private Entry() {
-      storeValue_ = com.google.protobuf.ByteString.EMPTY;
+    private Entry(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+    private static final Entry defaultInstance;
+    public static Entry getDefaultInstance() {
+      return defaultInstance;
     }
 
-    @java.lang.Override
+    public Entry getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+
+    private final com.google.protobuf.UnknownFieldSet unknownFields;
+    @Override
     public final com.google.protobuf.UnknownFieldSet
-    getUnknownFields() {
+        getUnknownFields() {
       return this.unknownFields;
     }
     private Entry(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
-      this();
-      if (extensionRegistry == null) {
-        throw new java.lang.NullPointerException();
-      }
+      initFields();
       int mutable_bitField0_ = 0;
       com.google.protobuf.UnknownFieldSet.Builder unknownFields =
           com.google.protobuf.UnknownFieldSet.newBuilder();
@@ -628,44 +566,49 @@ public final class CanalEntry {
             case 0:
               done = true;
               break;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                done = true;
+              }
+              break;
+            }
             case 10: {
-              com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder subBuilder = null;
-              if (header_ != null) {
+              Header.Builder subBuilder = null;
+              if (((bitField0_ & 0x00000001) == 0x00000001)) {
                 subBuilder = header_.toBuilder();
               }
-              header_ = input.readMessage(com.alibaba.otter.canal.protocol.CanalEntry.Header.parser(), extensionRegistry);
+              header_ = input.readMessage(Header.PARSER, extensionRegistry);
               if (subBuilder != null) {
                 subBuilder.mergeFrom(header_);
                 header_ = subBuilder.buildPartial();
               }
-
+              bitField0_ |= 0x00000001;
               break;
             }
             case 16: {
               int rawValue = input.readEnum();
-              entryTypePresentCase_ = 2;
-              entryTypePresent_ = rawValue;
+              EntryType value = EntryType.valueOf(rawValue);
+              if (value == null) {
+                unknownFields.mergeVarintField(2, rawValue);
+              } else {
+                bitField0_ |= 0x00000002;
+                entryType_ = value;
+              }
               break;
             }
             case 26: {
-
+              bitField0_ |= 0x00000004;
               storeValue_ = input.readBytes();
               break;
             }
-            default: {
-              if (!parseUnknownFieldProto3(
-                  input, unknownFields, extensionRegistry, tag)) {
-                done = true;
-              }
-              break;
-            }
           }
         }
       } catch (com.google.protobuf.InvalidProtocolBufferException e) {
         throw e.setUnfinishedMessage(this);
       } catch (java.io.IOException e) {
         throw new com.google.protobuf.InvalidProtocolBufferException(
-            e).setUnfinishedMessage(this);
+            e.getMessage()).setUnfinishedMessage(this);
       } finally {
         this.unknownFields = unknownFields.build();
         makeExtensionsImmutable();
@@ -673,124 +616,117 @@ public final class CanalEntry {
     }
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor;
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor;
     }
 
-    @java.lang.Override
-    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+    protected FieldAccessorTable
         internalGetFieldAccessorTable() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_fieldAccessorTable
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_fieldAccessorTable
           .ensureFieldAccessorsInitialized(
-              com.alibaba.otter.canal.protocol.CanalEntry.Entry.class, com.alibaba.otter.canal.protocol.CanalEntry.Entry.Builder.class);
+              Entry.class, Builder.class);
     }
 
-    private int entryTypePresentCase_ = 0;
-    private java.lang.Object entryTypePresent_;
-    public enum EntryTypePresentCase
-        implements com.google.protobuf.Internal.EnumLite {
-      ENTRYTYPE(2),
-      ENTRYTYPEPRESENT_NOT_SET(0);
-      private final int value;
-      private EntryTypePresentCase(int value) {
-        this.value = value;
-      }
-      /**
-       * @deprecated Use {@link #forNumber(int)} instead.
-       */
-      @java.lang.Deprecated
-      public static EntryTypePresentCase valueOf(int value) {
-        return forNumber(value);
-      }
-
-      public static EntryTypePresentCase forNumber(int value) {
-        switch (value) {
-          case 2: return ENTRYTYPE;
-          case 0: return ENTRYTYPEPRESENT_NOT_SET;
-          default: return null;
-        }
-      }
-      public int getNumber() {
-        return this.value;
+    public static com.google.protobuf.Parser PARSER =
+        new com.google.protobuf.AbstractParser() {
+      public Entry parsePartialFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return new Entry(input, extensionRegistry);
       }
     };
 
-    public EntryTypePresentCase
-    getEntryTypePresentCase() {
-      return EntryTypePresentCase.forNumber(
-          entryTypePresentCase_);
+    @Override
+    public com.google.protobuf.Parser getParserForType() {
+      return PARSER;
     }
 
+    private int bitField0_;
     public static final int HEADER_FIELD_NUMBER = 1;
-    private com.alibaba.otter.canal.protocol.CanalEntry.Header header_;
+    private Header header_;
     /**
+     * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+     *
      * 
            **协议头部信息*
      * 
      
-     *
-     * .com.alibaba.otter.canal.protocol.Header header = 1;
      */
     public boolean hasHeader() {
-      return header_ != null;
+      return ((bitField0_ & 0x00000001) == 0x00000001);
     }
     /**
+     * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+     *
      * 
            **协议头部信息*
      * 
      
-     *
-     * .com.alibaba.otter.canal.protocol.Header header = 1;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.Header getHeader() {
-      return header_ == null ? com.alibaba.otter.canal.protocol.CanalEntry.Header.getDefaultInstance() : header_;
+    public Header getHeader() {
+      return header_;
     }
     /**
+     * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+     *
      * 
            **协议头部信息*
      * 
      
-     *
-     * .com.alibaba.otter.canal.protocol.Header header = 1;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder getHeaderOrBuilder() {
-      return getHeader();
+    public HeaderOrBuilder getHeaderOrBuilder() {
+      return header_;
     }
 
     public static final int ENTRYTYPE_FIELD_NUMBER = 2;
+    private EntryType entryType_;
     /**
-     * .com.alibaba.otter.canal.protocol.EntryType entryType = 2;
+     * optional .com.alibaba.otter.canal.protocol.EntryType entryType = 2 [default = ROWDATA];
+     *
+     * 
      +     **打散后的事件类型*
+     * 
      
      */
-    public int getEntryTypeValue() {
-      if (entryTypePresentCase_ == 2) {
-        return (java.lang.Integer) entryTypePresent_;
-      }
-      return EntryType.ROWDATA_VALUE;
+    public boolean hasEntryType() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
     }
     /**
-     * .com.alibaba.otter.canal.protocol.EntryType entryType = 2;
+     * optional .com.alibaba.otter.canal.protocol.EntryType entryType = 2 [default = ROWDATA];
+     *
+     * 
      +     **打散后的事件类型*
+     * 
      
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.EntryType getEntryType() {
-      if (entryTypePresentCase_ == 2) {
-        @SuppressWarnings("deprecation")
-        com.alibaba.otter.canal.protocol.CanalEntry.EntryType result = com.alibaba.otter.canal.protocol.CanalEntry.EntryType.valueOf(
-            (java.lang.Integer) entryTypePresent_);
-        return result == null ? com.alibaba.otter.canal.protocol.CanalEntry.EntryType.UNRECOGNIZED : result;
-      }
-      return com.alibaba.otter.canal.protocol.CanalEntry.EntryType.TRANSACTIONBEGIN;
+    public EntryType getEntryType() {
+      return entryType_;
     }
 
     public static final int STOREVALUE_FIELD_NUMBER = 3;
     private com.google.protobuf.ByteString storeValue_;
     /**
+     * optional bytes storeValue = 3;
+     *
      * 
            **传输的二进制数组*
      * 
      
+     */
+    public boolean hasStoreValue() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    /**
+     * optional bytes storeValue = 3;
      *
-     * bytes storeValue = 3;
+     * 
      +     **传输的二进制数组*
+     * 
      
      */
     public com.google.protobuf.ByteString getStoreValue() {
       return storeValue_;
     }
 
+    private void initFields() {
+      header_ = Header.getDefaultInstance();
+      entryType_ = EntryType.ROWDATA;
+      storeValue_ = com.google.protobuf.ByteString.EMPTY;
+    }
     private byte memoizedIsInitialized = -1;
-    @java.lang.Override
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized == 1) return true;
@@ -800,218 +736,141 @@ public final class CanalEntry {
       return true;
     }
 
-    @java.lang.Override
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
-      if (header_ != null) {
-        output.writeMessage(1, getHeader());
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeMessage(1, header_);
       }
-      if (entryTypePresentCase_ == 2) {
-        output.writeEnum(2, ((java.lang.Integer) entryTypePresent_));
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeEnum(2, entryType_.getNumber());
       }
-      if (!storeValue_.isEmpty()) {
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
         output.writeBytes(3, storeValue_);
       }
-      unknownFields.writeTo(output);
+      getUnknownFields().writeTo(output);
     }
 
-    @java.lang.Override
+    private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
-      int size = memoizedSize;
+      int size = memoizedSerializedSize;
       if (size != -1) return size;
 
       size = 0;
-      if (header_ != null) {
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
-          .computeMessageSize(1, getHeader());
+          .computeMessageSize(1, header_);
       }
-      if (entryTypePresentCase_ == 2) {
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
         size += com.google.protobuf.CodedOutputStream
-          .computeEnumSize(2, ((java.lang.Integer) entryTypePresent_));
+          .computeEnumSize(2, entryType_.getNumber());
       }
-      if (!storeValue_.isEmpty()) {
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
         size += com.google.protobuf.CodedOutputStream
           .computeBytesSize(3, storeValue_);
       }
-      size += unknownFields.getSerializedSize();
-      memoizedSize = size;
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
       return size;
     }
 
-    @java.lang.Override
-    public boolean equals(final java.lang.Object obj) {
-      if (obj == this) {
-       return true;
-      }
-      if (!(obj instanceof com.alibaba.otter.canal.protocol.CanalEntry.Entry)) {
-        return super.equals(obj);
-      }
-      com.alibaba.otter.canal.protocol.CanalEntry.Entry other = (com.alibaba.otter.canal.protocol.CanalEntry.Entry) obj;
-
-      boolean result = true;
-      result = result && (hasHeader() == other.hasHeader());
-      if (hasHeader()) {
-        result = result && getHeader()
-            .equals(other.getHeader());
-      }
-      result = result && getStoreValue()
-          .equals(other.getStoreValue());
-      result = result && getEntryTypePresentCase().equals(
-          other.getEntryTypePresentCase());
-      if (!result) return false;
-      switch (entryTypePresentCase_) {
-        case 2:
-          result = result && getEntryTypeValue()
-              == other.getEntryTypeValue();
-          break;
-        case 0:
-        default:
-      }
-      result = result && unknownFields.equals(other.unknownFields);
-      return result;
+    private static final long serialVersionUID = 0L;
+    @Override
+    protected Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
     }
 
-    @java.lang.Override
-    public int hashCode() {
-      if (memoizedHashCode != 0) {
-        return memoizedHashCode;
-      }
-      int hash = 41;
-      hash = (19 * hash) + getDescriptor().hashCode();
-      if (hasHeader()) {
-        hash = (37 * hash) + HEADER_FIELD_NUMBER;
-        hash = (53 * hash) + getHeader().hashCode();
-      }
-      hash = (37 * hash) + STOREVALUE_FIELD_NUMBER;
-      hash = (53 * hash) + getStoreValue().hashCode();
-      switch (entryTypePresentCase_) {
-        case 2:
-          hash = (37 * hash) + ENTRYTYPE_FIELD_NUMBER;
-          hash = (53 * hash) + getEntryTypeValue();
-          break;
-        case 0:
-        default:
-      }
-      hash = (29 * hash) + unknownFields.hashCode();
-      memoizedHashCode = hash;
-      return hash;
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
-        java.nio.ByteBuffer data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
-        java.nio.ByteBuffer data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
+    public static Entry parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
+    public static Entry parseFrom(
         com.google.protobuf.ByteString data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(byte[] data)
+    public static Entry parseFrom(byte[] data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
+    public static Entry parseFrom(
         byte[] data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(java.io.InputStream input)
+    public static Entry parseFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
+    public static Entry parseFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseDelimitedFrom(java.io.InputStream input)
+    public static Entry parseDelimitedFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input);
+      return PARSER.parseDelimitedFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseDelimitedFrom(
+    public static Entry parseDelimitedFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseDelimitedFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
+    public static Entry parseFrom(
         com.google.protobuf.CodedInputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
+    public static Entry parseFrom(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
 
-    @java.lang.Override
+    public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
-    public static Builder newBuilder() {
-      return DEFAULT_INSTANCE.toBuilder();
-    }
-    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.Entry prototype) {
-      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
-    }
-    @java.lang.Override
-    public Builder toBuilder() {
-      return this == DEFAULT_INSTANCE
-          ? new Builder() : new Builder().mergeFrom(this);
+    public static Builder newBuilder(Entry prototype) {
+      return newBuilder().mergeFrom(prototype);
     }
+    public Builder toBuilder() { return newBuilder(this); }
 
-    @java.lang.Override
+    @Override
     protected Builder newBuilderForType(
-        com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        BuilderParent parent) {
       Builder builder = new Builder(parent);
       return builder;
     }
     /**
+     * Protobuf type {@code com.alibaba.otter.canal.protocol.Entry}
+     *
      * 
            ****************************************************************
      * message model
      *如果要在Enum中新增类型，确保以前的类型的下标值不变.
      ***************************************************************
      * 
      
-     *
-     * Protobuf type {@code com.alibaba.otter.canal.protocol.Entry}
      */
     public static final class Builder extends
-        com.google.protobuf.GeneratedMessageV3.Builder implements
+        com.google.protobuf.GeneratedMessage.Builder implements
         // @@protoc_insertion_point(builder_implements:com.alibaba.otter.canal.protocol.Entry)
-        com.alibaba.otter.canal.protocol.CanalEntry.EntryOrBuilder {
+        EntryOrBuilder {
       public static final com.google.protobuf.Descriptors.Descriptor
           getDescriptor() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor;
       }
 
-      @java.lang.Override
-      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      protected FieldAccessorTable
           internalGetFieldAccessorTable() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_fieldAccessorTable
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_fieldAccessorTable
             .ensureFieldAccessorsInitialized(
-                com.alibaba.otter.canal.protocol.CanalEntry.Entry.class, com.alibaba.otter.canal.protocol.CanalEntry.Entry.Builder.class);
+                Entry.class, Builder.class);
       }
 
       // Construct using com.alibaba.otter.canal.protocol.CanalEntry.Entry.newBuilder()
@@ -1020,148 +879,118 @@ public final class CanalEntry {
       }
 
       private Builder(
-          com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
       private void maybeForceBuilderInitialization() {
-        if (com.google.protobuf.GeneratedMessageV3
-                .alwaysUseFieldBuilders) {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getHeaderFieldBuilder();
         }
       }
-      @java.lang.Override
+      private static Builder create() {
+        return new Builder();
+      }
+
       public Builder clear() {
         super.clear();
         if (headerBuilder_ == null) {
-          header_ = null;
+          header_ = Header.getDefaultInstance();
         } else {
-          header_ = null;
-          headerBuilder_ = null;
+          headerBuilder_.clear();
         }
+        bitField0_ = (bitField0_ & ~0x00000001);
+        entryType_ = EntryType.ROWDATA;
+        bitField0_ = (bitField0_ & ~0x00000002);
         storeValue_ = com.google.protobuf.ByteString.EMPTY;
-
-        entryTypePresentCase_ = 0;
-        entryTypePresent_ = null;
+        bitField0_ = (bitField0_ & ~0x00000004);
         return this;
       }
 
-      @java.lang.Override
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.Entry getDefaultInstanceForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.Entry.getDefaultInstance();
+      public Entry getDefaultInstanceForType() {
+        return Entry.getDefaultInstance();
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.Entry build() {
-        com.alibaba.otter.canal.protocol.CanalEntry.Entry result = buildPartial();
+      public Entry build() {
+        Entry result = buildPartial();
         if (!result.isInitialized()) {
           throw newUninitializedMessageException(result);
         }
         return result;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.Entry buildPartial() {
-        com.alibaba.otter.canal.protocol.CanalEntry.Entry result = new com.alibaba.otter.canal.protocol.CanalEntry.Entry(this);
+      public Entry buildPartial() {
+        Entry result = new Entry(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
         if (headerBuilder_ == null) {
           result.header_ = header_;
         } else {
           result.header_ = headerBuilder_.build();
         }
-        if (entryTypePresentCase_ == 2) {
-          result.entryTypePresent_ = entryTypePresent_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.entryType_ = entryType_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
         }
         result.storeValue_ = storeValue_;
-        result.entryTypePresentCase_ = entryTypePresentCase_;
+        result.bitField0_ = to_bitField0_;
         onBuilt();
         return result;
       }
 
-      @java.lang.Override
-      public Builder clone() {
-        return (Builder) super.clone();
-      }
-      @java.lang.Override
-      public Builder setField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.setField(field, value);
-      }
-      @java.lang.Override
-      public Builder clearField(
-          com.google.protobuf.Descriptors.FieldDescriptor field) {
-        return (Builder) super.clearField(field);
-      }
-      @java.lang.Override
-      public Builder clearOneof(
-          com.google.protobuf.Descriptors.OneofDescriptor oneof) {
-        return (Builder) super.clearOneof(oneof);
-      }
-      @java.lang.Override
-      public Builder setRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          int index, java.lang.Object value) {
-        return (Builder) super.setRepeatedField(field, index, value);
-      }
-      @java.lang.Override
-      public Builder addRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.addRepeatedField(field, value);
-      }
-      @java.lang.Override
       public Builder mergeFrom(com.google.protobuf.Message other) {
-        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.Entry) {
-          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.Entry)other);
+        if (other instanceof Entry) {
+          return mergeFrom((Entry)other);
         } else {
           super.mergeFrom(other);
           return this;
         }
       }
 
-      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.Entry other) {
-        if (other == com.alibaba.otter.canal.protocol.CanalEntry.Entry.getDefaultInstance()) return this;
+      public Builder mergeFrom(Entry other) {
+        if (other == Entry.getDefaultInstance()) return this;
         if (other.hasHeader()) {
           mergeHeader(other.getHeader());
         }
-        if (other.getStoreValue() != com.google.protobuf.ByteString.EMPTY) {
+        if (other.hasEntryType()) {
+          setEntryType(other.getEntryType());
+        }
+        if (other.hasStoreValue()) {
           setStoreValue(other.getStoreValue());
         }
-        switch (other.getEntryTypePresentCase()) {
-          case ENTRYTYPE: {
-            setEntryTypeValue(other.getEntryTypeValue());
-            break;
-          }
-          case ENTRYTYPEPRESENT_NOT_SET: {
-            break;
-          }
-        }
-        this.mergeUnknownFields(other.unknownFields);
-        onChanged();
+        this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
 
-      @java.lang.Override
       public final boolean isInitialized() {
         return true;
       }
 
-      @java.lang.Override
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
           throws java.io.IOException {
-        com.alibaba.otter.canal.protocol.CanalEntry.Entry parsedMessage = null;
+        Entry parsedMessage = null;
         try {
           parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
         } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          parsedMessage = (com.alibaba.otter.canal.protocol.CanalEntry.Entry) e.getUnfinishedMessage();
-          throw e.unwrapIOException();
+          parsedMessage = (Entry) e.getUnfinishedMessage();
+          throw e;
         } finally {
           if (parsedMessage != null) {
             mergeFrom(parsedMessage);
@@ -1169,57 +998,43 @@ public final class CanalEntry {
         }
         return this;
       }
-      private int entryTypePresentCase_ = 0;
-      private java.lang.Object entryTypePresent_;
-      public EntryTypePresentCase
-          getEntryTypePresentCase() {
-        return EntryTypePresentCase.forNumber(
-            entryTypePresentCase_);
-      }
+      private int bitField0_;
 
-      public Builder clearEntryTypePresent() {
-        entryTypePresentCase_ = 0;
-        entryTypePresent_ = null;
-        onChanged();
-        return this;
-      }
-
-
-      private com.alibaba.otter.canal.protocol.CanalEntry.Header header_ = null;
-      private com.google.protobuf.SingleFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Header, com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder, com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder> headerBuilder_;
+      private Header header_ = Header.getDefaultInstance();
+      private com.google.protobuf.SingleFieldBuilder<
+          Header, Header.Builder, HeaderOrBuilder> headerBuilder_;
       /**
+       * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+       *
        * 
              **协议头部信息*
        * 
      
-       *
-       * .com.alibaba.otter.canal.protocol.Header header = 1;
        */
       public boolean hasHeader() {
-        return headerBuilder_ != null || header_ != null;
+        return ((bitField0_ & 0x00000001) == 0x00000001);
       }
       /**
+       * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+       *
        * 
              **协议头部信息*
        * 
      
-       *
-       * .com.alibaba.otter.canal.protocol.Header header = 1;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Header getHeader() {
+      public Header getHeader() {
         if (headerBuilder_ == null) {
-          return header_ == null ? com.alibaba.otter.canal.protocol.CanalEntry.Header.getDefaultInstance() : header_;
+          return header_;
         } else {
           return headerBuilder_.getMessage();
         }
       }
       /**
+       * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+       *
        * 
              **协议头部信息*
        * 
      
-       *
-       * .com.alibaba.otter.canal.protocol.Header header = 1;
        */
-      public Builder setHeader(com.alibaba.otter.canal.protocol.CanalEntry.Header value) {
+      public Builder setHeader(Header value) {
         if (headerBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -1229,39 +1044,40 @@ public final class CanalEntry {
         } else {
           headerBuilder_.setMessage(value);
         }
-
+        bitField0_ |= 0x00000001;
         return this;
       }
       /**
+       * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+       *
        * 
              **协议头部信息*
        * 
      
-       *
-       * .com.alibaba.otter.canal.protocol.Header header = 1;
        */
       public Builder setHeader(
-          com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder builderForValue) {
+          Header.Builder builderForValue) {
         if (headerBuilder_ == null) {
           header_ = builderForValue.build();
           onChanged();
         } else {
           headerBuilder_.setMessage(builderForValue.build());
         }
-
+        bitField0_ |= 0x00000001;
         return this;
       }
       /**
+       * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+       *
        * 
              **协议头部信息*
        * 
      
-       *
-       * .com.alibaba.otter.canal.protocol.Header header = 1;
        */
-      public Builder mergeHeader(com.alibaba.otter.canal.protocol.CanalEntry.Header value) {
+      public Builder mergeHeader(Header value) {
         if (headerBuilder_ == null) {
-          if (header_ != null) {
+          if (((bitField0_ & 0x00000001) == 0x00000001) &&
+              header_ != Header.getDefaultInstance()) {
             header_ =
-              com.alibaba.otter.canal.protocol.CanalEntry.Header.newBuilder(header_).mergeFrom(value).buildPartial();
+              Header.newBuilder(header_).mergeFrom(value).buildPartial();
           } else {
             header_ = value;
           }
@@ -1269,67 +1085,65 @@ public final class CanalEntry {
         } else {
           headerBuilder_.mergeFrom(value);
         }
-
+        bitField0_ |= 0x00000001;
         return this;
       }
       /**
+       * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+       *
        * 
              **协议头部信息*
        * 
      
-       *
-       * .com.alibaba.otter.canal.protocol.Header header = 1;
        */
       public Builder clearHeader() {
         if (headerBuilder_ == null) {
-          header_ = null;
+          header_ = Header.getDefaultInstance();
           onChanged();
         } else {
-          header_ = null;
-          headerBuilder_ = null;
+          headerBuilder_.clear();
         }
-
+        bitField0_ = (bitField0_ & ~0x00000001);
         return this;
       }
       /**
+       * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+       *
        * 
              **协议头部信息*
        * 
      
-       *
-       * .com.alibaba.otter.canal.protocol.Header header = 1;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder getHeaderBuilder() {
-        
+      public Header.Builder getHeaderBuilder() {
+        bitField0_ |= 0x00000001;
         onChanged();
         return getHeaderFieldBuilder().getBuilder();
       }
       /**
+       * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+       *
        * 
              **协议头部信息*
        * 
      
-       *
-       * .com.alibaba.otter.canal.protocol.Header header = 1;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder getHeaderOrBuilder() {
+      public HeaderOrBuilder getHeaderOrBuilder() {
         if (headerBuilder_ != null) {
           return headerBuilder_.getMessageOrBuilder();
         } else {
-          return header_ == null ?
-              com.alibaba.otter.canal.protocol.CanalEntry.Header.getDefaultInstance() : header_;
+          return header_;
         }
       }
       /**
+       * optional .com.alibaba.otter.canal.protocol.Header header = 1;
+       *
        * 
              **协议头部信息*
        * 
      
-       *
-       * .com.alibaba.otter.canal.protocol.Header header = 1;
        */
-      private com.google.protobuf.SingleFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Header, com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder, com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder> 
+      private com.google.protobuf.SingleFieldBuilder<
+          Header, Header.Builder, HeaderOrBuilder>
           getHeaderFieldBuilder() {
         if (headerBuilder_ == null) {
-          headerBuilder_ = new com.google.protobuf.SingleFieldBuilderV3<
-              com.alibaba.otter.canal.protocol.CanalEntry.Header, com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder, com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder>(
+          headerBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+              Header, Header.Builder, HeaderOrBuilder>(
                   getHeader(),
                   getParentForChildren(),
                   isClean());
@@ -1338,151 +1152,117 @@ public final class CanalEntry {
         return headerBuilder_;
       }
 
+      private EntryType entryType_ = EntryType.ROWDATA;
       /**
-       * .com.alibaba.otter.canal.protocol.EntryType entryType = 2;
+       * optional .com.alibaba.otter.canal.protocol.EntryType entryType = 2 [default = ROWDATA];
+       *
+       * 
      +       **打散后的事件类型*
+       * 
      
        */
-      public int getEntryTypeValue() {
-        if (entryTypePresentCase_ == 2) {
-          return ((java.lang.Integer) entryTypePresent_).intValue();
-        }
-        return 0;
+      public boolean hasEntryType() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
       }
       /**
-       * .com.alibaba.otter.canal.protocol.EntryType entryType = 2;
+       * optional .com.alibaba.otter.canal.protocol.EntryType entryType = 2 [default = ROWDATA];
+       *
+       * 
      +       **打散后的事件类型*
+       * 
      
        */
-      public Builder setEntryTypeValue(int value) {
-        entryTypePresentCase_ = 2;
-        entryTypePresent_ = value;
-        onChanged();
-        return this;
+      public EntryType getEntryType() {
+        return entryType_;
       }
       /**
-       * .com.alibaba.otter.canal.protocol.EntryType entryType = 2;
+       * optional .com.alibaba.otter.canal.protocol.EntryType entryType = 2 [default = ROWDATA];
+       *
+       * 
      +       **打散后的事件类型*
+       * 
      
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.EntryType getEntryType() {
-        if (entryTypePresentCase_ == 2) {
-          @SuppressWarnings("deprecation")
-          com.alibaba.otter.canal.protocol.CanalEntry.EntryType result = com.alibaba.otter.canal.protocol.CanalEntry.EntryType.valueOf(
-              (java.lang.Integer) entryTypePresent_);
-          return result == null ? com.alibaba.otter.canal.protocol.CanalEntry.EntryType.UNRECOGNIZED : result;
-        }
-        return com.alibaba.otter.canal.protocol.CanalEntry.EntryType.TRANSACTIONBEGIN;
-      }
-      /**
-       * .com.alibaba.otter.canal.protocol.EntryType entryType = 2;
-       */
-      public Builder setEntryType(com.alibaba.otter.canal.protocol.CanalEntry.EntryType value) {
+      public Builder setEntryType(EntryType value) {
         if (value == null) {
           throw new NullPointerException();
         }
-        entryTypePresentCase_ = 2;
-        entryTypePresent_ = value.getNumber();
+        bitField0_ |= 0x00000002;
+        entryType_ = value;
         onChanged();
         return this;
       }
       /**
-       * .com.alibaba.otter.canal.protocol.EntryType entryType = 2;
+       * optional .com.alibaba.otter.canal.protocol.EntryType entryType = 2 [default = ROWDATA];
+       *
+       * 
      +       **打散后的事件类型*
+       * 
      
        */
       public Builder clearEntryType() {
-        if (entryTypePresentCase_ == 2) {
-          entryTypePresentCase_ = 0;
-          entryTypePresent_ = null;
-          onChanged();
-        }
+        bitField0_ = (bitField0_ & ~0x00000002);
+        entryType_ = EntryType.ROWDATA;
+        onChanged();
         return this;
       }
 
       private com.google.protobuf.ByteString storeValue_ = com.google.protobuf.ByteString.EMPTY;
       /**
+       * optional bytes storeValue = 3;
+       *
        * 
              **传输的二进制数组*
        * 
      
+       */
+      public boolean hasStoreValue() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      /**
+       * optional bytes storeValue = 3;
        *
-       * bytes storeValue = 3;
+       * 
      +       **传输的二进制数组*
+       * 
      
        */
       public com.google.protobuf.ByteString getStoreValue() {
         return storeValue_;
       }
       /**
+       * optional bytes storeValue = 3;
+       *
        * 
              **传输的二进制数组*
        * 
      
-       *
-       * bytes storeValue = 3;
        */
       public Builder setStoreValue(com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000004;
         storeValue_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional bytes storeValue = 3;
+       *
        * 
              **传输的二进制数组*
        * 
      
-       *
-       * bytes storeValue = 3;
        */
       public Builder clearStoreValue() {
-        
+        bitField0_ = (bitField0_ & ~0x00000004);
         storeValue_ = getDefaultInstance().getStoreValue();
         onChanged();
         return this;
       }
-      @java.lang.Override
-      public final Builder setUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.setUnknownFieldsProto3(unknownFields);
-      }
-
-      @java.lang.Override
-      public final Builder mergeUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.mergeUnknownFields(unknownFields);
-      }
-
 
       // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Entry)
     }
 
-    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Entry)
-    private static final com.alibaba.otter.canal.protocol.CanalEntry.Entry DEFAULT_INSTANCE;
     static {
-      DEFAULT_INSTANCE = new com.alibaba.otter.canal.protocol.CanalEntry.Entry();
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry getDefaultInstance() {
-      return DEFAULT_INSTANCE;
-    }
-
-    private static final com.google.protobuf.Parser
-        PARSER = new com.google.protobuf.AbstractParser() {
-      @java.lang.Override
-      public Entry parsePartialFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return new Entry(input, extensionRegistry);
-      }
-    };
-
-    public static com.google.protobuf.Parser parser() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.google.protobuf.Parser getParserForType() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.alibaba.otter.canal.protocol.CanalEntry.Entry getDefaultInstanceForType() {
-      return DEFAULT_INSTANCE;
+      defaultInstance = new Entry(true);
+      defaultInstance.initFields();
     }
 
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Entry)
   }
 
   public interface HeaderOrBuilder extends
@@ -1490,246 +1270,335 @@ public final class CanalEntry {
       com.google.protobuf.MessageOrBuilder {
 
     /**
-     * int32 version = 1;
+     * optional int32 version = 1 [default = 1];
+     *
+     * 
      +     **协议的版本号*
+     * 
      
+     */
+    boolean hasVersion();
+    /**
+     * optional int32 version = 1 [default = 1];
+     *
+     * 
      +     **协议的版本号*
+     * 
      
      */
     int getVersion();
 
     /**
+     * optional string logfileName = 2;
+     *
      * 
            **binlog/redolog 文件名*
      * 
      
-     *
-     * string logfileName = 2;
      */
-    java.lang.String getLogfileName();
+    boolean hasLogfileName();
     /**
+     * optional string logfileName = 2;
+     *
      * 
            **binlog/redolog 文件名*
      * 
      
+     */
+    String getLogfileName();
+    /**
+     * optional string logfileName = 2;
      *
-     * string logfileName = 2;
+     * 
      +     **binlog/redolog 文件名*
+     * 
      
      */
     com.google.protobuf.ByteString
         getLogfileNameBytes();
 
     /**
+     * optional int64 logfileOffset = 3;
+     *
      * 
            **binlog/redolog 文件的偏移位置*
      * 
      
+     */
+    boolean hasLogfileOffset();
+    /**
+     * optional int64 logfileOffset = 3;
      *
-     * int64 logfileOffset = 3;
+     * 
      +     **binlog/redolog 文件的偏移位置*
+     * 
      
      */
     long getLogfileOffset();
 
     /**
+     * optional int64 serverId = 4;
+     *
      * 
            **服务端serverId*
      * 
      
+     */
+    boolean hasServerId();
+    /**
+     * optional int64 serverId = 4;
      *
-     * int64 serverId = 4;
+     * 
      +     **服务端serverId*
+     * 
      
      */
     long getServerId();
 
     /**
+     * optional string serverenCode = 5;
+     *
      * 
            ** 变更数据的编码 *
      * 
      
-     *
-     * string serverenCode = 5;
      */
-    java.lang.String getServerenCode();
+    boolean hasServerenCode();
     /**
+     * optional string serverenCode = 5;
+     *
      * 
            ** 变更数据的编码 *
      * 
      
+     */
+    String getServerenCode();
+    /**
+     * optional string serverenCode = 5;
      *
-     * string serverenCode = 5;
+     * 
      +     ** 变更数据的编码 *
+     * 
      
      */
     com.google.protobuf.ByteString
         getServerenCodeBytes();
 
     /**
+     * optional int64 executeTime = 6;
+     *
      * 
            **变更数据的执行时间 *
      * 
      
+     */
+    boolean hasExecuteTime();
+    /**
+     * optional int64 executeTime = 6;
      *
-     * int64 executeTime = 6;
+     * 
      +     **变更数据的执行时间 *
+     * 
      
      */
     long getExecuteTime();
 
     /**
-     * .com.alibaba.otter.canal.protocol.Type sourceType = 7;
+     * optional .com.alibaba.otter.canal.protocol.Type sourceType = 7 [default = MYSQL];
+     *
+     * 
      +     ** 变更数据的来源*
+     * 
      
      */
-    int getSourceTypeValue();
+    boolean hasSourceType();
     /**
-     * .com.alibaba.otter.canal.protocol.Type sourceType = 7;
+     * optional .com.alibaba.otter.canal.protocol.Type sourceType = 7 [default = MYSQL];
+     *
+     * 
      +     ** 变更数据的来源*
+     * 
      
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.Type getSourceType();
+    Type getSourceType();
 
     /**
+     * optional string schemaName = 8;
+     *
      * 
            ** 变更数据的schemaname*
      * 
      
-     *
-     * string schemaName = 8;
      */
-    java.lang.String getSchemaName();
+    boolean hasSchemaName();
     /**
+     * optional string schemaName = 8;
+     *
      * 
            ** 变更数据的schemaname*
      * 
      
+     */
+    String getSchemaName();
+    /**
+     * optional string schemaName = 8;
      *
-     * string schemaName = 8;
+     * 
      +     ** 变更数据的schemaname*
+     * 
      
      */
     com.google.protobuf.ByteString
         getSchemaNameBytes();
 
     /**
+     * optional string tableName = 9;
+     *
      * 
            **变更数据的tablename*
      * 
      
-     *
-     * string tableName = 9;
      */
-    java.lang.String getTableName();
+    boolean hasTableName();
     /**
+     * optional string tableName = 9;
+     *
      * 
            **变更数据的tablename*
      * 
      
+     */
+    String getTableName();
+    /**
+     * optional string tableName = 9;
      *
-     * string tableName = 9;
+     * 
      +     **变更数据的tablename*
+     * 
      
      */
     com.google.protobuf.ByteString
         getTableNameBytes();
 
     /**
+     * optional int64 eventLength = 10;
+     *
      * 
            **每个event的长度*
      * 
      
+     */
+    boolean hasEventLength();
+    /**
+     * optional int64 eventLength = 10;
      *
-     * int64 eventLength = 10;
+     * 
      +     **每个event的长度*
+     * 
      
      */
     long getEventLength();
 
     /**
-     * .com.alibaba.otter.canal.protocol.EventType eventType = 11;
+     * optional .com.alibaba.otter.canal.protocol.EventType eventType = 11 [default = UPDATE];
+     *
+     * 
      +     **数据变更类型*
+     * 
      
      */
-    int getEventTypeValue();
+    boolean hasEventType();
     /**
-     * .com.alibaba.otter.canal.protocol.EventType eventType = 11;
+     * optional .com.alibaba.otter.canal.protocol.EventType eventType = 11 [default = UPDATE];
+     *
+     * 
      +     **数据变更类型*
+     * 
      
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.EventType getEventType();
+    EventType getEventType();
 
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
      */
-    java.util.List 
+    java.util.List
         getPropsList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index);
+    Pair getProps(int index);
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
      */
     int getPropsCount();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
      */
-    java.util.List 
+    java.util.List
         getPropsOrBuilderList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
-        int index);
+    PairOrBuilder getPropsOrBuilder(int index);
 
     /**
+     * optional string gtid = 13;
+     *
      * 
            **当前事务的gitd*
      * 
      
-     *
-     * string gtid = 13;
      */
-    java.lang.String getGtid();
+    boolean hasGtid();
     /**
+     * optional string gtid = 13;
+     *
      * 
            **当前事务的gitd*
      * 
      
+     */
+    String getGtid();
+    /**
+     * optional string gtid = 13;
      *
-     * string gtid = 13;
+     * 
      +     **当前事务的gitd*
+     * 
      
      */
     com.google.protobuf.ByteString
         getGtidBytes();
-
-    public com.alibaba.otter.canal.protocol.CanalEntry.Header.VersionPresentCase getVersionPresentCase();
-
-    public com.alibaba.otter.canal.protocol.CanalEntry.Header.SourceTypePresentCase getSourceTypePresentCase();
-
-    public com.alibaba.otter.canal.protocol.CanalEntry.Header.EventTypePresentCase getEventTypePresentCase();
   }
   /**
+   * Protobuf type {@code com.alibaba.otter.canal.protocol.Header}
+   *
    * 
          **message Header*
    * 
      
-   *
-   * Protobuf type {@code com.alibaba.otter.canal.protocol.Header}
    */
-  public  static final class Header extends
-      com.google.protobuf.GeneratedMessageV3 implements
+  public static final class Header extends
+      com.google.protobuf.GeneratedMessage implements
       // @@protoc_insertion_point(message_implements:com.alibaba.otter.canal.protocol.Header)
       HeaderOrBuilder {
-  private static final long serialVersionUID = 0L;
     // Use Header.newBuilder() to construct.
-    private Header(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+    private Header(com.google.protobuf.GeneratedMessage.Builder builder) {
       super(builder);
+      this.unknownFields = builder.getUnknownFields();
     }
-    private Header() {
-      logfileName_ = "";
-      logfileOffset_ = 0L;
-      serverId_ = 0L;
-      serverenCode_ = "";
-      executeTime_ = 0L;
-      schemaName_ = "";
-      tableName_ = "";
-      eventLength_ = 0L;
-      props_ = java.util.Collections.emptyList();
-      gtid_ = "";
+    private Header(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+    private static final Header defaultInstance;
+    public static Header getDefaultInstance() {
+      return defaultInstance;
     }
 
-    @java.lang.Override
+    public Header getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+
+    private final com.google.protobuf.UnknownFieldSet unknownFields;
+    @Override
     public final com.google.protobuf.UnknownFieldSet
-    getUnknownFields() {
+        getUnknownFields() {
       return this.unknownFields;
     }
     private Header(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
-      this();
-      if (extensionRegistry == null) {
-        throw new java.lang.NullPointerException();
-      }
+      initFields();
       int mutable_bitField0_ = 0;
       com.google.protobuf.UnknownFieldSet.Builder unknownFields =
           com.google.protobuf.UnknownFieldSet.newBuilder();
@@ -1741,87 +1610,96 @@ public final class CanalEntry {
             case 0:
               done = true;
               break;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                done = true;
+              }
+              break;
+            }
             case 8: {
-              versionPresentCase_ = 1;
-              versionPresent_ = input.readInt32();
+              bitField0_ |= 0x00000001;
+              version_ = input.readInt32();
               break;
             }
             case 18: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              logfileName_ = s;
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000002;
+              logfileName_ = bs;
               break;
             }
             case 24: {
-
+              bitField0_ |= 0x00000004;
               logfileOffset_ = input.readInt64();
               break;
             }
             case 32: {
-
+              bitField0_ |= 0x00000008;
               serverId_ = input.readInt64();
               break;
             }
             case 42: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              serverenCode_ = s;
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000010;
+              serverenCode_ = bs;
               break;
             }
             case 48: {
-
+              bitField0_ |= 0x00000020;
               executeTime_ = input.readInt64();
               break;
             }
             case 56: {
               int rawValue = input.readEnum();
-              sourceTypePresentCase_ = 7;
-              sourceTypePresent_ = rawValue;
+              Type value = Type.valueOf(rawValue);
+              if (value == null) {
+                unknownFields.mergeVarintField(7, rawValue);
+              } else {
+                bitField0_ |= 0x00000040;
+                sourceType_ = value;
+              }
               break;
             }
             case 66: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              schemaName_ = s;
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000080;
+              schemaName_ = bs;
               break;
             }
             case 74: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              tableName_ = s;
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000100;
+              tableName_ = bs;
               break;
             }
             case 80: {
-
+              bitField0_ |= 0x00000200;
               eventLength_ = input.readInt64();
               break;
             }
             case 88: {
               int rawValue = input.readEnum();
-              eventTypePresentCase_ = 11;
-              eventTypePresent_ = rawValue;
+              EventType value = EventType.valueOf(rawValue);
+              if (value == null) {
+                unknownFields.mergeVarintField(11, rawValue);
+              } else {
+                bitField0_ |= 0x00000400;
+                eventType_ = value;
+              }
               break;
             }
             case 98: {
               if (!((mutable_bitField0_ & 0x00000800) == 0x00000800)) {
-                props_ = new java.util.ArrayList();
+                props_ = new java.util.ArrayList();
                 mutable_bitField0_ |= 0x00000800;
               }
-              props_.add(
-                  input.readMessage(com.alibaba.otter.canal.protocol.CanalEntry.Pair.parser(), extensionRegistry));
+              props_.add(input.readMessage(Pair.PARSER, extensionRegistry));
               break;
             }
             case 106: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              gtid_ = s;
-              break;
-            }
-            default: {
-              if (!parseUnknownFieldProto3(
-                  input, unknownFields, extensionRegistry, tag)) {
-                done = true;
-              }
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000800;
+              gtid_ = bs;
               break;
             }
           }
@@ -1830,7 +1708,7 @@ public final class CanalEntry {
         throw e.setUnfinishedMessage(this);
       } catch (java.io.IOException e) {
         throw new com.google.protobuf.InvalidProtocolBufferException(
-            e).setUnfinishedMessage(this);
+            e.getMessage()).setUnfinishedMessage(this);
       } finally {
         if (((mutable_bitField0_ & 0x00000800) == 0x00000800)) {
           props_ = java.util.Collections.unmodifiableList(props_);
@@ -1841,172 +1719,102 @@ public final class CanalEntry {
     }
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_descriptor;
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_descriptor;
     }
 
-    @java.lang.Override
-    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+    protected FieldAccessorTable
         internalGetFieldAccessorTable() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_fieldAccessorTable
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_fieldAccessorTable
           .ensureFieldAccessorsInitialized(
-              com.alibaba.otter.canal.protocol.CanalEntry.Header.class, com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder.class);
+              Header.class, Builder.class);
+    }
+
+    public static com.google.protobuf.Parser
       PARSER =
+        new com.google.protobuf.AbstractParser
      () {
+      public Header parsePartialFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return new Header(input, extensionRegistry);
+      }
+    };
+
+    @Override
+    public com.google.protobuf.Parser
       getParserForType() {
+      return PARSER;
     }
 
     private int bitField0_;
-    private int versionPresentCase_ = 0;
-    private java.lang.Object versionPresent_;
-    public enum VersionPresentCase
-        implements com.google.protobuf.Internal.EnumLite {
-      VERSION(1),
-      VERSIONPRESENT_NOT_SET(0);
-      private final int value;
-      private VersionPresentCase(int value) {
-        this.value = value;
-      }
-      /**
-       * @deprecated Use {@link #forNumber(int)} instead.
-       */
-      @java.lang.Deprecated
-      public static VersionPresentCase valueOf(int value) {
-        return forNumber(value);
-      }
-
-      public static VersionPresentCase forNumber(int value) {
-        switch (value) {
-          case 1: return VERSION;
-          case 0: return VERSIONPRESENT_NOT_SET;
-          default: return null;
-        }
-      }
-      public int getNumber() {
-        return this.value;
-      }
-    };
-
-    public VersionPresentCase
-    getVersionPresentCase() {
-      return VersionPresentCase.forNumber(
-          versionPresentCase_);
-    }
-
-    private int sourceTypePresentCase_ = 0;
-    private java.lang.Object sourceTypePresent_;
-    public enum SourceTypePresentCase
-        implements com.google.protobuf.Internal.EnumLite {
-      SOURCETYPE(7),
-      SOURCETYPEPRESENT_NOT_SET(0);
-      private final int value;
-      private SourceTypePresentCase(int value) {
-        this.value = value;
-      }
-      /**
-       * @deprecated Use {@link #forNumber(int)} instead.
-       */
-      @java.lang.Deprecated
-      public static SourceTypePresentCase valueOf(int value) {
-        return forNumber(value);
-      }
-
-      public static SourceTypePresentCase forNumber(int value) {
-        switch (value) {
-          case 7: return SOURCETYPE;
-          case 0: return SOURCETYPEPRESENT_NOT_SET;
-          default: return null;
-        }
-      }
-      public int getNumber() {
-        return this.value;
-      }
-    };
-
-    public SourceTypePresentCase
-    getSourceTypePresentCase() {
-      return SourceTypePresentCase.forNumber(
-          sourceTypePresentCase_);
-    }
-
-    private int eventTypePresentCase_ = 0;
-    private java.lang.Object eventTypePresent_;
-    public enum EventTypePresentCase
-        implements com.google.protobuf.Internal.EnumLite {
-      EVENTTYPE(11),
-      EVENTTYPEPRESENT_NOT_SET(0);
-      private final int value;
-      private EventTypePresentCase(int value) {
-        this.value = value;
-      }
-      /**
-       * @deprecated Use {@link #forNumber(int)} instead.
-       */
-      @java.lang.Deprecated
-      public static EventTypePresentCase valueOf(int value) {
-        return forNumber(value);
-      }
-
-      public static EventTypePresentCase forNumber(int value) {
-        switch (value) {
-          case 11: return EVENTTYPE;
-          case 0: return EVENTTYPEPRESENT_NOT_SET;
-          default: return null;
-        }
-      }
-      public int getNumber() {
-        return this.value;
-      }
-    };
-
-    public EventTypePresentCase
-    getEventTypePresentCase() {
-      return EventTypePresentCase.forNumber(
-          eventTypePresentCase_);
-    }
-
     public static final int VERSION_FIELD_NUMBER = 1;
+    private int version_;
     /**
-     * int32 version = 1;
+     * optional int32 version = 1 [default = 1];
+     *
+     * 
      +     **协议的版本号*
+     * 
      
+     */
+    public boolean hasVersion() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    /**
+     * optional int32 version = 1 [default = 1];
+     *
+     * 
      +     **协议的版本号*
+     * 
      
      */
     public int getVersion() {
-      if (versionPresentCase_ == 1) {
-        return (java.lang.Integer) versionPresent_;
-      }
-      return 1;
+      return version_;
     }
 
     public static final int LOGFILENAME_FIELD_NUMBER = 2;
-    private volatile java.lang.Object logfileName_;
+    private Object logfileName_;
     /**
+     * optional string logfileName = 2;
+     *
      * 
            **binlog/redolog 文件名*
      * 
      
-     *
-     * string logfileName = 2;
      */
-    public java.lang.String getLogfileName() {
-      java.lang.Object ref = logfileName_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasLogfileName() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    /**
+     * optional string logfileName = 2;
+     *
+     * 
      +     **binlog/redolog 文件名*
+     * 
      
+     */
+    public String getLogfileName() {
+      Object ref = logfileName_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        logfileName_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          logfileName_ = s;
+        }
         return s;
       }
     }
     /**
+     * optional string logfileName = 2;
+     *
      * 
            **binlog/redolog 文件名*
      * 
      
-     *
-     * string logfileName = 2;
      */
     public com.google.protobuf.ByteString
         getLogfileNameBytes() {
-      java.lang.Object ref = logfileName_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = logfileName_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         logfileName_ = b;
         return b;
       } else {
@@ -2017,11 +1825,21 @@ public final class CanalEntry {
     public static final int LOGFILEOFFSET_FIELD_NUMBER = 3;
     private long logfileOffset_;
     /**
+     * optional int64 logfileOffset = 3;
+     *
      * 
            **binlog/redolog 文件的偏移位置*
      * 
      
+     */
+    public boolean hasLogfileOffset() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    /**
+     * optional int64 logfileOffset = 3;
      *
-     * int64 logfileOffset = 3;
+     * 
      +     **binlog/redolog 文件的偏移位置*
+     * 
      
      */
     public long getLogfileOffset() {
       return logfileOffset_;
@@ -2030,51 +1848,73 @@ public final class CanalEntry {
     public static final int SERVERID_FIELD_NUMBER = 4;
     private long serverId_;
     /**
+     * optional int64 serverId = 4;
+     *
      * 
            **服务端serverId*
      * 
      
+     */
+    public boolean hasServerId() {
+      return ((bitField0_ & 0x00000008) == 0x00000008);
+    }
+    /**
+     * optional int64 serverId = 4;
      *
-     * int64 serverId = 4;
+     * 
      +     **服务端serverId*
+     * 
      
      */
     public long getServerId() {
       return serverId_;
     }
 
     public static final int SERVERENCODE_FIELD_NUMBER = 5;
-    private volatile java.lang.Object serverenCode_;
+    private Object serverenCode_;
     /**
+     * optional string serverenCode = 5;
+     *
      * 
            ** 变更数据的编码 *
      * 
      
-     *
-     * string serverenCode = 5;
      */
-    public java.lang.String getServerenCode() {
-      java.lang.Object ref = serverenCode_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasServerenCode() {
+      return ((bitField0_ & 0x00000010) == 0x00000010);
+    }
+    /**
+     * optional string serverenCode = 5;
+     *
+     * 
      +     ** 变更数据的编码 *
+     * 
      
+     */
+    public String getServerenCode() {
+      Object ref = serverenCode_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        serverenCode_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          serverenCode_ = s;
+        }
         return s;
       }
     }
     /**
+     * optional string serverenCode = 5;
+     *
      * 
            ** 变更数据的编码 *
      * 
      
-     *
-     * string serverenCode = 5;
      */
     public com.google.protobuf.ByteString
         getServerenCodeBytes() {
-      java.lang.Object ref = serverenCode_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = serverenCode_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         serverenCode_ = b;
         return b;
       } else {
@@ -2085,74 +1925,96 @@ public final class CanalEntry {
     public static final int EXECUTETIME_FIELD_NUMBER = 6;
     private long executeTime_;
     /**
+     * optional int64 executeTime = 6;
+     *
      * 
            **变更数据的执行时间 *
      * 
      
+     */
+    public boolean hasExecuteTime() {
+      return ((bitField0_ & 0x00000020) == 0x00000020);
+    }
+    /**
+     * optional int64 executeTime = 6;
      *
-     * int64 executeTime = 6;
+     * 
      +     **变更数据的执行时间 *
+     * 
      
      */
     public long getExecuteTime() {
       return executeTime_;
     }
 
     public static final int SOURCETYPE_FIELD_NUMBER = 7;
+    private Type sourceType_;
     /**
-     * .com.alibaba.otter.canal.protocol.Type sourceType = 7;
+     * optional .com.alibaba.otter.canal.protocol.Type sourceType = 7 [default = MYSQL];
+     *
+     * 
      +     ** 变更数据的来源*
+     * 
      
      */
-    public int getSourceTypeValue() {
-      if (sourceTypePresentCase_ == 7) {
-        return (java.lang.Integer) sourceTypePresent_;
-      }
-      return 0;
+    public boolean hasSourceType() {
+      return ((bitField0_ & 0x00000040) == 0x00000040);
     }
     /**
-     * .com.alibaba.otter.canal.protocol.Type sourceType = 7;
+     * optional .com.alibaba.otter.canal.protocol.Type sourceType = 7 [default = MYSQL];
+     *
+     * 
      +     ** 变更数据的来源*
+     * 
      
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.Type getSourceType() {
-      if (sourceTypePresentCase_ == 7) {
-        @SuppressWarnings("deprecation")
-        com.alibaba.otter.canal.protocol.CanalEntry.Type result = com.alibaba.otter.canal.protocol.CanalEntry.Type.valueOf(
-            (java.lang.Integer) sourceTypePresent_);
-        return result == null ? com.alibaba.otter.canal.protocol.CanalEntry.Type.UNRECOGNIZED : result;
-      }
-      return com.alibaba.otter.canal.protocol.CanalEntry.Type.TYPECOMPATIBLEPROTO2;
+    public Type getSourceType() {
+      return sourceType_;
     }
 
     public static final int SCHEMANAME_FIELD_NUMBER = 8;
-    private volatile java.lang.Object schemaName_;
+    private Object schemaName_;
     /**
+     * optional string schemaName = 8;
+     *
      * 
            ** 变更数据的schemaname*
      * 
      
-     *
-     * string schemaName = 8;
      */
-    public java.lang.String getSchemaName() {
-      java.lang.Object ref = schemaName_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasSchemaName() {
+      return ((bitField0_ & 0x00000080) == 0x00000080);
+    }
+    /**
+     * optional string schemaName = 8;
+     *
+     * 
      +     ** 变更数据的schemaname*
+     * 
      
+     */
+    public String getSchemaName() {
+      Object ref = schemaName_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        schemaName_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          schemaName_ = s;
+        }
         return s;
       }
     }
     /**
+     * optional string schemaName = 8;
+     *
      * 
            ** 变更数据的schemaname*
      * 
      
-     *
-     * string schemaName = 8;
      */
     public com.google.protobuf.ByteString
         getSchemaNameBytes() {
-      java.lang.Object ref = schemaName_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = schemaName_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         schemaName_ = b;
         return b;
       } else {
@@ -2161,40 +2023,52 @@ public final class CanalEntry {
     }
 
     public static final int TABLENAME_FIELD_NUMBER = 9;
-    private volatile java.lang.Object tableName_;
+    private Object tableName_;
     /**
+     * optional string tableName = 9;
+     *
      * 
            **变更数据的tablename*
      * 
      
-     *
-     * string tableName = 9;
      */
-    public java.lang.String getTableName() {
-      java.lang.Object ref = tableName_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasTableName() {
+      return ((bitField0_ & 0x00000100) == 0x00000100);
+    }
+    /**
+     * optional string tableName = 9;
+     *
+     * 
      +     **变更数据的tablename*
+     * 
      
+     */
+    public String getTableName() {
+      Object ref = tableName_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        tableName_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          tableName_ = s;
+        }
         return s;
       }
     }
     /**
+     * optional string tableName = 9;
+     *
      * 
            **变更数据的tablename*
      * 
      
-     *
-     * string tableName = 9;
      */
     public com.google.protobuf.ByteString
         getTableNameBytes() {
-      java.lang.Object ref = tableName_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = tableName_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         tableName_ = b;
         return b;
       } else {
@@ -2205,129 +2079,151 @@ public final class CanalEntry {
     public static final int EVENTLENGTH_FIELD_NUMBER = 10;
     private long eventLength_;
     /**
+     * optional int64 eventLength = 10;
+     *
      * 
            **每个event的长度*
      * 
      
+     */
+    public boolean hasEventLength() {
+      return ((bitField0_ & 0x00000200) == 0x00000200);
+    }
+    /**
+     * optional int64 eventLength = 10;
      *
-     * int64 eventLength = 10;
+     * 
      +     **每个event的长度*
+     * 
      
      */
     public long getEventLength() {
       return eventLength_;
     }
 
     public static final int EVENTTYPE_FIELD_NUMBER = 11;
+    private EventType eventType_;
     /**
-     * .com.alibaba.otter.canal.protocol.EventType eventType = 11;
+     * optional .com.alibaba.otter.canal.protocol.EventType eventType = 11 [default = UPDATE];
+     *
+     * 
      +     **数据变更类型*
+     * 
      
      */
-    public int getEventTypeValue() {
-      if (eventTypePresentCase_ == 11) {
-        return (java.lang.Integer) eventTypePresent_;
-      }
-      return 0;
+    public boolean hasEventType() {
+      return ((bitField0_ & 0x00000400) == 0x00000400);
     }
     /**
-     * .com.alibaba.otter.canal.protocol.EventType eventType = 11;
+     * optional .com.alibaba.otter.canal.protocol.EventType eventType = 11 [default = UPDATE];
+     *
+     * 
      +     **数据变更类型*
+     * 
      
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.EventType getEventType() {
-      if (eventTypePresentCase_ == 11) {
-        @SuppressWarnings("deprecation")
-        com.alibaba.otter.canal.protocol.CanalEntry.EventType result = com.alibaba.otter.canal.protocol.CanalEntry.EventType.valueOf(
-            (java.lang.Integer) eventTypePresent_);
-        return result == null ? com.alibaba.otter.canal.protocol.CanalEntry.EventType.UNRECOGNIZED : result;
-      }
-      return com.alibaba.otter.canal.protocol.CanalEntry.EventType.EVENTTYPECOMPATIBLEPROTO2;
+    public EventType getEventType() {
+      return eventType_;
     }
 
     public static final int PROPS_FIELD_NUMBER = 12;
-    private java.util.List props_;
+    private java.util.List props_;
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
      */
-    public java.util.List getPropsList() {
+    public java.util.List getPropsList() {
       return props_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
      */
-    public java.util.List 
+    public java.util.List
         getPropsOrBuilderList() {
       return props_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
      */
     public int getPropsCount() {
       return props_.size();
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+    public Pair getProps(int index) {
       return props_.get(index);
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+    public PairOrBuilder getPropsOrBuilder(
         int index) {
       return props_.get(index);
     }
 
     public static final int GTID_FIELD_NUMBER = 13;
-    private volatile java.lang.Object gtid_;
+    private Object gtid_;
     /**
+     * optional string gtid = 13;
+     *
      * 
            **当前事务的gitd*
      * 
      
-     *
-     * string gtid = 13;
      */
-    public java.lang.String getGtid() {
-      java.lang.Object ref = gtid_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasGtid() {
+      return ((bitField0_ & 0x00000800) == 0x00000800);
+    }
+    /**
+     * optional string gtid = 13;
+     *
+     * 
      +     **当前事务的gitd*
+     * 
      
+     */
+    public String getGtid() {
+      Object ref = gtid_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        gtid_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          gtid_ = s;
+        }
         return s;
       }
     }
     /**
+     * optional string gtid = 13;
+     *
      * 
            **当前事务的gitd*
      * 
      
-     *
-     * string gtid = 13;
      */
     public com.google.protobuf.ByteString
         getGtidBytes() {
-      java.lang.Object ref = gtid_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = gtid_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         gtid_ = b;
         return b;
       } else {
@@ -2335,8 +2231,22 @@ public final class CanalEntry {
       }
     }
 
+    private void initFields() {
+      version_ = 1;
+      logfileName_ = "";
+      logfileOffset_ = 0L;
+      serverId_ = 0L;
+      serverenCode_ = "";
+      executeTime_ = 0L;
+      sourceType_ = Type.MYSQL;
+      schemaName_ = "";
+      tableName_ = "";
+      eventLength_ = 0L;
+      eventType_ = EventType.UPDATE;
+      props_ = java.util.Collections.emptyList();
+      gtid_ = "";
+    }
     private byte memoizedIsInitialized = -1;
-    @java.lang.Override
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized == 1) return true;
@@ -2346,353 +2256,208 @@ public final class CanalEntry {
       return true;
     }
 
-    @java.lang.Override
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
-      if (versionPresentCase_ == 1) {
-        output.writeInt32(
-            1, (int)((java.lang.Integer) versionPresent_));
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeInt32(1, version_);
       }
-      if (!getLogfileNameBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 2, logfileName_);
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getLogfileNameBytes());
       }
-      if (logfileOffset_ != 0L) {
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
         output.writeInt64(3, logfileOffset_);
       }
-      if (serverId_ != 0L) {
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
         output.writeInt64(4, serverId_);
       }
-      if (!getServerenCodeBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 5, serverenCode_);
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        output.writeBytes(5, getServerenCodeBytes());
       }
-      if (executeTime_ != 0L) {
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
         output.writeInt64(6, executeTime_);
       }
-      if (sourceTypePresentCase_ == 7) {
-        output.writeEnum(7, ((java.lang.Integer) sourceTypePresent_));
+      if (((bitField0_ & 0x00000040) == 0x00000040)) {
+        output.writeEnum(7, sourceType_.getNumber());
       }
-      if (!getSchemaNameBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 8, schemaName_);
+      if (((bitField0_ & 0x00000080) == 0x00000080)) {
+        output.writeBytes(8, getSchemaNameBytes());
       }
-      if (!getTableNameBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 9, tableName_);
+      if (((bitField0_ & 0x00000100) == 0x00000100)) {
+        output.writeBytes(9, getTableNameBytes());
       }
-      if (eventLength_ != 0L) {
+      if (((bitField0_ & 0x00000200) == 0x00000200)) {
         output.writeInt64(10, eventLength_);
       }
-      if (eventTypePresentCase_ == 11) {
-        output.writeEnum(11, ((java.lang.Integer) eventTypePresent_));
+      if (((bitField0_ & 0x00000400) == 0x00000400)) {
+        output.writeEnum(11, eventType_.getNumber());
       }
       for (int i = 0; i < props_.size(); i++) {
         output.writeMessage(12, props_.get(i));
       }
-      if (!getGtidBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 13, gtid_);
+      if (((bitField0_ & 0x00000800) == 0x00000800)) {
+        output.writeBytes(13, getGtidBytes());
       }
-      unknownFields.writeTo(output);
+      getUnknownFields().writeTo(output);
     }
 
-    @java.lang.Override
+    private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
-      int size = memoizedSize;
+      int size = memoizedSerializedSize;
       if (size != -1) return size;
 
       size = 0;
-      if (versionPresentCase_ == 1) {
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
-          .computeInt32Size(
-              1, (int)((java.lang.Integer) versionPresent_));
+          .computeInt32Size(1, version_);
       }
-      if (!getLogfileNameBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, logfileName_);
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getLogfileNameBytes());
       }
-      if (logfileOffset_ != 0L) {
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
         size += com.google.protobuf.CodedOutputStream
           .computeInt64Size(3, logfileOffset_);
       }
-      if (serverId_ != 0L) {
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
         size += com.google.protobuf.CodedOutputStream
           .computeInt64Size(4, serverId_);
       }
-      if (!getServerenCodeBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(5, serverenCode_);
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(5, getServerenCodeBytes());
       }
-      if (executeTime_ != 0L) {
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
         size += com.google.protobuf.CodedOutputStream
           .computeInt64Size(6, executeTime_);
       }
-      if (sourceTypePresentCase_ == 7) {
+      if (((bitField0_ & 0x00000040) == 0x00000040)) {
         size += com.google.protobuf.CodedOutputStream
-          .computeEnumSize(7, ((java.lang.Integer) sourceTypePresent_));
+          .computeEnumSize(7, sourceType_.getNumber());
       }
-      if (!getSchemaNameBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(8, schemaName_);
+      if (((bitField0_ & 0x00000080) == 0x00000080)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(8, getSchemaNameBytes());
       }
-      if (!getTableNameBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(9, tableName_);
+      if (((bitField0_ & 0x00000100) == 0x00000100)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(9, getTableNameBytes());
       }
-      if (eventLength_ != 0L) {
+      if (((bitField0_ & 0x00000200) == 0x00000200)) {
         size += com.google.protobuf.CodedOutputStream
           .computeInt64Size(10, eventLength_);
       }
-      if (eventTypePresentCase_ == 11) {
+      if (((bitField0_ & 0x00000400) == 0x00000400)) {
         size += com.google.protobuf.CodedOutputStream
-          .computeEnumSize(11, ((java.lang.Integer) eventTypePresent_));
+          .computeEnumSize(11, eventType_.getNumber());
       }
       for (int i = 0; i < props_.size(); i++) {
         size += com.google.protobuf.CodedOutputStream
           .computeMessageSize(12, props_.get(i));
       }
-      if (!getGtidBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(13, gtid_);
+      if (((bitField0_ & 0x00000800) == 0x00000800)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(13, getGtidBytes());
       }
-      size += unknownFields.getSerializedSize();
-      memoizedSize = size;
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
       return size;
     }
 
-    @java.lang.Override
-    public boolean equals(final java.lang.Object obj) {
-      if (obj == this) {
-       return true;
-      }
-      if (!(obj instanceof com.alibaba.otter.canal.protocol.CanalEntry.Header)) {
-        return super.equals(obj);
-      }
-      com.alibaba.otter.canal.protocol.CanalEntry.Header other = (com.alibaba.otter.canal.protocol.CanalEntry.Header) obj;
-
-      boolean result = true;
-      result = result && getLogfileName()
-          .equals(other.getLogfileName());
-      result = result && (getLogfileOffset()
-          == other.getLogfileOffset());
-      result = result && (getServerId()
-          == other.getServerId());
-      result = result && getServerenCode()
-          .equals(other.getServerenCode());
-      result = result && (getExecuteTime()
-          == other.getExecuteTime());
-      result = result && getSchemaName()
-          .equals(other.getSchemaName());
-      result = result && getTableName()
-          .equals(other.getTableName());
-      result = result && (getEventLength()
-          == other.getEventLength());
-      result = result && getPropsList()
-          .equals(other.getPropsList());
-      result = result && getGtid()
-          .equals(other.getGtid());
-      result = result && getVersionPresentCase().equals(
-          other.getVersionPresentCase());
-      if (!result) return false;
-      switch (versionPresentCase_) {
-        case 1:
-          result = result && (getVersion()
-              == other.getVersion());
-          break;
-        case 0:
-        default:
-      }
-      result = result && getSourceTypePresentCase().equals(
-          other.getSourceTypePresentCase());
-      if (!result) return false;
-      switch (sourceTypePresentCase_) {
-        case 7:
-          result = result && getSourceTypeValue()
-              == other.getSourceTypeValue();
-          break;
-        case 0:
-        default:
-      }
-      result = result && getEventTypePresentCase().equals(
-          other.getEventTypePresentCase());
-      if (!result) return false;
-      switch (eventTypePresentCase_) {
-        case 11:
-          result = result && getEventTypeValue()
-              == other.getEventTypeValue();
-          break;
-        case 0:
-        default:
-      }
-      result = result && unknownFields.equals(other.unknownFields);
-      return result;
+    private static final long serialVersionUID = 0L;
+    @Override
+    protected Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
     }
 
-    @java.lang.Override
-    public int hashCode() {
-      if (memoizedHashCode != 0) {
-        return memoizedHashCode;
-      }
-      int hash = 41;
-      hash = (19 * hash) + getDescriptor().hashCode();
-      hash = (37 * hash) + LOGFILENAME_FIELD_NUMBER;
-      hash = (53 * hash) + getLogfileName().hashCode();
-      hash = (37 * hash) + LOGFILEOFFSET_FIELD_NUMBER;
-      hash = (53 * hash) + com.google.protobuf.Internal.hashLong(
-          getLogfileOffset());
-      hash = (37 * hash) + SERVERID_FIELD_NUMBER;
-      hash = (53 * hash) + com.google.protobuf.Internal.hashLong(
-          getServerId());
-      hash = (37 * hash) + SERVERENCODE_FIELD_NUMBER;
-      hash = (53 * hash) + getServerenCode().hashCode();
-      hash = (37 * hash) + EXECUTETIME_FIELD_NUMBER;
-      hash = (53 * hash) + com.google.protobuf.Internal.hashLong(
-          getExecuteTime());
-      hash = (37 * hash) + SCHEMANAME_FIELD_NUMBER;
-      hash = (53 * hash) + getSchemaName().hashCode();
-      hash = (37 * hash) + TABLENAME_FIELD_NUMBER;
-      hash = (53 * hash) + getTableName().hashCode();
-      hash = (37 * hash) + EVENTLENGTH_FIELD_NUMBER;
-      hash = (53 * hash) + com.google.protobuf.Internal.hashLong(
-          getEventLength());
-      if (getPropsCount() > 0) {
-        hash = (37 * hash) + PROPS_FIELD_NUMBER;
-        hash = (53 * hash) + getPropsList().hashCode();
-      }
-      hash = (37 * hash) + GTID_FIELD_NUMBER;
-      hash = (53 * hash) + getGtid().hashCode();
-      switch (versionPresentCase_) {
-        case 1:
-          hash = (37 * hash) + VERSION_FIELD_NUMBER;
-          hash = (53 * hash) + getVersion();
-          break;
-        case 0:
-        default:
-      }
-      switch (sourceTypePresentCase_) {
-        case 7:
-          hash = (37 * hash) + SOURCETYPE_FIELD_NUMBER;
-          hash = (53 * hash) + getSourceTypeValue();
-          break;
-        case 0:
-        default:
-      }
-      switch (eventTypePresentCase_) {
-        case 11:
-          hash = (37 * hash) + EVENTTYPE_FIELD_NUMBER;
-          hash = (53 * hash) + getEventTypeValue();
-          break;
-        case 0:
-        default:
-      }
-      hash = (29 * hash) + unknownFields.hashCode();
-      memoizedHashCode = hash;
-      return hash;
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
-        java.nio.ByteBuffer data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
-        java.nio.ByteBuffer data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
+    public static Header parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
+    public static Header parseFrom(
         com.google.protobuf.ByteString data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(byte[] data)
+    public static Header parseFrom(byte[] data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
+    public static Header parseFrom(
         byte[] data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(java.io.InputStream input)
+    public static Header parseFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
+    public static Header parseFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseDelimitedFrom(java.io.InputStream input)
+    public static Header parseDelimitedFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input);
+      return PARSER.parseDelimitedFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseDelimitedFrom(
+    public static Header parseDelimitedFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseDelimitedFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
+    public static Header parseFrom(
         com.google.protobuf.CodedInputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
+    public static Header parseFrom(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
 
-    @java.lang.Override
+    public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
-    public static Builder newBuilder() {
-      return DEFAULT_INSTANCE.toBuilder();
-    }
-    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.Header prototype) {
-      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
-    }
-    @java.lang.Override
-    public Builder toBuilder() {
-      return this == DEFAULT_INSTANCE
-          ? new Builder() : new Builder().mergeFrom(this);
+    public static Builder newBuilder(Header prototype) {
+      return newBuilder().mergeFrom(prototype);
     }
+    public Builder toBuilder() { return newBuilder(this); }
 
-    @java.lang.Override
+    @Override
     protected Builder newBuilderForType(
-        com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        BuilderParent parent) {
       Builder builder = new Builder(parent);
       return builder;
     }
     /**
+     * Protobuf type {@code com.alibaba.otter.canal.protocol.Header}
+     *
      * 
            **message Header*
      * 
      
-     *
-     * Protobuf type {@code com.alibaba.otter.canal.protocol.Header}
      */
     public static final class Builder extends
-        com.google.protobuf.GeneratedMessageV3.Builder implements
+        com.google.protobuf.GeneratedMessage.Builder implements
         // @@protoc_insertion_point(builder_implements:com.alibaba.otter.canal.protocol.Header)
-        com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder {
+        HeaderOrBuilder {
       public static final com.google.protobuf.Descriptors.Descriptor
           getDescriptor() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_descriptor;
       }
 
-      @java.lang.Override
-      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      protected FieldAccessorTable
           internalGetFieldAccessorTable() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_fieldAccessorTable
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_fieldAccessorTable
             .ensureFieldAccessorsInitialized(
-                com.alibaba.otter.canal.protocol.CanalEntry.Header.class, com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder.class);
+                Header.class, Builder.class);
       }
 
       // Construct using com.alibaba.otter.canal.protocol.CanalEntry.Header.newBuilder()
@@ -2701,35 +2466,43 @@ public final class CanalEntry {
       }
 
       private Builder(
-          com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
       private void maybeForceBuilderInitialization() {
-        if (com.google.protobuf.GeneratedMessageV3
-                .alwaysUseFieldBuilders) {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
           getPropsFieldBuilder();
         }
       }
-      @java.lang.Override
+      private static Builder create() {
+        return new Builder();
+      }
+
       public Builder clear() {
         super.clear();
+        version_ = 1;
+        bitField0_ = (bitField0_ & ~0x00000001);
         logfileName_ = "";
-
+        bitField0_ = (bitField0_ & ~0x00000002);
         logfileOffset_ = 0L;
-
+        bitField0_ = (bitField0_ & ~0x00000004);
         serverId_ = 0L;
-
+        bitField0_ = (bitField0_ & ~0x00000008);
         serverenCode_ = "";
-
+        bitField0_ = (bitField0_ & ~0x00000010);
         executeTime_ = 0L;
-
+        bitField0_ = (bitField0_ & ~0x00000020);
+        sourceType_ = Type.MYSQL;
+        bitField0_ = (bitField0_ & ~0x00000040);
         schemaName_ = "";
-
+        bitField0_ = (bitField0_ & ~0x00000080);
         tableName_ = "";
-
+        bitField0_ = (bitField0_ & ~0x00000100);
         eventLength_ = 0L;
-
+        bitField0_ = (bitField0_ & ~0x00000200);
+        eventType_ = EventType.UPDATE;
+        bitField0_ = (bitField0_ & ~0x00000400);
         if (propsBuilder_ == null) {
           props_ = java.util.Collections.emptyList();
           bitField0_ = (bitField0_ & ~0x00000800);
@@ -2737,58 +2510,79 @@ public final class CanalEntry {
           propsBuilder_.clear();
         }
         gtid_ = "";
-
-        versionPresentCase_ = 0;
-        versionPresent_ = null;
-        sourceTypePresentCase_ = 0;
-        sourceTypePresent_ = null;
-        eventTypePresentCase_ = 0;
-        eventTypePresent_ = null;
+        bitField0_ = (bitField0_ & ~0x00001000);
         return this;
       }
 
-      @java.lang.Override
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_descriptor;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.Header getDefaultInstanceForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.Header.getDefaultInstance();
+      public Header getDefaultInstanceForType() {
+        return Header.getDefaultInstance();
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.Header build() {
-        com.alibaba.otter.canal.protocol.CanalEntry.Header result = buildPartial();
+      public Header build() {
+        Header result = buildPartial();
         if (!result.isInitialized()) {
           throw newUninitializedMessageException(result);
         }
         return result;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.Header buildPartial() {
-        com.alibaba.otter.canal.protocol.CanalEntry.Header result = new com.alibaba.otter.canal.protocol.CanalEntry.Header(this);
+      public Header buildPartial() {
+        Header result = new Header(this);
         int from_bitField0_ = bitField0_;
         int to_bitField0_ = 0;
-        if (versionPresentCase_ == 1) {
-          result.versionPresent_ = versionPresent_;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.version_ = version_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
         }
         result.logfileName_ = logfileName_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
         result.logfileOffset_ = logfileOffset_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000008;
+        }
         result.serverId_ = serverId_;
+        if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+          to_bitField0_ |= 0x00000010;
+        }
         result.serverenCode_ = serverenCode_;
+        if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+          to_bitField0_ |= 0x00000020;
+        }
         result.executeTime_ = executeTime_;
-        if (sourceTypePresentCase_ == 7) {
-          result.sourceTypePresent_ = sourceTypePresent_;
+        if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
+          to_bitField0_ |= 0x00000040;
+        }
+        result.sourceType_ = sourceType_;
+        if (((from_bitField0_ & 0x00000080) == 0x00000080)) {
+          to_bitField0_ |= 0x00000080;
         }
         result.schemaName_ = schemaName_;
-        result.tableName_ = tableName_;
-        result.eventLength_ = eventLength_;
-        if (eventTypePresentCase_ == 11) {
-          result.eventTypePresent_ = eventTypePresent_;
+        if (((from_bitField0_ & 0x00000100) == 0x00000100)) {
+          to_bitField0_ |= 0x00000100;
         }
+        result.tableName_ = tableName_;
+        if (((from_bitField0_ & 0x00000200) == 0x00000200)) {
+          to_bitField0_ |= 0x00000200;
+        }
+        result.eventLength_ = eventLength_;
+        if (((from_bitField0_ & 0x00000400) == 0x00000400)) {
+          to_bitField0_ |= 0x00000400;
+        }
+        result.eventType_ = eventType_;
         if (propsBuilder_ == null) {
           if (((bitField0_ & 0x00000800) == 0x00000800)) {
             props_ = java.util.Collections.unmodifiableList(props_);
@@ -2798,87 +2592,67 @@ public final class CanalEntry {
         } else {
           result.props_ = propsBuilder_.build();
         }
+        if (((from_bitField0_ & 0x00001000) == 0x00001000)) {
+          to_bitField0_ |= 0x00000800;
+        }
         result.gtid_ = gtid_;
         result.bitField0_ = to_bitField0_;
-        result.versionPresentCase_ = versionPresentCase_;
-        result.sourceTypePresentCase_ = sourceTypePresentCase_;
-        result.eventTypePresentCase_ = eventTypePresentCase_;
         onBuilt();
         return result;
       }
 
-      @java.lang.Override
-      public Builder clone() {
-        return (Builder) super.clone();
-      }
-      @java.lang.Override
-      public Builder setField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.setField(field, value);
-      }
-      @java.lang.Override
-      public Builder clearField(
-          com.google.protobuf.Descriptors.FieldDescriptor field) {
-        return (Builder) super.clearField(field);
-      }
-      @java.lang.Override
-      public Builder clearOneof(
-          com.google.protobuf.Descriptors.OneofDescriptor oneof) {
-        return (Builder) super.clearOneof(oneof);
-      }
-      @java.lang.Override
-      public Builder setRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          int index, java.lang.Object value) {
-        return (Builder) super.setRepeatedField(field, index, value);
-      }
-      @java.lang.Override
-      public Builder addRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.addRepeatedField(field, value);
-      }
-      @java.lang.Override
       public Builder mergeFrom(com.google.protobuf.Message other) {
-        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.Header) {
-          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.Header)other);
+        if (other instanceof Header) {
+          return mergeFrom((Header)other);
         } else {
           super.mergeFrom(other);
           return this;
         }
       }
 
-      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.Header other) {
-        if (other == com.alibaba.otter.canal.protocol.CanalEntry.Header.getDefaultInstance()) return this;
-        if (!other.getLogfileName().isEmpty()) {
+      public Builder mergeFrom(Header other) {
+        if (other == Header.getDefaultInstance()) return this;
+        if (other.hasVersion()) {
+          setVersion(other.getVersion());
+        }
+        if (other.hasLogfileName()) {
+          bitField0_ |= 0x00000002;
           logfileName_ = other.logfileName_;
           onChanged();
         }
-        if (other.getLogfileOffset() != 0L) {
+        if (other.hasLogfileOffset()) {
           setLogfileOffset(other.getLogfileOffset());
         }
-        if (other.getServerId() != 0L) {
+        if (other.hasServerId()) {
           setServerId(other.getServerId());
         }
-        if (!other.getServerenCode().isEmpty()) {
+        if (other.hasServerenCode()) {
+          bitField0_ |= 0x00000010;
           serverenCode_ = other.serverenCode_;
           onChanged();
         }
-        if (other.getExecuteTime() != 0L) {
+        if (other.hasExecuteTime()) {
           setExecuteTime(other.getExecuteTime());
         }
-        if (!other.getSchemaName().isEmpty()) {
+        if (other.hasSourceType()) {
+          setSourceType(other.getSourceType());
+        }
+        if (other.hasSchemaName()) {
+          bitField0_ |= 0x00000080;
           schemaName_ = other.schemaName_;
           onChanged();
         }
-        if (!other.getTableName().isEmpty()) {
+        if (other.hasTableName()) {
+          bitField0_ |= 0x00000100;
           tableName_ = other.tableName_;
           onChanged();
         }
-        if (other.getEventLength() != 0L) {
+        if (other.hasEventLength()) {
           setEventLength(other.getEventLength());
         }
+        if (other.hasEventType()) {
+          setEventType(other.getEventType());
+        }
         if (propsBuilder_ == null) {
           if (!other.props_.isEmpty()) {
             if (props_.isEmpty()) {
@@ -2897,66 +2671,37 @@ public final class CanalEntry {
               propsBuilder_ = null;
               props_ = other.props_;
               bitField0_ = (bitField0_ & ~0x00000800);
-              propsBuilder_ = 
-                com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ?
+              propsBuilder_ =
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
                    getPropsFieldBuilder() : null;
             } else {
               propsBuilder_.addAllMessages(other.props_);
             }
           }
         }
-        if (!other.getGtid().isEmpty()) {
+        if (other.hasGtid()) {
+          bitField0_ |= 0x00001000;
           gtid_ = other.gtid_;
           onChanged();
         }
-        switch (other.getVersionPresentCase()) {
-          case VERSION: {
-            setVersion(other.getVersion());
-            break;
-          }
-          case VERSIONPRESENT_NOT_SET: {
-            break;
-          }
-        }
-        switch (other.getSourceTypePresentCase()) {
-          case SOURCETYPE: {
-            setSourceTypeValue(other.getSourceTypeValue());
-            break;
-          }
-          case SOURCETYPEPRESENT_NOT_SET: {
-            break;
-          }
-        }
-        switch (other.getEventTypePresentCase()) {
-          case EVENTTYPE: {
-            setEventTypeValue(other.getEventTypeValue());
-            break;
-          }
-          case EVENTTYPEPRESENT_NOT_SET: {
-            break;
-          }
-        }
-        this.mergeUnknownFields(other.unknownFields);
-        onChanged();
+        this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
 
-      @java.lang.Override
       public final boolean isInitialized() {
         return true;
       }
 
-      @java.lang.Override
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
           throws java.io.IOException {
-        com.alibaba.otter.canal.protocol.CanalEntry.Header parsedMessage = null;
+        Header parsedMessage = null;
         try {
           parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
         } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          parsedMessage = (com.alibaba.otter.canal.protocol.CanalEntry.Header) e.getUnfinishedMessage();
-          throw e.unwrapIOException();
+          parsedMessage = (Header) e.getUnfinishedMessage();
+          throw e;
         } finally {
           if (parsedMessage != null) {
             mergeFrom(parsedMessage);
@@ -2964,117 +2709,102 @@ public final class CanalEntry {
         }
         return this;
       }
-      private int versionPresentCase_ = 0;
-      private java.lang.Object versionPresent_;
-      public VersionPresentCase
-          getVersionPresentCase() {
-        return VersionPresentCase.forNumber(
-            versionPresentCase_);
-      }
-
-      public Builder clearVersionPresent() {
-        versionPresentCase_ = 0;
-        versionPresent_ = null;
-        onChanged();
-        return this;
-      }
-
-      private int sourceTypePresentCase_ = 0;
-      private java.lang.Object sourceTypePresent_;
-      public SourceTypePresentCase
-          getSourceTypePresentCase() {
-        return SourceTypePresentCase.forNumber(
-            sourceTypePresentCase_);
-      }
-
-      public Builder clearSourceTypePresent() {
-        sourceTypePresentCase_ = 0;
-        sourceTypePresent_ = null;
-        onChanged();
-        return this;
-      }
-
-      private int eventTypePresentCase_ = 0;
-      private java.lang.Object eventTypePresent_;
-      public EventTypePresentCase
-          getEventTypePresentCase() {
-        return EventTypePresentCase.forNumber(
-            eventTypePresentCase_);
-      }
-
-      public Builder clearEventTypePresent() {
-        eventTypePresentCase_ = 0;
-        eventTypePresent_ = null;
-        onChanged();
-        return this;
-      }
-
       private int bitField0_;
 
+      private int version_ = 1;
       /**
-       * int32 version = 1;
+       * optional int32 version = 1 [default = 1];
+       *
+       * 
      +       **协议的版本号*
+       * 
      
        */
-      public int getVersion() {
-        if (versionPresentCase_ == 1) {
-          return (java.lang.Integer) versionPresent_;
-        }
-        return 0;
+      public boolean hasVersion() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
       }
       /**
-       * int32 version = 1;
+       * optional int32 version = 1 [default = 1];
+       *
+       * 
      +       **协议的版本号*
+       * 
      
+       */
+      public int getVersion() {
+        return version_;
+      }
+      /**
+       * optional int32 version = 1 [default = 1];
+       *
+       * 
      +       **协议的版本号*
+       * 
      
        */
       public Builder setVersion(int value) {
-        versionPresentCase_ = 1;
-        versionPresent_ = value;
+        bitField0_ |= 0x00000001;
+        version_ = value;
         onChanged();
         return this;
       }
       /**
-       * int32 version = 1;
+       * optional int32 version = 1 [default = 1];
+       *
+       * 
      +       **协议的版本号*
+       * 
      
        */
       public Builder clearVersion() {
-        if (versionPresentCase_ == 1) {
-          versionPresentCase_ = 0;
-          versionPresent_ = null;
-          onChanged();
-        }
+        bitField0_ = (bitField0_ & ~0x00000001);
+        version_ = 1;
+        onChanged();
         return this;
       }
 
-      private java.lang.Object logfileName_ = "";
+      private Object logfileName_ = "";
       /**
+       * optional string logfileName = 2;
+       *
        * 
              **binlog/redolog 文件名*
        * 
      
-       *
-       * string logfileName = 2;
        */
-      public java.lang.String getLogfileName() {
-        java.lang.Object ref = logfileName_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasLogfileName() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      /**
+       * optional string logfileName = 2;
+       *
+       * 
      +       **binlog/redolog 文件名*
+       * 
      
+       */
+      public String getLogfileName() {
+        Object ref = logfileName_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          logfileName_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            logfileName_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
+       * optional string logfileName = 2;
+       *
        * 
              **binlog/redolog 文件名*
        * 
      
-       *
-       * string logfileName = 2;
        */
       public com.google.protobuf.ByteString
           getLogfileNameBytes() {
-        java.lang.Object ref = logfileName_;
+        Object ref = logfileName_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           logfileName_ = b;
           return b;
         } else {
@@ -3082,49 +2812,48 @@ public final class CanalEntry {
         }
       }
       /**
+       * optional string logfileName = 2;
+       *
        * 
              **binlog/redolog 文件名*
        * 
      
-       *
-       * string logfileName = 2;
        */
       public Builder setLogfileName(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000002;
         logfileName_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional string logfileName = 2;
+       *
        * 
              **binlog/redolog 文件名*
        * 
      
-       *
-       * string logfileName = 2;
        */
       public Builder clearLogfileName() {
-        
+        bitField0_ = (bitField0_ & ~0x00000002);
         logfileName_ = getDefaultInstance().getLogfileName();
         onChanged();
         return this;
       }
       /**
+       * optional string logfileName = 2;
+       *
        * 
              **binlog/redolog 文件名*
        * 
      
-       *
-       * string logfileName = 2;
        */
       public Builder setLogfileNameBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00000002;
         logfileName_ = value;
         onChanged();
         return this;
@@ -3132,37 +2861,47 @@ public final class CanalEntry {
 
       private long logfileOffset_ ;
       /**
+       * optional int64 logfileOffset = 3;
+       *
        * 
              **binlog/redolog 文件的偏移位置*
        * 
      
+       */
+      public boolean hasLogfileOffset() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      /**
+       * optional int64 logfileOffset = 3;
        *
-       * int64 logfileOffset = 3;
+       * 
      +       **binlog/redolog 文件的偏移位置*
+       * 
      
        */
       public long getLogfileOffset() {
         return logfileOffset_;
       }
       /**
+       * optional int64 logfileOffset = 3;
+       *
        * 
              **binlog/redolog 文件的偏移位置*
        * 
      
-       *
-       * int64 logfileOffset = 3;
        */
       public Builder setLogfileOffset(long value) {
-        
+        bitField0_ |= 0x00000004;
         logfileOffset_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional int64 logfileOffset = 3;
+       *
        * 
              **binlog/redolog 文件的偏移位置*
        * 
      
-       *
-       * int64 logfileOffset = 3;
        */
       public Builder clearLogfileOffset() {
-        
+        bitField0_ = (bitField0_ & ~0x00000004);
         logfileOffset_ = 0L;
         onChanged();
         return this;
@@ -3170,76 +2909,98 @@ public final class CanalEntry {
 
       private long serverId_ ;
       /**
+       * optional int64 serverId = 4;
+       *
        * 
              **服务端serverId*
        * 
      
+       */
+      public boolean hasServerId() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      /**
+       * optional int64 serverId = 4;
        *
-       * int64 serverId = 4;
+       * 
      +       **服务端serverId*
+       * 
      
        */
       public long getServerId() {
         return serverId_;
       }
       /**
+       * optional int64 serverId = 4;
+       *
        * 
              **服务端serverId*
        * 
      
-       *
-       * int64 serverId = 4;
        */
       public Builder setServerId(long value) {
-        
+        bitField0_ |= 0x00000008;
         serverId_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional int64 serverId = 4;
+       *
        * 
              **服务端serverId*
        * 
      
-       *
-       * int64 serverId = 4;
        */
       public Builder clearServerId() {
-        
+        bitField0_ = (bitField0_ & ~0x00000008);
         serverId_ = 0L;
         onChanged();
         return this;
       }
 
-      private java.lang.Object serverenCode_ = "";
+      private Object serverenCode_ = "";
       /**
+       * optional string serverenCode = 5;
+       *
        * 
              ** 变更数据的编码 *
        * 
      
-       *
-       * string serverenCode = 5;
        */
-      public java.lang.String getServerenCode() {
-        java.lang.Object ref = serverenCode_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasServerenCode() {
+        return ((bitField0_ & 0x00000010) == 0x00000010);
+      }
+      /**
+       * optional string serverenCode = 5;
+       *
+       * 
      +       ** 变更数据的编码 *
+       * 
      
+       */
+      public String getServerenCode() {
+        Object ref = serverenCode_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          serverenCode_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            serverenCode_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
+       * optional string serverenCode = 5;
+       *
        * 
              ** 变更数据的编码 *
        * 
      
-       *
-       * string serverenCode = 5;
        */
       public com.google.protobuf.ByteString
           getServerenCodeBytes() {
-        java.lang.Object ref = serverenCode_;
+        Object ref = serverenCode_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           serverenCode_ = b;
           return b;
         } else {
@@ -3247,49 +3008,48 @@ public final class CanalEntry {
         }
       }
       /**
+       * optional string serverenCode = 5;
+       *
        * 
              ** 变更数据的编码 *
        * 
      
-       *
-       * string serverenCode = 5;
        */
       public Builder setServerenCode(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000010;
         serverenCode_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional string serverenCode = 5;
+       *
        * 
              ** 变更数据的编码 *
        * 
      
-       *
-       * string serverenCode = 5;
        */
       public Builder clearServerenCode() {
-        
+        bitField0_ = (bitField0_ & ~0x00000010);
         serverenCode_ = getDefaultInstance().getServerenCode();
         onChanged();
         return this;
       }
       /**
+       * optional string serverenCode = 5;
+       *
        * 
              ** 变更数据的编码 *
        * 
      
-       *
-       * string serverenCode = 5;
        */
       public Builder setServerenCodeBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00000010;
         serverenCode_ = value;
         onChanged();
         return this;
@@ -3297,130 +3057,149 @@ public final class CanalEntry {
 
       private long executeTime_ ;
       /**
+       * optional int64 executeTime = 6;
+       *
        * 
              **变更数据的执行时间 *
        * 
      
+       */
+      public boolean hasExecuteTime() {
+        return ((bitField0_ & 0x00000020) == 0x00000020);
+      }
+      /**
+       * optional int64 executeTime = 6;
        *
-       * int64 executeTime = 6;
+       * 
      +       **变更数据的执行时间 *
+       * 
      
        */
       public long getExecuteTime() {
         return executeTime_;
       }
       /**
+       * optional int64 executeTime = 6;
+       *
        * 
              **变更数据的执行时间 *
        * 
      
-       *
-       * int64 executeTime = 6;
        */
       public Builder setExecuteTime(long value) {
-        
+        bitField0_ |= 0x00000020;
         executeTime_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional int64 executeTime = 6;
+       *
        * 
              **变更数据的执行时间 *
        * 
      
-       *
-       * int64 executeTime = 6;
        */
       public Builder clearExecuteTime() {
-        
+        bitField0_ = (bitField0_ & ~0x00000020);
         executeTime_ = 0L;
         onChanged();
         return this;
       }
 
+      private Type sourceType_ = Type.MYSQL;
       /**
-       * .com.alibaba.otter.canal.protocol.Type sourceType = 7;
+       * optional .com.alibaba.otter.canal.protocol.Type sourceType = 7 [default = MYSQL];
+       *
+       * 
      +       ** 变更数据的来源*
+       * 
      
        */
-      public int getSourceTypeValue() {
-        if (sourceTypePresentCase_ == 7) {
-          return ((java.lang.Integer) sourceTypePresent_).intValue();
-        }
-        return Type.MYSQL_VALUE;
+      public boolean hasSourceType() {
+        return ((bitField0_ & 0x00000040) == 0x00000040);
       }
       /**
-       * .com.alibaba.otter.canal.protocol.Type sourceType = 7;
+       * optional .com.alibaba.otter.canal.protocol.Type sourceType = 7 [default = MYSQL];
+       *
+       * 
      +       ** 变更数据的来源*
+       * 
      
        */
-      public Builder setSourceTypeValue(int value) {
-        sourceTypePresentCase_ = 7;
-        sourceTypePresent_ = value;
-        onChanged();
-        return this;
+      public Type getSourceType() {
+        return sourceType_;
       }
       /**
-       * .com.alibaba.otter.canal.protocol.Type sourceType = 7;
+       * optional .com.alibaba.otter.canal.protocol.Type sourceType = 7 [default = MYSQL];
+       *
+       * 
      +       ** 变更数据的来源*
+       * 
      
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Type getSourceType() {
-        if (sourceTypePresentCase_ == 7) {
-          @SuppressWarnings("deprecation")
-          com.alibaba.otter.canal.protocol.CanalEntry.Type result = com.alibaba.otter.canal.protocol.CanalEntry.Type.valueOf(
-              (java.lang.Integer) sourceTypePresent_);
-          return result == null ? com.alibaba.otter.canal.protocol.CanalEntry.Type.UNRECOGNIZED : result;
-        }
-        return com.alibaba.otter.canal.protocol.CanalEntry.Type.TYPECOMPATIBLEPROTO2;
-      }
-      /**
-       * .com.alibaba.otter.canal.protocol.Type sourceType = 7;
-       */
-      public Builder setSourceType(com.alibaba.otter.canal.protocol.CanalEntry.Type value) {
+      public Builder setSourceType(Type value) {
         if (value == null) {
           throw new NullPointerException();
         }
-        sourceTypePresentCase_ = 7;
-        sourceTypePresent_ = value.getNumber();
+        bitField0_ |= 0x00000040;
+        sourceType_ = value;
         onChanged();
         return this;
       }
       /**
-       * .com.alibaba.otter.canal.protocol.Type sourceType = 7;
+       * optional .com.alibaba.otter.canal.protocol.Type sourceType = 7 [default = MYSQL];
+       *
+       * 
      +       ** 变更数据的来源*
+       * 
      
        */
       public Builder clearSourceType() {
-        if (sourceTypePresentCase_ == 7) {
-          sourceTypePresentCase_ = 0;
-          sourceTypePresent_ = null;
-          onChanged();
-        }
+        bitField0_ = (bitField0_ & ~0x00000040);
+        sourceType_ = Type.MYSQL;
+        onChanged();
         return this;
       }
 
-      private java.lang.Object schemaName_ = "";
+      private Object schemaName_ = "";
       /**
+       * optional string schemaName = 8;
+       *
        * 
              ** 变更数据的schemaname*
        * 
      
-       *
-       * string schemaName = 8;
        */
-      public java.lang.String getSchemaName() {
-        java.lang.Object ref = schemaName_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasSchemaName() {
+        return ((bitField0_ & 0x00000080) == 0x00000080);
+      }
+      /**
+       * optional string schemaName = 8;
+       *
+       * 
      +       ** 变更数据的schemaname*
+       * 
      
+       */
+      public String getSchemaName() {
+        Object ref = schemaName_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          schemaName_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            schemaName_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
+       * optional string schemaName = 8;
+       *
        * 
              ** 变更数据的schemaname*
        * 
      
-       *
-       * string schemaName = 8;
        */
       public com.google.protobuf.ByteString
           getSchemaNameBytes() {
-        java.lang.Object ref = schemaName_;
+        Object ref = schemaName_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           schemaName_ = b;
           return b;
         } else {
@@ -3428,88 +3207,99 @@ public final class CanalEntry {
         }
       }
       /**
+       * optional string schemaName = 8;
+       *
        * 
              ** 变更数据的schemaname*
        * 
      
-       *
-       * string schemaName = 8;
        */
       public Builder setSchemaName(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000080;
         schemaName_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional string schemaName = 8;
+       *
        * 
              ** 变更数据的schemaname*
        * 
      
-       *
-       * string schemaName = 8;
        */
       public Builder clearSchemaName() {
-        
+        bitField0_ = (bitField0_ & ~0x00000080);
         schemaName_ = getDefaultInstance().getSchemaName();
         onChanged();
         return this;
       }
       /**
+       * optional string schemaName = 8;
+       *
        * 
              ** 变更数据的schemaname*
        * 
      
-       *
-       * string schemaName = 8;
        */
       public Builder setSchemaNameBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00000080;
         schemaName_ = value;
         onChanged();
         return this;
       }
 
-      private java.lang.Object tableName_ = "";
+      private Object tableName_ = "";
       /**
+       * optional string tableName = 9;
+       *
        * 
              **变更数据的tablename*
        * 
      
-       *
-       * string tableName = 9;
        */
-      public java.lang.String getTableName() {
-        java.lang.Object ref = tableName_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasTableName() {
+        return ((bitField0_ & 0x00000100) == 0x00000100);
+      }
+      /**
+       * optional string tableName = 9;
+       *
+       * 
      +       **变更数据的tablename*
+       * 
      
+       */
+      public String getTableName() {
+        Object ref = tableName_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          tableName_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            tableName_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
+       * optional string tableName = 9;
+       *
        * 
              **变更数据的tablename*
        * 
      
-       *
-       * string tableName = 9;
        */
       public com.google.protobuf.ByteString
           getTableNameBytes() {
-        java.lang.Object ref = tableName_;
+        Object ref = tableName_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           tableName_ = b;
           return b;
         } else {
@@ -3517,49 +3307,48 @@ public final class CanalEntry {
         }
       }
       /**
+       * optional string tableName = 9;
+       *
        * 
              **变更数据的tablename*
        * 
      
-       *
-       * string tableName = 9;
        */
       public Builder setTableName(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000100;
         tableName_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional string tableName = 9;
+       *
        * 
              **变更数据的tablename*
        * 
      
-       *
-       * string tableName = 9;
        */
       public Builder clearTableName() {
-        
+        bitField0_ = (bitField0_ & ~0x00000100);
         tableName_ = getDefaultInstance().getTableName();
         onChanged();
         return this;
       }
       /**
+       * optional string tableName = 9;
+       *
        * 
              **变更数据的tablename*
        * 
      
-       *
-       * string tableName = 9;
        */
       public Builder setTableNameBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00000100;
         tableName_ = value;
         onChanged();
         return this;
@@ -3567,116 +3356,123 @@ public final class CanalEntry {
 
       private long eventLength_ ;
       /**
+       * optional int64 eventLength = 10;
+       *
        * 
              **每个event的长度*
        * 
      
+       */
+      public boolean hasEventLength() {
+        return ((bitField0_ & 0x00000200) == 0x00000200);
+      }
+      /**
+       * optional int64 eventLength = 10;
        *
-       * int64 eventLength = 10;
+       * 
      +       **每个event的长度*
+       * 
      
        */
       public long getEventLength() {
         return eventLength_;
       }
       /**
+       * optional int64 eventLength = 10;
+       *
        * 
              **每个event的长度*
        * 
      
-       *
-       * int64 eventLength = 10;
        */
       public Builder setEventLength(long value) {
-        
+        bitField0_ |= 0x00000200;
         eventLength_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional int64 eventLength = 10;
+       *
        * 
              **每个event的长度*
        * 
      
-       *
-       * int64 eventLength = 10;
        */
       public Builder clearEventLength() {
-        
+        bitField0_ = (bitField0_ & ~0x00000200);
         eventLength_ = 0L;
         onChanged();
         return this;
       }
 
+      private EventType eventType_ = EventType.UPDATE;
       /**
-       * .com.alibaba.otter.canal.protocol.EventType eventType = 11;
+       * optional .com.alibaba.otter.canal.protocol.EventType eventType = 11 [default = UPDATE];
+       *
+       * 
      +       **数据变更类型*
+       * 
      
        */
-      public int getEventTypeValue() {
-        if (eventTypePresentCase_ == 11) {
-          return ((java.lang.Integer) eventTypePresent_).intValue();
-        }
-        return EventType.UPDATE_VALUE;
+      public boolean hasEventType() {
+        return ((bitField0_ & 0x00000400) == 0x00000400);
       }
       /**
-       * .com.alibaba.otter.canal.protocol.EventType eventType = 11;
+       * optional .com.alibaba.otter.canal.protocol.EventType eventType = 11 [default = UPDATE];
+       *
+       * 
      +       **数据变更类型*
+       * 
      
        */
-      public Builder setEventTypeValue(int value) {
-        eventTypePresentCase_ = 11;
-        eventTypePresent_ = value;
-        onChanged();
-        return this;
+      public EventType getEventType() {
+        return eventType_;
       }
       /**
-       * .com.alibaba.otter.canal.protocol.EventType eventType = 11;
+       * optional .com.alibaba.otter.canal.protocol.EventType eventType = 11 [default = UPDATE];
+       *
+       * 
      +       **数据变更类型*
+       * 
      
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.EventType getEventType() {
-        if (eventTypePresentCase_ == 11) {
-          @SuppressWarnings("deprecation")
-          com.alibaba.otter.canal.protocol.CanalEntry.EventType result = com.alibaba.otter.canal.protocol.CanalEntry.EventType.valueOf(
-              (java.lang.Integer) eventTypePresent_);
-          return result == null ? com.alibaba.otter.canal.protocol.CanalEntry.EventType.UNRECOGNIZED : result;
-        }
-        return com.alibaba.otter.canal.protocol.CanalEntry.EventType.EVENTTYPECOMPATIBLEPROTO2;
-      }
-      /**
-       * .com.alibaba.otter.canal.protocol.EventType eventType = 11;
-       */
-      public Builder setEventType(com.alibaba.otter.canal.protocol.CanalEntry.EventType value) {
+      public Builder setEventType(EventType value) {
         if (value == null) {
           throw new NullPointerException();
         }
-        eventTypePresentCase_ = 11;
-        eventTypePresent_ = value.getNumber();
+        bitField0_ |= 0x00000400;
+        eventType_ = value;
         onChanged();
         return this;
       }
       /**
-       * .com.alibaba.otter.canal.protocol.EventType eventType = 11;
+       * optional .com.alibaba.otter.canal.protocol.EventType eventType = 11 [default = UPDATE];
+       *
+       * 
      +       **数据变更类型*
+       * 
      
        */
       public Builder clearEventType() {
-        if (eventTypePresentCase_ == 11) {
-          eventTypePresentCase_ = 0;
-          eventTypePresent_ = null;
-          onChanged();
-        }
+        bitField0_ = (bitField0_ & ~0x00000400);
+        eventType_ = EventType.UPDATE;
+        onChanged();
         return this;
       }
 
-      private java.util.List props_ =
+      private java.util.List props_ =
         java.util.Collections.emptyList();
       private void ensurePropsIsMutable() {
         if (!((bitField0_ & 0x00000800) == 0x00000800)) {
-          props_ = new java.util.ArrayList(props_);
+          props_ = new java.util.ArrayList(props_);
           bitField0_ |= 0x00000800;
          }
       }
 
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> propsBuilder_;
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Pair, Pair.Builder, PairOrBuilder> propsBuilder_;
 
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
-      public java.util.List getPropsList() {
+      public java.util.List getPropsList() {
         if (propsBuilder_ == null) {
           return java.util.Collections.unmodifiableList(props_);
         } else {
@@ -3684,11 +3480,11 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
       public int getPropsCount() {
         if (propsBuilder_ == null) {
@@ -3698,13 +3494,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+      public Pair getProps(int index) {
         if (propsBuilder_ == null) {
           return props_.get(index);
         } else {
@@ -3712,14 +3508,14 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
       public Builder setProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+          int index, Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -3733,14 +3529,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
       public Builder setProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          int index, Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.set(index, builderForValue.build());
@@ -3751,13 +3547,13 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
-      public Builder addProps(com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+      public Builder addProps(Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -3771,14 +3567,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
       public Builder addProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+          int index, Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -3792,14 +3588,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
       public Builder addProps(
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.add(builderForValue.build());
@@ -3810,14 +3606,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
       public Builder addProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          int index, Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.add(index, builderForValue.build());
@@ -3828,14 +3624,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
       public Builder addAllProps(
-          java.lang.Iterable values) {
+          Iterable values) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           com.google.protobuf.AbstractMessageLite.Builder.addAll(
@@ -3847,11 +3643,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
       public Builder clearProps() {
         if (propsBuilder_ == null) {
@@ -3864,11 +3660,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
       public Builder removeProps(int index) {
         if (propsBuilder_ == null) {
@@ -3881,24 +3677,24 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder getPropsBuilder(
+      public Pair.Builder getPropsBuilder(
           int index) {
         return getPropsFieldBuilder().getBuilder(index);
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+      public PairOrBuilder getPropsOrBuilder(
           int index) {
         if (propsBuilder_ == null) {
           return props_.get(index);  } else {
@@ -3906,13 +3702,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
-      public java.util.List 
+      public java.util.List
            getPropsOrBuilderList() {
         if (propsBuilder_ != null) {
           return propsBuilder_.getMessageOrBuilderList();
@@ -3921,45 +3717,45 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder() {
+      public Pair.Builder addPropsBuilder() {
         return getPropsFieldBuilder().addBuilder(
-            com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+            Pair.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder(
+      public Pair.Builder addPropsBuilder(
           int index) {
         return getPropsFieldBuilder().addBuilder(
-            index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+            index, Pair.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
        */
-      public java.util.List 
+      public java.util.List
            getPropsBuilderList() {
         return getPropsFieldBuilder().getBuilderList();
       }
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> 
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Pair, Pair.Builder, PairOrBuilder>
           getPropsFieldBuilder() {
         if (propsBuilder_ == null) {
-          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3<
-              com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder>(
+          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              Pair, Pair.Builder, PairOrBuilder>(
                   props_,
                   ((bitField0_ & 0x00000800) == 0x00000800),
                   getParentForChildren(),
@@ -3969,40 +3765,52 @@ public final class CanalEntry {
         return propsBuilder_;
       }
 
-      private java.lang.Object gtid_ = "";
+      private Object gtid_ = "";
       /**
+       * optional string gtid = 13;
+       *
        * 
              **当前事务的gitd*
        * 
      
-       *
-       * string gtid = 13;
        */
-      public java.lang.String getGtid() {
-        java.lang.Object ref = gtid_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasGtid() {
+        return ((bitField0_ & 0x00001000) == 0x00001000);
+      }
+      /**
+       * optional string gtid = 13;
+       *
+       * 
      +       **当前事务的gitd*
+       * 
      
+       */
+      public String getGtid() {
+        Object ref = gtid_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          gtid_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            gtid_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
+       * optional string gtid = 13;
+       *
        * 
              **当前事务的gitd*
        * 
      
-       *
-       * string gtid = 13;
        */
       public com.google.protobuf.ByteString
           getGtidBytes() {
-        java.lang.Object ref = gtid_;
+        Object ref = gtid_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           gtid_ = b;
           return b;
         } else {
@@ -4010,104 +3818,62 @@ public final class CanalEntry {
         }
       }
       /**
+       * optional string gtid = 13;
+       *
        * 
              **当前事务的gitd*
        * 
      
-       *
-       * string gtid = 13;
        */
       public Builder setGtid(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00001000;
         gtid_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional string gtid = 13;
+       *
        * 
              **当前事务的gitd*
        * 
      
-       *
-       * string gtid = 13;
        */
       public Builder clearGtid() {
-        
+        bitField0_ = (bitField0_ & ~0x00001000);
         gtid_ = getDefaultInstance().getGtid();
         onChanged();
         return this;
       }
       /**
+       * optional string gtid = 13;
+       *
        * 
              **当前事务的gitd*
        * 
      
-       *
-       * string gtid = 13;
        */
       public Builder setGtidBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00001000;
         gtid_ = value;
         onChanged();
         return this;
       }
-      @java.lang.Override
-      public final Builder setUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.setUnknownFieldsProto3(unknownFields);
-      }
-
-      @java.lang.Override
-      public final Builder mergeUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.mergeUnknownFields(unknownFields);
-      }
-
 
       // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Header)
     }
 
-    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Header)
-    private static final com.alibaba.otter.canal.protocol.CanalEntry.Header DEFAULT_INSTANCE;
     static {
-      DEFAULT_INSTANCE = new com.alibaba.otter.canal.protocol.CanalEntry.Header();
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Header getDefaultInstance() {
-      return DEFAULT_INSTANCE;
-    }
-
-    private static final com.google.protobuf.Parser
      
-        PARSER = new com.google.protobuf.AbstractParser
      () {
-      @java.lang.Override
-      public Header parsePartialFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return new Header(input, extensionRegistry);
-      }
-    };
-
-    public static com.google.protobuf.Parser
       parser() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.google.protobuf.Parser
       getParserForType() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.alibaba.otter.canal.protocol.CanalEntry.Header getDefaultInstanceForType() {
-      return DEFAULT_INSTANCE;
+      defaultInstance = new Header(true);
+      defaultInstance.initFields();
     }
 
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Header)
   }
 
   public interface ColumnOrBuilder extends
@@ -4115,196 +3881,266 @@ public final class CanalEntry {
       com.google.protobuf.MessageOrBuilder {
 
     /**
+     * optional int32 index = 1;
+     *
      * 
            **字段下标*
      * 
      
+     */
+    boolean hasIndex();
+    /**
+     * optional int32 index = 1;
      *
-     * int32 index = 1;
+     * 
      +     **字段下标*
+     * 
      
      */
     int getIndex();
 
     /**
+     * optional int32 sqlType = 2;
+     *
      * 
            **字段java中类型*
      * 
      
+     */
+    boolean hasSqlType();
+    /**
+     * optional int32 sqlType = 2;
      *
-     * int32 sqlType = 2;
+     * 
      +     **字段java中类型*
+     * 
      
      */
     int getSqlType();
 
     /**
+     * optional string name = 3;
+     *
      * 
            **字段名称(忽略大小写)，在mysql中是没有的*
      * 
      
-     *
-     * string name = 3;
      */
-    java.lang.String getName();
+    boolean hasName();
     /**
+     * optional string name = 3;
+     *
      * 
            **字段名称(忽略大小写)，在mysql中是没有的*
      * 
      
+     */
+    String getName();
+    /**
+     * optional string name = 3;
      *
-     * string name = 3;
+     * 
      +     **字段名称(忽略大小写)，在mysql中是没有的*
+     * 
      
      */
     com.google.protobuf.ByteString
         getNameBytes();
 
     /**
+     * optional bool isKey = 4;
+     *
      * 
            **是否是主键*
      * 
      
+     */
+    boolean hasIsKey();
+    /**
+     * optional bool isKey = 4;
      *
-     * bool isKey = 4;
+     * 
      +     **是否是主键*
+     * 
      
      */
     boolean getIsKey();
 
     /**
+     * optional bool updated = 5;
+     *
      * 
            **如果EventType=UPDATE,用于标识这个字段值是否有修改*
      * 
      
+     */
+    boolean hasUpdated();
+    /**
+     * optional bool updated = 5;
      *
-     * bool updated = 5;
+     * 
      +     **如果EventType=UPDATE,用于标识这个字段值是否有修改*
+     * 
      
      */
     boolean getUpdated();
 
     /**
-     * bool isNull = 6;
+     * optional bool isNull = 6 [default = false];
+     *
+     * 
      +     ** 标识是否为空  *
+     * 
      
+     */
+    boolean hasIsNull();
+    /**
+     * optional bool isNull = 6 [default = false];
+     *
+     * 
      +     ** 标识是否为空  *
+     * 
      
      */
     boolean getIsNull();
 
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
      */
-    java.util.List 
+    java.util.List
         getPropsList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index);
+    Pair getProps(int index);
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
      */
     int getPropsCount();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
      */
-    java.util.List 
+    java.util.List
         getPropsOrBuilderList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
-        int index);
+    PairOrBuilder getPropsOrBuilder(int index);
 
     /**
+     * optional string value = 8;
+     *
      * 
            ** 字段值,timestamp,Datetime是一个时间格式的文本 *
      * 
      
-     *
-     * string value = 8;
      */
-    java.lang.String getValue();
+    boolean hasValue();
     /**
+     * optional string value = 8;
+     *
      * 
            ** 字段值,timestamp,Datetime是一个时间格式的文本 *
      * 
      
+     */
+    String getValue();
+    /**
+     * optional string value = 8;
      *
-     * string value = 8;
+     * 
      +     ** 字段值,timestamp,Datetime是一个时间格式的文本 *
+     * 
      
      */
     com.google.protobuf.ByteString
         getValueBytes();
 
     /**
+     * optional int32 length = 9;
+     *
      * 
            ** 对应数据对象原始长度 *
      * 
      
+     */
+    boolean hasLength();
+    /**
+     * optional int32 length = 9;
      *
-     * int32 length = 9;
+     * 
      +     ** 对应数据对象原始长度 *
+     * 
      
      */
     int getLength();
 
     /**
+     * optional string mysqlType = 10;
+     *
      * 
            **字段mysql类型*
      * 
      
-     *
-     * string mysqlType = 10;
      */
-    java.lang.String getMysqlType();
+    boolean hasMysqlType();
     /**
+     * optional string mysqlType = 10;
+     *
      * 
            **字段mysql类型*
      * 
      
+     */
+    String getMysqlType();
+    /**
+     * optional string mysqlType = 10;
      *
-     * string mysqlType = 10;
+     * 
      +     **字段mysql类型*
+     * 
      
      */
     com.google.protobuf.ByteString
         getMysqlTypeBytes();
-
-    public com.alibaba.otter.canal.protocol.CanalEntry.Column.IsNullPresentCase getIsNullPresentCase();
   }
   /**
+   * Protobuf type {@code com.alibaba.otter.canal.protocol.Column}
+   *
    * 
          **每个字段的数据结构*
    * 
      
-   *
-   * Protobuf type {@code com.alibaba.otter.canal.protocol.Column}
    */
-  public  static final class Column extends
-      com.google.protobuf.GeneratedMessageV3 implements
+  public static final class Column extends
+      com.google.protobuf.GeneratedMessage implements
       // @@protoc_insertion_point(message_implements:com.alibaba.otter.canal.protocol.Column)
       ColumnOrBuilder {
-  private static final long serialVersionUID = 0L;
     // Use Column.newBuilder() to construct.
-    private Column(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+    private Column(com.google.protobuf.GeneratedMessage.Builder builder) {
       super(builder);
+      this.unknownFields = builder.getUnknownFields();
     }
-    private Column() {
-      index_ = 0;
-      sqlType_ = 0;
-      name_ = "";
-      isKey_ = false;
-      updated_ = false;
-      props_ = java.util.Collections.emptyList();
-      value_ = "";
-      length_ = 0;
-      mysqlType_ = "";
+    private Column(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+    private static final Column defaultInstance;
+    public static Column getDefaultInstance() {
+      return defaultInstance;
     }
 
-    @java.lang.Override
+    public Column getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+
+    private final com.google.protobuf.UnknownFieldSet unknownFields;
+    @Override
     public final com.google.protobuf.UnknownFieldSet
-    getUnknownFields() {
+        getUnknownFields() {
       return this.unknownFields;
     }
     private Column(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
-      this();
-      if (extensionRegistry == null) {
-        throw new java.lang.NullPointerException();
-      }
+      initFields();
       int mutable_bitField0_ = 0;
       com.google.protobuf.UnknownFieldSet.Builder unknownFields =
           com.google.protobuf.UnknownFieldSet.newBuilder();
@@ -4316,68 +4152,67 @@ public final class CanalEntry {
             case 0:
               done = true;
               break;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                done = true;
+              }
+              break;
+            }
             case 8: {
-
+              bitField0_ |= 0x00000001;
               index_ = input.readInt32();
               break;
             }
             case 16: {
-
+              bitField0_ |= 0x00000002;
               sqlType_ = input.readInt32();
               break;
             }
             case 26: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              name_ = s;
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000004;
+              name_ = bs;
               break;
             }
             case 32: {
-
+              bitField0_ |= 0x00000008;
               isKey_ = input.readBool();
               break;
             }
             case 40: {
-
+              bitField0_ |= 0x00000010;
               updated_ = input.readBool();
               break;
             }
             case 48: {
-              isNullPresentCase_ = 6;
-              isNullPresent_ = input.readBool();
+              bitField0_ |= 0x00000020;
+              isNull_ = input.readBool();
               break;
             }
             case 58: {
               if (!((mutable_bitField0_ & 0x00000040) == 0x00000040)) {
-                props_ = new java.util.ArrayList();
+                props_ = new java.util.ArrayList();
                 mutable_bitField0_ |= 0x00000040;
               }
-              props_.add(
-                  input.readMessage(com.alibaba.otter.canal.protocol.CanalEntry.Pair.parser(), extensionRegistry));
+              props_.add(input.readMessage(Pair.PARSER, extensionRegistry));
               break;
             }
             case 66: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              value_ = s;
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000040;
+              value_ = bs;
               break;
             }
             case 72: {
-
+              bitField0_ |= 0x00000080;
               length_ = input.readInt32();
               break;
             }
             case 82: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              mysqlType_ = s;
-              break;
-            }
-            default: {
-              if (!parseUnknownFieldProto3(
-                  input, unknownFields, extensionRegistry, tag)) {
-                done = true;
-              }
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000100;
+              mysqlType_ = bs;
               break;
             }
           }
@@ -4386,7 +4221,7 @@ public final class CanalEntry {
         throw e.setUnfinishedMessage(this);
       } catch (java.io.IOException e) {
         throw new com.google.protobuf.InvalidProtocolBufferException(
-            e).setUnfinishedMessage(this);
+            e.getMessage()).setUnfinishedMessage(this);
       } finally {
         if (((mutable_bitField0_ & 0x00000040) == 0x00000040)) {
           props_ = java.util.Collections.unmodifiableList(props_);
@@ -4397,62 +4232,50 @@ public final class CanalEntry {
     }
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_descriptor;
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_descriptor;
     }
 
-    @java.lang.Override
-    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+    protected FieldAccessorTable
         internalGetFieldAccessorTable() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_fieldAccessorTable
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_fieldAccessorTable
           .ensureFieldAccessorsInitialized(
-              com.alibaba.otter.canal.protocol.CanalEntry.Column.class, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder.class);
+              Column.class, Builder.class);
     }
 
-    private int bitField0_;
-    private int isNullPresentCase_ = 0;
-    private java.lang.Object isNullPresent_;
-    public enum IsNullPresentCase
-        implements com.google.protobuf.Internal.EnumLite {
-      ISNULL(6),
-      ISNULLPRESENT_NOT_SET(0);
-      private final int value;
-      private IsNullPresentCase(int value) {
-        this.value = value;
-      }
-      /**
-       * @deprecated Use {@link #forNumber(int)} instead.
-       */
-      @java.lang.Deprecated
-      public static IsNullPresentCase valueOf(int value) {
-        return forNumber(value);
-      }
-
-      public static IsNullPresentCase forNumber(int value) {
-        switch (value) {
-          case 6: return ISNULL;
-          case 0: return ISNULLPRESENT_NOT_SET;
-          default: return null;
-        }
-      }
-      public int getNumber() {
-        return this.value;
+    public static com.google.protobuf.Parser PARSER =
+        new com.google.protobuf.AbstractParser() {
+      public Column parsePartialFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return new Column(input, extensionRegistry);
       }
     };
 
-    public IsNullPresentCase
-    getIsNullPresentCase() {
-      return IsNullPresentCase.forNumber(
-          isNullPresentCase_);
+    @Override
+    public com.google.protobuf.Parser getParserForType() {
+      return PARSER;
     }
 
+    private int bitField0_;
     public static final int INDEX_FIELD_NUMBER = 1;
     private int index_;
     /**
+     * optional int32 index = 1;
+     *
      * 
            **字段下标*
      * 
      
+     */
+    public boolean hasIndex() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    /**
+     * optional int32 index = 1;
      *
-     * int32 index = 1;
+     * 
      +     **字段下标*
+     * 
      
      */
     public int getIndex() {
       return index_;
@@ -4461,51 +4284,73 @@ public final class CanalEntry {
     public static final int SQLTYPE_FIELD_NUMBER = 2;
     private int sqlType_;
     /**
+     * optional int32 sqlType = 2;
+     *
      * 
            **字段java中类型*
      * 
      
+     */
+    public boolean hasSqlType() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    /**
+     * optional int32 sqlType = 2;
      *
-     * int32 sqlType = 2;
+     * 
      +     **字段java中类型*
+     * 
      
      */
     public int getSqlType() {
       return sqlType_;
     }
 
     public static final int NAME_FIELD_NUMBER = 3;
-    private volatile java.lang.Object name_;
+    private Object name_;
     /**
+     * optional string name = 3;
+     *
      * 
            **字段名称(忽略大小写)，在mysql中是没有的*
      * 
      
-     *
-     * string name = 3;
      */
-    public java.lang.String getName() {
-      java.lang.Object ref = name_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasName() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    /**
+     * optional string name = 3;
+     *
+     * 
      +     **字段名称(忽略大小写)，在mysql中是没有的*
+     * 
      
+     */
+    public String getName() {
+      Object ref = name_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        name_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          name_ = s;
+        }
         return s;
       }
     }
     /**
+     * optional string name = 3;
+     *
      * 
            **字段名称(忽略大小写)，在mysql中是没有的*
      * 
      
-     *
-     * string name = 3;
      */
     public com.google.protobuf.ByteString
         getNameBytes() {
-      java.lang.Object ref = name_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = name_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         name_ = b;
         return b;
       } else {
@@ -4516,11 +4361,21 @@ public final class CanalEntry {
     public static final int ISKEY_FIELD_NUMBER = 4;
     private boolean isKey_;
     /**
+     * optional bool isKey = 4;
+     *
      * 
            **是否是主键*
      * 
      
+     */
+    public boolean hasIsKey() {
+      return ((bitField0_ & 0x00000008) == 0x00000008);
+    }
+    /**
+     * optional bool isKey = 4;
      *
-     * bool isKey = 4;
+     * 
      +     **是否是主键*
+     * 
      
      */
     public boolean getIsKey() {
       return isKey_;
@@ -4529,117 +4384,151 @@ public final class CanalEntry {
     public static final int UPDATED_FIELD_NUMBER = 5;
     private boolean updated_;
     /**
+     * optional bool updated = 5;
+     *
      * 
            **如果EventType=UPDATE,用于标识这个字段值是否有修改*
      * 
      
+     */
+    public boolean hasUpdated() {
+      return ((bitField0_ & 0x00000010) == 0x00000010);
+    }
+    /**
+     * optional bool updated = 5;
      *
-     * bool updated = 5;
+     * 
      +     **如果EventType=UPDATE,用于标识这个字段值是否有修改*
+     * 
      
      */
     public boolean getUpdated() {
       return updated_;
     }
 
     public static final int ISNULL_FIELD_NUMBER = 6;
+    private boolean isNull_;
     /**
-     * bool isNull = 6;
+     * optional bool isNull = 6 [default = false];
+     *
+     * 
      +     ** 标识是否为空  *
+     * 
      
+     */
+    public boolean hasIsNull() {
+      return ((bitField0_ & 0x00000020) == 0x00000020);
+    }
+    /**
+     * optional bool isNull = 6 [default = false];
+     *
+     * 
      +     ** 标识是否为空  *
+     * 
      
      */
     public boolean getIsNull() {
-      if (isNullPresentCase_ == 6) {
-        return (java.lang.Boolean) isNullPresent_;
-      }
-      return false;
+      return isNull_;
     }
 
     public static final int PROPS_FIELD_NUMBER = 7;
-    private java.util.List props_;
+    private java.util.List props_;
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
      */
-    public java.util.List getPropsList() {
+    public java.util.List getPropsList() {
       return props_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
      */
-    public java.util.List 
+    public java.util.List
         getPropsOrBuilderList() {
       return props_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
      */
     public int getPropsCount() {
       return props_.size();
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+    public Pair getProps(int index) {
       return props_.get(index);
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+    public PairOrBuilder getPropsOrBuilder(
         int index) {
       return props_.get(index);
     }
 
     public static final int VALUE_FIELD_NUMBER = 8;
-    private volatile java.lang.Object value_;
+    private Object value_;
     /**
+     * optional string value = 8;
+     *
      * 
            ** 字段值,timestamp,Datetime是一个时间格式的文本 *
      * 
      
-     *
-     * string value = 8;
      */
-    public java.lang.String getValue() {
-      java.lang.Object ref = value_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasValue() {
+      return ((bitField0_ & 0x00000040) == 0x00000040);
+    }
+    /**
+     * optional string value = 8;
+     *
+     * 
      +     ** 字段值,timestamp,Datetime是一个时间格式的文本 *
+     * 
      
+     */
+    public String getValue() {
+      Object ref = value_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        value_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          value_ = s;
+        }
         return s;
       }
     }
     /**
+     * optional string value = 8;
+     *
      * 
            ** 字段值,timestamp,Datetime是一个时间格式的文本 *
      * 
      
-     *
-     * string value = 8;
      */
     public com.google.protobuf.ByteString
         getValueBytes() {
-      java.lang.Object ref = value_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = value_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         value_ = b;
         return b;
       } else {
@@ -4650,51 +4539,73 @@ public final class CanalEntry {
     public static final int LENGTH_FIELD_NUMBER = 9;
     private int length_;
     /**
+     * optional int32 length = 9;
+     *
      * 
            ** 对应数据对象原始长度 *
      * 
      
+     */
+    public boolean hasLength() {
+      return ((bitField0_ & 0x00000080) == 0x00000080);
+    }
+    /**
+     * optional int32 length = 9;
      *
-     * int32 length = 9;
+     * 
      +     ** 对应数据对象原始长度 *
+     * 
      
      */
     public int getLength() {
       return length_;
     }
 
     public static final int MYSQLTYPE_FIELD_NUMBER = 10;
-    private volatile java.lang.Object mysqlType_;
+    private Object mysqlType_;
     /**
+     * optional string mysqlType = 10;
+     *
      * 
            **字段mysql类型*
      * 
      
-     *
-     * string mysqlType = 10;
      */
-    public java.lang.String getMysqlType() {
-      java.lang.Object ref = mysqlType_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasMysqlType() {
+      return ((bitField0_ & 0x00000100) == 0x00000100);
+    }
+    /**
+     * optional string mysqlType = 10;
+     *
+     * 
      +     **字段mysql类型*
+     * 
      
+     */
+    public String getMysqlType() {
+      Object ref = mysqlType_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        mysqlType_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          mysqlType_ = s;
+        }
         return s;
       }
     }
     /**
+     * optional string mysqlType = 10;
+     *
      * 
            **字段mysql类型*
      * 
      
-     *
-     * string mysqlType = 10;
      */
     public com.google.protobuf.ByteString
         getMysqlTypeBytes() {
-      java.lang.Object ref = mysqlType_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = mysqlType_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         mysqlType_ = b;
         return b;
       } else {
@@ -4702,8 +4613,19 @@ public final class CanalEntry {
       }
     }
 
+    private void initFields() {
+      index_ = 0;
+      sqlType_ = 0;
+      name_ = "";
+      isKey_ = false;
+      updated_ = false;
+      isNull_ = false;
+      props_ = java.util.Collections.emptyList();
+      value_ = "";
+      length_ = 0;
+      mysqlType_ = "";
+    }
     private byte memoizedIsInitialized = -1;
-    @java.lang.Override
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized == 1) return true;
@@ -4713,291 +4635,187 @@ public final class CanalEntry {
       return true;
     }
 
-    @java.lang.Override
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
-      if (index_ != 0) {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
         output.writeInt32(1, index_);
       }
-      if (sqlType_ != 0) {
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
         output.writeInt32(2, sqlType_);
       }
-      if (!getNameBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 3, name_);
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeBytes(3, getNameBytes());
       }
-      if (isKey_ != false) {
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
         output.writeBool(4, isKey_);
       }
-      if (updated_ != false) {
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
         output.writeBool(5, updated_);
       }
-      if (isNullPresentCase_ == 6) {
-        output.writeBool(
-            6, (boolean)((java.lang.Boolean) isNullPresent_));
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
+        output.writeBool(6, isNull_);
       }
       for (int i = 0; i < props_.size(); i++) {
         output.writeMessage(7, props_.get(i));
       }
-      if (!getValueBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 8, value_);
+      if (((bitField0_ & 0x00000040) == 0x00000040)) {
+        output.writeBytes(8, getValueBytes());
       }
-      if (length_ != 0) {
+      if (((bitField0_ & 0x00000080) == 0x00000080)) {
         output.writeInt32(9, length_);
       }
-      if (!getMysqlTypeBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 10, mysqlType_);
+      if (((bitField0_ & 0x00000100) == 0x00000100)) {
+        output.writeBytes(10, getMysqlTypeBytes());
       }
-      unknownFields.writeTo(output);
+      getUnknownFields().writeTo(output);
     }
 
-    @java.lang.Override
+    private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
-      int size = memoizedSize;
+      int size = memoizedSerializedSize;
       if (size != -1) return size;
 
       size = 0;
-      if (index_ != 0) {
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
           .computeInt32Size(1, index_);
       }
-      if (sqlType_ != 0) {
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
         size += com.google.protobuf.CodedOutputStream
           .computeInt32Size(2, sqlType_);
       }
-      if (!getNameBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, name_);
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(3, getNameBytes());
       }
-      if (isKey_ != false) {
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
         size += com.google.protobuf.CodedOutputStream
           .computeBoolSize(4, isKey_);
       }
-      if (updated_ != false) {
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
         size += com.google.protobuf.CodedOutputStream
           .computeBoolSize(5, updated_);
       }
-      if (isNullPresentCase_ == 6) {
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
         size += com.google.protobuf.CodedOutputStream
-          .computeBoolSize(
-              6, (boolean)((java.lang.Boolean) isNullPresent_));
+          .computeBoolSize(6, isNull_);
       }
       for (int i = 0; i < props_.size(); i++) {
         size += com.google.protobuf.CodedOutputStream
           .computeMessageSize(7, props_.get(i));
       }
-      if (!getValueBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(8, value_);
+      if (((bitField0_ & 0x00000040) == 0x00000040)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(8, getValueBytes());
       }
-      if (length_ != 0) {
+      if (((bitField0_ & 0x00000080) == 0x00000080)) {
         size += com.google.protobuf.CodedOutputStream
           .computeInt32Size(9, length_);
       }
-      if (!getMysqlTypeBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(10, mysqlType_);
+      if (((bitField0_ & 0x00000100) == 0x00000100)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(10, getMysqlTypeBytes());
       }
-      size += unknownFields.getSerializedSize();
-      memoizedSize = size;
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
       return size;
     }
 
-    @java.lang.Override
-    public boolean equals(final java.lang.Object obj) {
-      if (obj == this) {
-       return true;
-      }
-      if (!(obj instanceof com.alibaba.otter.canal.protocol.CanalEntry.Column)) {
-        return super.equals(obj);
-      }
-      com.alibaba.otter.canal.protocol.CanalEntry.Column other = (com.alibaba.otter.canal.protocol.CanalEntry.Column) obj;
-
-      boolean result = true;
-      result = result && (getIndex()
-          == other.getIndex());
-      result = result && (getSqlType()
-          == other.getSqlType());
-      result = result && getName()
-          .equals(other.getName());
-      result = result && (getIsKey()
-          == other.getIsKey());
-      result = result && (getUpdated()
-          == other.getUpdated());
-      result = result && getPropsList()
-          .equals(other.getPropsList());
-      result = result && getValue()
-          .equals(other.getValue());
-      result = result && (getLength()
-          == other.getLength());
-      result = result && getMysqlType()
-          .equals(other.getMysqlType());
-      result = result && getIsNullPresentCase().equals(
-          other.getIsNullPresentCase());
-      if (!result) return false;
-      switch (isNullPresentCase_) {
-        case 6:
-          result = result && (getIsNull()
-              == other.getIsNull());
-          break;
-        case 0:
-        default:
-      }
-      result = result && unknownFields.equals(other.unknownFields);
-      return result;
+    private static final long serialVersionUID = 0L;
+    @Override
+    protected Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
     }
 
-    @java.lang.Override
-    public int hashCode() {
-      if (memoizedHashCode != 0) {
-        return memoizedHashCode;
-      }
-      int hash = 41;
-      hash = (19 * hash) + getDescriptor().hashCode();
-      hash = (37 * hash) + INDEX_FIELD_NUMBER;
-      hash = (53 * hash) + getIndex();
-      hash = (37 * hash) + SQLTYPE_FIELD_NUMBER;
-      hash = (53 * hash) + getSqlType();
-      hash = (37 * hash) + NAME_FIELD_NUMBER;
-      hash = (53 * hash) + getName().hashCode();
-      hash = (37 * hash) + ISKEY_FIELD_NUMBER;
-      hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(
-          getIsKey());
-      hash = (37 * hash) + UPDATED_FIELD_NUMBER;
-      hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(
-          getUpdated());
-      if (getPropsCount() > 0) {
-        hash = (37 * hash) + PROPS_FIELD_NUMBER;
-        hash = (53 * hash) + getPropsList().hashCode();
-      }
-      hash = (37 * hash) + VALUE_FIELD_NUMBER;
-      hash = (53 * hash) + getValue().hashCode();
-      hash = (37 * hash) + LENGTH_FIELD_NUMBER;
-      hash = (53 * hash) + getLength();
-      hash = (37 * hash) + MYSQLTYPE_FIELD_NUMBER;
-      hash = (53 * hash) + getMysqlType().hashCode();
-      switch (isNullPresentCase_) {
-        case 6:
-          hash = (37 * hash) + ISNULL_FIELD_NUMBER;
-          hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(
-              getIsNull());
-          break;
-        case 0:
-        default:
-      }
-      hash = (29 * hash) + unknownFields.hashCode();
-      memoizedHashCode = hash;
-      return hash;
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
-        java.nio.ByteBuffer data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
-        java.nio.ByteBuffer data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
+    public static Column parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
+    public static Column parseFrom(
         com.google.protobuf.ByteString data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(byte[] data)
+    public static Column parseFrom(byte[] data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
+    public static Column parseFrom(
         byte[] data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(java.io.InputStream input)
+    public static Column parseFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
+    public static Column parseFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseDelimitedFrom(java.io.InputStream input)
+    public static Column parseDelimitedFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input);
+      return PARSER.parseDelimitedFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseDelimitedFrom(
+    public static Column parseDelimitedFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseDelimitedFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
+    public static Column parseFrom(
         com.google.protobuf.CodedInputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
+    public static Column parseFrom(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
 
-    @java.lang.Override
+    public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
-    public static Builder newBuilder() {
-      return DEFAULT_INSTANCE.toBuilder();
-    }
-    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.Column prototype) {
-      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
-    }
-    @java.lang.Override
-    public Builder toBuilder() {
-      return this == DEFAULT_INSTANCE
-          ? new Builder() : new Builder().mergeFrom(this);
+    public static Builder newBuilder(Column prototype) {
+      return newBuilder().mergeFrom(prototype);
     }
+    public Builder toBuilder() { return newBuilder(this); }
 
-    @java.lang.Override
+    @Override
     protected Builder newBuilderForType(
-        com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        BuilderParent parent) {
       Builder builder = new Builder(parent);
       return builder;
     }
     /**
+     * Protobuf type {@code com.alibaba.otter.canal.protocol.Column}
+     *
      * 
            **每个字段的数据结构*
      * 
      
-     *
-     * Protobuf type {@code com.alibaba.otter.canal.protocol.Column}
      */
     public static final class Builder extends
-        com.google.protobuf.GeneratedMessageV3.Builder implements
+        com.google.protobuf.GeneratedMessage.Builder implements
         // @@protoc_insertion_point(builder_implements:com.alibaba.otter.canal.protocol.Column)
-        com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder {
+        ColumnOrBuilder {
       public static final com.google.protobuf.Descriptors.Descriptor
           getDescriptor() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_descriptor;
       }
 
-      @java.lang.Override
-      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      protected FieldAccessorTable
           internalGetFieldAccessorTable() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_fieldAccessorTable
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_fieldAccessorTable
             .ensureFieldAccessorsInitialized(
-                com.alibaba.otter.canal.protocol.CanalEntry.Column.class, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder.class);
+                Column.class, Builder.class);
       }
 
       // Construct using com.alibaba.otter.canal.protocol.CanalEntry.Column.newBuilder()
@@ -5006,29 +4824,33 @@ public final class CanalEntry {
       }
 
       private Builder(
-          com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
       private void maybeForceBuilderInitialization() {
-        if (com.google.protobuf.GeneratedMessageV3
-                .alwaysUseFieldBuilders) {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
           getPropsFieldBuilder();
         }
       }
-      @java.lang.Override
+      private static Builder create() {
+        return new Builder();
+      }
+
       public Builder clear() {
         super.clear();
         index_ = 0;
-
+        bitField0_ = (bitField0_ & ~0x00000001);
         sqlType_ = 0;
-
+        bitField0_ = (bitField0_ & ~0x00000002);
         name_ = "";
-
+        bitField0_ = (bitField0_ & ~0x00000004);
         isKey_ = false;
-
+        bitField0_ = (bitField0_ & ~0x00000008);
         updated_ = false;
-
+        bitField0_ = (bitField0_ & ~0x00000010);
+        isNull_ = false;
+        bitField0_ = (bitField0_ & ~0x00000020);
         if (propsBuilder_ == null) {
           props_ = java.util.Collections.emptyList();
           bitField0_ = (bitField0_ & ~0x00000040);
@@ -5036,49 +4858,63 @@ public final class CanalEntry {
           propsBuilder_.clear();
         }
         value_ = "";
-
+        bitField0_ = (bitField0_ & ~0x00000080);
         length_ = 0;
-
+        bitField0_ = (bitField0_ & ~0x00000100);
         mysqlType_ = "";
-
-        isNullPresentCase_ = 0;
-        isNullPresent_ = null;
+        bitField0_ = (bitField0_ & ~0x00000200);
         return this;
       }
 
-      @java.lang.Override
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_descriptor;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.Column getDefaultInstanceForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.Column.getDefaultInstance();
+      public Column getDefaultInstanceForType() {
+        return Column.getDefaultInstance();
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.Column build() {
-        com.alibaba.otter.canal.protocol.CanalEntry.Column result = buildPartial();
+      public Column build() {
+        Column result = buildPartial();
         if (!result.isInitialized()) {
           throw newUninitializedMessageException(result);
         }
         return result;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.Column buildPartial() {
-        com.alibaba.otter.canal.protocol.CanalEntry.Column result = new com.alibaba.otter.canal.protocol.CanalEntry.Column(this);
+      public Column buildPartial() {
+        Column result = new Column(this);
         int from_bitField0_ = bitField0_;
         int to_bitField0_ = 0;
-        result.index_ = index_;
-        result.sqlType_ = sqlType_;
-        result.name_ = name_;
-        result.isKey_ = isKey_;
-        result.updated_ = updated_;
-        if (isNullPresentCase_ == 6) {
-          result.isNullPresent_ = isNullPresent_;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
         }
+        result.index_ = index_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.sqlType_ = sqlType_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.name_ = name_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000008;
+        }
+        result.isKey_ = isKey_;
+        if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+          to_bitField0_ |= 0x00000010;
+        }
+        result.updated_ = updated_;
+        if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+          to_bitField0_ |= 0x00000020;
+        }
+        result.isNull_ = isNull_;
         if (propsBuilder_ == null) {
           if (((bitField0_ & 0x00000040) == 0x00000040)) {
             props_ = java.util.Collections.unmodifiableList(props_);
@@ -5088,75 +4924,54 @@ public final class CanalEntry {
         } else {
           result.props_ = propsBuilder_.build();
         }
+        if (((from_bitField0_ & 0x00000080) == 0x00000080)) {
+          to_bitField0_ |= 0x00000040;
+        }
         result.value_ = value_;
+        if (((from_bitField0_ & 0x00000100) == 0x00000100)) {
+          to_bitField0_ |= 0x00000080;
+        }
         result.length_ = length_;
+        if (((from_bitField0_ & 0x00000200) == 0x00000200)) {
+          to_bitField0_ |= 0x00000100;
+        }
         result.mysqlType_ = mysqlType_;
         result.bitField0_ = to_bitField0_;
-        result.isNullPresentCase_ = isNullPresentCase_;
         onBuilt();
         return result;
       }
 
-      @java.lang.Override
-      public Builder clone() {
-        return (Builder) super.clone();
-      }
-      @java.lang.Override
-      public Builder setField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.setField(field, value);
-      }
-      @java.lang.Override
-      public Builder clearField(
-          com.google.protobuf.Descriptors.FieldDescriptor field) {
-        return (Builder) super.clearField(field);
-      }
-      @java.lang.Override
-      public Builder clearOneof(
-          com.google.protobuf.Descriptors.OneofDescriptor oneof) {
-        return (Builder) super.clearOneof(oneof);
-      }
-      @java.lang.Override
-      public Builder setRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          int index, java.lang.Object value) {
-        return (Builder) super.setRepeatedField(field, index, value);
-      }
-      @java.lang.Override
-      public Builder addRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.addRepeatedField(field, value);
-      }
-      @java.lang.Override
       public Builder mergeFrom(com.google.protobuf.Message other) {
-        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.Column) {
-          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.Column)other);
+        if (other instanceof Column) {
+          return mergeFrom((Column)other);
         } else {
           super.mergeFrom(other);
           return this;
         }
       }
 
-      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.Column other) {
-        if (other == com.alibaba.otter.canal.protocol.CanalEntry.Column.getDefaultInstance()) return this;
-        if (other.getIndex() != 0) {
+      public Builder mergeFrom(Column other) {
+        if (other == Column.getDefaultInstance()) return this;
+        if (other.hasIndex()) {
           setIndex(other.getIndex());
         }
-        if (other.getSqlType() != 0) {
+        if (other.hasSqlType()) {
           setSqlType(other.getSqlType());
         }
-        if (!other.getName().isEmpty()) {
+        if (other.hasName()) {
+          bitField0_ |= 0x00000004;
           name_ = other.name_;
           onChanged();
         }
-        if (other.getIsKey() != false) {
+        if (other.hasIsKey()) {
           setIsKey(other.getIsKey());
         }
-        if (other.getUpdated() != false) {
+        if (other.hasUpdated()) {
           setUpdated(other.getUpdated());
         }
+        if (other.hasIsNull()) {
+          setIsNull(other.getIsNull());
+        }
         if (propsBuilder_ == null) {
           if (!other.props_.isEmpty()) {
             if (props_.isEmpty()) {
@@ -5175,55 +4990,45 @@ public final class CanalEntry {
               propsBuilder_ = null;
               props_ = other.props_;
               bitField0_ = (bitField0_ & ~0x00000040);
-              propsBuilder_ = 
-                com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ?
+              propsBuilder_ =
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
                    getPropsFieldBuilder() : null;
             } else {
               propsBuilder_.addAllMessages(other.props_);
             }
           }
         }
-        if (!other.getValue().isEmpty()) {
+        if (other.hasValue()) {
+          bitField0_ |= 0x00000080;
           value_ = other.value_;
           onChanged();
         }
-        if (other.getLength() != 0) {
+        if (other.hasLength()) {
           setLength(other.getLength());
         }
-        if (!other.getMysqlType().isEmpty()) {
+        if (other.hasMysqlType()) {
+          bitField0_ |= 0x00000200;
           mysqlType_ = other.mysqlType_;
           onChanged();
         }
-        switch (other.getIsNullPresentCase()) {
-          case ISNULL: {
-            setIsNull(other.getIsNull());
-            break;
-          }
-          case ISNULLPRESENT_NOT_SET: {
-            break;
-          }
-        }
-        this.mergeUnknownFields(other.unknownFields);
-        onChanged();
+        this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
 
-      @java.lang.Override
       public final boolean isInitialized() {
         return true;
       }
 
-      @java.lang.Override
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
           throws java.io.IOException {
-        com.alibaba.otter.canal.protocol.CanalEntry.Column parsedMessage = null;
+        Column parsedMessage = null;
         try {
           parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
         } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          parsedMessage = (com.alibaba.otter.canal.protocol.CanalEntry.Column) e.getUnfinishedMessage();
-          throw e.unwrapIOException();
+          parsedMessage = (Column) e.getUnfinishedMessage();
+          throw e;
         } finally {
           if (parsedMessage != null) {
             mergeFrom(parsedMessage);
@@ -5231,56 +5036,51 @@ public final class CanalEntry {
         }
         return this;
       }
-      private int isNullPresentCase_ = 0;
-      private java.lang.Object isNullPresent_;
-      public IsNullPresentCase
-          getIsNullPresentCase() {
-        return IsNullPresentCase.forNumber(
-            isNullPresentCase_);
-      }
-
-      public Builder clearIsNullPresent() {
-        isNullPresentCase_ = 0;
-        isNullPresent_ = null;
-        onChanged();
-        return this;
-      }
-
       private int bitField0_;
 
       private int index_ ;
       /**
+       * optional int32 index = 1;
+       *
        * 
              **字段下标*
        * 
      
+       */
+      public boolean hasIndex() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      /**
+       * optional int32 index = 1;
        *
-       * int32 index = 1;
+       * 
      +       **字段下标*
+       * 
      
        */
       public int getIndex() {
         return index_;
       }
       /**
+       * optional int32 index = 1;
+       *
        * 
              **字段下标*
        * 
      
-       *
-       * int32 index = 1;
        */
       public Builder setIndex(int value) {
-        
+        bitField0_ |= 0x00000001;
         index_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional int32 index = 1;
+       *
        * 
              **字段下标*
        * 
      
-       *
-       * int32 index = 1;
        */
       public Builder clearIndex() {
-        
+        bitField0_ = (bitField0_ & ~0x00000001);
         index_ = 0;
         onChanged();
         return this;
@@ -5288,76 +5088,98 @@ public final class CanalEntry {
 
       private int sqlType_ ;
       /**
+       * optional int32 sqlType = 2;
+       *
        * 
              **字段java中类型*
        * 
      
+       */
+      public boolean hasSqlType() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      /**
+       * optional int32 sqlType = 2;
        *
-       * int32 sqlType = 2;
+       * 
      +       **字段java中类型*
+       * 
      
        */
       public int getSqlType() {
         return sqlType_;
       }
       /**
+       * optional int32 sqlType = 2;
+       *
        * 
              **字段java中类型*
        * 
      
-       *
-       * int32 sqlType = 2;
        */
       public Builder setSqlType(int value) {
-        
+        bitField0_ |= 0x00000002;
         sqlType_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional int32 sqlType = 2;
+       *
        * 
              **字段java中类型*
        * 
      
-       *
-       * int32 sqlType = 2;
        */
       public Builder clearSqlType() {
-        
+        bitField0_ = (bitField0_ & ~0x00000002);
         sqlType_ = 0;
         onChanged();
         return this;
       }
 
-      private java.lang.Object name_ = "";
+      private Object name_ = "";
       /**
+       * optional string name = 3;
+       *
        * 
              **字段名称(忽略大小写)，在mysql中是没有的*
        * 
      
-       *
-       * string name = 3;
        */
-      public java.lang.String getName() {
-        java.lang.Object ref = name_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasName() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      /**
+       * optional string name = 3;
+       *
+       * 
      +       **字段名称(忽略大小写)，在mysql中是没有的*
+       * 
      
+       */
+      public String getName() {
+        Object ref = name_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          name_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            name_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
+       * optional string name = 3;
+       *
        * 
              **字段名称(忽略大小写)，在mysql中是没有的*
        * 
      
-       *
-       * string name = 3;
        */
       public com.google.protobuf.ByteString
           getNameBytes() {
-        java.lang.Object ref = name_;
+        Object ref = name_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           name_ = b;
           return b;
         } else {
@@ -5365,49 +5187,48 @@ public final class CanalEntry {
         }
       }
       /**
+       * optional string name = 3;
+       *
        * 
              **字段名称(忽略大小写)，在mysql中是没有的*
        * 
      
-       *
-       * string name = 3;
        */
       public Builder setName(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000004;
         name_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional string name = 3;
+       *
        * 
              **字段名称(忽略大小写)，在mysql中是没有的*
        * 
      
-       *
-       * string name = 3;
        */
       public Builder clearName() {
-        
+        bitField0_ = (bitField0_ & ~0x00000004);
         name_ = getDefaultInstance().getName();
         onChanged();
         return this;
       }
       /**
+       * optional string name = 3;
+       *
        * 
              **字段名称(忽略大小写)，在mysql中是没有的*
        * 
      
-       *
-       * string name = 3;
        */
       public Builder setNameBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00000004;
         name_ = value;
         onChanged();
         return this;
@@ -5415,37 +5236,47 @@ public final class CanalEntry {
 
       private boolean isKey_ ;
       /**
+       * optional bool isKey = 4;
+       *
        * 
              **是否是主键*
        * 
      
+       */
+      public boolean hasIsKey() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      /**
+       * optional bool isKey = 4;
        *
-       * bool isKey = 4;
+       * 
      +       **是否是主键*
+       * 
      
        */
       public boolean getIsKey() {
         return isKey_;
       }
       /**
+       * optional bool isKey = 4;
+       *
        * 
              **是否是主键*
        * 
      
-       *
-       * bool isKey = 4;
        */
       public Builder setIsKey(boolean value) {
-        
+        bitField0_ |= 0x00000008;
         isKey_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional bool isKey = 4;
+       *
        * 
              **是否是主键*
        * 
      
-       *
-       * bool isKey = 4;
        */
       public Builder clearIsKey() {
-        
+        bitField0_ = (bitField0_ & ~0x00000008);
         isKey_ = false;
         onChanged();
         return this;
@@ -5453,92 +5284,120 @@ public final class CanalEntry {
 
       private boolean updated_ ;
       /**
+       * optional bool updated = 5;
+       *
        * 
              **如果EventType=UPDATE,用于标识这个字段值是否有修改*
        * 
      
+       */
+      public boolean hasUpdated() {
+        return ((bitField0_ & 0x00000010) == 0x00000010);
+      }
+      /**
+       * optional bool updated = 5;
        *
-       * bool updated = 5;
+       * 
      +       **如果EventType=UPDATE,用于标识这个字段值是否有修改*
+       * 
      
        */
       public boolean getUpdated() {
         return updated_;
       }
       /**
+       * optional bool updated = 5;
+       *
        * 
              **如果EventType=UPDATE,用于标识这个字段值是否有修改*
        * 
      
-       *
-       * bool updated = 5;
        */
       public Builder setUpdated(boolean value) {
-        
+        bitField0_ |= 0x00000010;
         updated_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional bool updated = 5;
+       *
        * 
              **如果EventType=UPDATE,用于标识这个字段值是否有修改*
        * 
      
-       *
-       * bool updated = 5;
        */
       public Builder clearUpdated() {
-        
+        bitField0_ = (bitField0_ & ~0x00000010);
         updated_ = false;
         onChanged();
         return this;
       }
 
+      private boolean isNull_ ;
       /**
-       * bool isNull = 6;
+       * optional bool isNull = 6 [default = false];
+       *
+       * 
      +       ** 标识是否为空  *
+       * 
      
        */
-      public boolean getIsNull() {
-        if (isNullPresentCase_ == 6) {
-          return (java.lang.Boolean) isNullPresent_;
-        }
-        return false;
+      public boolean hasIsNull() {
+        return ((bitField0_ & 0x00000020) == 0x00000020);
       }
       /**
-       * bool isNull = 6;
+       * optional bool isNull = 6 [default = false];
+       *
+       * 
      +       ** 标识是否为空  *
+       * 
      
+       */
+      public boolean getIsNull() {
+        return isNull_;
+      }
+      /**
+       * optional bool isNull = 6 [default = false];
+       *
+       * 
      +       ** 标识是否为空  *
+       * 
      
        */
       public Builder setIsNull(boolean value) {
-        isNullPresentCase_ = 6;
-        isNullPresent_ = value;
+        bitField0_ |= 0x00000020;
+        isNull_ = value;
         onChanged();
         return this;
       }
       /**
-       * bool isNull = 6;
+       * optional bool isNull = 6 [default = false];
+       *
+       * 
      +       ** 标识是否为空  *
+       * 
      
        */
       public Builder clearIsNull() {
-        if (isNullPresentCase_ == 6) {
-          isNullPresentCase_ = 0;
-          isNullPresent_ = null;
-          onChanged();
-        }
+        bitField0_ = (bitField0_ & ~0x00000020);
+        isNull_ = false;
+        onChanged();
         return this;
       }
 
-      private java.util.List props_ =
+      private java.util.List props_ =
         java.util.Collections.emptyList();
       private void ensurePropsIsMutable() {
         if (!((bitField0_ & 0x00000040) == 0x00000040)) {
-          props_ = new java.util.ArrayList(props_);
+          props_ = new java.util.ArrayList(props_);
           bitField0_ |= 0x00000040;
          }
       }
 
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> propsBuilder_;
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Pair, Pair.Builder, PairOrBuilder> propsBuilder_;
 
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
-      public java.util.List getPropsList() {
+      public java.util.List getPropsList() {
         if (propsBuilder_ == null) {
           return java.util.Collections.unmodifiableList(props_);
         } else {
@@ -5546,11 +5405,11 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
       public int getPropsCount() {
         if (propsBuilder_ == null) {
@@ -5560,13 +5419,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+      public Pair getProps(int index) {
         if (propsBuilder_ == null) {
           return props_.get(index);
         } else {
@@ -5574,14 +5433,14 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
       public Builder setProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+          int index, Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -5595,14 +5454,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
       public Builder setProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          int index, Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.set(index, builderForValue.build());
@@ -5613,13 +5472,13 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
-      public Builder addProps(com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+      public Builder addProps(Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -5633,14 +5492,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
       public Builder addProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+          int index, Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -5654,14 +5513,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
       public Builder addProps(
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.add(builderForValue.build());
@@ -5672,14 +5531,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
       public Builder addProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          int index, Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.add(index, builderForValue.build());
@@ -5690,14 +5549,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
       public Builder addAllProps(
-          java.lang.Iterable values) {
+          Iterable values) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           com.google.protobuf.AbstractMessageLite.Builder.addAll(
@@ -5709,11 +5568,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
       public Builder clearProps() {
         if (propsBuilder_ == null) {
@@ -5726,11 +5585,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
       public Builder removeProps(int index) {
         if (propsBuilder_ == null) {
@@ -5743,24 +5602,24 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder getPropsBuilder(
+      public Pair.Builder getPropsBuilder(
           int index) {
         return getPropsFieldBuilder().getBuilder(index);
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+      public PairOrBuilder getPropsOrBuilder(
           int index) {
         if (propsBuilder_ == null) {
           return props_.get(index);  } else {
@@ -5768,13 +5627,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
-      public java.util.List 
+      public java.util.List
            getPropsOrBuilderList() {
         if (propsBuilder_ != null) {
           return propsBuilder_.getMessageOrBuilderList();
@@ -5783,45 +5642,45 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder() {
+      public Pair.Builder addPropsBuilder() {
         return getPropsFieldBuilder().addBuilder(
-            com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+            Pair.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder(
+      public Pair.Builder addPropsBuilder(
           int index) {
         return getPropsFieldBuilder().addBuilder(
-            index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+            index, Pair.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
        */
-      public java.util.List 
+      public java.util.List
            getPropsBuilderList() {
         return getPropsFieldBuilder().getBuilderList();
       }
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> 
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Pair, Pair.Builder, PairOrBuilder>
           getPropsFieldBuilder() {
         if (propsBuilder_ == null) {
-          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3<
-              com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder>(
+          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              Pair, Pair.Builder, PairOrBuilder>(
                   props_,
                   ((bitField0_ & 0x00000040) == 0x00000040),
                   getParentForChildren(),
@@ -5831,40 +5690,52 @@ public final class CanalEntry {
         return propsBuilder_;
       }
 
-      private java.lang.Object value_ = "";
+      private Object value_ = "";
       /**
+       * optional string value = 8;
+       *
        * 
              ** 字段值,timestamp,Datetime是一个时间格式的文本 *
        * 
      
-       *
-       * string value = 8;
        */
-      public java.lang.String getValue() {
-        java.lang.Object ref = value_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasValue() {
+        return ((bitField0_ & 0x00000080) == 0x00000080);
+      }
+      /**
+       * optional string value = 8;
+       *
+       * 
      +       ** 字段值,timestamp,Datetime是一个时间格式的文本 *
+       * 
      
+       */
+      public String getValue() {
+        Object ref = value_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          value_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            value_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
+       * optional string value = 8;
+       *
        * 
              ** 字段值,timestamp,Datetime是一个时间格式的文本 *
        * 
      
-       *
-       * string value = 8;
        */
       public com.google.protobuf.ByteString
           getValueBytes() {
-        java.lang.Object ref = value_;
+        Object ref = value_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           value_ = b;
           return b;
         } else {
@@ -5872,49 +5743,48 @@ public final class CanalEntry {
         }
       }
       /**
+       * optional string value = 8;
+       *
        * 
              ** 字段值,timestamp,Datetime是一个时间格式的文本 *
        * 
      
-       *
-       * string value = 8;
        */
       public Builder setValue(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000080;
         value_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional string value = 8;
+       *
        * 
              ** 字段值,timestamp,Datetime是一个时间格式的文本 *
        * 
      
-       *
-       * string value = 8;
        */
       public Builder clearValue() {
-        
+        bitField0_ = (bitField0_ & ~0x00000080);
         value_ = getDefaultInstance().getValue();
         onChanged();
         return this;
       }
       /**
+       * optional string value = 8;
+       *
        * 
              ** 字段值,timestamp,Datetime是一个时间格式的文本 *
        * 
      
-       *
-       * string value = 8;
        */
       public Builder setValueBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00000080;
         value_ = value;
         onChanged();
         return this;
@@ -5922,76 +5792,98 @@ public final class CanalEntry {
 
       private int length_ ;
       /**
+       * optional int32 length = 9;
+       *
        * 
              ** 对应数据对象原始长度 *
        * 
      
+       */
+      public boolean hasLength() {
+        return ((bitField0_ & 0x00000100) == 0x00000100);
+      }
+      /**
+       * optional int32 length = 9;
        *
-       * int32 length = 9;
+       * 
      +       ** 对应数据对象原始长度 *
+       * 
      
        */
       public int getLength() {
         return length_;
       }
       /**
+       * optional int32 length = 9;
+       *
        * 
              ** 对应数据对象原始长度 *
        * 
      
-       *
-       * int32 length = 9;
        */
       public Builder setLength(int value) {
-        
+        bitField0_ |= 0x00000100;
         length_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional int32 length = 9;
+       *
        * 
              ** 对应数据对象原始长度 *
        * 
      
-       *
-       * int32 length = 9;
        */
       public Builder clearLength() {
-        
+        bitField0_ = (bitField0_ & ~0x00000100);
         length_ = 0;
         onChanged();
         return this;
       }
 
-      private java.lang.Object mysqlType_ = "";
+      private Object mysqlType_ = "";
       /**
+       * optional string mysqlType = 10;
+       *
        * 
              **字段mysql类型*
        * 
      
-       *
-       * string mysqlType = 10;
        */
-      public java.lang.String getMysqlType() {
-        java.lang.Object ref = mysqlType_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasMysqlType() {
+        return ((bitField0_ & 0x00000200) == 0x00000200);
+      }
+      /**
+       * optional string mysqlType = 10;
+       *
+       * 
      +       **字段mysql类型*
+       * 
      
+       */
+      public String getMysqlType() {
+        Object ref = mysqlType_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          mysqlType_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            mysqlType_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
+       * optional string mysqlType = 10;
+       *
        * 
              **字段mysql类型*
        * 
      
-       *
-       * string mysqlType = 10;
        */
       public com.google.protobuf.ByteString
           getMysqlTypeBytes() {
-        java.lang.Object ref = mysqlType_;
+        Object ref = mysqlType_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           mysqlType_ = b;
           return b;
         } else {
@@ -5999,104 +5891,62 @@ public final class CanalEntry {
         }
       }
       /**
+       * optional string mysqlType = 10;
+       *
        * 
              **字段mysql类型*
        * 
      
-       *
-       * string mysqlType = 10;
        */
       public Builder setMysqlType(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000200;
         mysqlType_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional string mysqlType = 10;
+       *
        * 
              **字段mysql类型*
        * 
      
-       *
-       * string mysqlType = 10;
        */
       public Builder clearMysqlType() {
-        
+        bitField0_ = (bitField0_ & ~0x00000200);
         mysqlType_ = getDefaultInstance().getMysqlType();
         onChanged();
         return this;
       }
       /**
+       * optional string mysqlType = 10;
+       *
        * 
              **字段mysql类型*
        * 
      
-       *
-       * string mysqlType = 10;
        */
       public Builder setMysqlTypeBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00000200;
         mysqlType_ = value;
         onChanged();
         return this;
       }
-      @java.lang.Override
-      public final Builder setUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.setUnknownFieldsProto3(unknownFields);
-      }
-
-      @java.lang.Override
-      public final Builder mergeUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.mergeUnknownFields(unknownFields);
-      }
-
 
       // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Column)
     }
 
-    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Column)
-    private static final com.alibaba.otter.canal.protocol.CanalEntry.Column DEFAULT_INSTANCE;
     static {
-      DEFAULT_INSTANCE = new com.alibaba.otter.canal.protocol.CanalEntry.Column();
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Column getDefaultInstance() {
-      return DEFAULT_INSTANCE;
-    }
-
-    private static final com.google.protobuf.Parser
-        PARSER = new com.google.protobuf.AbstractParser() {
-      @java.lang.Override
-      public Column parsePartialFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return new Column(input, extensionRegistry);
-      }
-    };
-
-    public static com.google.protobuf.Parser parser() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.google.protobuf.Parser getParserForType() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.alibaba.otter.canal.protocol.CanalEntry.Column getDefaultInstanceForType() {
-      return DEFAULT_INSTANCE;
+      defaultInstance = new Column(true);
+      defaultInstance.initFields();
     }
 
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Column)
   }
 
   public interface RowDataOrBuilder extends
@@ -6104,168 +5954,168 @@ public final class CanalEntry {
       com.google.protobuf.MessageOrBuilder {
 
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+     *
      * 
            ** 字段信息，增量数据(修改前,删除前) *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
      */
-    java.util.List 
+    java.util.List
         getBeforeColumnsList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+     *
      * 
            ** 字段信息，增量数据(修改前,删除前) *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.Column getBeforeColumns(int index);
+    Column getBeforeColumns(int index);
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+     *
      * 
            ** 字段信息，增量数据(修改前,删除前) *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
      */
     int getBeforeColumnsCount();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+     *
      * 
            ** 字段信息，增量数据(修改前,删除前) *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
      */
-    java.util.List 
+    java.util.List
         getBeforeColumnsOrBuilderList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+     *
      * 
            ** 字段信息，增量数据(修改前,删除前) *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder getBeforeColumnsOrBuilder(
-        int index);
+    ColumnOrBuilder getBeforeColumnsOrBuilder(int index);
 
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+     *
      * 
            ** 字段信息，增量数据(修改后,新增后)  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
      */
-    java.util.List 
+    java.util.List
         getAfterColumnsList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+     *
      * 
            ** 字段信息，增量数据(修改后,新增后)  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.Column getAfterColumns(int index);
+    Column getAfterColumns(int index);
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+     *
      * 
            ** 字段信息，增量数据(修改后,新增后)  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
      */
     int getAfterColumnsCount();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+     *
      * 
            ** 字段信息，增量数据(修改后,新增后)  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
      */
-    java.util.List 
+    java.util.List
         getAfterColumnsOrBuilderList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+     *
      * 
            ** 字段信息，增量数据(修改后,新增后)  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder getAfterColumnsOrBuilder(
-        int index);
+    ColumnOrBuilder getAfterColumnsOrBuilder(int index);
 
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    java.util.List 
+    java.util.List
         getPropsList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index);
+    Pair getProps(int index);
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
     int getPropsCount();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    java.util.List 
+    java.util.List
         getPropsOrBuilderList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
-        int index);
+    PairOrBuilder getPropsOrBuilder(int index);
   }
   /**
    * Protobuf type {@code com.alibaba.otter.canal.protocol.RowData}
    */
-  public  static final class RowData extends
-      com.google.protobuf.GeneratedMessageV3 implements
+  public static final class RowData extends
+      com.google.protobuf.GeneratedMessage implements
       // @@protoc_insertion_point(message_implements:com.alibaba.otter.canal.protocol.RowData)
       RowDataOrBuilder {
-  private static final long serialVersionUID = 0L;
     // Use RowData.newBuilder() to construct.
-    private RowData(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+    private RowData(com.google.protobuf.GeneratedMessage.Builder builder) {
       super(builder);
+      this.unknownFields = builder.getUnknownFields();
     }
-    private RowData() {
-      beforeColumns_ = java.util.Collections.emptyList();
-      afterColumns_ = java.util.Collections.emptyList();
-      props_ = java.util.Collections.emptyList();
+    private RowData(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+    private static final RowData defaultInstance;
+    public static RowData getDefaultInstance() {
+      return defaultInstance;
     }
 
-    @java.lang.Override
+    public RowData getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+
+    private final com.google.protobuf.UnknownFieldSet unknownFields;
+    @Override
     public final com.google.protobuf.UnknownFieldSet
-    getUnknownFields() {
+        getUnknownFields() {
       return this.unknownFields;
     }
     private RowData(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
-      this();
-      if (extensionRegistry == null) {
-        throw new java.lang.NullPointerException();
-      }
+      initFields();
       int mutable_bitField0_ = 0;
       com.google.protobuf.UnknownFieldSet.Builder unknownFields =
           com.google.protobuf.UnknownFieldSet.newBuilder();
@@ -6277,38 +6127,35 @@ public final class CanalEntry {
             case 0:
               done = true;
               break;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                done = true;
+              }
+              break;
+            }
             case 10: {
               if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
-                beforeColumns_ = new java.util.ArrayList();
+                beforeColumns_ = new java.util.ArrayList();
                 mutable_bitField0_ |= 0x00000001;
               }
-              beforeColumns_.add(
-                  input.readMessage(com.alibaba.otter.canal.protocol.CanalEntry.Column.parser(), extensionRegistry));
+              beforeColumns_.add(input.readMessage(Column.PARSER, extensionRegistry));
               break;
             }
             case 18: {
               if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) {
-                afterColumns_ = new java.util.ArrayList();
+                afterColumns_ = new java.util.ArrayList();
                 mutable_bitField0_ |= 0x00000002;
               }
-              afterColumns_.add(
-                  input.readMessage(com.alibaba.otter.canal.protocol.CanalEntry.Column.parser(), extensionRegistry));
+              afterColumns_.add(input.readMessage(Column.PARSER, extensionRegistry));
               break;
             }
             case 26: {
               if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
-                props_ = new java.util.ArrayList();
+                props_ = new java.util.ArrayList();
                 mutable_bitField0_ |= 0x00000004;
               }
-              props_.add(
-                  input.readMessage(com.alibaba.otter.canal.protocol.CanalEntry.Pair.parser(), extensionRegistry));
-              break;
-            }
-            default: {
-              if (!parseUnknownFieldProto3(
-                  input, unknownFields, extensionRegistry, tag)) {
-                done = true;
-              }
+              props_.add(input.readMessage(Pair.PARSER, extensionRegistry));
               break;
             }
           }
@@ -6317,7 +6164,7 @@ public final class CanalEntry {
         throw e.setUnfinishedMessage(this);
       } catch (java.io.IOException e) {
         throw new com.google.protobuf.InvalidProtocolBufferException(
-            e).setUnfinishedMessage(this);
+            e.getMessage()).setUnfinishedMessage(this);
       } finally {
         if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
           beforeColumns_ = java.util.Collections.unmodifiableList(beforeColumns_);
@@ -6334,184 +6181,202 @@ public final class CanalEntry {
     }
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor;
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor;
     }
 
-    @java.lang.Override
-    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+    protected FieldAccessorTable
         internalGetFieldAccessorTable() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_fieldAccessorTable
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_fieldAccessorTable
           .ensureFieldAccessorsInitialized(
-              com.alibaba.otter.canal.protocol.CanalEntry.RowData.class, com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder.class);
+              RowData.class, Builder.class);
+    }
+
+    public static com.google.protobuf.Parser PARSER =
+        new com.google.protobuf.AbstractParser() {
+      public RowData parsePartialFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return new RowData(input, extensionRegistry);
+      }
+    };
+
+    @Override
+    public com.google.protobuf.Parser getParserForType() {
+      return PARSER;
     }
 
     public static final int BEFORECOLUMNS_FIELD_NUMBER = 1;
-    private java.util.List beforeColumns_;
+    private java.util.List beforeColumns_;
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+     *
      * 
            ** 字段信息，增量数据(修改前,删除前) *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
      */
-    public java.util.List getBeforeColumnsList() {
+    public java.util.List getBeforeColumnsList() {
       return beforeColumns_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+     *
      * 
            ** 字段信息，增量数据(修改前,删除前) *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
      */
-    public java.util.List 
+    public java.util.List
         getBeforeColumnsOrBuilderList() {
       return beforeColumns_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+     *
      * 
            ** 字段信息，增量数据(修改前,删除前) *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
      */
     public int getBeforeColumnsCount() {
       return beforeColumns_.size();
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+     *
      * 
            ** 字段信息，增量数据(修改前,删除前) *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.Column getBeforeColumns(int index) {
+    public Column getBeforeColumns(int index) {
       return beforeColumns_.get(index);
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+     *
      * 
            ** 字段信息，增量数据(修改前,删除前) *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder getBeforeColumnsOrBuilder(
+    public ColumnOrBuilder getBeforeColumnsOrBuilder(
         int index) {
       return beforeColumns_.get(index);
     }
 
     public static final int AFTERCOLUMNS_FIELD_NUMBER = 2;
-    private java.util.List afterColumns_;
+    private java.util.List afterColumns_;
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+     *
      * 
            ** 字段信息，增量数据(修改后,新增后)  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
      */
-    public java.util.List getAfterColumnsList() {
+    public java.util.List getAfterColumnsList() {
       return afterColumns_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+     *
      * 
            ** 字段信息，增量数据(修改后,新增后)  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
      */
-    public java.util.List 
+    public java.util.List
         getAfterColumnsOrBuilderList() {
       return afterColumns_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+     *
      * 
            ** 字段信息，增量数据(修改后,新增后)  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
      */
     public int getAfterColumnsCount() {
       return afterColumns_.size();
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+     *
      * 
            ** 字段信息，增量数据(修改后,新增后)  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.Column getAfterColumns(int index) {
+    public Column getAfterColumns(int index) {
       return afterColumns_.get(index);
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+     *
      * 
            ** 字段信息，增量数据(修改后,新增后)  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder getAfterColumnsOrBuilder(
+    public ColumnOrBuilder getAfterColumnsOrBuilder(
         int index) {
       return afterColumns_.get(index);
     }
 
     public static final int PROPS_FIELD_NUMBER = 3;
-    private java.util.List props_;
+    private java.util.List props_;
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    public java.util.List getPropsList() {
+    public java.util.List getPropsList() {
       return props_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    public java.util.List 
+    public java.util.List
         getPropsOrBuilderList() {
       return props_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
     public int getPropsCount() {
       return props_.size();
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+    public Pair getProps(int index) {
       return props_.get(index);
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+    public PairOrBuilder getPropsOrBuilder(
         int index) {
       return props_.get(index);
     }
 
+    private void initFields() {
+      beforeColumns_ = java.util.Collections.emptyList();
+      afterColumns_ = java.util.Collections.emptyList();
+      props_ = java.util.Collections.emptyList();
+    }
     private byte memoizedIsInitialized = -1;
-    @java.lang.Override
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized == 1) return true;
@@ -6521,9 +6386,9 @@ public final class CanalEntry {
       return true;
     }
 
-    @java.lang.Override
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
+      getSerializedSize();
       for (int i = 0; i < beforeColumns_.size(); i++) {
         output.writeMessage(1, beforeColumns_.get(i));
       }
@@ -6533,12 +6398,12 @@ public final class CanalEntry {
       for (int i = 0; i < props_.size(); i++) {
         output.writeMessage(3, props_.get(i));
       }
-      unknownFields.writeTo(output);
+      getUnknownFields().writeTo(output);
     }
 
-    @java.lang.Override
+    private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
-      int size = memoizedSize;
+      int size = memoizedSerializedSize;
       if (size != -1) return size;
 
       size = 0;
@@ -6554,143 +6419,81 @@ public final class CanalEntry {
         size += com.google.protobuf.CodedOutputStream
           .computeMessageSize(3, props_.get(i));
       }
-      size += unknownFields.getSerializedSize();
-      memoizedSize = size;
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
       return size;
     }
 
-    @java.lang.Override
-    public boolean equals(final java.lang.Object obj) {
-      if (obj == this) {
-       return true;
-      }
-      if (!(obj instanceof com.alibaba.otter.canal.protocol.CanalEntry.RowData)) {
-        return super.equals(obj);
-      }
-      com.alibaba.otter.canal.protocol.CanalEntry.RowData other = (com.alibaba.otter.canal.protocol.CanalEntry.RowData) obj;
-
-      boolean result = true;
-      result = result && getBeforeColumnsList()
-          .equals(other.getBeforeColumnsList());
-      result = result && getAfterColumnsList()
-          .equals(other.getAfterColumnsList());
-      result = result && getPropsList()
-          .equals(other.getPropsList());
-      result = result && unknownFields.equals(other.unknownFields);
-      return result;
+    private static final long serialVersionUID = 0L;
+    @Override
+    protected Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
     }
 
-    @java.lang.Override
-    public int hashCode() {
-      if (memoizedHashCode != 0) {
-        return memoizedHashCode;
-      }
-      int hash = 41;
-      hash = (19 * hash) + getDescriptor().hashCode();
-      if (getBeforeColumnsCount() > 0) {
-        hash = (37 * hash) + BEFORECOLUMNS_FIELD_NUMBER;
-        hash = (53 * hash) + getBeforeColumnsList().hashCode();
-      }
-      if (getAfterColumnsCount() > 0) {
-        hash = (37 * hash) + AFTERCOLUMNS_FIELD_NUMBER;
-        hash = (53 * hash) + getAfterColumnsList().hashCode();
-      }
-      if (getPropsCount() > 0) {
-        hash = (37 * hash) + PROPS_FIELD_NUMBER;
-        hash = (53 * hash) + getPropsList().hashCode();
-      }
-      hash = (29 * hash) + unknownFields.hashCode();
-      memoizedHashCode = hash;
-      return hash;
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
-        java.nio.ByteBuffer data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
-        java.nio.ByteBuffer data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
+    public static RowData parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
+    public static RowData parseFrom(
         com.google.protobuf.ByteString data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(byte[] data)
+    public static RowData parseFrom(byte[] data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
+    public static RowData parseFrom(
         byte[] data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(java.io.InputStream input)
+    public static RowData parseFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
+    public static RowData parseFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseDelimitedFrom(java.io.InputStream input)
+    public static RowData parseDelimitedFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input);
+      return PARSER.parseDelimitedFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseDelimitedFrom(
+    public static RowData parseDelimitedFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseDelimitedFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
+    public static RowData parseFrom(
         com.google.protobuf.CodedInputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
+    public static RowData parseFrom(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
 
-    @java.lang.Override
+    public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
-    public static Builder newBuilder() {
-      return DEFAULT_INSTANCE.toBuilder();
-    }
-    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.RowData prototype) {
-      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
-    }
-    @java.lang.Override
-    public Builder toBuilder() {
-      return this == DEFAULT_INSTANCE
-          ? new Builder() : new Builder().mergeFrom(this);
+    public static Builder newBuilder(RowData prototype) {
+      return newBuilder().mergeFrom(prototype);
     }
+    public Builder toBuilder() { return newBuilder(this); }
 
-    @java.lang.Override
+    @Override
     protected Builder newBuilderForType(
-        com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        BuilderParent parent) {
       Builder builder = new Builder(parent);
       return builder;
     }
@@ -6698,20 +6501,19 @@ public final class CanalEntry {
      * Protobuf type {@code com.alibaba.otter.canal.protocol.RowData}
      */
     public static final class Builder extends
-        com.google.protobuf.GeneratedMessageV3.Builder implements
+        com.google.protobuf.GeneratedMessage.Builder implements
         // @@protoc_insertion_point(builder_implements:com.alibaba.otter.canal.protocol.RowData)
-        com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder {
+        RowDataOrBuilder {
       public static final com.google.protobuf.Descriptors.Descriptor
           getDescriptor() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor;
       }
 
-      @java.lang.Override
-      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      protected FieldAccessorTable
           internalGetFieldAccessorTable() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_fieldAccessorTable
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_fieldAccessorTable
             .ensureFieldAccessorsInitialized(
-                com.alibaba.otter.canal.protocol.CanalEntry.RowData.class, com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder.class);
+                RowData.class, Builder.class);
       }
 
       // Construct using com.alibaba.otter.canal.protocol.CanalEntry.RowData.newBuilder()
@@ -6720,19 +6522,21 @@ public final class CanalEntry {
       }
 
       private Builder(
-          com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
       private void maybeForceBuilderInitialization() {
-        if (com.google.protobuf.GeneratedMessageV3
-                .alwaysUseFieldBuilders) {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
           getBeforeColumnsFieldBuilder();
           getAfterColumnsFieldBuilder();
           getPropsFieldBuilder();
         }
       }
-      @java.lang.Override
+      private static Builder create() {
+        return new Builder();
+      }
+
       public Builder clear() {
         super.clear();
         if (beforeColumnsBuilder_ == null) {
@@ -6756,29 +6560,29 @@ public final class CanalEntry {
         return this;
       }
 
-      @java.lang.Override
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.RowData getDefaultInstanceForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.RowData.getDefaultInstance();
+      public RowData getDefaultInstanceForType() {
+        return RowData.getDefaultInstance();
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.RowData build() {
-        com.alibaba.otter.canal.protocol.CanalEntry.RowData result = buildPartial();
+      public RowData build() {
+        RowData result = buildPartial();
         if (!result.isInitialized()) {
           throw newUninitializedMessageException(result);
         }
         return result;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.RowData buildPartial() {
-        com.alibaba.otter.canal.protocol.CanalEntry.RowData result = new com.alibaba.otter.canal.protocol.CanalEntry.RowData(this);
+      public RowData buildPartial() {
+        RowData result = new RowData(this);
         int from_bitField0_ = bitField0_;
         if (beforeColumnsBuilder_ == null) {
           if (((bitField0_ & 0x00000001) == 0x00000001)) {
@@ -6811,50 +6615,17 @@ public final class CanalEntry {
         return result;
       }
 
-      @java.lang.Override
-      public Builder clone() {
-        return (Builder) super.clone();
-      }
-      @java.lang.Override
-      public Builder setField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.setField(field, value);
-      }
-      @java.lang.Override
-      public Builder clearField(
-          com.google.protobuf.Descriptors.FieldDescriptor field) {
-        return (Builder) super.clearField(field);
-      }
-      @java.lang.Override
-      public Builder clearOneof(
-          com.google.protobuf.Descriptors.OneofDescriptor oneof) {
-        return (Builder) super.clearOneof(oneof);
-      }
-      @java.lang.Override
-      public Builder setRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          int index, java.lang.Object value) {
-        return (Builder) super.setRepeatedField(field, index, value);
-      }
-      @java.lang.Override
-      public Builder addRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.addRepeatedField(field, value);
-      }
-      @java.lang.Override
       public Builder mergeFrom(com.google.protobuf.Message other) {
-        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.RowData) {
-          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.RowData)other);
+        if (other instanceof RowData) {
+          return mergeFrom((RowData)other);
         } else {
           super.mergeFrom(other);
           return this;
         }
       }
 
-      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.RowData other) {
-        if (other == com.alibaba.otter.canal.protocol.CanalEntry.RowData.getDefaultInstance()) return this;
+      public Builder mergeFrom(RowData other) {
+        if (other == RowData.getDefaultInstance()) return this;
         if (beforeColumnsBuilder_ == null) {
           if (!other.beforeColumns_.isEmpty()) {
             if (beforeColumns_.isEmpty()) {
@@ -6873,8 +6644,8 @@ public final class CanalEntry {
               beforeColumnsBuilder_ = null;
               beforeColumns_ = other.beforeColumns_;
               bitField0_ = (bitField0_ & ~0x00000001);
-              beforeColumnsBuilder_ = 
-                com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ?
+              beforeColumnsBuilder_ =
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
                    getBeforeColumnsFieldBuilder() : null;
             } else {
               beforeColumnsBuilder_.addAllMessages(other.beforeColumns_);
@@ -6899,8 +6670,8 @@ public final class CanalEntry {
               afterColumnsBuilder_ = null;
               afterColumns_ = other.afterColumns_;
               bitField0_ = (bitField0_ & ~0x00000002);
-              afterColumnsBuilder_ = 
-                com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ?
+              afterColumnsBuilder_ =
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
                    getAfterColumnsFieldBuilder() : null;
             } else {
               afterColumnsBuilder_.addAllMessages(other.afterColumns_);
@@ -6925,35 +6696,32 @@ public final class CanalEntry {
               propsBuilder_ = null;
               props_ = other.props_;
               bitField0_ = (bitField0_ & ~0x00000004);
-              propsBuilder_ = 
-                com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ?
+              propsBuilder_ =
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
                    getPropsFieldBuilder() : null;
             } else {
               propsBuilder_.addAllMessages(other.props_);
             }
           }
         }
-        this.mergeUnknownFields(other.unknownFields);
-        onChanged();
+        this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
 
-      @java.lang.Override
       public final boolean isInitialized() {
         return true;
       }
 
-      @java.lang.Override
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
           throws java.io.IOException {
-        com.alibaba.otter.canal.protocol.CanalEntry.RowData parsedMessage = null;
+        RowData parsedMessage = null;
         try {
           parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
         } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          parsedMessage = (com.alibaba.otter.canal.protocol.CanalEntry.RowData) e.getUnfinishedMessage();
-          throw e.unwrapIOException();
+          parsedMessage = (RowData) e.getUnfinishedMessage();
+          throw e;
         } finally {
           if (parsedMessage != null) {
             mergeFrom(parsedMessage);
@@ -6963,26 +6731,26 @@ public final class CanalEntry {
       }
       private int bitField0_;
 
-      private java.util.List beforeColumns_ =
+      private java.util.List beforeColumns_ =
         java.util.Collections.emptyList();
       private void ensureBeforeColumnsIsMutable() {
         if (!((bitField0_ & 0x00000001) == 0x00000001)) {
-          beforeColumns_ = new java.util.ArrayList(beforeColumns_);
+          beforeColumns_ = new java.util.ArrayList(beforeColumns_);
           bitField0_ |= 0x00000001;
          }
       }
 
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Column, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder, com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder> beforeColumnsBuilder_;
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Column, Column.Builder, ColumnOrBuilder> beforeColumnsBuilder_;
 
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
-      public java.util.List getBeforeColumnsList() {
+      public java.util.List getBeforeColumnsList() {
         if (beforeColumnsBuilder_ == null) {
           return java.util.Collections.unmodifiableList(beforeColumns_);
         } else {
@@ -6990,11 +6758,11 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
       public int getBeforeColumnsCount() {
         if (beforeColumnsBuilder_ == null) {
@@ -7004,13 +6772,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Column getBeforeColumns(int index) {
+      public Column getBeforeColumns(int index) {
         if (beforeColumnsBuilder_ == null) {
           return beforeColumns_.get(index);
         } else {
@@ -7018,14 +6786,14 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
       public Builder setBeforeColumns(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column value) {
+          int index, Column value) {
         if (beforeColumnsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -7039,14 +6807,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
       public Builder setBeforeColumns(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder builderForValue) {
+          int index, Column.Builder builderForValue) {
         if (beforeColumnsBuilder_ == null) {
           ensureBeforeColumnsIsMutable();
           beforeColumns_.set(index, builderForValue.build());
@@ -7057,13 +6825,13 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
-      public Builder addBeforeColumns(com.alibaba.otter.canal.protocol.CanalEntry.Column value) {
+      public Builder addBeforeColumns(Column value) {
         if (beforeColumnsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -7077,14 +6845,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
       public Builder addBeforeColumns(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column value) {
+          int index, Column value) {
         if (beforeColumnsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -7098,14 +6866,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
       public Builder addBeforeColumns(
-          com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder builderForValue) {
+          Column.Builder builderForValue) {
         if (beforeColumnsBuilder_ == null) {
           ensureBeforeColumnsIsMutable();
           beforeColumns_.add(builderForValue.build());
@@ -7116,14 +6884,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
       public Builder addBeforeColumns(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder builderForValue) {
+          int index, Column.Builder builderForValue) {
         if (beforeColumnsBuilder_ == null) {
           ensureBeforeColumnsIsMutable();
           beforeColumns_.add(index, builderForValue.build());
@@ -7134,14 +6902,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
       public Builder addAllBeforeColumns(
-          java.lang.Iterable values) {
+          Iterable values) {
         if (beforeColumnsBuilder_ == null) {
           ensureBeforeColumnsIsMutable();
           com.google.protobuf.AbstractMessageLite.Builder.addAll(
@@ -7153,11 +6921,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
       public Builder clearBeforeColumns() {
         if (beforeColumnsBuilder_ == null) {
@@ -7170,11 +6938,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
       public Builder removeBeforeColumns(int index) {
         if (beforeColumnsBuilder_ == null) {
@@ -7187,24 +6955,24 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder getBeforeColumnsBuilder(
+      public Column.Builder getBeforeColumnsBuilder(
           int index) {
         return getBeforeColumnsFieldBuilder().getBuilder(index);
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder getBeforeColumnsOrBuilder(
+      public ColumnOrBuilder getBeforeColumnsOrBuilder(
           int index) {
         if (beforeColumnsBuilder_ == null) {
           return beforeColumns_.get(index);  } else {
@@ -7212,13 +6980,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
-      public java.util.List 
+      public java.util.List
            getBeforeColumnsOrBuilderList() {
         if (beforeColumnsBuilder_ != null) {
           return beforeColumnsBuilder_.getMessageOrBuilderList();
@@ -7227,45 +6995,45 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder addBeforeColumnsBuilder() {
+      public Column.Builder addBeforeColumnsBuilder() {
         return getBeforeColumnsFieldBuilder().addBuilder(
-            com.alibaba.otter.canal.protocol.CanalEntry.Column.getDefaultInstance());
+            Column.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder addBeforeColumnsBuilder(
+      public Column.Builder addBeforeColumnsBuilder(
           int index) {
         return getBeforeColumnsFieldBuilder().addBuilder(
-            index, com.alibaba.otter.canal.protocol.CanalEntry.Column.getDefaultInstance());
+            index, Column.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+       *
        * 
              ** 字段信息，增量数据(修改前,删除前) *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
        */
-      public java.util.List 
+      public java.util.List
            getBeforeColumnsBuilderList() {
         return getBeforeColumnsFieldBuilder().getBuilderList();
       }
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Column, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder, com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder> 
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Column, Column.Builder, ColumnOrBuilder>
           getBeforeColumnsFieldBuilder() {
         if (beforeColumnsBuilder_ == null) {
-          beforeColumnsBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3<
-              com.alibaba.otter.canal.protocol.CanalEntry.Column, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder, com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder>(
+          beforeColumnsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              Column, Column.Builder, ColumnOrBuilder>(
                   beforeColumns_,
                   ((bitField0_ & 0x00000001) == 0x00000001),
                   getParentForChildren(),
@@ -7275,26 +7043,26 @@ public final class CanalEntry {
         return beforeColumnsBuilder_;
       }
 
-      private java.util.List afterColumns_ =
+      private java.util.List afterColumns_ =
         java.util.Collections.emptyList();
       private void ensureAfterColumnsIsMutable() {
         if (!((bitField0_ & 0x00000002) == 0x00000002)) {
-          afterColumns_ = new java.util.ArrayList(afterColumns_);
+          afterColumns_ = new java.util.ArrayList(afterColumns_);
           bitField0_ |= 0x00000002;
          }
       }
 
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Column, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder, com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder> afterColumnsBuilder_;
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Column, Column.Builder, ColumnOrBuilder> afterColumnsBuilder_;
 
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
-      public java.util.List getAfterColumnsList() {
+      public java.util.List getAfterColumnsList() {
         if (afterColumnsBuilder_ == null) {
           return java.util.Collections.unmodifiableList(afterColumns_);
         } else {
@@ -7302,11 +7070,11 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
       public int getAfterColumnsCount() {
         if (afterColumnsBuilder_ == null) {
@@ -7316,13 +7084,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Column getAfterColumns(int index) {
+      public Column getAfterColumns(int index) {
         if (afterColumnsBuilder_ == null) {
           return afterColumns_.get(index);
         } else {
@@ -7330,14 +7098,14 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
       public Builder setAfterColumns(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column value) {
+          int index, Column value) {
         if (afterColumnsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -7351,14 +7119,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
       public Builder setAfterColumns(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder builderForValue) {
+          int index, Column.Builder builderForValue) {
         if (afterColumnsBuilder_ == null) {
           ensureAfterColumnsIsMutable();
           afterColumns_.set(index, builderForValue.build());
@@ -7369,13 +7137,13 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
-      public Builder addAfterColumns(com.alibaba.otter.canal.protocol.CanalEntry.Column value) {
+      public Builder addAfterColumns(Column value) {
         if (afterColumnsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -7389,14 +7157,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
       public Builder addAfterColumns(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column value) {
+          int index, Column value) {
         if (afterColumnsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -7410,14 +7178,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
       public Builder addAfterColumns(
-          com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder builderForValue) {
+          Column.Builder builderForValue) {
         if (afterColumnsBuilder_ == null) {
           ensureAfterColumnsIsMutable();
           afterColumns_.add(builderForValue.build());
@@ -7428,14 +7196,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
       public Builder addAfterColumns(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder builderForValue) {
+          int index, Column.Builder builderForValue) {
         if (afterColumnsBuilder_ == null) {
           ensureAfterColumnsIsMutable();
           afterColumns_.add(index, builderForValue.build());
@@ -7446,14 +7214,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
       public Builder addAllAfterColumns(
-          java.lang.Iterable values) {
+          Iterable values) {
         if (afterColumnsBuilder_ == null) {
           ensureAfterColumnsIsMutable();
           com.google.protobuf.AbstractMessageLite.Builder.addAll(
@@ -7465,11 +7233,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
       public Builder clearAfterColumns() {
         if (afterColumnsBuilder_ == null) {
@@ -7482,11 +7250,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
       public Builder removeAfterColumns(int index) {
         if (afterColumnsBuilder_ == null) {
@@ -7499,24 +7267,24 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder getAfterColumnsBuilder(
+      public Column.Builder getAfterColumnsBuilder(
           int index) {
         return getAfterColumnsFieldBuilder().getBuilder(index);
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder getAfterColumnsOrBuilder(
+      public ColumnOrBuilder getAfterColumnsOrBuilder(
           int index) {
         if (afterColumnsBuilder_ == null) {
           return afterColumns_.get(index);  } else {
@@ -7524,13 +7292,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
-      public java.util.List 
+      public java.util.List
            getAfterColumnsOrBuilderList() {
         if (afterColumnsBuilder_ != null) {
           return afterColumnsBuilder_.getMessageOrBuilderList();
@@ -7539,45 +7307,45 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder addAfterColumnsBuilder() {
+      public Column.Builder addAfterColumnsBuilder() {
         return getAfterColumnsFieldBuilder().addBuilder(
-            com.alibaba.otter.canal.protocol.CanalEntry.Column.getDefaultInstance());
+            Column.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder addAfterColumnsBuilder(
+      public Column.Builder addAfterColumnsBuilder(
           int index) {
         return getAfterColumnsFieldBuilder().addBuilder(
-            index, com.alibaba.otter.canal.protocol.CanalEntry.Column.getDefaultInstance());
+            index, Column.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+       *
        * 
              ** 字段信息，增量数据(修改后,新增后)  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
        */
-      public java.util.List 
+      public java.util.List
            getAfterColumnsBuilderList() {
         return getAfterColumnsFieldBuilder().getBuilderList();
       }
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Column, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder, com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder> 
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Column, Column.Builder, ColumnOrBuilder>
           getAfterColumnsFieldBuilder() {
         if (afterColumnsBuilder_ == null) {
-          afterColumnsBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3<
-              com.alibaba.otter.canal.protocol.CanalEntry.Column, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder, com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder>(
+          afterColumnsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              Column, Column.Builder, ColumnOrBuilder>(
                   afterColumns_,
                   ((bitField0_ & 0x00000002) == 0x00000002),
                   getParentForChildren(),
@@ -7587,26 +7355,26 @@ public final class CanalEntry {
         return afterColumnsBuilder_;
       }
 
-      private java.util.List props_ =
+      private java.util.List props_ =
         java.util.Collections.emptyList();
       private void ensurePropsIsMutable() {
         if (!((bitField0_ & 0x00000004) == 0x00000004)) {
-          props_ = new java.util.ArrayList(props_);
+          props_ = new java.util.ArrayList(props_);
           bitField0_ |= 0x00000004;
          }
       }
 
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> propsBuilder_;
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Pair, Pair.Builder, PairOrBuilder> propsBuilder_;
 
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public java.util.List getPropsList() {
+      public java.util.List getPropsList() {
         if (propsBuilder_ == null) {
           return java.util.Collections.unmodifiableList(props_);
         } else {
@@ -7614,11 +7382,11 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public int getPropsCount() {
         if (propsBuilder_ == null) {
@@ -7628,13 +7396,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+      public Pair getProps(int index) {
         if (propsBuilder_ == null) {
           return props_.get(index);
         } else {
@@ -7642,14 +7410,14 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder setProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+          int index, Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -7663,14 +7431,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder setProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          int index, Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.set(index, builderForValue.build());
@@ -7681,13 +7449,13 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public Builder addProps(com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+      public Builder addProps(Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -7701,14 +7469,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder addProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+          int index, Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -7722,14 +7490,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder addProps(
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.add(builderForValue.build());
@@ -7740,14 +7508,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder addProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          int index, Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.add(index, builderForValue.build());
@@ -7758,14 +7526,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder addAllProps(
-          java.lang.Iterable values) {
+          Iterable values) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           com.google.protobuf.AbstractMessageLite.Builder.addAll(
@@ -7777,11 +7545,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder clearProps() {
         if (propsBuilder_ == null) {
@@ -7794,11 +7562,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder removeProps(int index) {
         if (propsBuilder_ == null) {
@@ -7811,24 +7579,24 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder getPropsBuilder(
+      public Pair.Builder getPropsBuilder(
           int index) {
         return getPropsFieldBuilder().getBuilder(index);
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+      public PairOrBuilder getPropsOrBuilder(
           int index) {
         if (propsBuilder_ == null) {
           return props_.get(index);  } else {
@@ -7836,13 +7604,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public java.util.List 
+      public java.util.List
            getPropsOrBuilderList() {
         if (propsBuilder_ != null) {
           return propsBuilder_.getMessageOrBuilderList();
@@ -7851,45 +7619,45 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder() {
+      public Pair.Builder addPropsBuilder() {
         return getPropsFieldBuilder().addBuilder(
-            com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+            Pair.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder(
+      public Pair.Builder addPropsBuilder(
           int index) {
         return getPropsFieldBuilder().addBuilder(
-            index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+            index, Pair.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public java.util.List 
+      public java.util.List
            getPropsBuilderList() {
         return getPropsFieldBuilder().getBuilderList();
       }
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> 
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Pair, Pair.Builder, PairOrBuilder>
           getPropsFieldBuilder() {
         if (propsBuilder_ == null) {
-          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3<
-              com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder>(
+          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              Pair, Pair.Builder, PairOrBuilder>(
                   props_,
                   ((bitField0_ & 0x00000004) == 0x00000004),
                   getParentForChildren(),
@@ -7898,57 +7666,16 @@ public final class CanalEntry {
         }
         return propsBuilder_;
       }
-      @java.lang.Override
-      public final Builder setUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.setUnknownFieldsProto3(unknownFields);
-      }
-
-      @java.lang.Override
-      public final Builder mergeUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.mergeUnknownFields(unknownFields);
-      }
-
 
       // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.RowData)
     }
 
-    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.RowData)
-    private static final com.alibaba.otter.canal.protocol.CanalEntry.RowData DEFAULT_INSTANCE;
     static {
-      DEFAULT_INSTANCE = new com.alibaba.otter.canal.protocol.CanalEntry.RowData();
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData getDefaultInstance() {
-      return DEFAULT_INSTANCE;
-    }
-
-    private static final com.google.protobuf.Parser
-        PARSER = new com.google.protobuf.AbstractParser() {
-      @java.lang.Override
-      public RowData parsePartialFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return new RowData(input, extensionRegistry);
-      }
-    };
-
-    public static com.google.protobuf.Parser parser() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.google.protobuf.Parser getParserForType() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.alibaba.otter.canal.protocol.CanalEntry.RowData getDefaultInstanceForType() {
-      return DEFAULT_INSTANCE;
+      defaultInstance = new RowData(true);
+      defaultInstance.initFields();
     }
 
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.RowData)
   }
 
   public interface RowChangeOrBuilder extends
@@ -7956,193 +7683,232 @@ public final class CanalEntry {
       com.google.protobuf.MessageOrBuilder {
 
     /**
+     * optional int64 tableId = 1;
+     *
      * 
            **tableId,由数据库产生*
      * 
      
+     */
+    boolean hasTableId();
+    /**
+     * optional int64 tableId = 1;
      *
-     * int64 tableId = 1;
+     * 
      +     **tableId,由数据库产生*
+     * 
      
      */
     long getTableId();
 
     /**
-     * .com.alibaba.otter.canal.protocol.EventType eventType = 2;
+     * optional .com.alibaba.otter.canal.protocol.EventType eventType = 2 [default = UPDATE];
+     *
+     * 
      +     **数据变更类型*
+     * 
      
      */
-    int getEventTypeValue();
+    boolean hasEventType();
     /**
-     * .com.alibaba.otter.canal.protocol.EventType eventType = 2;
+     * optional .com.alibaba.otter.canal.protocol.EventType eventType = 2 [default = UPDATE];
+     *
+     * 
      +     **数据变更类型*
+     * 
      
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.EventType getEventType();
+    EventType getEventType();
 
     /**
-     * bool isDdl = 10;
+     * optional bool isDdl = 10 [default = false];
+     *
+     * 
      +     ** 标识是否是ddl语句  *
+     * 
      
+     */
+    boolean hasIsDdl();
+    /**
+     * optional bool isDdl = 10 [default = false];
+     *
+     * 
      +     ** 标识是否是ddl语句  *
+     * 
      
      */
     boolean getIsDdl();
 
     /**
+     * optional string sql = 11;
+     *
      * 
            ** ddl/query的sql语句  *
      * 
      
-     *
-     * string sql = 11;
      */
-    java.lang.String getSql();
+    boolean hasSql();
     /**
+     * optional string sql = 11;
+     *
      * 
            ** ddl/query的sql语句  *
      * 
      
+     */
+    String getSql();
+    /**
+     * optional string sql = 11;
      *
-     * string sql = 11;
+     * 
      +     ** ddl/query的sql语句  *
+     * 
      
      */
     com.google.protobuf.ByteString
         getSqlBytes();
 
     /**
+     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+     *
      * 
            ** 一次数据库变更可能存在多行  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
      */
-    java.util.List 
+    java.util.List
         getRowDatasList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+     *
      * 
            ** 一次数据库变更可能存在多行  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.RowData getRowDatas(int index);
+    RowData getRowDatas(int index);
     /**
+     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+     *
      * 
            ** 一次数据库变更可能存在多行  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
      */
     int getRowDatasCount();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+     *
      * 
            ** 一次数据库变更可能存在多行  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
      */
-    java.util.List 
+    java.util.List
         getRowDatasOrBuilderList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+     *
      * 
            ** 一次数据库变更可能存在多行  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder getRowDatasOrBuilder(
-        int index);
+    RowDataOrBuilder getRowDatasOrBuilder(int index);
 
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
      */
-    java.util.List 
+    java.util.List
         getPropsList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index);
+    Pair getProps(int index);
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
      */
     int getPropsCount();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
      */
-    java.util.List 
+    java.util.List
         getPropsOrBuilderList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
-        int index);
+    PairOrBuilder getPropsOrBuilder(int index);
 
     /**
+     * optional string ddlSchemaName = 14;
+     *
      * 
            ** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  *
      * 
      
-     *
-     * string ddlSchemaName = 14;
      */
-    java.lang.String getDdlSchemaName();
+    boolean hasDdlSchemaName();
     /**
+     * optional string ddlSchemaName = 14;
+     *
      * 
            ** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  *
      * 
      
+     */
+    String getDdlSchemaName();
+    /**
+     * optional string ddlSchemaName = 14;
      *
-     * string ddlSchemaName = 14;
+     * 
      +     ** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  *
+     * 
      
      */
     com.google.protobuf.ByteString
         getDdlSchemaNameBytes();
-
-    public com.alibaba.otter.canal.protocol.CanalEntry.RowChange.EventTypePresentCase getEventTypePresentCase();
-
-    public com.alibaba.otter.canal.protocol.CanalEntry.RowChange.IsDdlPresentCase getIsDdlPresentCase();
   }
   /**
+   * Protobuf type {@code com.alibaba.otter.canal.protocol.RowChange}
+   *
    * 
          **message row 每行变更数据的数据结构*
    * 
      
-   *
-   * Protobuf type {@code com.alibaba.otter.canal.protocol.RowChange}
    */
-  public  static final class RowChange extends
-      com.google.protobuf.GeneratedMessageV3 implements
+  public static final class RowChange extends
+      com.google.protobuf.GeneratedMessage implements
       // @@protoc_insertion_point(message_implements:com.alibaba.otter.canal.protocol.RowChange)
       RowChangeOrBuilder {
-  private static final long serialVersionUID = 0L;
     // Use RowChange.newBuilder() to construct.
-    private RowChange(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+    private RowChange(com.google.protobuf.GeneratedMessage.Builder builder) {
       super(builder);
+      this.unknownFields = builder.getUnknownFields();
     }
-    private RowChange() {
-      tableId_ = 0L;
-      sql_ = "";
-      rowDatas_ = java.util.Collections.emptyList();
-      props_ = java.util.Collections.emptyList();
-      ddlSchemaName_ = "";
+    private RowChange(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+    private static final RowChange defaultInstance;
+    public static RowChange getDefaultInstance() {
+      return defaultInstance;
     }
 
-    @java.lang.Override
+    public RowChange getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+
+    private final com.google.protobuf.UnknownFieldSet unknownFields;
+    @Override
     public final com.google.protobuf.UnknownFieldSet
-    getUnknownFields() {
+        getUnknownFields() {
       return this.unknownFields;
     }
     private RowChange(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
-      this();
-      if (extensionRegistry == null) {
-        throw new java.lang.NullPointerException();
-      }
+      initFields();
       int mutable_bitField0_ = 0;
       com.google.protobuf.UnknownFieldSet.Builder unknownFields =
           com.google.protobuf.UnknownFieldSet.newBuilder();
@@ -8154,57 +7920,60 @@ public final class CanalEntry {
             case 0:
               done = true;
               break;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                done = true;
+              }
+              break;
+            }
             case 8: {
-
+              bitField0_ |= 0x00000001;
               tableId_ = input.readInt64();
               break;
             }
             case 16: {
               int rawValue = input.readEnum();
-              eventTypePresentCase_ = 2;
-              eventTypePresent_ = rawValue;
+              EventType value = EventType.valueOf(rawValue);
+              if (value == null) {
+                unknownFields.mergeVarintField(2, rawValue);
+              } else {
+                bitField0_ |= 0x00000002;
+                eventType_ = value;
+              }
               break;
             }
             case 80: {
-              isDdlPresentCase_ = 10;
-              isDdlPresent_ = input.readBool();
+              bitField0_ |= 0x00000004;
+              isDdl_ = input.readBool();
               break;
             }
             case 90: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              sql_ = s;
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000008;
+              sql_ = bs;
               break;
             }
             case 98: {
               if (!((mutable_bitField0_ & 0x00000010) == 0x00000010)) {
-                rowDatas_ = new java.util.ArrayList();
+                rowDatas_ = new java.util.ArrayList();
                 mutable_bitField0_ |= 0x00000010;
               }
-              rowDatas_.add(
-                  input.readMessage(com.alibaba.otter.canal.protocol.CanalEntry.RowData.parser(), extensionRegistry));
+              rowDatas_.add(input.readMessage(RowData.PARSER, extensionRegistry));
               break;
             }
             case 106: {
               if (!((mutable_bitField0_ & 0x00000020) == 0x00000020)) {
-                props_ = new java.util.ArrayList();
+                props_ = new java.util.ArrayList();
                 mutable_bitField0_ |= 0x00000020;
               }
-              props_.add(
-                  input.readMessage(com.alibaba.otter.canal.protocol.CanalEntry.Pair.parser(), extensionRegistry));
+              props_.add(input.readMessage(Pair.PARSER, extensionRegistry));
               break;
             }
             case 114: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              ddlSchemaName_ = s;
-              break;
-            }
-            default: {
-              if (!parseUnknownFieldProto3(
-                  input, unknownFields, extensionRegistry, tag)) {
-                done = true;
-              }
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000010;
+              ddlSchemaName_ = bs;
               break;
             }
           }
@@ -8213,7 +7982,7 @@ public final class CanalEntry {
         throw e.setUnfinishedMessage(this);
       } catch (java.io.IOException e) {
         throw new com.google.protobuf.InvalidProtocolBufferException(
-            e).setUnfinishedMessage(this);
+            e.getMessage()).setUnfinishedMessage(this);
       } finally {
         if (((mutable_bitField0_ & 0x00000010) == 0x00000010)) {
           rowDatas_ = java.util.Collections.unmodifiableList(rowDatas_);
@@ -8227,172 +7996,148 @@ public final class CanalEntry {
     }
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor;
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor;
     }
 
-    @java.lang.Override
-    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+    protected FieldAccessorTable
         internalGetFieldAccessorTable() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_fieldAccessorTable
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_fieldAccessorTable
           .ensureFieldAccessorsInitialized(
-              com.alibaba.otter.canal.protocol.CanalEntry.RowChange.class, com.alibaba.otter.canal.protocol.CanalEntry.RowChange.Builder.class);
+              RowChange.class, Builder.class);
+    }
+
+    public static com.google.protobuf.Parser PARSER =
+        new com.google.protobuf.AbstractParser() {
+      public RowChange parsePartialFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return new RowChange(input, extensionRegistry);
+      }
+    };
+
+    @Override
+    public com.google.protobuf.Parser getParserForType() {
+      return PARSER;
     }
 
     private int bitField0_;
-    private int eventTypePresentCase_ = 0;
-    private java.lang.Object eventTypePresent_;
-    public enum EventTypePresentCase
-        implements com.google.protobuf.Internal.EnumLite {
-      EVENTTYPE(2),
-      EVENTTYPEPRESENT_NOT_SET(0);
-      private final int value;
-      private EventTypePresentCase(int value) {
-        this.value = value;
-      }
-      /**
-       * @deprecated Use {@link #forNumber(int)} instead.
-       */
-      @java.lang.Deprecated
-      public static EventTypePresentCase valueOf(int value) {
-        return forNumber(value);
-      }
-
-      public static EventTypePresentCase forNumber(int value) {
-        switch (value) {
-          case 2: return EVENTTYPE;
-          case 0: return EVENTTYPEPRESENT_NOT_SET;
-          default: return null;
-        }
-      }
-      public int getNumber() {
-        return this.value;
-      }
-    };
-
-    public EventTypePresentCase
-    getEventTypePresentCase() {
-      return EventTypePresentCase.forNumber(
-          eventTypePresentCase_);
-    }
-
-    private int isDdlPresentCase_ = 0;
-    private java.lang.Object isDdlPresent_;
-    public enum IsDdlPresentCase
-        implements com.google.protobuf.Internal.EnumLite {
-      ISDDL(10),
-      ISDDLPRESENT_NOT_SET(0);
-      private final int value;
-      private IsDdlPresentCase(int value) {
-        this.value = value;
-      }
-      /**
-       * @deprecated Use {@link #forNumber(int)} instead.
-       */
-      @java.lang.Deprecated
-      public static IsDdlPresentCase valueOf(int value) {
-        return forNumber(value);
-      }
-
-      public static IsDdlPresentCase forNumber(int value) {
-        switch (value) {
-          case 10: return ISDDL;
-          case 0: return ISDDLPRESENT_NOT_SET;
-          default: return null;
-        }
-      }
-      public int getNumber() {
-        return this.value;
-      }
-    };
-
-    public IsDdlPresentCase
-    getIsDdlPresentCase() {
-      return IsDdlPresentCase.forNumber(
-          isDdlPresentCase_);
-    }
-
     public static final int TABLEID_FIELD_NUMBER = 1;
     private long tableId_;
     /**
+     * optional int64 tableId = 1;
+     *
      * 
            **tableId,由数据库产生*
      * 
      
+     */
+    public boolean hasTableId() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    /**
+     * optional int64 tableId = 1;
      *
-     * int64 tableId = 1;
+     * 
      +     **tableId,由数据库产生*
+     * 
      
      */
     public long getTableId() {
       return tableId_;
     }
 
     public static final int EVENTTYPE_FIELD_NUMBER = 2;
+    private EventType eventType_;
     /**
-     * .com.alibaba.otter.canal.protocol.EventType eventType = 2;
+     * optional .com.alibaba.otter.canal.protocol.EventType eventType = 2 [default = UPDATE];
+     *
+     * 
      +     **数据变更类型*
+     * 
      
      */
-    public int getEventTypeValue() {
-      if (eventTypePresentCase_ == 2) {
-        return (java.lang.Integer) eventTypePresent_;
-      }
-      return 0;
+    public boolean hasEventType() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
     }
     /**
-     * .com.alibaba.otter.canal.protocol.EventType eventType = 2;
+     * optional .com.alibaba.otter.canal.protocol.EventType eventType = 2 [default = UPDATE];
+     *
+     * 
      +     **数据变更类型*
+     * 
      
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.EventType getEventType() {
-      if (eventTypePresentCase_ == 2) {
-        @SuppressWarnings("deprecation")
-        com.alibaba.otter.canal.protocol.CanalEntry.EventType result = com.alibaba.otter.canal.protocol.CanalEntry.EventType.valueOf(
-            (java.lang.Integer) eventTypePresent_);
-        return result == null ? com.alibaba.otter.canal.protocol.CanalEntry.EventType.UNRECOGNIZED : result;
-      }
-      return EventType.UPDATE;
+    public EventType getEventType() {
+      return eventType_;
     }
 
     public static final int ISDDL_FIELD_NUMBER = 10;
+    private boolean isDdl_;
     /**
-     * bool isDdl = 10;
+     * optional bool isDdl = 10 [default = false];
+     *
+     * 
      +     ** 标识是否是ddl语句  *
+     * 
      
+     */
+    public boolean hasIsDdl() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    /**
+     * optional bool isDdl = 10 [default = false];
+     *
+     * 
      +     ** 标识是否是ddl语句  *
+     * 
      
      */
     public boolean getIsDdl() {
-      if (isDdlPresentCase_ == 10) {
-        return (java.lang.Boolean) isDdlPresent_;
-      }
-      return false;
+      return isDdl_;
     }
 
     public static final int SQL_FIELD_NUMBER = 11;
-    private volatile java.lang.Object sql_;
+    private Object sql_;
     /**
+     * optional string sql = 11;
+     *
      * 
            ** ddl/query的sql语句  *
      * 
      
-     *
-     * string sql = 11;
      */
-    public java.lang.String getSql() {
-      java.lang.Object ref = sql_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasSql() {
+      return ((bitField0_ & 0x00000008) == 0x00000008);
+    }
+    /**
+     * optional string sql = 11;
+     *
+     * 
      +     ** ddl/query的sql语句  *
+     * 
      
+     */
+    public String getSql() {
+      Object ref = sql_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        sql_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          sql_ = s;
+        }
         return s;
       }
     }
     /**
+     * optional string sql = 11;
+     *
      * 
            ** ddl/query的sql语句  *
      * 
      
-     *
-     * string sql = 11;
      */
     public com.google.protobuf.ByteString
         getSqlBytes() {
-      java.lang.Object ref = sql_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = sql_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         sql_ = b;
         return b;
       } else {
@@ -8401,150 +8146,162 @@ public final class CanalEntry {
     }
 
     public static final int ROWDATAS_FIELD_NUMBER = 12;
-    private java.util.List rowDatas_;
+    private java.util.List rowDatas_;
     /**
+     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+     *
      * 
            ** 一次数据库变更可能存在多行  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
      */
-    public java.util.List getRowDatasList() {
+    public java.util.List getRowDatasList() {
       return rowDatas_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+     *
      * 
            ** 一次数据库变更可能存在多行  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
      */
-    public java.util.List 
+    public java.util.List
         getRowDatasOrBuilderList() {
       return rowDatas_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+     *
      * 
            ** 一次数据库变更可能存在多行  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
      */
     public int getRowDatasCount() {
       return rowDatas_.size();
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+     *
      * 
            ** 一次数据库变更可能存在多行  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.RowData getRowDatas(int index) {
+    public RowData getRowDatas(int index) {
       return rowDatas_.get(index);
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+     *
      * 
            ** 一次数据库变更可能存在多行  *
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder getRowDatasOrBuilder(
+    public RowDataOrBuilder getRowDatasOrBuilder(
         int index) {
       return rowDatas_.get(index);
     }
 
     public static final int PROPS_FIELD_NUMBER = 13;
-    private java.util.List props_;
+    private java.util.List props_;
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
      */
-    public java.util.List getPropsList() {
+    public java.util.List getPropsList() {
       return props_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
      */
-    public java.util.List 
+    public java.util.List
         getPropsOrBuilderList() {
       return props_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
      */
     public int getPropsCount() {
       return props_.size();
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+    public Pair getProps(int index) {
       return props_.get(index);
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+    public PairOrBuilder getPropsOrBuilder(
         int index) {
       return props_.get(index);
     }
 
     public static final int DDLSCHEMANAME_FIELD_NUMBER = 14;
-    private volatile java.lang.Object ddlSchemaName_;
+    private Object ddlSchemaName_;
     /**
+     * optional string ddlSchemaName = 14;
+     *
      * 
            ** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  *
      * 
      
-     *
-     * string ddlSchemaName = 14;
      */
-    public java.lang.String getDdlSchemaName() {
-      java.lang.Object ref = ddlSchemaName_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasDdlSchemaName() {
+      return ((bitField0_ & 0x00000010) == 0x00000010);
+    }
+    /**
+     * optional string ddlSchemaName = 14;
+     *
+     * 
      +     ** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  *
+     * 
      
+     */
+    public String getDdlSchemaName() {
+      Object ref = ddlSchemaName_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        ddlSchemaName_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          ddlSchemaName_ = s;
+        }
         return s;
       }
     }
     /**
+     * optional string ddlSchemaName = 14;
+     *
      * 
            ** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  *
      * 
      
-     *
-     * string ddlSchemaName = 14;
      */
     public com.google.protobuf.ByteString
         getDdlSchemaNameBytes() {
-      java.lang.Object ref = ddlSchemaName_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = ddlSchemaName_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         ddlSchemaName_ = b;
         return b;
       } else {
@@ -8552,8 +8309,16 @@ public final class CanalEntry {
       }
     }
 
+    private void initFields() {
+      tableId_ = 0L;
+      eventType_ = EventType.UPDATE;
+      isDdl_ = false;
+      sql_ = "";
+      rowDatas_ = java.util.Collections.emptyList();
+      props_ = java.util.Collections.emptyList();
+      ddlSchemaName_ = "";
+    }
     private byte memoizedIsInitialized = -1;
-    @java.lang.Override
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized == 1) return true;
@@ -8563,21 +8328,20 @@ public final class CanalEntry {
       return true;
     }
 
-    @java.lang.Override
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
-      if (tableId_ != 0L) {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
         output.writeInt64(1, tableId_);
       }
-      if (eventTypePresentCase_ == 2) {
-        output.writeEnum(2, ((java.lang.Integer) eventTypePresent_));
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeEnum(2, eventType_.getNumber());
       }
-      if (isDdlPresentCase_ == 10) {
-        output.writeBool(
-            10, (boolean)((java.lang.Boolean) isDdlPresent_));
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeBool(10, isDdl_);
       }
-      if (!getSqlBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 11, sql_);
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        output.writeBytes(11, getSqlBytes());
       }
       for (int i = 0; i < rowDatas_.size(); i++) {
         output.writeMessage(12, rowDatas_.get(i));
@@ -8585,33 +8349,33 @@ public final class CanalEntry {
       for (int i = 0; i < props_.size(); i++) {
         output.writeMessage(13, props_.get(i));
       }
-      if (!getDdlSchemaNameBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 14, ddlSchemaName_);
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        output.writeBytes(14, getDdlSchemaNameBytes());
       }
-      unknownFields.writeTo(output);
+      getUnknownFields().writeTo(output);
     }
 
-    @java.lang.Override
+    private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
-      int size = memoizedSize;
+      int size = memoizedSerializedSize;
       if (size != -1) return size;
 
       size = 0;
-      if (tableId_ != 0L) {
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
           .computeInt64Size(1, tableId_);
       }
-      if (eventTypePresentCase_ == 2) {
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
         size += com.google.protobuf.CodedOutputStream
-          .computeEnumSize(2, ((java.lang.Integer) eventTypePresent_));
+          .computeEnumSize(2, eventType_.getNumber());
       }
-      if (isDdlPresentCase_ == 10) {
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
         size += com.google.protobuf.CodedOutputStream
-          .computeBoolSize(
-              10, (boolean)((java.lang.Boolean) isDdlPresent_));
+          .computeBoolSize(10, isDdl_);
       }
-      if (!getSqlBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(11, sql_);
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(11, getSqlBytes());
       }
       for (int i = 0; i < rowDatas_.size(); i++) {
         size += com.google.protobuf.CodedOutputStream
@@ -8621,217 +8385,109 @@ public final class CanalEntry {
         size += com.google.protobuf.CodedOutputStream
           .computeMessageSize(13, props_.get(i));
       }
-      if (!getDdlSchemaNameBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(14, ddlSchemaName_);
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(14, getDdlSchemaNameBytes());
       }
-      size += unknownFields.getSerializedSize();
-      memoizedSize = size;
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
       return size;
     }
 
-    @java.lang.Override
-    public boolean equals(final java.lang.Object obj) {
-      if (obj == this) {
-       return true;
-      }
-      if (!(obj instanceof com.alibaba.otter.canal.protocol.CanalEntry.RowChange)) {
-        return super.equals(obj);
-      }
-      com.alibaba.otter.canal.protocol.CanalEntry.RowChange other = (com.alibaba.otter.canal.protocol.CanalEntry.RowChange) obj;
-
-      boolean result = true;
-      result = result && (getTableId()
-          == other.getTableId());
-      result = result && getSql()
-          .equals(other.getSql());
-      result = result && getRowDatasList()
-          .equals(other.getRowDatasList());
-      result = result && getPropsList()
-          .equals(other.getPropsList());
-      result = result && getDdlSchemaName()
-          .equals(other.getDdlSchemaName());
-      result = result && getEventTypePresentCase().equals(
-          other.getEventTypePresentCase());
-      if (!result) return false;
-      switch (eventTypePresentCase_) {
-        case 2:
-          result = result && getEventTypeValue()
-              == other.getEventTypeValue();
-          break;
-        case 0:
-        default:
-      }
-      result = result && getIsDdlPresentCase().equals(
-          other.getIsDdlPresentCase());
-      if (!result) return false;
-      switch (isDdlPresentCase_) {
-        case 10:
-          result = result && (getIsDdl()
-              == other.getIsDdl());
-          break;
-        case 0:
-        default:
-      }
-      result = result && unknownFields.equals(other.unknownFields);
-      return result;
+    private static final long serialVersionUID = 0L;
+    @Override
+    protected Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
     }
 
-    @java.lang.Override
-    public int hashCode() {
-      if (memoizedHashCode != 0) {
-        return memoizedHashCode;
-      }
-      int hash = 41;
-      hash = (19 * hash) + getDescriptor().hashCode();
-      hash = (37 * hash) + TABLEID_FIELD_NUMBER;
-      hash = (53 * hash) + com.google.protobuf.Internal.hashLong(
-          getTableId());
-      hash = (37 * hash) + SQL_FIELD_NUMBER;
-      hash = (53 * hash) + getSql().hashCode();
-      if (getRowDatasCount() > 0) {
-        hash = (37 * hash) + ROWDATAS_FIELD_NUMBER;
-        hash = (53 * hash) + getRowDatasList().hashCode();
-      }
-      if (getPropsCount() > 0) {
-        hash = (37 * hash) + PROPS_FIELD_NUMBER;
-        hash = (53 * hash) + getPropsList().hashCode();
-      }
-      hash = (37 * hash) + DDLSCHEMANAME_FIELD_NUMBER;
-      hash = (53 * hash) + getDdlSchemaName().hashCode();
-      switch (eventTypePresentCase_) {
-        case 2:
-          hash = (37 * hash) + EVENTTYPE_FIELD_NUMBER;
-          hash = (53 * hash) + getEventTypeValue();
-          break;
-        case 0:
-        default:
-      }
-      switch (isDdlPresentCase_) {
-        case 10:
-          hash = (37 * hash) + ISDDL_FIELD_NUMBER;
-          hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(
-              getIsDdl());
-          break;
-        case 0:
-        default:
-      }
-      hash = (29 * hash) + unknownFields.hashCode();
-      memoizedHashCode = hash;
-      return hash;
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
-        java.nio.ByteBuffer data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
-        java.nio.ByteBuffer data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
+    public static RowChange parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
+    public static RowChange parseFrom(
         com.google.protobuf.ByteString data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(byte[] data)
+    public static RowChange parseFrom(byte[] data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
+    public static RowChange parseFrom(
         byte[] data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(java.io.InputStream input)
+    public static RowChange parseFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
+    public static RowChange parseFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseDelimitedFrom(java.io.InputStream input)
+    public static RowChange parseDelimitedFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input);
+      return PARSER.parseDelimitedFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseDelimitedFrom(
+    public static RowChange parseDelimitedFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseDelimitedFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
+    public static RowChange parseFrom(
         com.google.protobuf.CodedInputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
+    public static RowChange parseFrom(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
 
-    @java.lang.Override
+    public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
-    public static Builder newBuilder() {
-      return DEFAULT_INSTANCE.toBuilder();
-    }
-    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.RowChange prototype) {
-      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
-    }
-    @java.lang.Override
-    public Builder toBuilder() {
-      return this == DEFAULT_INSTANCE
-          ? new Builder() : new Builder().mergeFrom(this);
+    public static Builder newBuilder(RowChange prototype) {
+      return newBuilder().mergeFrom(prototype);
     }
+    public Builder toBuilder() { return newBuilder(this); }
 
-    @java.lang.Override
+    @Override
     protected Builder newBuilderForType(
-        com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        BuilderParent parent) {
       Builder builder = new Builder(parent);
       return builder;
     }
     /**
+     * Protobuf type {@code com.alibaba.otter.canal.protocol.RowChange}
+     *
      * 
            **message row 每行变更数据的数据结构*
      * 
      
-     *
-     * Protobuf type {@code com.alibaba.otter.canal.protocol.RowChange}
      */
     public static final class Builder extends
-        com.google.protobuf.GeneratedMessageV3.Builder implements
+        com.google.protobuf.GeneratedMessage.Builder implements
         // @@protoc_insertion_point(builder_implements:com.alibaba.otter.canal.protocol.RowChange)
-        com.alibaba.otter.canal.protocol.CanalEntry.RowChangeOrBuilder {
+        RowChangeOrBuilder {
       public static final com.google.protobuf.Descriptors.Descriptor
           getDescriptor() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor;
       }
 
-      @java.lang.Override
-      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      protected FieldAccessorTable
           internalGetFieldAccessorTable() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_fieldAccessorTable
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_fieldAccessorTable
             .ensureFieldAccessorsInitialized(
-                com.alibaba.otter.canal.protocol.CanalEntry.RowChange.class, com.alibaba.otter.canal.protocol.CanalEntry.RowChange.Builder.class);
+                RowChange.class, Builder.class);
       }
 
       // Construct using com.alibaba.otter.canal.protocol.CanalEntry.RowChange.newBuilder()
@@ -8840,24 +8496,30 @@ public final class CanalEntry {
       }
 
       private Builder(
-          com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
       private void maybeForceBuilderInitialization() {
-        if (com.google.protobuf.GeneratedMessageV3
-                .alwaysUseFieldBuilders) {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
           getRowDatasFieldBuilder();
           getPropsFieldBuilder();
         }
       }
-      @java.lang.Override
+      private static Builder create() {
+        return new Builder();
+      }
+
       public Builder clear() {
         super.clear();
         tableId_ = 0L;
-
+        bitField0_ = (bitField0_ & ~0x00000001);
+        eventType_ = EventType.UPDATE;
+        bitField0_ = (bitField0_ & ~0x00000002);
+        isDdl_ = false;
+        bitField0_ = (bitField0_ & ~0x00000004);
         sql_ = "";
-
+        bitField0_ = (bitField0_ & ~0x00000008);
         if (rowDatasBuilder_ == null) {
           rowDatas_ = java.util.Collections.emptyList();
           bitField0_ = (bitField0_ & ~0x00000010);
@@ -8871,45 +8533,49 @@ public final class CanalEntry {
           propsBuilder_.clear();
         }
         ddlSchemaName_ = "";
-
-        eventTypePresentCase_ = 0;
-        eventTypePresent_ = null;
-        isDdlPresentCase_ = 0;
-        isDdlPresent_ = null;
+        bitField0_ = (bitField0_ & ~0x00000040);
         return this;
       }
 
-      @java.lang.Override
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.RowChange getDefaultInstanceForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.RowChange.getDefaultInstance();
+      public RowChange getDefaultInstanceForType() {
+        return RowChange.getDefaultInstance();
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.RowChange build() {
-        com.alibaba.otter.canal.protocol.CanalEntry.RowChange result = buildPartial();
+      public RowChange build() {
+        RowChange result = buildPartial();
         if (!result.isInitialized()) {
           throw newUninitializedMessageException(result);
         }
         return result;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.RowChange buildPartial() {
-        com.alibaba.otter.canal.protocol.CanalEntry.RowChange result = new com.alibaba.otter.canal.protocol.CanalEntry.RowChange(this);
+      public RowChange buildPartial() {
+        RowChange result = new RowChange(this);
         int from_bitField0_ = bitField0_;
         int to_bitField0_ = 0;
-        result.tableId_ = tableId_;
-        if (eventTypePresentCase_ == 2) {
-          result.eventTypePresent_ = eventTypePresent_;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
         }
-        if (isDdlPresentCase_ == 10) {
-          result.isDdlPresent_ = isDdlPresent_;
+        result.tableId_ = tableId_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.eventType_ = eventType_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.isDdl_ = isDdl_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000008;
         }
         result.sql_ = sql_;
         if (rowDatasBuilder_ == null) {
@@ -8930,62 +8596,37 @@ public final class CanalEntry {
         } else {
           result.props_ = propsBuilder_.build();
         }
+        if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
+          to_bitField0_ |= 0x00000010;
+        }
         result.ddlSchemaName_ = ddlSchemaName_;
         result.bitField0_ = to_bitField0_;
-        result.eventTypePresentCase_ = eventTypePresentCase_;
-        result.isDdlPresentCase_ = isDdlPresentCase_;
         onBuilt();
         return result;
       }
 
-      @java.lang.Override
-      public Builder clone() {
-        return (Builder) super.clone();
-      }
-      @java.lang.Override
-      public Builder setField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.setField(field, value);
-      }
-      @java.lang.Override
-      public Builder clearField(
-          com.google.protobuf.Descriptors.FieldDescriptor field) {
-        return (Builder) super.clearField(field);
-      }
-      @java.lang.Override
-      public Builder clearOneof(
-          com.google.protobuf.Descriptors.OneofDescriptor oneof) {
-        return (Builder) super.clearOneof(oneof);
-      }
-      @java.lang.Override
-      public Builder setRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          int index, java.lang.Object value) {
-        return (Builder) super.setRepeatedField(field, index, value);
-      }
-      @java.lang.Override
-      public Builder addRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.addRepeatedField(field, value);
-      }
-      @java.lang.Override
       public Builder mergeFrom(com.google.protobuf.Message other) {
-        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.RowChange) {
-          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.RowChange)other);
+        if (other instanceof RowChange) {
+          return mergeFrom((RowChange)other);
         } else {
           super.mergeFrom(other);
           return this;
         }
       }
 
-      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.RowChange other) {
-        if (other == com.alibaba.otter.canal.protocol.CanalEntry.RowChange.getDefaultInstance()) return this;
-        if (other.getTableId() != 0L) {
+      public Builder mergeFrom(RowChange other) {
+        if (other == RowChange.getDefaultInstance()) return this;
+        if (other.hasTableId()) {
           setTableId(other.getTableId());
         }
-        if (!other.getSql().isEmpty()) {
+        if (other.hasEventType()) {
+          setEventType(other.getEventType());
+        }
+        if (other.hasIsDdl()) {
+          setIsDdl(other.getIsDdl());
+        }
+        if (other.hasSql()) {
+          bitField0_ |= 0x00000008;
           sql_ = other.sql_;
           onChanged();
         }
@@ -9007,8 +8648,8 @@ public final class CanalEntry {
               rowDatasBuilder_ = null;
               rowDatas_ = other.rowDatas_;
               bitField0_ = (bitField0_ & ~0x00000010);
-              rowDatasBuilder_ = 
-                com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ?
+              rowDatasBuilder_ =
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
                    getRowDatasFieldBuilder() : null;
             } else {
               rowDatasBuilder_.addAllMessages(other.rowDatas_);
@@ -9033,57 +8674,37 @@ public final class CanalEntry {
               propsBuilder_ = null;
               props_ = other.props_;
               bitField0_ = (bitField0_ & ~0x00000020);
-              propsBuilder_ = 
-                com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ?
+              propsBuilder_ =
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
                    getPropsFieldBuilder() : null;
             } else {
               propsBuilder_.addAllMessages(other.props_);
             }
           }
         }
-        if (!other.getDdlSchemaName().isEmpty()) {
+        if (other.hasDdlSchemaName()) {
+          bitField0_ |= 0x00000040;
           ddlSchemaName_ = other.ddlSchemaName_;
           onChanged();
         }
-        switch (other.getEventTypePresentCase()) {
-          case EVENTTYPE: {
-            setEventTypeValue(other.getEventTypeValue());
-            break;
-          }
-          case EVENTTYPEPRESENT_NOT_SET: {
-            break;
-          }
-        }
-        switch (other.getIsDdlPresentCase()) {
-          case ISDDL: {
-            setIsDdl(other.getIsDdl());
-            break;
-          }
-          case ISDDLPRESENT_NOT_SET: {
-            break;
-          }
-        }
-        this.mergeUnknownFields(other.unknownFields);
-        onChanged();
+        this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
 
-      @java.lang.Override
       public final boolean isInitialized() {
         return true;
       }
 
-      @java.lang.Override
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
           throws java.io.IOException {
-        com.alibaba.otter.canal.protocol.CanalEntry.RowChange parsedMessage = null;
+        RowChange parsedMessage = null;
         try {
           parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
         } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          parsedMessage = (com.alibaba.otter.canal.protocol.CanalEntry.RowChange) e.getUnfinishedMessage();
-          throw e.unwrapIOException();
+          parsedMessage = (RowChange) e.getUnfinishedMessage();
+          throw e;
         } finally {
           if (parsedMessage != null) {
             mergeFrom(parsedMessage);
@@ -9091,194 +8712,201 @@ public final class CanalEntry {
         }
         return this;
       }
-      private int eventTypePresentCase_ = 0;
-      private java.lang.Object eventTypePresent_;
-      public EventTypePresentCase
-          getEventTypePresentCase() {
-        return EventTypePresentCase.forNumber(
-            eventTypePresentCase_);
-      }
-
-      public Builder clearEventTypePresent() {
-        eventTypePresentCase_ = 0;
-        eventTypePresent_ = null;
-        onChanged();
-        return this;
-      }
-
-      private int isDdlPresentCase_ = 0;
-      private java.lang.Object isDdlPresent_;
-      public IsDdlPresentCase
-          getIsDdlPresentCase() {
-        return IsDdlPresentCase.forNumber(
-            isDdlPresentCase_);
-      }
-
-      public Builder clearIsDdlPresent() {
-        isDdlPresentCase_ = 0;
-        isDdlPresent_ = null;
-        onChanged();
-        return this;
-      }
-
       private int bitField0_;
 
       private long tableId_ ;
       /**
+       * optional int64 tableId = 1;
+       *
        * 
              **tableId,由数据库产生*
        * 
      
+       */
+      public boolean hasTableId() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      /**
+       * optional int64 tableId = 1;
        *
-       * int64 tableId = 1;
+       * 
      +       **tableId,由数据库产生*
+       * 
      
        */
       public long getTableId() {
         return tableId_;
       }
       /**
+       * optional int64 tableId = 1;
+       *
        * 
              **tableId,由数据库产生*
        * 
      
-       *
-       * int64 tableId = 1;
        */
       public Builder setTableId(long value) {
-        
+        bitField0_ |= 0x00000001;
         tableId_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional int64 tableId = 1;
+       *
        * 
              **tableId,由数据库产生*
        * 
      
-       *
-       * int64 tableId = 1;
        */
       public Builder clearTableId() {
-        
+        bitField0_ = (bitField0_ & ~0x00000001);
         tableId_ = 0L;
         onChanged();
         return this;
       }
 
+      private EventType eventType_ = EventType.UPDATE;
       /**
-       * .com.alibaba.otter.canal.protocol.EventType eventType = 2;
+       * optional .com.alibaba.otter.canal.protocol.EventType eventType = 2 [default = UPDATE];
+       *
+       * 
      +       **数据变更类型*
+       * 
      
        */
-      public int getEventTypeValue() {
-        if (eventTypePresentCase_ == 2) {
-          return ((java.lang.Integer) eventTypePresent_).intValue();
-        }
-        return 0;
+      public boolean hasEventType() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
       }
       /**
-       * .com.alibaba.otter.canal.protocol.EventType eventType = 2;
+       * optional .com.alibaba.otter.canal.protocol.EventType eventType = 2 [default = UPDATE];
+       *
+       * 
      +       **数据变更类型*
+       * 
      
        */
-      public Builder setEventTypeValue(int value) {
-        eventTypePresentCase_ = 2;
-        eventTypePresent_ = value;
-        onChanged();
-        return this;
+      public EventType getEventType() {
+        return eventType_;
       }
       /**
-       * .com.alibaba.otter.canal.protocol.EventType eventType = 2;
+       * optional .com.alibaba.otter.canal.protocol.EventType eventType = 2 [default = UPDATE];
+       *
+       * 
      +       **数据变更类型*
+       * 
      
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.EventType getEventType() {
-        if (eventTypePresentCase_ == 2) {
-          @SuppressWarnings("deprecation")
-          com.alibaba.otter.canal.protocol.CanalEntry.EventType result = com.alibaba.otter.canal.protocol.CanalEntry.EventType.valueOf(
-              (java.lang.Integer) eventTypePresent_);
-          return result == null ? com.alibaba.otter.canal.protocol.CanalEntry.EventType.UNRECOGNIZED : result;
-        }
-        return com.alibaba.otter.canal.protocol.CanalEntry.EventType.EVENTTYPECOMPATIBLEPROTO2;
-      }
-      /**
-       * .com.alibaba.otter.canal.protocol.EventType eventType = 2;
-       */
-      public Builder setEventType(com.alibaba.otter.canal.protocol.CanalEntry.EventType value) {
+      public Builder setEventType(EventType value) {
         if (value == null) {
           throw new NullPointerException();
         }
-        eventTypePresentCase_ = 2;
-        eventTypePresent_ = value.getNumber();
+        bitField0_ |= 0x00000002;
+        eventType_ = value;
         onChanged();
         return this;
       }
       /**
-       * .com.alibaba.otter.canal.protocol.EventType eventType = 2;
+       * optional .com.alibaba.otter.canal.protocol.EventType eventType = 2 [default = UPDATE];
+       *
+       * 
      +       **数据变更类型*
+       * 
      
        */
       public Builder clearEventType() {
-        if (eventTypePresentCase_ == 2) {
-          eventTypePresentCase_ = 0;
-          eventTypePresent_ = null;
-          onChanged();
-        }
+        bitField0_ = (bitField0_ & ~0x00000002);
+        eventType_ = EventType.UPDATE;
+        onChanged();
         return this;
       }
 
+      private boolean isDdl_ ;
       /**
-       * bool isDdl = 10;
+       * optional bool isDdl = 10 [default = false];
+       *
+       * 
      +       ** 标识是否是ddl语句  *
+       * 
      
        */
-      public boolean getIsDdl() {
-        if (isDdlPresentCase_ == 10) {
-          return (java.lang.Boolean) isDdlPresent_;
-        }
-        return false;
+      public boolean hasIsDdl() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
       }
       /**
-       * bool isDdl = 10;
+       * optional bool isDdl = 10 [default = false];
+       *
+       * 
      +       ** 标识是否是ddl语句  *
+       * 
      
+       */
+      public boolean getIsDdl() {
+        return isDdl_;
+      }
+      /**
+       * optional bool isDdl = 10 [default = false];
+       *
+       * 
      +       ** 标识是否是ddl语句  *
+       * 
      
        */
       public Builder setIsDdl(boolean value) {
-        isDdlPresentCase_ = 10;
-        isDdlPresent_ = value;
+        bitField0_ |= 0x00000004;
+        isDdl_ = value;
         onChanged();
         return this;
       }
       /**
-       * bool isDdl = 10;
+       * optional bool isDdl = 10 [default = false];
+       *
+       * 
      +       ** 标识是否是ddl语句  *
+       * 
      
        */
       public Builder clearIsDdl() {
-        if (isDdlPresentCase_ == 10) {
-          isDdlPresentCase_ = 0;
-          isDdlPresent_ = null;
-          onChanged();
-        }
+        bitField0_ = (bitField0_ & ~0x00000004);
+        isDdl_ = false;
+        onChanged();
         return this;
       }
 
-      private java.lang.Object sql_ = "";
+      private Object sql_ = "";
       /**
+       * optional string sql = 11;
+       *
        * 
              ** ddl/query的sql语句  *
        * 
      
-       *
-       * string sql = 11;
        */
-      public java.lang.String getSql() {
-        java.lang.Object ref = sql_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasSql() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      /**
+       * optional string sql = 11;
+       *
+       * 
      +       ** ddl/query的sql语句  *
+       * 
      
+       */
+      public String getSql() {
+        Object ref = sql_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          sql_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            sql_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
+       * optional string sql = 11;
+       *
        * 
              ** ddl/query的sql语句  *
        * 
      
-       *
-       * string sql = 11;
        */
       public com.google.protobuf.ByteString
           getSqlBytes() {
-        java.lang.Object ref = sql_;
+        Object ref = sql_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           sql_ = b;
           return b;
         } else {
@@ -9286,74 +8914,73 @@ public final class CanalEntry {
         }
       }
       /**
+       * optional string sql = 11;
+       *
        * 
              ** ddl/query的sql语句  *
        * 
      
-       *
-       * string sql = 11;
        */
       public Builder setSql(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000008;
         sql_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional string sql = 11;
+       *
        * 
              ** ddl/query的sql语句  *
        * 
      
-       *
-       * string sql = 11;
        */
       public Builder clearSql() {
-        
+        bitField0_ = (bitField0_ & ~0x00000008);
         sql_ = getDefaultInstance().getSql();
         onChanged();
         return this;
       }
       /**
+       * optional string sql = 11;
+       *
        * 
              ** ddl/query的sql语句  *
        * 
      
-       *
-       * string sql = 11;
        */
       public Builder setSqlBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00000008;
         sql_ = value;
         onChanged();
         return this;
       }
 
-      private java.util.List rowDatas_ =
+      private java.util.List rowDatas_ =
         java.util.Collections.emptyList();
       private void ensureRowDatasIsMutable() {
         if (!((bitField0_ & 0x00000010) == 0x00000010)) {
-          rowDatas_ = new java.util.ArrayList(rowDatas_);
+          rowDatas_ = new java.util.ArrayList(rowDatas_);
           bitField0_ |= 0x00000010;
          }
       }
 
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.RowData, com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder, com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder> rowDatasBuilder_;
+      private com.google.protobuf.RepeatedFieldBuilder<
+          RowData, RowData.Builder, RowDataOrBuilder> rowDatasBuilder_;
 
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
-      public java.util.List getRowDatasList() {
+      public java.util.List getRowDatasList() {
         if (rowDatasBuilder_ == null) {
           return java.util.Collections.unmodifiableList(rowDatas_);
         } else {
@@ -9361,11 +8988,11 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
       public int getRowDatasCount() {
         if (rowDatasBuilder_ == null) {
@@ -9375,13 +9002,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.RowData getRowDatas(int index) {
+      public RowData getRowDatas(int index) {
         if (rowDatasBuilder_ == null) {
           return rowDatas_.get(index);
         } else {
@@ -9389,14 +9016,14 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
       public Builder setRowDatas(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.RowData value) {
+          int index, RowData value) {
         if (rowDatasBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -9410,14 +9037,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
       public Builder setRowDatas(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder builderForValue) {
+          int index, RowData.Builder builderForValue) {
         if (rowDatasBuilder_ == null) {
           ensureRowDatasIsMutable();
           rowDatas_.set(index, builderForValue.build());
@@ -9428,13 +9055,13 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
-      public Builder addRowDatas(com.alibaba.otter.canal.protocol.CanalEntry.RowData value) {
+      public Builder addRowDatas(RowData value) {
         if (rowDatasBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -9448,14 +9075,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
       public Builder addRowDatas(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.RowData value) {
+          int index, RowData value) {
         if (rowDatasBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -9469,14 +9096,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
       public Builder addRowDatas(
-          com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder builderForValue) {
+          RowData.Builder builderForValue) {
         if (rowDatasBuilder_ == null) {
           ensureRowDatasIsMutable();
           rowDatas_.add(builderForValue.build());
@@ -9487,14 +9114,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
       public Builder addRowDatas(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder builderForValue) {
+          int index, RowData.Builder builderForValue) {
         if (rowDatasBuilder_ == null) {
           ensureRowDatasIsMutable();
           rowDatas_.add(index, builderForValue.build());
@@ -9505,14 +9132,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
       public Builder addAllRowDatas(
-          java.lang.Iterable values) {
+          Iterable values) {
         if (rowDatasBuilder_ == null) {
           ensureRowDatasIsMutable();
           com.google.protobuf.AbstractMessageLite.Builder.addAll(
@@ -9524,11 +9151,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
       public Builder clearRowDatas() {
         if (rowDatasBuilder_ == null) {
@@ -9541,11 +9168,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
       public Builder removeRowDatas(int index) {
         if (rowDatasBuilder_ == null) {
@@ -9558,24 +9185,24 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder getRowDatasBuilder(
+      public RowData.Builder getRowDatasBuilder(
           int index) {
         return getRowDatasFieldBuilder().getBuilder(index);
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder getRowDatasOrBuilder(
+      public RowDataOrBuilder getRowDatasOrBuilder(
           int index) {
         if (rowDatasBuilder_ == null) {
           return rowDatas_.get(index);  } else {
@@ -9583,13 +9210,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
-      public java.util.List 
+      public java.util.List
            getRowDatasOrBuilderList() {
         if (rowDatasBuilder_ != null) {
           return rowDatasBuilder_.getMessageOrBuilderList();
@@ -9598,45 +9225,45 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder addRowDatasBuilder() {
+      public RowData.Builder addRowDatasBuilder() {
         return getRowDatasFieldBuilder().addBuilder(
-            com.alibaba.otter.canal.protocol.CanalEntry.RowData.getDefaultInstance());
+            RowData.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder addRowDatasBuilder(
+      public RowData.Builder addRowDatasBuilder(
           int index) {
         return getRowDatasFieldBuilder().addBuilder(
-            index, com.alibaba.otter.canal.protocol.CanalEntry.RowData.getDefaultInstance());
+            index, RowData.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+       *
        * 
              ** 一次数据库变更可能存在多行  *
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
        */
-      public java.util.List 
+      public java.util.List
            getRowDatasBuilderList() {
         return getRowDatasFieldBuilder().getBuilderList();
       }
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.RowData, com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder, com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder> 
+      private com.google.protobuf.RepeatedFieldBuilder<
+          RowData, RowData.Builder, RowDataOrBuilder>
           getRowDatasFieldBuilder() {
         if (rowDatasBuilder_ == null) {
-          rowDatasBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3<
-              com.alibaba.otter.canal.protocol.CanalEntry.RowData, com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder, com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder>(
+          rowDatasBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              RowData, RowData.Builder, RowDataOrBuilder>(
                   rowDatas_,
                   ((bitField0_ & 0x00000010) == 0x00000010),
                   getParentForChildren(),
@@ -9646,26 +9273,26 @@ public final class CanalEntry {
         return rowDatasBuilder_;
       }
 
-      private java.util.List props_ =
+      private java.util.List props_ =
         java.util.Collections.emptyList();
       private void ensurePropsIsMutable() {
         if (!((bitField0_ & 0x00000020) == 0x00000020)) {
-          props_ = new java.util.ArrayList(props_);
+          props_ = new java.util.ArrayList(props_);
           bitField0_ |= 0x00000020;
          }
       }
 
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> propsBuilder_;
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Pair, Pair.Builder, PairOrBuilder> propsBuilder_;
 
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
-      public java.util.List getPropsList() {
+      public java.util.List getPropsList() {
         if (propsBuilder_ == null) {
           return java.util.Collections.unmodifiableList(props_);
         } else {
@@ -9673,11 +9300,11 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
       public int getPropsCount() {
         if (propsBuilder_ == null) {
@@ -9687,13 +9314,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+      public Pair getProps(int index) {
         if (propsBuilder_ == null) {
           return props_.get(index);
         } else {
@@ -9701,14 +9328,14 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
       public Builder setProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+          int index, Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -9722,14 +9349,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
       public Builder setProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          int index, Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.set(index, builderForValue.build());
@@ -9740,13 +9367,13 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
-      public Builder addProps(com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+      public Builder addProps(Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -9760,14 +9387,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
       public Builder addProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+          int index, Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -9781,14 +9408,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
       public Builder addProps(
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.add(builderForValue.build());
@@ -9799,14 +9426,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
       public Builder addProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          int index, Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.add(index, builderForValue.build());
@@ -9817,14 +9444,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
       public Builder addAllProps(
-          java.lang.Iterable values) {
+          Iterable values) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           com.google.protobuf.AbstractMessageLite.Builder.addAll(
@@ -9836,11 +9463,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
       public Builder clearProps() {
         if (propsBuilder_ == null) {
@@ -9853,11 +9480,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
       public Builder removeProps(int index) {
         if (propsBuilder_ == null) {
@@ -9870,24 +9497,24 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder getPropsBuilder(
+      public Pair.Builder getPropsBuilder(
           int index) {
         return getPropsFieldBuilder().getBuilder(index);
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+      public PairOrBuilder getPropsOrBuilder(
           int index) {
         if (propsBuilder_ == null) {
           return props_.get(index);  } else {
@@ -9895,13 +9522,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
-      public java.util.List 
+      public java.util.List
            getPropsOrBuilderList() {
         if (propsBuilder_ != null) {
           return propsBuilder_.getMessageOrBuilderList();
@@ -9910,45 +9537,45 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder() {
+      public Pair.Builder addPropsBuilder() {
         return getPropsFieldBuilder().addBuilder(
-            com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+            Pair.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder(
+      public Pair.Builder addPropsBuilder(
           int index) {
         return getPropsFieldBuilder().addBuilder(
-            index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+            index, Pair.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
        */
-      public java.util.List 
+      public java.util.List
            getPropsBuilderList() {
         return getPropsFieldBuilder().getBuilderList();
       }
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> 
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Pair, Pair.Builder, PairOrBuilder>
           getPropsFieldBuilder() {
         if (propsBuilder_ == null) {
-          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3<
-              com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder>(
+          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              Pair, Pair.Builder, PairOrBuilder>(
                   props_,
                   ((bitField0_ & 0x00000020) == 0x00000020),
                   getParentForChildren(),
@@ -9958,40 +9585,52 @@ public final class CanalEntry {
         return propsBuilder_;
       }
 
-      private java.lang.Object ddlSchemaName_ = "";
+      private Object ddlSchemaName_ = "";
       /**
+       * optional string ddlSchemaName = 14;
+       *
        * 
              ** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  *
        * 
      
-       *
-       * string ddlSchemaName = 14;
        */
-      public java.lang.String getDdlSchemaName() {
-        java.lang.Object ref = ddlSchemaName_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasDdlSchemaName() {
+        return ((bitField0_ & 0x00000040) == 0x00000040);
+      }
+      /**
+       * optional string ddlSchemaName = 14;
+       *
+       * 
      +       ** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  *
+       * 
      
+       */
+      public String getDdlSchemaName() {
+        Object ref = ddlSchemaName_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          ddlSchemaName_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            ddlSchemaName_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
+       * optional string ddlSchemaName = 14;
+       *
        * 
              ** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  *
        * 
      
-       *
-       * string ddlSchemaName = 14;
        */
       public com.google.protobuf.ByteString
           getDdlSchemaNameBytes() {
-        java.lang.Object ref = ddlSchemaName_;
+        Object ref = ddlSchemaName_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           ddlSchemaName_ = b;
           return b;
         } else {
@@ -9999,104 +9638,62 @@ public final class CanalEntry {
         }
       }
       /**
+       * optional string ddlSchemaName = 14;
+       *
        * 
              ** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  *
        * 
      
-       *
-       * string ddlSchemaName = 14;
        */
       public Builder setDdlSchemaName(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000040;
         ddlSchemaName_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional string ddlSchemaName = 14;
+       *
        * 
              ** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  *
        * 
      
-       *
-       * string ddlSchemaName = 14;
        */
       public Builder clearDdlSchemaName() {
-        
+        bitField0_ = (bitField0_ & ~0x00000040);
         ddlSchemaName_ = getDefaultInstance().getDdlSchemaName();
         onChanged();
         return this;
       }
       /**
+       * optional string ddlSchemaName = 14;
+       *
        * 
              ** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  *
        * 
      
-       *
-       * string ddlSchemaName = 14;
        */
       public Builder setDdlSchemaNameBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00000040;
         ddlSchemaName_ = value;
         onChanged();
         return this;
       }
-      @java.lang.Override
-      public final Builder setUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.setUnknownFieldsProto3(unknownFields);
-      }
-
-      @java.lang.Override
-      public final Builder mergeUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.mergeUnknownFields(unknownFields);
-      }
-
 
       // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.RowChange)
     }
 
-    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.RowChange)
-    private static final com.alibaba.otter.canal.protocol.CanalEntry.RowChange DEFAULT_INSTANCE;
     static {
-      DEFAULT_INSTANCE = new com.alibaba.otter.canal.protocol.CanalEntry.RowChange();
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange getDefaultInstance() {
-      return DEFAULT_INSTANCE;
-    }
-
-    private static final com.google.protobuf.Parser
-        PARSER = new com.google.protobuf.AbstractParser() {
-      @java.lang.Override
-      public RowChange parsePartialFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return new RowChange(input, extensionRegistry);
-      }
-    };
-
-    public static com.google.protobuf.Parser parser() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.google.protobuf.Parser getParserForType() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.alibaba.otter.canal.protocol.CanalEntry.RowChange getDefaultInstanceForType() {
-      return DEFAULT_INSTANCE;
+      defaultInstance = new RowChange(true);
+      defaultInstance.initFields();
     }
 
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.RowChange)
   }
 
   public interface TransactionBeginOrBuilder extends
@@ -10104,121 +9701,146 @@ public final class CanalEntry {
       com.google.protobuf.MessageOrBuilder {
 
     /**
+     * optional int64 executeTime = 1;
+     *
      * 
            **已废弃，请使用header里的executeTime*
      * 
      
+     */
+    boolean hasExecuteTime();
+    /**
+     * optional int64 executeTime = 1;
      *
-     * int64 executeTime = 1;
+     * 
      +     **已废弃，请使用header里的executeTime*
+     * 
      
      */
     long getExecuteTime();
 
     /**
+     * optional string transactionId = 2;
+     *
      * 
            **已废弃，Begin里不提供事务id*
      * 
      
-     *
-     * string transactionId = 2;
      */
-    java.lang.String getTransactionId();
+    boolean hasTransactionId();
     /**
+     * optional string transactionId = 2;
+     *
      * 
            **已废弃，Begin里不提供事务id*
      * 
      
+     */
+    String getTransactionId();
+    /**
+     * optional string transactionId = 2;
      *
-     * string transactionId = 2;
+     * 
      +     **已废弃，Begin里不提供事务id*
+     * 
      
      */
     com.google.protobuf.ByteString
         getTransactionIdBytes();
 
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    java.util.List 
+    java.util.List
         getPropsList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index);
+    Pair getProps(int index);
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
     int getPropsCount();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    java.util.List 
+    java.util.List
         getPropsOrBuilderList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
-        int index);
+    PairOrBuilder getPropsOrBuilder(int index);
 
     /**
+     * optional int64 threadId = 4;
+     *
      * 
            **执行的thread Id*
      * 
      
+     */
+    boolean hasThreadId();
+    /**
+     * optional int64 threadId = 4;
      *
-     * int64 threadId = 4;
+     * 
      +     **执行的thread Id*
+     * 
      
      */
     long getThreadId();
   }
   /**
+   * Protobuf type {@code com.alibaba.otter.canal.protocol.TransactionBegin}
+   *
    * 
          **开始事务的一些信息*
    * 
      
-   *
-   * Protobuf type {@code com.alibaba.otter.canal.protocol.TransactionBegin}
    */
-  public  static final class TransactionBegin extends
-      com.google.protobuf.GeneratedMessageV3 implements
+  public static final class TransactionBegin extends
+      com.google.protobuf.GeneratedMessage implements
       // @@protoc_insertion_point(message_implements:com.alibaba.otter.canal.protocol.TransactionBegin)
       TransactionBeginOrBuilder {
-  private static final long serialVersionUID = 0L;
     // Use TransactionBegin.newBuilder() to construct.
-    private TransactionBegin(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+    private TransactionBegin(com.google.protobuf.GeneratedMessage.Builder builder) {
       super(builder);
+      this.unknownFields = builder.getUnknownFields();
     }
-    private TransactionBegin() {
-      executeTime_ = 0L;
-      transactionId_ = "";
-      props_ = java.util.Collections.emptyList();
-      threadId_ = 0L;
+    private TransactionBegin(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+    private static final TransactionBegin defaultInstance;
+    public static TransactionBegin getDefaultInstance() {
+      return defaultInstance;
     }
 
-    @java.lang.Override
+    public TransactionBegin getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+
+    private final com.google.protobuf.UnknownFieldSet unknownFields;
+    @Override
     public final com.google.protobuf.UnknownFieldSet
-    getUnknownFields() {
+        getUnknownFields() {
       return this.unknownFields;
     }
     private TransactionBegin(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
-      this();
-      if (extensionRegistry == null) {
-        throw new java.lang.NullPointerException();
-      }
+      initFields();
       int mutable_bitField0_ = 0;
       com.google.protobuf.UnknownFieldSet.Builder unknownFields =
           com.google.protobuf.UnknownFieldSet.newBuilder();
@@ -10230,45 +9852,44 @@ public final class CanalEntry {
             case 0:
               done = true;
               break;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                done = true;
+              }
+              break;
+            }
             case 8: {
-
+              bitField0_ |= 0x00000001;
               executeTime_ = input.readInt64();
               break;
             }
             case 18: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              transactionId_ = s;
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000002;
+              transactionId_ = bs;
               break;
             }
             case 26: {
               if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
-                props_ = new java.util.ArrayList();
+                props_ = new java.util.ArrayList();
                 mutable_bitField0_ |= 0x00000004;
               }
-              props_.add(
-                  input.readMessage(com.alibaba.otter.canal.protocol.CanalEntry.Pair.parser(), extensionRegistry));
+              props_.add(input.readMessage(Pair.PARSER, extensionRegistry));
               break;
             }
             case 32: {
-
+              bitField0_ |= 0x00000004;
               threadId_ = input.readInt64();
               break;
             }
-            default: {
-              if (!parseUnknownFieldProto3(
-                  input, unknownFields, extensionRegistry, tag)) {
-                done = true;
-              }
-              break;
-            }
           }
         }
       } catch (com.google.protobuf.InvalidProtocolBufferException e) {
         throw e.setUnfinishedMessage(this);
       } catch (java.io.IOException e) {
         throw new com.google.protobuf.InvalidProtocolBufferException(
-            e).setUnfinishedMessage(this);
+            e.getMessage()).setUnfinishedMessage(this);
       } finally {
         if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
           props_ = java.util.Collections.unmodifiableList(props_);
@@ -10279,66 +9900,102 @@ public final class CanalEntry {
     }
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor;
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor;
     }
 
-    @java.lang.Override
-    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+    protected FieldAccessorTable
         internalGetFieldAccessorTable() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_fieldAccessorTable
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_fieldAccessorTable
           .ensureFieldAccessorsInitialized(
-              com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin.class, com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin.Builder.class);
+              TransactionBegin.class, Builder.class);
+    }
+
+    public static com.google.protobuf.Parser PARSER =
+        new com.google.protobuf.AbstractParser() {
+      public TransactionBegin parsePartialFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return new TransactionBegin(input, extensionRegistry);
+      }
+    };
+
+    @Override
+    public com.google.protobuf.Parser getParserForType() {
+      return PARSER;
     }
 
     private int bitField0_;
     public static final int EXECUTETIME_FIELD_NUMBER = 1;
     private long executeTime_;
     /**
+     * optional int64 executeTime = 1;
+     *
      * 
            **已废弃，请使用header里的executeTime*
      * 
      
+     */
+    public boolean hasExecuteTime() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    /**
+     * optional int64 executeTime = 1;
      *
-     * int64 executeTime = 1;
+     * 
      +     **已废弃，请使用header里的executeTime*
+     * 
      
      */
     public long getExecuteTime() {
       return executeTime_;
     }
 
     public static final int TRANSACTIONID_FIELD_NUMBER = 2;
-    private volatile java.lang.Object transactionId_;
+    private Object transactionId_;
     /**
+     * optional string transactionId = 2;
+     *
      * 
            **已废弃，Begin里不提供事务id*
      * 
      
-     *
-     * string transactionId = 2;
      */
-    public java.lang.String getTransactionId() {
-      java.lang.Object ref = transactionId_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasTransactionId() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    /**
+     * optional string transactionId = 2;
+     *
+     * 
      +     **已废弃，Begin里不提供事务id*
+     * 
      
+     */
+    public String getTransactionId() {
+      Object ref = transactionId_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        transactionId_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          transactionId_ = s;
+        }
         return s;
       }
     }
     /**
+     * optional string transactionId = 2;
+     *
      * 
            **已废弃，Begin里不提供事务id*
      * 
      
-     *
-     * string transactionId = 2;
      */
     public com.google.protobuf.ByteString
         getTransactionIdBytes() {
-      java.lang.Object ref = transactionId_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = transactionId_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         transactionId_ = b;
         return b;
       } else {
@@ -10347,56 +10004,56 @@ public final class CanalEntry {
     }
 
     public static final int PROPS_FIELD_NUMBER = 3;
-    private java.util.List props_;
+    private java.util.List props_;
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    public java.util.List getPropsList() {
+    public java.util.List getPropsList() {
       return props_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    public java.util.List 
+    public java.util.List
         getPropsOrBuilderList() {
       return props_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
     public int getPropsCount() {
       return props_.size();
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+    public Pair getProps(int index) {
       return props_.get(index);
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+    public PairOrBuilder getPropsOrBuilder(
         int index) {
       return props_.get(index);
     }
@@ -10404,18 +10061,33 @@ public final class CanalEntry {
     public static final int THREADID_FIELD_NUMBER = 4;
     private long threadId_;
     /**
+     * optional int64 threadId = 4;
+     *
      * 
            **执行的thread Id*
      * 
      
+     */
+    public boolean hasThreadId() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    /**
+     * optional int64 threadId = 4;
      *
-     * int64 threadId = 4;
+     * 
      +     **执行的thread Id*
+     * 
      
      */
     public long getThreadId() {
       return threadId_;
     }
 
+    private void initFields() {
+      executeTime_ = 0L;
+      transactionId_ = "";
+      props_ = java.util.Collections.emptyList();
+      threadId_ = 0L;
+    }
     private byte memoizedIsInitialized = -1;
-    @java.lang.Override
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized == 1) return true;
@@ -10425,209 +10097,145 @@ public final class CanalEntry {
       return true;
     }
 
-    @java.lang.Override
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
-      if (executeTime_ != 0L) {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
         output.writeInt64(1, executeTime_);
       }
-      if (!getTransactionIdBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 2, transactionId_);
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getTransactionIdBytes());
       }
       for (int i = 0; i < props_.size(); i++) {
         output.writeMessage(3, props_.get(i));
       }
-      if (threadId_ != 0L) {
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
         output.writeInt64(4, threadId_);
       }
-      unknownFields.writeTo(output);
+      getUnknownFields().writeTo(output);
     }
 
-    @java.lang.Override
+    private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
-      int size = memoizedSize;
+      int size = memoizedSerializedSize;
       if (size != -1) return size;
 
       size = 0;
-      if (executeTime_ != 0L) {
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
           .computeInt64Size(1, executeTime_);
       }
-      if (!getTransactionIdBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, transactionId_);
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getTransactionIdBytes());
       }
       for (int i = 0; i < props_.size(); i++) {
         size += com.google.protobuf.CodedOutputStream
           .computeMessageSize(3, props_.get(i));
       }
-      if (threadId_ != 0L) {
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
         size += com.google.protobuf.CodedOutputStream
           .computeInt64Size(4, threadId_);
       }
-      size += unknownFields.getSerializedSize();
-      memoizedSize = size;
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
       return size;
     }
 
-    @java.lang.Override
-    public boolean equals(final java.lang.Object obj) {
-      if (obj == this) {
-       return true;
-      }
-      if (!(obj instanceof com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin)) {
-        return super.equals(obj);
-      }
-      com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin other = (com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin) obj;
-
-      boolean result = true;
-      result = result && (getExecuteTime()
-          == other.getExecuteTime());
-      result = result && getTransactionId()
-          .equals(other.getTransactionId());
-      result = result && getPropsList()
-          .equals(other.getPropsList());
-      result = result && (getThreadId()
-          == other.getThreadId());
-      result = result && unknownFields.equals(other.unknownFields);
-      return result;
+    private static final long serialVersionUID = 0L;
+    @Override
+    protected Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
     }
 
-    @java.lang.Override
-    public int hashCode() {
-      if (memoizedHashCode != 0) {
-        return memoizedHashCode;
-      }
-      int hash = 41;
-      hash = (19 * hash) + getDescriptor().hashCode();
-      hash = (37 * hash) + EXECUTETIME_FIELD_NUMBER;
-      hash = (53 * hash) + com.google.protobuf.Internal.hashLong(
-          getExecuteTime());
-      hash = (37 * hash) + TRANSACTIONID_FIELD_NUMBER;
-      hash = (53 * hash) + getTransactionId().hashCode();
-      if (getPropsCount() > 0) {
-        hash = (37 * hash) + PROPS_FIELD_NUMBER;
-        hash = (53 * hash) + getPropsList().hashCode();
-      }
-      hash = (37 * hash) + THREADID_FIELD_NUMBER;
-      hash = (53 * hash) + com.google.protobuf.Internal.hashLong(
-          getThreadId());
-      hash = (29 * hash) + unknownFields.hashCode();
-      memoizedHashCode = hash;
-      return hash;
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
-        java.nio.ByteBuffer data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
-        java.nio.ByteBuffer data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
+    public static TransactionBegin parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
+    public static TransactionBegin parseFrom(
         com.google.protobuf.ByteString data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(byte[] data)
+    public static TransactionBegin parseFrom(byte[] data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
+    public static TransactionBegin parseFrom(
         byte[] data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(java.io.InputStream input)
+    public static TransactionBegin parseFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
+    public static TransactionBegin parseFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseDelimitedFrom(java.io.InputStream input)
+    public static TransactionBegin parseDelimitedFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input);
+      return PARSER.parseDelimitedFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseDelimitedFrom(
+    public static TransactionBegin parseDelimitedFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseDelimitedFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
+    public static TransactionBegin parseFrom(
         com.google.protobuf.CodedInputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
+    public static TransactionBegin parseFrom(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
 
-    @java.lang.Override
+    public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
-    public static Builder newBuilder() {
-      return DEFAULT_INSTANCE.toBuilder();
-    }
-    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin prototype) {
-      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
-    }
-    @java.lang.Override
-    public Builder toBuilder() {
-      return this == DEFAULT_INSTANCE
-          ? new Builder() : new Builder().mergeFrom(this);
+    public static Builder newBuilder(TransactionBegin prototype) {
+      return newBuilder().mergeFrom(prototype);
     }
+    public Builder toBuilder() { return newBuilder(this); }
 
-    @java.lang.Override
+    @Override
     protected Builder newBuilderForType(
-        com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        BuilderParent parent) {
       Builder builder = new Builder(parent);
       return builder;
     }
     /**
+     * Protobuf type {@code com.alibaba.otter.canal.protocol.TransactionBegin}
+     *
      * 
            **开始事务的一些信息*
      * 
      
-     *
-     * Protobuf type {@code com.alibaba.otter.canal.protocol.TransactionBegin}
      */
     public static final class Builder extends
-        com.google.protobuf.GeneratedMessageV3.Builder implements
+        com.google.protobuf.GeneratedMessage.Builder implements
         // @@protoc_insertion_point(builder_implements:com.alibaba.otter.canal.protocol.TransactionBegin)
-        com.alibaba.otter.canal.protocol.CanalEntry.TransactionBeginOrBuilder {
+        TransactionBeginOrBuilder {
       public static final com.google.protobuf.Descriptors.Descriptor
           getDescriptor() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor;
       }
 
-      @java.lang.Override
-      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      protected FieldAccessorTable
           internalGetFieldAccessorTable() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_fieldAccessorTable
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_fieldAccessorTable
             .ensureFieldAccessorsInitialized(
-                com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin.class, com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin.Builder.class);
+                TransactionBegin.class, Builder.class);
       }
 
       // Construct using com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin.newBuilder()
@@ -10636,23 +10244,25 @@ public final class CanalEntry {
       }
 
       private Builder(
-          com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
       private void maybeForceBuilderInitialization() {
-        if (com.google.protobuf.GeneratedMessageV3
-                .alwaysUseFieldBuilders) {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
           getPropsFieldBuilder();
         }
       }
-      @java.lang.Override
+      private static Builder create() {
+        return new Builder();
+      }
+
       public Builder clear() {
         super.clear();
         executeTime_ = 0L;
-
+        bitField0_ = (bitField0_ & ~0x00000001);
         transactionId_ = "";
-
+        bitField0_ = (bitField0_ & ~0x00000002);
         if (propsBuilder_ == null) {
           props_ = java.util.Collections.emptyList();
           bitField0_ = (bitField0_ & ~0x00000004);
@@ -10660,36 +10270,42 @@ public final class CanalEntry {
           propsBuilder_.clear();
         }
         threadId_ = 0L;
-
+        bitField0_ = (bitField0_ & ~0x00000008);
         return this;
       }
 
-      @java.lang.Override
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin getDefaultInstanceForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin.getDefaultInstance();
+      public TransactionBegin getDefaultInstanceForType() {
+        return TransactionBegin.getDefaultInstance();
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin build() {
-        com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin result = buildPartial();
+      public TransactionBegin build() {
+        TransactionBegin result = buildPartial();
         if (!result.isInitialized()) {
           throw newUninitializedMessageException(result);
         }
         return result;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin buildPartial() {
-        com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin result = new com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin(this);
+      public TransactionBegin buildPartial() {
+        TransactionBegin result = new TransactionBegin(this);
         int from_bitField0_ = bitField0_;
         int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
         result.executeTime_ = executeTime_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
         result.transactionId_ = transactionId_;
         if (propsBuilder_ == null) {
           if (((bitField0_ & 0x00000004) == 0x00000004)) {
@@ -10700,60 +10316,31 @@ public final class CanalEntry {
         } else {
           result.props_ = propsBuilder_.build();
         }
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000004;
+        }
         result.threadId_ = threadId_;
         result.bitField0_ = to_bitField0_;
         onBuilt();
         return result;
       }
 
-      @java.lang.Override
-      public Builder clone() {
-        return (Builder) super.clone();
-      }
-      @java.lang.Override
-      public Builder setField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.setField(field, value);
-      }
-      @java.lang.Override
-      public Builder clearField(
-          com.google.protobuf.Descriptors.FieldDescriptor field) {
-        return (Builder) super.clearField(field);
-      }
-      @java.lang.Override
-      public Builder clearOneof(
-          com.google.protobuf.Descriptors.OneofDescriptor oneof) {
-        return (Builder) super.clearOneof(oneof);
-      }
-      @java.lang.Override
-      public Builder setRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          int index, java.lang.Object value) {
-        return (Builder) super.setRepeatedField(field, index, value);
-      }
-      @java.lang.Override
-      public Builder addRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.addRepeatedField(field, value);
-      }
-      @java.lang.Override
       public Builder mergeFrom(com.google.protobuf.Message other) {
-        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin) {
-          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin)other);
+        if (other instanceof TransactionBegin) {
+          return mergeFrom((TransactionBegin)other);
         } else {
           super.mergeFrom(other);
           return this;
         }
       }
 
-      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin other) {
-        if (other == com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin.getDefaultInstance()) return this;
-        if (other.getExecuteTime() != 0L) {
+      public Builder mergeFrom(TransactionBegin other) {
+        if (other == TransactionBegin.getDefaultInstance()) return this;
+        if (other.hasExecuteTime()) {
           setExecuteTime(other.getExecuteTime());
         }
-        if (!other.getTransactionId().isEmpty()) {
+        if (other.hasTransactionId()) {
+          bitField0_ |= 0x00000002;
           transactionId_ = other.transactionId_;
           onChanged();
         }
@@ -10775,38 +10362,35 @@ public final class CanalEntry {
               propsBuilder_ = null;
               props_ = other.props_;
               bitField0_ = (bitField0_ & ~0x00000004);
-              propsBuilder_ = 
-                com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ?
+              propsBuilder_ =
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
                    getPropsFieldBuilder() : null;
             } else {
               propsBuilder_.addAllMessages(other.props_);
             }
           }
         }
-        if (other.getThreadId() != 0L) {
+        if (other.hasThreadId()) {
           setThreadId(other.getThreadId());
         }
-        this.mergeUnknownFields(other.unknownFields);
-        onChanged();
+        this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
 
-      @java.lang.Override
       public final boolean isInitialized() {
         return true;
       }
 
-      @java.lang.Override
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
           throws java.io.IOException {
-        com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parsedMessage = null;
+        TransactionBegin parsedMessage = null;
         try {
           parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
         } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          parsedMessage = (com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin) e.getUnfinishedMessage();
-          throw e.unwrapIOException();
+          parsedMessage = (TransactionBegin) e.getUnfinishedMessage();
+          throw e;
         } finally {
           if (parsedMessage != null) {
             mergeFrom(parsedMessage);
@@ -10818,76 +10402,98 @@ public final class CanalEntry {
 
       private long executeTime_ ;
       /**
+       * optional int64 executeTime = 1;
+       *
        * 
              **已废弃，请使用header里的executeTime*
        * 
      
+       */
+      public boolean hasExecuteTime() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      /**
+       * optional int64 executeTime = 1;
        *
-       * int64 executeTime = 1;
+       * 
      +       **已废弃，请使用header里的executeTime*
+       * 
      
        */
       public long getExecuteTime() {
         return executeTime_;
       }
       /**
+       * optional int64 executeTime = 1;
+       *
        * 
              **已废弃，请使用header里的executeTime*
        * 
      
-       *
-       * int64 executeTime = 1;
        */
       public Builder setExecuteTime(long value) {
-        
+        bitField0_ |= 0x00000001;
         executeTime_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional int64 executeTime = 1;
+       *
        * 
              **已废弃，请使用header里的executeTime*
        * 
      
-       *
-       * int64 executeTime = 1;
        */
       public Builder clearExecuteTime() {
-        
+        bitField0_ = (bitField0_ & ~0x00000001);
         executeTime_ = 0L;
         onChanged();
         return this;
       }
 
-      private java.lang.Object transactionId_ = "";
+      private Object transactionId_ = "";
       /**
+       * optional string transactionId = 2;
+       *
        * 
              **已废弃，Begin里不提供事务id*
        * 
      
-       *
-       * string transactionId = 2;
        */
-      public java.lang.String getTransactionId() {
-        java.lang.Object ref = transactionId_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasTransactionId() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      /**
+       * optional string transactionId = 2;
+       *
+       * 
      +       **已废弃，Begin里不提供事务id*
+       * 
      
+       */
+      public String getTransactionId() {
+        Object ref = transactionId_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          transactionId_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            transactionId_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
+       * optional string transactionId = 2;
+       *
        * 
              **已废弃，Begin里不提供事务id*
        * 
      
-       *
-       * string transactionId = 2;
        */
       public com.google.protobuf.ByteString
           getTransactionIdBytes() {
-        java.lang.Object ref = transactionId_;
+        Object ref = transactionId_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           transactionId_ = b;
           return b;
         } else {
@@ -10895,74 +10501,73 @@ public final class CanalEntry {
         }
       }
       /**
+       * optional string transactionId = 2;
+       *
        * 
              **已废弃，Begin里不提供事务id*
        * 
      
-       *
-       * string transactionId = 2;
        */
       public Builder setTransactionId(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000002;
         transactionId_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional string transactionId = 2;
+       *
        * 
              **已废弃，Begin里不提供事务id*
        * 
      
-       *
-       * string transactionId = 2;
        */
       public Builder clearTransactionId() {
-        
+        bitField0_ = (bitField0_ & ~0x00000002);
         transactionId_ = getDefaultInstance().getTransactionId();
         onChanged();
         return this;
       }
       /**
+       * optional string transactionId = 2;
+       *
        * 
              **已废弃，Begin里不提供事务id*
        * 
      
-       *
-       * string transactionId = 2;
        */
       public Builder setTransactionIdBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00000002;
         transactionId_ = value;
         onChanged();
         return this;
       }
 
-      private java.util.List props_ =
+      private java.util.List props_ =
         java.util.Collections.emptyList();
       private void ensurePropsIsMutable() {
         if (!((bitField0_ & 0x00000004) == 0x00000004)) {
-          props_ = new java.util.ArrayList(props_);
+          props_ = new java.util.ArrayList(props_);
           bitField0_ |= 0x00000004;
          }
       }
 
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> propsBuilder_;
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Pair, Pair.Builder, PairOrBuilder> propsBuilder_;
 
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public java.util.List getPropsList() {
+      public java.util.List getPropsList() {
         if (propsBuilder_ == null) {
           return java.util.Collections.unmodifiableList(props_);
         } else {
@@ -10970,11 +10575,11 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public int getPropsCount() {
         if (propsBuilder_ == null) {
@@ -10984,13 +10589,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+      public Pair getProps(int index) {
         if (propsBuilder_ == null) {
           return props_.get(index);
         } else {
@@ -10998,14 +10603,14 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder setProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+          int index, Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -11019,14 +10624,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder setProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          int index, Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.set(index, builderForValue.build());
@@ -11037,13 +10642,13 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public Builder addProps(com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+      public Builder addProps(Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -11057,14 +10662,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder addProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+          int index, Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -11078,14 +10683,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder addProps(
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.add(builderForValue.build());
@@ -11096,14 +10701,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder addProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          int index, Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.add(index, builderForValue.build());
@@ -11114,14 +10719,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder addAllProps(
-          java.lang.Iterable values) {
+          Iterable values) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           com.google.protobuf.AbstractMessageLite.Builder.addAll(
@@ -11133,11 +10738,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder clearProps() {
         if (propsBuilder_ == null) {
@@ -11150,11 +10755,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder removeProps(int index) {
         if (propsBuilder_ == null) {
@@ -11167,24 +10772,24 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder getPropsBuilder(
+      public Pair.Builder getPropsBuilder(
           int index) {
         return getPropsFieldBuilder().getBuilder(index);
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+      public PairOrBuilder getPropsOrBuilder(
           int index) {
         if (propsBuilder_ == null) {
           return props_.get(index);  } else {
@@ -11192,13 +10797,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public java.util.List 
+      public java.util.List
            getPropsOrBuilderList() {
         if (propsBuilder_ != null) {
           return propsBuilder_.getMessageOrBuilderList();
@@ -11207,45 +10812,45 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder() {
+      public Pair.Builder addPropsBuilder() {
         return getPropsFieldBuilder().addBuilder(
-            com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+            Pair.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder(
+      public Pair.Builder addPropsBuilder(
           int index) {
         return getPropsFieldBuilder().addBuilder(
-            index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+            index, Pair.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public java.util.List 
+      public java.util.List
            getPropsBuilderList() {
         return getPropsFieldBuilder().getBuilderList();
       }
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> 
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Pair, Pair.Builder, PairOrBuilder>
           getPropsFieldBuilder() {
         if (propsBuilder_ == null) {
-          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3<
-              com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder>(
+          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              Pair, Pair.Builder, PairOrBuilder>(
                   props_,
                   ((bitField0_ & 0x00000004) == 0x00000004),
                   getParentForChildren(),
@@ -11257,92 +10862,61 @@ public final class CanalEntry {
 
       private long threadId_ ;
       /**
+       * optional int64 threadId = 4;
+       *
        * 
              **执行的thread Id*
        * 
      
+       */
+      public boolean hasThreadId() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      /**
+       * optional int64 threadId = 4;
        *
-       * int64 threadId = 4;
+       * 
      +       **执行的thread Id*
+       * 
      
        */
       public long getThreadId() {
         return threadId_;
       }
       /**
+       * optional int64 threadId = 4;
+       *
        * 
              **执行的thread Id*
        * 
      
-       *
-       * int64 threadId = 4;
        */
       public Builder setThreadId(long value) {
-        
+        bitField0_ |= 0x00000008;
         threadId_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional int64 threadId = 4;
+       *
        * 
              **执行的thread Id*
        * 
      
-       *
-       * int64 threadId = 4;
        */
       public Builder clearThreadId() {
-        
+        bitField0_ = (bitField0_ & ~0x00000008);
         threadId_ = 0L;
         onChanged();
         return this;
       }
-      @java.lang.Override
-      public final Builder setUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.setUnknownFieldsProto3(unknownFields);
-      }
-
-      @java.lang.Override
-      public final Builder mergeUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.mergeUnknownFields(unknownFields);
-      }
-
 
       // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.TransactionBegin)
     }
 
-    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.TransactionBegin)
-    private static final com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin DEFAULT_INSTANCE;
     static {
-      DEFAULT_INSTANCE = new com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin();
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin getDefaultInstance() {
-      return DEFAULT_INSTANCE;
-    }
-
-    private static final com.google.protobuf.Parser
-        PARSER = new com.google.protobuf.AbstractParser() {
-      @java.lang.Override
-      public TransactionBegin parsePartialFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return new TransactionBegin(input, extensionRegistry);
-      }
-    };
-
-    public static com.google.protobuf.Parser parser() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.google.protobuf.Parser getParserForType() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin getDefaultInstanceForType() {
-      return DEFAULT_INSTANCE;
+      defaultInstance = new TransactionBegin(true);
+      defaultInstance.initFields();
     }
 
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.TransactionBegin)
   }
 
   public interface TransactionEndOrBuilder extends
@@ -11350,111 +10924,129 @@ public final class CanalEntry {
       com.google.protobuf.MessageOrBuilder {
 
     /**
+     * optional int64 executeTime = 1;
+     *
      * 
            **已废弃，请使用header里的executeTime*
      * 
      
+     */
+    boolean hasExecuteTime();
+    /**
+     * optional int64 executeTime = 1;
      *
-     * int64 executeTime = 1;
+     * 
      +     **已废弃，请使用header里的executeTime*
+     * 
      
      */
     long getExecuteTime();
 
     /**
+     * optional string transactionId = 2;
+     *
      * 
            **事务号*
      * 
      
-     *
-     * string transactionId = 2;
      */
-    java.lang.String getTransactionId();
+    boolean hasTransactionId();
     /**
+     * optional string transactionId = 2;
+     *
      * 
            **事务号*
      * 
      
+     */
+    String getTransactionId();
+    /**
+     * optional string transactionId = 2;
      *
-     * string transactionId = 2;
+     * 
      +     **事务号*
+     * 
      
      */
     com.google.protobuf.ByteString
         getTransactionIdBytes();
 
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    java.util.List 
+    java.util.List
         getPropsList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index);
+    Pair getProps(int index);
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
     int getPropsCount();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    java.util.List 
+    java.util.List
         getPropsOrBuilderList();
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
-        int index);
+    PairOrBuilder getPropsOrBuilder(int index);
   }
   /**
+   * Protobuf type {@code com.alibaba.otter.canal.protocol.TransactionEnd}
+   *
    * 
          **结束事务的一些信息*
    * 
      
-   *
-   * Protobuf type {@code com.alibaba.otter.canal.protocol.TransactionEnd}
    */
-  public  static final class TransactionEnd extends
-      com.google.protobuf.GeneratedMessageV3 implements
+  public static final class TransactionEnd extends
+      com.google.protobuf.GeneratedMessage implements
       // @@protoc_insertion_point(message_implements:com.alibaba.otter.canal.protocol.TransactionEnd)
       TransactionEndOrBuilder {
-  private static final long serialVersionUID = 0L;
     // Use TransactionEnd.newBuilder() to construct.
-    private TransactionEnd(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+    private TransactionEnd(com.google.protobuf.GeneratedMessage.Builder builder) {
       super(builder);
+      this.unknownFields = builder.getUnknownFields();
     }
-    private TransactionEnd() {
-      executeTime_ = 0L;
-      transactionId_ = "";
-      props_ = java.util.Collections.emptyList();
+    private TransactionEnd(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+    private static final TransactionEnd defaultInstance;
+    public static TransactionEnd getDefaultInstance() {
+      return defaultInstance;
     }
 
-    @java.lang.Override
+    public TransactionEnd getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+
+    private final com.google.protobuf.UnknownFieldSet unknownFields;
+    @Override
     public final com.google.protobuf.UnknownFieldSet
-    getUnknownFields() {
+        getUnknownFields() {
       return this.unknownFields;
     }
     private TransactionEnd(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
-      this();
-      if (extensionRegistry == null) {
-        throw new java.lang.NullPointerException();
-      }
+      initFields();
       int mutable_bitField0_ = 0;
       com.google.protobuf.UnknownFieldSet.Builder unknownFields =
           com.google.protobuf.UnknownFieldSet.newBuilder();
@@ -11466,31 +11058,30 @@ public final class CanalEntry {
             case 0:
               done = true;
               break;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                done = true;
+              }
+              break;
+            }
             case 8: {
-
+              bitField0_ |= 0x00000001;
               executeTime_ = input.readInt64();
               break;
             }
             case 18: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              transactionId_ = s;
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000002;
+              transactionId_ = bs;
               break;
             }
             case 26: {
               if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
-                props_ = new java.util.ArrayList();
+                props_ = new java.util.ArrayList();
                 mutable_bitField0_ |= 0x00000004;
               }
-              props_.add(
-                  input.readMessage(com.alibaba.otter.canal.protocol.CanalEntry.Pair.parser(), extensionRegistry));
-              break;
-            }
-            default: {
-              if (!parseUnknownFieldProto3(
-                  input, unknownFields, extensionRegistry, tag)) {
-                done = true;
-              }
+              props_.add(input.readMessage(Pair.PARSER, extensionRegistry));
               break;
             }
           }
@@ -11499,7 +11090,7 @@ public final class CanalEntry {
         throw e.setUnfinishedMessage(this);
       } catch (java.io.IOException e) {
         throw new com.google.protobuf.InvalidProtocolBufferException(
-            e).setUnfinishedMessage(this);
+            e.getMessage()).setUnfinishedMessage(this);
       } finally {
         if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
           props_ = java.util.Collections.unmodifiableList(props_);
@@ -11510,66 +11101,102 @@ public final class CanalEntry {
     }
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor;
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor;
     }
 
-    @java.lang.Override
-    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+    protected FieldAccessorTable
         internalGetFieldAccessorTable() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_fieldAccessorTable
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_fieldAccessorTable
           .ensureFieldAccessorsInitialized(
-              com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd.class, com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd.Builder.class);
+              TransactionEnd.class, Builder.class);
+    }
+
+    public static com.google.protobuf.Parser PARSER =
+        new com.google.protobuf.AbstractParser() {
+      public TransactionEnd parsePartialFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return new TransactionEnd(input, extensionRegistry);
+      }
+    };
+
+    @Override
+    public com.google.protobuf.Parser getParserForType() {
+      return PARSER;
     }
 
     private int bitField0_;
     public static final int EXECUTETIME_FIELD_NUMBER = 1;
     private long executeTime_;
     /**
+     * optional int64 executeTime = 1;
+     *
      * 
            **已废弃，请使用header里的executeTime*
      * 
      
+     */
+    public boolean hasExecuteTime() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    /**
+     * optional int64 executeTime = 1;
      *
-     * int64 executeTime = 1;
+     * 
      +     **已废弃，请使用header里的executeTime*
+     * 
      
      */
     public long getExecuteTime() {
       return executeTime_;
     }
 
     public static final int TRANSACTIONID_FIELD_NUMBER = 2;
-    private volatile java.lang.Object transactionId_;
+    private Object transactionId_;
     /**
+     * optional string transactionId = 2;
+     *
      * 
            **事务号*
      * 
      
-     *
-     * string transactionId = 2;
      */
-    public java.lang.String getTransactionId() {
-      java.lang.Object ref = transactionId_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasTransactionId() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    /**
+     * optional string transactionId = 2;
+     *
+     * 
      +     **事务号*
+     * 
      
+     */
+    public String getTransactionId() {
+      Object ref = transactionId_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        transactionId_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          transactionId_ = s;
+        }
         return s;
       }
     }
     /**
+     * optional string transactionId = 2;
+     *
      * 
            **事务号*
      * 
      
-     *
-     * string transactionId = 2;
      */
     public com.google.protobuf.ByteString
         getTransactionIdBytes() {
-      java.lang.Object ref = transactionId_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = transactionId_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         transactionId_ = b;
         return b;
       } else {
@@ -11578,62 +11205,66 @@ public final class CanalEntry {
     }
 
     public static final int PROPS_FIELD_NUMBER = 3;
-    private java.util.List props_;
+    private java.util.List props_;
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    public java.util.List getPropsList() {
+    public java.util.List getPropsList() {
       return props_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    public java.util.List 
+    public java.util.List
         getPropsOrBuilderList() {
       return props_;
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
     public int getPropsCount() {
       return props_.size();
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+    public Pair getProps(int index) {
       return props_.get(index);
     }
     /**
+     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
      */
-    public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+    public PairOrBuilder getPropsOrBuilder(
         int index) {
       return props_.get(index);
     }
 
+    private void initFields() {
+      executeTime_ = 0L;
+      transactionId_ = "";
+      props_ = java.util.Collections.emptyList();
+    }
     private byte memoizedIsInitialized = -1;
-    @java.lang.Override
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized == 1) return true;
@@ -11643,197 +11274,138 @@ public final class CanalEntry {
       return true;
     }
 
-    @java.lang.Override
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
-      if (executeTime_ != 0L) {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
         output.writeInt64(1, executeTime_);
       }
-      if (!getTransactionIdBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 2, transactionId_);
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getTransactionIdBytes());
       }
       for (int i = 0; i < props_.size(); i++) {
         output.writeMessage(3, props_.get(i));
       }
-      unknownFields.writeTo(output);
+      getUnknownFields().writeTo(output);
     }
 
-    @java.lang.Override
+    private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
-      int size = memoizedSize;
+      int size = memoizedSerializedSize;
       if (size != -1) return size;
 
       size = 0;
-      if (executeTime_ != 0L) {
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
           .computeInt64Size(1, executeTime_);
       }
-      if (!getTransactionIdBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, transactionId_);
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getTransactionIdBytes());
       }
       for (int i = 0; i < props_.size(); i++) {
         size += com.google.protobuf.CodedOutputStream
           .computeMessageSize(3, props_.get(i));
       }
-      size += unknownFields.getSerializedSize();
-      memoizedSize = size;
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
       return size;
     }
 
-    @java.lang.Override
-    public boolean equals(final java.lang.Object obj) {
-      if (obj == this) {
-       return true;
-      }
-      if (!(obj instanceof com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd)) {
-        return super.equals(obj);
-      }
-      com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd other = (com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd) obj;
-
-      boolean result = true;
-      result = result && (getExecuteTime()
-          == other.getExecuteTime());
-      result = result && getTransactionId()
-          .equals(other.getTransactionId());
-      result = result && getPropsList()
-          .equals(other.getPropsList());
-      result = result && unknownFields.equals(other.unknownFields);
-      return result;
+    private static final long serialVersionUID = 0L;
+    @Override
+    protected Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
     }
 
-    @java.lang.Override
-    public int hashCode() {
-      if (memoizedHashCode != 0) {
-        return memoizedHashCode;
-      }
-      int hash = 41;
-      hash = (19 * hash) + getDescriptor().hashCode();
-      hash = (37 * hash) + EXECUTETIME_FIELD_NUMBER;
-      hash = (53 * hash) + com.google.protobuf.Internal.hashLong(
-          getExecuteTime());
-      hash = (37 * hash) + TRANSACTIONID_FIELD_NUMBER;
-      hash = (53 * hash) + getTransactionId().hashCode();
-      if (getPropsCount() > 0) {
-        hash = (37 * hash) + PROPS_FIELD_NUMBER;
-        hash = (53 * hash) + getPropsList().hashCode();
-      }
-      hash = (29 * hash) + unknownFields.hashCode();
-      memoizedHashCode = hash;
-      return hash;
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
-        java.nio.ByteBuffer data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
-        java.nio.ByteBuffer data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
+    public static TransactionEnd parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
+    public static TransactionEnd parseFrom(
         com.google.protobuf.ByteString data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(byte[] data)
+    public static TransactionEnd parseFrom(byte[] data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
+    public static TransactionEnd parseFrom(
         byte[] data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(java.io.InputStream input)
+    public static TransactionEnd parseFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
+    public static TransactionEnd parseFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseDelimitedFrom(java.io.InputStream input)
+    public static TransactionEnd parseDelimitedFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input);
+      return PARSER.parseDelimitedFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseDelimitedFrom(
+    public static TransactionEnd parseDelimitedFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseDelimitedFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
+    public static TransactionEnd parseFrom(
         com.google.protobuf.CodedInputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
+    public static TransactionEnd parseFrom(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
 
-    @java.lang.Override
+    public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
-    public static Builder newBuilder() {
-      return DEFAULT_INSTANCE.toBuilder();
-    }
-    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd prototype) {
-      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
-    }
-    @java.lang.Override
-    public Builder toBuilder() {
-      return this == DEFAULT_INSTANCE
-          ? new Builder() : new Builder().mergeFrom(this);
+    public static Builder newBuilder(TransactionEnd prototype) {
+      return newBuilder().mergeFrom(prototype);
     }
+    public Builder toBuilder() { return newBuilder(this); }
 
-    @java.lang.Override
+    @Override
     protected Builder newBuilderForType(
-        com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        BuilderParent parent) {
       Builder builder = new Builder(parent);
       return builder;
     }
     /**
+     * Protobuf type {@code com.alibaba.otter.canal.protocol.TransactionEnd}
+     *
      * 
            **结束事务的一些信息*
      * 
      
-     *
-     * Protobuf type {@code com.alibaba.otter.canal.protocol.TransactionEnd}
      */
     public static final class Builder extends
-        com.google.protobuf.GeneratedMessageV3.Builder implements
+        com.google.protobuf.GeneratedMessage.Builder implements
         // @@protoc_insertion_point(builder_implements:com.alibaba.otter.canal.protocol.TransactionEnd)
-        com.alibaba.otter.canal.protocol.CanalEntry.TransactionEndOrBuilder {
+        TransactionEndOrBuilder {
       public static final com.google.protobuf.Descriptors.Descriptor
           getDescriptor() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor;
       }
 
-      @java.lang.Override
-      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      protected FieldAccessorTable
           internalGetFieldAccessorTable() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_fieldAccessorTable
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_fieldAccessorTable
             .ensureFieldAccessorsInitialized(
-                com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd.class, com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd.Builder.class);
+                TransactionEnd.class, Builder.class);
       }
 
       // Construct using com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd.newBuilder()
@@ -11842,23 +11414,25 @@ public final class CanalEntry {
       }
 
       private Builder(
-          com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
       private void maybeForceBuilderInitialization() {
-        if (com.google.protobuf.GeneratedMessageV3
-                .alwaysUseFieldBuilders) {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
           getPropsFieldBuilder();
         }
       }
-      @java.lang.Override
+      private static Builder create() {
+        return new Builder();
+      }
+
       public Builder clear() {
         super.clear();
         executeTime_ = 0L;
-
+        bitField0_ = (bitField0_ & ~0x00000001);
         transactionId_ = "";
-
+        bitField0_ = (bitField0_ & ~0x00000002);
         if (propsBuilder_ == null) {
           props_ = java.util.Collections.emptyList();
           bitField0_ = (bitField0_ & ~0x00000004);
@@ -11868,32 +11442,38 @@ public final class CanalEntry {
         return this;
       }
 
-      @java.lang.Override
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd getDefaultInstanceForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd.getDefaultInstance();
+      public TransactionEnd getDefaultInstanceForType() {
+        return TransactionEnd.getDefaultInstance();
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd build() {
-        com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd result = buildPartial();
+      public TransactionEnd build() {
+        TransactionEnd result = buildPartial();
         if (!result.isInitialized()) {
           throw newUninitializedMessageException(result);
         }
         return result;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd buildPartial() {
-        com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd result = new com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd(this);
+      public TransactionEnd buildPartial() {
+        TransactionEnd result = new TransactionEnd(this);
         int from_bitField0_ = bitField0_;
         int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
         result.executeTime_ = executeTime_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
         result.transactionId_ = transactionId_;
         if (propsBuilder_ == null) {
           if (((bitField0_ & 0x00000004) == 0x00000004)) {
@@ -11909,54 +11489,22 @@ public final class CanalEntry {
         return result;
       }
 
-      @java.lang.Override
-      public Builder clone() {
-        return (Builder) super.clone();
-      }
-      @java.lang.Override
-      public Builder setField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.setField(field, value);
-      }
-      @java.lang.Override
-      public Builder clearField(
-          com.google.protobuf.Descriptors.FieldDescriptor field) {
-        return (Builder) super.clearField(field);
-      }
-      @java.lang.Override
-      public Builder clearOneof(
-          com.google.protobuf.Descriptors.OneofDescriptor oneof) {
-        return (Builder) super.clearOneof(oneof);
-      }
-      @java.lang.Override
-      public Builder setRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          int index, java.lang.Object value) {
-        return (Builder) super.setRepeatedField(field, index, value);
-      }
-      @java.lang.Override
-      public Builder addRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.addRepeatedField(field, value);
-      }
-      @java.lang.Override
       public Builder mergeFrom(com.google.protobuf.Message other) {
-        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd) {
-          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd)other);
+        if (other instanceof TransactionEnd) {
+          return mergeFrom((TransactionEnd)other);
         } else {
           super.mergeFrom(other);
           return this;
         }
       }
 
-      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd other) {
-        if (other == com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd.getDefaultInstance()) return this;
-        if (other.getExecuteTime() != 0L) {
+      public Builder mergeFrom(TransactionEnd other) {
+        if (other == TransactionEnd.getDefaultInstance()) return this;
+        if (other.hasExecuteTime()) {
           setExecuteTime(other.getExecuteTime());
         }
-        if (!other.getTransactionId().isEmpty()) {
+        if (other.hasTransactionId()) {
+          bitField0_ |= 0x00000002;
           transactionId_ = other.transactionId_;
           onChanged();
         }
@@ -11978,35 +11526,32 @@ public final class CanalEntry {
               propsBuilder_ = null;
               props_ = other.props_;
               bitField0_ = (bitField0_ & ~0x00000004);
-              propsBuilder_ = 
-                com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ?
+              propsBuilder_ =
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
                    getPropsFieldBuilder() : null;
             } else {
               propsBuilder_.addAllMessages(other.props_);
             }
           }
         }
-        this.mergeUnknownFields(other.unknownFields);
-        onChanged();
+        this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
 
-      @java.lang.Override
       public final boolean isInitialized() {
         return true;
       }
 
-      @java.lang.Override
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
           throws java.io.IOException {
-        com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parsedMessage = null;
+        TransactionEnd parsedMessage = null;
         try {
           parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
         } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          parsedMessage = (com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd) e.getUnfinishedMessage();
-          throw e.unwrapIOException();
+          parsedMessage = (TransactionEnd) e.getUnfinishedMessage();
+          throw e;
         } finally {
           if (parsedMessage != null) {
             mergeFrom(parsedMessage);
@@ -12018,76 +11563,98 @@ public final class CanalEntry {
 
       private long executeTime_ ;
       /**
+       * optional int64 executeTime = 1;
+       *
        * 
              **已废弃，请使用header里的executeTime*
        * 
      
+       */
+      public boolean hasExecuteTime() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      /**
+       * optional int64 executeTime = 1;
        *
-       * int64 executeTime = 1;
+       * 
      +       **已废弃，请使用header里的executeTime*
+       * 
      
        */
       public long getExecuteTime() {
         return executeTime_;
       }
       /**
+       * optional int64 executeTime = 1;
+       *
        * 
              **已废弃，请使用header里的executeTime*
        * 
      
-       *
-       * int64 executeTime = 1;
        */
       public Builder setExecuteTime(long value) {
-        
+        bitField0_ |= 0x00000001;
         executeTime_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional int64 executeTime = 1;
+       *
        * 
              **已废弃，请使用header里的executeTime*
        * 
      
-       *
-       * int64 executeTime = 1;
        */
       public Builder clearExecuteTime() {
-        
+        bitField0_ = (bitField0_ & ~0x00000001);
         executeTime_ = 0L;
         onChanged();
         return this;
       }
 
-      private java.lang.Object transactionId_ = "";
+      private Object transactionId_ = "";
       /**
+       * optional string transactionId = 2;
+       *
        * 
              **事务号*
        * 
      
-       *
-       * string transactionId = 2;
        */
-      public java.lang.String getTransactionId() {
-        java.lang.Object ref = transactionId_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasTransactionId() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      /**
+       * optional string transactionId = 2;
+       *
+       * 
      +       **事务号*
+       * 
      
+       */
+      public String getTransactionId() {
+        Object ref = transactionId_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          transactionId_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            transactionId_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
+       * optional string transactionId = 2;
+       *
        * 
              **事务号*
        * 
      
-       *
-       * string transactionId = 2;
        */
       public com.google.protobuf.ByteString
           getTransactionIdBytes() {
-        java.lang.Object ref = transactionId_;
+        Object ref = transactionId_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           transactionId_ = b;
           return b;
         } else {
@@ -12095,74 +11662,73 @@ public final class CanalEntry {
         }
       }
       /**
+       * optional string transactionId = 2;
+       *
        * 
              **事务号*
        * 
      
-       *
-       * string transactionId = 2;
        */
       public Builder setTransactionId(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000002;
         transactionId_ = value;
         onChanged();
         return this;
       }
       /**
+       * optional string transactionId = 2;
+       *
        * 
              **事务号*
        * 
      
-       *
-       * string transactionId = 2;
        */
       public Builder clearTransactionId() {
-        
+        bitField0_ = (bitField0_ & ~0x00000002);
         transactionId_ = getDefaultInstance().getTransactionId();
         onChanged();
         return this;
       }
       /**
+       * optional string transactionId = 2;
+       *
        * 
              **事务号*
        * 
      
-       *
-       * string transactionId = 2;
        */
       public Builder setTransactionIdBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00000002;
         transactionId_ = value;
         onChanged();
         return this;
       }
 
-      private java.util.List props_ =
+      private java.util.List props_ =
         java.util.Collections.emptyList();
       private void ensurePropsIsMutable() {
         if (!((bitField0_ & 0x00000004) == 0x00000004)) {
-          props_ = new java.util.ArrayList(props_);
+          props_ = new java.util.ArrayList(props_);
           bitField0_ |= 0x00000004;
          }
       }
 
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> propsBuilder_;
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Pair, Pair.Builder, PairOrBuilder> propsBuilder_;
 
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public java.util.List getPropsList() {
+      public java.util.List getPropsList() {
         if (propsBuilder_ == null) {
           return java.util.Collections.unmodifiableList(props_);
         } else {
@@ -12170,11 +11736,11 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public int getPropsCount() {
         if (propsBuilder_ == null) {
@@ -12184,13 +11750,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+      public Pair getProps(int index) {
         if (propsBuilder_ == null) {
           return props_.get(index);
         } else {
@@ -12198,14 +11764,14 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder setProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+          int index, Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -12219,14 +11785,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder setProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          int index, Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.set(index, builderForValue.build());
@@ -12237,13 +11803,13 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public Builder addProps(com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+      public Builder addProps(Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -12257,14 +11823,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder addProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+          int index, Pair value) {
         if (propsBuilder_ == null) {
           if (value == null) {
             throw new NullPointerException();
@@ -12278,14 +11844,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder addProps(
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.add(builderForValue.build());
@@ -12296,14 +11862,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder addProps(
-          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+          int index, Pair.Builder builderForValue) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           props_.add(index, builderForValue.build());
@@ -12314,14 +11880,14 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder addAllProps(
-          java.lang.Iterable values) {
+          Iterable values) {
         if (propsBuilder_ == null) {
           ensurePropsIsMutable();
           com.google.protobuf.AbstractMessageLite.Builder.addAll(
@@ -12333,11 +11899,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder clearProps() {
         if (propsBuilder_ == null) {
@@ -12350,11 +11916,11 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
       public Builder removeProps(int index) {
         if (propsBuilder_ == null) {
@@ -12367,24 +11933,24 @@ public final class CanalEntry {
         return this;
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder getPropsBuilder(
+      public Pair.Builder getPropsBuilder(
           int index) {
         return getPropsFieldBuilder().getBuilder(index);
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+      public PairOrBuilder getPropsOrBuilder(
           int index) {
         if (propsBuilder_ == null) {
           return props_.get(index);  } else {
@@ -12392,13 +11958,13 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public java.util.List 
+      public java.util.List
            getPropsOrBuilderList() {
         if (propsBuilder_ != null) {
           return propsBuilder_.getMessageOrBuilderList();
@@ -12407,45 +11973,45 @@ public final class CanalEntry {
         }
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder() {
+      public Pair.Builder addPropsBuilder() {
         return getPropsFieldBuilder().addBuilder(
-            com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+            Pair.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder(
+      public Pair.Builder addPropsBuilder(
           int index) {
         return getPropsFieldBuilder().addBuilder(
-            index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+            index, Pair.getDefaultInstance());
       }
       /**
+       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+       *
        * 
              **预留扩展*
        * 
      
-       *
-       * repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
        */
-      public java.util.List 
+      public java.util.List
            getPropsBuilderList() {
         return getPropsFieldBuilder().getBuilderList();
       }
-      private com.google.protobuf.RepeatedFieldBuilderV3<
-          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> 
+      private com.google.protobuf.RepeatedFieldBuilder<
+          Pair, Pair.Builder, PairOrBuilder>
           getPropsFieldBuilder() {
         if (propsBuilder_ == null) {
-          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3<
-              com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder>(
+          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              Pair, Pair.Builder, PairOrBuilder>(
                   props_,
                   ((bitField0_ & 0x00000004) == 0x00000004),
                   getParentForChildren(),
@@ -12454,57 +12020,16 @@ public final class CanalEntry {
         }
         return propsBuilder_;
       }
-      @java.lang.Override
-      public final Builder setUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.setUnknownFieldsProto3(unknownFields);
-      }
-
-      @java.lang.Override
-      public final Builder mergeUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.mergeUnknownFields(unknownFields);
-      }
-
 
       // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.TransactionEnd)
     }
 
-    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.TransactionEnd)
-    private static final com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd DEFAULT_INSTANCE;
     static {
-      DEFAULT_INSTANCE = new com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd();
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd getDefaultInstance() {
-      return DEFAULT_INSTANCE;
-    }
-
-    private static final com.google.protobuf.Parser
-        PARSER = new com.google.protobuf.AbstractParser() {
-      @java.lang.Override
-      public TransactionEnd parsePartialFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return new TransactionEnd(input, extensionRegistry);
-      }
-    };
-
-    public static com.google.protobuf.Parser parser() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.google.protobuf.Parser getParserForType() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd getDefaultInstanceForType() {
-      return DEFAULT_INSTANCE;
+      defaultInstance = new TransactionEnd(true);
+      defaultInstance.initFields();
     }
 
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.TransactionEnd)
   }
 
   public interface PairOrBuilder extends
@@ -12512,59 +12037,71 @@ public final class CanalEntry {
       com.google.protobuf.MessageOrBuilder {
 
     /**
-     * string key = 1;
+     * optional string key = 1;
      */
-    java.lang.String getKey();
+    boolean hasKey();
     /**
-     * string key = 1;
+     * optional string key = 1;
+     */
+    String getKey();
+    /**
+     * optional string key = 1;
      */
     com.google.protobuf.ByteString
         getKeyBytes();
 
     /**
-     * string value = 2;
+     * optional string value = 2;
      */
-    java.lang.String getValue();
+    boolean hasValue();
     /**
-     * string value = 2;
+     * optional string value = 2;
+     */
+    String getValue();
+    /**
+     * optional string value = 2;
      */
     com.google.protobuf.ByteString
         getValueBytes();
   }
   /**
+   * Protobuf type {@code com.alibaba.otter.canal.protocol.Pair}
+   *
    * 
          **预留扩展*
    * 
      
-   *
-   * Protobuf type {@code com.alibaba.otter.canal.protocol.Pair}
    */
-  public  static final class Pair extends
-      com.google.protobuf.GeneratedMessageV3 implements
+  public static final class Pair extends
+      com.google.protobuf.GeneratedMessage implements
       // @@protoc_insertion_point(message_implements:com.alibaba.otter.canal.protocol.Pair)
       PairOrBuilder {
-  private static final long serialVersionUID = 0L;
     // Use Pair.newBuilder() to construct.
-    private Pair(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+    private Pair(com.google.protobuf.GeneratedMessage.Builder builder) {
       super(builder);
+      this.unknownFields = builder.getUnknownFields();
     }
-    private Pair() {
-      key_ = "";
-      value_ = "";
+    private Pair(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+    private static final Pair defaultInstance;
+    public static Pair getDefaultInstance() {
+      return defaultInstance;
     }
 
-    @java.lang.Override
+    public Pair getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+
+    private final com.google.protobuf.UnknownFieldSet unknownFields;
+    @Override
     public final com.google.protobuf.UnknownFieldSet
-    getUnknownFields() {
+        getUnknownFields() {
       return this.unknownFields;
     }
     private Pair(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
-      this();
-      if (extensionRegistry == null) {
-        throw new java.lang.NullPointerException();
-      }
+      initFields();
       int mutable_bitField0_ = 0;
       com.google.protobuf.UnknownFieldSet.Builder unknownFields =
           com.google.protobuf.UnknownFieldSet.newBuilder();
@@ -12576,23 +12113,23 @@ public final class CanalEntry {
             case 0:
               done = true;
               break;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                done = true;
+              }
+              break;
+            }
             case 10: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              key_ = s;
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000001;
+              key_ = bs;
               break;
             }
             case 18: {
-              java.lang.String s = input.readStringRequireUtf8();
-
-              value_ = s;
-              break;
-            }
-            default: {
-              if (!parseUnknownFieldProto3(
-                  input, unknownFields, extensionRegistry, tag)) {
-                done = true;
-              }
+              com.google.protobuf.ByteString bs = input.readBytes();
+              bitField0_ |= 0x00000002;
+              value_ = bs;
               break;
             }
           }
@@ -12601,7 +12138,7 @@ public final class CanalEntry {
         throw e.setUnfinishedMessage(this);
       } catch (java.io.IOException e) {
         throw new com.google.protobuf.InvalidProtocolBufferException(
-            e).setUnfinishedMessage(this);
+            e.getMessage()).setUnfinishedMessage(this);
       } finally {
         this.unknownFields = unknownFields.build();
         makeExtensionsImmutable();
@@ -12609,44 +12146,67 @@ public final class CanalEntry {
     }
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor;
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor;
     }
 
-    @java.lang.Override
-    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+    protected FieldAccessorTable
         internalGetFieldAccessorTable() {
-      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_fieldAccessorTable
+      return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_fieldAccessorTable
           .ensureFieldAccessorsInitialized(
-              com.alibaba.otter.canal.protocol.CanalEntry.Pair.class, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder.class);
+              Pair.class, Builder.class);
     }
 
+    public static com.google.protobuf.Parser PARSER =
+        new com.google.protobuf.AbstractParser() {
+      public Pair parsePartialFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return new Pair(input, extensionRegistry);
+      }
+    };
+
+    @Override
+    public com.google.protobuf.Parser getParserForType() {
+      return PARSER;
+    }
+
+    private int bitField0_;
     public static final int KEY_FIELD_NUMBER = 1;
-    private volatile java.lang.Object key_;
+    private Object key_;
     /**
-     * string key = 1;
+     * optional string key = 1;
      */
-    public java.lang.String getKey() {
-      java.lang.Object ref = key_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasKey() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    /**
+     * optional string key = 1;
+     */
+    public String getKey() {
+      Object ref = key_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        key_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          key_ = s;
+        }
         return s;
       }
     }
     /**
-     * string key = 1;
+     * optional string key = 1;
      */
     public com.google.protobuf.ByteString
         getKeyBytes() {
-      java.lang.Object ref = key_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = key_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         key_ = b;
         return b;
       } else {
@@ -12655,32 +12215,40 @@ public final class CanalEntry {
     }
 
     public static final int VALUE_FIELD_NUMBER = 2;
-    private volatile java.lang.Object value_;
+    private Object value_;
     /**
-     * string value = 2;
+     * optional string value = 2;
      */
-    public java.lang.String getValue() {
-      java.lang.Object ref = value_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
+    public boolean hasValue() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    /**
+     * optional string value = 2;
+     */
+    public String getValue() {
+      Object ref = value_;
+      if (ref instanceof String) {
+        return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs = 
+        com.google.protobuf.ByteString bs =
             (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        value_ = s;
+        String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          value_ = s;
+        }
         return s;
       }
     }
     /**
-     * string value = 2;
+     * optional string value = 2;
      */
     public com.google.protobuf.ByteString
         getValueBytes() {
-      java.lang.Object ref = value_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
+      Object ref = value_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b =
             com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
+                (String) ref);
         value_ = b;
         return b;
       } else {
@@ -12688,8 +12256,11 @@ public final class CanalEntry {
       }
     }
 
+    private void initFields() {
+      key_ = "";
+      value_ = "";
+    }
     private byte memoizedIsInitialized = -1;
-    @java.lang.Override
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized == 1) return true;
@@ -12699,182 +12270,131 @@ public final class CanalEntry {
       return true;
     }
 
-    @java.lang.Override
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
-      if (!getKeyBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 1, key_);
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getKeyBytes());
       }
-      if (!getValueBytes().isEmpty()) {
-        com.google.protobuf.GeneratedMessageV3.writeString(output, 2, value_);
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getValueBytes());
       }
-      unknownFields.writeTo(output);
+      getUnknownFields().writeTo(output);
     }
 
-    @java.lang.Override
+    private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
-      int size = memoizedSize;
+      int size = memoizedSerializedSize;
       if (size != -1) return size;
 
       size = 0;
-      if (!getKeyBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, key_);
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getKeyBytes());
       }
-      if (!getValueBytes().isEmpty()) {
-        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, value_);
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getValueBytes());
       }
-      size += unknownFields.getSerializedSize();
-      memoizedSize = size;
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
       return size;
     }
 
-    @java.lang.Override
-    public boolean equals(final java.lang.Object obj) {
-      if (obj == this) {
-       return true;
-      }
-      if (!(obj instanceof com.alibaba.otter.canal.protocol.CanalEntry.Pair)) {
-        return super.equals(obj);
-      }
-      com.alibaba.otter.canal.protocol.CanalEntry.Pair other = (com.alibaba.otter.canal.protocol.CanalEntry.Pair) obj;
-
-      boolean result = true;
-      result = result && getKey()
-          .equals(other.getKey());
-      result = result && getValue()
-          .equals(other.getValue());
-      result = result && unknownFields.equals(other.unknownFields);
-      return result;
+    private static final long serialVersionUID = 0L;
+    @Override
+    protected Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
     }
 
-    @java.lang.Override
-    public int hashCode() {
-      if (memoizedHashCode != 0) {
-        return memoizedHashCode;
-      }
-      int hash = 41;
-      hash = (19 * hash) + getDescriptor().hashCode();
-      hash = (37 * hash) + KEY_FIELD_NUMBER;
-      hash = (53 * hash) + getKey().hashCode();
-      hash = (37 * hash) + VALUE_FIELD_NUMBER;
-      hash = (53 * hash) + getValue().hashCode();
-      hash = (29 * hash) + unknownFields.hashCode();
-      memoizedHashCode = hash;
-      return hash;
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
-        java.nio.ByteBuffer data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
-        java.nio.ByteBuffer data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
+    public static Pair parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
+    public static Pair parseFrom(
         com.google.protobuf.ByteString data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(byte[] data)
+    public static Pair parseFrom(byte[] data)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
+    public static Pair parseFrom(
         byte[] data,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws com.google.protobuf.InvalidProtocolBufferException {
       return PARSER.parseFrom(data, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(java.io.InputStream input)
+    public static Pair parseFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
+    public static Pair parseFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseDelimitedFrom(java.io.InputStream input)
+    public static Pair parseDelimitedFrom(java.io.InputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input);
+      return PARSER.parseDelimitedFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseDelimitedFrom(
+    public static Pair parseDelimitedFrom(
         java.io.InputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseDelimitedFrom(input, extensionRegistry);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
+    public static Pair parseFrom(
         com.google.protobuf.CodedInputStream input)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input);
+      return PARSER.parseFrom(input);
     }
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
+    public static Pair parseFrom(
         com.google.protobuf.CodedInputStream input,
         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
         throws java.io.IOException {
-      return com.google.protobuf.GeneratedMessageV3
-          .parseWithIOException(PARSER, input, extensionRegistry);
+      return PARSER.parseFrom(input, extensionRegistry);
     }
 
-    @java.lang.Override
+    public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
-    public static Builder newBuilder() {
-      return DEFAULT_INSTANCE.toBuilder();
-    }
-    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.Pair prototype) {
-      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
-    }
-    @java.lang.Override
-    public Builder toBuilder() {
-      return this == DEFAULT_INSTANCE
-          ? new Builder() : new Builder().mergeFrom(this);
+    public static Builder newBuilder(Pair prototype) {
+      return newBuilder().mergeFrom(prototype);
     }
+    public Builder toBuilder() { return newBuilder(this); }
 
-    @java.lang.Override
+    @Override
     protected Builder newBuilderForType(
-        com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        BuilderParent parent) {
       Builder builder = new Builder(parent);
       return builder;
     }
     /**
+     * Protobuf type {@code com.alibaba.otter.canal.protocol.Pair}
+     *
      * 
            **预留扩展*
      * 
      
-     *
-     * Protobuf type {@code com.alibaba.otter.canal.protocol.Pair}
      */
     public static final class Builder extends
-        com.google.protobuf.GeneratedMessageV3.Builder implements
+        com.google.protobuf.GeneratedMessage.Builder implements
         // @@protoc_insertion_point(builder_implements:com.alibaba.otter.canal.protocol.Pair)
-        com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder {
+        PairOrBuilder {
       public static final com.google.protobuf.Descriptors.Descriptor
           getDescriptor() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor;
       }
 
-      @java.lang.Override
-      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      protected FieldAccessorTable
           internalGetFieldAccessorTable() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_fieldAccessorTable
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_fieldAccessorTable
             .ensureFieldAccessorsInitialized(
-                com.alibaba.otter.canal.protocol.CanalEntry.Pair.class, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder.class);
+                Pair.class, Builder.class);
       }
 
       // Construct using com.alibaba.otter.canal.protocol.CanalEntry.Pair.newBuilder()
@@ -12883,127 +12403,104 @@ public final class CanalEntry {
       }
 
       private Builder(
-          com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
       private void maybeForceBuilderInitialization() {
-        if (com.google.protobuf.GeneratedMessageV3
-                .alwaysUseFieldBuilders) {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
         }
       }
-      @java.lang.Override
+      private static Builder create() {
+        return new Builder();
+      }
+
       public Builder clear() {
         super.clear();
         key_ = "";
-
+        bitField0_ = (bitField0_ & ~0x00000001);
         value_ = "";
-
+        bitField0_ = (bitField0_ & ~0x00000002);
         return this;
       }
 
-      @java.lang.Override
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor;
+        return CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getDefaultInstanceForType() {
-        return com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance();
+      public Pair getDefaultInstanceForType() {
+        return Pair.getDefaultInstance();
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair build() {
-        com.alibaba.otter.canal.protocol.CanalEntry.Pair result = buildPartial();
+      public Pair build() {
+        Pair result = buildPartial();
         if (!result.isInitialized()) {
           throw newUninitializedMessageException(result);
         }
         return result;
       }
 
-      @java.lang.Override
-      public com.alibaba.otter.canal.protocol.CanalEntry.Pair buildPartial() {
-        com.alibaba.otter.canal.protocol.CanalEntry.Pair result = new com.alibaba.otter.canal.protocol.CanalEntry.Pair(this);
+      public Pair buildPartial() {
+        Pair result = new Pair(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
         result.key_ = key_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
         result.value_ = value_;
+        result.bitField0_ = to_bitField0_;
         onBuilt();
         return result;
       }
 
-      @java.lang.Override
-      public Builder clone() {
-        return (Builder) super.clone();
-      }
-      @java.lang.Override
-      public Builder setField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.setField(field, value);
-      }
-      @java.lang.Override
-      public Builder clearField(
-          com.google.protobuf.Descriptors.FieldDescriptor field) {
-        return (Builder) super.clearField(field);
-      }
-      @java.lang.Override
-      public Builder clearOneof(
-          com.google.protobuf.Descriptors.OneofDescriptor oneof) {
-        return (Builder) super.clearOneof(oneof);
-      }
-      @java.lang.Override
-      public Builder setRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          int index, java.lang.Object value) {
-        return (Builder) super.setRepeatedField(field, index, value);
-      }
-      @java.lang.Override
-      public Builder addRepeatedField(
-          com.google.protobuf.Descriptors.FieldDescriptor field,
-          java.lang.Object value) {
-        return (Builder) super.addRepeatedField(field, value);
-      }
-      @java.lang.Override
       public Builder mergeFrom(com.google.protobuf.Message other) {
-        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.Pair) {
-          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.Pair)other);
+        if (other instanceof Pair) {
+          return mergeFrom((Pair)other);
         } else {
           super.mergeFrom(other);
           return this;
         }
       }
 
-      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.Pair other) {
-        if (other == com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance()) return this;
-        if (!other.getKey().isEmpty()) {
+      public Builder mergeFrom(Pair other) {
+        if (other == Pair.getDefaultInstance()) return this;
+        if (other.hasKey()) {
+          bitField0_ |= 0x00000001;
           key_ = other.key_;
           onChanged();
         }
-        if (!other.getValue().isEmpty()) {
+        if (other.hasValue()) {
+          bitField0_ |= 0x00000002;
           value_ = other.value_;
           onChanged();
         }
-        this.mergeUnknownFields(other.unknownFields);
-        onChanged();
+        this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
 
-      @java.lang.Override
       public final boolean isInitialized() {
         return true;
       }
 
-      @java.lang.Override
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
           throws java.io.IOException {
-        com.alibaba.otter.canal.protocol.CanalEntry.Pair parsedMessage = null;
+        Pair parsedMessage = null;
         try {
           parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
         } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          parsedMessage = (com.alibaba.otter.canal.protocol.CanalEntry.Pair) e.getUnfinishedMessage();
-          throw e.unwrapIOException();
+          parsedMessage = (Pair) e.getUnfinishedMessage();
+          throw e;
         } finally {
           if (parsedMessage != null) {
             mergeFrom(parsedMessage);
@@ -13011,33 +12508,42 @@ public final class CanalEntry {
         }
         return this;
       }
+      private int bitField0_;
 
-      private java.lang.Object key_ = "";
+      private Object key_ = "";
       /**
-       * string key = 1;
+       * optional string key = 1;
        */
-      public java.lang.String getKey() {
-        java.lang.Object ref = key_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasKey() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      /**
+       * optional string key = 1;
+       */
+      public String getKey() {
+        Object ref = key_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          key_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            key_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
-       * string key = 1;
+       * optional string key = 1;
        */
       public com.google.protobuf.ByteString
           getKeyBytes() {
-        java.lang.Object ref = key_;
+        Object ref = key_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           key_ = b;
           return b;
         } else {
@@ -13045,68 +12551,75 @@ public final class CanalEntry {
         }
       }
       /**
-       * string key = 1;
+       * optional string key = 1;
        */
       public Builder setKey(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000001;
         key_ = value;
         onChanged();
         return this;
       }
       /**
-       * string key = 1;
+       * optional string key = 1;
        */
       public Builder clearKey() {
-        
+        bitField0_ = (bitField0_ & ~0x00000001);
         key_ = getDefaultInstance().getKey();
         onChanged();
         return this;
       }
       /**
-       * string key = 1;
+       * optional string key = 1;
        */
       public Builder setKeyBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00000001;
         key_ = value;
         onChanged();
         return this;
       }
 
-      private java.lang.Object value_ = "";
+      private Object value_ = "";
       /**
-       * string value = 2;
+       * optional string value = 2;
        */
-      public java.lang.String getValue() {
-        java.lang.Object ref = value_;
-        if (!(ref instanceof java.lang.String)) {
+      public boolean hasValue() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      /**
+       * optional string value = 2;
+       */
+      public String getValue() {
+        Object ref = value_;
+        if (!(ref instanceof String)) {
           com.google.protobuf.ByteString bs =
               (com.google.protobuf.ByteString) ref;
-          java.lang.String s = bs.toStringUtf8();
-          value_ = s;
+          String s = bs.toStringUtf8();
+          if (bs.isValidUtf8()) {
+            value_ = s;
+          }
           return s;
         } else {
-          return (java.lang.String) ref;
+          return (String) ref;
         }
       }
       /**
-       * string value = 2;
+       * optional string value = 2;
        */
       public com.google.protobuf.ByteString
           getValueBytes() {
-        java.lang.Object ref = value_;
+        Object ref = value_;
         if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString b =
               com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
+                  (String) ref);
           value_ = b;
           return b;
         } else {
@@ -13114,197 +12627,151 @@ public final class CanalEntry {
         }
       }
       /**
-       * string value = 2;
+       * optional string value = 2;
        */
       public Builder setValue(
-          java.lang.String value) {
+          String value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  
+  bitField0_ |= 0x00000002;
         value_ = value;
         onChanged();
         return this;
       }
       /**
-       * string value = 2;
+       * optional string value = 2;
        */
       public Builder clearValue() {
-        
+        bitField0_ = (bitField0_ & ~0x00000002);
         value_ = getDefaultInstance().getValue();
         onChanged();
         return this;
       }
       /**
-       * string value = 2;
+       * optional string value = 2;
        */
       public Builder setValueBytes(
           com.google.protobuf.ByteString value) {
         if (value == null) {
     throw new NullPointerException();
   }
-  checkByteStringIsUtf8(value);
-        
+  bitField0_ |= 0x00000002;
         value_ = value;
         onChanged();
         return this;
       }
-      @java.lang.Override
-      public final Builder setUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.setUnknownFieldsProto3(unknownFields);
-      }
-
-      @java.lang.Override
-      public final Builder mergeUnknownFields(
-          final com.google.protobuf.UnknownFieldSet unknownFields) {
-        return super.mergeUnknownFields(unknownFields);
-      }
-
 
       // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Pair)
     }
 
-    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Pair)
-    private static final com.alibaba.otter.canal.protocol.CanalEntry.Pair DEFAULT_INSTANCE;
     static {
-      DEFAULT_INSTANCE = new com.alibaba.otter.canal.protocol.CanalEntry.Pair();
-    }
-
-    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair getDefaultInstance() {
-      return DEFAULT_INSTANCE;
-    }
-
-    private static final com.google.protobuf.Parser
-        PARSER = new com.google.protobuf.AbstractParser() {
-      @java.lang.Override
-      public Pair parsePartialFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return new Pair(input, extensionRegistry);
-      }
-    };
-
-    public static com.google.protobuf.Parser parser() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.google.protobuf.Parser getParserForType() {
-      return PARSER;
-    }
-
-    @java.lang.Override
-    public com.alibaba.otter.canal.protocol.CanalEntry.Pair getDefaultInstanceForType() {
-      return DEFAULT_INSTANCE;
+      defaultInstance = new Pair(true);
+      defaultInstance.initFields();
     }
 
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Pair)
   }
 
   private static final com.google.protobuf.Descriptors.Descriptor
     internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor;
-  private static final 
-    com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
       internal_static_com_alibaba_otter_canal_protocol_Entry_fieldAccessorTable;
   private static final com.google.protobuf.Descriptors.Descriptor
     internal_static_com_alibaba_otter_canal_protocol_Header_descriptor;
-  private static final 
-    com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
       internal_static_com_alibaba_otter_canal_protocol_Header_fieldAccessorTable;
   private static final com.google.protobuf.Descriptors.Descriptor
     internal_static_com_alibaba_otter_canal_protocol_Column_descriptor;
-  private static final 
-    com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
       internal_static_com_alibaba_otter_canal_protocol_Column_fieldAccessorTable;
   private static final com.google.protobuf.Descriptors.Descriptor
     internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor;
-  private static final 
-    com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
       internal_static_com_alibaba_otter_canal_protocol_RowData_fieldAccessorTable;
   private static final com.google.protobuf.Descriptors.Descriptor
     internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor;
-  private static final 
-    com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
       internal_static_com_alibaba_otter_canal_protocol_RowChange_fieldAccessorTable;
   private static final com.google.protobuf.Descriptors.Descriptor
     internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor;
-  private static final 
-    com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
       internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_fieldAccessorTable;
   private static final com.google.protobuf.Descriptors.Descriptor
     internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor;
-  private static final 
-    com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
       internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_fieldAccessorTable;
   private static final com.google.protobuf.Descriptors.Descriptor
     internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor;
-  private static final 
-    com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
       internal_static_com_alibaba_otter_canal_protocol_Pair_fieldAccessorTable;
 
   public static com.google.protobuf.Descriptors.FileDescriptor
       getDescriptor() {
     return descriptor;
   }
-  private static  com.google.protobuf.Descriptors.FileDescriptor
+  private static com.google.protobuf.Descriptors.FileDescriptor
       descriptor;
   static {
-    java.lang.String[] descriptorData = {
+    String[] descriptorData = {
       "\n\023EntryProtocol.proto\022 com.alibaba.otter" +
-      ".canal.protocol\"\254\001\n\005Entry\0228\n\006header\030\001 \001(" +
+      ".canal.protocol\"\236\001\n\005Entry\0228\n\006header\030\001 \001(" +
       "\0132(.com.alibaba.otter.canal.protocol.Hea" +
-      "der\022@\n\tentryType\030\002 \001(\0162+.com.alibaba.ott" +
-      "er.canal.protocol.EntryTypeH\000\022\022\n\nstoreVa" +
-      "lue\030\003 \001(\014B\023\n\021entryType_present\"\303\003\n\006Heade" +
-      "r\022\021\n\007version\030\001 \001(\005H\000\022\023\n\013logfileName\030\002 \001(" +
-      "\t\022\025\n\rlogfileOffset\030\003 \001(\003\022\020\n\010serverId\030\004 \001" +
-      "(\003\022\024\n\014serverenCode\030\005 \001(\t\022\023\n\013executeTime\030" +
-      "\006 \001(\003\022<\n\nsourceType\030\007 \001(\0162&.com.alibaba." +
-      "otter.canal.protocol.TypeH\001\022\022\n\nschemaNam" +
-      "e\030\010 \001(\t\022\021\n\ttableName\030\t \001(\t\022\023\n\013eventLengt" +
-      "h\030\n \001(\003\022@\n\teventType\030\013 \001(\0162+.com.alibaba" +
-      ".otter.canal.protocol.EventTypeH\002\0225\n\005pro" +
-      "ps\030\014 \003(\0132&.com.alibaba.otter.canal.proto" +
-      "col.Pair\022\014\n\004gtid\030\r \001(\tB\021\n\017version_presen" +
-      "tB\024\n\022sourceType_presentB\023\n\021eventType_pre" +
-      "sent\"\343\001\n\006Column\022\r\n\005index\030\001 \001(\005\022\017\n\007sqlTyp" +
-      "e\030\002 \001(\005\022\014\n\004name\030\003 \001(\t\022\r\n\005isKey\030\004 \001(\010\022\017\n\007" +
-      "updated\030\005 \001(\010\022\020\n\006isNull\030\006 \001(\010H\000\0225\n\005props" +
-      "\030\007 \003(\0132&.com.alibaba.otter.canal.protoco" +
-      "l.Pair\022\r\n\005value\030\010 \001(\t\022\016\n\006length\030\t \001(\005\022\021\n" +
-      "\tmysqlType\030\n \001(\tB\020\n\016isNull_present\"\301\001\n\007R" +
-      "owData\022?\n\rbeforeColumns\030\001 \003(\0132(.com.alib" +
-      "aba.otter.canal.protocol.Column\022>\n\014after" +
-      "Columns\030\002 \003(\0132(.com.alibaba.otter.canal." +
-      "protocol.Column\0225\n\005props\030\003 \003(\0132&.com.ali" +
-      "baba.otter.canal.protocol.Pair\"\255\002\n\tRowCh" +
-      "ange\022\017\n\007tableId\030\001 \001(\003\022@\n\teventType\030\002 \001(\016" +
-      "2+.com.alibaba.otter.canal.protocol.Even" +
-      "tTypeH\000\022\017\n\005isDdl\030\n \001(\010H\001\022\013\n\003sql\030\013 \001(\t\022;\n" +
-      "\010rowDatas\030\014 \003(\0132).com.alibaba.otter.cana" +
-      "l.protocol.RowData\0225\n\005props\030\r \003(\0132&.com." +
-      "alibaba.otter.canal.protocol.Pair\022\025\n\rddl" +
-      "SchemaName\030\016 \001(\tB\023\n\021eventType_presentB\017\n" +
-      "\risDdl_present\"\207\001\n\020TransactionBegin\022\023\n\013e" +
-      "xecuteTime\030\001 \001(\003\022\025\n\rtransactionId\030\002 \001(\t\022" +
-      "5\n\005props\030\003 \003(\0132&.com.alibaba.otter.canal" +
-      ".protocol.Pair\022\020\n\010threadId\030\004 \001(\003\"s\n\016Tran" +
-      "sactionEnd\022\023\n\013executeTime\030\001 \001(\003\022\025\n\rtrans" +
-      "actionId\030\002 \001(\t\0225\n\005props\030\003 \003(\0132&.com.alib" +
-      "aba.otter.canal.protocol.Pair\"\"\n\004Pair\022\013\n" +
-      "\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\t*^\n\tEntryType\022\024" +
-      "\n\020TRANSACTIONBEGIN\020\000\022\013\n\007ROWDATA\020\001\022\022\n\016TRA" +
-      "NSACTIONEND\020\002\022\r\n\tHEARTBEAT\020\003\022\013\n\007GTIDLOG\020" +
-      "\004*\345\001\n\tEventType\022\035\n\031EVENTTYPECOMPATIBLEPR" +
-      "OTO2\020\000\022\n\n\006INSERT\020\001\022\n\n\006UPDATE\020\002\022\n\n\006DELETE" +
-      "\020\003\022\n\n\006CREATE\020\004\022\t\n\005ALTER\020\005\022\t\n\005ERASE\020\006\022\t\n\005" +
-      "QUERY\020\007\022\014\n\010TRUNCATE\020\010\022\n\n\006RENAME\020\t\022\n\n\006CIN" +
-      "DEX\020\n\022\n\n\006DINDEX\020\013\022\010\n\004GTID\020\014\022\014\n\010XACOMMIT\020" +
-      "\r\022\016\n\nXAROLLBACK\020\016\022\016\n\nMHEARTBEAT\020\017*B\n\004Typ" +
-      "e\022\030\n\024TYPECOMPATIBLEPROTO2\020\000\022\n\n\006ORACLE\020\001\022" +
-      "\t\n\005MYSQL\020\002\022\t\n\005PGSQL\020\003B0\n com.alibaba.ott" +
-      "er.canal.protocolB\nCanalEntryH\001b\006proto3"
+      "der\022G\n\tentryType\030\002 \001(\0162+.com.alibaba.ott" +
+      "er.canal.protocol.EntryType:\007ROWDATA\022\022\n\n" +
+      "storeValue\030\003 \001(\014\"\221\003\n\006Header\022\022\n\007version\030\001" +
+      " \001(\005:\0011\022\023\n\013logfileName\030\002 \001(\t\022\025\n\rlogfileO" +
+      "ffset\030\003 \001(\003\022\020\n\010serverId\030\004 \001(\003\022\024\n\014servere" +
+      "nCode\030\005 \001(\t\022\023\n\013executeTime\030\006 \001(\003\022A\n\nsour" +
+      "ceType\030\007 \001(\0162&.com.alibaba.otter.canal.p",
+      "rotocol.Type:\005MYSQL\022\022\n\nschemaName\030\010 \001(\t\022" +
+      "\021\n\ttableName\030\t \001(\t\022\023\n\013eventLength\030\n \001(\003\022" +
+      "F\n\teventType\030\013 \001(\0162+.com.alibaba.otter.c" +
+      "anal.protocol.EventType:\006UPDATE\0225\n\005props" +
+      "\030\014 \003(\0132&.com.alibaba.otter.canal.protoco" +
+      "l.Pair\022\014\n\004gtid\030\r \001(\t\"\326\001\n\006Column\022\r\n\005index" +
+      "\030\001 \001(\005\022\017\n\007sqlType\030\002 \001(\005\022\014\n\004name\030\003 \001(\t\022\r\n" +
+      "\005isKey\030\004 \001(\010\022\017\n\007updated\030\005 \001(\010\022\025\n\006isNull\030" +
+      "\006 \001(\010:\005false\0225\n\005props\030\007 \003(\0132&.com.alibab" +
+      "a.otter.canal.protocol.Pair\022\r\n\005value\030\010 \001",
+      "(\t\022\016\n\006length\030\t \001(\005\022\021\n\tmysqlType\030\n \001(\t\"\301\001" +
+      "\n\007RowData\022?\n\rbeforeColumns\030\001 \003(\0132(.com.a" +
+      "libaba.otter.canal.protocol.Column\022>\n\014af" +
+      "terColumns\030\002 \003(\0132(.com.alibaba.otter.can" +
+      "al.protocol.Column\0225\n\005props\030\003 \003(\0132&.com." +
+      "alibaba.otter.canal.protocol.Pair\"\222\002\n\tRo" +
+      "wChange\022\017\n\007tableId\030\001 \001(\003\022F\n\teventType\030\002 " +
+      "\001(\0162+.com.alibaba.otter.canal.protocol.E" +
+      "ventType:\006UPDATE\022\024\n\005isDdl\030\n \001(\010:\005false\022\013" +
+      "\n\003sql\030\013 \001(\t\022;\n\010rowDatas\030\014 \003(\0132).com.alib",
+      "aba.otter.canal.protocol.RowData\0225\n\005prop" +
+      "s\030\r \003(\0132&.com.alibaba.otter.canal.protoc" +
+      "ol.Pair\022\025\n\rddlSchemaName\030\016 \001(\t\"\207\001\n\020Trans" +
+      "actionBegin\022\023\n\013executeTime\030\001 \001(\003\022\025\n\rtran" +
+      "sactionId\030\002 \001(\t\0225\n\005props\030\003 \003(\0132&.com.ali" +
+      "baba.otter.canal.protocol.Pair\022\020\n\010thread" +
+      "Id\030\004 \001(\003\"s\n\016TransactionEnd\022\023\n\013executeTim" +
+      "e\030\001 \001(\003\022\025\n\rtransactionId\030\002 \001(\t\0225\n\005props\030" +
+      "\003 \003(\0132&.com.alibaba.otter.canal.protocol" +
+      ".Pair\"\"\n\004Pair\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(",
+      "\t*^\n\tEntryType\022\024\n\020TRANSACTIONBEGIN\020\001\022\013\n\007" +
+      "ROWDATA\020\002\022\022\n\016TRANSACTIONEND\020\003\022\r\n\tHEARTBE" +
+      "AT\020\004\022\013\n\007GTIDLOG\020\005*\306\001\n\tEventType\022\n\n\006INSER" +
+      "T\020\001\022\n\n\006UPDATE\020\002\022\n\n\006DELETE\020\003\022\n\n\006CREATE\020\004\022" +
+      "\t\n\005ALTER\020\005\022\t\n\005ERASE\020\006\022\t\n\005QUERY\020\007\022\014\n\010TRUN" +
+      "CATE\020\010\022\n\n\006RENAME\020\t\022\n\n\006CINDEX\020\n\022\n\n\006DINDEX" +
+      "\020\013\022\010\n\004GTID\020\014\022\014\n\010XACOMMIT\020\r\022\016\n\nXAROLLBACK" +
+      "\020\016\022\016\n\nMHEARTBEAT\020\017*(\n\004Type\022\n\n\006ORACLE\020\001\022\t" +
+      "\n\005MYSQL\020\002\022\t\n\005PGSQL\020\003B0\n com.alibaba.otte" +
+      "r.canal.protocolB\nCanalEntryH\001"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
         new com.google.protobuf.Descriptors.FileDescriptor.    InternalDescriptorAssigner() {
@@ -13321,51 +12788,51 @@ public final class CanalEntry {
     internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor =
       getDescriptor().getMessageTypes().get(0);
     internal_static_com_alibaba_otter_canal_protocol_Entry_fieldAccessorTable = new
-      com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+      com.google.protobuf.GeneratedMessage.FieldAccessorTable(
         internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor,
-        new java.lang.String[] { "Header", "EntryType", "StoreValue", "EntryTypePresent", });
+        new String[] { "Header", "EntryType", "StoreValue", });
     internal_static_com_alibaba_otter_canal_protocol_Header_descriptor =
       getDescriptor().getMessageTypes().get(1);
     internal_static_com_alibaba_otter_canal_protocol_Header_fieldAccessorTable = new
-      com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+      com.google.protobuf.GeneratedMessage.FieldAccessorTable(
         internal_static_com_alibaba_otter_canal_protocol_Header_descriptor,
-        new java.lang.String[] { "Version", "LogfileName", "LogfileOffset", "ServerId", "ServerenCode", "ExecuteTime", "SourceType", "SchemaName", "TableName", "EventLength", "EventType", "Props", "Gtid", "VersionPresent", "SourceTypePresent", "EventTypePresent", });
+        new String[] { "Version", "LogfileName", "LogfileOffset", "ServerId", "ServerenCode", "ExecuteTime", "SourceType", "SchemaName", "TableName", "EventLength", "EventType", "Props", "Gtid", });
     internal_static_com_alibaba_otter_canal_protocol_Column_descriptor =
       getDescriptor().getMessageTypes().get(2);
     internal_static_com_alibaba_otter_canal_protocol_Column_fieldAccessorTable = new
-      com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+      com.google.protobuf.GeneratedMessage.FieldAccessorTable(
         internal_static_com_alibaba_otter_canal_protocol_Column_descriptor,
-        new java.lang.String[] { "Index", "SqlType", "Name", "IsKey", "Updated", "IsNull", "Props", "Value", "Length", "MysqlType", "IsNullPresent", });
+        new String[] { "Index", "SqlType", "Name", "IsKey", "Updated", "IsNull", "Props", "Value", "Length", "MysqlType", });
     internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor =
       getDescriptor().getMessageTypes().get(3);
     internal_static_com_alibaba_otter_canal_protocol_RowData_fieldAccessorTable = new
-      com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+      com.google.protobuf.GeneratedMessage.FieldAccessorTable(
         internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor,
-        new java.lang.String[] { "BeforeColumns", "AfterColumns", "Props", });
+        new String[] { "BeforeColumns", "AfterColumns", "Props", });
     internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor =
       getDescriptor().getMessageTypes().get(4);
     internal_static_com_alibaba_otter_canal_protocol_RowChange_fieldAccessorTable = new
-      com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+      com.google.protobuf.GeneratedMessage.FieldAccessorTable(
         internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor,
-        new java.lang.String[] { "TableId", "EventType", "IsDdl", "Sql", "RowDatas", "Props", "DdlSchemaName", "EventTypePresent", "IsDdlPresent", });
+        new String[] { "TableId", "EventType", "IsDdl", "Sql", "RowDatas", "Props", "DdlSchemaName", });
     internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor =
       getDescriptor().getMessageTypes().get(5);
     internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_fieldAccessorTable = new
-      com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+      com.google.protobuf.GeneratedMessage.FieldAccessorTable(
         internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor,
-        new java.lang.String[] { "ExecuteTime", "TransactionId", "Props", "ThreadId", });
+        new String[] { "ExecuteTime", "TransactionId", "Props", "ThreadId", });
     internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor =
       getDescriptor().getMessageTypes().get(6);
     internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_fieldAccessorTable = new
-      com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+      com.google.protobuf.GeneratedMessage.FieldAccessorTable(
         internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor,
-        new java.lang.String[] { "ExecuteTime", "TransactionId", "Props", });
+        new String[] { "ExecuteTime", "TransactionId", "Props", });
     internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor =
       getDescriptor().getMessageTypes().get(7);
     internal_static_com_alibaba_otter_canal_protocol_Pair_fieldAccessorTable = new
-      com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+      com.google.protobuf.GeneratedMessage.FieldAccessorTable(
         internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor,
-        new java.lang.String[] { "Key", "Value", });
+        new String[] { "Key", "Value", });
   }
 
   // @@protoc_insertion_point(outer_class_scope)
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/ClientIdentity.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/ClientIdentity.java
index aaa568e0..bae04a79 100644
--- a/protocol/src/main/java/com/alibaba/otter/canal/protocol/ClientIdentity.java
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/ClientIdentity.java
@@ -13,9 +13,10 @@ import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
  */
 public class ClientIdentity implements Serializable {
 
-    private String destination;
-    private short  clientId;
-    private String filter;
+    private static final long serialVersionUID = -8262100681930834834L;
+    private String            destination;
+    private short             clientId;
+    private String            filter;
 
     public ClientIdentity(){
 
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java
index e6aab531..2b958b1c 100644
--- a/protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java
@@ -1,10 +1,10 @@
 package com.alibaba.otter.canal.protocol;
 
 import java.io.Serializable;
-import java.util.*;
+import java.util.List;
+import java.util.Map;
 
-import com.google.common.collect.Table;
-import com.google.protobuf.ByteString;
+import com.google.common.collect.Lists;
 
 /**
  * @author machengyuan 2018-9-13 下午10:31:14
@@ -13,12 +13,15 @@ import com.google.protobuf.ByteString;
 public class FlatMessage implements Serializable {
 
     private static final long         serialVersionUID = -3386650678735860050L;
-
     private long                      id;
     private String                    database;
     private String                    table;
+    private List              pkNames;
     private Boolean                   isDdl;
     private String                    type;
+    // binlog executeTime
+    private Long                      es;
+    // dml build timeStamp
     private Long                      ts;
     private String                    sql;
     private Map      sqlType;
@@ -57,6 +60,21 @@ public class FlatMessage implements Serializable {
         this.table = table;
     }
 
+    public List getPkNames() {
+        return pkNames;
+    }
+
+    public void addPkName(String pkName) {
+        if (this.pkNames == null) {
+            this.pkNames = Lists.newArrayList();
+        }
+        this.pkNames.add(pkName);
+    }
+
+    public void setPkNames(List pkNames) {
+        this.pkNames = pkNames;
+    }
+
     public Boolean getIsDdl() {
         return isDdl;
     }
@@ -121,166 +139,18 @@ public class FlatMessage implements Serializable {
         this.old = old;
     }
 
-    public static List messageConverter(Message message) {
-        try {
-            if (message == null) {
-                return null;
-            }
-
-            List flatMessages = new ArrayList<>();
-
-            List rawEntries = message.getRawEntries();
-
-            for (ByteString byteString : rawEntries) {
-                CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString);
-                if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
-                    || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
-                    continue;
-                }
-
-                CanalEntry.RowChange rowChange;
-                try {
-                    rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
-                } catch (Exception e) {
-                    throw new RuntimeException(
-                        "ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
-                        e);
-                }
-
-                CanalEntry.EventType eventType = rowChange.getEventType();
-
-                FlatMessage flatMessage = new FlatMessage(message.getId());
-                flatMessages.add(flatMessage);
-                flatMessage.setDatabase(entry.getHeader().getSchemaName());
-                flatMessage.setTable(entry.getHeader().getTableName());
-                flatMessage.setIsDdl(rowChange.getIsDdl());
-                flatMessage.setType(eventType.toString());
-                flatMessage.setTs(System.currentTimeMillis());
-                flatMessage.setSql(rowChange.getSql());
-
-                if (!rowChange.getIsDdl()) {
-                    Map sqlType = new LinkedHashMap<>();
-                    Map mysqlType = new LinkedHashMap<>();
-                    List> data = new ArrayList<>();
-                    List> old = new ArrayList<>();
-
-                    Set updateSet = new HashSet<>();
-                    for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
-                        if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
-                            && eventType != CanalEntry.EventType.DELETE) {
-                            continue;
-                        }
-
-                        Map row = new LinkedHashMap<>();
-                        List columns;
-
-                        if (eventType == CanalEntry.EventType.DELETE) {
-                            columns = rowData.getBeforeColumnsList();
-                        } else {
-                            columns = rowData.getAfterColumnsList();
-                        }
-
-                        for (CanalEntry.Column column : columns) {
-                            sqlType.put(column.getName(), column.getSqlType());
-                            mysqlType.put(column.getName(), column.getMysqlType());
-                            row.put(column.getName(), column.getValue());
-                            // 获取update为true的字段
-                            if (column.getUpdated()) {
-                                updateSet.add(column.getName());
-                            }
-                        }
-                        if (!row.isEmpty()) {
-                            data.add(row);
-                        }
-
-                        if (eventType == CanalEntry.EventType.UPDATE) {
-                            Map rowOld = new LinkedHashMap<>();
-                            for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
-                                if (updateSet.contains(column.getName())) {
-                                    rowOld.put(column.getName(), column.getValue());
-                                }
-                            }
-                            // update操作将记录修改前的值
-                            if (!rowOld.isEmpty()) {
-                                old.add(rowOld);
-                            }
-                        }
-                    }
-                    if (!sqlType.isEmpty()) {
-                        flatMessage.setSqlType(sqlType);
-                    }
-                    if (!mysqlType.isEmpty()) {
-                        flatMessage.setMysqlType(mysqlType);
-                    }
-                    if (!data.isEmpty()) {
-                        flatMessage.setData(data);
-                    }
-                    if (!old.isEmpty()) {
-                        flatMessage.setOld(old);
-                    }
-                }
-            }
-            return flatMessages;
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
+    public Long getEs() {
+        return es;
     }
 
-    public static FlatMessage[] messagePartition(FlatMessage flatMessage, Integer partitionsNum,
-                                                 Table pkHashConfig) {
-        FlatMessage[] partitionMessages = new FlatMessage[partitionsNum];
-
-        String pk = pkHashConfig.get(flatMessage.getDatabase(), flatMessage.getTable());
-        if (pk == null || flatMessage.getIsDdl()) {
-            partitionMessages[0] = flatMessage;
-        } else {
-            if (flatMessage.getData() != null) {
-                int idx = 0;
-                for (Map row : flatMessage.getData()) {
-                    String value = row.get(pk);
-                    if (value == null) {
-                        value = "";
-                    }
-                    int hash = value.hashCode();
-                    int pkHash = Math.abs(hash) % partitionsNum;
-
-                    FlatMessage flatMessageTmp = partitionMessages[pkHash];
-                    if (flatMessageTmp == null) {
-                        flatMessageTmp = new FlatMessage(flatMessage.getId());
-                        partitionMessages[pkHash] = flatMessageTmp;
-                        flatMessageTmp.setDatabase(flatMessage.getDatabase());
-                        flatMessageTmp.setTable(flatMessage.getTable());
-                        flatMessageTmp.setIsDdl(flatMessage.getIsDdl());
-                        flatMessageTmp.setType(flatMessage.getType());
-                        flatMessageTmp.setSql(flatMessage.getSql());
-                        flatMessageTmp.setSqlType(flatMessage.getSqlType());
-                        flatMessageTmp.setMysqlType(flatMessage.getMysqlType());
-                    }
-                    List> data = flatMessageTmp.getData();
-                    if (data == null) {
-                        data = new ArrayList<>();
-                        flatMessageTmp.setData(data);
-                    }
-                    data.add(row);
-                    if (flatMessage.getOld() != null && !flatMessage.getOld().isEmpty()) {
-                        List> old = flatMessageTmp.getOld();
-                        if (old == null) {
-                            old = new ArrayList<>();
-                            flatMessageTmp.setOld(old);
-                        }
-                        old.add(flatMessage.getOld().get(idx));
-                    }
-                    idx++;
-                }
-            }
-        }
-        return partitionMessages;
+    public void setEs(Long es) {
+        this.es = es;
     }
 
     @Override
     public String toString() {
-        return "FlatMessage{" + "id=" + id + ", database='" + database + '\'' + ", table='" + table + '\'' + ", isDdl="
-               + isDdl + ", type='" + type + '\'' + ", ts=" + ts + ", sql='" + sql + '\'' + ", sqlType=" + sqlType
-               + ", mysqlType=" + mysqlType + ", data=" + data + ", old=" + old + '}';
+        return "FlatMessage [id=" + id + ", database=" + database + ", table=" + table + ", isDdl=" + isDdl + ", type="
+               + type + ", es=" + es + ", ts=" + ts + ", sql=" + sql + ", sqlType=" + sqlType + ", mysqlType="
+               + mysqlType + ", data=" + data + ", old=" + old + "]";
     }
 }
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java
index 35adf15d..3fa691ad 100644
--- a/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java
@@ -17,9 +17,7 @@ import com.google.protobuf.ByteString;
 public class Message implements Serializable {
 
     private static final long      serialVersionUID = 1234034768477580009L;
-
     private long                   id;
-    @Deprecated
     private List entries          = new ArrayList();
     // row data for performance, see:
     // https://github.com/alibaba/canal/issues/726
@@ -39,6 +37,7 @@ public class Message implements Serializable {
         } else {
             this.entries = entries == null ? new ArrayList() : entries;
         }
+        this.raw = raw;
     }
 
     public Message(long id){
diff --git a/server/pom.xml b/server/pom.xml
index a2eb437d..709a14e0 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -3,7 +3,7 @@
 	
 		com.alibaba.otter
 		canal
-		1.1.1-SNAPSHOT
+		1.1.3-SNAPSHOT
 		../pom.xml
 	
 	canal.server
@@ -26,11 +26,6 @@
 			${project.version}
 		
 		
-		
-			org.yaml
-			snakeyaml
-			1.17
-		
 		
 			org.apache.kafka
 			kafka_2.11
@@ -42,6 +37,10 @@
 				
 			
 		
+		
+			org.apache.rocketmq
+			rocketmq-client
+		
 		
 		
 			org.jboss.netty
@@ -55,5 +54,9 @@
 			junit
 			test
 		
+		
+			com.aliyun.openservices
+			aliware-apache-rocketmq-cloud
+		
 	
 
diff --git a/server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java b/server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java
new file mode 100644
index 00000000..fda5dc65
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java
@@ -0,0 +1,79 @@
+package com.alibaba.otter.canal.common;
+
+import java.util.List;
+
+import org.apache.kafka.common.errors.SerializationException;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalPacket;
+import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
+import com.alibaba.otter.canal.protocol.Message;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.WireFormat;
+
+public class CanalMessageSerializer {
+
+    @SuppressWarnings("deprecation")
+    public static byte[] serializer(Message data, boolean filterTransactionEntry) {
+        try {
+            if (data != null) {
+                if (data.getId() != -1) {
+                    if (data.isRaw() && !CollectionUtils.isEmpty(data.getRawEntries())) {
+                        // for performance
+                        List rowEntries = data.getRawEntries();
+                        // message size
+                        int messageSize = 0;
+                        messageSize += CodedOutputStream.computeInt64Size(1, data.getId());
+
+                        int dataSize = 0;
+                        for (int i = 0; i < rowEntries.size(); i++) {
+                            dataSize += CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i));
+                        }
+                        messageSize += dataSize;
+                        messageSize += 1 * rowEntries.size();
+                        // packet size
+                        int size = 0;
+                        size += CodedOutputStream.computeEnumSize(3, PacketType.MESSAGES.getNumber());
+                        size += CodedOutputStream.computeTagSize(5)
+                                + CodedOutputStream.computeRawVarint32Size(messageSize) + messageSize;
+                        // build data
+                        byte[] body = new byte[size];
+                        CodedOutputStream output = CodedOutputStream.newInstance(body);
+                        output.writeEnum(3, PacketType.MESSAGES.getNumber());
+
+                        output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+                        output.writeRawVarint32(messageSize);
+                        // message
+                        output.writeInt64(1, data.getId());
+                        for (int i = 0; i < rowEntries.size(); i++) {
+                            output.writeBytes(2, rowEntries.get(i));
+                        }
+                        output.checkNoSpaceLeft();
+                        return body;
+                    } else if (!CollectionUtils.isEmpty(data.getEntries())) {
+                        // mq模式只会走到非rowEntry模式
+                        CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
+                        for (CanalEntry.Entry entry : data.getEntries()) {
+                            if (filterTransactionEntry
+                                && (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) {
+                                continue;
+                            }
+
+                            messageBuilder.addMessages(entry.toByteString());
+                        }
+
+                        CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
+                        packetBuilder.setType(PacketType.MESSAGES);
+                        packetBuilder.setBody(messageBuilder.build().toByteString());
+                        return packetBuilder.build().toByteArray();
+                    }
+                }
+            }
+        } catch (Exception e) {
+            throw new SerializationException("Error when serializing message to byte[] ");
+        }
+        return null;
+    }
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java b/server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java
new file mode 100644
index 00000000..598204f0
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java
@@ -0,0 +1,592 @@
+package com.alibaba.otter.canal.common;
+
+import java.util.*;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.protocol.Message;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * process MQ Message utils
+ *
+ * @author agapple 2018年12月11日 下午1:28:32
+ */
+public class MQMessageUtils {
+
+    @SuppressWarnings("deprecation")
+    private static Map>    partitionDatas    = MigrateMap
+        .makeComputingMap(new MapMaker().softValues(), new Function>() {
+
+                                                                                 public List apply(String pkHashConfigs) {
+                                                                                     List datas = Lists
+                                                                                         .newArrayList();
+                                                                                     String[] pkHashConfigArray = StringUtils
+                                                                                         .split(pkHashConfigs, ",");
+                                                                                     // schema.table:id^name
+                                                                                     for (String pkHashConfig : pkHashConfigArray) {
+                                                                                         PartitionData data = new PartitionData();
+                                                                                         int i = pkHashConfig
+                                                                                             .lastIndexOf(":");
+                                                                                         if (i > 0) {
+                                                                                             String pkStr = pkHashConfig
+                                                                                                 .substring(i + 1);
+                                                                                             if (pkStr.equalsIgnoreCase(
+                                                                                                 "$pk$")) {
+                                                                                                 data.hashMode.autoPkHash = true;
+                                                                                             } else {
+                                                                                                 data.hashMode.pkNames = Lists
+                                                                                                     .newArrayList(
+                                                                                                         StringUtils
+                                                                                                             .split(
+                                                                                                                 pkStr,
+                                                                                                                 '^'));
+                                                                                             }
+
+                                                                                             pkHashConfig = pkHashConfig
+                                                                                                 .substring(0, i);
+                                                                                         } else {
+                                                                                             data.hashMode.tableHash = true;
+                                                                                         }
+
+                                                                                         if (!isWildCard(
+                                                                                             pkHashConfig)) {
+                                                                                             data.simpleName = pkHashConfig;
+                                                                                         } else {
+                                                                                             data.regexFilter = new AviaterRegexFilter(
+                                                                                                 pkHashConfig);
+                                                                                         }
+                                                                                         datas.add(data);
+                                                                                     }
+
+                                                                                     return datas;
+                                                                                 }
+                                                                             });
+
+    @SuppressWarnings("deprecation")
+    private static Map> dynamicTopicDatas = MigrateMap
+        .makeComputingMap(new MapMaker().softValues(), new Function>() {
+
+                                                                                 public List apply(String pkHashConfigs) {
+                                                                                     List datas = Lists
+                                                                                         .newArrayList();
+                                                                                     String[] dynamicTopicArray = StringUtils
+                                                                                         .split(pkHashConfigs, ",");
+                                                                                     // schema.table
+                                                                                     for (String dynamicTopic : dynamicTopicArray) {
+                                                                                         DynamicTopicData data = new DynamicTopicData();
+
+                                                                                         if (!isWildCard(
+                                                                                             dynamicTopic)) {
+                                                                                             data.simpleName = dynamicTopic;
+                                                                                         } else {
+                                                                                             if (dynamicTopic
+                                                                                                 .contains("\\.")) {
+                                                                                                 data.tableRegexFilter = new AviaterRegexFilter(
+                                                                                                     dynamicTopic);
+                                                                                             } else {
+                                                                                                 data.schemaRegexFilter = new AviaterRegexFilter(
+                                                                                                     dynamicTopic);
+                                                                                             }
+                                                                                         }
+                                                                                         datas.add(data);
+                                                                                     }
+
+                                                                                     return datas;
+                                                                                 }
+                                                                             });
+
+    /**
+     * 按 schema 或者 schema+table 将 message 分配到对应topic
+     *
+     * @param message 原message
+     * @param defaultTopic 默认topic
+     * @param dynamicTopicConfigs 动态topic规则
+     * @return 分隔后的message map
+     */
+    public static Map messageTopics(Message message, String defaultTopic, String dynamicTopicConfigs) {
+        List entries;
+        if (message.isRaw()) {
+            List rawEntries = message.getRawEntries();
+            entries = new ArrayList<>(rawEntries.size());
+            for (ByteString byteString : rawEntries) {
+                CanalEntry.Entry entry;
+                try {
+                    entry = CanalEntry.Entry.parseFrom(byteString);
+                } catch (InvalidProtocolBufferException e) {
+                    throw new RuntimeException(e);
+                }
+                entries.add(entry);
+            }
+        } else {
+            entries = message.getEntries();
+        }
+        Map messages = new HashMap<>();
+        for (CanalEntry.Entry entry : entries) {
+            if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
+                continue;
+            }
+
+            String schemaName = entry.getHeader().getSchemaName();
+            String tableName = entry.getHeader().getTableName();
+
+            if (StringUtils.isEmpty(schemaName) || StringUtils.isEmpty(tableName)) {
+                put2MapMessage(messages, message.getId(), defaultTopic, entry);
+            } else {
+                Set topics = matchTopics(schemaName + "." + tableName, dynamicTopicConfigs);
+                if (topics != null) {
+                    for (String topic : topics) {
+                        put2MapMessage(messages, message.getId(), topic, entry);
+                    }
+                } else {
+                    topics = matchTopics(schemaName, dynamicTopicConfigs);
+                    if (topics != null) {
+                        for (String topic : topics) {
+                            put2MapMessage(messages, message.getId(), topic, entry);
+                        }
+                    } else {
+                        put2MapMessage(messages, message.getId(), defaultTopic, entry);
+                    }
+                }
+            }
+        }
+        return messages;
+    }
+
+    /**
+     * 将 message 分区
+     *
+     * @param partitionsNum 分区数
+     * @param pkHashConfigs 分区库表主键正则表达式
+     * @return 分区message数组
+     */
+    @SuppressWarnings("unchecked")
+    public static Message[] messagePartition(Message message, Integer partitionsNum, String pkHashConfigs) {
+        if (partitionsNum == null) {
+            partitionsNum = 1;
+        }
+        Message[] partitionMessages = new Message[partitionsNum];
+        List[] partitionEntries = new List[partitionsNum];
+        for (int i = 0; i < partitionsNum; i++) {
+            partitionEntries[i] = new ArrayList<>();
+        }
+
+        List entries;
+        if (message.isRaw()) {
+            List rawEntries = message.getRawEntries();
+            entries = new ArrayList<>(rawEntries.size());
+            for (ByteString byteString : rawEntries) {
+                Entry entry;
+                try {
+                    entry = Entry.parseFrom(byteString);
+                } catch (InvalidProtocolBufferException e) {
+                    throw new RuntimeException(e);
+                }
+                entries.add(entry);
+            }
+        } else {
+            entries = message.getEntries();
+        }
+
+        for (Entry entry : entries) {
+            CanalEntry.RowChange rowChange;
+            try {
+                rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+            } catch (Exception e) {
+                throw new RuntimeException(e.getMessage(), e);
+            }
+
+            if (rowChange.getIsDdl()) {
+                partitionEntries[0].add(entry);
+            } else {
+                if (rowChange.getRowDatasList() != null && !rowChange.getRowDatasList().isEmpty()) {
+                    String database = entry.getHeader().getSchemaName();
+                    String table = entry.getHeader().getTableName();
+                    HashMode hashMode = getPartitionHashColumns(database + "." + table, pkHashConfigs);
+                    if (hashMode == null) {
+                        // 如果都没有匹配，发送到第一个分区
+                        partitionEntries[0].add(entry);
+                    } else {
+                        for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
+                            int hashCode = table.hashCode();
+                            if (hashMode.autoPkHash) {
+                                // isEmpty use default pkNames
+                                for (CanalEntry.Column column : rowData.getAfterColumnsList()) {
+                                    if (column.getIsKey()) {
+                                        hashCode = hashCode ^ column.getValue().hashCode();
+                                    }
+                                }
+                            } else if (!hashMode.tableHash) {
+                                for (CanalEntry.Column column : rowData.getAfterColumnsList()) {
+                                    if (checkPkNamesHasContain(hashMode.pkNames, column.getName())) {
+                                        hashCode = hashCode ^ column.getValue().hashCode();
+                                    }
+                                }
+                            }
+
+                            int pkHash = Math.abs(hashCode) % partitionsNum;
+                            pkHash = Math.abs(pkHash);
+                            partitionEntries[pkHash].add(entry);
+                        }
+                    }
+                } else {
+                    // 针对stmt/mixed binlog格式的query事件
+                    partitionEntries[0].add(entry);
+                }
+            }
+        }
+
+        for (int i = 0; i < partitionsNum; i++) {
+            List entriesTmp = partitionEntries[i];
+            if (!entriesTmp.isEmpty()) {
+                partitionMessages[i] = new Message(message.getId(), entriesTmp);
+            }
+        }
+
+        return partitionMessages;
+    }
+
+    /**
+     * 将Message转换为FlatMessage
+     *
+     * @param message 原生message
+     * @return FlatMessage列表
+     */
+    public static List messageConverter(Message message) {
+        try {
+            if (message == null) {
+                return null;
+            }
+
+            List flatMessages = new ArrayList<>();
+            List entrys = null;
+            if (message.isRaw()) {
+                List rawEntries = message.getRawEntries();
+                entrys = new ArrayList(rawEntries.size());
+                for (ByteString byteString : rawEntries) {
+                    CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString);
+                    entrys.add(entry);
+                }
+            } else {
+                entrys = message.getEntries();
+            }
+
+            for (CanalEntry.Entry entry : entrys) {
+                if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                    || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
+                    continue;
+                }
+
+                CanalEntry.RowChange rowChange;
+                try {
+                    rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+                } catch (Exception e) {
+                    throw new RuntimeException(
+                        "ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
+                        e);
+                }
+
+                CanalEntry.EventType eventType = rowChange.getEventType();
+
+                FlatMessage flatMessage = new FlatMessage(message.getId());
+                flatMessages.add(flatMessage);
+                flatMessage.setDatabase(entry.getHeader().getSchemaName());
+                flatMessage.setTable(entry.getHeader().getTableName());
+                flatMessage.setIsDdl(rowChange.getIsDdl());
+                flatMessage.setType(eventType.toString());
+                flatMessage.setEs(entry.getHeader().getExecuteTime());
+                flatMessage.setTs(System.currentTimeMillis());
+                flatMessage.setSql(rowChange.getSql());
+
+                if (!rowChange.getIsDdl()) {
+                    Map sqlType = new LinkedHashMap<>();
+                    Map mysqlType = new LinkedHashMap<>();
+                    List> data = new ArrayList<>();
+                    List> old = new ArrayList<>();
+
+                    Set updateSet = new HashSet<>();
+                    for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
+                        if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
+                            && eventType != CanalEntry.EventType.DELETE) {
+                            continue;
+                        }
+
+                        Map row = new LinkedHashMap<>();
+                        List columns;
+
+                        if (eventType == CanalEntry.EventType.DELETE) {
+                            columns = rowData.getBeforeColumnsList();
+                        } else {
+                            columns = rowData.getAfterColumnsList();
+                        }
+
+                        for (CanalEntry.Column column : columns) {
+                            if (column.getIsKey()) {
+                                flatMessage.addPkName(column.getName());
+                            }
+                            sqlType.put(column.getName(), column.getSqlType());
+                            mysqlType.put(column.getName(), column.getMysqlType());
+                            if (column.getIsNull()) {
+                                row.put(column.getName(), null);
+                            } else {
+                                row.put(column.getName(), column.getValue());
+                            }
+                            // 获取update为true的字段
+                            if (column.getUpdated()) {
+                                updateSet.add(column.getName());
+                            }
+                        }
+                        if (!row.isEmpty()) {
+                            data.add(row);
+                        }
+
+                        if (eventType == CanalEntry.EventType.UPDATE) {
+                            Map rowOld = new LinkedHashMap<>();
+                            for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
+                                if (updateSet.contains(column.getName())) {
+                                    if (column.getIsNull()) {
+                                        rowOld.put(column.getName(), null);
+                                    } else {
+                                        rowOld.put(column.getName(), column.getValue());
+                                    }
+                                }
+                            }
+                            // update操作将记录修改前的值
+                            if (!rowOld.isEmpty()) {
+                                old.add(rowOld);
+                            }
+                        }
+                    }
+                    if (!sqlType.isEmpty()) {
+                        flatMessage.setSqlType(sqlType);
+                    }
+                    if (!mysqlType.isEmpty()) {
+                        flatMessage.setMysqlType(mysqlType);
+                    }
+                    if (!data.isEmpty()) {
+                        flatMessage.setData(data);
+                    }
+                    if (!old.isEmpty()) {
+                        flatMessage.setOld(old);
+                    }
+                }
+            }
+            return flatMessages;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * 将FlatMessage按指定的字段值hash拆分
+     *
+     * @param flatMessage flatMessage
+     * @param partitionsNum 分区数量
+     * @param pkHashConfigs hash映射
+     * @return 拆分后的flatMessage数组
+     */
+    public static FlatMessage[] messagePartition(FlatMessage flatMessage, Integer partitionsNum, String pkHashConfigs) {
+        if (partitionsNum == null) {
+            partitionsNum = 1;
+        }
+        FlatMessage[] partitionMessages = new FlatMessage[partitionsNum];
+
+        if (flatMessage.getIsDdl()) {
+            partitionMessages[0] = flatMessage;
+        } else {
+            if (flatMessage.getData() != null && !flatMessage.getData().isEmpty()) {
+                String database = flatMessage.getDatabase();
+                String table = flatMessage.getTable();
+                HashMode hashMode = getPartitionHashColumns(database + "." + table, pkHashConfigs);
+                if (hashMode == null) {
+                    // 如果都没有匹配，发送到第一个分区
+                    partitionMessages[0] = flatMessage;
+                } else {
+                    List pkNames = hashMode.pkNames;
+                    if (hashMode.autoPkHash) {
+                        pkNames = flatMessage.getPkNames();
+                    }
+
+                    int idx = 0;
+                    for (Map row : flatMessage.getData()) {
+                        int hashCode = table.hashCode();
+                        if (!hashMode.tableHash) {
+                            for (String pkName : pkNames) {
+                                String value = row.get(pkName);
+                                if (value == null) {
+                                    value = "";
+                                }
+                                hashCode = hashCode ^ value.hashCode();
+                            }
+                        }
+
+                        int pkHash = Math.abs(hashCode) % partitionsNum;
+                        // math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
+                        pkHash = Math.abs(pkHash);
+
+                        FlatMessage flatMessageTmp = partitionMessages[pkHash];
+                        if (flatMessageTmp == null) {
+                            flatMessageTmp = new FlatMessage(flatMessage.getId());
+                            partitionMessages[pkHash] = flatMessageTmp;
+                            flatMessageTmp.setDatabase(flatMessage.getDatabase());
+                            flatMessageTmp.setTable(flatMessage.getTable());
+                            flatMessageTmp.setIsDdl(flatMessage.getIsDdl());
+                            flatMessageTmp.setType(flatMessage.getType());
+                            flatMessageTmp.setSql(flatMessage.getSql());
+                            flatMessageTmp.setSqlType(flatMessage.getSqlType());
+                            flatMessageTmp.setMysqlType(flatMessage.getMysqlType());
+                            flatMessageTmp.setEs(flatMessage.getEs());
+                            flatMessageTmp.setTs(flatMessage.getTs());
+                        }
+                        List> data = flatMessageTmp.getData();
+                        if (data == null) {
+                            data = new ArrayList<>();
+                            flatMessageTmp.setData(data);
+                        }
+                        data.add(row);
+                        if (flatMessage.getOld() != null && !flatMessage.getOld().isEmpty()) {
+                            List> old = flatMessageTmp.getOld();
+                            if (old == null) {
+                                old = new ArrayList<>();
+                                flatMessageTmp.setOld(old);
+                            }
+                            old.add(flatMessage.getOld().get(idx));
+                        }
+                        idx++;
+                    }
+                }
+            } else {
+                // 针对stmt/mixed binlog格式的query事件
+                partitionMessages[0] = flatMessage;
+            }
+        }
+        return partitionMessages;
+    }
+
+    /**
+     * match return List , not match return null
+     */
+    public static HashMode getPartitionHashColumns(String name, String pkHashConfigs) {
+        if (StringUtils.isEmpty(pkHashConfigs)) {
+            return null;
+        }
+
+        List datas = partitionDatas.get(pkHashConfigs);
+        for (PartitionData data : datas) {
+            if (data.simpleName != null) {
+                if (data.simpleName.equalsIgnoreCase(name)) {
+                    return data.hashMode;
+                }
+            } else {
+                if (data.regexFilter.filter(name)) {
+                    return data.hashMode;
+                }
+            }
+        }
+
+        return null;
+    }
+
+    private static Set matchTopics(String name, String dynamicTopicConfigs) {
+        String[] router = StringUtils.split(dynamicTopicConfigs, ';');
+        Set topics = new HashSet<>();
+        for (String item : router) {
+            int i = item.indexOf(":");
+            if (i > -1) {
+                String topic = item.substring(0, i).trim();
+                String topicConfigs = item.substring(i + 1).trim();
+                if (matchDynamicTopic(name, topicConfigs)) {
+                    topics.add(topic);
+                }
+            } else if (matchDynamicTopic(name, item)) {
+                topics.add(name);
+            }
+        }
+        return topics.isEmpty() ? null : topics;
+    }
+
+    public static boolean matchDynamicTopic(String name, String dynamicTopicConfigs) {
+        if (StringUtils.isEmpty(dynamicTopicConfigs)) {
+            return false;
+        }
+
+        boolean res = false;
+        List datas = dynamicTopicDatas.get(dynamicTopicConfigs);
+        for (DynamicTopicData data : datas) {
+            if (data.simpleName != null) {
+                if (data.simpleName.equalsIgnoreCase(name)) {
+                    res = true;
+                    break;
+                }
+            } else if (name.contains(".")) {
+                if (data.tableRegexFilter != null && data.tableRegexFilter.filter(name)) {
+                    res = true;
+                    break;
+                }
+            } else {
+                if (data.schemaRegexFilter != null && data.schemaRegexFilter.filter(name)) {
+                    res = true;
+                    break;
+                }
+            }
+        }
+        return res;
+    }
+
+    public static boolean checkPkNamesHasContain(List pkNames, String name) {
+        for (String pkName : pkNames) {
+            if (pkName.equalsIgnoreCase(name)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private static boolean isWildCard(String value) {
+        // not contaiins '.' ?
+        return StringUtils.containsAny(value,
+            new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$', '^' });
+    }
+
+    private static void put2MapMessage(Map messageMap, Long messageId, String topicName,
+                                       CanalEntry.Entry entry) {
+        Message message = messageMap.get(topicName);
+        if (message == null) {
+            message = new Message(messageId, new ArrayList());
+            messageMap.put(topicName, message);
+        }
+        message.getEntries().add(entry);
+    }
+
+    public static class PartitionData {
+
+        public String             simpleName;
+        public AviaterRegexFilter regexFilter;
+        public HashMode           hashMode = new HashMode();
+    }
+
+    public static class HashMode {
+
+        public boolean      autoPkHash = false;
+        public boolean      tableHash  = false;
+        public List pkNames    = Lists.newArrayList();
+    }
+
+    public static class DynamicTopicData {
+
+        public String             simpleName;
+        public AviaterRegexFilter schemaRegexFilter;
+        public AviaterRegexFilter tableRegexFilter;
+    }
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java b/server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
new file mode 100644
index 00000000..4ee28c5e
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
@@ -0,0 +1,224 @@
+package com.alibaba.otter.canal.common;
+
+import java.util.Properties;
+
+/**
+ * kafka 配置项
+ *
+ * @author machengyuan 2018-6-11 下午05:30:49
+ * @version 1.0.0
+ */
+public class MQProperties {
+
+    private String     servers                = "127.0.0.1:6667";
+    private int        retries                = 0;
+    private int        batchSize              = 16384;
+    private int        lingerMs               = 1;
+    private int        maxRequestSize         = 1048576;
+    private long       bufferMemory           = 33554432L;
+    private boolean    filterTransactionEntry = true;
+    private String     producerGroup          = "Canal-Producer";
+    private int        canalBatchSize         = 50;
+    private Long       canalGetTimeout        = 100L;
+    private boolean    flatMessage            = true;
+    private String     compressionType        = "none";
+    private String     acks                   = "all";
+    private String     aliyunAccessKey        = "";
+    private String     aliyunSecretKey        = "";
+    private boolean    transaction            = false;           // 是否开启事务
+    private Properties properties             = new Properties();
+
+    public static class CanalDestination {
+
+        private String  canalDestination;
+        private String  topic;
+        private Integer partition;
+        private Integer partitionsNum;
+        private String  partitionHash;
+        private String  dynamicTopic;
+
+        public String getCanalDestination() {
+            return canalDestination;
+        }
+
+        public void setCanalDestination(String canalDestination) {
+            this.canalDestination = canalDestination;
+        }
+
+        public String getTopic() {
+            return topic;
+        }
+
+        public void setTopic(String topic) {
+            this.topic = topic;
+        }
+
+        public Integer getPartition() {
+            return partition;
+        }
+
+        public void setPartition(Integer partition) {
+            this.partition = partition;
+        }
+
+        public Integer getPartitionsNum() {
+            return partitionsNum;
+        }
+
+        public void setPartitionsNum(Integer partitionsNum) {
+            this.partitionsNum = partitionsNum;
+        }
+
+        public String getPartitionHash() {
+            return partitionHash;
+        }
+
+        public void setPartitionHash(String partitionHash) {
+            this.partitionHash = partitionHash;
+        }
+
+        public String getDynamicTopic() {
+            return dynamicTopic;
+        }
+
+        public void setDynamicTopic(String dynamicTopic) {
+            this.dynamicTopic = dynamicTopic;
+        }
+    }
+
+    public String getServers() {
+        return servers;
+    }
+
+    public void setServers(String servers) {
+        this.servers = servers;
+    }
+
+    public int getRetries() {
+        return retries;
+    }
+
+    public void setRetries(int retries) {
+        this.retries = retries;
+    }
+
+    public int getBatchSize() {
+        return batchSize;
+    }
+
+    public void setBatchSize(int batchSize) {
+        this.batchSize = batchSize;
+    }
+
+    public int getLingerMs() {
+        return lingerMs;
+    }
+
+    public void setLingerMs(int lingerMs) {
+        this.lingerMs = lingerMs;
+    }
+
+    public long getBufferMemory() {
+        return bufferMemory;
+    }
+
+    public void setBufferMemory(long bufferMemory) {
+        this.bufferMemory = bufferMemory;
+    }
+
+    public int getCanalBatchSize() {
+        return canalBatchSize;
+    }
+
+    public void setCanalBatchSize(int canalBatchSize) {
+        this.canalBatchSize = canalBatchSize;
+    }
+
+    public Long getCanalGetTimeout() {
+        return canalGetTimeout;
+    }
+
+    public void setCanalGetTimeout(Long canalGetTimeout) {
+        this.canalGetTimeout = canalGetTimeout;
+    }
+
+    public boolean getFlatMessage() {
+        return flatMessage;
+    }
+
+    public void setFlatMessage(boolean flatMessage) {
+        this.flatMessage = flatMessage;
+    }
+
+    public boolean isFilterTransactionEntry() {
+        return filterTransactionEntry;
+    }
+
+    public void setFilterTransactionEntry(boolean filterTransactionEntry) {
+        this.filterTransactionEntry = filterTransactionEntry;
+    }
+
+    public String getProducerGroup() {
+        return producerGroup;
+    }
+
+    public void setProducerGroup(String producerGroup) {
+        this.producerGroup = producerGroup;
+    }
+
+    public String getCompressionType() {
+        return compressionType;
+    }
+
+    public void setCompressionType(String compressionType) {
+        this.compressionType = compressionType;
+    }
+
+    public String getAcks() {
+        return acks;
+    }
+
+    public void setAcks(String acks) {
+        this.acks = acks;
+    }
+
+    public String getAliyunAccessKey() {
+        return aliyunAccessKey;
+    }
+
+    public void setAliyunAccessKey(String aliyunAccessKey) {
+        this.aliyunAccessKey = aliyunAccessKey;
+    }
+
+    public String getAliyunSecretKey() {
+        return aliyunSecretKey;
+    }
+
+    public void setAliyunSecretKey(String aliyunSecretKey) {
+        this.aliyunSecretKey = aliyunSecretKey;
+    }
+
+    public int getMaxRequestSize() {
+        return maxRequestSize;
+    }
+
+    public void setMaxRequestSize(int maxRequestSize) {
+        this.maxRequestSize = maxRequestSize;
+    }
+
+    public boolean getTransaction() {
+        return transaction;
+    }
+
+    public void setTransaction(boolean transaction) {
+        this.transaction = transaction;
+    }
+
+    public Properties getProperties() {
+        return properties;
+    }
+
+    public void setProperties(Properties properties) {
+        this.properties = properties;
+    }
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java b/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
index 84656fb1..12c048aa 100644
--- a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
+++ b/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
@@ -1,20 +1,25 @@
 package com.alibaba.otter.canal.kafka;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
-import java.util.concurrent.Future;
+import java.util.concurrent.ExecutionException;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.clients.producer.RecordMetadata;
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.common.MQMessageUtils;
+import com.alibaba.otter.canal.common.MQProperties;
 import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.spi.CanalMQProducer;
 
 /**
  * kafka producer 主操作类
@@ -22,7 +27,7 @@ import com.alibaba.otter.canal.protocol.Message;
  * @author machengyuan 2018-6-11 下午05:30:49
  * @version 1.0.0
  */
-public class CanalKafkaProducer {
+public class CanalKafkaProducer implements CanalMQProducer {
 
     private static final Logger       logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
 
@@ -30,18 +35,30 @@ public class CanalKafkaProducer {
 
     private Producer  producer2;                                                 // 用于扁平message的数据投递
 
-    private KafkaProperties           kafkaProperties;
+    private MQProperties              kafkaProperties;
 
-    public void init(KafkaProperties kafkaProperties) {
+    @Override
+    public void init(MQProperties kafkaProperties) {
         this.kafkaProperties = kafkaProperties;
         Properties properties = new Properties();
         properties.put("bootstrap.servers", kafkaProperties.getServers());
-        properties.put("acks", "all");
-        properties.put("retries", kafkaProperties.getRetries());
+        properties.put("acks", kafkaProperties.getAcks());
+        properties.put("compression.type", kafkaProperties.getCompressionType());
         properties.put("batch.size", kafkaProperties.getBatchSize());
         properties.put("linger.ms", kafkaProperties.getLingerMs());
+        properties.put("max.request.size", kafkaProperties.getMaxRequestSize());
         properties.put("buffer.memory", kafkaProperties.getBufferMemory());
         properties.put("key.serializer", StringSerializer.class.getName());
+
+        if (!kafkaProperties.getProperties().isEmpty()) {
+            properties.putAll(kafkaProperties.getProperties());
+        }
+
+        if (kafkaProperties.getTransaction()) {
+            properties.put("transactional.id", "canal-transactional-id");
+        } else {
+            properties.put("retries", kafkaProperties.getRetries());
+        }
         if (!kafkaProperties.getFlatMessage()) {
             properties.put("value.serializer", MessageSerializer.class.getName());
             producer = new KafkaProducer(properties);
@@ -49,10 +66,16 @@ public class CanalKafkaProducer {
             properties.put("value.serializer", StringSerializer.class.getName());
             producer2 = new KafkaProducer(properties);
         }
-
-        // producer.initTransactions();
+        if (kafkaProperties.getTransaction()) {
+            if (!kafkaProperties.getFlatMessage()) {
+                producer.initTransactions();
+            } else {
+                producer2.initTransactions();
+            }
+        }
     }
 
+    @Override
     public void stop() {
         try {
             logger.info("## stop the kafka producer");
@@ -69,46 +92,128 @@ public class CanalKafkaProducer {
         }
     }
 
-    public void send(KafkaProperties.Topic topic, Message message, Callback callback) {
+    @Override
+    public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) {
+        // 开启事务，需要kafka版本支持
+        Producer producerTmp;
+        if (!kafkaProperties.getFlatMessage()) {
+            producerTmp = producer;
+        } else {
+            producerTmp = producer2;
+        }
+
+        if (kafkaProperties.getTransaction()) {
+            producerTmp.beginTransaction();
+        }
         try {
-            // producer.beginTransaction();
-            if (!kafkaProperties.getFlatMessage()) {
-                ProducerRecord record;
-                if (topic.getPartition() != null) {
-                    record = new ProducerRecord(topic.getTopic(), topic.getPartition(), null, message);
-                } else {
-                    record = new ProducerRecord(topic.getTopic(), message);
-                }
+            if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) {
+                // 动态topic
+                Map messageMap = MQMessageUtils
+                    .messageTopics(message, canalDestination.getTopic(), canalDestination.getDynamicTopic());
 
-                producer.send(record);
-            } else {
-                // 发送扁平数据json
-                List flatMessages = FlatMessage.messageConverter(message);
-                if (flatMessages != null) {
-                    for (FlatMessage flatMessage : flatMessages) {
-                        ProducerRecord record = new ProducerRecord(topic.getTopic(),
-                            JSON.toJSONString(flatMessage));
-                        producer2.send(record);
+                for (Map.Entry entry : messageMap.entrySet()) {
+                    String topicName = entry.getKey().replace('.', '_');
+                    Message messageSub = entry.getValue();
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("## Send message to kafka topic: " + topicName);
                     }
+                    send(canalDestination, topicName, messageSub);
                 }
+            } else {
+                send(canalDestination, canalDestination.getTopic(), message);
+            }
+            if (kafkaProperties.getTransaction()) {
+                producerTmp.commitTransaction();
             }
-
-            // producer.commitTransaction();
             callback.commit();
-            if (logger.isDebugEnabled()) {
-                logger.debug("send message to kafka topic: {}", topic.getTopic());
-            }
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
-            // producer.abortTransaction();
+            if (kafkaProperties.getTransaction()) {
+                producerTmp.abortTransaction();
+            }
             callback.rollback();
         }
     }
 
-    public interface Callback {
+    private void send(MQProperties.CanalDestination canalDestination, String topicName,
+                      Message message) throws Exception {
+        if (!kafkaProperties.getFlatMessage()) {
+            ProducerRecord record = null;
+            if (canalDestination.getPartition() != null) {
+                record = new ProducerRecord<>(topicName, canalDestination.getPartition(), null, message);
+            } else {
+                if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) {
+                    Message[] messages = MQMessageUtils.messagePartition(message,
+                        canalDestination.getPartitionsNum(),
+                        canalDestination.getPartitionHash());
+                    int length = messages.length;
+                    for (int i = 0; i < length; i++) {
+                        Message messagePartition = messages[i];
+                        if (messagePartition != null) {
+                            record = new ProducerRecord<>(topicName, i, null, messagePartition);
+                        }
+                    }
+                } else {
+                    record = new ProducerRecord<>(topicName, 0, null, message);
+                }
+            }
 
-        void commit();
+            if (record != null) {
+                if (kafkaProperties.getTransaction()) {
+                    producer.send(record);
+                } else {
+                    producer.send(record).get();
+                }
 
-        void rollback();
+                if (logger.isDebugEnabled()) {
+                    logger.debug("Send  message to kafka topic: [{}], packet: {}", topicName, message.toString());
+                }
+            }
+        } else {
+            // 发送扁平数据json
+            List flatMessages = MQMessageUtils.messageConverter(message);
+            if (flatMessages != null) {
+                for (FlatMessage flatMessage : flatMessages) {
+                    if (StringUtils.isEmpty(canalDestination.getPartitionHash())) {
+                        Integer partition = canalDestination.getPartition();
+                        if (partition == null) {
+                            partition = 0;
+                        }
+                        produce(topicName, partition, flatMessage);
+                    } else {
+                        FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
+                            canalDestination.getPartitionsNum(),
+                            canalDestination.getPartitionHash());
+                        int length = partitionFlatMessage.length;
+                        for (int i = 0; i < length; i++) {
+                            FlatMessage flatMessagePart = partitionFlatMessage[i];
+                            if (flatMessagePart != null) {
+                                produce(topicName, i, flatMessagePart);
+                            }
+                        }
+                    }
+
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("Send flat message to kafka topic: [{}], packet: {}",
+                            topicName,
+                            JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue));
+                    }
+                }
+            }
+        }
     }
+
+    private void produce(String topicName, int partition, FlatMessage flatMessage) throws ExecutionException,
+                                                                                   InterruptedException {
+        ProducerRecord record = new ProducerRecord(topicName,
+            partition,
+            null,
+            JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue));
+        if (kafkaProperties.getTransaction()) {
+            producer2.send(record);
+        } else {
+            producer2.send(record).get();
+        }
+    }
+
 }
diff --git a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaStarter.java b/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaStarter.java
deleted file mode 100644
index 5b73bf3b..00000000
--- a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaStarter.java
+++ /dev/null
@@ -1,167 +0,0 @@
-package com.alibaba.otter.canal.kafka;
-
-import java.io.FileInputStream;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.yaml.snakeyaml.Yaml;
-
-import com.alibaba.otter.canal.kafka.KafkaProperties.CanalDestination;
-import com.alibaba.otter.canal.kafka.KafkaProperties.Topic;
-import com.alibaba.otter.canal.protocol.ClientIdentity;
-import com.alibaba.otter.canal.protocol.Message;
-import com.alibaba.otter.canal.server.CanalServerStarter;
-import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
-
-/**
- * kafka 启动类
- *
- * @author machengyuan 2018-6-11 下午05:30:49
- * @version 1.0.0
- */
-public class CanalKafkaStarter implements CanalServerStarter {
-
-    private static final Logger logger               = LoggerFactory.getLogger(CanalKafkaStarter.class);
-
-    private static final String CLASSPATH_URL_PREFIX = "classpath:";
-
-    private volatile boolean    running              = false;
-
-    private ExecutorService     executorService;
-
-    private CanalKafkaProducer  canalKafkaProducer;
-
-    private KafkaProperties     kafkaProperties;
-
-    public void init() {
-        try {
-            logger.info("## load kafka configurations");
-            String conf = System.getProperty("kafka.conf", "classpath:kafka.yml");
-
-            if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
-                conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
-                kafkaProperties = new Yaml().loadAs(CanalKafkaStarter.class.getClassLoader().getResourceAsStream(conf),
-                    KafkaProperties.class);
-            } else {
-                kafkaProperties = new Yaml().loadAs(new FileInputStream(conf), KafkaProperties.class);
-            }
-
-            // 初始化 kafka producer
-            canalKafkaProducer = new CanalKafkaProducer();
-            canalKafkaProducer.init(kafkaProperties);
-            // set filterTransactionEntry
-            // if (kafkaProperties.isFilterTransactionEntry()) {
-            // System.setProperty("canal.instance.filter.transaction.entry", "true");
-            // }
-            // 对应每个instance启动一个worker线程
-            List destinations = kafkaProperties.getCanalDestinations();
-
-            executorService = Executors.newFixedThreadPool(destinations.size());
-
-            logger.info("## start the kafka workers.");
-            for (final CanalDestination destination : destinations) {
-                executorService.execute(new Runnable() {
-
-                    @Override
-                    public void run() {
-                        worker(destination);
-                    }
-                });
-            }
-            running = true;
-            logger.info("## the kafka workers is running now ......");
-            Runtime.getRuntime().addShutdownHook(new Thread() {
-
-                public void run() {
-                    try {
-                        logger.info("## stop the kafka workers");
-                        running = false;
-                        executorService.shutdown();
-                        canalKafkaProducer.stop();
-                    } catch (Throwable e) {
-                        logger.warn("##something goes wrong when stopping kafka workers:", e);
-                    } finally {
-                        logger.info("## canal kafka is down.");
-                    }
-                }
-
-            });
-
-        } catch (Throwable e) {
-            logger.error("## Something goes wrong when starting up the canal kafka workers:", e);
-            System.exit(0);
-        }
-    }
-
-    private void worker(CanalDestination destination) {
-        while (!running)
-            ;
-        logger.info("## start the canal consumer: {}.", destination.getCanalDestination());
-        final CanalServerWithEmbedded server = CanalServerWithEmbedded.instance();
-        final ClientIdentity clientIdentity = new ClientIdentity(destination.getCanalDestination(), (short) 1001, "");
-        while (running) {
-            try {
-                if (!server.getCanalInstances().containsKey(clientIdentity.getDestination())) {
-                    try {
-                        Thread.sleep(3000);
-                    } catch (InterruptedException e) {
-                        // ignore
-                    }
-                    continue;
-                }
-                server.subscribe(clientIdentity);
-                logger.info("## the canal consumer {} is running now ......", destination.getCanalDestination());
-
-                while (running) {
-                    Message message;
-                    if (kafkaProperties.getCanalGetTimeout() != null) {
-                        message = server.getWithoutAck(clientIdentity,
-                            kafkaProperties.getCanalBatchSize(),
-                            kafkaProperties.getCanalGetTimeout(),
-                            TimeUnit.MILLISECONDS);
-                    } else {
-                        message = server.getWithoutAck(clientIdentity, kafkaProperties.getCanalBatchSize());
-                    }
-
-                    final long batchId = message.getId();
-                    try {
-                        int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
-                        if (batchId != -1 && size != 0) {
-                            Topic topic = new Topic();
-                            topic.setTopic(destination.getTopic());
-                            topic.setPartition(destination.getPartition());
-                            canalKafkaProducer.send(topic, message, new CanalKafkaProducer.Callback() {
-
-                                @Override
-                                public void commit() {
-                                    server.ack(clientIdentity, batchId); // 提交确认
-                                }
-
-                                @Override
-                                public void rollback() {
-                                    server.rollback(clientIdentity, batchId);
-                                }
-                            }); // 发送message到topic
-                        } else {
-                            try {
-                                Thread.sleep(100);
-                            } catch (InterruptedException e) {
-                                // ignore
-                            }
-                        }
-
-                    } catch (Exception e) {
-                        logger.error(e.getMessage(), e);
-                    }
-                }
-            } catch (Exception e) {
-                logger.error("process error!", e);
-            }
-        }
-    }
-}
diff --git a/server/src/main/java/com/alibaba/otter/canal/kafka/KafkaProperties.java b/server/src/main/java/com/alibaba/otter/canal/kafka/KafkaProperties.java
deleted file mode 100644
index 82707ba4..00000000
--- a/server/src/main/java/com/alibaba/otter/canal/kafka/KafkaProperties.java
+++ /dev/null
@@ -1,188 +0,0 @@
-package com.alibaba.otter.canal.kafka;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * kafka 配置项
- *
- * @author machengyuan 2018-6-11 下午05:30:49
- * @version 1.0.0
- */
-public class KafkaProperties {
-
-    private String                 servers                = "localhost:6667";
-    private int                    retries                = 0;
-    private int                    batchSize              = 16384;
-    private int                    lingerMs               = 1;
-    private long                   bufferMemory           = 33554432L;
-    private boolean                filterTransactionEntry = true;
-    private int                    canalBatchSize         = 50;
-    private Long                   canalGetTimeout;
-    private boolean                flatMessage            = true;
-
-    private List canalDestinations      = new ArrayList();
-
-    public static class CanalDestination {
-
-        private String     canalDestination;
-        private String     topic;
-        private Integer    partition;
-        private Set topics = new HashSet();
-
-        public String getCanalDestination() {
-            return canalDestination;
-        }
-
-        public void setCanalDestination(String canalDestination) {
-            this.canalDestination = canalDestination;
-        }
-
-        public String getTopic() {
-            return topic;
-        }
-
-        public void setTopic(String topic) {
-            this.topic = topic;
-        }
-
-        public Integer getPartition() {
-            return partition;
-        }
-
-        public void setPartition(Integer partition) {
-            this.partition = partition;
-        }
-
-        public Set getTopics() {
-            return topics;
-        }
-
-        public void setTopics(Set topics) {
-            this.topics = topics;
-        }
-    }
-
-    public static class Topic {
-
-        private String  topic;
-        private Integer partition;
-
-        public String getTopic() {
-            return topic;
-        }
-
-        public void setTopic(String topic) {
-            this.topic = topic;
-        }
-
-        public Integer getPartition() {
-            return partition;
-        }
-
-        public void setPartition(Integer partition) {
-            this.partition = partition;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-
-            Topic topic1 = (Topic) o;
-
-            if (topic != null ? !topic.equals(topic1.topic) : topic1.topic != null) return false;
-            return partition != null ? partition.equals(topic1.partition) : topic1.partition == null;
-        }
-
-        @Override
-        public int hashCode() {
-            int result = topic != null ? topic.hashCode() : 0;
-            result = 31 * result + (partition != null ? partition.hashCode() : 0);
-            return result;
-        }
-    }
-
-    public String getServers() {
-        return servers;
-    }
-
-    public void setServers(String servers) {
-        this.servers = servers;
-    }
-
-    public int getRetries() {
-        return retries;
-    }
-
-    public void setRetries(int retries) {
-        this.retries = retries;
-    }
-
-    public int getBatchSize() {
-        return batchSize;
-    }
-
-    public void setBatchSize(int batchSize) {
-        this.batchSize = batchSize;
-    }
-
-    public int getLingerMs() {
-        return lingerMs;
-    }
-
-    public void setLingerMs(int lingerMs) {
-        this.lingerMs = lingerMs;
-    }
-
-    public long getBufferMemory() {
-        return bufferMemory;
-    }
-
-    public void setBufferMemory(long bufferMemory) {
-        this.bufferMemory = bufferMemory;
-    }
-
-    public int getCanalBatchSize() {
-        return canalBatchSize;
-    }
-
-    public void setCanalBatchSize(int canalBatchSize) {
-        this.canalBatchSize = canalBatchSize;
-    }
-
-    public Long getCanalGetTimeout() {
-        return canalGetTimeout;
-    }
-
-    public void setCanalGetTimeout(Long canalGetTimeout) {
-        this.canalGetTimeout = canalGetTimeout;
-    }
-
-    public boolean getFlatMessage() {
-        return flatMessage;
-    }
-
-    public void setFlatMessage(boolean flatMessage) {
-        this.flatMessage = flatMessage;
-    }
-
-    public List getCanalDestinations() {
-        return canalDestinations;
-    }
-
-    public void setCanalDestinations(List canalDestinations) {
-        this.canalDestinations = canalDestinations;
-    }
-
-    public boolean isFilterTransactionEntry() {
-        return filterTransactionEntry;
-    }
-
-    public void setFilterTransactionEntry(boolean filterTransactionEntry) {
-        this.filterTransactionEntry = filterTransactionEntry;
-    }
-
-}
diff --git a/server/src/main/java/com/alibaba/otter/canal/kafka/MessageSerializer.java b/server/src/main/java/com/alibaba/otter/canal/kafka/MessageSerializer.java
index 49db5aa1..39c297d8 100644
--- a/server/src/main/java/com/alibaba/otter/canal/kafka/MessageSerializer.java
+++ b/server/src/main/java/com/alibaba/otter/canal/kafka/MessageSerializer.java
@@ -1,19 +1,12 @@
 package com.alibaba.otter.canal.kafka;
 
-import java.util.List;
 import java.util.Map;
 
-import org.apache.kafka.common.errors.SerializationException;
+import org.apache.commons.lang.BooleanUtils;
 import org.apache.kafka.common.serialization.Serializer;
-import org.springframework.util.CollectionUtils;
 
-import com.alibaba.otter.canal.protocol.CanalEntry;
-import com.alibaba.otter.canal.protocol.CanalPacket;
-import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
+import com.alibaba.otter.canal.common.CanalMessageSerializer;
 import com.alibaba.otter.canal.protocol.Message;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.CodedOutputStream;
-import com.google.protobuf.WireFormat;
 
 /**
  * Kafka Message类的序列化
@@ -23,66 +16,20 @@ import com.google.protobuf.WireFormat;
  */
 public class MessageSerializer implements Serializer {
 
+    private boolean filterTransactionEntry = false;
+
+    public MessageSerializer(){
+        this.filterTransactionEntry = BooleanUtils.toBoolean(System.getProperty("canal.instance.filter.transaction.entry",
+            "false"));
+    }
+
     @Override
     public void configure(Map configs, boolean isKey) {
     }
 
     @Override
     public byte[] serialize(String topic, Message data) {
-        try {
-            if (data != null) {
-                if (data.getId() != -1) {
-                    if (data.isRaw() && !CollectionUtils.isEmpty(data.getRawEntries())) {
-                        // for performance
-                        List rowEntries = data.getRawEntries();
-                        // message size
-                        int messageSize = 0;
-                        messageSize += CodedOutputStream.computeInt64Size(1, data.getId());
-
-                        int dataSize = 0;
-                        for (int i = 0; i < rowEntries.size(); i++) {
-                            dataSize += CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i));
-                        }
-                        messageSize += dataSize;
-                        messageSize += 1 * rowEntries.size();
-                        // packet size
-                        int size = 0;
-                        size += CodedOutputStream.computeEnumSize(3,
-                            PacketType.MESSAGES.getNumber());
-                        size += CodedOutputStream.computeTagSize(5)
-                                + CodedOutputStream.computeRawVarint32Size(messageSize)
-                                + messageSize;
-                        // build data
-                        byte[] body = new byte[size];
-                        CodedOutputStream output = CodedOutputStream.newInstance(body);
-                        output.writeEnum(3, PacketType.MESSAGES.getNumber());
-
-                        output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
-                        output.writeRawVarint32(messageSize);
-                        // message
-                        output.writeInt64(1, data.getId());
-                        for (int i = 0; i < rowEntries.size(); i++) {
-                            output.writeBytes(2, rowEntries.get(i));
-                        }
-                        output.checkNoSpaceLeft();
-                        return body;
-                    } else if (!CollectionUtils.isEmpty(data.getEntries())) {
-                        CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
-                        for (CanalEntry.Entry entry : data.getEntries()) {
-                            messageBuilder.addMessages(entry.toByteString());
-                        }
-
-                        CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
-                        packetBuilder.setType(PacketType.MESSAGES);
-                        packetBuilder.setBody(messageBuilder.build().toByteString());
-                        return packetBuilder.build().toByteArray();
-                    }
-                }
-            }
-        } catch (Exception e) {
-            throw new SerializationException("Error when serializing message to byte[] ");
-        }
-        return null;
+        return CanalMessageSerializer.serializer(data, filterTransactionEntry);
     }
 
     @Override
diff --git a/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java b/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java
new file mode 100644
index 00000000..ff7db991
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java
@@ -0,0 +1,230 @@
+package com.alibaba.otter.canal.rocketmq;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.MessageQueueSelector;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.common.CanalMessageSerializer;
+import com.alibaba.otter.canal.common.MQMessageUtils;
+import com.alibaba.otter.canal.common.MQProperties;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.server.exception.CanalServerException;
+import com.alibaba.otter.canal.spi.CanalMQProducer;
+import com.aliyun.openservices.apache.api.impl.authority.SessionCredentials;
+import com.aliyun.openservices.apache.api.impl.rocketmq.ClientRPCHook;
+
+public class CanalRocketMQProducer implements CanalMQProducer {
+
+    private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class);
+
+    private DefaultMQProducer   defaultMQProducer;
+
+    private MQProperties        mqProperties;
+
+    @Override
+    public void init(MQProperties rocketMQProperties) {
+        this.mqProperties = rocketMQProperties;
+        RPCHook rpcHook = null;
+        if (rocketMQProperties.getAliyunAccessKey().length() > 0
+            && rocketMQProperties.getAliyunSecretKey().length() > 0) {
+            SessionCredentials sessionCredentials = new SessionCredentials();
+            sessionCredentials.setAccessKey(rocketMQProperties.getAliyunAccessKey());
+            sessionCredentials.setSecretKey(rocketMQProperties.getAliyunSecretKey());
+            rpcHook = new ClientRPCHook(sessionCredentials);
+        }
+
+        defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(), rpcHook);
+        defaultMQProducer.setNamesrvAddr(rocketMQProperties.getServers());
+        defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetries());
+        defaultMQProducer.setVipChannelEnabled(false);
+        logger.info("##Start RocketMQ producer##");
+        try {
+            defaultMQProducer.start();
+        } catch (MQClientException ex) {
+            throw new CanalServerException("Start RocketMQ producer error", ex);
+        }
+    }
+
+    @Override
+    public void send(final MQProperties.CanalDestination destination, com.alibaba.otter.canal.protocol.Message data,
+                     Callback callback) {
+        try {
+            if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
+                // 动态topic
+                Map messageMap = MQMessageUtils
+                    .messageTopics(data, destination.getTopic(), destination.getDynamicTopic());
+
+                for (Map.Entry entry : messageMap.entrySet()) {
+                    String topicName = entry.getKey().replace('.', '_');
+                    com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
+                    send(destination, topicName, messageSub);
+                }
+            } else {
+                send(destination, destination.getTopic(), data);
+            }
+            callback.commit();
+        } catch (Exception e) {
+            callback.rollback();
+        }
+    }
+
+    public void send(final MQProperties.CanalDestination destination, String topicName,
+                     com.alibaba.otter.canal.protocol.Message data) throws Exception {
+        if (!mqProperties.getFlatMessage()) {
+            try {
+                if (destination.getPartition() != null) {
+                    Message message = new Message(topicName,
+                        CanalMessageSerializer.serializer(data, mqProperties.isFilterTransactionEntry()));
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("send message:{} to destination:{}, partition: {}",
+                            message,
+                            destination.getCanalDestination(),
+                            destination.getPartition());
+                    }
+                    this.defaultMQProducer.send(message, new MessageQueueSelector() {
+
+                        @Override
+                        public MessageQueue select(List mqs, Message msg, Object arg) {
+                            int partition = 0;
+                            if (destination.getPartition() != null) {
+                                partition = destination.getPartition();
+                            }
+                            return mqs.get(partition);
+                        }
+                    }, null);
+                } else {
+                    if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
+                        com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils
+                            .messagePartition(data, destination.getPartitionsNum(), destination.getPartitionHash());
+                        int length = messages.length;
+                        for (int i = 0; i < length; i++) {
+                            com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
+                            if (dataPartition != null) {
+                                if (logger.isDebugEnabled()) {
+                                    logger.debug("flatMessagePart: {}, partition: {}",
+                                        JSON.toJSONString(dataPartition, SerializerFeature.WriteMapNullValue),
+                                        i);
+                                }
+                                final int index = i;
+                                try {
+                                    Message message = new Message(topicName,
+                                        CanalMessageSerializer.serializer(dataPartition,
+                                            mqProperties.isFilterTransactionEntry()));
+                                    this.defaultMQProducer.send(message, new MessageQueueSelector() {
+
+                                        @Override
+                                        public MessageQueue select(List mqs, Message msg, Object arg) {
+                                            if (index > mqs.size()) {
+                                                throw new CanalServerException("partition number is error,config num:"
+                                                                               + destination.getPartitionsNum()
+                                                                               + ", mq num: " + mqs.size());
+                                            }
+                                            return mqs.get(index);
+                                        }
+                                    }, null);
+                                } catch (Exception e) {
+                                    logger.error("send flat message to hashed partition error", e);
+                                    throw e;
+                                }
+                            }
+                        }
+                    }
+                }
+            } catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) {
+                logger.error("Send message error!", e);
+                throw e;
+            }
+        } else {
+            List flatMessages = MQMessageUtils.messageConverter(data);
+            if (flatMessages != null) {
+                for (FlatMessage flatMessage : flatMessages) {
+                    if (destination.getPartition() != null) {
+                        try {
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("send message: {} to topic: {} fixed partition: {}",
+                                    JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue),
+                                    topicName,
+                                    destination.getPartition());
+                            }
+                            Message message = new Message(topicName,
+                                JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue).getBytes());
+                            this.defaultMQProducer.send(message, new MessageQueueSelector() {
+
+                                @Override
+                                public MessageQueue select(List mqs, Message msg, Object arg) {
+                                    return mqs.get(destination.getPartition());
+                                }
+                            }, null);
+                        } catch (Exception e) {
+                            logger.error("send flat message to fixed partition error", e);
+                            throw e;
+                        }
+                    } else {
+                        if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
+                            FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
+                                destination.getPartitionsNum(),
+                                destination.getPartitionHash());
+                            int length = partitionFlatMessage.length;
+                            for (int i = 0; i < length; i++) {
+                                FlatMessage flatMessagePart = partitionFlatMessage[i];
+                                if (flatMessagePart != null) {
+                                    if (logger.isDebugEnabled()) {
+                                        logger.debug("flatMessagePart: {}, partition: {}",
+                                            JSON.toJSONString(flatMessagePart, SerializerFeature.WriteMapNullValue),
+                                            i);
+                                    }
+                                    final int index = i;
+                                    try {
+                                        Message message = new Message(topicName,
+                                            JSON.toJSONString(flatMessagePart, SerializerFeature.WriteMapNullValue)
+                                                .getBytes());
+                                        this.defaultMQProducer.send(message, new MessageQueueSelector() {
+
+                                            @Override
+                                            public MessageQueue select(List mqs, Message msg,
+                                                                       Object arg) {
+                                                if (index > mqs.size()) {
+                                                    throw new CanalServerException(
+                                                        "partition number is error,config num:"
+                                                                                   + destination.getPartitionsNum()
+                                                                                   + ", mq num: " + mqs.size());
+                                                }
+                                                return mqs.get(index);
+                                            }
+                                        }, null);
+                                    } catch (Exception e) {
+                                        logger.error("send flat message to hashed partition error", e);
+                                        throw e;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        if (logger.isDebugEnabled()) {
+            logger.debug("send message to rocket topic: {}", destination.getTopic());
+        }
+    }
+
+    @Override
+    public void stop() {
+        logger.info("## Stop RocketMQ producer##");
+        this.defaultMQProducer.shutdown();
+    }
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java b/server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java
new file mode 100644
index 00000000..b24bb7cf
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java
@@ -0,0 +1,228 @@
+package com.alibaba.otter.canal.server;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.common.MQProperties;
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.core.CanalMQConfig;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
+import com.alibaba.otter.canal.spi.CanalMQProducer;
+
+public class CanalMQStarter {
+
+    private static final Logger          logger         = LoggerFactory.getLogger(CanalMQStarter.class);
+
+    private volatile boolean             running        = false;
+
+    private ExecutorService              executorService;
+
+    private CanalMQProducer              canalMQProducer;
+
+    private MQProperties                 properties;
+
+    private CanalServerWithEmbedded      canalServer;
+
+    private Map canalMQWorks   = new ConcurrentHashMap<>();
+
+    private static Thread                shutdownThread = null;
+
+    public CanalMQStarter(CanalMQProducer canalMQProducer){
+        this.canalMQProducer = canalMQProducer;
+    }
+
+    public synchronized void start(MQProperties properties) {
+        try {
+            if (running) {
+                return;
+            }
+            this.properties = properties;
+            canalMQProducer.init(properties);
+            // set filterTransactionEntry
+            if (properties.isFilterTransactionEntry()) {
+                System.setProperty("canal.instance.filter.transaction.entry", "true");
+            }
+
+            if (properties.getFlatMessage()) {
+                // 针对flat message模式,设置为raw避免ByteString->Entry的二次解析
+                System.setProperty("canal.instance.memory.rawEntry", "false");
+            }
+
+            canalServer = CanalServerWithEmbedded.instance();
+
+            // 对应每个instance启动一个worker线程
+            executorService = Executors.newCachedThreadPool();
+            logger.info("## start the MQ workers.");
+
+            String[] destinations = StringUtils.split(System.getProperty("canal.destinations"), ",");
+            for (String destination : destinations) {
+                destination = destination.trim();
+                CanalMQRunnable canalMQRunnable = new CanalMQRunnable(destination);
+                canalMQWorks.put(destination, canalMQRunnable);
+                executorService.execute(canalMQRunnable);
+            }
+
+            running = true;
+            logger.info("## the MQ workers is running now ......");
+
+            shutdownThread = new Thread() {
+
+                public void run() {
+                    try {
+                        logger.info("## stop the MQ workers");
+                        running = false;
+                        executorService.shutdown();
+                        canalMQProducer.stop();
+                    } catch (Throwable e) {
+                        logger.warn("##something goes wrong when stopping MQ workers:", e);
+                    } finally {
+                        logger.info("## canal MQ is down.");
+                    }
+                }
+
+            };
+
+            Runtime.getRuntime().addShutdownHook(shutdownThread);
+        } catch (Throwable e) {
+            logger.error("## Something goes wrong when starting up the canal MQ workers:", e);
+        }
+    }
+
+    public synchronized void destroy() {
+        running = false;
+        if (executorService != null) {
+            executorService.shutdown();
+        }
+        if (canalMQProducer != null) {
+            canalMQProducer.stop();
+        }
+        if (shutdownThread != null) {
+            Runtime.getRuntime().removeShutdownHook(shutdownThread);
+            shutdownThread = null;
+        }
+    }
+
+    public synchronized void startDestination(String destination) {
+        CanalInstance canalInstance = canalServer.getCanalInstances().get(destination);
+        if (canalInstance != null) {
+            stopDestination(destination);
+            CanalMQRunnable canalMQRunnable = new CanalMQRunnable(destination);
+            canalMQWorks.put(canalInstance.getDestination(), canalMQRunnable);
+            executorService.execute(canalMQRunnable);
+            logger.info("## Start the MQ work of destination:" + destination);
+        }
+    }
+
+    public synchronized void stopDestination(String destination) {
+        CanalMQRunnable canalMQRunable = canalMQWorks.get(destination);
+        if (canalMQRunable != null) {
+            canalMQRunable.stop();
+            canalMQWorks.remove(destination);
+            logger.info("## Stop the MQ work of destination:" + destination);
+        }
+    }
+
+    private void worker(String destination, AtomicBoolean destinationRunning) {
+        while (!running || !destinationRunning.get())
+            ;
+        logger.info("## start the MQ producer: {}.", destination);
+
+        final ClientIdentity clientIdentity = new ClientIdentity(destination, (short) 1001, "");
+        while (running && destinationRunning.get()) {
+            try {
+                CanalInstance canalInstance = canalServer.getCanalInstances().get(destination);
+                if (canalInstance == null) {
+                    try {
+                        Thread.sleep(3000);
+                    } catch (InterruptedException e) {
+                        // ignore
+                    }
+                    continue;
+                }
+                MQProperties.CanalDestination canalDestination = new MQProperties.CanalDestination();
+                canalDestination.setCanalDestination(destination);
+                CanalMQConfig mqConfig = canalInstance.getMqConfig();
+                canalDestination.setTopic(mqConfig.getTopic());
+                canalDestination.setPartition(mqConfig.getPartition());
+                canalDestination.setDynamicTopic(mqConfig.getDynamicTopic());
+                canalDestination.setPartitionsNum(mqConfig.getPartitionsNum());
+                canalDestination.setPartitionHash(mqConfig.getPartitionHash());
+
+                canalServer.subscribe(clientIdentity);
+                logger.info("## the MQ producer: {} is running now ......", destination);
+
+                Long getTimeout = properties.getCanalGetTimeout();
+                int getBatchSize = properties.getCanalBatchSize();
+                while (running && destinationRunning.get()) {
+                    Message message;
+                    if (getTimeout != null && getTimeout > 0) {
+                        message = canalServer
+                            .getWithoutAck(clientIdentity, getBatchSize, getTimeout, TimeUnit.MILLISECONDS);
+                    } else {
+                        message = canalServer.getWithoutAck(clientIdentity, getBatchSize);
+                    }
+
+                    final long batchId = message.getId();
+                    try {
+                        int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
+                        if (batchId != -1 && size != 0) {
+                            canalMQProducer.send(canalDestination, message, new CanalMQProducer.Callback() {
+
+                                @Override
+                                public void commit() {
+                                    canalServer.ack(clientIdentity, batchId); // 提交确认
+                                }
+
+                                @Override
+                                public void rollback() {
+                                    canalServer.rollback(clientIdentity, batchId);
+                                }
+                            }); // 发送message到topic
+                        } else {
+                            try {
+                                Thread.sleep(100);
+                            } catch (InterruptedException e) {
+                                // ignore
+                            }
+                        }
+
+                    } catch (Exception e) {
+                        logger.error(e.getMessage(), e);
+                    }
+                }
+            } catch (Exception e) {
+                logger.error("process error!", e);
+            }
+        }
+    }
+
+    private class CanalMQRunnable implements Runnable {
+
+        private String destination;
+
+        CanalMQRunnable(String destination){
+            this.destination = destination;
+        }
+
+        private AtomicBoolean running = new AtomicBoolean(true);
+
+        @Override
+        public void run() {
+            worker(destination, running);
+        }
+
+        public void stop() {
+            running.set(false);
+        }
+    }
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java b/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java
index a4f74c90..b07062ee 100644
--- a/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java
+++ b/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java
@@ -1,11 +1,12 @@
 package com.alibaba.otter.canal.server.embedded;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.ServiceLoader;
 import java.util.concurrent.TimeUnit;
 
-import com.alibaba.otter.canal.spi.CanalMetricsProvider;
-import com.alibaba.otter.canal.spi.CanalMetricsService;
-import com.alibaba.otter.canal.spi.NopCanalMetricsService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.MDC;
@@ -14,6 +15,7 @@ import org.springframework.util.CollectionUtils;
 import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
 import com.alibaba.otter.canal.instance.core.CanalInstance;
 import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
+import com.alibaba.otter.canal.protocol.CanalEntry;
 import com.alibaba.otter.canal.protocol.ClientIdentity;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.protocol.position.LogPosition;
@@ -22,7 +24,11 @@ import com.alibaba.otter.canal.protocol.position.PositionRange;
 import com.alibaba.otter.canal.server.CanalServer;
 import com.alibaba.otter.canal.server.CanalService;
 import com.alibaba.otter.canal.server.exception.CanalServerException;
+import com.alibaba.otter.canal.spi.CanalMetricsProvider;
+import com.alibaba.otter.canal.spi.CanalMetricsService;
+import com.alibaba.otter.canal.spi.NopCanalMetricsService;
 import com.alibaba.otter.canal.store.CanalEventStore;
+import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer;
 import com.alibaba.otter.canal.store.model.Event;
 import com.alibaba.otter.canal.store.model.Events;
 import com.google.common.base.Function;
@@ -40,12 +46,12 @@ import com.google.protobuf.ByteString;
  */
 public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements CanalServer, CanalService {
 
-    private static final Logger        logger           = LoggerFactory.getLogger(CanalServerWithEmbedded.class);
+    private static final Logger        logger  = LoggerFactory.getLogger(CanalServerWithEmbedded.class);
     private Map canalInstances;
     // private Map lastRollbackPostions;
     private CanalInstanceGenerator     canalInstanceGenerator;
     private int                        metricsPort;
-    private CanalMetricsService        metrics          = NopCanalMetricsService.NOP;
+    private CanalMetricsService        metrics = NopCanalMetricsService.NOP;
 
     private static class SingletonHolder {
 
@@ -207,7 +213,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
      * b. 如果timeout不为null
      *    1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
      *    2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
-     *
+     * 
      * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
      *
    */ @@ -239,12 +245,23 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C } else { // 记录到流式信息 Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange()); - List entrys = Lists.transform(events.getEvents(), new Function() { + boolean raw = isRaw(canalInstance.getEventStore()); + List entrys = null; + if (raw) { + entrys = Lists.transform(events.getEvents(), new Function() { - public ByteString apply(Event input) { - return input.getRawEntry(); - } - }); + public ByteString apply(Event input) { + return input.getRawEntry(); + } + }); + } else { + entrys = Lists.transform(events.getEvents(), new Function() { + + public CanalEntry.Entry apply(Event input) { + return input.getEntry(); + } + }); + } if (logger.isInfoEnabled()) { logger.info("get successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]", clientIdentity.getClientId(), @@ -255,7 +272,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C } // 直接提交ack ack(clientIdentity, batchId); - return new Message(batchId, true, entrys); + return new Message(batchId, raw, entrys); } } } @@ -283,7 +300,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C * b. 如果timeout不为null * 1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回 * 2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少 - * + * * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题) * */ @@ -311,7 +328,8 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C } if (CollectionUtils.isEmpty(events.getEvents())) { - // logger.debug("getWithoutAck successfully, clientId:{} batchSize:{} but result + // logger.debug("getWithoutAck successfully, clientId:{} + // batchSize:{} but result // is null", // clientIdentity.getClientId(), // batchSize); @@ -319,12 +337,23 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C } else { // 记录到流式信息 Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange()); - List entrys = Lists.transform(events.getEvents(), new Function() { + boolean raw = isRaw(canalInstance.getEventStore()); + List entrys = null; + if (raw) { + entrys = Lists.transform(events.getEvents(), new Function() { - public ByteString apply(Event input) { - return input.getRawEntry(); - } - }); + public ByteString apply(Event input) { + return input.getRawEntry(); + } + }); + } else { + entrys = Lists.transform(events.getEvents(), new Function() { + + public CanalEntry.Entry apply(Event input) { + return input.getEntry(); + } + }); + } if (logger.isInfoEnabled()) { logger.info("getWithoutAck successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]", clientIdentity.getClientId(), @@ -333,7 +362,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C batchId, events.getPositionRange()); } - return new Message(batchId, true, entrys); + return new Message(batchId, raw, entrys); } } @@ -515,17 +544,25 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C // 发现provider, 进行初始化 if (list.size() > 1) { logger.warn("Found more than one CanalMetricsProvider, use the first one."); - //报告冲突 + // 报告冲突 for (CanalMetricsProvider p : list) { logger.warn("Found CanalMetricsProvider: {}.", p.getClass().getName()); } } - //默认使用第一个 + // 默认使用第一个 CanalMetricsProvider provider = list.get(0); this.metrics = provider.getService(); } } + private boolean isRaw(CanalEventStore eventStore) { + if (eventStore instanceof MemoryEventStoreWithBuffer) { + return ((MemoryEventStoreWithBuffer) eventStore).isRaw(); + } + + return true; + } + // ========= setter ========== public void setCanalInstanceGenerator(CanalInstanceGenerator canalInstanceGenerator) { diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java index fe8c5538..ba0432b8 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java @@ -3,7 +3,6 @@ package com.alibaba.otter.canal.server.netty.handler; import java.util.List; import java.util.concurrent.TimeUnit; -import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.jboss.netty.buffer.ChannelBuffer; @@ -34,6 +33,7 @@ import com.alibaba.otter.canal.protocol.ClientIdentity; import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; import com.alibaba.otter.canal.server.netty.NettyUtils; +import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator; import com.google.protobuf.ByteString; import com.google.protobuf.CodedOutputStream; import com.google.protobuf.WireFormat; @@ -50,6 +50,7 @@ public class SessionHandler extends SimpleChannelHandler { this.embeddedServer = embeddedServer; } + @SuppressWarnings({ "deprecation" }) public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { logger.info("message receives in session handler..."); long start = System.nanoTime(); diff --git a/server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java new file mode 100644 index 00000000..9ddbc80d --- /dev/null +++ b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java @@ -0,0 +1,37 @@ +package com.alibaba.otter.canal.spi; + +import java.io.IOException; + +import com.alibaba.otter.canal.common.MQProperties; +import com.alibaba.otter.canal.protocol.Message; + +public interface CanalMQProducer { + + /** + * Init producer. + * + * @param mqProperties MQ config + */ + void init(MQProperties mqProperties); + + /** + * Send canal message to related topic + * + * @param canalDestination canal mq destination + * @param message canal message + * @throws IOException + */ + void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) throws IOException; + + /** + * Stop MQ producer service + */ + void stop(); + + interface Callback { + + void commit(); + + void rollback(); + } +} diff --git a/server/src/test/java/com/alibaba/otter/canal/server/ProtocolTest.java b/server/src/test/java/com/alibaba/otter/canal/server/ProtocolTest.java index cb078ddf..110c3d72 100644 --- a/server/src/test/java/com/alibaba/otter/canal/server/ProtocolTest.java +++ b/server/src/test/java/com/alibaba/otter/canal/server/ProtocolTest.java @@ -56,6 +56,7 @@ public class ProtocolTest { } } + @SuppressWarnings("deprecation") private byte[] buildData(Message message) throws IOException { List rowEntries = message.getRawEntries(); // message size diff --git a/sink/pom.xml b/sink/pom.xml index 1a1fef1e..5e706819 100644 --- a/sink/pom.xml +++ b/sink/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.1.1-SNAPSHOT + 1.1.3-SNAPSHOT ../pom.xml com.alibaba.otter diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java index 7afde6b6..27df9074 100644 --- a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java +++ b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java @@ -20,6 +20,7 @@ import com.alibaba.otter.canal.sink.CanalEventDownStreamHandler; import com.alibaba.otter.canal.sink.CanalEventSink; import com.alibaba.otter.canal.sink.exception.CanalSinkException; import com.alibaba.otter.canal.store.CanalEventStore; +import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer; import com.alibaba.otter.canal.store.model.Event; /** @@ -33,13 +34,17 @@ public class EntryEventSink extends AbstractCanalEventSink eventStore; - protected boolean filterTransactionEntry = false; // 是否需要过滤事务头/尾 + protected boolean filterTransactionEntry = false; // 是否需要尽可能过滤事务头/尾 protected boolean filterEmtryTransactionEntry = true; // 是否需要过滤空的事务头/尾 protected long emptyTransactionInterval = 5 * 1000; // 空的事务输出的频率 - protected long emptyTransctionThresold = 8192; // 超过1024个事务头，输出一个 + protected long emptyTransctionThresold = 8192; // 超过8192个事务头，输出一个 + + protected volatile long lastTransactionTimestamp = 0L; + protected AtomicLong lastTransactionCount = new AtomicLong(0L); protected volatile long lastEmptyTransactionTimestamp = 0L; protected AtomicLong lastEmptyTransactionCount = new AtomicLong(0L); - private AtomicLong eventsSinkBlockingTime = new AtomicLong(0L); + protected AtomicLong eventsSinkBlockingTime = new AtomicLong(0L); + protected boolean raw; public EntryEventSink(){ addHandler(new HeartBeatEntryEventHandler()); @@ -49,6 +54,10 @@ public class EntryEventSink extends AbstractCanalEventSink entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException { - List rowDatas = entrys; - if (filterTransactionEntry) { - rowDatas = new ArrayList(); - for (CanalEntry.Entry entry : entrys) { - if (entry.getEntryType() == EntryType.ROWDATA) { - rowDatas.add(entry); - } - } - } - - return sinkData(rowDatas, remoteAddress); + return sinkData(entrys, remoteAddress); } private boolean sinkData(List entrys, InetSocketAddress remoteAddress) @@ -97,17 +96,27 @@ public class EntryEventSink extends AbstractCanalEventSink com.alibaba.otter canal - 1.1.1-SNAPSHOT + 1.1.3-SNAPSHOT ../pom.xml com.alibaba.otter diff --git a/store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java b/store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java index 5fdb0323..7f5eeb47 100644 --- a/store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java +++ b/store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java @@ -7,6 +7,8 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; +import org.apache.commons.lang.StringUtils; + import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalEntry.EventType; import com.alibaba.otter.canal.protocol.position.LogPosition; @@ -37,14 +39,14 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple private static final long INIT_SEQUENCE = -1; private int bufferSize = 16 * 1024; - private int bufferMemUnit = 1024; // memsize的单位，默认为1kb大小 + private int bufferMemUnit = 1024; // memsize的单位，默认为1kb大小 private int indexMask; private Event[] entries; // 记录下put/get/ack操作的三个下标 - private AtomicLong putSequence = new AtomicLong(INIT_SEQUENCE); // 代表当前put操作最后一次写操作发生的位置 - private AtomicLong getSequence = new AtomicLong(INIT_SEQUENCE); // 代表当前get操作读取的最后一条的位置 - private AtomicLong ackSequence = new AtomicLong(INIT_SEQUENCE); // 代表当前ack操作的最后一条的位置 + private AtomicLong putSequence = new AtomicLong(INIT_SEQUENCE); // 代表当前put操作最后一次写操作发生的位置 + private AtomicLong getSequence = new AtomicLong(INIT_SEQUENCE); // 代表当前get操作读取的最后一条的位置 + private AtomicLong ackSequence = new AtomicLong(INIT_SEQUENCE); // 代表当前ack操作的最后一条的位置 // 记录下put/get/ack操作的三个memsize大小 private AtomicLong putMemSize = new AtomicLong(0); @@ -66,8 +68,9 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple private Condition notFull = lock.newCondition(); private Condition notEmpty = lock.newCondition(); - private BatchMode batchMode = BatchMode.ITEMSIZE; // 默认为内存大小模式 + private BatchMode batchMode = BatchMode.ITEMSIZE; // 默认为内存大小模式 private boolean ddlIsolation = false; + private boolean raw = true; // 针对entry是否开启raw模式 public MemoryEventStoreWithBuffer(){ @@ -335,7 +338,8 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple for (int i = entrys.size() - 1; i >= 0; i--) { Event event = entrys.get(i); - if (CanalEntry.EntryType.TRANSACTIONBEGIN == event.getEntryType() + // GTID模式,ack的位点必须是事务结尾,因为下一次订阅的时候mysql会发送这个gtid之后的next,如果在事务头就记录了会丢这最后一个事务 + if ((CanalEntry.EntryType.TRANSACTIONBEGIN == event.getEntryType() && StringUtils.isEmpty(event.getGtid())) || CanalEntry.EntryType.TRANSACTIONEND == event.getEntryType() || isDdl(event.getEventType())) { // 将事务头/尾设置可被为ack的点 range.setAck(CanalEventUtils.createPosition(event)); @@ -625,6 +629,14 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple this.ddlIsolation = ddlIsolation; } + public boolean isRaw() { + return raw; + } + + public void setRaw(boolean raw) { + this.raw = raw; + } + public AtomicLong getPutSequence() { return putSequence; } @@ -668,4 +680,6 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple public AtomicLong getAckTableRows() { return ackTableRows; } + + } diff --git a/store/src/main/java/com/alibaba/otter/canal/store/model/Event.java b/store/src/main/java/com/alibaba/otter/canal/store/model/Event.java index 3fed5251..db1618e1 100644 --- a/store/src/main/java/com/alibaba/otter/canal/store/model/Event.java +++ b/store/src/main/java/com/alibaba/otter/canal/store/model/Event.java @@ -34,10 +34,17 @@ public class Event implements Serializable { private long rawLength; private int rowsCount; + // ==== https://github.com/alibaba/canal/issues/1019 + private CanalEntry.Entry entry; + public Event(){ } public Event(LogIdentity logIdentity, CanalEntry.Entry entry){ + this(logIdentity, entry, true); + } + + public Event(LogIdentity logIdentity, CanalEntry.Entry entry, boolean raw){ this.logIdentity = logIdentity; this.entryType = entry.getEntryType(); this.executeTime = entry.getHeader().getExecuteTime(); @@ -46,9 +53,6 @@ public class Event implements Serializable { this.serverId = entry.getHeader().getServerId(); this.gtid = entry.getHeader().getGtid(); this.eventType = entry.getHeader().getEventType(); - // build raw - this.rawEntry = entry.toByteString(); - this.rawLength = rawEntry.size(); if (entryType == EntryType.ROWDATA) { List props = entry.getHeader().getPropsList(); if (props != null) { @@ -60,6 +64,16 @@ public class Event implements Serializable { } } } + + if (raw) { + // build raw + this.rawEntry = entry.toByteString(); + this.rawLength = rawEntry.size(); + } else { + this.entry = entry; + // 按照6倍的event length预估 + this.rawLength = entry.getHeader().getEventLength() * 6; + } } public LogIdentity getLogIdentity() { @@ -150,6 +164,14 @@ public class Event implements Serializable { this.rowsCount = rowsCount; } + public CanalEntry.Entry getEntry() { + return entry; + } + + public void setEntry(CanalEntry.Entry entry) { + this.entry = entry; + } + public String toString() { return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); }