>();
+ private static final ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean();
+
+ static {
+ // regist Converter
+ convertUtilsBean.register(SqlTimestampConverter.SQL_TIMESTAMP, Date.class);
+ convertUtilsBean.register(SqlTimestampConverter.SQL_TIMESTAMP, Time.class);
+ convertUtilsBean.register(SqlTimestampConverter.SQL_TIMESTAMP, Timestamp.class);
+ convertUtilsBean.register(ByteArrayConverter.SQL_BYTES, byte[].class);
+
+ // bool
+ sqlTypeToJavaTypeMap.put(Types.BOOLEAN, Boolean.class);
+
+ // int
+ sqlTypeToJavaTypeMap.put(Types.TINYINT, Integer.class);
+ sqlTypeToJavaTypeMap.put(Types.SMALLINT, Integer.class);
+ sqlTypeToJavaTypeMap.put(Types.INTEGER, Integer.class);
+
+ // long
+ sqlTypeToJavaTypeMap.put(Types.BIGINT, Long.class);
+ // mysql bit最多64位，无符号
+ sqlTypeToJavaTypeMap.put(Types.BIT, BigInteger.class);
+
+ // decimal
+ sqlTypeToJavaTypeMap.put(Types.REAL, Float.class);
+ sqlTypeToJavaTypeMap.put(Types.FLOAT, Float.class);
+ sqlTypeToJavaTypeMap.put(Types.DOUBLE, Double.class);
+ sqlTypeToJavaTypeMap.put(Types.NUMERIC, BigDecimal.class);
+ sqlTypeToJavaTypeMap.put(Types.DECIMAL, BigDecimal.class);
+
+ // date
+ sqlTypeToJavaTypeMap.put(Types.DATE, Date.class);
+ sqlTypeToJavaTypeMap.put(Types.TIME, Time.class);
+ sqlTypeToJavaTypeMap.put(Types.TIMESTAMP, Timestamp.class);
+
+ // blob
+ sqlTypeToJavaTypeMap.put(Types.BLOB, byte[].class);
+
+ // byte[]
+ sqlTypeToJavaTypeMap.put(Types.REF, byte[].class);
+ sqlTypeToJavaTypeMap.put(Types.OTHER, byte[].class);
+ sqlTypeToJavaTypeMap.put(Types.ARRAY, byte[].class);
+ sqlTypeToJavaTypeMap.put(Types.STRUCT, byte[].class);
+ sqlTypeToJavaTypeMap.put(Types.SQLXML, byte[].class);
+ sqlTypeToJavaTypeMap.put(Types.BINARY, byte[].class);
+ sqlTypeToJavaTypeMap.put(Types.DATALINK, byte[].class);
+ sqlTypeToJavaTypeMap.put(Types.DISTINCT, byte[].class);
+ sqlTypeToJavaTypeMap.put(Types.VARBINARY, byte[].class);
+ sqlTypeToJavaTypeMap.put(Types.JAVA_OBJECT, byte[].class);
+ sqlTypeToJavaTypeMap.put(Types.LONGVARBINARY, byte[].class);
+
+ // String
+ sqlTypeToJavaTypeMap.put(Types.CHAR, String.class);
+ sqlTypeToJavaTypeMap.put(Types.VARCHAR, String.class);
+ sqlTypeToJavaTypeMap.put(Types.LONGVARCHAR, String.class);
+ sqlTypeToJavaTypeMap.put(Types.LONGNVARCHAR, String.class);
+ sqlTypeToJavaTypeMap.put(Types.NCHAR, String.class);
+ sqlTypeToJavaTypeMap.put(Types.NVARCHAR, String.class);
+ sqlTypeToJavaTypeMap.put(Types.NCLOB, String.class);
+ sqlTypeToJavaTypeMap.put(Types.CLOB, String.class);
+ }
+
+ /**
+ * 将指定java.sql.Types的ResultSet value转换成相应的String
+ *
+ * @param rs
+ * @param index
+ * @param sqlType
+ * @return
+ * @throws SQLException
+ */
+ public static String sqlValueToString(ResultSet rs, int index, int sqlType) throws SQLException {
+ Class requiredType = sqlTypeToJavaTypeMap.get(sqlType);
+ if (requiredType == null) {
+ throw new IllegalArgumentException("unknow java.sql.Types - " + sqlType);
+ }
+
+ return getResultSetValue(rs, index, requiredType);
+ }
+
+ /**
+ * sqlValueToString方法的逆向过程
+ *
+ * @param value
+ * @param sqlType
+ * @param isTextRequired
+ * @param isEmptyStringNulled
+ * @return
+ */
+ public static Object stringToSqlValue(String value, int sqlType, boolean isRequired, boolean isEmptyStringNulled) {
+ // 设置变量
+ String sourceValue = value;
+ if (SqlUtils.isTextType(sqlType)) {
+ if ((sourceValue == null) || (StringUtils.isEmpty(sourceValue) && isEmptyStringNulled)) {
+ return isRequired ? REQUIRED_FIELD_NULL_SUBSTITUTE : null;
+ } else {
+ return sourceValue;
+ }
+ } else {
+ if (StringUtils.isEmpty(sourceValue)) {
+ return null;
+ } else {
+ Class requiredType = sqlTypeToJavaTypeMap.get(sqlType);
+ if (requiredType == null) {
+ throw new IllegalArgumentException("unknow java.sql.Types - " + sqlType);
+ } else if (requiredType.equals(String.class)) {
+ return sourceValue;
+ } else if (isNumeric(sqlType)) {
+ return convertUtilsBean.convert(sourceValue.trim(), requiredType);
+ } else {
+ return convertUtilsBean.convert(sourceValue, requiredType);
+ }
+ }
+ }
+ }
+
+ public static String encoding(String source, int sqlType, String sourceEncoding, String targetEncoding) {
+ switch (sqlType) {
+ case Types.CHAR:
+ case Types.VARCHAR:
+ case Types.LONGVARCHAR:
+ case Types.NCHAR:
+ case Types.NVARCHAR:
+ case Types.LONGNVARCHAR:
+ case Types.CLOB:
+ case Types.NCLOB:
+ if (false == StringUtils.isEmpty(source)) {
+ String fromEncoding = StringUtils.isBlank(sourceEncoding) ? "UTF-8" : sourceEncoding;
+ String toEncoding = StringUtils.isBlank(targetEncoding) ? "UTF-8" : targetEncoding;
+
+ // if (false == StringUtils.equalsIgnoreCase(fromEncoding,
+ // toEncoding)) {
+ try {
+ return new String(source.getBytes(fromEncoding), toEncoding);
+ } catch (UnsupportedEncodingException e) {
+ throw new IllegalArgumentException(e.getMessage(), e);
+ }
+ // }
+ }
+ }
+
+ return source;
+ }
+
+ /**
+ * Retrieve a JDBC column value from a ResultSet, using the specified value
+ * type.
+ *
+ * Uses the specifically typed ResultSet accessor methods, falling back to
+ * {@link #getResultSetValue(ResultSet, int)} for unknown types.
+ *
+ * Note that the returned value may not be assignable to the specified
+ * required type, in case of an unknown type. Calling code needs to deal
+ * with this case appropriately, e.g. throwing a corresponding exception.
+ *
+ * @param rs is the ResultSet holding the data
+ * @param index is the column index
+ * @param requiredType the required value type (may be null)
+ * @return the value object
+ * @throws SQLException if thrown by the JDBC API
+ */
+ private static String getResultSetValue(ResultSet rs, int index, Class requiredType) throws SQLException {
+ if (requiredType == null) {
+ return getResultSetValue(rs, index);
+ }
+
+ Object value = null;
+ boolean wasNullCheck = false;
+
+ // Explicitly extract typed value, as far as possible.
+ if (String.class.equals(requiredType)) {
+ value = rs.getString(index);
+ } else if (boolean.class.equals(requiredType) || Boolean.class.equals(requiredType)) {
+ value = Boolean.valueOf(rs.getBoolean(index));
+ wasNullCheck = true;
+ } else if (byte.class.equals(requiredType) || Byte.class.equals(requiredType)) {
+ value = new Byte(rs.getByte(index));
+ wasNullCheck = true;
+ } else if (short.class.equals(requiredType) || Short.class.equals(requiredType)) {
+ value = new Short(rs.getShort(index));
+ wasNullCheck = true;
+ } else if (int.class.equals(requiredType) || Integer.class.equals(requiredType)) {
+ value = new Long(rs.getLong(index));
+ wasNullCheck = true;
+ } else if (long.class.equals(requiredType) || Long.class.equals(requiredType)) {
+ value = rs.getBigDecimal(index);
+ wasNullCheck = true;
+ } else if (float.class.equals(requiredType) || Float.class.equals(requiredType)) {
+ value = new Float(rs.getFloat(index));
+ wasNullCheck = true;
+ } else if (double.class.equals(requiredType) || Double.class.equals(requiredType)
+ || Number.class.equals(requiredType)) {
+ value = new Double(rs.getDouble(index));
+ wasNullCheck = true;
+ } else if (Time.class.equals(requiredType)) {
+ // try {
+ // value = rs.getTime(index);
+ // } catch (SQLException e) {
+ value = rs.getString(index);// 尝试拿为string对象，0000无法用Time表示
+ // if (value == null && !rs.wasNull()) {
+ // value = "00:00:00"; //
+ // mysql设置了zeroDateTimeBehavior=convertToNull，出现0值时返回为null
+ // }
+ // }
+ } else if (Timestamp.class.equals(requiredType) || Date.class.equals(requiredType)) {
+ // try {
+ // value = convertTimestamp(rs.getTimestamp(index));
+ // } catch (SQLException e) {
+ // 尝试拿为string对象，0000-00-00 00:00:00无法用Timestamp 表示
+ value = rs.getString(index);
+ // if (value == null && !rs.wasNull()) {
+ // value = "0000:00:00 00:00:00"; //
+ // mysql设置了zeroDateTimeBehavior=convertToNull，出现0值时返回为null
+ // }
+ // }
+ } else if (BigDecimal.class.equals(requiredType)) {
+ value = rs.getBigDecimal(index);
+ } else if (BigInteger.class.equals(requiredType)) {
+ value = rs.getBigDecimal(index);
+ } else if (Blob.class.equals(requiredType)) {
+ value = rs.getBlob(index);
+ } else if (Clob.class.equals(requiredType)) {
+ value = rs.getClob(index);
+ } else if (byte[].class.equals(requiredType)) {
+ try {
+ byte[] bytes = rs.getBytes(index);
+ if (bytes == null) {
+ value = null;
+ } else {
+ value = new String(bytes, "ISO-8859-1");// 将binary转化为iso-8859-1的字符串
+ }
+ } catch (UnsupportedEncodingException e) {
+ throw new SQLException(e);
+ }
+ } else {
+ // Some unknown type desired -> rely on getObject.
+ value = getResultSetValue(rs, index);
+ }
+
+ // Perform was-null check if demanded (for results that the
+ // JDBC driver returns as primitives).
+ if (wasNullCheck && (value != null) && rs.wasNull()) {
+ value = null;
+ }
+
+ return (value == null) ? null : convertUtilsBean.convert(value);
+ }
+
+ /**
+ * Retrieve a JDBC column value from a ResultSet, using the most appropriate
+ * value type. The returned value should be a detached value object, not
+ * having any ties to the active ResultSet: in particular, it should not be
+ * a Blob or Clob object but rather a byte array respectively String
+ * representation.
+ *
+ * Uses the getObject(index) method, but includes additional
+ * "hacks" to get around Oracle 10g returning a non-standard object for its
+ * TIMESTAMP datatype and a java.sql.Date for DATE columns
+ * leaving out the time portion: These columns will explicitly be extracted
+ * as standard java.sql.Timestamp object.
+ *
+ * @param rs is the ResultSet holding the data
+ * @param index is the column index
+ * @return the value object
+ * @throws SQLException if thrown by the JDBC API
+ * @see Blob
+ * @see Clob
+ * @see Timestamp
+ */
+ private static String getResultSetValue(ResultSet rs, int index) throws SQLException {
+ Object obj = rs.getObject(index);
+ return (obj == null) ? null : convertUtilsBean.convert(obj);
+ }
+
+ // private static Object convertTimestamp(Timestamp timestamp) {
+ // return (timestamp == null) ? null : timestamp.getTime();
+ // }
+
+ /**
+ * Check whether the given SQL type is numeric.
+ */
+ public static boolean isNumeric(int sqlType) {
+ return (Types.BIT == sqlType) || (Types.BIGINT == sqlType) || (Types.DECIMAL == sqlType)
+ || (Types.DOUBLE == sqlType) || (Types.FLOAT == sqlType) || (Types.INTEGER == sqlType)
+ || (Types.NUMERIC == sqlType) || (Types.REAL == sqlType) || (Types.SMALLINT == sqlType)
+ || (Types.TINYINT == sqlType);
+ }
+
+ public static boolean isTextType(int sqlType) {
+ if (sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.CLOB || sqlType == Types.LONGVARCHAR
+ || sqlType == Types.NCHAR || sqlType == Types.NVARCHAR || sqlType == Types.NCLOB
+ || sqlType == Types.LONGNVARCHAR) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
diff --git a/example/src/main/resources/client-spring.xml b/example/src/main/resources/client-spring.xml
new file mode 100644
index 00000000..29eba1af
--- /dev/null
+++ b/example/src/main/resources/client-spring.xml
@@ -0,0 +1,53 @@
+
+
+
+
+
+
+
+
+ classpath:client.properties
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/example/src/main/resources/client.properties b/example/src/main/resources/client.properties
new file mode 100644
index 00000000..c5142912
--- /dev/null
+++ b/example/src/main/resources/client.properties
@@ -0,0 +1,16 @@
+# client 配置
+zk.servers=127.0.0.1:2181
+# 5 * 1024
+client.batch.size=5120
+client.debug=false
+client.destination=example
+client.username=canal
+client.password=canal
+client.exceptionstrategy=1
+client.retrytimes=3
+client.filter=.*\\..*
+
+# 同步目标: mysql 配置
+target.mysql.url=jdbc:mysql://127.0.0.1:4306
+target.mysql.username=root
+target.mysql.password=123456
diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
index bca5ba13..039a80b7 100644
--- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
+++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
@@ -6,6 +6,11 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.HistoryTableMetaCache;
+
+import com.alibaba.otter.canal.meta.FileMixedMetaManager;
+
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -20,13 +25,7 @@ import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
import com.alibaba.otter.canal.instance.core.AbstractCanalInstance;
import com.alibaba.otter.canal.instance.manager.model.Canal;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter;
-import com.alibaba.otter.canal.instance.manager.model.CanalParameter.DataSourcing;
-import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode;
-import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode;
-import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode;
-import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType;
-import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode;
-import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageScavengeMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.*;
import com.alibaba.otter.canal.meta.MemoryMetaManager;
import com.alibaba.otter.canal.meta.PeriodMixedMetaManager;
import com.alibaba.otter.canal.meta.ZooKeeperMetaManager;
@@ -37,12 +36,7 @@ import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
-import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
-import com.alibaba.otter.canal.parse.index.FailbackLogPositionManager;
-import com.alibaba.otter.canal.parse.index.MemoryLogPositionManager;
-import com.alibaba.otter.canal.parse.index.MetaLogPositionManager;
-import com.alibaba.otter.canal.parse.index.PeriodMixedLogPositionManager;
-import com.alibaba.otter.canal.parse.index.ZooKeeperLogPositionManager;
+import com.alibaba.otter.canal.parse.index.*;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.sink.entry.EntryEventSink;
@@ -120,6 +114,11 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
zooKeeperMetaManager.setZkClientx(getZkclientx());
((PeriodMixedMetaManager) metaManager).setZooKeeperMetaManager(zooKeeperMetaManager);
+ } else if (mode.isLocalFile()){
+ FileMixedMetaManager fileMixedMetaManager = new FileMixedMetaManager();
+ fileMixedMetaManager.setDataDir(parameters.getDataDir());
+ fileMixedMetaManager.setPeriod(parameters.getMetaFileFlushPeriod());
+ metaManager = fileMixedMetaManager;
} else {
throw new CanalException("unsupport MetaMode for " + mode);
}
@@ -242,6 +241,13 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
mysqlEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
// 数据库信息参数
mysqlEventParser.setSlaveId(parameters.getSlaveId());
+ mysqlEventParser.setTableMetaStorageFactory(parameters.getTableMetaStorageFactory());
+ // Ctrip callback
+// mysqlEventParser.setCallback(parameters.getCallback());
+// HistoryTableMetaCache cache = new HistoryTableMetaCache();
+// cache.init(parameters.getEntries());
+// mysqlEventParser.setHistoryTableMetaCache(cache);
+
if (!CollectionUtils.isEmpty(dbAddresses)) {
mysqlEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0),
parameters.getDbUsername(),
diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
index 16c929ca..d707c679 100644
--- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
+++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
@@ -5,6 +5,7 @@ import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorageFactory;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.ToStringBuilder;
@@ -28,8 +29,10 @@ public class CanalParameter implements Serializable {
private Long zkClusterId; // zk集群id，为管理方便
private List zkClusters; // zk集群地址
+ private String dataDir = "../conf"; // 默认本地文件数据的目录默认是conf
// meta相关参数
private MetaMode metaMode = MetaMode.MEMORY; // meta机制
+ private Integer metaFileFlushPeriod = 1000; // meta刷新间隔
// storage存储
private Integer transactionSize = 1024; // 支持处理的transaction事务大小
@@ -106,6 +109,9 @@ public class CanalParameter implements Serializable {
private Long standbyLogfileOffest = null;
private Long standbyTimestamp = null;
+ // Ctrip Table Meta
+ TableMetaStorageFactory tableMetaStorageFactory;
+
public static enum RunMode {
/** 嵌入式 */
@@ -243,7 +249,9 @@ public class CanalParameter implements Serializable {
/** 文件存储模式 */
ZOOKEEPER,
/** 混合模式，内存+文件 */
- MIXED;
+ MIXED,
+ /** 本地文件存储模式*/
+ LOCAL_FILE;
public boolean isMemory() {
return this.equals(MetaMode.MEMORY);
@@ -256,6 +264,10 @@ public class CanalParameter implements Serializable {
public boolean isMixed() {
return this.equals(MetaMode.MIXED);
}
+
+ public boolean isLocalFile(){
+ return this.equals(MetaMode.LOCAL_FILE);
+ }
}
public static enum IndexMode {
@@ -390,6 +402,22 @@ public class CanalParameter implements Serializable {
return storageMode;
}
+ public String getDataDir() {
+ return dataDir;
+ }
+
+ public void setDataDir(String dataDir) {
+ this.dataDir = dataDir;
+ }
+
+ public Integer getMetaFileFlushPeriod() {
+ return metaFileFlushPeriod;
+ }
+
+ public void setMetaFileFlushPeriod(Integer metaFileFlushPeriod) {
+ this.metaFileFlushPeriod = metaFileFlushPeriod;
+ }
+
public void setStorageMode(StorageMode storageMode) {
this.storageMode = storageMode;
}
@@ -859,6 +887,14 @@ public class CanalParameter implements Serializable {
this.blackFilter = blackFilter;
}
+ public TableMetaStorageFactory getTableMetaStorageFactory() {
+ return tableMetaStorageFactory;
+ }
+
+ public void setTableMetaStorageFactory(TableMetaStorageFactory tableMetaStorageFactory) {
+ this.tableMetaStorageFactory = tableMetaStorageFactory;
+ }
+
public String toString() {
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
}
diff --git a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/MessageDeserializer.java b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/MessageDeserializer.java
index 1cb3daba..70f2fc06 100644
--- a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/MessageDeserializer.java
+++ b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/MessageDeserializer.java
@@ -29,7 +29,9 @@ public class MessageDeserializer implements Deserializer {
@Override
public Message deserialize(String topic, byte[] data) {
try {
- if (data == null) return null;
+ if (data == null) {
+ return null;
+ }
else {
CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
switch (p.getType()) {
diff --git a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningData.java b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningData.java
index 3eab85ed..7bbbabd7 100644
--- a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningData.java
+++ b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningData.java
@@ -8,8 +8,8 @@ package com.alibaba.otter.canal.kafka.client.running;
*/
public class ClientRunningData {
- private String groupId;
- private String address;
+ private String groupId;
+ private String address;
private boolean active = true;
public String getGroupId() {
diff --git a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningMonitor.java b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningMonitor.java
index 3bdbfe6e..0f61211e 100644
--- a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningMonitor.java
+++ b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningMonitor.java
@@ -1,12 +1,11 @@
package com.alibaba.otter.canal.kafka.client.running;
-import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
-import com.alibaba.otter.canal.common.utils.AddressUtils;
-import com.alibaba.otter.canal.common.utils.BooleanMutex;
-import com.alibaba.otter.canal.common.utils.JsonUtils;
-import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
-import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
-import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import java.text.MessageFormat;
+import java.util.Random;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
import org.I0Itec.zkclient.IZkDataListener;
import org.I0Itec.zkclient.exception.ZkException;
import org.I0Itec.zkclient.exception.ZkInterruptedException;
@@ -17,11 +16,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
-import java.text.MessageFormat;
-import java.util.Random;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.common.utils.AddressUtils;
+import com.alibaba.otter.canal.common.utils.BooleanMutex;
+import com.alibaba.otter.canal.common.utils.JsonUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+
/**
* kafka client running状态信息
@@ -31,13 +33,18 @@ import java.util.concurrent.TimeUnit;
*/
public class ClientRunningMonitor extends AbstractCanalLifeCycle {
- private static final String TOPIC_ROOT_NODE = ZookeeperPathUtils.CANAL_ROOT_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "topics";
+ private static final String TOPIC_ROOT_NODE = ZookeeperPathUtils.CANAL_ROOT_NODE
+ + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "topics";
- private static final String TOPIC_NODE = TOPIC_ROOT_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "{0}";
+ private static final String TOPIC_NODE = TOPIC_ROOT_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR
+ + "{0}";
- private static final String TOPIC_CLIENTID_NODE = TOPIC_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "{1}";
+ private static final String TOPIC_CLIENTID_NODE = TOPIC_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR
+ + "{1}";
- private static final String TOPIC_CLIENTID_RUNNING_NODE = TOPIC_CLIENTID_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + ZookeeperPathUtils.RUNNING_NODE;
+ private static final String TOPIC_CLIENTID_RUNNING_NODE = TOPIC_CLIENTID_NODE
+ + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR
+ + ZookeeperPathUtils.RUNNING_NODE;
private static String getTopicClientRunning(String topic, String groupId) {
return MessageFormat.format(TOPIC_CLIENTID_RUNNING_NODE, topic, groupId);
@@ -47,21 +54,21 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
return MessageFormat.format(TOPIC_CLIENTID_NODE, topic, groupId);
}
- private static final Logger logger = LoggerFactory.getLogger(ClientRunningMonitor.class);
- private ZkClientx zkClient;
- private String topic;
- private ClientRunningData clientData;
- private IZkDataListener dataListener;
- private BooleanMutex mutex = new BooleanMutex(false);
- private volatile boolean release = false;
+ private static final Logger logger = LoggerFactory.getLogger(ClientRunningMonitor.class);
+ private ZkClientx zkClient;
+ private String topic;
+ private ClientRunningData clientData;
+ private IZkDataListener dataListener;
+ private BooleanMutex mutex = new BooleanMutex(false);
+ private volatile boolean release = false;
private volatile ClientRunningData activeData;
- private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1);
- private ClientRunningListener listener;
- private int delayTime = 5;
+ private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1);
+ private ClientRunningListener listener;
+ private int delayTime = 5;
- private static Integer virtualPort;
+ private static Integer virtualPort;
- public ClientRunningMonitor() {
+ public ClientRunningMonitor(){
if (virtualPort == null) {
Random rand = new Random();
virtualPort = rand.nextInt(9000) + 1000;
@@ -108,7 +115,6 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
}
-
public void start() {
super.start();
@@ -123,7 +129,7 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
String path = getTopicClientRunning(this.topic, clientData.getGroupId());
zkClient.unsubscribeDataChanges(path, dataListener);
releaseRunning(); // 尝试一下release
- //Fix issue #697
+ // Fix issue #697
if (delayExector != null) {
delayExector.shutdown();
}
@@ -159,13 +165,12 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
}
}
} catch (ZkNoNodeException e) {
- zkClient.createPersistent(getClientIdNodePath(this.topic, clientData.getGroupId()),
- true); // 尝试创建父节点
+ zkClient.createPersistent(getClientIdNodePath(this.topic, clientData.getGroupId()), true); // 尝试创建父节点
initRunning();
} catch (Throwable t) {
logger.error(MessageFormat.format("There is an error when execute initRunning method, with destination [{0}].",
- topic),
- t);
+ topic),
+ t);
// 出现任何异常尝试release
releaseRunning();
throw new CanalClientException("something goes wrong in initRunning method. ", t);
@@ -187,7 +192,8 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
*/
public boolean check() {
String path = getTopicClientRunning(this.topic, clientData.getGroupId());
- //ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
+ // ZookeeperPathUtils.getDestinationClientRunning(this.destination,
+ // clientData.getClientId());
try {
byte[] bytes = zkClient.readData(path);
ClientRunningData eventData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
@@ -196,8 +202,8 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
boolean result = isMine(activeData.getAddress());
if (!result) {
logger.warn("canal is running in [{}] , but not in [{}]",
- activeData.getAddress(),
- clientData.getAddress());
+ activeData.getAddress(),
+ clientData.getAddress());
}
return result;
} catch (ZkNoNodeException e) {
@@ -235,7 +241,7 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
if (listener != null) {
// 触发回调
listener.processActiveEnter();
- this.clientData.setAddress(/*address*/AddressUtils.getHostIp() + ":" + virtualPort);
+ this.clientData.setAddress(/* address */AddressUtils.getHostIp() + ":" + virtualPort);
String path = getTopicClientRunning(this.topic, clientData.getGroupId());
// 序列化
diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java
index e0b2d1d8..573f0eac 100644
--- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java
+++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java
@@ -17,9 +17,9 @@ import com.alibaba.otter.canal.deployer.CanalController;
*/
public class CanalServerStarter {
- private static final String CLASSPATH_URL_PREFIX = "classpath:";
- private static final Logger logger = LoggerFactory.getLogger(CanalServerStarter.class);
- private volatile static boolean running = false;
+ private static final String CLASSPATH_URL_PREFIX = "classpath:";
+ private static final Logger logger = LoggerFactory.getLogger(CanalServerStarter.class);
+ private volatile static boolean running = false;
public static void init() {
try {
diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java
index 1543abf9..4543aac1 100644
--- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java
+++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java
@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.kafka.producer;
+import java.io.IOException;
import java.util.Properties;
import org.apache.kafka.clients.producer.KafkaProducer;
@@ -10,7 +11,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.kafka.producer.KafkaProperties.Topic;
-import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
/**
@@ -49,21 +49,33 @@ public class CanalKafkaProducer {
}
}
- public void send(Topic topic, Message message) {
- boolean valid = false;
- if (message != null && !message.getEntries().isEmpty()) {
- for (CanalEntry.Entry entry : message.getEntries()) {
- if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN
- && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) {
- valid = true;
- break;
- }
- }
- }
- if (!valid) {
- return;
- }
+ public void send(Topic topic, Message message) throws IOException {
+ // set canal.instance.filter.transaction.entry = true
+ // boolean valid = false;
+ // if (message != null) {
+ // if (message.isRaw() && !message.getRawEntries().isEmpty()) {
+ // for (ByteString byteString : message.getRawEntries()) {
+ // CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString);
+ // if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN
+ // && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) {
+ // valid = true;
+ // break;
+ // }
+ // }
+ // } else if (!message.getEntries().isEmpty()){
+ // for (CanalEntry.Entry entry : message.getEntries()) {
+ // if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN
+ // && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) {
+ // valid = true;
+ // break;
+ // }
+ // }
+ // }
+ // }
+ // if (!valid) {
+ // return;
+ // }
ProducerRecord record;
if (topic.getPartition() != null) {
record = new ProducerRecord(topic.getTopic(), topic.getPartition(), null, message);
diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java
index 0ff018c2..0eedfa7a 100644
--- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java
+++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java
@@ -52,7 +52,10 @@ public class CanalKafkaStarter {
// 初始化 kafka producer
canalKafkaProducer = new CanalKafkaProducer();
canalKafkaProducer.init(kafkaProperties);
-
+ // set filterTransactionEntry
+ if (kafkaProperties.isFilterTransactionEntry()) {
+ System.setProperty("canal.instance.filter.transaction.entry", "true");
+ }
// 对应每个instance启动一个worker线程
List destinations = kafkaProperties.getCanalDestinations();
@@ -118,7 +121,7 @@ public class CanalKafkaStarter {
Message message = server.getWithoutAck(clientIdentity, kafkaProperties.getCanalBatchSize()); // 获取指定数量的数据
long batchId = message.getId();
try {
- int size = message.getEntries().size();
+ int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
if (batchId != -1 && size != 0) {
if (!StringUtils.isEmpty(destination.getTopic())) {
Topic topic = new Topic();
diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/KafkaProperties.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/KafkaProperties.java
index 0b346137..8e193aac 100644
--- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/KafkaProperties.java
+++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/KafkaProperties.java
@@ -13,15 +13,15 @@ import java.util.Set;
*/
public class KafkaProperties {
- private String servers = "localhost:6667";
- private int retries = 0;
- private int batchSize = 16384;
- private int lingerMs = 1;
- private long bufferMemory = 33554432L;
+ private String servers = "localhost:6667";
+ private int retries = 0;
+ private int batchSize = 16384;
+ private int lingerMs = 1;
+ private long bufferMemory = 33554432L;
+ private boolean filterTransactionEntry = true;
+ private int canalBatchSize = 5;
- private int canalBatchSize = 5;
-
- private List canalDestinations = new ArrayList();
+ private List canalDestinations = new ArrayList();
public static class CanalDestination {
@@ -158,4 +158,13 @@ public class KafkaProperties {
public void setCanalDestinations(List canalDestinations) {
this.canalDestinations = canalDestinations;
}
+
+ public boolean isFilterTransactionEntry() {
+ return filterTransactionEntry;
+ }
+
+ public void setFilterTransactionEntry(boolean filterTransactionEntry) {
+ this.filterTransactionEntry = filterTransactionEntry;
+ }
+
}
diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java
index 339e4f8c..02336004 100644
--- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java
+++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java
@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.kafka.producer;
+import java.util.List;
import java.util.Map;
import org.apache.kafka.common.errors.SerializationException;
@@ -8,7 +9,11 @@ import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalPacket;
+import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
import com.alibaba.otter.canal.protocol.Message;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.WireFormat;
/**
* Kafka Message类的序列化
@@ -25,22 +30,59 @@ public class MessageSerializer implements Serializer {
@Override
public byte[] serialize(String topic, Message data) {
try {
- if (data == null) return null;
- else {
- CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
- if (data.getId() != -1 && !CollectionUtils.isEmpty(data.getEntries())) {
- for (CanalEntry.Entry entry : data.getEntries()) {
- messageBuilder.addMessages(entry.toByteString());
+ if (data != null) {
+ if (data.getId() != -1) {
+ if (data.isRaw() && !CollectionUtils.isEmpty(data.getRawEntries())) {
+ // for performance
+ List rowEntries = data.getRawEntries();
+ // message size
+ int messageSize = 0;
+ messageSize += com.google.protobuf.CodedOutputStream.computeInt64Size(1, data.getId());
+
+ int dataSize = 0;
+ for (int i = 0; i < rowEntries.size(); i++) {
+ dataSize += com.google.protobuf.CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i));
+ }
+ messageSize += dataSize;
+ messageSize += 1 * rowEntries.size();
+ // packet size
+ int size = 0;
+ size += com.google.protobuf.CodedOutputStream.computeEnumSize(3,
+ PacketType.MESSAGES.getNumber());
+ size += com.google.protobuf.CodedOutputStream.computeTagSize(5)
+ + com.google.protobuf.CodedOutputStream.computeRawVarint32Size(messageSize)
+ + messageSize;
+ // build data
+ byte[] body = new byte[size];
+ CodedOutputStream output = CodedOutputStream.newInstance(body);
+ output.writeEnum(3, PacketType.MESSAGES.getNumber());
+
+ output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+ output.writeRawVarint32(messageSize);
+ // message
+ output.writeInt64(1, data.getId());
+ for (int i = 0; i < rowEntries.size(); i++) {
+ output.writeBytes(2, rowEntries.get(i));
+ }
+ output.checkNoSpaceLeft();
+ return body;
+ } else if (!CollectionUtils.isEmpty(data.getEntries())) {
+ CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
+ for (CanalEntry.Entry entry : data.getEntries()) {
+ messageBuilder.addMessages(entry.toByteString());
+ }
+
+ CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
+ packetBuilder.setType(CanalPacket.PacketType.MESSAGES);
+ packetBuilder.setBody(messageBuilder.build().toByteString());
+ return packetBuilder.build().toByteArray();
}
}
- CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
- packetBuilder.setType(CanalPacket.PacketType.MESSAGES);
- packetBuilder.setBody(messageBuilder.build().toByteString());
- return packetBuilder.build().toByteArray();
}
} catch (Exception e) {
throw new SerializationException("Error when serializing message to byte[] ");
}
+ return null;
}
@Override
diff --git a/kafka/src/main/resources/kafka.yml b/kafka/src/main/resources/kafka.yml
index ab214319..d6ccc51a 100644
--- a/kafka/src/main/resources/kafka.yml
+++ b/kafka/src/main/resources/kafka.yml
@@ -5,6 +5,7 @@ lingerMs: 1
bufferMemory: 33554432
# canal的批次大小，单位 k
canalBatchSize: 50
+filterTransactionEntry: true
canalDestinations:
- canalDestination: example
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/exception/PositionNotFoundException.java b/parse/src/main/java/com/alibaba/otter/canal/parse/exception/PositionNotFoundException.java
new file mode 100644
index 00000000..d5552a9f
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/exception/PositionNotFoundException.java
@@ -0,0 +1,30 @@
+package com.alibaba.otter.canal.parse.exception;
+
+/**
+ * @author chengjin.lyf on 2018/7/20 下午2:54
+ * @since 1.0.25
+ */
+public class PositionNotFoundException extends CanalParseException {
+
+ private static final long serialVersionUID = -7382448928116244017L;
+
+ public PositionNotFoundException(String errorCode) {
+ super(errorCode);
+ }
+
+ public PositionNotFoundException(String errorCode, Throwable cause) {
+ super(errorCode, cause);
+ }
+
+ public PositionNotFoundException(String errorCode, String errorDesc) {
+ super(errorCode, errorDesc);
+ }
+
+ public PositionNotFoundException(String errorCode, String errorDesc, Throwable cause) {
+ super(errorCode, errorDesc, cause);
+ }
+
+ public PositionNotFoundException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/exception/ServerIdNotMatchException.java b/parse/src/main/java/com/alibaba/otter/canal/parse/exception/ServerIdNotMatchException.java
new file mode 100644
index 00000000..35a6b790
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/exception/ServerIdNotMatchException.java
@@ -0,0 +1,30 @@
+package com.alibaba.otter.canal.parse.exception;
+
+import com.alibaba.otter.canal.common.CanalException;
+
+/**
+ * @author chengjin.lyf on 2018/8/8 下午1:07
+ * @since 1.0.25
+ */
+public class ServerIdNotMatchException extends CanalException{
+
+ public ServerIdNotMatchException(String errorCode) {
+ super(errorCode);
+ }
+
+ public ServerIdNotMatchException(String errorCode, Throwable cause) {
+ super(errorCode, cause);
+ }
+
+ public ServerIdNotMatchException(String errorCode, String errorDesc) {
+ super(errorCode, errorDesc);
+ }
+
+ public ServerIdNotMatchException(String errorCode, String errorDesc, Throwable cause) {
+ super(errorCode, errorDesc, cause);
+ }
+
+ public ServerIdNotMatchException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
index 54bb14e7..0034ba2b 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
@@ -8,6 +8,7 @@ import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
+import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang.math.RandomUtils;
@@ -94,6 +95,10 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
.availableProcessors() * 60 / 100; // 60%的能力跑解析,剩余部分处理网络
protected int parallelBufferSize = 256; // 必须为2的幂
protected MultiStageCoprocessor multiStageCoprocessor;
+ protected ParserExceptionHandler parserExceptionHandler;
+ protected long serverId;
+
+
protected abstract BinlogParser buildParser();
@@ -170,11 +175,16 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
preDump(erosaConnection);
erosaConnection.connect();// 链接
+
+ long queryServerId = erosaConnection.queryServerId();
+ if (queryServerId != 0){
+ serverId = queryServerId;
+ }
// 4. 获取最后的位置信息
EntryPosition position = findStartPosition(erosaConnection);
final EntryPosition startPosition = position;
if (startPosition == null) {
- throw new CanalParseException("can't find start position for " + destination);
+ throw new PositionNotFoundException("can't find start position for " + destination);
}
if (!processTableMeta(startPosition)) {
@@ -277,6 +287,9 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
runningInfo.getAddress().toString()), e);
sendAlarm(destination, ExceptionUtils.getFullStackTrace(e));
}
+ if (parserExceptionHandler!=null){
+ parserExceptionHandler.handle(e);
+ }
} finally {
// 重新置为中断状态
Thread.interrupted();
@@ -303,7 +316,12 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
transactionBuffer.reset();// 重置一下缓冲队列，重新记录数据
binlogParser.reset();// 重新置位
if (multiStageCoprocessor != null) {
- multiStageCoprocessor.reset();
+ // 处理 RejectedExecutionException
+ try {
+ multiStageCoprocessor.reset();
+ } catch (Throwable t) {
+ logger.debug("multi processor rejected:", t);
+ }
}
if (running) {
@@ -331,6 +349,11 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
stopHeartBeat(); // 先停止心跳
parseThread.interrupt(); // 尝试中断
eventSink.interrupt();
+
+ if (multiStageCoprocessor != null && multiStageCoprocessor.isStart()) {
+ multiStageCoprocessor.stop();
+ }
+
try {
parseThread.join();// 等待其结束
} catch (InterruptedException e) {
@@ -343,10 +366,6 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
if (transactionBuffer.isStart()) {
transactionBuffer.stop();
}
-
- if (multiStageCoprocessor != null && multiStageCoprocessor.isStart()) {
- multiStageCoprocessor.stop();
- }
}
protected boolean consumeTheEventAndProfilingIfNecessary(List entrys) throws CanalSinkException,
@@ -609,4 +628,19 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
this.parallelBufferSize = parallelBufferSize;
}
+ public ParserExceptionHandler getParserExceptionHandler() {
+ return parserExceptionHandler;
+ }
+
+ public void setParserExceptionHandler(ParserExceptionHandler parserExceptionHandler) {
+ this.parserExceptionHandler = parserExceptionHandler;
+ }
+
+ public long getServerId() {
+ return serverId;
+ }
+
+ public void setServerId(long serverId) {
+ this.serverId = serverId;
+ }
}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java
index 6beb4a9e..5355e2e1 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java
@@ -40,4 +40,6 @@ public interface ErosaConnection {
public void dump(GTIDSet gtidSet, MultiStageCoprocessor coprocessor) throws IOException;
ErosaConnection fork();
+
+ public long queryServerId() throws IOException;
}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java
index 1c5c495d..96e063e9 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java
@@ -21,9 +21,9 @@ public interface MultiStageCoprocessor extends CanalLifeCycle {
/**
* 网络数据投递
*/
- public void publish(LogBuffer buffer);
+ public boolean publish(LogBuffer buffer);
- public void publish(LogBuffer buffer, String binlogFileName);
+ public boolean publish(LogBuffer buffer, String binlogFileName);
public void reset();
}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ParserExceptionHandler.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ParserExceptionHandler.java
new file mode 100644
index 00000000..d0face01
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ParserExceptionHandler.java
@@ -0,0 +1,9 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+/**
+ * @author chengjin.lyf on 2018/7/20 下午3:55
+ * @since 1.0.25
+ */
+public interface ParserExceptionHandler {
+ void handle(Throwable e);
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java
index 8188cf83..c0d2f190 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java
@@ -7,6 +7,7 @@ import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.filter.CanalEventFilter;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
+import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
@@ -91,8 +92,16 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
public void start() throws CanalParseException {
if (enableTsdb) {
if (tableMetaTSDB == null) {
- // 初始化
- tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml);
+ synchronized (CanalEventParser.class) {
+ try {
+ // 设置当前正在加载的通道，加载spring查找文件时会用到该变量
+ System.setProperty("canal.instance.destination", destination);
+ // 初始化
+ tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml);
+ } finally {
+ System.setProperty("canal.instance.destination", "");
+ }
+ }
}
}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
index f97c4ccd..0358dfda 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
@@ -4,6 +4,7 @@ import java.io.File;
import java.io.IOException;
import java.util.List;
+import com.alibaba.otter.canal.parse.exception.ServerIdNotMatchException;
import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@@ -36,6 +37,9 @@ public class LocalBinLogConnection implements ErosaConnection {
private String directory;
private int bufferSize = 16 * 1024;
private boolean running = false;
+ private long serverId;
+ private FileParserListener parserListener;
+
public LocalBinLogConnection(){
}
@@ -96,6 +100,9 @@ public class LocalBinLogConnection implements ErosaConnection {
if (event == null) {
continue;
}
+ if (serverId != 0 && event.getServerId() != serverId){
+ throw new ServerIdNotMatchException("unexpected serverId "+serverId + " in binlog file !");
+ }
if (!func.sink(event)) {
needContinue = false;
@@ -103,8 +110,9 @@ public class LocalBinLogConnection implements ErosaConnection {
}
}
+ fetcher.close(); // 关闭上一个文件
+ parserFinish(current.getName());
if (needContinue) {// 读取下一个
- fetcher.close(); // 关闭上一个文件
File nextFile;
if (needWait) {
@@ -160,6 +168,11 @@ public class LocalBinLogConnection implements ErosaConnection {
while (fetcher.fetch()) {
LogEvent event = decoder.decode(fetcher, context);
if (event != null) {
+
+ if (serverId != 0 && event.getServerId() != serverId){
+ throw new ServerIdNotMatchException("unexpected serverId "+serverId + " in binlog file !");
+ }
+
if (event.getWhen() > timestampSeconds) {
break;
}
@@ -221,11 +234,16 @@ public class LocalBinLogConnection implements ErosaConnection {
while (fetcher.fetch()) {
LogBuffer buffer = fetcher.duplicate();
fetcher.consume(fetcher.limit());
- coprocessor.publish(buffer, binlogfilename); // set filename
+ // set filename
+ if (!coprocessor.publish(buffer, binlogfilename)) {
+ needContinue = false;
+ break;
+ }
}
+ fetcher.close(); // 关闭上一个文件
+ parserFinish(binlogfilename);
if (needContinue) {// 读取下一个
- fetcher.close(); // 关闭上一个文件
File nextFile;
if (needWait) {
@@ -254,6 +272,12 @@ public class LocalBinLogConnection implements ErosaConnection {
}
}
+ private void parserFinish(String fileName){
+ if (parserListener != null){
+ parserListener.onFinish(fileName);
+ }
+ }
+
@Override
public void dump(long timestampMills, MultiStageCoprocessor coprocessor) throws IOException {
List currentBinlogs = binlogs.currentBinlogs();
@@ -282,6 +306,11 @@ public class LocalBinLogConnection implements ErosaConnection {
while (fetcher.fetch()) {
LogEvent event = decoder.decode(fetcher, context);
if (event != null) {
+
+ if (serverId != 0 && event.getServerId() != serverId){
+ throw new ServerIdNotMatchException("unexpected serverId "+serverId + " in binlog file !");
+ }
+
if (event.getWhen() > timestampSeconds) {
break;
}
@@ -340,6 +369,11 @@ public class LocalBinLogConnection implements ErosaConnection {
return connection;
}
+ @Override
+ public long queryServerId() {
+ return 0;
+ }
+
public boolean isNeedWait() {
return needWait;
}
@@ -364,4 +398,20 @@ public class LocalBinLogConnection implements ErosaConnection {
this.bufferSize = bufferSize;
}
+ public long getServerId() {
+ return serverId;
+ }
+
+ public void setServerId(long serverId) {
+ this.serverId = serverId;
+ }
+
+ public void setParserListener(FileParserListener parserListener) {
+ this.parserListener = parserListener;
+ }
+
+ public interface FileParserListener{
+ void onFinish(String fileName);
+ }
+
}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
index 46f7e300..5f681a54 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
@@ -9,8 +9,10 @@ import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.math.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlQueryExecutor;
@@ -206,7 +208,9 @@ public class MysqlConnection implements ErosaConnection {
while (fetcher.fetch()) {
LogBuffer buffer = fetcher.duplicate();
fetcher.consume(fetcher.limit());
- coprocessor.publish(buffer);
+ if (!coprocessor.publish(buffer)) {
+ break;
+ }
}
} finally {
fetcher.close();
@@ -230,7 +234,9 @@ public class MysqlConnection implements ErosaConnection {
while (fetcher.fetch()) {
LogBuffer buffer = fetcher.duplicate();
fetcher.consume(fetcher.limit());
- coprocessor.publish(buffer);
+ if (!coprocessor.publish(buffer)) {
+ break;
+ }
}
} finally {
fetcher.close();
@@ -320,6 +326,16 @@ public class MysqlConnection implements ErosaConnection {
return connection;
}
+ @Override
+ public long queryServerId() throws IOException {
+ ResultSetPacket resultSetPacket = query("show variables like 'server_id'");
+ List fieldValues = resultSetPacket.getFieldValues();
+ if (fieldValues == null || fieldValues.size() != 2){
+ return 0;
+ }
+ return NumberUtils.toLong(fieldValues.get(1));
+ }
+
// ====================== help method ====================
/**
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
index 6c2b9d4d..a3d1ba01 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
@@ -10,6 +10,11 @@ import java.util.Map;
import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicLong;
+import com.alibaba.otter.canal.parse.inbound.*;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheInterface;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheWithStorage;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorage;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorageFactory;
import org.apache.commons.lang.StringUtils;
import org.springframework.util.CollectionUtils;
@@ -20,9 +25,6 @@ import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.ha.CanalHAController;
-import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
-import com.alibaba.otter.canal.parse.inbound.HeartBeatCallback;
-import com.alibaba.otter.canal.parse.inbound.SinkFunction;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.BinlogFormat;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.BinlogImage;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert;
@@ -53,16 +55,16 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
private int receiveBufferSize = 64 * 1024;
private int sendBufferSize = 64 * 1024;
// 数据库信息
- private AuthenticationInfo masterInfo; // 主库
- private AuthenticationInfo standbyInfo; // 备库
+ protected AuthenticationInfo masterInfo; // 主库
+ protected AuthenticationInfo standbyInfo; // 备库
// binlog信息
- private EntryPosition masterPosition;
- private EntryPosition standbyPosition;
+ protected EntryPosition masterPosition;
+ protected EntryPosition standbyPosition;
private long slaveId; // 链接到mysql的slave
// 心跳检查信息
private String detectingSQL; // 心跳sql
private MysqlConnection metaConnection; // 查询meta信息的链接
- private TableMetaCache tableMetaCache; // 对应meta
+ private TableMetaCacheInterface tableMetaCache; // 对应meta
// cache
private int fallbackIntervalInSeconds = 60; // 切换回退时间
private BinlogFormat[] supportBinlogFormats; // 支持的binlogFormat,如果设置会执行强校验
@@ -72,6 +74,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
private int dumpErrorCount = 0; // binlogDump失败异常计数
private int dumpErrorCountThreshold = 2; // binlogDump失败异常计数阀值
+ private TableMetaStorageFactory tableMetaStorageFactory;
+
protected ErosaConnection buildErosaConnection() {
return buildMysqlConnection(this.runningInfo);
}
@@ -125,7 +129,13 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
}
- tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB);
+
+ TableMetaStorage storage = null;
+ if (tableMetaStorageFactory != null) {
+ storage = tableMetaStorageFactory.getTableMetaStorage();
+ }
+
+ tableMetaCache = new TableMetaCacheWithStorage(metaConnection, storage);
((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache);
}
}
@@ -643,6 +653,9 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
}
EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
+ if (isGTIDMode && fields.size() > 4) {
+ endPosition.setGtid(fields.get(4));
+ }
return endPosition;
} catch (IOException e) {
throw new CanalParseException("command : 'show master status' has an error!", e);
@@ -908,4 +921,11 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
this.dumpErrorCountThreshold = dumpErrorCountThreshold;
}
+ public TableMetaStorageFactory getTableMetaStorageFactory() {
+ return tableMetaStorageFactory;
+ }
+
+ public void setTableMetaStorageFactory(TableMetaStorageFactory tableMetaStorageFactory) {
+ this.tableMetaStorageFactory = tableMetaStorageFactory;
+ }
}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
index 76c70723..e93a1d20 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
@@ -52,17 +52,17 @@ import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
*/
public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implements MultiStageCoprocessor {
- private LogEventConvert logEventConvert;
- private EventTransactionBuffer transactionBuffer;
- private ErosaConnection connection;
+ private LogEventConvert logEventConvert;
+ private EventTransactionBuffer transactionBuffer;
+ private ErosaConnection connection;
- private int parserThreadCount;
- private int ringBufferSize;
- private RingBuffer disruptorMsgBuffer;
- private ExecutorService parserExecutor;
- private ExecutorService stageExecutor;
- private String destination;
- private CanalParseException exception;
+ private int parserThreadCount;
+ private int ringBufferSize;
+ private RingBuffer disruptorMsgBuffer;
+ private ExecutorService parserExecutor;
+ private ExecutorService stageExecutor;
+ private String destination;
+ private volatile CanalParseException exception;
public MysqlMultiStageCoprocessor(int ringBufferSize, int parserThreadCount, LogEventConvert logEventConvert,
EventTransactionBuffer transactionBuffer, String destination){
@@ -135,22 +135,31 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
} catch (Throwable e) {
// ignore
}
- disruptorMsgBuffer = null;
super.stop();
}
/**
* 网络数据投递
*/
- public void publish(LogBuffer buffer) {
- publish(buffer, null);
+ public boolean publish(LogBuffer buffer) {
+ return publish(buffer, null);
}
- public void publish(LogBuffer buffer, String binlogFileName) {
- if (!isStart() && exception != null) {
- throw exception;
+ public boolean publish(LogBuffer buffer, String binlogFileName) {
+ if (!isStart()) {
+ if (exception != null) {
+ throw exception;
+ }
+ return false;
}
+ /**
+ * 由于改为processor仅终止自身stage而不是stop，那么需要由incident标识coprocessor是否正常工作。
+ * 让dump线程能够及时感知
+ */
+ if (exception != null) {
+ throw exception;
+ }
boolean interupted = false;
do {
try {
@@ -168,10 +177,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
interupted = Thread.interrupted();
}
} while (!interupted && isStart());
-
- if (exception != null) {
- throw exception;
- }
+ return isStart();
}
@Override
@@ -197,10 +203,15 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
try {
LogBuffer buffer = event.getBuffer();
if (StringUtils.isNotEmpty(event.getBinlogFileName())
- && !context.getLogPosition().getFileName().equals(event.getBinlogFileName())) {
+ && (context.getLogPosition() == null
+ || !context.getLogPosition().getFileName().equals(event.getBinlogFileName()))) {
// set roate binlog file name
- context.setLogPosition(new LogPosition(event.getBinlogFileName(), context.getLogPosition()
- .getPosition()));
+ if (context.getLogPosition() == null){
+ context.setLogPosition(new LogPosition(event.getBinlogFileName(), 0));
+ }else{
+ context.setLogPosition(new LogPosition(event.getBinlogFileName(), context.getLogPosition()
+ .getPosition()));
+ }
}
LogEvent logEvent = decoder.decode(buffer, context);
@@ -249,9 +260,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
@Override
public void onShutdown() {
- if (isStart()) {
- stop();
- }
+
}
}
@@ -287,9 +296,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
@Override
public void onShutdown() {
- if (isStart()) {
- stop();
- }
+
}
}
@@ -328,9 +335,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
@Override
public void onShutdown() {
- if (isStart()) {
- stop();
- }
+
}
}
@@ -397,7 +402,6 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
@Override
public void handleEventException(final Throwable ex, final long sequence, final Object event) {
- throw new RuntimeException(ex);
}
@Override
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java
index a85962cc..222d762a 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java
@@ -64,9 +64,6 @@ public class DirectLogFetcher extends LogFetcher {
if ("1".equals(dbsemi)) {
issemi = true;
}
- // 和mysql driver一样，提供buffer机制，提升读取binlog速度
- // this.input = new
- // BufferedInputStream(channel.socket().getInputStream(), 16384);
}
/**
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
index 09af808f..72369f07 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
@@ -10,6 +10,9 @@ import java.util.Arrays;
import java.util.BitSet;
import java.util.List;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheInterface;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorage;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception.NoHistoryException;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.slf4j.Logger;
@@ -87,7 +90,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
private volatile AviaterRegexFilter nameFilter; // 运行时引用可能会有变化，比如规则发生变化时
private volatile AviaterRegexFilter nameBlackFilter;
- private TableMetaCache tableMetaCache;
+
+ private TableMetaCacheInterface tableMetaCache;
+ private String binlogFileName = "mysql-bin.000001";
+
private Charset charset = Charset.defaultCharset();
private boolean filterQueryDcl = false;
private boolean filterQueryDml = false;
@@ -262,7 +268,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
if (!isSeek) {
// 使用新的表结构元数据管理方式
EntryPosition position = createPosition(event.getHeader());
- tableMetaCache.apply(position, event.getDbName(), queryString, null);
+ String fulltbName = schemaName+"."+tableName;
+ tableMetaCache.apply(position, fulltbName, queryString, null);
}
Header header = createHeader(event.getHeader(), schemaName, tableName, type);
@@ -933,7 +940,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
this.nameBlackFilter = nameBlackFilter;
}
- public void setTableMetaCache(TableMetaCache tableMetaCache) {
+ public void setTableMetaCache(TableMetaCacheInterface tableMetaCache) {
this.tableMetaCache = tableMetaCache;
}
@@ -960,5 +967,4 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
public void setGtidSet(GTIDSet gtidSet) {
this.gtidSet = gtidSet;
}
-
}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
index 4c1cef3c..5c782aa1 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
@@ -6,6 +6,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheInterface;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
@@ -29,7 +30,7 @@ import com.google.common.cache.LoadingCache;
* @author jianghang 2013-1-17 下午10:15:16
* @version 1.0.0
*/
-public class TableMetaCache {
+public class TableMetaCache implements TableMetaCacheInterface {
public static final String COLUMN_NAME = "COLUMN_NAME";
public static final String COLUMN_TYPE = "COLUMN_TYPE";
@@ -99,6 +100,10 @@ public class TableMetaCache {
String createDDL = packet.getFieldValues().get(1);
MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
memoryTableMeta.apply(DatabaseTableMeta.INIT_POSITION, schema, createDDL, null);
+ String[] strings = table.split("\\.");
+ if (strings.length > 1) {
+ table = strings[1];
+ }
TableMeta tableMeta = memoryTableMeta.find(schema, table);
return tableMeta.getFields();
} else {
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java
new file mode 100644
index 00000000..d95c8af1
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java
@@ -0,0 +1,260 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.rds;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
+
+import io.netty.handler.codec.http.HttpResponseStatus;
+
+/**
+ * @author chengjin.lyf on 2018/8/7 下午3:10
+ * @since 1.0.25
+ */
+public class BinlogDownloadQueue {
+
+ private static final Logger logger = LoggerFactory.getLogger(BinlogDownloadQueue.class);
+ private static final int TIMEOUT = 10000;
+
+ private LinkedBlockingQueue downloadQueue = new LinkedBlockingQueue();
+ private LinkedBlockingQueue taskQueue = new LinkedBlockingQueue();
+ private LinkedList binlogList;
+ private final int batchSize;
+ private Thread downloadThread;
+ public boolean running = true;
+ private final String destDir;
+ private String hostId;
+ private int currentSize;
+ private String lastDownload;
+
+ public BinlogDownloadQueue(List downloadQueue, int batchSize, String destDir) throws IOException {
+ this.binlogList = new LinkedList(downloadQueue);
+ this.batchSize = batchSize;
+ this.destDir = destDir;
+ this.currentSize = 0;
+ prepareBinlogList();
+ cleanDir();
+ }
+
+ private void prepareBinlogList(){
+ for (BinlogFile binlog : this.binlogList) {
+ String fileName = StringUtils.substringBetween(binlog.getDownloadLink(), "mysql-bin.", "?");
+ binlog.setFileName(fileName);
+ }
+ Collections.sort(this.binlogList, new Comparator() {
+ @Override
+ public int compare(BinlogFile o1, BinlogFile o2) {
+ return o1.getFileName().compareTo(o2.getFileName());
+ }
+ });
+ }
+
+ public void cleanDir() throws IOException {
+ File destDirFile = new File(destDir);
+ FileUtils.forceMkdir(destDirFile);
+ FileUtils.cleanDirectory(destDirFile);
+ }
+
+ public void silenceDownload() {
+ if (downloadThread != null) {
+ return;
+ }
+ downloadThread = new Thread(new DownloadThread());
+ downloadThread.start();
+ }
+
+
+ public BinlogFile tryOne() throws IOException {
+ BinlogFile binlogFile = binlogList.poll();
+ download(binlogFile);
+ hostId = binlogFile.getHostInstanceID();
+ this.currentSize ++;
+ return binlogFile;
+ }
+
+ public void notifyNotMatch(){
+ this.currentSize --;
+ filter(hostId);
+ }
+
+ private void filter(String hostInstanceId){
+ Iterator it = binlogList.iterator();
+ while (it.hasNext()){
+ BinlogFile bf = it.next();
+ if(bf.getHostInstanceID().equalsIgnoreCase(hostInstanceId)){
+ it.remove();
+ }else{
+ hostId = bf.getHostInstanceID();
+ }
+ }
+ }
+
+ public boolean isLastFile(String fileName){
+ String needCompareName = lastDownload;
+ if (StringUtils.isNotEmpty(needCompareName) && StringUtils.endsWith(needCompareName, "tar")){
+ needCompareName = needCompareName.substring(0, needCompareName.indexOf("."));
+ }
+ return fileName.equalsIgnoreCase(needCompareName) && binlogList.isEmpty();
+ }
+
+ public void prepare() throws InterruptedException {
+ for (int i = this.currentSize; i < batchSize && !binlogList.isEmpty(); i++) {
+ BinlogFile binlogFile = null;
+ while (!binlogList.isEmpty()){
+ binlogFile = binlogList.poll();
+ if (!binlogFile.getHostInstanceID().equalsIgnoreCase(hostId)){
+ continue;
+ }
+ break;
+ }
+ if (binlogFile == null){
+ break;
+ }
+ this.downloadQueue.put(binlogFile);
+ this.lastDownload = "mysql-bin." + binlogFile.getFileName();
+ this.currentSize ++;
+ }
+ }
+
+ public void downOne(){
+ this.currentSize --;
+ }
+
+ public void release(){
+ running = false;
+ this.currentSize = 0;
+ binlogList.clear();
+ downloadQueue.clear();
+ }
+
+ private void download(BinlogFile binlogFile) throws IOException {
+ String downloadLink = binlogFile.getDownloadLink();
+ String fileName = binlogFile.getFileName();
+ HttpGet httpGet = new HttpGet(downloadLink);
+ CloseableHttpClient httpClient = HttpClientBuilder.create()
+ .setMaxConnPerRoute(50)
+ .setMaxConnTotal(100)
+ .build();
+ RequestConfig requestConfig = RequestConfig.custom()
+ .setConnectTimeout(TIMEOUT)
+ .setConnectionRequestTimeout(TIMEOUT)
+ .setSocketTimeout(TIMEOUT)
+ .build();
+ httpGet.setConfig(requestConfig);
+ HttpResponse response = httpClient.execute(httpGet);
+ int statusCode = response.getStatusLine().getStatusCode();
+ if (statusCode != HttpResponseStatus.OK.code()) {
+ throw new RuntimeException("download failed , url:" + downloadLink + " , statusCode:"
+ + statusCode);
+ }
+ saveFile(new File(destDir), "mysql-bin." + fileName, response);
+ }
+
+ private static void saveFile(File parentFile, String fileName, HttpResponse response) throws IOException {
+ InputStream is = response.getEntity().getContent();
+ long totalSize = Long.parseLong(response.getFirstHeader("Content-Length").getValue());
+ if(response.getFirstHeader("Content-Disposition")!=null){
+ fileName = response.getFirstHeader("Content-Disposition").getValue();
+ fileName = StringUtils.substringAfter(fileName, "filename=");
+ }
+ boolean isTar = StringUtils.endsWith(fileName, ".tar");
+ FileUtils.forceMkdir(parentFile);
+ FileOutputStream fos = null;
+ try {
+ if (isTar) {
+ TarArchiveInputStream tais = new TarArchiveInputStream(is);
+ TarArchiveEntry tarArchiveEntry = null;
+ while ((tarArchiveEntry = tais.getNextTarEntry()) != null) {
+ String name = tarArchiveEntry.getName();
+ File tarFile = new File(parentFile, name + ".tmp");
+ logger.info("start to download file " + tarFile.getName());
+ BufferedOutputStream bos = null;
+ try {
+ bos = new BufferedOutputStream(new FileOutputStream(tarFile));
+ int read = -1;
+ byte[] buffer = new byte[1024];
+ while ((read = tais.read(buffer)) != -1) {
+ bos.write(buffer, 0, read);
+ }
+ logger.info("download file " + tarFile.getName() + " end!");
+ tarFile.renameTo(new File(parentFile, name));
+ } finally {
+ IOUtils.closeQuietly(bos);
+ }
+ }
+ tais.close();
+ } else {
+ File file = new File(parentFile, fileName + ".tmp");
+ if (!file.isFile()) {
+ file.createNewFile();
+ }
+ try {
+ fos = new FileOutputStream(file);
+ byte[] buffer = new byte[1024];
+ int len;
+ long copySize = 0;
+ long nextPrintProgress = 0;
+ logger.info("start to download file " + file.getName());
+ while ((len = is.read(buffer)) != -1) {
+ fos.write(buffer, 0, len);
+ copySize += len;
+ long progress = copySize * 100 / totalSize;
+ if (progress >= nextPrintProgress) {
+ logger.info("download " + file.getName() + " progress : " + progress
+ + "% , download size : " + copySize + ", total size : " + totalSize);
+ nextPrintProgress += 10;
+ }
+ }
+ logger.info("download file " + file.getName() + " end!");
+ fos.flush();
+ } finally {
+ IOUtils.closeQuietly(fos);
+ }
+ file.renameTo(new File(parentFile, fileName));
+ }
+ } finally {
+ IOUtils.closeQuietly(fos);
+ }
+ }
+
+ public void execute(Runnable runnable) throws InterruptedException {
+ taskQueue.put(runnable);
+ }
+
+ private class DownloadThread implements Runnable {
+
+ @Override
+ public void run() {
+ while (running) {
+ try {
+ BinlogFile binlogFile = downloadQueue.poll(5000, TimeUnit.MILLISECONDS);
+ if (binlogFile != null){
+ download(binlogFile);
+ }
+ Runnable runnable = taskQueue.poll(5000, TimeUnit.MILLISECONDS);
+ if (runnable != null){
+ runnable.run();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ }
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java
new file mode 100644
index 00000000..5bcc8688
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java
@@ -0,0 +1,149 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.rds;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+
+import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
+import com.alibaba.otter.canal.parse.inbound.ParserExceptionHandler;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
+
+/**
+ * @author chengjin.lyf on 2018/7/20 上午10:52
+ * @since 1.0.25
+ */
+public class RdsBinlogEventParserProxy extends MysqlEventParser {
+
+ private String rdsOpenApiUrl = "https://rds.aliyuncs.com/"; // openapi地址
+ private String accesskey; // 云账号的ak
+ private String secretkey; // 云账号sk
+ private String instanceId; // rds实例id
+ private Long startTime;
+ private Long endTime;
+ private String directory; //binlog 目录
+ private int batchSize = 4; //最多下载的binlog文件数量
+
+ private RdsLocalBinlogEventParser rdsBinlogEventParser = new RdsLocalBinlogEventParser();
+ private ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactory() {
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r, "rds-binlog-daemon-thread");
+ t.setDaemon(true);
+ return t;
+ }
+ });
+
+ @Override
+ public void start() {
+ final ParserExceptionHandler targetHandler = this.getParserExceptionHandler();
+ rdsBinlogEventParser.setLogPositionManager(this.getLogPositionManager());
+ rdsBinlogEventParser.setDestination(destination);
+ rdsBinlogEventParser.setAlarmHandler(this.getAlarmHandler());
+ rdsBinlogEventParser.setConnectionCharset(this.connectionCharset);
+ rdsBinlogEventParser.setConnectionCharsetNumber(this.connectionCharsetNumber);
+ rdsBinlogEventParser.setEnableTsdb(this.enableTsdb);
+ rdsBinlogEventParser.setEventBlackFilter(this.eventBlackFilter);
+ rdsBinlogEventParser.setFilterQueryDcl(this.filterQueryDcl);
+ rdsBinlogEventParser.setFilterQueryDdl(this.filterQueryDdl);
+ rdsBinlogEventParser.setFilterQueryDml(this.filterQueryDml);
+ rdsBinlogEventParser.setFilterRows(this.filterRows);
+ rdsBinlogEventParser.setFilterTableError(this.filterTableError);
+ rdsBinlogEventParser.setIsGTIDMode(this.isGTIDMode);
+ rdsBinlogEventParser.setMasterInfo(this.masterInfo);
+ rdsBinlogEventParser.setEventFilter(this.eventFilter);
+ rdsBinlogEventParser.setMasterPosition(this.masterPosition);
+ rdsBinlogEventParser.setTransactionSize(this.transactionSize);
+ rdsBinlogEventParser.setUrl(this.rdsOpenApiUrl);
+ rdsBinlogEventParser.setAccesskey(this.accesskey);
+ rdsBinlogEventParser.setSecretkey(this.secretkey);
+ rdsBinlogEventParser.setInstanceId(this.instanceId);
+ rdsBinlogEventParser.setEventSink(eventSink);
+ rdsBinlogEventParser.setDirectory(directory);
+ rdsBinlogEventParser.setBatchSize(batchSize);
+ rdsBinlogEventParser.setFinishListener(new RdsLocalBinlogEventParser.ParseFinishListener() {
+ @Override
+ public void onFinish() {
+ executorService.execute(new Runnable() {
+ @Override
+ public void run() {
+ rdsBinlogEventParser.stop();
+ RdsBinlogEventParserProxy.this.start();
+ }
+ });
+
+ }
+ });
+ this.setParserExceptionHandler(new ParserExceptionHandler() {
+
+ @Override
+ public void handle(Throwable e) {
+ handleMysqlParserException(e);
+ if (targetHandler != null) {
+ targetHandler.handle(e);
+ }
+ }
+ });
+ super.start();
+ }
+
+ public void handleMysqlParserException(Throwable throwable) {
+ if (throwable instanceof PositionNotFoundException) {
+ logger.info("remove rds not found position, try download rds binlog!");
+ executorService.execute(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ logger.info("stop mysql parser!");
+ RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this;
+ long serverId = rdsBinlogEventParserProxy.getServerId();
+ rdsBinlogEventParser.setServerId(serverId);
+ rdsBinlogEventParserProxy.stop();
+ logger.info("start rds mysql binlog parser!");
+ rdsBinlogEventParser.start();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ }
+ }
+
+ @Override
+ public void stop() {
+ super.stop();
+ }
+
+ @Override
+ public boolean isStart() {
+ return super.isStart();
+ }
+
+ public void setRdsOpenApiUrl(String rdsOpenApiUrl) {
+ this.rdsOpenApiUrl = rdsOpenApiUrl;
+ }
+
+
+ public void setAccesskey(String accesskey) {
+ this.accesskey = accesskey;
+ }
+
+
+ public void setSecretkey(String secretkey) {
+ this.secretkey = secretkey;
+ }
+
+
+ public void setInstanceId(String instanceId) {
+ this.instanceId = instanceId;
+ }
+
+ public void setDirectory(String directory) {
+ this.directory = directory;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java
index eb638739..dced6d8d 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java
@@ -1,5 +1,9 @@
package com.alibaba.otter.canal.parse.inbound.mysql.rds;
+import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
+import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.DescribeBinlogFileResult;
+import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsItem;
+import com.alibaba.otter.canal.parse.inbound.mysql.rds.request.DescribeBinlogFilesRequest;
import io.netty.handler.codec.http.HttpResponseStatus;
import java.io.BufferedOutputStream;
@@ -9,16 +13,11 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.UUID;
+import java.util.*;
import javax.crypto.Mac;
import javax.crypto.SecretKey;
@@ -56,6 +55,52 @@ public class RdsBinlogOpenApi {
private static final String API_VERSION = "2014-08-15";
private static final String SIGNATURE_VERSION = "1.0";
+
+ public static List listBinlogFiles(String url, String ak, String sk, String dbInstanceId, Date startTime,
+ Date endTime) {
+ DescribeBinlogFilesRequest request = new DescribeBinlogFilesRequest();
+ if (StringUtils.isNotEmpty(url)){
+ try {
+ URI uri = new URI(url);
+ request.setEndPoint(uri.getHost());
+ } catch (URISyntaxException e) {
+ logger.error("resolve url host failed, will use default rds endpoint!");
+ }
+ }
+ request.setStartDate(startTime);
+ request.setEndDate(endTime);
+ request.setPageNumber(1);
+ request.setPageSize(100);
+ request.setRdsInstanceId(dbInstanceId);
+ request.setAccessKeyId(ak);
+ request.setAccessKeySecret(sk);
+ DescribeBinlogFileResult result = null;
+ int retryTime = 3;
+ while (true){
+ try{
+ result = request.doAction();
+ break;
+ }catch (Exception e){
+ if(retryTime-- <= 0){
+ throw new RuntimeException(e);
+ }
+ try {
+ Thread.sleep(100L);
+ } catch (InterruptedException e1) {
+ }
+ }
+ }
+ if (result == null){
+ return Collections.EMPTY_LIST;
+ }
+ RdsItem rdsItem = result.getItems();
+ if (rdsItem != null){
+ return rdsItem.getBinLogFile();
+ }
+ return Collections.EMPTY_LIST;
+ }
+
+
public static void downloadBinlogFiles(String url, String ak, String sk, String dbInstanceId, Date startTime,
Date endTime, File destDir) throws Throwable {
int pageSize = 100;
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java
index 68be38e5..40cb3b13 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java
@@ -2,14 +2,23 @@ package com.alibaba.otter.canal.parse.inbound.mysql.rds;
import java.io.File;
import java.util.Date;
+import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.math.NumberUtils;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
+import com.alibaba.otter.canal.parse.exception.ServerIdNotMatchException;
+import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
+import com.alibaba.otter.canal.parse.inbound.ParserExceptionHandler;
+import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinLogConnection;
import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser;
+import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
/**
* 基于rds binlog备份文件的复制
@@ -17,47 +26,102 @@ import com.alibaba.otter.canal.protocol.position.EntryPosition;
* @author agapple 2017年10月15日 下午1:27:36
* @since 1.0.25
*/
-public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements CanalEventParser {
+public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements CanalEventParser, LocalBinLogConnection.FileParserListener {
private String url = "https://rds.aliyuncs.com/"; // openapi地址
- private String accesskey; // 云账号的ak
- private String secretkey; // 云账号sk
- private String instanceId; // rds实例id
- private Long startTime;
- private Long endTime;
+ private String accesskey; // 云账号的ak
+ private String secretkey; // 云账号sk
+ private String instanceId; // rds实例id
+ private Long startTime;
+ private Long endTime;
+ private BinlogDownloadQueue binlogDownloadQueue;
+ private ParseFinishListener finishListener;
+ private int batchSize;
public RdsLocalBinlogEventParser(){
}
public void start() throws CanalParseException {
try {
- Assert.notNull(startTime);
Assert.notNull(accesskey);
Assert.notNull(secretkey);
Assert.notNull(instanceId);
Assert.notNull(url);
+ Assert.notNull(directory);
+
if (endTime == null) {
endTime = System.currentTimeMillis();
}
- RdsBinlogOpenApi.downloadBinlogFiles(url,
- accesskey,
+ EntryPosition entryPosition = findStartPosition(null);
+ if (entryPosition == null) {
+ throw new PositionNotFoundException("position not found!");
+ }
+ long startTimeInMill = entryPosition.getTimestamp();
+ startTime = startTimeInMill;
+ List binlogFiles = RdsBinlogOpenApi.listBinlogFiles(url, accesskey,
secretkey,
instanceId,
new Date(startTime),
- new Date(endTime),
- new File(directory));
-
- // 更新一下时间戳
- masterPosition = new EntryPosition(startTime);
+ new Date(endTime));
+ binlogDownloadQueue = new BinlogDownloadQueue(binlogFiles, batchSize, directory);
+ binlogDownloadQueue.silenceDownload();
+ needWait = true;
+ parallel = false;
+ // try to download one file,use to test server id
+ binlogDownloadQueue.tryOne();
} catch (Throwable e) {
logger.error("download binlog failed", e);
throw new CanalParseException(e);
}
+ setParserExceptionHandler(new ParserExceptionHandler() {
+ @Override
+ public void handle(Throwable e) {
+ handleMysqlParserException(e);
+ }
+ });
super.start();
}
+ private void handleMysqlParserException(Throwable throwable) {
+ if (throwable instanceof ServerIdNotMatchException) {
+ logger.error("server id not match, try download another rds binlog!");
+ binlogDownloadQueue.notifyNotMatch();
+ try {
+ binlogDownloadQueue.cleanDir();
+ binlogDownloadQueue.prepare();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ try {
+ binlogDownloadQueue.execute(new Runnable() {
+
+ @Override
+ public void run() {
+ RdsLocalBinlogEventParser.super.stop();
+ RdsLocalBinlogEventParser.super.start();
+ }
+ });
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+ }
+
+ @Override
+ protected ErosaConnection buildErosaConnection() {
+ ErosaConnection connection = super.buildErosaConnection();
+ if (connection instanceof LocalBinLogConnection) {
+ LocalBinLogConnection localBinLogConnection = (LocalBinLogConnection) connection;
+ localBinLogConnection.setNeedWait(true);
+ localBinLogConnection.setServerId(serverId);
+ localBinLogConnection.setParserListener(this);
+ }
+ return connection;
+ }
+
public String getUrl() {
return url;
}
@@ -68,44 +132,81 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
}
}
- public String getAccesskey() {
- return accesskey;
- }
public void setAccesskey(String accesskey) {
this.accesskey = accesskey;
}
- public String getSecretkey() {
- return secretkey;
- }
public void setSecretkey(String secretkey) {
this.secretkey = secretkey;
}
- public String getInstanceId() {
- return instanceId;
- }
public void setInstanceId(String instanceId) {
this.instanceId = instanceId;
}
- public Long getStartTime() {
- return startTime;
- }
public void setStartTime(Long startTime) {
this.startTime = startTime;
}
- public Long getEndTime() {
- return endTime;
- }
public void setEndTime(Long endTime) {
this.endTime = endTime;
}
+ @Override
+ public void onFinish(String fileName) {
+ try {
+ binlogDownloadQueue.downOne();
+ File needDeleteFile = new File(directory + File.separator + fileName);
+ if (needDeleteFile.exists()){
+ needDeleteFile.delete();
+ }
+ // 处理下logManager位点问题
+ LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
+ EntryPosition position = logPosition.getPostion();
+ if (position != null){
+ LogPosition newLogPosition = new LogPosition();
+ String journalName = position.getJournalName();
+ int sepIdx = journalName.indexOf(".");
+ String fileIndex = journalName.substring(sepIdx+1);
+ int index = NumberUtils.toInt(fileIndex) + 1;
+ String newJournalName = journalName.substring(0, sepIdx) + "." + StringUtils.leftPad(String.valueOf(index), fileIndex.length(), "0");
+ newLogPosition.setPostion(new EntryPosition(newJournalName, 4L, position.getTimestamp(), position.getServerId()));
+ newLogPosition.setIdentity(logPosition.getIdentity());
+ logPositionManager.persistLogPosition(destination, newLogPosition);
+ }
+
+ if (binlogDownloadQueue.isLastFile(fileName)) {
+ logger.info("all file parse complete, switch to mysql parser!");
+ finishListener.onFinish();
+ return;
+ }
+ binlogDownloadQueue.prepare();
+ } catch (Exception e) {
+ logger.error("prepare download binlog file failed!", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void stop() {
+ this.binlogDownloadQueue.release();
+ super.stop();
+ }
+
+ public void setFinishListener(ParseFinishListener finishListener) {
+ this.finishListener = finishListener;
+ }
+
+ public interface ParseFinishListener{
+ void onFinish();
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/BinlogFile.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/BinlogFile.java
new file mode 100644
index 00000000..e6b354d7
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/BinlogFile.java
@@ -0,0 +1,72 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.rds.data;
+
+/**
+ * @author chengjin.lyf on 2018/8/7 下午2:26
+ * @since 1.0.25
+ */
+public class BinlogFile {
+
+ private Long FileSize;
+ private String LogBeginTime;
+ private String LogEndTime;
+ private String DownloadLink;
+ private String HostInstanceID;
+ private String LinkExpiredTime;
+ private String fileName;
+
+ public Long getFileSize() {
+ return FileSize;
+ }
+
+ public void setFileSize(Long fileSize) {
+ FileSize = fileSize;
+ }
+
+ public String getLogBeginTime() {
+ return LogBeginTime;
+ }
+
+ public void setLogBeginTime(String logBeginTime) {
+ LogBeginTime = logBeginTime;
+ }
+
+ public String getLogEndTime() {
+ return LogEndTime;
+ }
+
+ public void setLogEndTime(String logEndTime) {
+ LogEndTime = logEndTime;
+ }
+
+ public String getDownloadLink() {
+ return DownloadLink;
+ }
+
+ public void setDownloadLink(String downloadLink) {
+ DownloadLink = downloadLink;
+ }
+
+ public String getHostInstanceID() {
+ return HostInstanceID;
+ }
+
+ public void setHostInstanceID(String hostInstanceID) {
+ HostInstanceID = hostInstanceID;
+ }
+
+ public String getLinkExpiredTime() {
+ return LinkExpiredTime;
+ }
+
+ public void setLinkExpiredTime(String linkExpiredTime) {
+ LinkExpiredTime = linkExpiredTime;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public void setFileName(String fileName) {
+ this.fileName = fileName;
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/DescribeBinlogFileResult.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/DescribeBinlogFileResult.java
new file mode 100644
index 00000000..d8d54e44
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/DescribeBinlogFileResult.java
@@ -0,0 +1,62 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.rds.data;
+
+/**
+ * @author chengjin.lyf on 2018/8/7 下午2:26
+ * @since 1.0.25
+ */
+public class DescribeBinlogFileResult {
+ private RdsItem Items;
+ private long PageNumber;
+ private long TotalRecordCount;
+ private long TotalFileSize;
+ private String RequestId;
+ private long PageRecordCount;
+
+ public RdsItem getItems() {
+ return Items;
+ }
+
+ public void setItems(RdsItem items) {
+ Items = items;
+ }
+
+ public long getPageNumber() {
+ return PageNumber;
+ }
+
+ public void setPageNumber(long pageNumber) {
+ PageNumber = pageNumber;
+ }
+
+ public long getTotalRecordCount() {
+ return TotalRecordCount;
+ }
+
+ public void setTotalRecordCount(long totalRecordCount) {
+ TotalRecordCount = totalRecordCount;
+ }
+
+ public long getTotalFileSize() {
+ return TotalFileSize;
+ }
+
+ public void setTotalFileSize(long totalFileSize) {
+ TotalFileSize = totalFileSize;
+ }
+
+ public String getRequestId() {
+ return RequestId;
+ }
+
+ public void setRequestId(String requestId) {
+ RequestId = requestId;
+ }
+
+ public long getPageRecordCount() {
+ return PageRecordCount;
+ }
+
+ public void setPageRecordCount(long pageRecordCount) {
+ PageRecordCount = pageRecordCount;
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsBackupPolicy.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsBackupPolicy.java
new file mode 100644
index 00000000..b6ca7c53
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsBackupPolicy.java
@@ -0,0 +1,69 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.rds.data;
+
+/**
+ * @author chengjin.lyf on 2018/8/7 下午2:26
+ * @since 1.0.25
+ */
+public class RdsBackupPolicy {
+
+ /**
+ * 数据备份保留天数（7到730天）。
+ */
+ private String BackupRetentionPeriod;
+ /**
+ * 数据备份时间，格式：HH:mmZ- HH:mm Z。
+ */
+ private String PreferredBackupTime;
+ /**
+ * 数据备份周期。Monday：周一；Tuesday：周二；Wednesday：周三；Thursday：周四；Friday：周五；Saturday：周六；Sunday：周日。
+ */
+ private String PreferredBackupPeriod;
+ /**
+ * 日志备份状态。Enable：开启；Disabled：关闭。
+ */
+ private boolean BackupLog;
+ /**
+ * 日志备份保留天数（7到730天）。
+ */
+ private int LogBackupRetentionPeriod;
+
+ public String getBackupRetentionPeriod() {
+ return BackupRetentionPeriod;
+ }
+
+ public void setBackupRetentionPeriod(String backupRetentionPeriod) {
+ BackupRetentionPeriod = backupRetentionPeriod;
+ }
+
+ public String getPreferredBackupTime() {
+ return PreferredBackupTime;
+ }
+
+ public void setPreferredBackupTime(String preferredBackupTime) {
+ PreferredBackupTime = preferredBackupTime;
+ }
+
+ public String getPreferredBackupPeriod() {
+ return PreferredBackupPeriod;
+ }
+
+ public void setPreferredBackupPeriod(String preferredBackupPeriod) {
+ PreferredBackupPeriod = preferredBackupPeriod;
+ }
+
+ public boolean isBackupLog() {
+ return BackupLog;
+ }
+
+ public void setBackupLog(boolean backupLog) {
+ BackupLog = backupLog;
+ }
+
+ public int getLogBackupRetentionPeriod() {
+ return LogBackupRetentionPeriod;
+ }
+
+ public void setLogBackupRetentionPeriod(int logBackupRetentionPeriod) {
+ LogBackupRetentionPeriod = logBackupRetentionPeriod;
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsItem.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsItem.java
new file mode 100644
index 00000000..a2cfaffe
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsItem.java
@@ -0,0 +1,19 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.rds.data;
+
+import java.util.List;
+
+/**
+ * @author chengjin.lyf on 2018/8/7 下午2:26
+ * @since 1.0.25
+ */
+public class RdsItem {
+ private List BinLogFile;
+
+ public List getBinLogFile() {
+ return BinLogFile;
+ }
+
+ public void setBinLogFile(List binLogFile) {
+ BinLogFile = binLogFile;
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java
new file mode 100644
index 00000000..976968f6
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java
@@ -0,0 +1,250 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.rds.request;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import javax.net.ssl.SSLContext;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.HttpClientConnectionManager;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.TrustStrategy;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.ssl.SSLContexts;
+import org.apache.http.util.EntityUtils;
+
+import io.netty.handler.codec.http.HttpResponseStatus;
+
+/**
+ * @author chengjin.lyf on 2018/8/7 下午2:26
+ * @since 1.0.25
+ */
+public abstract class AbstractRequest {
+
+ /**
+ * 要求的编码格式
+ */
+ private static final String ENCODING = "UTF-8";
+ /**
+ * 要求的sign签名算法
+ */
+ private static final String MAC_NAME = "HmacSHA1";
+
+ private String accessKeyId;
+
+ private String accessKeySecret;
+
+ /**
+ * api 版本
+ *
+ */
+ private String version;
+
+ private String endPoint = "rds.aliyuncs.com";
+
+ private String protocol = "http";
+
+ public void setProtocol(String protocol) {
+ this.protocol = protocol;
+ }
+
+ private int timeout = (int) TimeUnit.MINUTES.toMillis(1);
+
+
+ private Map treeMap = new TreeMap();
+
+ public void putQueryString(String name, String value){
+ if (StringUtils.isBlank(name) || StringUtils.isBlank(value)){
+ return;
+ }
+ treeMap.put(name, value);
+ }
+
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+
+ public void setEndPoint(String endPoint) {
+ this.endPoint = endPoint;
+ }
+
+ public void setAccessKeyId(String accessKeyId) {
+ this.accessKeyId = accessKeyId;
+ }
+
+ public void setAccessKeySecret(String accessKeySecret) {
+ this.accessKeySecret = accessKeySecret;
+ }
+
+ /**
+ * 使用 HMAC-SHA1 签名方法对对encryptText进行签名
+ *
+ * @param encryptText 被签名的字符串
+ * @param encryptKey 密钥
+ * @return
+ * @throws Exception
+ */
+ private byte[] HmacSHA1Encrypt(String encryptText, String encryptKey) throws Exception {
+ byte[] data = encryptKey.getBytes(ENCODING);
+ // 根据给定的字节数组构造一个密钥,第二参数指定一个密钥算法的名称
+ SecretKey secretKey = new SecretKeySpec(data, MAC_NAME);
+ // 生成一个指定 Mac 算法 的 Mac 对象
+ Mac mac = Mac.getInstance(MAC_NAME);
+ // 用给定密钥初始化 Mac 对象
+ mac.init(secretKey);
+
+ byte[] text = encryptText.getBytes(ENCODING);
+ // 完成 Mac 操作
+ return mac.doFinal(text);
+ }
+
+ private String base64(byte input[]) throws UnsupportedEncodingException {
+ return new String(Base64.encodeBase64(input), ENCODING);
+ }
+
+ private String concatQueryString(Map parameters) throws UnsupportedEncodingException {
+ if (null == parameters) {
+ return null;
+ }
+ StringBuilder urlBuilder = new StringBuilder("");
+ for (Map.Entry entry : parameters.entrySet()) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ urlBuilder.append(encode(key));
+ if (val != null) {
+ urlBuilder.append("=").append(encode(val));
+ }
+ urlBuilder.append("&");
+ }
+ int strIndex = urlBuilder.length();
+ if (parameters.size() > 0) {
+ urlBuilder.deleteCharAt(strIndex - 1);
+ }
+ return urlBuilder.toString();
+ }
+
+ private String encode(String value) throws UnsupportedEncodingException {
+ return URLEncoder.encode(value, "UTF-8");
+ }
+
+ private String makeSignature(TreeMap paramMap) throws Exception {
+ String cqs = concatQueryString(paramMap);
+ cqs = encode(cqs);
+ cqs = cqs.replaceAll("\\+", "%20");
+ cqs = cqs.replaceAll("\\*", "%2A");
+ cqs = cqs.replaceAll("%7E", "~");
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append("GET").append("&").append(encode("/")).append("&").append(cqs);
+ return base64(HmacSHA1Encrypt(stringBuilder.toString(), accessKeySecret + "&"));
+ }
+
+ public final String formatUTCTZ(Date date) {
+ SimpleDateFormat sdf = new SimpleDateFormat("YYYY-MM-dd'T'HH:mm:ss'Z'");
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+ return sdf.format(date);
+ }
+
+ private void fillCommonParam(Map p) {
+ p.put("Format", "JSON");
+ p.put("Version", version);
+ p.put("AccessKeyId", accessKeyId);
+ p.put("SignatureMethod", "HMAC-SHA1"); //此处不能用变量 MAC_NAME
+ p.put("Timestamp", formatUTCTZ(new Date()));
+ p.put("SignatureVersion", "1.0");
+ p.put("SignatureNonce", UUID.randomUUID().toString());
+ }
+
+ private String makeRequestString(Map param) throws Exception {
+ fillCommonParam(param);
+ String sign = makeSignature(new TreeMap(param));
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry entry : param.entrySet()) {
+ builder.append(encode(entry.getKey())).append("=").append(encode(entry.getValue())).append("&");
+ }
+ builder.append("Signature").append("=").append(sign);
+ return builder.toString();
+ }
+
+ /**
+ * 执行http请求
+ *
+ * @param getMethod
+ * @return
+ * @throws IOException
+ */
+ private final HttpResponse executeHttpRequest(HttpGet getMethod, String host) throws Exception {
+ SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
+
+ @Override
+ public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
+ return true;
+ }
+ }).build();
+ SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext,
+ new String[] { "TLSv1" },
+ null,
+ SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ Registry registry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .register("https", sslsf)
+ .build();
+ HttpClientConnectionManager httpClientConnectionManager = new PoolingHttpClientConnectionManager(registry);
+ CloseableHttpClient httpClient = HttpClientBuilder.create()
+ .setMaxConnPerRoute(50)
+ .setMaxConnTotal(100)
+ .setConnectionManager(httpClientConnectionManager)
+ .build();
+ RequestConfig requestConfig = RequestConfig.custom()
+ .setConnectTimeout(timeout)
+ .setConnectionRequestTimeout(timeout)
+ .setSocketTimeout(timeout)
+ .build();
+ getMethod.setConfig(requestConfig);
+ HttpResponse response = httpClient.execute(getMethod);
+ int statusCode = response.getStatusLine().getStatusCode();
+ if (statusCode != HttpResponseStatus.OK.code() && statusCode != HttpResponseStatus.PARTIAL_CONTENT.code()) {
+ String result = EntityUtils.toString(response.getEntity());
+ throw new RuntimeException("return error !" + response.getStatusLine().getReasonPhrase() + ", " + result);
+ }
+ return response;
+ }
+
+ protected abstract T processResult(HttpResponse response) throws Exception;
+
+ protected void processBefore(){
+
+ }
+
+ public final T doAction() throws Exception {
+ processBefore();
+ String requestStr = makeRequestString(treeMap);
+ HttpGet httpGet = new HttpGet(protocol + "://" +endPoint + "?" + requestStr);
+ HttpResponse response = executeHttpRequest(httpGet, endPoint);
+ if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+ String result = EntityUtils.toString(response.getEntity());
+ throw new RuntimeException("http request failed! " + result);
+ }
+ return processResult(response);
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBackupPolicyRequest.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBackupPolicyRequest.java
new file mode 100644
index 00000000..0fbdfc47
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBackupPolicyRequest.java
@@ -0,0 +1,41 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.rds.request;
+
+import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsBackupPolicy;
+import org.apache.http.HttpResponse;
+import org.apache.http.util.EntityUtils;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+
+/**
+ * rds 备份策略查询
+ * @author chengjin.lyf on 2018/8/7 下午3:41
+ * @since 1.0.25
+ */
+public class DescribeBackupPolicyRequest extends AbstractRequest {
+
+
+ public DescribeBackupPolicyRequest() {
+ setVersion("2014-08-15");
+ putQueryString("Action", "DescribeBackupPolicy");
+
+ }
+
+
+ public void setRdsInstanceId(String rdsInstanceId) {
+ putQueryString("DBInstanceId", rdsInstanceId);
+ }
+
+ @Override
+ protected RdsBackupPolicy processResult(HttpResponse response) throws Exception {
+ String result = EntityUtils.toString(response.getEntity());
+ JSONObject jsonObj = JSON.parseObject(result);
+ RdsBackupPolicy policy = new RdsBackupPolicy();
+ policy.setBackupRetentionPeriod(jsonObj.getString("BackupRetentionPeriod"));
+ policy.setBackupLog(jsonObj.getString("BackupLog").equalsIgnoreCase("Enable"));
+ policy.setLogBackupRetentionPeriod(jsonObj.getIntValue("LogBackupRetentionPeriod"));
+ policy.setPreferredBackupPeriod(jsonObj.getString("PreferredBackupPeriod"));
+ policy.setPreferredBackupTime(jsonObj.getString("PreferredBackupTime"));
+ return policy;
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBinlogFilesRequest.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBinlogFilesRequest.java
new file mode 100644
index 00000000..1c91c255
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBinlogFilesRequest.java
@@ -0,0 +1,56 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.rds.request;
+
+import java.util.Date;
+
+import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.DescribeBinlogFileResult;
+import org.apache.http.HttpResponse;
+import org.apache.http.util.EntityUtils;
+
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.TypeReference;
+
+/**
+ * @author chengjin.lyf on 2018/8/7 下午3:41
+ * @since 1.0.25
+ */
+public class DescribeBinlogFilesRequest extends AbstractRequest {
+
+
+ public DescribeBinlogFilesRequest() {
+ setVersion("2014-08-15");
+ putQueryString("Action", "DescribeBinlogFiles");
+
+ }
+
+ public void setRdsInstanceId(String rdsInstanceId) {
+ putQueryString("DBInstanceId", rdsInstanceId);
+ }
+
+ public void setPageSize(int pageSize) {
+ putQueryString("PageSize", String.valueOf(pageSize));
+ }
+
+ public void setPageNumber(int pageNumber) {
+ putQueryString("PageNumber", String.valueOf(pageNumber));
+ }
+
+ public void setStartDate(Date startDate) {
+ putQueryString("StartTime" , formatUTCTZ(startDate));
+ }
+
+ public void setEndDate(Date endDate) {
+ putQueryString("EndTime" , formatUTCTZ(endDate));
+ }
+
+ public void setResourceOwnerId(Long resourceOwnerId) {
+ putQueryString("ResourceOwnerId", String.valueOf(resourceOwnerId));
+ }
+
+ @Override
+ protected DescribeBinlogFileResult processResult(HttpResponse response) throws Exception {
+ String result = EntityUtils.toString(response.getEntity());
+ DescribeBinlogFileResult describeBinlogFileResult = JSONObject.parseObject(result, new TypeReference() {
+ });
+ return describeBinlogFileResult;
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java
new file mode 100644
index 00000000..0964627d
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java
@@ -0,0 +1,191 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
+
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
+import com.alibaba.otter.canal.parse.inbound.TableMeta;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception.CacheConnectionNull;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception.NoHistoryException;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+import java.io.IOException;
+import java.util.*;
+
+public class HistoryTableMetaCache {
+ private TableMetaStorage tableMetaStorage;
+ private MysqlConnection metaConnection;
+ private LoadingCache> cache; // 第一层：数据库名.表名，第二层时间戳，TableMeta
+
+ public HistoryTableMetaCache() {
+ cache = CacheBuilder.newBuilder().build(new CacheLoader>() {
+ @Override
+ public Map load(String tableName) throws Exception {
+ Long timestamp = new Date().getTime();
+ String[] strs = tableName.split("\\.");
+ String schema = strs[0];
+ if (tableMetaStorage != null) {
+ init(tableMetaStorage.fetchByTableName(tableName)); // 从存储中读取表的历史ddl
+ }
+ ResultSetPacket resultSetPacket = connectionQuery("show create table " + tableName); // 获取当前ddl
+ String currentDdl = resultSetPacket.getFieldValues().get(1);
+ if (cache.asMap().containsKey(tableName)) {
+ Map tableMetaMap = cache.getUnchecked(tableName);
+ if (tableMetaMap.isEmpty()) {
+ put(schema, tableName, currentDdl, timestamp - 1000L); // 放入当前schema，取时间为当前时间-1s
+ } else { // 如果table存在历史
+ Iterator iterator = tableMetaMap.keySet().iterator();
+ Long firstTimestamp = iterator.next();
+ TableMeta first = tableMetaMap.get(firstTimestamp); // 拿第一条ddl
+ if (!first.getDdl().equalsIgnoreCase(currentDdl)) { // 当前ddl与历史第一条不一致，放入当前ddl
+ put(schema, tableName, currentDdl, calculateNewTimestamp(firstTimestamp)); // 计算放入的timestamp,设为第一条时间+1s
+ }
+ }
+ } else {
+ put(schema, tableName, currentDdl, timestamp - 1000L); // 放入当前schema
+ }
+ return cache.get(tableName);
+ }
+ });
+ }
+
+ public void init(List entries) throws IOException {
+ if (entries == null) {
+ return;
+ }
+ for (TableMetaEntry entry : entries) {
+ try {
+ put(entry.getSchema(), entry.getTable(), entry.getDdl(), entry.getTimestamp());
+ } catch (CacheConnectionNull cacheConnectionNull) {
+ cacheConnectionNull.printStackTrace();
+ }
+ }
+ }
+
+ public TableMeta put(String schema, String table, String ddl, Long timestamp) throws CacheConnectionNull, IOException {
+ ResultSetPacket resultSetPacket;
+ if (!(ddl.contains("CREATE TABLE") || ddl.contains("create table"))) { // 尝试直接从数据库拉取CREATE TABLE的DDL
+ resultSetPacket = connectionQuery("show create table " + table);
+ ddl = resultSetPacket.getFieldValues().get(1);
+ } else { // CREATE TABLE 的 DDL
+ resultSetPacket = new ResultSetPacket();
+ List fields = new ArrayList();
+ String[] strings = table.split("\\.");
+ String shortTable = table;
+ if (strings.length > 1) {
+ shortTable = strings[1];
+ }
+ fields.add(0, shortTable);
+ fields.add(1, ddl);
+ resultSetPacket.setFieldValues(fields);
+ if (metaConnection != null) {
+ resultSetPacket.setSourceAddress(metaConnection.getAddress());
+ }
+ }
+ Map tableMetaMap;
+ if (!cache.asMap().containsKey(table)) {
+ tableMetaMap = new TreeMap(new Comparator() {
+ @Override
+ public int compare(Long o1, Long o2) {
+ return o2.compareTo(o1);
+ }
+ });
+ cache.put(table, tableMetaMap);
+ } else {
+ tableMetaMap = cache.getUnchecked(table);
+ }
+ eliminate(tableMetaMap); // 淘汰旧的TableMeta
+ TableMeta tableMeta = new TableMeta(schema, table, TableMetaCache.parseTableMeta(schema, table, resultSetPacket));
+ if (tableMeta.getDdl() == null) { // 生成的TableMeta有时DDL为null
+ tableMeta.setDdl(ddl);
+ }
+ tableMetaMap.put(timestamp, tableMeta);
+ return tableMeta;
+ }
+
+ public TableMeta get(String schema, String table, Long timestamp) throws NoHistoryException, CacheConnectionNull {
+ Map tableMetaMap = cache.getUnchecked(table);
+ Iterator iterator = tableMetaMap.keySet().iterator();
+ Long selected = null;
+ while(iterator.hasNext()) {
+ Long temp = iterator.next();
+ if (timestamp > temp) {
+ selected = temp;
+ break;
+ }
+ }
+
+ if (selected == null) {
+ iterator = tableMetaMap.keySet().iterator();
+ if (iterator.hasNext()) {
+ selected = iterator.next();
+ } else {
+ throw new NoHistoryException(schema, table);
+ }
+ }
+
+ return tableMetaMap.get(selected);
+ }
+
+ public void clearTableMeta() {
+ cache.invalidateAll();
+ }
+
+ public void clearTableMetaWithSchemaName(String schema) {
+ for (String tableName : cache.asMap().keySet()) {
+ String[] strs = tableName.split("\\.");
+ if (schema.equalsIgnoreCase(strs[0])) {
+ cache.invalidate(tableName);
+ }
+ }
+ }
+
+ public void clearTableMeta(String schema, String table) {
+ if (!table.contains(".")) {
+ table = schema+"."+table;
+ }
+ cache.invalidate(table);
+ }
+
+ // eliminate older table meta in cache
+ private void eliminate(Map tableMetaMap) {
+ int MAX_CAPABILITY = 20;
+ if (tableMetaMap.keySet().size() < MAX_CAPABILITY) {
+ return;
+ }
+ Iterator iterator = tableMetaMap.keySet().iterator();
+ while(iterator.hasNext()) {
+ iterator.next();
+ }
+ iterator.remove();
+ }
+
+ private Long calculateNewTimestamp(Long oldTimestamp) {
+ return oldTimestamp + 1000;
+ }
+
+ private ResultSetPacket connectionQuery(String query) throws CacheConnectionNull, IOException {
+ if (metaConnection == null) {
+ throw new CacheConnectionNull();
+ }
+ try {
+ return metaConnection.query(query);
+ } catch (IOException e) {
+ try {
+ metaConnection.reconnect();
+ return metaConnection.query(query);
+ } catch (IOException e1) {
+ throw e1;
+ }
+ }
+ }
+
+ public void setMetaConnection(MysqlConnection metaConnection) {
+ this.metaConnection = metaConnection;
+ }
+
+ public void setTableMetaStorage(TableMetaStorage tableMetaStorage) {
+ this.tableMetaStorage = tableMetaStorage;
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheInterface.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheInterface.java
new file mode 100644
index 00000000..5196ee08
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheInterface.java
@@ -0,0 +1,20 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
+
+import com.alibaba.otter.canal.parse.inbound.TableMeta;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+
+public interface TableMetaCacheInterface {
+
+ TableMeta getTableMeta(String schema, String table, boolean useCache, EntryPosition position);
+
+ void clearTableMeta();
+
+ void clearTableMetaWithSchemaName(String schema);
+
+ void clearTableMeta(String schema, String table);
+
+ boolean apply(EntryPosition position, String schema, String ddl, String extra);
+
+ boolean isOnRDS();
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheWithStorage.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheWithStorage.java
new file mode 100644
index 00000000..eb43158b
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheWithStorage.java
@@ -0,0 +1,105 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
+
+import com.alibaba.otter.canal.parse.inbound.TableMeta;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+public class TableMetaCacheWithStorage implements TableMetaCacheInterface {
+
+ private static Logger logger = LoggerFactory.getLogger(TableMetaCacheWithStorage.class);
+ private TableMetaStorage tableMetaStorage; // TableMeta存储
+ private HistoryTableMetaCache cache = new HistoryTableMetaCache(); // cache
+
+ public TableMetaCacheWithStorage(MysqlConnection con, TableMetaStorage tableMetaStorage) {
+ this.tableMetaStorage = tableMetaStorage;
+ InetSocketAddress address = con.getAddress();
+ this.tableMetaStorage.setDbAddress(address.getHostName()+":"+address.getPort());
+ cache.setMetaConnection(con);
+ cache.setTableMetaStorage(tableMetaStorage);
+ if (tableMetaStorage != null) {
+ try {
+ cache.init(tableMetaStorage.fetch()); // 初始化，从存储拉取TableMeta
+ } catch (IOException e) {
+ logger.error(e.getMessage());
+ }
+ }
+ }
+
+ @Override
+ public boolean apply(EntryPosition position, String fullTableName, String ddl, String extra) {
+ String[] strs = fullTableName.split("\\.");
+ String schema = strs[0];
+ if (schema.equalsIgnoreCase("null")) { // ddl schema为null，放弃处理
+ return false;
+ }
+ try {
+ TableMeta tableMeta = cache.get(schema, fullTableName, position.getTimestamp());
+ if (!compare(tableMeta, ddl)) { // 获取最近的TableMeta，进行比对
+ TableMeta result = cache.put(schema, fullTableName, ddl, calTimestamp(position.getTimestamp()));
+ if (tableMetaStorage != null && result != null) { // 储存
+ tableMetaStorage.store(schema, fullTableName, result.getDdl(), calTimestamp(position.getTimestamp()));
+ }
+ }
+ return true;
+ } catch (Exception e) {
+ logger.error(e.toString());
+ }
+
+ return false;
+ }
+
+ @Override
+ public boolean isOnRDS() {
+ return false;
+ }
+
+ /***
+ *
+ * @param schema dbname
+ * @param table tablename
+ * @param useCache unused
+ * @param position timestamp
+ * @return
+ */
+ @Override
+ public TableMeta getTableMeta(String schema, String table, boolean useCache, EntryPosition position) {
+ String fulltbName = schema + "." + table;
+ try {
+ return cache.get(schema, fulltbName, position.getTimestamp());
+ } catch (Exception e) {
+ logger.error(e.toString());
+ }
+ return null;
+ }
+
+ @Override
+ public void clearTableMeta() {
+ cache.clearTableMeta();
+ }
+
+ @Override
+ public void clearTableMetaWithSchemaName(String schema) {
+ cache.clearTableMetaWithSchemaName(schema);
+ }
+
+ @Override
+ public void clearTableMeta(String schema, String table) {
+ cache.clearTableMeta(schema, table);
+ }
+
+ private boolean compare(TableMeta tableMeta, String ddl) {
+ if (tableMeta == null) {
+ return false;
+ }
+ return tableMeta.getDdl().equalsIgnoreCase(ddl);
+ }
+
+ private Long calTimestamp(Long timestamp) {
+ return timestamp;
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaEntry.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaEntry.java
new file mode 100644
index 00000000..03e63ed7
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaEntry.java
@@ -0,0 +1,55 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
+
+import java.io.Serializable;
+
+public class TableMetaEntry implements Serializable {
+
+ private static final long serialVersionUID = -1350200637109107904L;
+
+ private String dbAddress;
+ private String schema;
+ private String table;
+ private String ddl;
+ private Long timestamp;
+
+
+ public String getSchema() {
+ return schema;
+ }
+
+ public void setSchema(String schema) {
+ this.schema = schema;
+ }
+
+ public String getTable() {
+ return table;
+ }
+
+ public void setTable(String table) {
+ this.table = table;
+ }
+
+ public String getDdl() {
+ return ddl;
+ }
+
+ public void setDdl(String ddl) {
+ this.ddl = ddl;
+ }
+
+ public Long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(Long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public String getDbAddress() {
+ return dbAddress;
+ }
+
+ public void setDbAddress(String dbAddress) {
+ this.dbAddress = dbAddress;
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorage.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorage.java
new file mode 100644
index 00000000..ac9d7a0a
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorage.java
@@ -0,0 +1,18 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
+
+import java.util.List;
+
+public interface TableMetaStorage {
+
+ void store(String schema, String table, String ddl, Long timestamp);
+
+ List fetch();
+
+ List fetchByTableName(String tableName);
+
+ String getDbName();
+
+ String getDbAddress();
+
+ void setDbAddress(String address);
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorageFactory.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorageFactory.java
new file mode 100644
index 00000000..0e532ebb
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorageFactory.java
@@ -0,0 +1,9 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
+
+public interface TableMetaStorageFactory {
+
+ TableMetaStorage getTableMetaStorage();
+
+ String getDbName();
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/CacheConnectionNull.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/CacheConnectionNull.java
new file mode 100644
index 00000000..1dadd78a
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/CacheConnectionNull.java
@@ -0,0 +1,9 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception;
+
+public class CacheConnectionNull extends Exception{
+
+ @Override
+ public String toString() {
+ return "CacheConnectionNull";
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/NoHistoryException.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/NoHistoryException.java
new file mode 100644
index 00000000..d8f1d16b
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/NoHistoryException.java
@@ -0,0 +1,21 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception;
+
+public class NoHistoryException extends Exception{
+
+ private String dbName;
+ private String tbName;
+
+ public NoHistoryException(String dbName, String tbName) {
+ this.dbName = dbName;
+ this.tbName = tbName;
+ }
+
+ public void printTableName() {
+ System.out.println(dbName+"."+tbName);
+ }
+
+ @Override
+ public String toString() {
+ return "NioHistoryException: " + dbName + " " + tbName;
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaCallback.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaCallback.java
new file mode 100644
index 00000000..661986a1
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaCallback.java
@@ -0,0 +1,14 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql;
+
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaEntry;
+
+import java.util.List;
+
+public interface MySqlTableMetaCallback {
+
+ void save(String dbAddress, String schema, String table,String ddl, Long timestamp);
+
+ List fetch(String dbAddress, String dbName);
+
+ List fetch(String dbAddress, String dbName, String tableName);
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorage.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorage.java
new file mode 100644
index 00000000..f120ef80
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorage.java
@@ -0,0 +1,48 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql;
+
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaEntry;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorage;
+
+import java.util.List;
+
+public class MySqlTableMetaStorage implements TableMetaStorage {
+ private MySqlTableMetaCallback mySqlTableMetaCallback;
+ private String dbName;
+ private String dbAddress;
+
+ MySqlTableMetaStorage(MySqlTableMetaCallback callback, String dbName) {
+ mySqlTableMetaCallback = callback;
+ this.dbName = dbName;
+ }
+
+
+ @Override
+ public void store(String schema, String table, String ddl, Long timestamp) {
+ mySqlTableMetaCallback.save(dbAddress, schema, table, ddl, timestamp);
+ }
+
+ @Override
+ public List fetch() {
+ return mySqlTableMetaCallback.fetch(dbAddress, dbName);
+ }
+
+ @Override
+ public List fetchByTableName(String tableName) {
+ return mySqlTableMetaCallback.fetch(dbAddress, dbName, tableName);
+ }
+
+ @Override
+ public String getDbName() {
+ return dbName;
+ }
+
+ @Override
+ public String getDbAddress() {
+ return dbAddress;
+ }
+
+ @Override
+ public void setDbAddress(String address) {
+ this.dbAddress = address;
+ }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorageFactory.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorageFactory.java
new file mode 100644
index 00000000..0eee235d
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorageFactory.java
@@ -0,0 +1,26 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql;
+
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorage;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorageFactory;
+
+public class MySqlTableMetaStorageFactory implements TableMetaStorageFactory {
+
+ private MySqlTableMetaCallback mySQLTableMetaCallback;
+ private String dbName;
+
+ public MySqlTableMetaStorageFactory(MySqlTableMetaCallback callback, String dbName) {
+ mySQLTableMetaCallback = callback;
+ this.dbName = dbName;
+ }
+
+ @Override
+ public TableMetaStorage getTableMetaStorage() {
+ return new MySqlTableMetaStorage(mySQLTableMetaCallback, dbName);
+ }
+
+ @Override
+ public String getDbName() {
+ return dbName;
+ }
+
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogDumpPerformanceTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogDumpPerformanceTest.java
new file mode 100644
index 00000000..8ca9b044
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogDumpPerformanceTest.java
@@ -0,0 +1,89 @@
+package com.alibaba.otter.canal.parse;
+
+import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
+import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.CanalEventSink;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+
+public class MysqlBinlogDumpPerformanceTest {
+
+ public static void main(String args[]) {
+ final MysqlEventParser controller = new MysqlEventParser();
+ final EntryPosition startPosition = new EntryPosition("mysql-bin.001699", 120L, 100L);
+ controller.setConnectionCharset(Charset.forName("UTF-8"));
+ controller.setSlaveId(3344L);
+ controller.setDetectingEnable(false);
+ controller.setFilterQueryDml(true);
+ controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3328), "root", "hello"));
+ controller.setMasterPosition(startPosition);
+ controller.setEnableTsdb(false);
+ controller.setDestination("example");
+ controller.setTsdbSpringXml("classpath:spring/tsdb/h2-tsdb.xml");
+ // controller.setEventFilter(new AviaterRegexFilter("test\\..*"));
+ // controller.setEventBlackFilter(new
+ // AviaterRegexFilter("canal_tsdb\\..*"));
+ controller.setParallel(true);
+ controller.setParallelBufferSize(256);
+ controller.setParallelThreadSize(16);
+ controller.setIsGTIDMode(false);
+ final AtomicLong sum = new AtomicLong(0);
+ final AtomicLong last = new AtomicLong(0);
+ final AtomicLong start = new AtomicLong(System.currentTimeMillis());
+ final AtomicLong end = new AtomicLong(0);
+ controller.setEventSink(new AbstractCanalEventSinkTest>() {
+
+ public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination)
+ throws CanalSinkException,
+ InterruptedException {
+
+ sum.addAndGet(entrys.size());
+ long current = sum.get();
+ if (current - last.get() >= 100000) {
+ end.set(System.currentTimeMillis());
+ long tps = ((current - last.get()) * 1000) / (end.get() - start.get());
+ System.out.println(" total : " + sum + " , cost : " + (end.get() - start.get()) + " , tps : " + tps);
+ last.set(current);
+ start.set(end.get());
+ }
+ return true;
+ }
+
+ });
+ controller.setLogPositionManager(new AbstractLogPositionManager() {
+
+ @Override
+ public LogPosition getLatestIndexBy(String destination) {
+ return null;
+ }
+
+ @Override
+ public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
+ }
+ });
+
+ controller.start();
+
+ try {
+ Thread.sleep(100 * 1000 * 1000L);
+ } catch (InterruptedException e) {
+ }
+ controller.stop();
+ }
+
+ public static abstract class AbstractCanalEventSinkTest extends AbstractCanalLifeCycle implements CanalEventSink {
+
+ public void interrupt() {
+ }
+ }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogEventPerformanceTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogEventPerformanceTest.java
new file mode 100644
index 00000000..e92fecb5
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogEventPerformanceTest.java
@@ -0,0 +1,82 @@
+package com.alibaba.otter.canal.parse;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
+import com.taobao.tddl.dbsync.binlog.LogContext;
+import com.taobao.tddl.dbsync.binlog.LogDecoder;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+public class MysqlBinlogEventPerformanceTest {
+
+ protected static Charset charset = Charset.forName("utf-8");
+
+ public static void main(String args[]) {
+ DirectLogFetcher fetcher = new DirectLogFetcher();
+ try {
+ MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
+ connector.connect();
+ updateSettings(connector);
+ sendBinlogDump(connector, "mysql-bin.000006", 120L, 3);
+ fetcher.start(connector.getChannel());
+ LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
+ LogContext context = new LogContext();
+ AtomicLong sum = new AtomicLong(0);
+ long start = System.currentTimeMillis();
+ long last = 0;
+ long end = 0;
+ while (fetcher.fetch()) {
+ decoder.decode(fetcher, context);
+ sum.incrementAndGet();
+ long current = sum.get();
+ if (current - last >= 100000) {
+ end = System.currentTimeMillis();
+ long tps = ((current - last) * 1000) / (end - start);
+ System.out.println(" total : " + sum + " , cost : " + (end - start) + " , tps : " + tps);
+ last = current;
+ start = end;
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ fetcher.close();
+ } catch (IOException e) {
+ }
+ }
+ }
+
+ private static void sendBinlogDump(MysqlConnector connector, String binlogfilename, Long binlogPosition, int slaveId)
+ throws IOException {
+ BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket();
+ binlogDumpCmd.binlogFileName = binlogfilename;
+ binlogDumpCmd.binlogPosition = binlogPosition;
+ binlogDumpCmd.slaveServerId = slaveId;
+ byte[] cmdBody = binlogDumpCmd.toBytes();
+
+ HeaderPacket binlogDumpHeader = new HeaderPacket();
+ binlogDumpHeader.setPacketBodyLength(cmdBody.length);
+ binlogDumpHeader.setPacketSequenceNumber((byte) 0x00);
+ PacketManager.writePkg(connector.getChannel(), binlogDumpHeader.toBytes(), cmdBody);
+ }
+
+ private static void updateSettings(MysqlConnector connector) throws IOException {
+ update("set @master_binlog_checksum= '@@global.binlog_checksum'", connector);
+ update("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'", connector);
+ }
+
+ public static void update(String cmd, MysqlConnector connector) throws IOException {
+ MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector);
+ exector.update(cmd);
+ }
+
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java
new file mode 100644
index 00000000..79b88375
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java
@@ -0,0 +1,192 @@
+package com.alibaba.otter.canal.parse;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.util.BitSet;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogContext;
+import com.taobao.tddl.dbsync.binlog.LogDecoder;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
+import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
+
+public class MysqlBinlogParsePerformanceTest {
+
+ protected static Charset charset = Charset.forName("utf-8");
+
+ public static void main(String args[]) {
+ DirectLogFetcher fetcher = new DirectLogFetcher();
+ try {
+ MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
+ connector.connect();
+ updateSettings(connector);
+ sendBinlogDump(connector, "mysql-bin.000006", 120L, 3);
+ fetcher.start(connector.getChannel());
+ final BlockingQueue buffer = new ArrayBlockingQueue(1024);
+ Thread thread = new Thread(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ consumer(buffer);
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ thread.start();
+
+ while (fetcher.fetch()) {
+ buffer.put(fetcher.duplicate());
+ fetcher.consume(fetcher.limit());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ fetcher.close();
+ } catch (IOException e) {
+ }
+ }
+ }
+
+ public static void consumer(BlockingQueue buffer) throws IOException, InterruptedException {
+ LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
+ LogContext context = new LogContext();
+
+ AtomicLong sum = new AtomicLong(0);
+ long start = System.currentTimeMillis();
+ long last = 0;
+ long end = 0;
+ while (true) {
+ LogEvent event = null;
+ event = decoder.decode(buffer.take(), context);
+ int eventType = event.getHeader().getType();
+ switch (eventType) {
+ case LogEvent.ROTATE_EVENT:
+ break;
+ case LogEvent.WRITE_ROWS_EVENT_V1:
+ case LogEvent.WRITE_ROWS_EVENT:
+ parseRowsEvent((WriteRowsLogEvent) event, sum);
+ break;
+ case LogEvent.UPDATE_ROWS_EVENT_V1:
+ case LogEvent.UPDATE_ROWS_EVENT:
+ parseRowsEvent((UpdateRowsLogEvent) event, sum);
+ break;
+ case LogEvent.DELETE_ROWS_EVENT_V1:
+ case LogEvent.DELETE_ROWS_EVENT:
+ parseRowsEvent((DeleteRowsLogEvent) event, sum);
+ break;
+ case LogEvent.XID_EVENT:
+ sum.incrementAndGet();
+ break;
+ case LogEvent.QUERY_EVENT:
+ sum.incrementAndGet();
+ break;
+ default:
+ break;
+ }
+
+ long current = sum.get();
+ if (current - last >= 100000) {
+ end = System.currentTimeMillis();
+ long tps = ((current - last) * 1000) / (end - start);
+ System.out.println(" total : " + sum + " , cost : " + (end - start) + " , tps : " + tps);
+ last = current;
+ start = end;
+ }
+ }
+ }
+
+ private static void sendBinlogDump(MysqlConnector connector, String binlogfilename, Long binlogPosition, int slaveId)
+ throws IOException {
+ BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket();
+ binlogDumpCmd.binlogFileName = binlogfilename;
+ binlogDumpCmd.binlogPosition = binlogPosition;
+ binlogDumpCmd.slaveServerId = slaveId;
+ byte[] cmdBody = binlogDumpCmd.toBytes();
+
+ HeaderPacket binlogDumpHeader = new HeaderPacket();
+ binlogDumpHeader.setPacketBodyLength(cmdBody.length);
+ binlogDumpHeader.setPacketSequenceNumber((byte) 0x00);
+ PacketManager.writePkg(connector.getChannel(), binlogDumpHeader.toBytes(), cmdBody);
+ }
+
+ private static void updateSettings(MysqlConnector connector) throws IOException {
+ update("set @master_binlog_checksum= '@@global.binlog_checksum'", connector);
+ update("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'", connector);
+ }
+
+ public static void update(String cmd, MysqlConnector connector) throws IOException {
+ MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector);
+ exector.update(cmd);
+ }
+
+ public static void parseRowsEvent(RowsLogEvent event, AtomicLong sum) {
+ try {
+ RowsLogBuffer buffer = event.getRowsBuf(charset.name());
+ BitSet columns = event.getColumns();
+ BitSet changeColumns = event.getChangeColumns();
+ while (buffer.nextOneRow(columns)) {
+ int type = event.getHeader().getType();
+ if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
+ parseOneRow(event, buffer, columns, true);
+ } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
+ parseOneRow(event, buffer, columns, false);
+ } else {
+ parseOneRow(event, buffer, columns, false);
+ if (!buffer.nextOneRow(changeColumns)) {
+ break;
+ }
+ parseOneRow(event, buffer, changeColumns, true);
+ }
+
+ sum.incrementAndGet();
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("parse row data failed.", e);
+ }
+ }
+
+ public static void parseOneRow(RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter)
+ throws UnsupportedEncodingException {
+ TableMapLogEvent map = event.getTable();
+ if (map == null) {
+ throw new RuntimeException("not found TableMap with tid=" + event.getTableId());
+ }
+
+ final int columnCnt = map.getColumnCnt();
+ final ColumnInfo[] columnInfo = map.getColumnInfo();
+ for (int i = 0; i < columnCnt; i++) {
+ if (!cols.get(i)) {
+ continue;
+ }
+
+ ColumnInfo info = columnInfo[i];
+ buffer.nextValue(info.type, info.meta);
+ if (buffer.isNull()) {
+ } else {
+ buffer.getValue();
+ }
+ }
+ }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java
index 6849c5b3..6c11c637 100644
--- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java
@@ -41,6 +41,9 @@ public class MysqlDumpTest {
controller.setTsdbSpringXml("classpath:tsdb/h2-tsdb.xml");
controller.setEventFilter(new AviaterRegexFilter("test\\..*"));
controller.setEventBlackFilter(new AviaterRegexFilter("canal_tsdb\\..*"));
+ controller.setParallel(true);
+ controller.setParallelBufferSize(256);
+ controller.setParallelThreadSize(2);
controller.setIsGTIDMode(false);
controller.setEventSink(new AbstractCanalEventSinkTest>() {
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogEventParserProxyTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogEventParserProxyTest.java
new file mode 100644
index 00000000..18a217b8
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogEventParserProxyTest.java
@@ -0,0 +1,153 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.helper.TimeoutChecker;
+import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaEntry;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaCallback;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaStorageFactory;
+import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+
+/**
+ * @author chengjin.lyf on 2018/7/21 下午5:24
+ * @since 1.0.25
+ */
+public class RdsBinlogEventParserProxyTest {
+
+ private static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
+ private static final String MYSQL_ADDRESS = "";
+ private static final String USERNAME = "";
+ private static final String PASSWORD = "";
+ public static final String DBNAME = "";
+ public static final String TBNAME = "";
+ public static final String DDL = "";
+
+
+ @Test
+ public void test_timestamp() throws InterruptedException {
+ final TimeoutChecker timeoutChecker = new TimeoutChecker(3000 * 1000);
+ final AtomicLong entryCount = new AtomicLong(0);
+ final EntryPosition entryPosition = new EntryPosition();
+
+ final RdsBinlogEventParserProxy controller = new RdsBinlogEventParserProxy();
+ Calendar calendar = Calendar.getInstance();
+ calendar.add(Calendar.DAY_OF_YEAR, -1);
+ final EntryPosition defaultPosition = buildPosition(null, null, calendar.getTimeInMillis());
+ controller.setSlaveId(3344L);
+ controller.setDetectingEnable(false);
+ controller.setDetectingSQL(DETECTING_SQL);
+ controller.setMasterInfo(buildAuthentication());
+ controller.setMasterPosition(defaultPosition);
+ controller.setInstanceId("");
+ controller.setAccesskey("");
+ controller.setSecretkey("");
+ controller.setBatchSize(4);
+// controller.setRdsOpenApiUrl("https://rds.aliyuncs.com/");
+ controller.setTableMetaStorageFactory(new MySqlTableMetaStorageFactory(new MySqlTableMetaCallback() {
+ @Override
+ public void save(String dbAddress, String schema, String table, String ddl, Long timestamp) {
+
+ }
+
+ @Override
+ public List fetch(String dbAddress, String dbName) {
+ TableMetaEntry tableMeta = new TableMetaEntry();
+ tableMeta.setSchema(DBNAME);
+ tableMeta.setTable(TBNAME);
+ tableMeta.setDdl(DDL);
+ tableMeta.setTimestamp(new Date().getTime());
+ List entries = new ArrayList();
+ entries.add(tableMeta);
+ return entries;
+ }
+
+ @Override
+ public List fetch(String dbAddress, String dbName, String tableName) {
+ TableMetaEntry tableMeta = new TableMetaEntry();
+ tableMeta.setSchema(DBNAME);
+ tableMeta.setTable(TBNAME);
+ tableMeta.setDdl(DDL);
+ tableMeta.setTimestamp(new Date().getTime());
+ List entries = new ArrayList();
+ entries.add(tableMeta);
+ return entries;
+ }
+ }, DBNAME));
+ controller.setEventSink(new AbstractCanalEventSinkTest>() {
+
+ @Override
+ public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination)
+ throws CanalSinkException {
+ for (CanalEntry.Entry entry : entrys) {
+ if (entry.getEntryType() != CanalEntry.EntryType.HEARTBEAT) {
+ entryCount.incrementAndGet();
+
+ String logfilename = entry.getHeader().getLogfileName();
+ long logfileoffset = entry.getHeader().getLogfileOffset();
+ long executeTime = entry.getHeader().getExecuteTime();
+
+ entryPosition.setJournalName(logfilename);
+ entryPosition.setPosition(logfileoffset);
+ entryPosition.setTimestamp(executeTime);
+ break;
+ }
+ }
+ return true;
+ }
+ });
+
+ controller.setLogPositionManager(new AbstractLogPositionManager() {
+
+ private LogPosition logPosition;
+ public void persistLogPosition(String destination, LogPosition logPosition) {
+ System.out.println(logPosition);
+ this.logPosition = logPosition;
+ }
+
+ public LogPosition getLatestIndexBy(String destination) {
+ return logPosition;
+ }
+ });
+
+ controller.start();
+ timeoutChecker.waitForIdle();
+
+ if (controller.isStart()) {
+ controller.stop();
+ }
+
+ // check
+ Assert.assertTrue(entryCount.get() > 0);
+
+ // 对比第一条数据和起始的position相同
+ Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
+ Assert.assertTrue(entryPosition.getPosition() <= 6163L);
+ Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
+ }
+
+
+ // ======================== helper method =======================
+
+ private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {
+ return new EntryPosition(binlogFile, offest, timestamp);
+ }
+
+ private AuthenticationInfo buildAuthentication() {
+ return new AuthenticationInfo(new InetSocketAddress(MYSQL_ADDRESS, 3306), USERNAME, PASSWORD);
+ }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/NoStorageTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/NoStorageTest.java
new file mode 100644
index 00000000..04fd95ea
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/NoStorageTest.java
@@ -0,0 +1,38 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
+
+import com.alibaba.otter.canal.parse.inbound.TableMeta;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import org.junit.Test;
+
+import java.net.InetSocketAddress;
+import java.util.Date;
+
+public class NoStorageTest {
+ final String DBNAME = "testdb";
+ final String TBNAME = "testtb";
+ final String DDL = "CREATE TABLE `testtb` (\n" +
+ " `id` int(11) NOT NULL AUTO_INCREMENT,\n" +
+ " `name` varchar(2048) DEFAULT NULL,\n" +
+ " `datachange_lasttime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最晚更新时间',\n" +
+ " `otter_testcol` varchar(45) DEFAULT NULL,\n" +
+ " `otter_testcol1` varchar(45) DEFAULT NULL,\n" +
+ " `otter_testcol2` varchar(45) DEFAULT NULL,\n" +
+ " `otter_testcol3` varchar(45) DEFAULT NULL,\n" +
+ " `otter_testcol4` varchar(45) DEFAULT NULL,\n" +
+ " `otter_testcol5` varchar(45) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`id`)\n" +
+ " ) ENGINE=InnoDB AUTO_INCREMENT=58333898 DEFAULT CHARSET=utf8mb4";
+ @Test
+ public void nostorage() {
+ MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
+ TableMetaCacheWithStorage tableMetaCacheWithStorage = new TableMetaCacheWithStorage(connection, null);
+ EntryPosition entryPosition = new EntryPosition();
+ entryPosition.setTimestamp(new Date().getTime());
+ String fullTableName = DBNAME + "." + TBNAME;
+ tableMetaCacheWithStorage.apply(entryPosition, fullTableName, DDL, null);
+ entryPosition.setTimestamp(new Date().getTime() + 1000L);
+ TableMeta result = tableMetaCacheWithStorage.getTableMeta(DBNAME, TBNAME, false, entryPosition);
+ assert result.getDdl().equalsIgnoreCase(DDL);
+ }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/StorageTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/StorageTest.java
new file mode 100644
index 00000000..0ec16607
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/StorageTest.java
@@ -0,0 +1,77 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
+
+import com.alibaba.otter.canal.parse.inbound.TableMeta;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaCallback;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaStorageFactory;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.Position;
+import org.junit.Test;
+
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+public class StorageTest {
+
+ final String DBNAME = "testdb";
+ final String TBNAME = "testtb";
+ final String DDL = "CREATE TABLE `testtb` (\n" +
+ " `id` int(11) NOT NULL AUTO_INCREMENT,\n" +
+ " `name` varchar(2048) DEFAULT NULL,\n" +
+ " `datachange_lasttime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最晚更新时间',\n" +
+ " `otter_testcol` varchar(45) DEFAULT NULL,\n" +
+ " `otter_testcol1` varchar(45) DEFAULT NULL,\n" +
+ " `otter_testcol2` varchar(45) DEFAULT NULL,\n" +
+ " `otter_testcol3` varchar(45) DEFAULT NULL,\n" +
+ " `otter_testcol4` varchar(45) DEFAULT NULL,\n" +
+ " `otter_testcol5` varchar(45) DEFAULT NULL,\n" +
+ " PRIMARY KEY (`id`)\n" +
+ " ) ENGINE=InnoDB AUTO_INCREMENT=58333898 DEFAULT CHARSET=utf8mb4";
+
+ @Test
+ public void storage() {
+
+ MySqlTableMetaStorageFactory factory = new MySqlTableMetaStorageFactory(new MySqlTableMetaCallback() {
+ @Override
+ public void save(String dbAddress, String schema, String table, String ddl, Long timestamp) {
+
+ }
+
+ @Override
+ public List fetch(String dbAddress, String dbName) {
+ TableMetaEntry tableMeta = new TableMetaEntry();
+ tableMeta.setSchema(DBNAME);
+ tableMeta.setTable(TBNAME);
+ tableMeta.setDdl(DDL);
+ tableMeta.setTimestamp(new Date().getTime());
+ List entries = new ArrayList();
+ entries.add(tableMeta);
+ return entries;
+ }
+
+ @Override
+ public List fetch(String dbAddress, String dbName, String tableName) {
+ TableMetaEntry tableMeta = new TableMetaEntry();
+ tableMeta.setSchema(DBNAME);
+ tableMeta.setTable(TBNAME);
+ tableMeta.setDdl(DDL);
+ tableMeta.setTimestamp(new Date().getTime());
+ List entries = new ArrayList();
+ entries.add(tableMeta);
+ return entries;
+ }
+ }, DBNAME);
+ MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
+ TableMetaCacheWithStorage tableMetaCacheWithStorage = new TableMetaCacheWithStorage(connection, factory.getTableMetaStorage());
+ EntryPosition entryPosition = new EntryPosition();
+ entryPosition.setTimestamp(new Date().getTime());
+ String fullTableName = DBNAME + "." + TBNAME;
+ tableMetaCacheWithStorage.apply(entryPosition, fullTableName, DDL, null);
+
+ entryPosition.setTimestamp(new Date().getTime() + 1000L);
+ TableMeta result = tableMetaCacheWithStorage.getTableMeta(DBNAME, TBNAME, false, entryPosition);
+ assert result.getDdl().equalsIgnoreCase(DDL);
+ }
+}
diff --git a/pom.xml b/pom.xml
index 1e8123ef..56ac821a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -66,16 +66,6 @@
false
-
- alibaba
- http://code.alibabatech.com/mvn/releases/
-
- true
-
-
- false
-
-
sonatype
sonatype
@@ -106,8 +96,8 @@
true
true
- 1.6
- 1.6
+ 1.7
+ 1.7
UTF-8
3.2.9.RELEASE
@@ -129,6 +119,7 @@
example
kafka
kafka-client
+ prometheus
@@ -256,7 +247,7 @@
com.alibaba.fastsql
fastsql
- 2.0.0_preview_520
+ 2.0.0_preview_540
com.alibaba
@@ -341,7 +332,7 @@
org.apache.maven.plugins
maven-compiler-plugin
- 3.2
+ 3.8.0
${java_source_version}
${java_target_version}
diff --git a/prometheus/pom.xml b/prometheus/pom.xml
new file mode 100644
index 00000000..16cf8acf
--- /dev/null
+++ b/prometheus/pom.xml
@@ -0,0 +1,67 @@
+
+
+
+ canal
+ com.alibaba.otter
+ 1.0.26-SNAPSHOT
+
+ 4.0.0
+
+ com.alibaba.otter
+ canal.prometheus
+ 1.0.26-SNAPSHOT
+ canal prometheus module for otter ${project.version}
+
+
+
+ org.aspectj
+ aspectjrt
+ 1.8.9
+
+
+ org.aspectj
+ aspectjweaver
+ 1.8.9
+
+
+ org.jctools
+ jctools-core
+ 2.1.2
+
+
+ io.prometheus
+ simpleclient
+ 0.4.0
+
+
+ io.prometheus
+ simpleclient_hotspot
+ 0.4.0
+
+
+ io.prometheus
+ simpleclient_httpserver
+ 0.4.0
+
+
+ io.prometheus
+ simpleclient_pushgateway
+ 0.4.0
+
+
+ com.alibaba.otter
+ canal.instance.core
+ ${project.version}
+ provided
+
+
+ com.alibaba.otter
+ canal.server
+ ${project.version}
+ provided
+
+
+
+
diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java
new file mode 100644
index 00000000..8acf40b8
--- /dev/null
+++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java
@@ -0,0 +1,98 @@
+package com.alibaba.otter.canal.prometheus;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.prometheus.impl.InstanceMetaCollector;
+import com.alibaba.otter.canal.prometheus.impl.MemoryStoreCollector;
+import com.alibaba.otter.canal.prometheus.impl.PrometheusCanalEventDownStreamHandler;
+import com.alibaba.otter.canal.sink.CanalEventSink;
+import com.alibaba.otter.canal.sink.entry.EntryEventSink;
+import com.alibaba.otter.canal.store.CanalStoreException;
+import io.prometheus.client.Collector;
+import io.prometheus.client.CollectorRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * @author Chuanyi Li
+ */
+public class CanalInstanceExports {
+
+ private static final Logger logger = LoggerFactory.getLogger(CanalInstanceExports.class);
+
+ public static final String[] labels = {"destination"};
+
+ public static final List labelList = Collections.singletonList(labels[0]);
+
+ private final String destination;
+
+ private Collector storeCollector;
+
+ private Collector delayCollector;
+
+ private Collector metaCollector;
+
+ private CanalInstanceExports(CanalInstance instance) {
+ this.destination = instance.getDestination();
+ initDelayGauge(instance);
+ initStoreCollector(instance);
+ initMetaCollector(instance);
+ }
+
+
+
+ static CanalInstanceExports forInstance(CanalInstance instance) {
+ return new CanalInstanceExports(instance);
+ }
+
+ void register() {
+ if (delayCollector != null) {
+ delayCollector.register();
+ }
+ if (storeCollector != null) {
+ storeCollector.register();
+ }
+ if (metaCollector != null) {
+ metaCollector.register();
+ }
+ }
+
+ void unregister() {
+ if (delayCollector != null) {
+ CollectorRegistry.defaultRegistry.unregister(delayCollector);
+ }
+ if (storeCollector != null) {
+ CollectorRegistry.defaultRegistry.unregister(storeCollector);
+ }
+ if (metaCollector != null) {
+ CollectorRegistry.defaultRegistry.unregister(metaCollector);
+ }
+ }
+
+ private void initDelayGauge(CanalInstance instance) {
+ CanalEventSink sink = instance.getEventSink();
+ if (sink instanceof EntryEventSink) {
+ EntryEventSink entryEventSink = (EntryEventSink) sink;
+ // TODO ensure not to add handler again
+ PrometheusCanalEventDownStreamHandler handler = new PrometheusCanalEventDownStreamHandler(destination);
+ entryEventSink.addHandler(handler);
+ delayCollector = handler.getCollector();
+ } else {
+ logger.warn("This impl register metrics for only EntryEventSink, skip.");
+ }
+ }
+
+ private void initStoreCollector(CanalInstance instance) {
+ try {
+ storeCollector = new MemoryStoreCollector(instance.getEventStore(), destination);
+ } catch (CanalStoreException cse) {
+ logger.warn("Failed to register metrics for destination {}.", destination, cse);
+ }
+ }
+
+ private void initMetaCollector(CanalInstance instance) {
+ metaCollector = new InstanceMetaCollector(instance);
+ }
+}
diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java
new file mode 100644
index 00000000..8eef79ba
--- /dev/null
+++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java
@@ -0,0 +1,21 @@
+package com.alibaba.otter.canal.prometheus;
+
+import com.alibaba.otter.canal.prometheus.impl.InboundThroughputAspect;
+import com.alibaba.otter.canal.prometheus.impl.OutboundThroughputAspect;
+
+/**
+ * @author Chuanyi Li
+ */
+public class CanalServerExports {
+
+ private static boolean initialized = false;
+
+ public static synchronized void initialize() {
+ if (!initialized) {
+ InboundThroughputAspect.getCollector().register();
+ OutboundThroughputAspect.getCollector().register();
+ initialized = true;
+ }
+ }
+
+}
diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusProvider.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusProvider.java
new file mode 100644
index 00000000..12b8675f
--- /dev/null
+++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusProvider.java
@@ -0,0 +1,15 @@
+package com.alibaba.otter.canal.prometheus;
+
+import com.alibaba.otter.canal.spi.CanalMetricsProvider;
+import com.alibaba.otter.canal.spi.CanalMetricsService;
+
+/**
+ * @author Chuanyi Li
+ */
+public class PrometheusProvider implements CanalMetricsProvider {
+
+ @Override
+ public CanalMetricsService getService() {
+ return PrometheusService.getInstance();
+ }
+}
diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java
new file mode 100644
index 00000000..c7942235
--- /dev/null
+++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java
@@ -0,0 +1,110 @@
+package com.alibaba.otter.canal.prometheus;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.spi.CanalMetricsService;
+import io.prometheus.client.exporter.HTTPServer;
+import io.prometheus.client.hotspot.DefaultExports;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author Chuanyi Li
+ */
+public class PrometheusService implements CanalMetricsService {
+
+ private static final Logger logger = LoggerFactory.getLogger(PrometheusService.class);
+
+ private final Map exports = new ConcurrentHashMap();
+
+ private volatile boolean running = false;
+
+ private HTTPServer server;
+
+ private PrometheusService() {
+ }
+
+ private static class SingletonHolder {
+ private static final PrometheusService SINGLETON = new PrometheusService();
+ }
+
+ public static PrometheusService getInstance() {
+ return SingletonHolder.SINGLETON;
+ }
+
+ @Override
+ public void initialize() {
+ try {
+ //TODO 1.Configurable port
+ //TODO 2.Https
+ server = new HTTPServer(11112);
+ } catch (IOException e) {
+ logger.warn("Unable to start prometheus HTTPServer.", e);
+ return;
+ }
+ try {
+ // JVM exports
+ DefaultExports.initialize();
+ // Canal server level exports
+ CanalServerExports.initialize();
+ } catch (Throwable t) {
+ logger.warn("Unable to initialize server exports.", t);
+ }
+
+ running = true;
+ }
+
+ @Override
+ public void terminate() {
+ running = false;
+ // Normally, service should be terminated at canal shutdown.
+ // No need to unregister instance exports explicitly.
+ // But for the sake of safety, unregister them.
+ for (CanalInstanceExports ie : exports.values()) {
+ ie.unregister();
+ }
+ if (server != null) {
+ server.stop();
+ }
+ }
+
+ @Override
+ public boolean isRunning() {
+ return running;
+ }
+
+ @Override
+ public void register(CanalInstance instance) {
+ if (instance.isStart()) {
+ logger.warn("Cannot register metrics for destination {} that is running.", instance.getDestination());
+ return;
+ }
+ try {
+ CanalInstanceExports export = CanalInstanceExports.forInstance(instance);
+ export.register();
+ exports.put(instance.getDestination(), export);
+ } catch (Throwable t) {
+ logger.warn("Unable to register instance exports for {}.", instance.getDestination(), t);
+ }
+ logger.info("Register metrics for destination {}.", instance.getDestination());
+ }
+
+ @Override
+ public void unregister(CanalInstance instance) {
+ if (instance.isStart()) {
+ logger.warn("Try unregister metrics after destination {} is stopped.", instance.getDestination());
+ }
+ try {
+ CanalInstanceExports export = exports.remove(instance.getDestination());
+ if (export != null) {
+ export.unregister();
+ }
+ } catch (Throwable t) {
+ logger.warn("Unable to unregister instance exports for {}.", instance.getDestination(), t);
+ }
+ logger.info("Unregister metrics for destination {}.", instance.getDestination());
+ }
+}
diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java
new file mode 100644
index 00000000..eabc256c
--- /dev/null
+++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java
@@ -0,0 +1,79 @@
+package com.alibaba.otter.canal.prometheus.impl;
+
+import io.prometheus.client.Collector;
+import io.prometheus.client.CounterMetricFamily;
+import org.aspectj.lang.annotation.After;
+import org.aspectj.lang.annotation.Aspect;
+import org.aspectj.lang.annotation.Pointcut;
+import org.jctools.maps.ConcurrentAutoTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Chuanyi Li
+ */
+@Aspect
+public class InboundThroughputAspect {
+
+ private static final Logger logger = LoggerFactory.getLogger(InboundThroughputAspect.class);
+
+ /**
+ * Support highly scalable counters
+ * @see ConcurrentAutoTable
+ */
+ private static final ConcurrentAutoTable total = new ConcurrentAutoTable();
+
+ private static final Collector collector = new InboundThroughputCollector();
+
+ public static Collector getCollector() {
+ return collector;
+ }
+
+ @Pointcut("call(byte[] com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..))")
+ public void read() {}
+
+ @Pointcut("call(void com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..)) ")
+ public void readBytes() {}
+
+ //nested read, just eliminate them.
+ @Pointcut("withincode(* com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..))")
+ public void nestedCall() {}
+
+ @After("read() && !nestedCall() && args(len, ..)")
+ public void recordRead(int len) {
+ accumulateBytes(len);
+ }
+
+ @After("readBytes() && !nestedCall() && args(.., len, timeout)")
+ public void recordReadBytes(int len, int timeout) {
+ accumulateBytes(len);
+ }
+
+ private void accumulateBytes(int count) {
+ try {
+ total.add(count);
+ } catch (Throwable t) {
+ //Catch every Throwable, rather than break the business logic.
+ logger.warn("Error while accumulate inbound bytes.", t);
+ }
+ }
+
+ public static class InboundThroughputCollector extends Collector {
+
+ private InboundThroughputCollector() {}
+
+ @Override
+ public List collect() {
+ List mfs = new ArrayList();
+ CounterMetricFamily bytes = new CounterMetricFamily("canal_net_inbound_bytes",
+ "Total socket inbound bytes of canal server.",
+ total.get());
+ mfs.add(bytes);
+ return mfs;
+ }
+ }
+
+}
diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java
new file mode 100644
index 00000000..5562b8fd
--- /dev/null
+++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java
@@ -0,0 +1,64 @@
+package com.alibaba.otter.canal.prometheus.impl;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring;
+import com.alibaba.otter.canal.meta.CanalMetaManager;
+import com.alibaba.otter.canal.prometheus.CanalInstanceExports;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import io.prometheus.client.Collector;
+import io.prometheus.client.GaugeMetricFamily;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author Chuanyi Li
+ */
+public class InstanceMetaCollector extends Collector {
+
+ private static final List InfoLabel = Arrays.asList("destination", "mode");
+
+ private CanalMetaManager metaManager;
+
+ private final String destination;
+
+ private final String mode;
+
+ private final String subsHelp;
+
+ public InstanceMetaCollector(CanalInstance instance) {
+ if (instance == null) {
+ throw new IllegalArgumentException("CanalInstance must not be null.");
+ }
+ if (instance instanceof CanalInstanceWithSpring) {
+ mode = "spring";
+ } else {
+ mode = "manager";
+ }
+ this.metaManager = instance.getMetaManager();
+ this.destination = instance.getDestination();
+ this.subsHelp = "Subscriptions of canal instance " + destination;
+ }
+
+ @Override
+ public List collect() {
+ List mfs = new ArrayList();
+ GaugeMetricFamily instanceInfo = new GaugeMetricFamily(
+ "canal_instance",
+ "Canal instance",
+ InfoLabel);
+ instanceInfo.addMetric(Arrays.asList(destination, mode), 1);
+ mfs.add(instanceInfo);
+ if (metaManager.isStart()) {
+ // client id = hardcode 1001, 目前没有意义
+ List subs = metaManager.listAllSubscribeInfo(destination);
+ GaugeMetricFamily subscriptions = new GaugeMetricFamily(
+ "canal_instance_subscription",
+ subsHelp, CanalInstanceExports.labelList);
+ subscriptions.addMetric(Arrays.asList(destination), subs.size());
+ mfs.add(subscriptions);
+ }
+ return mfs;
+ }
+}
diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java
new file mode 100644
index 00000000..79fdca4a
--- /dev/null
+++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java
@@ -0,0 +1,75 @@
+package com.alibaba.otter.canal.prometheus.impl;
+
+import com.alibaba.otter.canal.prometheus.CanalInstanceExports;
+import com.alibaba.otter.canal.store.CanalEventStore;
+import com.alibaba.otter.canal.store.CanalStoreException;
+import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer;
+import io.prometheus.client.Collector;
+import io.prometheus.client.CounterMetricFamily;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * @author Chuanyi Li
+ */
+public class MemoryStoreCollector extends Collector {
+
+ private static final Class clazz = MemoryEventStoreWithBuffer.class;
+
+ private final String destination;
+
+ private final AtomicLong putSequence;
+
+ private final AtomicLong ackSequence;
+
+ private final String putHelp;
+
+ private final String ackHelp;
+
+ public MemoryStoreCollector(CanalEventStore store, String destination) {
+ this.destination = destination;
+ if (!(store instanceof MemoryEventStoreWithBuffer)) {
+ throw new IllegalArgumentException("EventStore must be MemoryEventStoreWithBuffer");
+ }
+ MemoryEventStoreWithBuffer ms = (MemoryEventStoreWithBuffer) store;
+ putSequence = getDeclaredValue(ms, "putSequence");
+ ackSequence = getDeclaredValue(ms, "ackSequence");
+ putHelp = "Produced sequence of canal instance " + destination;
+ ackHelp = "Consumed sequence of canal instance " + destination;
+ }
+
+ @Override
+ public List collect() {
+ List mfs = new ArrayList();
+ CounterMetricFamily put = new CounterMetricFamily("canal_instance_store_produce_seq",
+ putHelp, Arrays.asList(CanalInstanceExports.labels));
+ put.addMetric(Collections.singletonList(destination), putSequence.doubleValue());
+ mfs.add(put);
+ CounterMetricFamily ack = new CounterMetricFamily("canal_instance_store_consume_seq",
+ ackHelp, Arrays.asList(CanalInstanceExports.labels));
+ ack.addMetric(Collections.singletonList(destination), ackSequence.doubleValue());
+ mfs.add(ack);
+ return mfs;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static T getDeclaredValue(MemoryEventStoreWithBuffer store, String name) {
+ T value;
+ try {
+ Field putField = clazz.getDeclaredField(name);
+ putField.setAccessible(true);
+ value = (T) putField.get(store);
+ } catch (NoSuchFieldException e) {
+ throw new CanalStoreException(e);
+ } catch (IllegalAccessException e) {
+ throw new CanalStoreException(e);
+ }
+ return value;
+ }
+
+}
diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java
new file mode 100644
index 00000000..60dd808d
--- /dev/null
+++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java
@@ -0,0 +1,80 @@
+package com.alibaba.otter.canal.prometheus.impl;
+
+import io.prometheus.client.Collector;
+import io.prometheus.client.CounterMetricFamily;
+import org.aspectj.lang.annotation.After;
+import org.aspectj.lang.annotation.Aspect;
+import org.aspectj.lang.annotation.Pointcut;
+import org.jboss.netty.channel.Channel;
+import org.jctools.maps.ConcurrentAutoTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.alibaba.otter.canal.server.netty.NettyUtils.HEADER_LENGTH;
+
+/**
+ * @author Chuanyi Li
+ */
+@Aspect
+public class OutboundThroughputAspect {
+ private static final Logger logger = LoggerFactory.getLogger(OutboundThroughputAspect.class);
+
+ /**
+ * Support highly scalable counters
+ * @see ConcurrentAutoTable
+ */
+ private static final ConcurrentAutoTable total = new ConcurrentAutoTable();
+
+ private static final Collector collector = new OutboundThroughputCollector();
+
+ public static Collector getCollector() {
+ return collector;
+ }
+
+ @Pointcut("call(* com.alibaba.otter.canal.server.netty.NettyUtils.write(..))")
+ public void write() {}
+
+ //nested read, just eliminate them.
+ @Pointcut("withincode(* com.alibaba.otter.canal.server.netty.NettyUtils.write(..))")
+ public void nestedCall() {}
+
+ @After("write() && !nestedCall() && args(ch, bytes, ..)")
+ public void recordWriteBytes(Channel ch, byte[] bytes) {
+ if (bytes != null) {
+ accumulateBytes(HEADER_LENGTH + bytes.length);
+ }
+ }
+
+ @After("write() && !nestedCall() && args(ch, buf, ..)")
+ public void recordWriteBuffer(Channel ch, ByteBuffer buf) {
+ if (buf != null) {
+ total.add(HEADER_LENGTH + buf.limit());
+ }
+ }
+ private void accumulateBytes(int count) {
+ try {
+ total.add(count);
+ } catch (Throwable t) {
+ //Catch every Throwable, rather than break the business logic.
+ logger.warn("Error while accumulate inbound bytes.", t);
+ }
+ }
+
+ public static class OutboundThroughputCollector extends Collector {
+
+ private OutboundThroughputCollector() {}
+
+ @Override public List collect() {
+ List mfs = new ArrayList();
+ CounterMetricFamily bytes = new CounterMetricFamily("canal_net_outbound_bytes",
+ "Total socket outbound bytes of canal server.",
+ total.get());
+ mfs.add(bytes);
+ return mfs;
+ }
+ }
+}
diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java
new file mode 100644
index 00000000..c55a7eac
--- /dev/null
+++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java
@@ -0,0 +1,73 @@
+package com.alibaba.otter.canal.prometheus.impl;
+
+import com.alibaba.otter.canal.prometheus.CanalInstanceExports;
+import com.alibaba.otter.canal.sink.AbstractCanalEventDownStreamHandler;
+import com.alibaba.otter.canal.store.model.Event;
+import io.prometheus.client.Collector;
+import io.prometheus.client.GaugeMetricFamily;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * @author Chuanyi Li
+ */
+public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDownStreamHandler> {
+
+ private final Collector collector;
+
+ private long latestExecuteTime = 0L;
+
+ private static final String DELAY_NAME = "canal_instance_traffic_delay";
+
+ private final String delayHelpName;
+
+ private final List labelValues;
+
+ public PrometheusCanalEventDownStreamHandler(final String destination) {
+ this.delayHelpName = "Traffic delay of canal instance " + destination + " in seconds.";
+ this.labelValues = Collections.singletonList(destination);
+ collector = new Collector() {
+ @Override
+ public List collect() {
+ List mfs = new ArrayList();
+ long now = System.currentTimeMillis();
+ GaugeMetricFamily delay = new GaugeMetricFamily(
+ DELAY_NAME,
+ delayHelpName,
+ CanalInstanceExports.labelList);
+ double d = 0.0;
+ if (latestExecuteTime > 0) {
+ d = now - latestExecuteTime;
+ }
+ d = d > 0.0 ? (d / 1000) : 0.0;
+ delay.addMetric(labelValues, d);
+ mfs.add(delay);
+ return mfs;
+ }
+ };
+ }
+
+ @Override
+ public List before(List events) {
+ // TODO utilize MySQL master heartbeat packet to refresh delay if always no more events coming
+ // see: https://dev.mysql.com/worklog/task/?id=342
+ // heartbeats are sent by the master only if there is no
+ // more unsent events in the actual binlog file for a period longer that
+ // master_heartbeat_period.
+ if (events != null && !events.isEmpty()) {
+ Event last = events.get(events.size() - 1);
+ long ts = last.getExecuteTime();
+ if (ts > latestExecuteTime) {
+ latestExecuteTime = ts;
+ }
+ }
+ return events;
+ }
+
+ public Collector getCollector() {
+ return this.collector;
+ }
+
+}
diff --git a/prometheus/src/main/resources/META-INF/aop.xml b/prometheus/src/main/resources/META-INF/aop.xml
new file mode 100644
index 00000000..8bd31d0c
--- /dev/null
+++ b/prometheus/src/main/resources/META-INF/aop.xml
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+
diff --git a/prometheus/src/main/resources/META-INF/services/com.alibaba.otter.canal.spi.CanalMetricsProvider b/prometheus/src/main/resources/META-INF/services/com.alibaba.otter.canal.spi.CanalMetricsProvider
new file mode 100644
index 00000000..9adbf8d1
--- /dev/null
+++ b/prometheus/src/main/resources/META-INF/services/com.alibaba.otter.canal.spi.CanalMetricsProvider
@@ -0,0 +1 @@
+com.alibaba.otter.canal.prometheus.PrometheusProvider
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java b/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java
index 722bb2bc..ccf21020 100644
--- a/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java
+++ b/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java
@@ -1,11 +1,11 @@
package com.alibaba.otter.canal.server.embedded;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.concurrent.TimeUnit;
+import com.alibaba.otter.canal.spi.CanalMetricsProvider;
+import com.alibaba.otter.canal.spi.CanalMetricsService;
+import com.alibaba.otter.canal.spi.NopCanalMetricsService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
@@ -33,17 +33,18 @@ import com.google.protobuf.ByteString;
/**
* 嵌入式版本实现
- *
+ *
* @author jianghang 2012-7-12 下午01:34:00
* @author zebin.xuzb
* @version 1.0.0
*/
public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements CanalServer, CanalService {
- private static final Logger logger = LoggerFactory.getLogger(CanalServerWithEmbedded.class);
+ private static final Logger logger = LoggerFactory.getLogger(CanalServerWithEmbedded.class);
private Map canalInstances;
// private Map lastRollbackPostions;
private CanalInstanceGenerator canalInstanceGenerator;
+ private CanalMetricsService metrics = NopCanalMetricsService.NOP;
private static class SingletonHolder {
@@ -61,7 +62,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
public void start() {
if (!isStart()) {
super.start();
-
+ // 如果存在provider,则启动metrics service
+ loadCanalMetrics();
+ metrics.initialize();
canalInstances = MigrateMap.makeComputingMap(new Function() {
public CanalInstance apply(String destination) {
@@ -92,6 +95,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
logger.error(String.format("stop CanalInstance[%s] has an error", entry.getKey()), e);
}
}
+ metrics.terminate();
}
public void start(final String destination) {
@@ -99,6 +103,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
if (!canalInstance.isStart()) {
try {
MDC.put("destination", destination);
+ if (metrics.isRunning()) {
+ metrics.register(canalInstance);
+ }
canalInstance.start();
logger.info("start CanalInstances[{}] successfully", destination);
} finally {
@@ -114,6 +121,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
try {
MDC.put("destination", destination);
canalInstance.stop();
+ if (metrics.isRunning()) {
+ metrics.unregister(canalInstance);
+ }
logger.info("stop CanalInstances[{}] successfully", destination);
} finally {
MDC.remove("destination");
@@ -176,7 +186,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
/**
* 获取数据
- *
+ *
*
* 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
*
@@ -188,14 +198,14 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
/**
* 获取数据，可以指定超时时间.
- *
+ *
*
* 几种case:
* a. 如果timeout为null，则采用tryGet方式，即时获取
* b. 如果timeout不为null
* 1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
* 2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
- *
+ *
* 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
*
*/
@@ -251,7 +261,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
/**
* 不指定 position 获取事件。canal 会记住此 client 最新的 position。
* 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
- *
+ *
*
* 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
*
@@ -264,14 +274,14 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
/**
* 不指定 position 获取事件。canal 会记住此 client 最新的 position。
* 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
- *
+ *
*
* 几种case:
* a. 如果timeout为null，则采用tryGet方式，即时获取
* b. 如果timeout不为null
* 1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
* 2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
- *
+ *
* 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
*
*/
@@ -342,7 +352,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
/**
* 进行 batch id 的确认。确认之后，小于等于此 batchId 的 Message 都会被确认。
- *
+ *
*
* 注意：进行反馈时必须按照batchId的顺序进行ack(需有客户端保证)
*
@@ -492,6 +502,27 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
}
}
+ private void loadCanalMetrics() {
+ ServiceLoader providers = ServiceLoader.load(CanalMetricsProvider.class);
+ List list = new ArrayList();
+ for (CanalMetricsProvider provider : providers) {
+ list.add(provider);
+ }
+ if (!list.isEmpty()) {
+ // 发现provider, 进行初始化
+ if (list.size() > 1) {
+ logger.warn("Found more than one CanalMetricsProvider, use the first one.");
+ //报告冲突
+ for (CanalMetricsProvider p : list) {
+ logger.warn("Found CanalMetricsProvider: {}.", p.getClass().getName());
+ }
+ }
+ //默认使用第一个
+ CanalMetricsProvider provider = list.get(0);
+ this.metrics = provider.getService();
+ }
+ }
+
// ========= setter ==========
public void setCanalInstanceGenerator(CanalInstanceGenerator canalInstanceGenerator) {
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java
index 531c817f..20532400 100644
--- a/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java
+++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java
@@ -2,8 +2,12 @@ package com.alibaba.otter.canal.server.netty;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.List;
+import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.buffer.CompositeChannelBuffer;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.Channels;
@@ -19,9 +23,23 @@ import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
public class NettyUtils {
private static final Logger logger = LoggerFactory.getLogger(NettyUtils.class);
- private static int HEADER_LENGTH = 4;
+ public static int HEADER_LENGTH = 4;
public static Timer hashedWheelTimer = new HashedWheelTimer();
+ public static void write(Channel channel, ByteBuffer body, ChannelFutureListener channelFutureListner) {
+ byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.limit()).array();
+ List components = new ArrayList(2);
+ components.add(ChannelBuffers.wrappedBuffer(ByteOrder.BIG_ENDIAN, header));
+ components.add(ChannelBuffers.wrappedBuffer(body));
+
+ if (channelFutureListner == null) {
+ Channels.write(channel, new CompositeChannelBuffer(ByteOrder.BIG_ENDIAN, components));
+ } else {
+ Channels.write(channel, new CompositeChannelBuffer(ByteOrder.BIG_ENDIAN, components))
+ .addListener(channelFutureListner);
+ }
+ }
+
public static void write(Channel channel, byte[] body, ChannelFutureListener channelFutureListner) {
byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.length).array();
if (channelFutureListner == null) {
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java
index 8f7b4278..11bba043 100644
--- a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java
+++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java
@@ -33,13 +33,10 @@ import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import com.alibaba.otter.canal.server.netty.NettyUtils;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.WireFormat;
-/**
- * 处理具体的客户端请求
- *
- * @author jianghang 2012-10-24 下午02:21:13
- * @version 1.0.0
- */
public class SessionHandler extends SimpleChannelHandler {
private static final Logger logger = LoggerFactory.getLogger(SessionHandler.class);
@@ -76,7 +73,7 @@ public class SessionHandler extends SimpleChannelHandler {
}
embeddedServer.subscribe(clientIdentity);
- ctx.setAttachment(clientIdentity);// 设置状态数据
+ // ctx.setAttachment(clientIdentity);// 设置状态数据
NettyUtils.ack(ctx.getChannel(), null);
} else {
NettyUtils.error(401,
@@ -130,27 +127,74 @@ public class SessionHandler extends SimpleChannelHandler {
}
// }
- Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
- packetBuilder.setType(PacketType.MESSAGES);
+ if (message.getId() != -1 && message.isRaw()) {
+ List rowEntries = message.getRawEntries();
+ // message size
+ int messageSize = 0;
+ messageSize += com.google.protobuf.CodedOutputStream.computeInt64Size(1, message.getId());
- Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
- messageBuilder.setBatchId(message.getId());
- if (message.getId() != -1) {
- if (message.isRaw()) {
- // for performance
- if (!CollectionUtils.isEmpty(message.getRawEntries())) {
+ int dataSize = 0;
+ for (int i = 0; i < rowEntries.size(); i++) {
+ dataSize += com.google.protobuf.CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i));
+ }
+ messageSize += dataSize;
+ messageSize += 1 * rowEntries.size();
+ // packet size
+ int size = 0;
+ size += com.google.protobuf.CodedOutputStream.computeEnumSize(3,
+ PacketType.MESSAGES.getNumber());
+ size += com.google.protobuf.CodedOutputStream.computeTagSize(5)
+ + com.google.protobuf.CodedOutputStream.computeRawVarint32Size(messageSize)
+ + messageSize;
+ // recyle bytes
+ // ByteBuffer byteBuffer = (ByteBuffer)
+ // ctx.getAttachment();
+ // if (byteBuffer != null && size <=
+ // byteBuffer.capacity()) {
+ // byteBuffer.clear();
+ // } else {
+ // byteBuffer =
+ // ByteBuffer.allocate(size).order(ByteOrder.BIG_ENDIAN);
+ // ctx.setAttachment(byteBuffer);
+ // }
+ // CodedOutputStream output =
+ // CodedOutputStream.newInstance(byteBuffer);
+ byte[] body = new byte[size];
+ CodedOutputStream output = CodedOutputStream.newInstance(body);
+ output.writeEnum(3, PacketType.MESSAGES.getNumber());
+
+ output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+ output.writeRawVarint32(messageSize);
+ // message
+ output.writeInt64(1, message.getId());
+ for (int i = 0; i < rowEntries.size(); i++) {
+ output.writeBytes(2, rowEntries.get(i));
+ }
+ output.checkNoSpaceLeft();
+ NettyUtils.write(ctx.getChannel(), body, null);
+
+ // output.flush();
+ // byteBuffer.flip();
+ // NettyUtils.write(ctx.getChannel(), byteBuffer,
+ // null);
+ } else {
+ Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
+ packetBuilder.setType(PacketType.MESSAGES);
+
+ Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
+ messageBuilder.setBatchId(message.getId());
+ if (message.getId() != -1) {
+ if (message.isRaw() && !CollectionUtils.isEmpty(message.getRawEntries())) {
messageBuilder.addAllMessages(message.getRawEntries());
- }
- } else {
- if (!CollectionUtils.isEmpty(message.getEntries())) {
+ } else if (!CollectionUtils.isEmpty(message.getEntries())) {
for (Entry entry : message.getEntries()) {
messageBuilder.addMessages(entry.toByteString());
}
}
}
+ packetBuilder.setBody(messageBuilder.build().toByteString());
+ NettyUtils.write(ctx.getChannel(), packetBuilder.build().toByteArray(), null);// 输出数据
}
- packetBuilder.setBody(messageBuilder.build().toByteString());
- NettyUtils.write(ctx.getChannel(), packetBuilder.build().toByteArray(), null);// 输出数据
} else {
NettyUtils.error(401,
MessageFormatter.format("destination or clientId is null", get.toString()).getMessage(),
diff --git a/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsProvider.java b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsProvider.java
new file mode 100644
index 00000000..ad0ceddf
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsProvider.java
@@ -0,0 +1,24 @@
+package com.alibaba.otter.canal.spi;
+
+/**
+ * Use java service provider mechanism to provide {@link CanalMetricsService}.
+ *
+ * Example:
+ * {@code
+ * ServiceLoader providers = ServiceLoader.load(CanalMetricsProvider.class);
+ * List list = new ArrayList();
+ * for (CanalMetricsProvider provider : providers) {
+ * list.add(provider);
+ * }
+ * }
+ *
+ * @author Chuanyi Li
+ */
+public interface CanalMetricsProvider {
+
+ /**
+ * @return Impl of {@link CanalMetricsService}
+ */
+ CanalMetricsService getService();
+
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsService.java b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsService.java
new file mode 100644
index 00000000..6d823be8
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsService.java
@@ -0,0 +1,42 @@
+package com.alibaba.otter.canal.spi;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+
+/**
+ * Canal server/instance metrics for export.
+ *
+ * Designed to be created by service provider.
+ *
+ * @see CanalMetricsProvider
+ * @author Chuanyi Li
+ */
+public interface CanalMetricsService {
+
+ /**
+ * Initialization on canal server startup.
+ */
+ void initialize();
+
+ /**
+ * Clean-up at canal server stop phase.
+ */
+ void terminate();
+
+ /**
+ * @return {@code true} if the metrics service is running, otherwise {@code false}.
+ */
+ boolean isRunning();
+
+ /**
+ * Register instance level metrics for specified instance.
+ * @param instance {@link CanalInstance}
+ */
+ void register(CanalInstance instance);
+
+ /**
+ * Unregister instance level metrics for specified instance.
+ * @param instance {@link CanalInstance}
+ */
+ void unregister(CanalInstance instance);
+
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/spi/NopCanalMetricsService.java b/server/src/main/java/com/alibaba/otter/canal/spi/NopCanalMetricsService.java
new file mode 100644
index 00000000..0d6ad088
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/spi/NopCanalMetricsService.java
@@ -0,0 +1,38 @@
+package com.alibaba.otter.canal.spi;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+
+/**
+ * @author Chuanyi Li
+ */
+public class NopCanalMetricsService implements CanalMetricsService {
+
+ public static final NopCanalMetricsService NOP = new NopCanalMetricsService();
+
+ private NopCanalMetricsService() {}
+
+ @Override
+ public void initialize() {
+
+ }
+
+ @Override
+ public void terminate() {
+
+ }
+
+ @Override
+ public boolean isRunning() {
+ return false;
+ }
+
+ @Override
+ public void register(CanalInstance instance) {
+
+ }
+
+ @Override
+ public void unregister(CanalInstance instance) {
+
+ }
+}
diff --git a/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java
new file mode 100644
index 00000000..e986b6ff
--- /dev/null
+++ b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java
@@ -0,0 +1,53 @@
+package com.alibaba.otter.canal.server;
+
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+
+import com.alibaba.otter.canal.instance.manager.model.Canal;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.*;
+
+public class CanalServerWithEmbedded_FileModeTest extends BaseCanalServerWithEmbededTest {
+
+ protected Canal buildCanal() {
+ Canal canal = new Canal();
+ canal.setId(1L);
+ canal.setName(DESTINATION);
+ canal.setDesc("my standalone server test ");
+
+ CanalParameter parameter = new CanalParameter();
+
+ parameter.setMetaMode(MetaMode.LOCAL_FILE);
+ parameter.setDataDir("./conf");
+ parameter.setMetaFileFlushPeriod(1000);
+ parameter.setHaMode(HAMode.HEARTBEAT);
+ parameter.setIndexMode(IndexMode.MEMORY_META_FAILBACK);
+
+ parameter.setStorageMode(StorageMode.MEMORY);
+ parameter.setMemoryStorageBufferSize(32 * 1024);
+
+ parameter.setSourcingType(SourcingType.MYSQL);
+ parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
+ new InetSocketAddress(MYSQL_ADDRESS, 3306)));
+ parameter.setDbUsername(USERNAME);
+ parameter.setDbPassword(PASSWORD);
+ parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":332L,\"timestamp\":\"1505998863000\"}",
+ "{\"journalName\":\"mysql-bin.000001\",\"position\":332L,\"timestamp\":\"1505998863000\"}"));
+
+ parameter.setSlaveId(1234L);
+
+ parameter.setDefaultConnectionTimeoutInSeconds(30);
+ parameter.setConnectionCharset("UTF-8");
+ parameter.setConnectionCharsetNumber((byte) 33);
+ parameter.setReceiveBufferSize(8 * 1024);
+ parameter.setSendBufferSize(8 * 1024);
+
+ parameter.setDetectingEnable(false);
+ parameter.setDetectingIntervalInSeconds(10);
+ parameter.setDetectingRetryTimes(3);
+ parameter.setDetectingSQL(DETECTING_SQL);
+
+ canal.setCanalParameter(parameter);
+ return canal;
+ }
+}
\ No newline at end of file
diff --git a/server/src/test/java/com/alibaba/otter/canal/server/ProtocolTest.java b/server/src/test/java/com/alibaba/otter/canal/server/ProtocolTest.java
new file mode 100644
index 00000000..cb078ddf
--- /dev/null
+++ b/server/src/test/java/com/alibaba/otter/canal/server/ProtocolTest.java
@@ -0,0 +1,92 @@
+package com.alibaba.otter.canal.server;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.CanalEntry.Header;
+import com.alibaba.otter.canal.protocol.CanalPacket.Compression;
+import com.alibaba.otter.canal.protocol.CanalPacket.Messages;
+import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
+import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.WireFormat;
+
+public class ProtocolTest {
+
+ @Test
+ public void testSimple() throws IOException {
+ Header.Builder headerBuilder = Header.newBuilder();
+ headerBuilder.setLogfileName("mysql-bin.000001");
+ headerBuilder.setLogfileOffset(1024);
+ headerBuilder.setExecuteTime(1024);
+ Entry.Builder entryBuilder = Entry.newBuilder();
+ entryBuilder.setHeader(headerBuilder.build());
+ entryBuilder.setEntryType(EntryType.ROWDATA);
+ Entry entry = entryBuilder.build();
+ Message message = new Message(3, true, Arrays.asList(entry.toByteString()));
+
+ byte[] body = buildData(message);
+ Packet packet = Packet.parseFrom(body);
+ switch (packet.getType()) {
+ case MESSAGES: {
+ if (!packet.getCompression().equals(Compression.NONE)) {
+ throw new CanalClientException("compression is not supported in this connector");
+ }
+
+ Messages messages = Messages.parseFrom(packet.getBody());
+ Message result = new Message(messages.getBatchId());
+ for (ByteString byteString : messages.getMessagesList()) {
+ result.addEntry(Entry.parseFrom(byteString));
+ }
+
+ System.out.println(result);
+ break;
+ }
+ default: {
+ throw new CanalClientException("unexpected packet type: " + packet.getType());
+ }
+ }
+ }
+
+ private byte[] buildData(Message message) throws IOException {
+ List rowEntries = message.getRawEntries();
+ // message size
+ int messageSize = 0;
+ messageSize += com.google.protobuf.CodedOutputStream.computeInt64Size(1, message.getId());
+
+ int dataSize = 0;
+ for (int i = 0; i < rowEntries.size(); i++) {
+ dataSize += com.google.protobuf.CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i));
+ }
+ messageSize += dataSize;
+ messageSize += 1 * rowEntries.size();
+ // packet size
+ int size = 0;
+ size += com.google.protobuf.CodedOutputStream.computeEnumSize(3, PacketType.MESSAGES.getNumber());
+ size += com.google.protobuf.CodedOutputStream.computeTagSize(5)
+ + com.google.protobuf.CodedOutputStream.computeRawVarint32Size(messageSize) + messageSize;
+ // TODO recyle bytes[]
+ byte[] body = new byte[size];
+ CodedOutputStream output = CodedOutputStream.newInstance(body);
+ output.writeEnum(3, PacketType.MESSAGES.getNumber());
+
+ output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+ output.writeRawVarint32(messageSize);
+ // message
+ output.writeInt64(1, message.getId());
+ for (int i = 0; i < rowEntries.size(); i++) {
+ output.writeBytes(2, rowEntries.get(i));
+ }
+ output.checkNoSpaceLeft();
+
+ return body;
+ }
+}
From b53131553ed974544f881791302f53a939ed45f6 Mon Sep 17 00:00:00 2001
From: "charles.lin" <848497337@qq.com>
Date: Thu, 9 Aug 2018 02:01:44 +0800
Subject: [PATCH 2/3] Commit test case
---
.../mysql/RdsBinlogEventParserProxyTest.java | 30 -------------------
1 file changed, 30 deletions(-)
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogEventParserProxyTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogEventParserProxyTest.java
index 18a217b8..c552a787 100644
--- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogEventParserProxyTest.java
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogEventParserProxyTest.java
@@ -58,36 +58,6 @@ public class RdsBinlogEventParserProxyTest {
controller.setSecretkey("");
controller.setBatchSize(4);
// controller.setRdsOpenApiUrl("https://rds.aliyuncs.com/");
- controller.setTableMetaStorageFactory(new MySqlTableMetaStorageFactory(new MySqlTableMetaCallback() {
- @Override
- public void save(String dbAddress, String schema, String table, String ddl, Long timestamp) {
-
- }
-
- @Override
- public List fetch(String dbAddress, String dbName) {
- TableMetaEntry tableMeta = new TableMetaEntry();
- tableMeta.setSchema(DBNAME);
- tableMeta.setTable(TBNAME);
- tableMeta.setDdl(DDL);
- tableMeta.setTimestamp(new Date().getTime());
- List entries = new ArrayList();
- entries.add(tableMeta);
- return entries;
- }
-
- @Override
- public List fetch(String dbAddress, String dbName, String tableName) {
- TableMetaEntry tableMeta = new TableMetaEntry();
- tableMeta.setSchema(DBNAME);
- tableMeta.setTable(TBNAME);
- tableMeta.setDdl(DDL);
- tableMeta.setTimestamp(new Date().getTime());
- List entries = new ArrayList();
- entries.add(tableMeta);
- return entries;
- }
- }, DBNAME));
controller.setEventSink(new AbstractCanalEventSinkTest>() {
@Override
From b57ac0bcdf9f512d8e2fada584f4b0ed97e14390 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E4=B8=83=E9=94=8B?=
Date: Thu, 9 Aug 2018 16:56:13 +0800
Subject: [PATCH 3/3] fixed merge and blob encoding example
---
.../example/AbstractCanalClientTest.java | 15 +-
.../exception/PositionNotFoundException.java | 30 --
.../exception/ServerIdNotMatchException.java | 30 --
.../parse/inbound/AbstractEventParser.java | 30 +-
.../canal/parse/inbound/ErosaConnection.java | 2 -
.../parse/inbound/ParserExceptionHandler.java | 9 -
.../inbound/mysql/LocalBinLogConnection.java | 50 +---
.../parse/inbound/mysql/MysqlConnection.java | 12 -
.../parse/inbound/mysql/MysqlEventParser.java | 8 +-
.../mysql/MysqlMultiStageCoprocessor.java | 11 +-
.../inbound/mysql/dbsync/LogEventConvert.java | 15 +-
.../inbound/mysql/dbsync/TableMetaCache.java | 7 +-
.../mysql/rds/BinlogDownloadQueue.java | 260 ------------------
.../mysql/rds/RdsBinlogEventParserProxy.java | 149 ----------
.../inbound/mysql/rds/RdsBinlogOpenApi.java | 61 +---
.../mysql/rds/RdsLocalBinlogEventParser.java | 159 ++---------
.../inbound/mysql/rds/data/BinlogFile.java | 72 -----
.../rds/data/DescribeBinlogFileResult.java | 62 -----
.../mysql/rds/data/RdsBackupPolicy.java | 69 -----
.../parse/inbound/mysql/rds/data/RdsItem.java | 19 --
.../mysql/rds/request/AbstractRequest.java | 250 -----------------
.../request/DescribeBackupPolicyRequest.java | 41 ---
.../request/DescribeBinlogFilesRequest.java | 56 ----
.../tablemeta/HistoryTableMetaCache.java | 191 -------------
.../tablemeta/TableMetaCacheInterface.java | 20 --
.../tablemeta/TableMetaCacheWithStorage.java | 105 -------
.../mysql/tablemeta/TableMetaEntry.java | 55 ----
.../mysql/tablemeta/TableMetaStorage.java | 18 --
.../tablemeta/TableMetaStorageFactory.java | 9 -
.../exception/CacheConnectionNull.java | 9 -
.../exception/NoHistoryException.java | 21 --
.../impl/mysql/MySqlTableMetaCallback.java | 14 -
.../impl/mysql/MySqlTableMetaStorage.java | 48 ----
.../mysql/MySqlTableMetaStorageFactory.java | 26 --
.../mysql/RdsBinlogEventParserProxyTest.java | 123 ---------
.../mysql/tablemeta/NoStorageTest.java | 38 +++
.../inbound/mysql/tablemeta/StorageTest.java | 77 ++++++
37 files changed, 181 insertions(+), 1990 deletions(-)
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/exception/PositionNotFoundException.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/exception/ServerIdNotMatchException.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ParserExceptionHandler.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/BinlogFile.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/DescribeBinlogFileResult.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsBackupPolicy.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsItem.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBackupPolicyRequest.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBinlogFilesRequest.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheInterface.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheWithStorage.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaEntry.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorage.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorageFactory.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/CacheConnectionNull.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/NoHistoryException.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaCallback.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorage.java
delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorageFactory.java
delete mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogEventParserProxyTest.java
create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/NoStorageTest.java
create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/StorageTest.java
diff --git a/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java b/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java
index d1ddd88e..01e3df71 100644
--- a/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java
+++ b/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java
@@ -1,9 +1,9 @@
package com.alibaba.otter.canal.example;
+import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
-import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.SystemUtils;
@@ -254,7 +254,18 @@ public class AbstractCanalClientTest {
protected void printColumn(List columns) {
for (Column column : columns) {
StringBuilder builder = new StringBuilder();
- builder.append(column.getName() + " : " + column.getValue());
+ try {
+ if (StringUtils.containsIgnoreCase(column.getMysqlType(), "BLOB")
+ || StringUtils.containsIgnoreCase(column.getMysqlType(), "BINARY")) {
+ // get value bytes
+ builder.append(column.getName() + " : "
+ + new String(column.getValue().getBytes("ISO-8859-1"), "UTF-8"));
+ } else {
+ builder.append(column.getName() + " : " + column.getValue());
+ }
+ } catch (UnsupportedEncodingException e) {
+ }
+
builder.append(" type=" + column.getMysqlType());
if (column.getUpdated()) {
builder.append(" update=" + column.getUpdated());
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/exception/PositionNotFoundException.java b/parse/src/main/java/com/alibaba/otter/canal/parse/exception/PositionNotFoundException.java
deleted file mode 100644
index d5552a9f..00000000
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/exception/PositionNotFoundException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package com.alibaba.otter.canal.parse.exception;
-
-/**
- * @author chengjin.lyf on 2018/7/20 下午2:54
- * @since 1.0.25
- */
-public class PositionNotFoundException extends CanalParseException {
-
- private static final long serialVersionUID = -7382448928116244017L;
-
- public PositionNotFoundException(String errorCode) {
- super(errorCode);
- }
-
- public PositionNotFoundException(String errorCode, Throwable cause) {
- super(errorCode, cause);
- }
-
- public PositionNotFoundException(String errorCode, String errorDesc) {
- super(errorCode, errorDesc);
- }
-
- public PositionNotFoundException(String errorCode, String errorDesc, Throwable cause) {
- super(errorCode, errorDesc, cause);
- }
-
- public PositionNotFoundException(Throwable cause) {
- super(cause);
- }
-}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/exception/ServerIdNotMatchException.java b/parse/src/main/java/com/alibaba/otter/canal/parse/exception/ServerIdNotMatchException.java
deleted file mode 100644
index 35a6b790..00000000
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/exception/ServerIdNotMatchException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package com.alibaba.otter.canal.parse.exception;
-
-import com.alibaba.otter.canal.common.CanalException;
-
-/**
- * @author chengjin.lyf on 2018/8/8 下午1:07
- * @since 1.0.25
- */
-public class ServerIdNotMatchException extends CanalException{
-
- public ServerIdNotMatchException(String errorCode) {
- super(errorCode);
- }
-
- public ServerIdNotMatchException(String errorCode, Throwable cause) {
- super(errorCode, cause);
- }
-
- public ServerIdNotMatchException(String errorCode, String errorDesc) {
- super(errorCode, errorDesc);
- }
-
- public ServerIdNotMatchException(String errorCode, String errorDesc, Throwable cause) {
- super(errorCode, errorDesc, cause);
- }
-
- public ServerIdNotMatchException(Throwable cause) {
- super(cause);
- }
-}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
index 0034ba2b..7f7d39ac 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
@@ -8,7 +8,6 @@ import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
-import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang.math.RandomUtils;
@@ -95,10 +94,6 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
.availableProcessors() * 60 / 100; // 60%的能力跑解析,剩余部分处理网络
protected int parallelBufferSize = 256; // 必须为2的幂
protected MultiStageCoprocessor multiStageCoprocessor;
- protected ParserExceptionHandler parserExceptionHandler;
- protected long serverId;
-
-
protected abstract BinlogParser buildParser();
@@ -175,16 +170,11 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
preDump(erosaConnection);
erosaConnection.connect();// 链接
-
- long queryServerId = erosaConnection.queryServerId();
- if (queryServerId != 0){
- serverId = queryServerId;
- }
// 4. 获取最后的位置信息
EntryPosition position = findStartPosition(erosaConnection);
final EntryPosition startPosition = position;
if (startPosition == null) {
- throw new PositionNotFoundException("can't find start position for " + destination);
+ throw new CanalParseException("can't find start position for " + destination);
}
if (!processTableMeta(startPosition)) {
@@ -287,9 +277,6 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
runningInfo.getAddress().toString()), e);
sendAlarm(destination, ExceptionUtils.getFullStackTrace(e));
}
- if (parserExceptionHandler!=null){
- parserExceptionHandler.handle(e);
- }
} finally {
// 重新置为中断状态
Thread.interrupted();
@@ -628,19 +615,4 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle
this.parallelBufferSize = parallelBufferSize;
}
- public ParserExceptionHandler getParserExceptionHandler() {
- return parserExceptionHandler;
- }
-
- public void setParserExceptionHandler(ParserExceptionHandler parserExceptionHandler) {
- this.parserExceptionHandler = parserExceptionHandler;
- }
-
- public long getServerId() {
- return serverId;
- }
-
- public void setServerId(long serverId) {
- this.serverId = serverId;
- }
}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java
index 5355e2e1..6beb4a9e 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java
@@ -40,6 +40,4 @@ public interface ErosaConnection {
public void dump(GTIDSet gtidSet, MultiStageCoprocessor coprocessor) throws IOException;
ErosaConnection fork();
-
- public long queryServerId() throws IOException;
}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ParserExceptionHandler.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ParserExceptionHandler.java
deleted file mode 100644
index d0face01..00000000
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ParserExceptionHandler.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound;
-
-/**
- * @author chengjin.lyf on 2018/7/20 下午3:55
- * @since 1.0.25
- */
-public interface ParserExceptionHandler {
- void handle(Throwable e);
-}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
index 0358dfda..6c59465b 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
@@ -4,7 +4,6 @@ import java.io.File;
import java.io.IOException;
import java.util.List;
-import com.alibaba.otter.canal.parse.exception.ServerIdNotMatchException;
import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@@ -37,9 +36,6 @@ public class LocalBinLogConnection implements ErosaConnection {
private String directory;
private int bufferSize = 16 * 1024;
private boolean running = false;
- private long serverId;
- private FileParserListener parserListener;
-
public LocalBinLogConnection(){
}
@@ -100,9 +96,6 @@ public class LocalBinLogConnection implements ErosaConnection {
if (event == null) {
continue;
}
- if (serverId != 0 && event.getServerId() != serverId){
- throw new ServerIdNotMatchException("unexpected serverId "+serverId + " in binlog file !");
- }
if (!func.sink(event)) {
needContinue = false;
@@ -110,9 +103,8 @@ public class LocalBinLogConnection implements ErosaConnection {
}
}
- fetcher.close(); // 关闭上一个文件
- parserFinish(current.getName());
if (needContinue) {// 读取下一个
+ fetcher.close(); // 关闭上一个文件
File nextFile;
if (needWait) {
@@ -168,11 +160,6 @@ public class LocalBinLogConnection implements ErosaConnection {
while (fetcher.fetch()) {
LogEvent event = decoder.decode(fetcher, context);
if (event != null) {
-
- if (serverId != 0 && event.getServerId() != serverId){
- throw new ServerIdNotMatchException("unexpected serverId "+serverId + " in binlog file !");
- }
-
if (event.getWhen() > timestampSeconds) {
break;
}
@@ -241,9 +228,8 @@ public class LocalBinLogConnection implements ErosaConnection {
}
}
- fetcher.close(); // 关闭上一个文件
- parserFinish(binlogfilename);
if (needContinue) {// 读取下一个
+ fetcher.close(); // 关闭上一个文件
File nextFile;
if (needWait) {
@@ -272,12 +258,6 @@ public class LocalBinLogConnection implements ErosaConnection {
}
}
- private void parserFinish(String fileName){
- if (parserListener != null){
- parserListener.onFinish(fileName);
- }
- }
-
@Override
public void dump(long timestampMills, MultiStageCoprocessor coprocessor) throws IOException {
List currentBinlogs = binlogs.currentBinlogs();
@@ -306,11 +286,6 @@ public class LocalBinLogConnection implements ErosaConnection {
while (fetcher.fetch()) {
LogEvent event = decoder.decode(fetcher, context);
if (event != null) {
-
- if (serverId != 0 && event.getServerId() != serverId){
- throw new ServerIdNotMatchException("unexpected serverId "+serverId + " in binlog file !");
- }
-
if (event.getWhen() > timestampSeconds) {
break;
}
@@ -369,11 +344,6 @@ public class LocalBinLogConnection implements ErosaConnection {
return connection;
}
- @Override
- public long queryServerId() {
- return 0;
- }
-
public boolean isNeedWait() {
return needWait;
}
@@ -398,20 +368,4 @@ public class LocalBinLogConnection implements ErosaConnection {
this.bufferSize = bufferSize;
}
- public long getServerId() {
- return serverId;
- }
-
- public void setServerId(long serverId) {
- this.serverId = serverId;
- }
-
- public void setParserListener(FileParserListener parserListener) {
- this.parserListener = parserListener;
- }
-
- public interface FileParserListener{
- void onFinish(String fileName);
- }
-
}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
index 5f681a54..d01a210e 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
@@ -9,10 +9,8 @@ import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.math.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlQueryExecutor;
@@ -326,16 +324,6 @@ public class MysqlConnection implements ErosaConnection {
return connection;
}
- @Override
- public long queryServerId() throws IOException {
- ResultSetPacket resultSetPacket = query("show variables like 'server_id'");
- List fieldValues = resultSetPacket.getFieldValues();
- if (fieldValues == null || fieldValues.size() != 2){
- return 0;
- }
- return NumberUtils.toLong(fieldValues.get(1));
- }
-
// ====================== help method ====================
/**
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
index 8faaaa61..53e4e5e5 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
@@ -53,11 +53,11 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
private int receiveBufferSize = 64 * 1024;
private int sendBufferSize = 64 * 1024;
// 数据库信息
- protected AuthenticationInfo masterInfo; // 主库
- protected AuthenticationInfo standbyInfo; // 备库
+ private AuthenticationInfo masterInfo; // 主库
+ private AuthenticationInfo standbyInfo; // 备库
// binlog信息
- protected EntryPosition masterPosition;
- protected EntryPosition standbyPosition;
+ private EntryPosition masterPosition;
+ private EntryPosition standbyPosition;
private long slaveId; // 链接到mysql的slave
// 心跳检查信息
private String detectingSQL; // 心跳sql
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
index e93a1d20..281a05f2 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
@@ -203,15 +203,10 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
try {
LogBuffer buffer = event.getBuffer();
if (StringUtils.isNotEmpty(event.getBinlogFileName())
- && (context.getLogPosition() == null
- || !context.getLogPosition().getFileName().equals(event.getBinlogFileName()))) {
+ && !context.getLogPosition().getFileName().equals(event.getBinlogFileName())) {
// set roate binlog file name
- if (context.getLogPosition() == null){
- context.setLogPosition(new LogPosition(event.getBinlogFileName(), 0));
- }else{
- context.setLogPosition(new LogPosition(event.getBinlogFileName(), context.getLogPosition()
- .getPosition()));
- }
+ context.setLogPosition(new LogPosition(event.getBinlogFileName(), context.getLogPosition()
+ .getPosition()));
}
LogEvent logEvent = decoder.decode(buffer, context);
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
index f25e80f4..2f9b5304 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
@@ -10,9 +10,6 @@ import java.util.Arrays;
import java.util.BitSet;
import java.util.List;
-import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheInterface;
-import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorage;
-import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception.NoHistoryException;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.slf4j.Logger;
@@ -90,10 +87,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
private volatile AviaterRegexFilter nameFilter; // 运行时引用可能会有变化，比如规则发生变化时
private volatile AviaterRegexFilter nameBlackFilter;
-
- private TableMetaCacheInterface tableMetaCache;
- private String binlogFileName = "mysql-bin.000001";
-
+ private TableMetaCache tableMetaCache;
private Charset charset = Charset.defaultCharset();
private boolean filterQueryDcl = false;
private boolean filterQueryDml = false;
@@ -268,8 +262,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
if (!isSeek) {
// 使用新的表结构元数据管理方式
EntryPosition position = createPosition(event.getHeader());
- String fulltbName = schemaName+"."+tableName;
- tableMetaCache.apply(position, fulltbName, queryString, null);
+ tableMetaCache.apply(position, event.getDbName(), queryString, null);
}
Header header = createHeader(event.getHeader(), schemaName, tableName, type);
@@ -766,6 +759,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
} else {
// byte数组，直接使用iso-8859-1保留对应编码，浪费内存
columnBuilder.setValue(new String((byte[]) value, ISO_8859_1));
+ // columnBuilder.setValueBytes(ByteString.copyFrom((byte[])
+ // value));
javaType = Types.BLOB;
}
break;
@@ -944,7 +939,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
this.nameBlackFilter = nameBlackFilter;
}
- public void setTableMetaCache(TableMetaCacheInterface tableMetaCache) {
+ public void setTableMetaCache(TableMetaCache tableMetaCache) {
this.tableMetaCache = tableMetaCache;
}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
index 5c782aa1..4c1cef3c 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
@@ -6,7 +6,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheInterface;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
@@ -30,7 +29,7 @@ import com.google.common.cache.LoadingCache;
* @author jianghang 2013-1-17 下午10:15:16
* @version 1.0.0
*/
-public class TableMetaCache implements TableMetaCacheInterface {
+public class TableMetaCache {
public static final String COLUMN_NAME = "COLUMN_NAME";
public static final String COLUMN_TYPE = "COLUMN_TYPE";
@@ -100,10 +99,6 @@ public class TableMetaCache implements TableMetaCacheInterface {
String createDDL = packet.getFieldValues().get(1);
MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
memoryTableMeta.apply(DatabaseTableMeta.INIT_POSITION, schema, createDDL, null);
- String[] strings = table.split("\\.");
- if (strings.length > 1) {
- table = strings[1];
- }
TableMeta tableMeta = memoryTableMeta.find(schema, table);
return tableMeta.getFields();
} else {
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java
deleted file mode 100644
index d95c8af1..00000000
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java
+++ /dev/null
@@ -1,260 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
-
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午3:10
- * @since 1.0.25
- */
-public class BinlogDownloadQueue {
-
- private static final Logger logger = LoggerFactory.getLogger(BinlogDownloadQueue.class);
- private static final int TIMEOUT = 10000;
-
- private LinkedBlockingQueue downloadQueue = new LinkedBlockingQueue();
- private LinkedBlockingQueue taskQueue = new LinkedBlockingQueue();
- private LinkedList binlogList;
- private final int batchSize;
- private Thread downloadThread;
- public boolean running = true;
- private final String destDir;
- private String hostId;
- private int currentSize;
- private String lastDownload;
-
- public BinlogDownloadQueue(List downloadQueue, int batchSize, String destDir) throws IOException {
- this.binlogList = new LinkedList(downloadQueue);
- this.batchSize = batchSize;
- this.destDir = destDir;
- this.currentSize = 0;
- prepareBinlogList();
- cleanDir();
- }
-
- private void prepareBinlogList(){
- for (BinlogFile binlog : this.binlogList) {
- String fileName = StringUtils.substringBetween(binlog.getDownloadLink(), "mysql-bin.", "?");
- binlog.setFileName(fileName);
- }
- Collections.sort(this.binlogList, new Comparator() {
- @Override
- public int compare(BinlogFile o1, BinlogFile o2) {
- return o1.getFileName().compareTo(o2.getFileName());
- }
- });
- }
-
- public void cleanDir() throws IOException {
- File destDirFile = new File(destDir);
- FileUtils.forceMkdir(destDirFile);
- FileUtils.cleanDirectory(destDirFile);
- }
-
- public void silenceDownload() {
- if (downloadThread != null) {
- return;
- }
- downloadThread = new Thread(new DownloadThread());
- downloadThread.start();
- }
-
-
- public BinlogFile tryOne() throws IOException {
- BinlogFile binlogFile = binlogList.poll();
- download(binlogFile);
- hostId = binlogFile.getHostInstanceID();
- this.currentSize ++;
- return binlogFile;
- }
-
- public void notifyNotMatch(){
- this.currentSize --;
- filter(hostId);
- }
-
- private void filter(String hostInstanceId){
- Iterator it = binlogList.iterator();
- while (it.hasNext()){
- BinlogFile bf = it.next();
- if(bf.getHostInstanceID().equalsIgnoreCase(hostInstanceId)){
- it.remove();
- }else{
- hostId = bf.getHostInstanceID();
- }
- }
- }
-
- public boolean isLastFile(String fileName){
- String needCompareName = lastDownload;
- if (StringUtils.isNotEmpty(needCompareName) && StringUtils.endsWith(needCompareName, "tar")){
- needCompareName = needCompareName.substring(0, needCompareName.indexOf("."));
- }
- return fileName.equalsIgnoreCase(needCompareName) && binlogList.isEmpty();
- }
-
- public void prepare() throws InterruptedException {
- for (int i = this.currentSize; i < batchSize && !binlogList.isEmpty(); i++) {
- BinlogFile binlogFile = null;
- while (!binlogList.isEmpty()){
- binlogFile = binlogList.poll();
- if (!binlogFile.getHostInstanceID().equalsIgnoreCase(hostId)){
- continue;
- }
- break;
- }
- if (binlogFile == null){
- break;
- }
- this.downloadQueue.put(binlogFile);
- this.lastDownload = "mysql-bin." + binlogFile.getFileName();
- this.currentSize ++;
- }
- }
-
- public void downOne(){
- this.currentSize --;
- }
-
- public void release(){
- running = false;
- this.currentSize = 0;
- binlogList.clear();
- downloadQueue.clear();
- }
-
- private void download(BinlogFile binlogFile) throws IOException {
- String downloadLink = binlogFile.getDownloadLink();
- String fileName = binlogFile.getFileName();
- HttpGet httpGet = new HttpGet(downloadLink);
- CloseableHttpClient httpClient = HttpClientBuilder.create()
- .setMaxConnPerRoute(50)
- .setMaxConnTotal(100)
- .build();
- RequestConfig requestConfig = RequestConfig.custom()
- .setConnectTimeout(TIMEOUT)
- .setConnectionRequestTimeout(TIMEOUT)
- .setSocketTimeout(TIMEOUT)
- .build();
- httpGet.setConfig(requestConfig);
- HttpResponse response = httpClient.execute(httpGet);
- int statusCode = response.getStatusLine().getStatusCode();
- if (statusCode != HttpResponseStatus.OK.code()) {
- throw new RuntimeException("download failed , url:" + downloadLink + " , statusCode:"
- + statusCode);
- }
- saveFile(new File(destDir), "mysql-bin." + fileName, response);
- }
-
- private static void saveFile(File parentFile, String fileName, HttpResponse response) throws IOException {
- InputStream is = response.getEntity().getContent();
- long totalSize = Long.parseLong(response.getFirstHeader("Content-Length").getValue());
- if(response.getFirstHeader("Content-Disposition")!=null){
- fileName = response.getFirstHeader("Content-Disposition").getValue();
- fileName = StringUtils.substringAfter(fileName, "filename=");
- }
- boolean isTar = StringUtils.endsWith(fileName, ".tar");
- FileUtils.forceMkdir(parentFile);
- FileOutputStream fos = null;
- try {
- if (isTar) {
- TarArchiveInputStream tais = new TarArchiveInputStream(is);
- TarArchiveEntry tarArchiveEntry = null;
- while ((tarArchiveEntry = tais.getNextTarEntry()) != null) {
- String name = tarArchiveEntry.getName();
- File tarFile = new File(parentFile, name + ".tmp");
- logger.info("start to download file " + tarFile.getName());
- BufferedOutputStream bos = null;
- try {
- bos = new BufferedOutputStream(new FileOutputStream(tarFile));
- int read = -1;
- byte[] buffer = new byte[1024];
- while ((read = tais.read(buffer)) != -1) {
- bos.write(buffer, 0, read);
- }
- logger.info("download file " + tarFile.getName() + " end!");
- tarFile.renameTo(new File(parentFile, name));
- } finally {
- IOUtils.closeQuietly(bos);
- }
- }
- tais.close();
- } else {
- File file = new File(parentFile, fileName + ".tmp");
- if (!file.isFile()) {
- file.createNewFile();
- }
- try {
- fos = new FileOutputStream(file);
- byte[] buffer = new byte[1024];
- int len;
- long copySize = 0;
- long nextPrintProgress = 0;
- logger.info("start to download file " + file.getName());
- while ((len = is.read(buffer)) != -1) {
- fos.write(buffer, 0, len);
- copySize += len;
- long progress = copySize * 100 / totalSize;
- if (progress >= nextPrintProgress) {
- logger.info("download " + file.getName() + " progress : " + progress
- + "% , download size : " + copySize + ", total size : " + totalSize);
- nextPrintProgress += 10;
- }
- }
- logger.info("download file " + file.getName() + " end!");
- fos.flush();
- } finally {
- IOUtils.closeQuietly(fos);
- }
- file.renameTo(new File(parentFile, fileName));
- }
- } finally {
- IOUtils.closeQuietly(fos);
- }
- }
-
- public void execute(Runnable runnable) throws InterruptedException {
- taskQueue.put(runnable);
- }
-
- private class DownloadThread implements Runnable {
-
- @Override
- public void run() {
- while (running) {
- try {
- BinlogFile binlogFile = downloadQueue.poll(5000, TimeUnit.MILLISECONDS);
- if (binlogFile != null){
- download(binlogFile);
- }
- Runnable runnable = taskQueue.poll(5000, TimeUnit.MILLISECONDS);
- if (runnable != null){
- runnable.run();
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- }
- }
-}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java
deleted file mode 100644
index 5bcc8688..00000000
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java
+++ /dev/null
@@ -1,149 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadFactory;
-
-import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
-import com.alibaba.otter.canal.parse.inbound.ParserExceptionHandler;
-import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
-
-/**
- * @author chengjin.lyf on 2018/7/20 上午10:52
- * @since 1.0.25
- */
-public class RdsBinlogEventParserProxy extends MysqlEventParser {
-
- private String rdsOpenApiUrl = "https://rds.aliyuncs.com/"; // openapi地址
- private String accesskey; // 云账号的ak
- private String secretkey; // 云账号sk
- private String instanceId; // rds实例id
- private Long startTime;
- private Long endTime;
- private String directory; //binlog 目录
- private int batchSize = 4; //最多下载的binlog文件数量
-
- private RdsLocalBinlogEventParser rdsBinlogEventParser = new RdsLocalBinlogEventParser();
- private ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactory() {
-
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new Thread(r, "rds-binlog-daemon-thread");
- t.setDaemon(true);
- return t;
- }
- });
-
- @Override
- public void start() {
- final ParserExceptionHandler targetHandler = this.getParserExceptionHandler();
- rdsBinlogEventParser.setLogPositionManager(this.getLogPositionManager());
- rdsBinlogEventParser.setDestination(destination);
- rdsBinlogEventParser.setAlarmHandler(this.getAlarmHandler());
- rdsBinlogEventParser.setConnectionCharset(this.connectionCharset);
- rdsBinlogEventParser.setConnectionCharsetNumber(this.connectionCharsetNumber);
- rdsBinlogEventParser.setEnableTsdb(this.enableTsdb);
- rdsBinlogEventParser.setEventBlackFilter(this.eventBlackFilter);
- rdsBinlogEventParser.setFilterQueryDcl(this.filterQueryDcl);
- rdsBinlogEventParser.setFilterQueryDdl(this.filterQueryDdl);
- rdsBinlogEventParser.setFilterQueryDml(this.filterQueryDml);
- rdsBinlogEventParser.setFilterRows(this.filterRows);
- rdsBinlogEventParser.setFilterTableError(this.filterTableError);
- rdsBinlogEventParser.setIsGTIDMode(this.isGTIDMode);
- rdsBinlogEventParser.setMasterInfo(this.masterInfo);
- rdsBinlogEventParser.setEventFilter(this.eventFilter);
- rdsBinlogEventParser.setMasterPosition(this.masterPosition);
- rdsBinlogEventParser.setTransactionSize(this.transactionSize);
- rdsBinlogEventParser.setUrl(this.rdsOpenApiUrl);
- rdsBinlogEventParser.setAccesskey(this.accesskey);
- rdsBinlogEventParser.setSecretkey(this.secretkey);
- rdsBinlogEventParser.setInstanceId(this.instanceId);
- rdsBinlogEventParser.setEventSink(eventSink);
- rdsBinlogEventParser.setDirectory(directory);
- rdsBinlogEventParser.setBatchSize(batchSize);
- rdsBinlogEventParser.setFinishListener(new RdsLocalBinlogEventParser.ParseFinishListener() {
- @Override
- public void onFinish() {
- executorService.execute(new Runnable() {
- @Override
- public void run() {
- rdsBinlogEventParser.stop();
- RdsBinlogEventParserProxy.this.start();
- }
- });
-
- }
- });
- this.setParserExceptionHandler(new ParserExceptionHandler() {
-
- @Override
- public void handle(Throwable e) {
- handleMysqlParserException(e);
- if (targetHandler != null) {
- targetHandler.handle(e);
- }
- }
- });
- super.start();
- }
-
- public void handleMysqlParserException(Throwable throwable) {
- if (throwable instanceof PositionNotFoundException) {
- logger.info("remove rds not found position, try download rds binlog!");
- executorService.execute(new Runnable() {
-
- @Override
- public void run() {
- try {
- logger.info("stop mysql parser!");
- RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this;
- long serverId = rdsBinlogEventParserProxy.getServerId();
- rdsBinlogEventParser.setServerId(serverId);
- rdsBinlogEventParserProxy.stop();
- logger.info("start rds mysql binlog parser!");
- rdsBinlogEventParser.start();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
- }
- }
-
- @Override
- public void stop() {
- super.stop();
- }
-
- @Override
- public boolean isStart() {
- return super.isStart();
- }
-
- public void setRdsOpenApiUrl(String rdsOpenApiUrl) {
- this.rdsOpenApiUrl = rdsOpenApiUrl;
- }
-
-
- public void setAccesskey(String accesskey) {
- this.accesskey = accesskey;
- }
-
-
- public void setSecretkey(String secretkey) {
- this.secretkey = secretkey;
- }
-
-
- public void setInstanceId(String instanceId) {
- this.instanceId = instanceId;
- }
-
- public void setDirectory(String directory) {
- this.directory = directory;
- }
-
- public void setBatchSize(int batchSize) {
- this.batchSize = batchSize;
- }
-}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java
index dced6d8d..eb638739 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java
@@ -1,9 +1,5 @@
package com.alibaba.otter.canal.parse.inbound.mysql.rds;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.DescribeBinlogFileResult;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsItem;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.request.DescribeBinlogFilesRequest;
import io.netty.handler.codec.http.HttpResponseStatus;
import java.io.BufferedOutputStream;
@@ -13,11 +9,16 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.UUID;
import javax.crypto.Mac;
import javax.crypto.SecretKey;
@@ -55,52 +56,6 @@ public class RdsBinlogOpenApi {
private static final String API_VERSION = "2014-08-15";
private static final String SIGNATURE_VERSION = "1.0";
-
- public static List listBinlogFiles(String url, String ak, String sk, String dbInstanceId, Date startTime,
- Date endTime) {
- DescribeBinlogFilesRequest request = new DescribeBinlogFilesRequest();
- if (StringUtils.isNotEmpty(url)){
- try {
- URI uri = new URI(url);
- request.setEndPoint(uri.getHost());
- } catch (URISyntaxException e) {
- logger.error("resolve url host failed, will use default rds endpoint!");
- }
- }
- request.setStartDate(startTime);
- request.setEndDate(endTime);
- request.setPageNumber(1);
- request.setPageSize(100);
- request.setRdsInstanceId(dbInstanceId);
- request.setAccessKeyId(ak);
- request.setAccessKeySecret(sk);
- DescribeBinlogFileResult result = null;
- int retryTime = 3;
- while (true){
- try{
- result = request.doAction();
- break;
- }catch (Exception e){
- if(retryTime-- <= 0){
- throw new RuntimeException(e);
- }
- try {
- Thread.sleep(100L);
- } catch (InterruptedException e1) {
- }
- }
- }
- if (result == null){
- return Collections.EMPTY_LIST;
- }
- RdsItem rdsItem = result.getItems();
- if (rdsItem != null){
- return rdsItem.getBinLogFile();
- }
- return Collections.EMPTY_LIST;
- }
-
-
public static void downloadBinlogFiles(String url, String ak, String sk, String dbInstanceId, Date startTime,
Date endTime, File destDir) throws Throwable {
int pageSize = 100;
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java
index 40cb3b13..68be38e5 100644
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java
@@ -2,23 +2,14 @@ package com.alibaba.otter.canal.parse.inbound.mysql.rds;
import java.io.File;
import java.util.Date;
-import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.math.NumberUtils;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
-import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
-import com.alibaba.otter.canal.parse.exception.ServerIdNotMatchException;
-import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
-import com.alibaba.otter.canal.parse.inbound.ParserExceptionHandler;
-import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinLogConnection;
import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
-import com.alibaba.otter.canal.protocol.position.LogPosition;
/**
* 基于rds binlog备份文件的复制
@@ -26,102 +17,47 @@ import com.alibaba.otter.canal.protocol.position.LogPosition;
* @author agapple 2017年10月15日 下午1:27:36
* @since 1.0.25
*/
-public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements CanalEventParser, LocalBinLogConnection.FileParserListener {
+public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements CanalEventParser {
private String url = "https://rds.aliyuncs.com/"; // openapi地址
- private String accesskey; // 云账号的ak
- private String secretkey; // 云账号sk
- private String instanceId; // rds实例id
- private Long startTime;
- private Long endTime;
- private BinlogDownloadQueue binlogDownloadQueue;
- private ParseFinishListener finishListener;
- private int batchSize;
+ private String accesskey; // 云账号的ak
+ private String secretkey; // 云账号sk
+ private String instanceId; // rds实例id
+ private Long startTime;
+ private Long endTime;
public RdsLocalBinlogEventParser(){
}
public void start() throws CanalParseException {
try {
+ Assert.notNull(startTime);
Assert.notNull(accesskey);
Assert.notNull(secretkey);
Assert.notNull(instanceId);
Assert.notNull(url);
- Assert.notNull(directory);
-
if (endTime == null) {
endTime = System.currentTimeMillis();
}
- EntryPosition entryPosition = findStartPosition(null);
- if (entryPosition == null) {
- throw new PositionNotFoundException("position not found!");
- }
- long startTimeInMill = entryPosition.getTimestamp();
- startTime = startTimeInMill;
- List binlogFiles = RdsBinlogOpenApi.listBinlogFiles(url, accesskey,
+ RdsBinlogOpenApi.downloadBinlogFiles(url,
+ accesskey,
secretkey,
instanceId,
new Date(startTime),
- new Date(endTime));
- binlogDownloadQueue = new BinlogDownloadQueue(binlogFiles, batchSize, directory);
- binlogDownloadQueue.silenceDownload();
- needWait = true;
- parallel = false;
- // try to download one file,use to test server id
- binlogDownloadQueue.tryOne();
+ new Date(endTime),
+ new File(directory));
+
+ // 更新一下时间戳
+ masterPosition = new EntryPosition(startTime);
} catch (Throwable e) {
logger.error("download binlog failed", e);
throw new CanalParseException(e);
}
- setParserExceptionHandler(new ParserExceptionHandler() {
- @Override
- public void handle(Throwable e) {
- handleMysqlParserException(e);
- }
- });
super.start();
}
- private void handleMysqlParserException(Throwable throwable) {
- if (throwable instanceof ServerIdNotMatchException) {
- logger.error("server id not match, try download another rds binlog!");
- binlogDownloadQueue.notifyNotMatch();
- try {
- binlogDownloadQueue.cleanDir();
- binlogDownloadQueue.prepare();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- try {
- binlogDownloadQueue.execute(new Runnable() {
-
- @Override
- public void run() {
- RdsLocalBinlogEventParser.super.stop();
- RdsLocalBinlogEventParser.super.start();
- }
- });
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
-
- }
- }
-
- @Override
- protected ErosaConnection buildErosaConnection() {
- ErosaConnection connection = super.buildErosaConnection();
- if (connection instanceof LocalBinLogConnection) {
- LocalBinLogConnection localBinLogConnection = (LocalBinLogConnection) connection;
- localBinLogConnection.setNeedWait(true);
- localBinLogConnection.setServerId(serverId);
- localBinLogConnection.setParserListener(this);
- }
- return connection;
- }
-
public String getUrl() {
return url;
}
@@ -132,81 +68,44 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
}
}
+ public String getAccesskey() {
+ return accesskey;
+ }
public void setAccesskey(String accesskey) {
this.accesskey = accesskey;
}
+ public String getSecretkey() {
+ return secretkey;
+ }
public void setSecretkey(String secretkey) {
this.secretkey = secretkey;
}
+ public String getInstanceId() {
+ return instanceId;
+ }
public void setInstanceId(String instanceId) {
this.instanceId = instanceId;
}
+ public Long getStartTime() {
+ return startTime;
+ }
public void setStartTime(Long startTime) {
this.startTime = startTime;
}
+ public Long getEndTime() {
+ return endTime;
+ }
public void setEndTime(Long endTime) {
this.endTime = endTime;
}
- @Override
- public void onFinish(String fileName) {
- try {
- binlogDownloadQueue.downOne();
- File needDeleteFile = new File(directory + File.separator + fileName);
- if (needDeleteFile.exists()){
- needDeleteFile.delete();
- }
- // 处理下logManager位点问题
- LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
- EntryPosition position = logPosition.getPostion();
- if (position != null){
- LogPosition newLogPosition = new LogPosition();
- String journalName = position.getJournalName();
- int sepIdx = journalName.indexOf(".");
- String fileIndex = journalName.substring(sepIdx+1);
- int index = NumberUtils.toInt(fileIndex) + 1;
- String newJournalName = journalName.substring(0, sepIdx) + "." + StringUtils.leftPad(String.valueOf(index), fileIndex.length(), "0");
- newLogPosition.setPostion(new EntryPosition(newJournalName, 4L, position.getTimestamp(), position.getServerId()));
- newLogPosition.setIdentity(logPosition.getIdentity());
- logPositionManager.persistLogPosition(destination, newLogPosition);
- }
-
- if (binlogDownloadQueue.isLastFile(fileName)) {
- logger.info("all file parse complete, switch to mysql parser!");
- finishListener.onFinish();
- return;
- }
- binlogDownloadQueue.prepare();
- } catch (Exception e) {
- logger.error("prepare download binlog file failed!", e);
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void stop() {
- this.binlogDownloadQueue.release();
- super.stop();
- }
-
- public void setFinishListener(ParseFinishListener finishListener) {
- this.finishListener = finishListener;
- }
-
- public interface ParseFinishListener{
- void onFinish();
- }
-
- public void setBatchSize(int batchSize) {
- this.batchSize = batchSize;
- }
}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/BinlogFile.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/BinlogFile.java
deleted file mode 100644
index e6b354d7..00000000
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/BinlogFile.java
+++ /dev/null
@@ -1,72 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.data;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午2:26
- * @since 1.0.25
- */
-public class BinlogFile {
-
- private Long FileSize;
- private String LogBeginTime;
- private String LogEndTime;
- private String DownloadLink;
- private String HostInstanceID;
- private String LinkExpiredTime;
- private String fileName;
-
- public Long getFileSize() {
- return FileSize;
- }
-
- public void setFileSize(Long fileSize) {
- FileSize = fileSize;
- }
-
- public String getLogBeginTime() {
- return LogBeginTime;
- }
-
- public void setLogBeginTime(String logBeginTime) {
- LogBeginTime = logBeginTime;
- }
-
- public String getLogEndTime() {
- return LogEndTime;
- }
-
- public void setLogEndTime(String logEndTime) {
- LogEndTime = logEndTime;
- }
-
- public String getDownloadLink() {
- return DownloadLink;
- }
-
- public void setDownloadLink(String downloadLink) {
- DownloadLink = downloadLink;
- }
-
- public String getHostInstanceID() {
- return HostInstanceID;
- }
-
- public void setHostInstanceID(String hostInstanceID) {
- HostInstanceID = hostInstanceID;
- }
-
- public String getLinkExpiredTime() {
- return LinkExpiredTime;
- }
-
- public void setLinkExpiredTime(String linkExpiredTime) {
- LinkExpiredTime = linkExpiredTime;
- }
-
- public String getFileName() {
- return fileName;
- }
-
- public void setFileName(String fileName) {
- this.fileName = fileName;
- }
-}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/DescribeBinlogFileResult.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/DescribeBinlogFileResult.java
deleted file mode 100644
index d8d54e44..00000000
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/DescribeBinlogFileResult.java
+++ /dev/null
@@ -1,62 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.data;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午2:26
- * @since 1.0.25
- */
-public class DescribeBinlogFileResult {
- private RdsItem Items;
- private long PageNumber;
- private long TotalRecordCount;
- private long TotalFileSize;
- private String RequestId;
- private long PageRecordCount;
-
- public RdsItem getItems() {
- return Items;
- }
-
- public void setItems(RdsItem items) {
- Items = items;
- }
-
- public long getPageNumber() {
- return PageNumber;
- }
-
- public void setPageNumber(long pageNumber) {
- PageNumber = pageNumber;
- }
-
- public long getTotalRecordCount() {
- return TotalRecordCount;
- }
-
- public void setTotalRecordCount(long totalRecordCount) {
- TotalRecordCount = totalRecordCount;
- }
-
- public long getTotalFileSize() {
- return TotalFileSize;
- }
-
- public void setTotalFileSize(long totalFileSize) {
- TotalFileSize = totalFileSize;
- }
-
- public String getRequestId() {
- return RequestId;
- }
-
- public void setRequestId(String requestId) {
- RequestId = requestId;
- }
-
- public long getPageRecordCount() {
- return PageRecordCount;
- }
-
- public void setPageRecordCount(long pageRecordCount) {
- PageRecordCount = pageRecordCount;
- }
-}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsBackupPolicy.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsBackupPolicy.java
deleted file mode 100644
index b6ca7c53..00000000
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsBackupPolicy.java
+++ /dev/null
@@ -1,69 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.data;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午2:26
- * @since 1.0.25
- */
-public class RdsBackupPolicy {
-
- /**
- * 数据备份保留天数（7到730天）。
- */
- private String BackupRetentionPeriod;
- /**
- * 数据备份时间，格式：HH:mmZ- HH:mm Z。
- */
- private String PreferredBackupTime;
- /**
- * 数据备份周期。Monday：周一；Tuesday：周二；Wednesday：周三；Thursday：周四；Friday：周五；Saturday：周六；Sunday：周日。
- */
- private String PreferredBackupPeriod;
- /**
- * 日志备份状态。Enable：开启；Disabled：关闭。
- */
- private boolean BackupLog;
- /**
- * 日志备份保留天数（7到730天）。
- */
- private int LogBackupRetentionPeriod;
-
- public String getBackupRetentionPeriod() {
- return BackupRetentionPeriod;
- }
-
- public void setBackupRetentionPeriod(String backupRetentionPeriod) {
- BackupRetentionPeriod = backupRetentionPeriod;
- }
-
- public String getPreferredBackupTime() {
- return PreferredBackupTime;
- }
-
- public void setPreferredBackupTime(String preferredBackupTime) {
- PreferredBackupTime = preferredBackupTime;
- }
-
- public String getPreferredBackupPeriod() {
- return PreferredBackupPeriod;
- }
-
- public void setPreferredBackupPeriod(String preferredBackupPeriod) {
- PreferredBackupPeriod = preferredBackupPeriod;
- }
-
- public boolean isBackupLog() {
- return BackupLog;
- }
-
- public void setBackupLog(boolean backupLog) {
- BackupLog = backupLog;
- }
-
- public int getLogBackupRetentionPeriod() {
- return LogBackupRetentionPeriod;
- }
-
- public void setLogBackupRetentionPeriod(int logBackupRetentionPeriod) {
- LogBackupRetentionPeriod = logBackupRetentionPeriod;
- }
-}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsItem.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsItem.java
deleted file mode 100644
index a2cfaffe..00000000
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsItem.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.data;
-
-import java.util.List;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午2:26
- * @since 1.0.25
- */
-public class RdsItem {
- private List BinLogFile;
-
- public List getBinLogFile() {
- return BinLogFile;
- }
-
- public void setBinLogFile(List binLogFile) {
- BinLogFile = binLogFile;
- }
-}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java
deleted file mode 100644
index 976968f6..00000000
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java
+++ /dev/null
@@ -1,250 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.request;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-import javax.crypto.Mac;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
-import javax.net.ssl.SSLContext;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.config.Registry;
-import org.apache.http.config.RegistryBuilder;
-import org.apache.http.conn.HttpClientConnectionManager;
-import org.apache.http.conn.socket.PlainConnectionSocketFactory;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.TrustStrategy;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.ssl.SSLContexts;
-import org.apache.http.util.EntityUtils;
-
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午2:26
- * @since 1.0.25
- */
-public abstract class AbstractRequest {
-
- /**
- * 要求的编码格式
- */
- private static final String ENCODING = "UTF-8";
- /**
- * 要求的sign签名算法
- */
- private static final String MAC_NAME = "HmacSHA1";
-
- private String accessKeyId;
-
- private String accessKeySecret;
-
- /**
- * api 版本
- *
- */
- private String version;
-
- private String endPoint = "rds.aliyuncs.com";
-
- private String protocol = "http";
-
- public void setProtocol(String protocol) {
- this.protocol = protocol;
- }
-
- private int timeout = (int) TimeUnit.MINUTES.toMillis(1);
-
-
- private Map treeMap = new TreeMap();
-
- public void putQueryString(String name, String value){
- if (StringUtils.isBlank(name) || StringUtils.isBlank(value)){
- return;
- }
- treeMap.put(name, value);
- }
-
-
- public void setVersion(String version) {
- this.version = version;
- }
-
-
- public void setEndPoint(String endPoint) {
- this.endPoint = endPoint;
- }
-
- public void setAccessKeyId(String accessKeyId) {
- this.accessKeyId = accessKeyId;
- }
-
- public void setAccessKeySecret(String accessKeySecret) {
- this.accessKeySecret = accessKeySecret;
- }
-
- /**
- * 使用 HMAC-SHA1 签名方法对对encryptText进行签名
- *
- * @param encryptText 被签名的字符串
- * @param encryptKey 密钥
- * @return
- * @throws Exception
- */
- private byte[] HmacSHA1Encrypt(String encryptText, String encryptKey) throws Exception {
- byte[] data = encryptKey.getBytes(ENCODING);
- // 根据给定的字节数组构造一个密钥,第二参数指定一个密钥算法的名称
- SecretKey secretKey = new SecretKeySpec(data, MAC_NAME);
- // 生成一个指定 Mac 算法 的 Mac 对象
- Mac mac = Mac.getInstance(MAC_NAME);
- // 用给定密钥初始化 Mac 对象
- mac.init(secretKey);
-
- byte[] text = encryptText.getBytes(ENCODING);
- // 完成 Mac 操作
- return mac.doFinal(text);
- }
-
- private String base64(byte input[]) throws UnsupportedEncodingException {
- return new String(Base64.encodeBase64(input), ENCODING);
- }
-
- private String concatQueryString(Map parameters) throws UnsupportedEncodingException {
- if (null == parameters) {
- return null;
- }
- StringBuilder urlBuilder = new StringBuilder("");
- for (Map.Entry entry : parameters.entrySet()) {
- String key = entry.getKey();
- String val = entry.getValue();
- urlBuilder.append(encode(key));
- if (val != null) {
- urlBuilder.append("=").append(encode(val));
- }
- urlBuilder.append("&");
- }
- int strIndex = urlBuilder.length();
- if (parameters.size() > 0) {
- urlBuilder.deleteCharAt(strIndex - 1);
- }
- return urlBuilder.toString();
- }
-
- private String encode(String value) throws UnsupportedEncodingException {
- return URLEncoder.encode(value, "UTF-8");
- }
-
- private String makeSignature(TreeMap paramMap) throws Exception {
- String cqs = concatQueryString(paramMap);
- cqs = encode(cqs);
- cqs = cqs.replaceAll("\\+", "%20");
- cqs = cqs.replaceAll("\\*", "%2A");
- cqs = cqs.replaceAll("%7E", "~");
- StringBuilder stringBuilder = new StringBuilder();
- stringBuilder.append("GET").append("&").append(encode("/")).append("&").append(cqs);
- return base64(HmacSHA1Encrypt(stringBuilder.toString(), accessKeySecret + "&"));
- }
-
- public final String formatUTCTZ(Date date) {
- SimpleDateFormat sdf = new SimpleDateFormat("YYYY-MM-dd'T'HH:mm:ss'Z'");
- sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
- return sdf.format(date);
- }
-
- private void fillCommonParam(Map p) {
- p.put("Format", "JSON");
- p.put("Version", version);
- p.put("AccessKeyId", accessKeyId);
- p.put("SignatureMethod", "HMAC-SHA1"); //此处不能用变量 MAC_NAME
- p.put("Timestamp", formatUTCTZ(new Date()));
- p.put("SignatureVersion", "1.0");
- p.put("SignatureNonce", UUID.randomUUID().toString());
- }
-
- private String makeRequestString(Map param) throws Exception {
- fillCommonParam(param);
- String sign = makeSignature(new TreeMap(param));
- StringBuilder builder = new StringBuilder();
- for (Map.Entry entry : param.entrySet()) {
- builder.append(encode(entry.getKey())).append("=").append(encode(entry.getValue())).append("&");
- }
- builder.append("Signature").append("=").append(sign);
- return builder.toString();
- }
-
- /**
- * 执行http请求
- *
- * @param getMethod
- * @return
- * @throws IOException
- */
- private final HttpResponse executeHttpRequest(HttpGet getMethod, String host) throws Exception {
- SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
-
- @Override
- public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
- return true;
- }
- }).build();
- SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext,
- new String[] { "TLSv1" },
- null,
- SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
- Registry registry = RegistryBuilder.create()
- .register("http", PlainConnectionSocketFactory.INSTANCE)
- .register("https", sslsf)
- .build();
- HttpClientConnectionManager httpClientConnectionManager = new PoolingHttpClientConnectionManager(registry);
- CloseableHttpClient httpClient = HttpClientBuilder.create()
- .setMaxConnPerRoute(50)
- .setMaxConnTotal(100)
- .setConnectionManager(httpClientConnectionManager)
- .build();
- RequestConfig requestConfig = RequestConfig.custom()
- .setConnectTimeout(timeout)
- .setConnectionRequestTimeout(timeout)
- .setSocketTimeout(timeout)
- .build();
- getMethod.setConfig(requestConfig);
- HttpResponse response = httpClient.execute(getMethod);
- int statusCode = response.getStatusLine().getStatusCode();
- if (statusCode != HttpResponseStatus.OK.code() && statusCode != HttpResponseStatus.PARTIAL_CONTENT.code()) {
- String result = EntityUtils.toString(response.getEntity());
- throw new RuntimeException("return error !" + response.getStatusLine().getReasonPhrase() + ", " + result);
- }
- return response;
- }
-
- protected abstract T processResult(HttpResponse response) throws Exception;
-
- protected void processBefore(){
-
- }
-
- public final T doAction() throws Exception {
- processBefore();
- String requestStr = makeRequestString(treeMap);
- HttpGet httpGet = new HttpGet(protocol + "://" +endPoint + "?" + requestStr);
- HttpResponse response = executeHttpRequest(httpGet, endPoint);
- if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
- String result = EntityUtils.toString(response.getEntity());
- throw new RuntimeException("http request failed! " + result);
- }
- return processResult(response);
- }
-}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBackupPolicyRequest.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBackupPolicyRequest.java
deleted file mode 100644
index 0fbdfc47..00000000
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBackupPolicyRequest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.request;
-
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsBackupPolicy;
-import org.apache.http.HttpResponse;
-import org.apache.http.util.EntityUtils;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.JSONObject;
-
-/**
- * rds 备份策略查询
- * @author chengjin.lyf on 2018/8/7 下午3:41
- * @since 1.0.25
- */
-public class DescribeBackupPolicyRequest extends AbstractRequest {
-
-
- public DescribeBackupPolicyRequest() {
- setVersion("2014-08-15");
- putQueryString("Action", "DescribeBackupPolicy");
-
- }
-
-
- public void setRdsInstanceId(String rdsInstanceId) {
- putQueryString("DBInstanceId", rdsInstanceId);
- }
-
- @Override
- protected RdsBackupPolicy processResult(HttpResponse response) throws Exception {
- String result = EntityUtils.toString(response.getEntity());
- JSONObject jsonObj = JSON.parseObject(result);
- RdsBackupPolicy policy = new RdsBackupPolicy();
- policy.setBackupRetentionPeriod(jsonObj.getString("BackupRetentionPeriod"));
- policy.setBackupLog(jsonObj.getString("BackupLog").equalsIgnoreCase("Enable"));
- policy.setLogBackupRetentionPeriod(jsonObj.getIntValue("LogBackupRetentionPeriod"));
- policy.setPreferredBackupPeriod(jsonObj.getString("PreferredBackupPeriod"));
- policy.setPreferredBackupTime(jsonObj.getString("PreferredBackupTime"));
- return policy;
- }
-}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBinlogFilesRequest.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBinlogFilesRequest.java
deleted file mode 100644
index 1c91c255..00000000
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBinlogFilesRequest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.request;
-
-import java.util.Date;
-
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.DescribeBinlogFileResult;
-import org.apache.http.HttpResponse;
-import org.apache.http.util.EntityUtils;
-
-import com.alibaba.fastjson.JSONObject;
-import com.alibaba.fastjson.TypeReference;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午3:41
- * @since 1.0.25
- */
-public class DescribeBinlogFilesRequest extends AbstractRequest {
-
-
- public DescribeBinlogFilesRequest() {
- setVersion("2014-08-15");
- putQueryString("Action", "DescribeBinlogFiles");
-
- }
-
- public void setRdsInstanceId(String rdsInstanceId) {
- putQueryString("DBInstanceId", rdsInstanceId);
- }
-
- public void setPageSize(int pageSize) {
- putQueryString("PageSize", String.valueOf(pageSize));
- }
-
- public void setPageNumber(int pageNumber) {
- putQueryString("PageNumber", String.valueOf(pageNumber));
- }
-
- public void setStartDate(Date startDate) {
- putQueryString("StartTime" , formatUTCTZ(startDate));
- }
-
- public void setEndDate(Date endDate) {
- putQueryString("EndTime" , formatUTCTZ(endDate));
- }
-
- public void setResourceOwnerId(Long resourceOwnerId) {
- putQueryString("ResourceOwnerId", String.valueOf(resourceOwnerId));
- }
-
- @Override
- protected DescribeBinlogFileResult processResult(HttpResponse response) throws Exception {
- String result = EntityUtils.toString(response.getEntity());
- DescribeBinlogFileResult describeBinlogFileResult = JSONObject.parseObject(result, new TypeReference() {
- });
- return describeBinlogFileResult;
- }
-}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java
deleted file mode 100644
index 0964627d..00000000
--- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java
+++ /dev/null
@@ -1,191 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
-
-import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
-import com.alibaba.otter.canal.parse.inbound.TableMeta;
-import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
-import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache;
-import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception.CacheConnectionNull;
-import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception.NoHistoryException;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-
-import java.io.IOException;
-import java.util.*;
-
-public class HistoryTableMetaCache {
- private TableMetaStorage tableMetaStorage;
- private MysqlConnection metaConnection;
- private LoadingCache> cache; // 第一层：数据库名.表名，第二层时间戳，TableMeta
-
- public HistoryTableMetaCache() {
- cache = CacheBuilder.newBuilder().build(new CacheLoader>() {
- @Override
- public Map load(String tableName) throws Exception {
- Long timestamp = new Date().getTime();
- String[] strs = tableName.split("\\.");
- String schema = strs[0];
- if (tableMetaStorage != null) {
- init(tableMetaStorage.fetchByTableName(tableName)); // 从存储中读取表的历史ddl
- }
- ResultSetPacket resultSetPacket = connectionQuery("show create table " + tableName); // 获取当前ddl
- String currentDdl = resultSetPacket.getFieldValues().get(1);
- if (cache.asMap().containsKey(tableName)) {
- Map tableMetaMap = cache.getUnchecked(tableName);
- if (tableMetaMap.isEmpty()) {
- put(schema, tableName, currentDdl, timestamp - 1000L); // 放入当前schema，取时间为当前时间-1s
- } else { // 如果table存在历史
- Iterator iterator = tableMetaMap.keySet().iterator();
- Long firstTimestamp = iterator.next();
- TableMeta first = tableMetaMap.get(firstTimestamp); // 拿第一条ddl
- if (!first.getDdl().equalsIgnoreCase(currentDdl)) { // 当前ddl与历史第一条不一致，放入当前ddl
- put(schema, tableName, currentDdl, calculateNewTimestamp(firstTimestamp)); // 计算放入的timestamp,设为第一条时间+1s
- }
- }
- } else {
- put(schema, tableName, currentDdl, timestamp - 1000L); // 放入当前schema
- }
- return cache.get(tableName);
- }
- });
- }
-
- public void init(List entries) throws IOException {
- if (entries == null) {
- return;
- }
- for (TableMetaEntry entry : entries) {
- try {
- put(entry.getSchema(), entry.getTable(), entry.getDdl(), entry.getTimestamp());
- } catch (CacheConnectionNull cacheConnectionNull) {
- cacheConnectionNull.printStackTrace();
- }
- }
- }
-
- public TableMeta put(String schema, String table, String ddl, Long timestamp) throws CacheConnectionNull, IOException {
- ResultSetPacket resultSetPacket;
- if (!(ddl.contains("CREATE TABLE") || ddl.contains("create table"))) { // 尝试直接从数据库拉取CREATE TABLE的DDL
- resultSetPacket = connectionQuery("show create table " + table);
- ddl = resultSetPacket.getFieldValues().get(1);
- } else { // CREATE TABLE 的 DDL
- resultSetPacket = new ResultSetPacket();
- List fields = new ArrayList();
- String[] strings = table.split("\\.");
- String shortTable = table;
- if (strings.length > 1) {
- shortTable = strings[1];
- }
- fields.add(0, shortTable);
- fields.add(1, ddl);
- resultSetPacket.setFieldValues(fields);
- if (metaConnection != null) {
- resultSetPacket.setSourceAddress(metaConnection.getAddress());
- }
- }
- Map tableMetaMap;
- if (!cache.asMap().containsKey(table)) {
- tableMetaMap = new TreeMap(new Comparator() {
- @Override
- public int compare(Long o1, Long o2) {
- return o2.compareTo(o1);
- }
- });
- cache.put(table, tableMetaMap);
- } else {
- tableMetaMap = cache.getUnchecked(table);
- }
- eliminate(tableMetaMap); // 淘汰旧的TableMeta
- TableMeta tableMeta = new TableMeta(schema, table, TableMetaCache.parseTableMeta(schema, table, resultSetPacket));
- if (tableMeta.getDdl() == null) { // 生成的TableMeta有时DDL为null
- tableMeta.setDdl(ddl);
- }
- tableMetaMap.put(timestamp, tableMeta);
- return tableMeta;
- }
-
- public TableMeta get(String schema, String table, Long timestamp) throws NoHistoryException, CacheConnectionNull {
- Map tableMetaMap = cache.getUnchecked(table);
- Iterator iterator = tableMetaMap.keySet().iterator();
- Long selected = null;
- while(iterator.hasNext()) {
- Long temp = iterator.next();
- if (timestamp > temp) {
- selected = temp;
- break;
- }
- }
-
- if (selected == null) {
- iterator = tableMetaMap.keySet().iterator();
- if (iterator.hasNext()) {
- selected = iterator.next();
- } else {
- throw new NoHistoryException(schema, table);
- }
- }
-
- return tableMetaMap.get(selected);
- }
-
- public void clearTableMeta() {
- cache.invalidateAll();
- }
-
- public void clearTableMetaWithSchemaName(String schema) {
- for (String tableName : cache.asMap().keySet()) {
- String[] strs = tableName.split("\\.");
- if (schema.equalsIgnoreCase(strs[0])) {
- cache.invalidate(tableName);
- }
- }
- }
-
- public void clearTableMeta(String schema, String table) {
- if (!table.contains(".")) {
- table = schema+"."+table;
- }
- cache.invalidate(table);
- }
-
- // eliminate older table meta in cache
- private void eliminate(Map tableMetaMap) {
- int MAX_CAPABILITY = 20;
- if (tableMetaMap.keySet().size() < MAX_CAPABILITY) {
- return;
- }
- Iterator