chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed

Browse Source
This commit is contained in:
agapple
2019-02-25 15:18:04 +08:00
parent 1473a4a6b5
commit 072420c32b
7 changed files with 71 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java
View File

@@ -2,7 +2,12 @@ package com.alibaba.otter.canal.client.adapter.support;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.math.BigInteger; import java.math.BigInteger;
import java.sql.*; import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import java.sql.Types;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.slf4j.Logger; import org.slf4j.Logger;
@@ -67,14 +72,14 @@ public class JdbcTypeUtil {
} }
} }
private static boolean isText(String columnType) {
return "LONGTEXT".equalsIgnoreCase(columnType) || "MEDIUMTEXT".equalsIgnoreCase(columnType)
|| "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType);
}
public static Object typeConvert(String columnName, String value, int sqlType, String mysqlType) { public static Object typeConvert(String columnName, String value, int sqlType, String mysqlType) {
if (value == null if (value == null
|| (value.equals("") || (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) {
&& !("longtext".equals(mysqlType)
|| "text".equals(mysqlType)
|| sqlType == Types.CHAR
|| sqlType == Types.VARCHAR
|| sqlType == Types.LONGVARCHAR))) {
return null; return null;
} }
@@ -124,7 +129,7 @@ public class JdbcTypeUtil {
value = value.trim().replace(" ", "T"); value = value.trim().replace(" ", "T");
DateTime dt = new DateTime(value); DateTime dt = new DateTime(value);
res = new Date(dt.toDate().getTime()); res = new Date(dt.toDate().getTime());
break; break;
} }
case Types.TIME: case Types.TIME:
value = "T" + value; value = "T" + value;

6
filter/src/test/java/com/alibaba/otter/canal/filter/AviaterFilterTest.java
View File

@@ -50,6 +50,12 @@ public class AviaterFilterTest {
result = filter.filter("s3.t2"); result = filter.filter("s3.t2");
Assert.assertEquals(false, result); Assert.assertEquals(false, result);
result = filter.filter("S1.S2");
Assert.assertEquals(true, result);
result = filter.filter("S2.S1");
Assert.assertEquals(true, result);
AviaterRegexFilter filter2 = new AviaterRegexFilter("s1\\..*,s2.t1"); AviaterRegexFilter filter2 = new AviaterRegexFilter("s1\\..*,s2.t1");
result = filter2.filter("s1.t1"); result = filter2.filter("s1.t1");

4
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
View File

@@ -294,7 +294,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
mysqlEventParser.setFallbackIntervalInSeconds(parameters.getFallbackIntervalInSeconds()); mysqlEventParser.setFallbackIntervalInSeconds(parameters.getFallbackIntervalInSeconds());
mysqlEventParser.setProfilingEnabled(false); mysqlEventParser.setProfilingEnabled(false);
mysqlEventParser.setFilterTableError(parameters.getFilterTableError()); mysqlEventParser.setFilterTableError(parameters.getFilterTableError());
mysqlEventParser.setParallel(parameters.isParallel()); mysqlEventParser.setParallel(parameters.getParallel());
mysqlEventParser.setIsGTIDMode(BooleanUtils.toBoolean(parameters.getGtidEnable())); mysqlEventParser.setIsGTIDMode(BooleanUtils.toBoolean(parameters.getGtidEnable()));
// tsdb // tsdb
if (parameters.getTsdbSnapshotInterval() != null) { if (parameters.getTsdbSnapshotInterval() != null) {
@@ -341,7 +341,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
localBinlogEventParser.setDetectingEnable(parameters.getDetectingEnable()); localBinlogEventParser.setDetectingEnable(parameters.getDetectingEnable());
localBinlogEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds()); localBinlogEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
localBinlogEventParser.setFilterTableError(parameters.getFilterTableError()); localBinlogEventParser.setFilterTableError(parameters.getFilterTableError());
localBinlogEventParser.setParallel(parameters.isParallel()); localBinlogEventParser.setParallel(parameters.getParallel());
// 数据库信息，反查表结构时需要 // 数据库信息，反查表结构时需要
if (!CollectionUtils.isEmpty(dbAddresses)) { if (!CollectionUtils.isEmpty(dbAddresses)) {
localBinlogEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0), localBinlogEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0),

7
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
View File

@@ -118,8 +118,7 @@ public class CanalParameter implements Serializable {
private String standbyLogfileName = null; // standby起始位置 private String standbyLogfileName = null; // standby起始位置
private Long standbyLogfileOffest = null; private Long standbyLogfileOffest = null;
private Long standbyTimestamp = null; private Long standbyTimestamp = null;
private boolean parallel = true; private Boolean parallel = Boolean.FALSE;
public static enum RunMode { public static enum RunMode {
@@ -984,11 +983,11 @@ public class CanalParameter implements Serializable {
this.tsdbSnapshotExpire = tsdbSnapshotExpire; this.tsdbSnapshotExpire = tsdbSnapshotExpire;
} }
public boolean isParallel() { public Boolean getParallel() {
return parallel; return parallel;
} }
public void setParallel(boolean parallel) { public void setParallel(Boolean parallel) {
this.parallel = parallel; this.parallel = parallel;
} }

78
server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java
View File

@@ -1,6 +1,12 @@
package com.alibaba.otter.canal.common; package com.alibaba.otter.canal.common;
import java.util.*; import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
@@ -24,46 +30,36 @@ import com.google.protobuf.InvalidProtocolBufferException;
public class MQMessageUtils { public class MQMessageUtils {
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private static Map<String, List<PartitionData>> partitionDatas = MigrateMap private static Map<String, List<PartitionData>> partitionDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
.makeComputingMap(new MapMaker().softValues(), new Function<String, List<PartitionData>>() { new Function<String, List<PartitionData>>() {
public List<PartitionData> apply(String pkHashConfigs) { public List<PartitionData> apply(String pkHashConfigs) {
List<PartitionData> datas = Lists List<PartitionData> datas = Lists.newArrayList();
.newArrayList(); String[] pkHashConfigArray = StringUtils.split(pkHashConfigs,
String[] pkHashConfigArray = StringUtils ",");
.split(pkHashConfigs, ",");
// schema.table:id^name // schema.table:id^name
for (String pkHashConfig : pkHashConfigArray) { for (String pkHashConfig : pkHashConfigArray) {
PartitionData data = new PartitionData(); PartitionData data = new PartitionData();
int i = pkHashConfig int i = pkHashConfig.lastIndexOf(":");
.lastIndexOf(":");
if (i > 0) { if (i > 0) {
String pkStr = pkHashConfig String pkStr = pkHashConfig.substring(i + 1);
.substring(i + 1); if (pkStr.equalsIgnoreCase("$pk$")) {
if (pkStr.equalsIgnoreCase(
"$pk$")) {
data.hashMode.autoPkHash = true; data.hashMode.autoPkHash = true;
} else { } else {
data.hashMode.pkNames = Lists data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,
.newArrayList( '^'));
StringUtils
.split(
pkStr,
'^'));
} }
pkHashConfig = pkHashConfig pkHashConfig = pkHashConfig.substring(0,
.substring(0, i); i);
} else { } else {
data.hashMode.tableHash = true; data.hashMode.tableHash = true;
} }
if (!isWildCard( if (!isWildCard(pkHashConfig)) {
pkHashConfig)) {
data.simpleName = pkHashConfig; data.simpleName = pkHashConfig;
} else { } else {
data.regexFilter = new AviaterRegexFilter( data.regexFilter = new AviaterRegexFilter(pkHashConfig);
pkHashConfig);
} }
datas.add(data); datas.add(data);
} }
@@ -73,29 +69,24 @@ public class MQMessageUtils {
}); });
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
.makeComputingMap(new MapMaker().softValues(), new Function<String, List<DynamicTopicData>>() { new Function<String, List<DynamicTopicData>>() {
public List<DynamicTopicData> apply(String pkHashConfigs) { public List<DynamicTopicData> apply(String pkHashConfigs) {
List<DynamicTopicData> datas = Lists List<DynamicTopicData> datas = Lists.newArrayList();
.newArrayList(); String[] dynamicTopicArray = StringUtils.split(pkHashConfigs,
String[] dynamicTopicArray = StringUtils ",");
.split(pkHashConfigs, ",");
// schema.table // schema.table
for (String dynamicTopic : dynamicTopicArray) { for (String dynamicTopic : dynamicTopicArray) {
DynamicTopicData data = new DynamicTopicData(); DynamicTopicData data = new DynamicTopicData();
if (!isWildCard( if (!isWildCard(dynamicTopic)) {
dynamicTopic)) {
data.simpleName = dynamicTopic; data.simpleName = dynamicTopic;
} else { } else {
if (dynamicTopic if (dynamicTopic.contains("\\.")) {
.contains("\\.")) { data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
data.tableRegexFilter = new AviaterRegexFilter(
dynamicTopic);
} else { } else {
data.schemaRegexFilter = new AviaterRegexFilter( data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
dynamicTopic);
} }
} }
datas.add(data); datas.add(data);
@@ -291,9 +282,8 @@ public class MQMessageUtils {
try { try {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException( throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:"
"ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), + entry.toString(), e);
e);
} }
CanalEntry.EventType eventType = rowChange.getEventType(); CanalEntry.EventType eventType = rowChange.getEventType();
@@ -555,8 +545,8 @@ public class MQMessageUtils {
private static boolean isWildCard(String value) { private static boolean isWildCard(String value) {
// not contaiins '.' ? // not contaiins '.' ?
return StringUtils.containsAny(value, return StringUtils.containsAny(value, new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$',
new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$', '^' }); '^' });
} }
private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName, private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName,

16
server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java
View File

@@ -133,8 +133,14 @@ public class CanalMQStarter {
} }
private void worker(String destination, AtomicBoolean destinationRunning) { private void worker(String destination, AtomicBoolean destinationRunning) {
while (!running || !destinationRunning.get()) while (!running || !destinationRunning.get()) {
; try {
Thread.sleep(100);
} catch (InterruptedException e) {
// ignore
}
}
logger.info("## start the MQ producer: {}.", destination); logger.info("## start the MQ producer: {}.", destination);
final ClientIdentity clientIdentity = new ClientIdentity(destination, (short) 1001, ""); final ClientIdentity clientIdentity = new ClientIdentity(destination, (short) 1001, "");
@@ -166,8 +172,10 @@ public class CanalMQStarter {
while (running && destinationRunning.get()) { while (running && destinationRunning.get()) {
Message message; Message message;
if (getTimeout != null && getTimeout > 0) { if (getTimeout != null && getTimeout > 0) {
message = canalServer message = canalServer.getWithoutAck(clientIdentity,
.getWithoutAck(clientIdentity, getBatchSize, getTimeout, TimeUnit.MILLISECONDS); getBatchSize,
getTimeout,
TimeUnit.MILLISECONDS);
} else { } else {
message = canalServer.getWithoutAck(clientIdentity, getBatchSize); message = canalServer.getWithoutAck(clientIdentity, getBatchSize);
} }

2
sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/GroupEventSink.java
View File

@@ -51,7 +51,7 @@ public class GroupEventSink extends EntryEventSink {
try { try {
barrier.await(event);// 进行timeline的归并调度处理 barrier.await(event);// 进行timeline的归并调度处理
if (filterTransactionEntry) { if (filterTransactionEntry) {
return super.doSink(Arrays.asList(event)); super.doSink(Arrays.asList(event));
} else if (i == size - 1) { } else if (i == size - 1) {
// 针对事务数据，只有到最后一条数据都通过后，才进行sink操作，保证原子性 // 针对事务数据，只有到最后一条数据都通过后，才进行sink操作，保证原子性
// 同时批量sink，也要保证在最后一条数据释放状态之前写出数据，否则就有并发问题 // 同时批量sink，也要保证在最后一条数据释放状态之前写出数据，否则就有并发问题