chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed #1538

Browse Source
This commit is contained in:
mcy
2019-02-28 17:29:06 +08:00
parent f21abc13af
commit e7f069850f
5 changed files with 98 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java
View File

@@ -4,8 +4,6 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.*;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -116,23 +114,33 @@ public class JdbcTypeUtil {
break;
case Types.DATE:
if (!value.startsWith("0000-00-00")) {
value = value.trim().replace(" ", "T");
DateTime dt = new DateTime(value, DateTimeZone.forID("+08:00"));
res = new Date(dt.toDate().getTime());
java.util.Date date = Util.parseDate(value);
if (date != null) {
res = new Date(date.getTime());
} else {
res = null;
}
} else {
res = null;
}
break;
case Types.TIME:
value = "T" + value;
DateTime dt = new DateTime(value, DateTimeZone.forID("+08:00"));
res = new Time(dt.toDate().getTime());
case Types.TIME: {
java.util.Date date = Util.parseDate(value);
if (date != null) {
res = new Time(date.getTime());
} else {
res = null;
}
break;
}
case Types.TIMESTAMP:
if (!value.startsWith("0000-00-00")) {
value = value.trim().replace(" ", "T");
dt = new DateTime(value, DateTimeZone.forID("+08:00"));
res = new Timestamp(dt.toDate().getTime());
java.util.Date date = Util.parseDate(value);
if (date != null) {
res = new Timestamp(date.getTime());
} else {
res = null;
}
} else {
res = null;
}

35
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java
View File

@@ -6,12 +6,15 @@ import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Date;
import java.util.function.Consumer;
import java.util.function.Function;
import javax.sql.DataSource;
import org.apache.commons.lang.StringUtils;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -136,4 +139,36 @@ public class Util {
return column;
}
/**
* 通用日期时间字符解析
*
* @param datetimeStr 日期时间字符串
* @return Date
*/
public static Date parseDate(String datetimeStr) {
if (StringUtils.isEmpty(datetimeStr)) {
return null;
}
datetimeStr = datetimeStr.trim();
if (datetimeStr.contains("-")) {
if (datetimeStr.contains(":")) {
datetimeStr = datetimeStr.replace(" ", "T");
}
} else if (datetimeStr.contains(":")) {
datetimeStr = "T" + datetimeStr;
}
DateTime dateTime;
if (datetimeStr.startsWith("1940-06-03") || datetimeStr.startsWith("1941-03-16")
|| datetimeStr.startsWith("1986-05-04") || datetimeStr.startsWith("1987-04-12")
|| datetimeStr.startsWith("1988-04-10") || datetimeStr.startsWith("1989-04-16")
|| datetimeStr.startsWith("1990-04-15") || datetimeStr.startsWith("1991-04-14")) {
dateTime = new DateTime(datetimeStr, DateTimeZone.forID("+08:00"));
} else {
dateTime = new DateTime(datetimeStr);
}
return dateTime.toDate();
}
}

22
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java
View File

@@ -12,7 +12,6 @@ import javax.sql.DataSource;
import org.apache.commons.codec.binary.Base64;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -21,6 +20,7 @@ import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem;
import com.alibaba.otter.canal.client.adapter.support.Util;
/**
* ES 同步工具同类
@@ -104,14 +104,14 @@ public class ESSyncUtil {
}
} else if ("date".equals(esType)) {
if (val instanceof java.sql.Time) {
DateTime dateTime = new DateTime(((java.sql.Time) val).getTime(), DateTimeZone.forID("+08:00"));
DateTime dateTime = new DateTime(((java.sql.Time) val).getTime());
if (dateTime.getMillisOfSecond() != 0) {
res = dateTime.toString("HH:mm:ss.SSS");
} else {
res = dateTime.toString("HH:mm:ss");
}
} else if (val instanceof java.sql.Timestamp) {
DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime(), DateTimeZone.forID("+08:00"));
DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime());
if (dateTime.getMillisOfSecond() != 0) {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00");
} else {
@@ -120,9 +120,9 @@ public class ESSyncUtil {
} else if (val instanceof java.sql.Date || val instanceof Date) {
DateTime dateTime;
if (val instanceof java.sql.Date) {
dateTime = new DateTime(((java.sql.Date) val).getTime(), DateTimeZone.forID("+08:00"));
dateTime = new DateTime(((java.sql.Date) val).getTime());
} else {
dateTime = new DateTime(((Date) val).getTime(), DateTimeZone.forID("+08:00"));
dateTime = new DateTime(((Date) val).getTime());
}
if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
&& dateTime.getMillisOfSecond() == 0) {
@@ -135,7 +135,7 @@ public class ESSyncUtil {
}
}
} else if (val instanceof Long) {
DateTime dateTime = new DateTime(((Long) val).longValue(), DateTimeZone.forID("+08:00"));
DateTime dateTime = new DateTime(((Long) val).longValue());
if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
&& dateTime.getMillisOfSecond() == 0) {
res = dateTime.toString("yyyy-MM-dd");
@@ -149,17 +149,23 @@ public class ESSyncUtil {
if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' '
&& v.charAt(13) == ':' && v.charAt(16) == ':') {
String dt = v.substring(0, 10) + "T" + v.substring(11);
DateTime dateTime = new DateTime(dt, DateTimeZone.forID("+08:00"));
Date date = Util.parseDate(dt);
if (date != null) {
DateTime dateTime = new DateTime(date);
if (dateTime.getMillisOfSecond() != 0) {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00");
} else {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00");
}
}
} else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') {
DateTime dateTime = new DateTime(v, DateTimeZone.forID("+08:00"));
Date date = Util.parseDate(v);
if (date != null) {
DateTime dateTime = new DateTime(date);
res = dateTime.toString("yyyy-MM-dd");
}
}
}
} else if ("binary".equals(esType)) {
if (val instanceof byte[]) {
Base64 base64 = new Base64();

22
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java
View File

@@ -7,6 +7,7 @@ import java.math.RoundingMode;
import java.sql.Timestamp;
import java.util.Date;
import com.alibaba.otter.canal.client.adapter.support.Util;
import org.apache.hadoop.hbase.util.Bytes;
import org.joda.time.DateTime;
@@ -390,7 +391,7 @@ public class PhTypeUtil {
String dateStr = (String) v;
Date date;
try {
date = parseDatetime(dateStr);
date = Util.parseDate(dateStr);
if (date != null) {
encodeLong(date.getTime(), b, 0);
}
@@ -419,7 +420,7 @@ public class PhTypeUtil {
String dateStr = (String) v;
Date date;
try {
date = parseDatetime(dateStr);
date = Util.parseDate(dateStr);
if (date != null) {
encodeUnsignedLong(date.getTime(), b, 0);
}
@@ -600,22 +601,9 @@ public class PhTypeUtil {
private static void checkForSufficientLength(byte[] b, int offset, int requiredLength) {
if (b.length < offset + requiredLength) {
throw new RuntimeException("Expected length of at least " + requiredLength + " bytes, but had "
+ (b.length - offset));
throw new RuntimeException(
"Expected length of at least " + requiredLength + " bytes, but had " + (b.length - offset));
}
}
private static Date parseDatetime(String dateStr) {
Date date = null;
int len = dateStr.length();
if (len == 10 && dateStr.charAt(4) == '-' && dateStr.charAt(7) == '-') {
date = new DateTime(dateStr, DateTimeZone.forID("+08:00")).toDate();
} else if (len == 8 && dateStr.charAt(2) == ':' && dateStr.charAt(5) == ':') {
date = new DateTime("T" + dateStr, DateTimeZone.forID("+08:00")).toDate();
} else if (len >= 19 && dateStr.charAt(4) == '-' && dateStr.charAt(7) == '-' && dateStr.charAt(13) == ':'
&& dateStr.charAt(16) == ':') {
date = new DateTime(dateStr.replace(" ", "T"), DateTimeZone.forID("+08:00")).toDate();
}
return date;
}
}

30
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java
View File

@@ -10,10 +10,9 @@ import java.util.LinkedHashMap;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.joda.time.DateTime;
import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig;
import org.joda.time.DateTimeZone;
import com.alibaba.otter.canal.client.adapter.support.Util;
public class SyncUtil {
@@ -197,9 +196,12 @@ public class SyncUtil {
} else if (value instanceof String) {
String v = (String) value;
if (!v.startsWith("0000-00-00")) {
v = v.trim().replace(" ", "T");
DateTime dt = new DateTime(v, DateTimeZone.forID("+08:00"));
pstmt.setDate(i, new Date(dt.toDate().getTime()));
java.util.Date date = Util.parseDate(v);
if (date != null) {
pstmt.setDate(i, new Date(date.getTime()));
} else {
pstmt.setNull(i, type);
}
} else {
pstmt.setNull(i, type);
}
@@ -214,9 +216,12 @@ public class SyncUtil {
pstmt.setTime(i, new java.sql.Time(((java.util.Date) value).getTime()));
} else if (value instanceof String) {
String v = (String) value;
v = "T" + v;
DateTime dt = new DateTime(v, DateTimeZone.forID("+08:00"));
pstmt.setTime(i, new Time(dt.toDate().getTime()));
java.util.Date date = Util.parseDate(v);
if (date != null) {
pstmt.setTime(i, new Time(date.getTime()));
} else {
pstmt.setNull(i, type);
}
} else {
pstmt.setNull(i, type);
}
@@ -229,9 +234,12 @@ public class SyncUtil {
} else if (value instanceof String) {
String v = (String) value;
if (!v.startsWith("0000-00-00")) {
v = v.trim().replace(" ", "T");
DateTime dt = new DateTime(v, DateTimeZone.forID("+08:00"));
pstmt.setTimestamp(i, new Timestamp(dt.toDate().getTime()));
java.util.Date date = Util.parseDate(v);
if (date != null) {
pstmt.setTimestamp(i, new Timestamp(date.getTime()));
} else {
pstmt.setNull(i, type);
}
} else {
pstmt.setNull(i, type);
}