From e7f069850f74b27b5a481bf56b711b6d3cdad330 Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 28 Feb 2019 17:29:06 +0800 Subject: [PATCH] fixed #1538 --- .../client/adapter/support/JdbcTypeUtil.java | 32 ++++++++++------- .../canal/client/adapter/support/Util.java | 35 +++++++++++++++++++ .../client/adapter/es/support/ESSyncUtil.java | 32 ++++++++++------- .../adapter/hbase/support/PhTypeUtil.java | 22 +++--------- .../client/adapter/rdb/support/SyncUtil.java | 30 ++++++++++------ 5 files changed, 98 insertions(+), 53 deletions(-) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java index 6843665b..cc1e4e7e 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java @@ -4,8 +4,6 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.sql.*; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -116,23 +114,33 @@ public class JdbcTypeUtil { break; case Types.DATE: if (!value.startsWith("0000-00-00")) { - value = value.trim().replace(" ", "T"); - DateTime dt = new DateTime(value, DateTimeZone.forID("+08:00")); - res = new Date(dt.toDate().getTime()); + java.util.Date date = Util.parseDate(value); + if (date != null) { + res = new Date(date.getTime()); + } else { + res = null; + } } else { res = null; } break; - case Types.TIME: - value = "T" + value; - DateTime dt = new DateTime(value, DateTimeZone.forID("+08:00")); - res = new Time(dt.toDate().getTime()); + case Types.TIME: { + java.util.Date date = Util.parseDate(value); + if (date != null) { + res = new Time(date.getTime()); + } else { + res = null; + } break; + } case Types.TIMESTAMP: if (!value.startsWith("0000-00-00")) { - value = value.trim().replace(" ", "T"); - dt = new DateTime(value, DateTimeZone.forID("+08:00")); - res = new Timestamp(dt.toDate().getTime()); + java.util.Date date = Util.parseDate(value); + if (date != null) { + res = new Timestamp(date.getTime()); + } else { + res = null; + } } else { res = null; } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java index 919974a1..42cc2ca1 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java @@ -6,12 +6,15 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Date; import java.util.function.Consumer; import java.util.function.Function; import javax.sql.DataSource; import org.apache.commons.lang.StringUtils; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -136,4 +139,36 @@ public class Util { return column; } + + /** + * 通用日期时间字符解析 + * + * @param datetimeStr 日期时间字符串 + * @return Date + */ + public static Date parseDate(String datetimeStr) { + if (StringUtils.isEmpty(datetimeStr)) { + return null; + } + datetimeStr = datetimeStr.trim(); + if (datetimeStr.contains("-")) { + if (datetimeStr.contains(":")) { + datetimeStr = datetimeStr.replace(" ", "T"); + } + } else if (datetimeStr.contains(":")) { + datetimeStr = "T" + datetimeStr; + } + + DateTime dateTime; + if (datetimeStr.startsWith("1940-06-03") || datetimeStr.startsWith("1941-03-16") + || datetimeStr.startsWith("1986-05-04") || datetimeStr.startsWith("1987-04-12") + || datetimeStr.startsWith("1988-04-10") || datetimeStr.startsWith("1989-04-16") + || datetimeStr.startsWith("1990-04-15") || datetimeStr.startsWith("1991-04-14")) { + dateTime = new DateTime(datetimeStr, DateTimeZone.forID("+08:00")); + } else { + dateTime = new DateTime(datetimeStr); + } + + return dateTime.toDate(); + } } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java index 74ca70cd..84cb2f78 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java @@ -12,7 +12,6 @@ import javax.sql.DataSource; import org.apache.commons.codec.binary.Base64; import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,6 +20,7 @@ import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; +import com.alibaba.otter.canal.client.adapter.support.Util; /** * ES 同步工具同类 @@ -104,14 +104,14 @@ public class ESSyncUtil { } } else if ("date".equals(esType)) { if (val instanceof java.sql.Time) { - DateTime dateTime = new DateTime(((java.sql.Time) val).getTime(), DateTimeZone.forID("+08:00")); + DateTime dateTime = new DateTime(((java.sql.Time) val).getTime()); if (dateTime.getMillisOfSecond() != 0) { res = dateTime.toString("HH:mm:ss.SSS"); } else { res = dateTime.toString("HH:mm:ss"); } } else if (val instanceof java.sql.Timestamp) { - DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime(), DateTimeZone.forID("+08:00")); + DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime()); if (dateTime.getMillisOfSecond() != 0) { res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); } else { @@ -120,9 +120,9 @@ public class ESSyncUtil { } else if (val instanceof java.sql.Date || val instanceof Date) { DateTime dateTime; if (val instanceof java.sql.Date) { - dateTime = new DateTime(((java.sql.Date) val).getTime(), DateTimeZone.forID("+08:00")); + dateTime = new DateTime(((java.sql.Date) val).getTime()); } else { - dateTime = new DateTime(((Date) val).getTime(), DateTimeZone.forID("+08:00")); + dateTime = new DateTime(((Date) val).getTime()); } if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0 && dateTime.getMillisOfSecond() == 0) { @@ -135,7 +135,7 @@ public class ESSyncUtil { } } } else if (val instanceof Long) { - DateTime dateTime = new DateTime(((Long) val).longValue(), DateTimeZone.forID("+08:00")); + DateTime dateTime = new DateTime(((Long) val).longValue()); if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0 && dateTime.getMillisOfSecond() == 0) { res = dateTime.toString("yyyy-MM-dd"); @@ -149,15 +149,21 @@ public class ESSyncUtil { if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' ' && v.charAt(13) == ':' && v.charAt(16) == ':') { String dt = v.substring(0, 10) + "T" + v.substring(11); - DateTime dateTime = new DateTime(dt, DateTimeZone.forID("+08:00")); - if (dateTime.getMillisOfSecond() != 0) { - res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); - } else { - res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00"); + Date date = Util.parseDate(dt); + if (date != null) { + DateTime dateTime = new DateTime(date); + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00"); + } } } else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') { - DateTime dateTime = new DateTime(v, DateTimeZone.forID("+08:00")); - res = dateTime.toString("yyyy-MM-dd"); + Date date = Util.parseDate(v); + if (date != null) { + DateTime dateTime = new DateTime(date); + res = dateTime.toString("yyyy-MM-dd"); + } } } } else if ("binary".equals(esType)) { diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java index f738e054..6ff684e4 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java @@ -7,6 +7,7 @@ import java.math.RoundingMode; import java.sql.Timestamp; import java.util.Date; +import com.alibaba.otter.canal.client.adapter.support.Util; import org.apache.hadoop.hbase.util.Bytes; import org.joda.time.DateTime; @@ -390,7 +391,7 @@ public class PhTypeUtil { String dateStr = (String) v; Date date; try { - date = parseDatetime(dateStr); + date = Util.parseDate(dateStr); if (date != null) { encodeLong(date.getTime(), b, 0); } @@ -419,7 +420,7 @@ public class PhTypeUtil { String dateStr = (String) v; Date date; try { - date = parseDatetime(dateStr); + date = Util.parseDate(dateStr); if (date != null) { encodeUnsignedLong(date.getTime(), b, 0); } @@ -600,22 +601,9 @@ public class PhTypeUtil { private static void checkForSufficientLength(byte[] b, int offset, int requiredLength) { if (b.length < offset + requiredLength) { - throw new RuntimeException("Expected length of at least " + requiredLength + " bytes, but had " - + (b.length - offset)); + throw new RuntimeException( + "Expected length of at least " + requiredLength + " bytes, but had " + (b.length - offset)); } } - private static Date parseDatetime(String dateStr) { - Date date = null; - int len = dateStr.length(); - if (len == 10 && dateStr.charAt(4) == '-' && dateStr.charAt(7) == '-') { - date = new DateTime(dateStr, DateTimeZone.forID("+08:00")).toDate(); - } else if (len == 8 && dateStr.charAt(2) == ':' && dateStr.charAt(5) == ':') { - date = new DateTime("T" + dateStr, DateTimeZone.forID("+08:00")).toDate(); - } else if (len >= 19 && dateStr.charAt(4) == '-' && dateStr.charAt(7) == '-' && dateStr.charAt(13) == ':' - && dateStr.charAt(16) == ':') { - date = new DateTime(dateStr.replace(" ", "T"), DateTimeZone.forID("+08:00")).toDate(); - } - return date; - } } diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java index ea1977e9..a8670766 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java @@ -10,10 +10,9 @@ import java.util.LinkedHashMap; import java.util.Map; import org.apache.commons.lang.StringUtils; -import org.joda.time.DateTime; import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; -import org.joda.time.DateTimeZone; +import com.alibaba.otter.canal.client.adapter.support.Util; public class SyncUtil { @@ -197,9 +196,12 @@ public class SyncUtil { } else if (value instanceof String) { String v = (String) value; if (!v.startsWith("0000-00-00")) { - v = v.trim().replace(" ", "T"); - DateTime dt = new DateTime(v, DateTimeZone.forID("+08:00")); - pstmt.setDate(i, new Date(dt.toDate().getTime())); + java.util.Date date = Util.parseDate(v); + if (date != null) { + pstmt.setDate(i, new Date(date.getTime())); + } else { + pstmt.setNull(i, type); + } } else { pstmt.setNull(i, type); } @@ -214,9 +216,12 @@ public class SyncUtil { pstmt.setTime(i, new java.sql.Time(((java.util.Date) value).getTime())); } else if (value instanceof String) { String v = (String) value; - v = "T" + v; - DateTime dt = new DateTime(v, DateTimeZone.forID("+08:00")); - pstmt.setTime(i, new Time(dt.toDate().getTime())); + java.util.Date date = Util.parseDate(v); + if (date != null) { + pstmt.setTime(i, new Time(date.getTime())); + } else { + pstmt.setNull(i, type); + } } else { pstmt.setNull(i, type); } @@ -229,9 +234,12 @@ public class SyncUtil { } else if (value instanceof String) { String v = (String) value; if (!v.startsWith("0000-00-00")) { - v = v.trim().replace(" ", "T"); - DateTime dt = new DateTime(v, DateTimeZone.forID("+08:00")); - pstmt.setTimestamp(i, new Timestamp(dt.toDate().getTime())); + java.util.Date date = Util.parseDate(v); + if (date != null) { + pstmt.setTimestamp(i, new Timestamp(date.getTime())); + } else { + pstmt.setNull(i, type); + } } else { pstmt.setNull(i, type); }