chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue #287 , support chinese & empty string valuewq

Browse Source
This commit is contained in:
agapple
2017-03-28 00:25:11 +08:00
parent 2eb05b5934
commit 73627df082
2 changed files with 23 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java
View File

@@ -52,23 +52,24 @@ public class JsonConversion {
public static final int VALUE_ENTRY_SIZE_SMALL = (1 + SMALL_OFFSET_SIZE); public static final int VALUE_ENTRY_SIZE_SMALL = (1 + SMALL_OFFSET_SIZE);
public static final int VALUE_ENTRY_SIZE_LARGE = (1 + LARGE_OFFSET_SIZE); public static final int VALUE_ENTRY_SIZE_LARGE = (1 + LARGE_OFFSET_SIZE);
public static Json_Value parse_value(int type, LogBuffer buffer, long len) { public static Json_Value parse_value(int type, LogBuffer buffer, long len, String charsetName) {
buffer = buffer.duplicate(buffer.position(), (int) len); buffer = buffer.duplicate(buffer.position(), (int) len);
switch (type) { switch (type) {
case JSONB_TYPE_SMALL_OBJECT: case JSONB_TYPE_SMALL_OBJECT:
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false); return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false, charsetName);
case JSONB_TYPE_LARGE_OBJECT: case JSONB_TYPE_LARGE_OBJECT:
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true); return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true, charsetName);
case JSONB_TYPE_SMALL_ARRAY: case JSONB_TYPE_SMALL_ARRAY:
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false); return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false, charsetName);
case JSONB_TYPE_LARGE_ARRAY: case JSONB_TYPE_LARGE_ARRAY:
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true); return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true, charsetName);
default: default:
return parse_scalar(type, buffer, len); return parse_scalar(type, buffer, len, charsetName);
} }
} }
private static Json_Value parse_array_or_object(Json_enum_type type, LogBuffer buffer, long len, boolean large) { private static Json_Value parse_array_or_object(Json_enum_type type, LogBuffer buffer, long len, boolean large,
String charsetName) {
long offset_size = large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE; long offset_size = large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
if (len < 2 * offset_size) { if (len < 2 * offset_size) {
throw new IllegalArgumentException("illegal json data"); throw new IllegalArgumentException("illegal json data");
@@ -95,7 +96,7 @@ public class JsonConversion {
return large ? buffer.getUint32() : buffer.getUint16(); return large ? buffer.getUint32() : buffer.getUint16();
} }
private static Json_Value parse_scalar(int type, LogBuffer buffer, long len) { private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, String charsetName) {
switch (type) { switch (type) {
case JSONB_TYPE_LITERAL: case JSONB_TYPE_LITERAL:
/* purecov: inspected */ /* purecov: inspected */
@@ -146,12 +147,12 @@ public class JsonConversion {
} }
} }
if (str_len == 0 || len < n + str_len) { if (len < n + str_len) {
throw new IllegalArgumentException("illegal json data"); throw new IllegalArgumentException("illegal json data");
} }
return new Json_Value(Json_enum_type.STRING, buffer.rewind() return new Json_Value(Json_enum_type.STRING, buffer.rewind()
.forward((int) n) .forward((int) n)
.getFixString((int) str_len)); .getFixString((int) str_len, charsetName));
case JSONB_TYPE_OPAQUE: case JSONB_TYPE_OPAQUE:
/* /*
* There should always be at least one byte, which tells the * There should always be at least one byte, which tells the
@@ -237,7 +238,7 @@ public class JsonConversion {
this.m_large = large; this.m_large = large;
} }
public String key(int i) { public String key(int i, String charsetName) {
m_data.rewind(); m_data.rewind();
int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE; int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL; int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
@@ -250,10 +251,10 @@ public class JsonConversion {
// entry, always two // entry, always two
// bytes. // bytes.
long key_length = m_data.getUint16(); long key_length = m_data.getUint16();
return m_data.rewind().forward((int) key_offset).getFixString((int) key_length); return m_data.rewind().forward((int) key_offset).getFixString((int) key_length, charsetName);
} }
public Json_Value element(int i) { public Json_Value element(int i, String charsetName) {
m_data.rewind(); m_data.rewind();
int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE; int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL; int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
@@ -266,13 +267,13 @@ public class JsonConversion {
int type = m_data.forward(entry_offset).getUint8(); int type = m_data.forward(entry_offset).getUint8();
if (type == JSONB_TYPE_INT16 || type == JSONB_TYPE_UINT16 || type == JSONB_TYPE_LITERAL if (type == JSONB_TYPE_INT16 || type == JSONB_TYPE_UINT16 || type == JSONB_TYPE_LITERAL
|| (m_large && (type == JSONB_TYPE_INT32 || type == JSONB_TYPE_UINT32))) { || (m_large && (type == JSONB_TYPE_INT32 || type == JSONB_TYPE_UINT32))) {
return parse_scalar(type, m_data, value_entry_size - 1); return parse_scalar(type, m_data, value_entry_size - 1, charsetName);
} }
int value_offset = (int) read_offset_or_size(m_data, m_large); int value_offset = (int) read_offset_or_size(m_data, m_large);
return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset); return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset, charsetName);
} }
public StringBuilder toJsonString(StringBuilder buf) { public StringBuilder toJsonString(StringBuilder buf, String charsetName) {
switch (m_type) { switch (m_type) {
case OBJECT: case OBJECT:
buf.append("{"); buf.append("{");
@@ -280,9 +281,9 @@ public class JsonConversion {
if (i > 0) { if (i > 0) {
buf.append(", "); buf.append(", ");
} }
buf.append('"').append(key(i)).append('"'); buf.append('"').append(key(i, charsetName)).append('"');
buf.append(": "); buf.append(": ");
element(i).toJsonString(buf); element(i, charsetName).toJsonString(buf, charsetName);
} }
buf.append("}"); buf.append("}");
break; break;
@@ -292,7 +293,7 @@ public class JsonConversion {
if (i > 0) { if (i > 0) {
buf.append(", "); buf.append(", ");
} }
element(i).toJsonString(buf); element(i, charsetName).toJsonString(buf, charsetName);
} }
buf.append("]"); buf.append("]");
break; break;
@@ -361,7 +362,7 @@ public class JsonConversion {
} }
buf.append('"').append(text).append('"'); buf.append('"').append(text).append('"');
} else { } else {
text = m_data.getFixString((int) m_length); text = m_data.getFixString((int) m_length, charsetName);
buf.append('"').append(text).append('"'); buf.append('"').append(text).append('"');
} }

4
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java
View File

@@ -952,9 +952,9 @@ public final class RowsLogBuffer {
len = buffer.getUint16(); len = buffer.getUint16();
buffer.forward(meta - 2); buffer.forward(meta - 2);
int position = buffer.position(); int position = buffer.position();
Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(), buffer, len - 1); Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(), buffer, len - 1, charsetName);
StringBuilder builder = new StringBuilder(); StringBuilder builder = new StringBuilder();
jsonValue.toJsonString(builder); jsonValue.toJsonString(builder, charsetName);
value = builder.toString(); value = builder.toString();
buffer.position(position + len); buffer.position(position + len);
// byte[] binary = new byte[len]; // byte[] binary = new byte[len];