mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
fixed test
This commit is contained in:
@@ -13,7 +13,6 @@ import org.apache.kafka.common.TopicPartition;
|
|||||||
import org.apache.kafka.common.serialization.StringDeserializer;
|
import org.apache.kafka.common.serialization.StringDeserializer;
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSON;
|
import com.alibaba.fastjson.JSON;
|
||||||
import com.alibaba.otter.canal.client.CanalMessageDeserializer;
|
|
||||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||||
import com.alibaba.otter.canal.protocol.Message;
|
import com.alibaba.otter.canal.protocol.Message;
|
||||||
|
|
||||||
@@ -50,7 +49,7 @@ public class KafkaCanalConnector {
|
|||||||
properties.put("max.poll.records", "100");
|
properties.put("max.poll.records", "100");
|
||||||
properties.put("key.deserializer", StringDeserializer.class.getName());
|
properties.put("key.deserializer", StringDeserializer.class.getName());
|
||||||
if (!flatMessage) {
|
if (!flatMessage) {
|
||||||
properties.put("value.deserializer", CanalMessageDeserializer.class.getName());
|
properties.put("value.deserializer", MessageDeserializer.class.getName());
|
||||||
} else {
|
} else {
|
||||||
properties.put("value.deserializer", StringDeserializer.class.getName());
|
properties.put("value.deserializer", StringDeserializer.class.getName());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
package com.alibaba.otter.canal.client.kafka;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.kafka.common.serialization.Deserializer;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.client.CanalMessageDeserializer;
|
||||||
|
import com.alibaba.otter.canal.protocol.Message;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Kafka Message类的反序列化
|
||||||
|
*
|
||||||
|
* @author machengyuan @ 2018-6-12
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class MessageDeserializer implements Deserializer<Message> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void configure(Map<String, ?> configs, boolean isKey) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Message deserialize(String topic1, byte[] data) {
|
||||||
|
return CanalMessageDeserializer.deserializer(data);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
// nothing to do
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1082,10 +1082,10 @@ public final class RowsLogBuffer {
|
|||||||
buffer.fillBytes(binary, 0, len);
|
buffer.fillBytes(binary, 0, len);
|
||||||
|
|
||||||
/* Warning unsupport cloumn type */
|
/* Warning unsupport cloumn type */
|
||||||
logger.warn(String.format("!! Unsupport column type MYSQL_TYPE_GEOMETRY: meta=%d (%04X), len = %d",
|
// logger.warn(String.format("!! Unsupport column type MYSQL_TYPE_GEOMETRY: meta=%d (%04X), len = %d",
|
||||||
meta,
|
// meta,
|
||||||
meta,
|
// meta,
|
||||||
len));
|
// len));
|
||||||
javaType = Types.BINARY;
|
javaType = Types.BINARY;
|
||||||
value = binary;
|
value = binary;
|
||||||
length = len;
|
length = len;
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
package com.alibaba.otter.canal.example;
|
package com.alibaba.otter.canal.example;
|
||||||
|
|
||||||
import java.net.InetSocketAddress;
|
import java.net.InetSocketAddress;
|
||||||
import java.util.concurrent.ArrayBlockingQueue;
|
import java.util.concurrent.ArrayBlockingQueue;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.CanalConnector;
|
import com.alibaba.otter.canal.client.CanalConnector;
|
||||||
import com.alibaba.otter.canal.client.CanalConnectors;
|
import com.alibaba.otter.canal.client.CanalConnectors;
|
||||||
@@ -44,7 +46,7 @@ public class SimpleCanalClientPermanceTest {
|
|||||||
connector.connect();
|
connector.connect();
|
||||||
connector.subscribe();
|
connector.subscribe();
|
||||||
while (true) {
|
while (true) {
|
||||||
Message message = connector.getWithoutAck(batchSize);
|
Message message = connector.getWithoutAck(batchSize, 100L, TimeUnit.MILLISECONDS);
|
||||||
long batchId = message.getId();
|
long batchId = message.getId();
|
||||||
int size = message.getRawEntries().size();
|
int size = message.getRawEntries().size();
|
||||||
sum += size;
|
sum += size;
|
||||||
@@ -53,11 +55,13 @@ public class SimpleCanalClientPermanceTest {
|
|||||||
queue.add(batchId);
|
queue.add(batchId);
|
||||||
if (count % 10 == 0) {
|
if (count % 10 == 0) {
|
||||||
end = System.currentTimeMillis();
|
end = System.currentTimeMillis();
|
||||||
long tps = (perSum * 1000) / (end - start);
|
if (end - start != 0) {
|
||||||
System.out.println(" total : " + sum + " , current : " + perSum + " , cost : " + (end - start)
|
long tps = (perSum * 1000) / (end - start);
|
||||||
+ " , tps : " + tps);
|
System.out.println(" total : " + sum + " , current : " + perSum + " , cost : " + (end - start)
|
||||||
start = end;
|
+ " , tps : " + tps);
|
||||||
perSum = 0;
|
start = end;
|
||||||
|
perSum = 0;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ public class SessionHandler extends SimpleChannelHandler {
|
|||||||
this.embeddedServer = embeddedServer;
|
this.embeddedServer = embeddedServer;
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings({ "deprecation", "deprecation" })
|
@SuppressWarnings({ "deprecation" })
|
||||||
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
|
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
|
||||||
logger.info("message receives in session handler...");
|
logger.info("message receives in session handler...");
|
||||||
long start = System.nanoTime();
|
long start = System.nanoTime();
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ public class ProtocolTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("deprecation")
|
||||||
private byte[] buildData(Message message) throws IOException {
|
private byte[] buildData(Message message) throws IOException {
|
||||||
List<ByteString> rowEntries = message.getRawEntries();
|
List<ByteString> rowEntries = message.getRawEntries();
|
||||||
// message size
|
// message size
|
||||||
|
|||||||
Reference in New Issue
Block a user