mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
Merge branch 'develop-tmp'
# Conflicts: # kafka-client/pom.xml # kafka/pom.xml
This commit is contained in:
machey
@@ -25,6 +25,29 @@
|
||||
<artifactId>canal.instance.manager</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<!-- kafka -->
|
||||
<dependency>
|
||||
<groupId>org.yaml</groupId>
|
||||
<artifactId>snakeyaml</artifactId>
|
||||
<version>1.17</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka_2.11</artifactId>
|
||||
<version>1.1.1</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<!--kafka_2.11_1.1.1 exclusion掉了netty 的依赖，但CanalServerWithNetty 依赖 netty3,升级kafka至 1.1.1 需要显示加入,否则会启动失败 -->
|
||||
<dependency>
|
||||
<groupId>org.jboss.netty</groupId>
|
||||
<artifactId>netty</artifactId>
|
||||
<version>3.2.2.Final</version>
|
||||
</dependency>
|
||||
|
||||
<!-- test dependency -->
|
||||
<dependency>
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package com.alibaba.otter.canal.kafka;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
import org.apache.kafka.clients.producer.Producer;
|
||||
import org.apache.kafka.clients.producer.ProducerRecord;
|
||||
import org.apache.kafka.common.serialization.StringSerializer;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
|
||||
/**
|
||||
* kafka producer 主操作类
|
||||
*
|
||||
* @author machengyuan 2018-6-11 下午05:30:49
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class CanalKafkaProducer {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
|
||||
|
||||
private Producer<String, Message> producer;
|
||||
|
||||
public void init(KafkaProperties kafkaProperties) {
|
||||
Properties properties = new Properties();
|
||||
properties.put("bootstrap.servers", kafkaProperties.getServers());
|
||||
properties.put("acks", "all");
|
||||
properties.put("retries", kafkaProperties.getRetries());
|
||||
properties.put("batch.size", kafkaProperties.getBatchSize());
|
||||
properties.put("linger.ms", kafkaProperties.getLingerMs());
|
||||
properties.put("buffer.memory", kafkaProperties.getBufferMemory());
|
||||
properties.put("key.serializer", StringSerializer.class.getName());
|
||||
properties.put("value.serializer", MessageSerializer.class.getName());
|
||||
producer = new KafkaProducer<String, Message>(properties);
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
try {
|
||||
logger.info("## stop the kafka producer");
|
||||
producer.close();
|
||||
} catch (Throwable e) {
|
||||
logger.warn("##something goes wrong when stopping kafka producer:", e);
|
||||
} finally {
|
||||
logger.info("## kafka producer is down.");
|
||||
}
|
||||
}
|
||||
|
||||
public void send(KafkaProperties.Topic topic, Message message) throws IOException {
|
||||
// set canal.instance.filter.transaction.entry = true
|
||||
|
||||
// boolean valid = false;
|
||||
// if (message != null) {
|
||||
// if (message.isRaw() && !message.getRawEntries().isEmpty()) {
|
||||
// for (ByteString byteString : message.getRawEntries()) {
|
||||
// CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString);
|
||||
// if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN
|
||||
// && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) {
|
||||
// valid = true;
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// } else if (!message.getEntries().isEmpty()){
|
||||
// for (CanalEntry.Entry entry : message.getEntries()) {
|
||||
// if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN
|
||||
// && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) {
|
||||
// valid = true;
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if (!valid) {
|
||||
// return;
|
||||
// }
|
||||
ProducerRecord<String, Message> record;
|
||||
if (topic.getPartition() != null) {
|
||||
record = new ProducerRecord<String, Message>(topic.getTopic(), topic.getPartition(), null, message);
|
||||
} else {
|
||||
record = new ProducerRecord<String, Message>(topic.getTopic(), message);
|
||||
}
|
||||
producer.send(record);
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("send message to kafka topic: {} \n {}", topic.getTopic(), message.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package com.alibaba.otter.canal.kafka;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.yaml.snakeyaml.Yaml;
|
||||
|
||||
import com.alibaba.otter.canal.kafka.KafkaProperties.CanalDestination;
|
||||
import com.alibaba.otter.canal.kafka.KafkaProperties.Topic;
|
||||
import com.alibaba.otter.canal.protocol.ClientIdentity;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.alibaba.otter.canal.server.CanalServerStarter;
|
||||
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
|
||||
|
||||
/**
|
||||
* kafka 启动类
|
||||
*
|
||||
* @author machengyuan 2018-6-11 下午05:30:49
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class CanalKafkaStarter implements CanalServerStarter {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaStarter.class);
|
||||
|
||||
private static final String CLASSPATH_URL_PREFIX = "classpath:";
|
||||
|
||||
private volatile boolean running = false;
|
||||
|
||||
private ExecutorService executorService;
|
||||
|
||||
private CanalKafkaProducer canalKafkaProducer;
|
||||
|
||||
private KafkaProperties kafkaProperties;
|
||||
|
||||
public void init() {
|
||||
try {
|
||||
logger.info("## load kafka configurations");
|
||||
String conf = System.getProperty("kafka.conf", "classpath:kafka.yml");
|
||||
|
||||
if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
|
||||
conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
|
||||
kafkaProperties = new Yaml().loadAs(CanalKafkaStarter.class.getClassLoader().getResourceAsStream(conf),
|
||||
KafkaProperties.class);
|
||||
} else {
|
||||
kafkaProperties = new Yaml().loadAs(new FileInputStream(conf), KafkaProperties.class);
|
||||
}
|
||||
|
||||
// 初始化 kafka producer
|
||||
canalKafkaProducer = new CanalKafkaProducer();
|
||||
canalKafkaProducer.init(kafkaProperties);
|
||||
// set filterTransactionEntry
|
||||
if (kafkaProperties.isFilterTransactionEntry()) {
|
||||
System.setProperty("canal.instance.filter.transaction.entry", "true");
|
||||
}
|
||||
// 对应每个instance启动一个worker线程
|
||||
List<CanalDestination> destinations = kafkaProperties.getCanalDestinations();
|
||||
|
||||
executorService = Executors.newFixedThreadPool(destinations.size());
|
||||
|
||||
logger.info("## start the kafka workers.");
|
||||
for (final CanalDestination destination : destinations) {
|
||||
executorService.execute(new Runnable() {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
worker(destination);
|
||||
}
|
||||
});
|
||||
}
|
||||
running = true;
|
||||
logger.info("## the kafka workers is running now ......");
|
||||
Runtime.getRuntime().addShutdownHook(new Thread() {
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
logger.info("## stop the kafka workers");
|
||||
running = false;
|
||||
executorService.shutdown();
|
||||
canalKafkaProducer.stop();
|
||||
} catch (Throwable e) {
|
||||
logger.warn("##something goes wrong when stopping kafka workers:", e);
|
||||
} finally {
|
||||
logger.info("## canal kafka is down.");
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
} catch (Throwable e) {
|
||||
logger.error("## Something goes wrong when starting up the canal kafka workers:", e);
|
||||
System.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
private void worker(CanalDestination destination) {
|
||||
while (!running)
|
||||
;
|
||||
logger.info("## start the canal consumer: {}.", destination.getCanalDestination());
|
||||
CanalServerWithEmbedded server = CanalServerWithEmbedded.instance();
|
||||
ClientIdentity clientIdentity = new ClientIdentity(destination.getCanalDestination(), (short) 1001, "");
|
||||
while (running) {
|
||||
try {
|
||||
if (!server.getCanalInstances().containsKey(clientIdentity.getDestination())) {
|
||||
try {
|
||||
Thread.sleep(3000);
|
||||
} catch (InterruptedException e) {
|
||||
// ignore
|
||||
}
|
||||
continue;
|
||||
}
|
||||
server.subscribe(clientIdentity);
|
||||
logger.info("## the canal consumer {} is running now ......", destination.getCanalDestination());
|
||||
|
||||
while (running) {
|
||||
Message message = server.getWithoutAck(clientIdentity, kafkaProperties.getCanalBatchSize()); // 获取指定数量的数据
|
||||
long batchId = message.getId();
|
||||
try {
|
||||
int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
|
||||
if (batchId != -1 && size != 0) {
|
||||
if (!StringUtils.isEmpty(destination.getTopic())) {
|
||||
Topic topic = new Topic();
|
||||
topic.setTopic(destination.getTopic());
|
||||
topic.setPartition(destination.getPartition());
|
||||
destination.getTopics().add(topic);
|
||||
}
|
||||
for (Topic topic : destination.getTopics()) {
|
||||
canalKafkaProducer.send(topic, message); // 发送message到所有topic
|
||||
}
|
||||
}
|
||||
|
||||
if (batchId != -1) {
|
||||
server.ack(clientIdentity, batchId); // 提交确认
|
||||
}
|
||||
} catch (Exception e) {
|
||||
server.rollback(clientIdentity);
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("process error!", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package com.alibaba.otter.canal.kafka;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* kafka 配置项
|
||||
*
|
||||
* @author machengyuan 2018-6-11 下午05:30:49
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class KafkaProperties {
|
||||
|
||||
private String servers = "localhost:6667";
|
||||
private int retries = 0;
|
||||
private int batchSize = 16384;
|
||||
private int lingerMs = 1;
|
||||
private long bufferMemory = 33554432L;
|
||||
private boolean filterTransactionEntry = true;
|
||||
private int canalBatchSize = 5;
|
||||
|
||||
private List<CanalDestination> canalDestinations = new ArrayList<CanalDestination>();
|
||||
|
||||
public static class CanalDestination {
|
||||
|
||||
private String canalDestination;
|
||||
private String topic;
|
||||
private Integer partition;
|
||||
private Set<Topic> topics = new HashSet<Topic>();
|
||||
|
||||
public String getCanalDestination() {
|
||||
return canalDestination;
|
||||
}
|
||||
|
||||
public void setCanalDestination(String canalDestination) {
|
||||
this.canalDestination = canalDestination;
|
||||
}
|
||||
|
||||
public String getTopic() {
|
||||
return topic;
|
||||
}
|
||||
|
||||
public void setTopic(String topic) {
|
||||
this.topic = topic;
|
||||
}
|
||||
|
||||
public Integer getPartition() {
|
||||
return partition;
|
||||
}
|
||||
|
||||
public void setPartition(Integer partition) {
|
||||
this.partition = partition;
|
||||
}
|
||||
|
||||
public Set<Topic> getTopics() {
|
||||
return topics;
|
||||
}
|
||||
|
||||
public void setTopics(Set<Topic> topics) {
|
||||
this.topics = topics;
|
||||
}
|
||||
}
|
||||
|
||||
public static class Topic {
|
||||
|
||||
private String topic;
|
||||
private Integer partition;
|
||||
|
||||
public String getTopic() {
|
||||
return topic;
|
||||
}
|
||||
|
||||
public void setTopic(String topic) {
|
||||
this.topic = topic;
|
||||
}
|
||||
|
||||
public Integer getPartition() {
|
||||
return partition;
|
||||
}
|
||||
|
||||
public void setPartition(Integer partition) {
|
||||
this.partition = partition;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
|
||||
Topic topic1 = (Topic) o;
|
||||
|
||||
if (topic != null ? !topic.equals(topic1.topic) : topic1.topic != null) return false;
|
||||
return partition != null ? partition.equals(topic1.partition) : topic1.partition == null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = topic != null ? topic.hashCode() : 0;
|
||||
result = 31 * result + (partition != null ? partition.hashCode() : 0);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
public String getServers() {
|
||||
return servers;
|
||||
}
|
||||
|
||||
public void setServers(String servers) {
|
||||
this.servers = servers;
|
||||
}
|
||||
|
||||
public int getRetries() {
|
||||
return retries;
|
||||
}
|
||||
|
||||
public void setRetries(int retries) {
|
||||
this.retries = retries;
|
||||
}
|
||||
|
||||
public int getBatchSize() {
|
||||
return batchSize;
|
||||
}
|
||||
|
||||
public void setBatchSize(int batchSize) {
|
||||
this.batchSize = batchSize;
|
||||
}
|
||||
|
||||
public int getLingerMs() {
|
||||
return lingerMs;
|
||||
}
|
||||
|
||||
public void setLingerMs(int lingerMs) {
|
||||
this.lingerMs = lingerMs;
|
||||
}
|
||||
|
||||
public long getBufferMemory() {
|
||||
return bufferMemory;
|
||||
}
|
||||
|
||||
public void setBufferMemory(long bufferMemory) {
|
||||
this.bufferMemory = bufferMemory;
|
||||
}
|
||||
|
||||
public int getCanalBatchSize() {
|
||||
return canalBatchSize;
|
||||
}
|
||||
|
||||
public void setCanalBatchSize(int canalBatchSize) {
|
||||
this.canalBatchSize = canalBatchSize;
|
||||
}
|
||||
|
||||
public List<CanalDestination> getCanalDestinations() {
|
||||
return canalDestinations;
|
||||
}
|
||||
|
||||
public void setCanalDestinations(List<CanalDestination> canalDestinations) {
|
||||
this.canalDestinations = canalDestinations;
|
||||
}
|
||||
|
||||
public boolean isFilterTransactionEntry() {
|
||||
return filterTransactionEntry;
|
||||
}
|
||||
|
||||
public void setFilterTransactionEntry(boolean filterTransactionEntry) {
|
||||
this.filterTransactionEntry = filterTransactionEntry;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package com.alibaba.otter.canal.kafka;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.kafka.common.errors.SerializationException;
|
||||
import org.apache.kafka.common.serialization.Serializer;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import com.alibaba.otter.canal.protocol.CanalEntry;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.google.protobuf.ByteString;
|
||||
import com.google.protobuf.CodedOutputStream;
|
||||
import com.google.protobuf.WireFormat;
|
||||
|
||||
/**
|
||||
* Kafka Message类的序列化
|
||||
*
|
||||
* @author machengyuan 2018-6-11 下午05:30:49
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class MessageSerializer implements Serializer<Message> {
|
||||
|
||||
@Override
|
||||
public void configure(Map<String, ?> configs, boolean isKey) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] serialize(String topic, Message data) {
|
||||
try {
|
||||
if (data != null) {
|
||||
if (data.getId() != -1) {
|
||||
if (data.isRaw() && !CollectionUtils.isEmpty(data.getRawEntries())) {
|
||||
// for performance
|
||||
List<ByteString> rowEntries = data.getRawEntries();
|
||||
// message size
|
||||
int messageSize = 0;
|
||||
messageSize += CodedOutputStream.computeInt64Size(1, data.getId());
|
||||
|
||||
int dataSize = 0;
|
||||
for (int i = 0; i < rowEntries.size(); i++) {
|
||||
dataSize += CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i));
|
||||
}
|
||||
messageSize += dataSize;
|
||||
messageSize += 1 * rowEntries.size();
|
||||
// packet size
|
||||
int size = 0;
|
||||
size += CodedOutputStream.computeEnumSize(3,
|
||||
PacketType.MESSAGES.getNumber());
|
||||
size += CodedOutputStream.computeTagSize(5)
|
||||
+ CodedOutputStream.computeRawVarint32Size(messageSize)
|
||||
+ messageSize;
|
||||
// build data
|
||||
byte[] body = new byte[size];
|
||||
CodedOutputStream output = CodedOutputStream.newInstance(body);
|
||||
output.writeEnum(3, PacketType.MESSAGES.getNumber());
|
||||
|
||||
output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
|
||||
output.writeRawVarint32(messageSize);
|
||||
// message
|
||||
output.writeInt64(1, data.getId());
|
||||
for (int i = 0; i < rowEntries.size(); i++) {
|
||||
output.writeBytes(2, rowEntries.get(i));
|
||||
}
|
||||
output.checkNoSpaceLeft();
|
||||
return body;
|
||||
} else if (!CollectionUtils.isEmpty(data.getEntries())) {
|
||||
CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
|
||||
for (CanalEntry.Entry entry : data.getEntries()) {
|
||||
messageBuilder.addMessages(entry.toByteString());
|
||||
}
|
||||
|
||||
CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
|
||||
packetBuilder.setType(PacketType.MESSAGES);
|
||||
packetBuilder.setBody(messageBuilder.build().toByteString());
|
||||
return packetBuilder.build().toByteArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new SerializationException("Error when serializing message to byte[] ");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
// nothing to do
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.alibaba.otter.canal.server;
|
||||
|
||||
/**
|
||||
* 外部服务如Kafka, RocketMQ启动接口
|
||||
*
|
||||
* @author machengyuan 2018-8-23 下午05:20:29
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public interface CanalServerStarter {
|
||||
|
||||
void init();
|
||||
}
|
||||
Reference in New Issue
Block a user