chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

移除kafka client基于zk的HA机制

Browse Source
This commit is contained in:
mcy
2018-08-24 10:52:04 +08:00
parent b7de1e5070
commit 6b94f396ff
5 changed files with 99 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java
View File

@@ -20,9 +20,9 @@ public class CanalClientConfig {
private String bootstrapServers; private String bootstrapServers;
private List<KafkaTopic> kafkaTopics = new ArrayList<>(); private List<KafkaTopic> kafkaTopics;
private List<CanalInstance> canalInstances = new ArrayList<>(); private List<CanalInstance> canalInstances;
public String getCanalServerHost() { public String getCanalServerHost() {
return canalServerHost; return canalServerHost;

5
client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java
View File

@@ -91,7 +91,7 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
private void process() { private void process() {
while (!running) while (!running)
; ;
ExecutorService executor = Executors.newFixedThreadPool(1); ExecutorService executor = Executors.newSingleThreadExecutor();
final AtomicBoolean executing = new AtomicBoolean(true); final AtomicBoolean executing = new AtomicBoolean(true);
while (running) { while (running) {
try { try {
@@ -142,7 +142,8 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
} }
}); });
while (executing.get()) { // keeping kafka client active // 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端
while (executing.get()) {
connector.ack(); connector.ack();
Thread.sleep(500); Thread.sleep(500);
} }

17
client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java
View File

@@ -43,7 +43,7 @@ public class CanalAdapterLoader {
*/ */
public void init() { public void init() {
// canal instances 和 kafka topics 配置不能同时为空 // canal instances 和 kafka topics 配置不能同时为空
if (canalClientConfig.getCanalInstances().isEmpty() && canalClientConfig.getKafkaTopics().isEmpty()) { if (canalClientConfig.getCanalInstances() == null && canalClientConfig.getKafkaTopics() == null) {
throw new RuntimeException("Blank config property: canalInstances or canalKafkaTopics"); throw new RuntimeException("Blank config property: canalInstances or canalKafkaTopics");
} }
@@ -58,11 +58,13 @@ public class CanalAdapterLoader {
} }
String zkHosts = this.canalClientConfig.getZookeeperHosts(); String zkHosts = this.canalClientConfig.getZookeeperHosts();
if (zkHosts == null && sa == null) { // if (zkHosts == null && sa == null) {
throw new RuntimeException("Blank config property: canalServerHost or zookeeperHosts"); // throw new RuntimeException("Blank config property: canalServerHost or
} // zookeeperHosts");
// }
// 初始化canal-client的适配器 // 初始化canal-client的适配器
if (canalClientConfig.getCanalInstances() != null) {
for (CanalClientConfig.CanalInstance instance : canalClientConfig.getCanalInstances()) { for (CanalClientConfig.CanalInstance instance : canalClientConfig.getCanalInstances()) {
List<List<CanalOuterAdapter>> canalOuterAdapterGroups = new ArrayList<>(); List<List<CanalOuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
@@ -83,8 +85,10 @@ public class CanalAdapterLoader {
worker.start(); worker.start();
logger.info("Start adapter for canal instance: {} succeed", instance.getInstance()); logger.info("Start adapter for canal instance: {} succeed", instance.getInstance());
} }
}
// 初始化canal-client-kafka的适配器 // 初始化canal-client-kafka的适配器
if (canalClientConfig.getKafkaTopics() != null) {
for (CanalClientConfig.KafkaTopic kafkaTopic : canalClientConfig.getKafkaTopics()) { for (CanalClientConfig.KafkaTopic kafkaTopic : canalClientConfig.getKafkaTopics()) {
for (CanalClientConfig.Group group : kafkaTopic.getGroups()) { for (CanalClientConfig.Group group : kafkaTopic.getGroups()) {
List<List<CanalOuterAdapter>> canalOuterAdapterGroups = new ArrayList<>(); List<List<CanalOuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
@@ -98,8 +102,8 @@ public class CanalAdapterLoader {
} }
canalOuterAdapterGroups.add(canalOuterAdapters); canalOuterAdapterGroups.add(canalOuterAdapters);
String zkServers = canalClientConfig.getZookeeperHosts(); // String zkServers = canalClientConfig.getZookeeperHosts();
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(zkServers, CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(zkHosts,
canalClientConfig.getBootstrapServers(), canalClientConfig.getBootstrapServers(),
kafkaTopic.getTopic(), kafkaTopic.getTopic(),
group.getGroupId(), group.getGroupId(),
@@ -111,6 +115,7 @@ public class CanalAdapterLoader {
} }
} }
} }
}
private void loadConnector(CanalOuterAdapterConfiguration config, List<CanalOuterAdapter> canalOutConnectors) { private void loadConnector(CanalOuterAdapterConfiguration config, List<CanalOuterAdapter> canalOutConnectors) {
try { try {

19
client-launcher/src/main/resources/canal-client.yml
View File

@@ -1,6 +1,6 @@
canalServerHost: 127.0.0.1:11111 canalServerHost: 127.0.0.1:11111
#zookeeperHosts: 127.0.0.1:2181 #zookeeperHosts: slave1:2181
#bootstrapServers: kafka1.mytest.com:9092,kafka2.mytest.com:9092 #bootstrapServers: slave1:6667,slave2:6667
canalInstances: canalInstances:
- instance: example - instance: example
@@ -10,13 +10,12 @@ canalInstances:
- name: hbase - name: hbase
hosts: slave1:2181 hosts: slave1:2181
properties: {znodeParent: "/hbase-unsecure"} properties: {znodeParent: "/hbase-unsecure"}
#kafkaTopics: #kafkaTopics:
#- topic: devmysql4308 #- topic: example
# groups: # groups:
# - groupId: devmysql4308_es # - groupId: example_g1
# adapters: # outAdapters:
# - name: es # - name: logger
# hosts: # - name: hbase
# zkHosts: # hosts: slave1:2181
# properties: {clusterName: es-service-test} # properties: {znodeParent: "/hbase-unsecure"}

66
client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java
View File

@@ -10,10 +10,7 @@ import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringDeserializer;
import com.alibaba.otter.canal.client.kafka.running.ClientRunningData; import com.alibaba.otter.canal.client.kafka.running.ClientRunningData;
import com.alibaba.otter.canal.client.kafka.running.ClientRunningListener;
import com.alibaba.otter.canal.client.kafka.running.ClientRunningMonitor;
import com.alibaba.otter.canal.common.utils.AddressUtils; import com.alibaba.otter.canal.common.utils.AddressUtils;
import com.alibaba.otter.canal.common.utils.BooleanMutex;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx; import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.exception.CanalClientException; import com.alibaba.otter.canal.protocol.exception.CanalClientException;
@@ -30,13 +27,13 @@ public class KafkaCanalConnector {
private String topic; private String topic;
private Integer partition; private Integer partition;
private Properties properties; private Properties properties;
private ClientRunningMonitor runningMonitor; // 运行控制 // private ClientRunningMonitor runningMonitor; // 运行控制
// private BooleanMutex mutex = new BooleanMutex(false);
private ZkClientx zkClientx; private ZkClientx zkClientx;
private BooleanMutex mutex = new BooleanMutex(false);
private volatile boolean connected = false; private volatile boolean connected = false;
private volatile boolean running = false; private volatile boolean running = false;
public KafkaCanalConnector(String zkServers, String servers, String topic, Integer partition, String groupId) { public KafkaCanalConnector(String zkServers, String servers, String topic, Integer partition, String groupId){
this.topic = topic; this.topic = topic;
this.partition = partition; this.partition = partition;
@@ -45,7 +42,7 @@ public class KafkaCanalConnector {
properties.put("group.id", groupId); properties.put("group.id", groupId);
properties.put("enable.auto.commit", false); properties.put("enable.auto.commit", false);
properties.put("auto.commit.interval.ms", "1000"); properties.put("auto.commit.interval.ms", "1000");
properties.put("auto.offset.reset", "latest"); //如果没有offset则从最后的offset开始读 properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读
properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置 properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置
properties.put("session.timeout.ms", "30000"); // 默认为30秒 properties.put("session.timeout.ms", "30000"); // 默认为30秒
properties.put("max.poll.records", "1"); // 所以一次只取一条数据 properties.put("max.poll.records", "1"); // 所以一次只取一条数据
@@ -59,19 +56,19 @@ public class KafkaCanalConnector {
clientData.setGroupId(groupId); clientData.setGroupId(groupId);
clientData.setAddress(AddressUtils.getHostIp()); clientData.setAddress(AddressUtils.getHostIp());
runningMonitor = new ClientRunningMonitor(); // runningMonitor = new ClientRunningMonitor();
runningMonitor.setTopic(topic); // runningMonitor.setTopic(topic);
runningMonitor.setZkClient(zkClientx); // runningMonitor.setZkClient(zkClientx);
runningMonitor.setClientData(clientData); // runningMonitor.setClientData(clientData);
runningMonitor.setListener(new ClientRunningListener() { // runningMonitor.setListener(new ClientRunningListener() {
public void processActiveEnter() { // public void processActiveEnter() {
mutex.set(true); // mutex.set(true);
} // }
//
public void processActiveExit() { // public void processActiveExit() {
mutex.set(false); // mutex.set(false);
} // }
}); // });
} }
} }
@@ -96,11 +93,11 @@ public class KafkaCanalConnector {
return; return;
} }
if (runningMonitor != null) { // if (runningMonitor != null) {
if (!runningMonitor.isStart()) { // if (!runningMonitor.isStart()) {
runningMonitor.start(); // runningMonitor.start();
} // }
} // }
connected = true; connected = true;
@@ -116,9 +113,9 @@ public class KafkaCanalConnector {
kafkaConsumer.close(); kafkaConsumer.close();
connected = false; connected = false;
if (runningMonitor.isStart()) { // if (runningMonitor.isStart()) {
runningMonitor.stop(); // runningMonitor.stop();
} // }
} }
private void waitClientRunning() { private void waitClientRunning() {
@@ -129,12 +126,12 @@ public class KafkaCanalConnector {
} }
running = true; running = true;
mutex.get();// 阻塞等待 // mutex.get();// 阻塞等待
} else { } else {
// 单机模式直接设置为running // 单机模式直接设置为running
running = true; running = true;
} }
} catch (InterruptedException e) { } catch (Exception e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
throw new CanalClientException(e); throw new CanalClientException(e);
} }
@@ -142,7 +139,8 @@ public class KafkaCanalConnector {
public boolean checkValid() { public boolean checkValid() {
if (zkClientx != null) { if (zkClientx != null) {
return mutex.state(); // return mutex.state();
return true;
} else { } else {
return true;// 默认都放过 return true;// 默认都放过
} }
@@ -235,9 +233,9 @@ public class KafkaCanalConnector {
public void stopRunning() { public void stopRunning() {
if (running) { if (running) {
running = false; // 设置为非running状态 running = false; // 设置为非running状态
if (!mutex.state()) { // if (!mutex.state()) {
mutex.set(true); // 中断阻塞 // mutex.set(true); // 中断阻塞
} // }
} }
} }
} }