chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed rocketmq ak/sk

Browse Source
This commit is contained in:
agapple
2018-11-23 09:56:55 +08:00
parent aa89453bf6
commit ec235f3efd
9 changed files with 56 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java
View File

@@ -30,6 +30,10 @@ public class CanalClientConfig {
private Long timeout; private Long timeout;
// 模式 tcp kafka rocketMQ // 模式 tcp kafka rocketMQ
private String mode = "tcp"; private String mode = "tcp";
// aliyun ak/sk
private String accessKey;
private String secretKey;
// canal adapters 配置 // canal adapters 配置
private List<CanalAdapter> canalAdapters; private List<CanalAdapter> canalAdapters;
@@ -105,6 +109,22 @@ public class CanalClientConfig {
this.mode = mode; this.mode = mode;
} }
public String getAccessKey() {
return accessKey;
}
public void setAccessKey(String accessKey) {
this.accessKey = accessKey;
}
public String getSecretKey() {
return secretKey;
}
public void setSecretKey(String secretKey) {
this.secretKey = secretKey;
}
public List<CanalAdapter> getCanalAdapters() { public List<CanalAdapter> getCanalAdapters() {
return canalAdapters; return canalAdapters;
} }

46
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java
View File

@@ -27,8 +27,7 @@ import org.slf4j.LoggerFactory;
*/ */
public class ExtensionLoader<T> { public class ExtensionLoader<T> {
private static final Logger logger = LoggerFactory private static final Logger logger = LoggerFactory.getLogger(ExtensionLoader.class);
.getLogger(ExtensionLoader.class);
private static final String SERVICES_DIRECTORY = "META-INF/services/"; private static final String SERVICES_DIRECTORY = "META-INF/services/";
@@ -36,8 +35,7 @@ public class ExtensionLoader<T> {
private static final String DEFAULT_CLASSLOADER_POLICY = "internal"; private static final String DEFAULT_CLASSLOADER_POLICY = "internal";
private static final Pattern NAME_SEPARATOR = Pattern private static final Pattern NAME_SEPARATOR = Pattern.compile("\\s*[,]+\\s*");
.compile("\\s*[,]+\\s*");
private static final ConcurrentMap<Class<?>, ExtensionLoader<?>> EXTENSION_LOADERS = new ConcurrentHashMap<>(); private static final ConcurrentMap<Class<?>, ExtensionLoader<?>> EXTENSION_LOADERS = new ConcurrentHashMap<>();
@@ -173,8 +171,7 @@ public class ExtensionLoader<T> {
return instance; return instance;
} catch (Throwable t) { } catch (Throwable t) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+ ") could not be instantiated: " + t.getMessage(), + ") could not be instantiated: " + t.getMessage(), t);
t);
} }
} }
@@ -194,8 +191,7 @@ public class ExtensionLoader<T> {
return instance; return instance;
} catch (Throwable t) { } catch (Throwable t) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+ ") could not be instantiated: " + t.getMessage(), + ") could not be instantiated: " + t.getMessage(), t);
t);
} }
} }
@@ -334,10 +330,12 @@ public class ExtensionLoader<T> {
// Class.forName(line, true, // Class.forName(line, true,
// classLoader); // classLoader);
if (!type.isAssignableFrom(clazz)) { if (!type.isAssignableFrom(clazz)) {
throw new IllegalStateException( throw new IllegalStateException("Error when load extension class(interface: "
"Error when load extension class(interface: " + type + type
+ ", class line: " + clazz.getName() + ", class line: "
+ "), class " + clazz.getName() + clazz.getName()
+ "), class "
+ clazz.getName()
+ "is not subtype of interface."); + "is not subtype of interface.");
} else { } else {
try { try {
@@ -355,9 +353,9 @@ public class ExtensionLoader<T> {
extensionClasses.put(n, clazz); extensionClasses.put(n, clazz);
} else if (c != clazz) { } else if (c != clazz) {
cachedNames.remove(clazz); cachedNames.remove(clazz);
throw new IllegalStateException( throw new IllegalStateException("Duplicate extension "
"Duplicate extension " + type.getName() + " name " + type.getName()
+ n + " on " + " name " + n + " on "
+ c.getName() + " and " + c.getName() + " and "
+ clazz.getName()); + clazz.getName());
} }
@@ -367,9 +365,12 @@ public class ExtensionLoader<T> {
} }
} }
} catch (Throwable t) { } catch (Throwable t) {
IllegalStateException e = new IllegalStateException( IllegalStateException e = new IllegalStateException("Failed to load extension class(interface: "
"Failed to load extension class(interface: " + type + ", class line: " + type
+ line + ") in " + url + ", class line: "
+ line
+ ") in "
+ url
+ ", cause: " + ", cause: "
+ t.getMessage(), + t.getMessage(),
t); t);
@@ -384,18 +385,17 @@ public class ExtensionLoader<T> {
} }
} catch (Throwable t) { } catch (Throwable t) {
logger.error("Exception when load extension class(interface: " + type + ", class file: " + url logger.error("Exception when load extension class(interface: " + type + ", class file: " + url
+ ") in " + url, + ") in " + url, t);
t);
} }
} // end of while urls } // end of while urls
} }
} catch (Throwable t) { } catch (Throwable t) {
logger.error( logger.error("Exception when load extension class(interface: " + type + ", description file: " + fileName
"Exception when load extension class(interface: " + type + ", description file: " + fileName + ").", + ").", t);
t);
} }
} }
@SuppressWarnings("unused")
private static ClassLoader findClassLoader() { private static ClassLoader findClassLoader() {
return ExtensionLoader.class.getClassLoader(); return ExtensionLoader.class.getClassLoader();
} }

1
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java
View File

@@ -3,7 +3,6 @@ package com.alibaba.otter.canal.adapter.launcher;
import org.springframework.boot.Banner; import org.springframework.boot.Banner;
import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
/** /**
* 启动入口 * 启动入口

3
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java
View File

@@ -64,11 +64,12 @@ public class SyncSwitch {
} }
} }
@SuppressWarnings("resource")
private synchronized void startListen(String destination, BooleanMutex mutex) { private synchronized void startListen(String destination, BooleanMutex mutex) {
try { try {
String path = SYN_SWITCH_ZK_NODE + destination; String path = SYN_SWITCH_ZK_NODE + destination;
CuratorFramework curator = curatorClient.getCurator(); CuratorFramework curator = curatorClient.getCurator();
final NodeCache nodeCache = new NodeCache(curator, path); NodeCache nodeCache = new NodeCache(curator, path);
nodeCache.start(); nodeCache.start();
nodeCache.getListenable().addListener(() -> initMutex(curator, destination, mutex)); nodeCache.getListenable().addListener(() -> initMutex(curator, destination, mutex));
} catch (Exception e) { } catch (Exception e) {

1
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java
View File

@@ -47,6 +47,7 @@ public class AdapterCanalConfig extends CanalClientConfig {
return srcDataSources; return srcDataSources;
} }
@SuppressWarnings("resource")
public void setSrcDataSources(Map<String, DatasourceConfig> srcDataSources) { public void setSrcDataSources(Map<String, DatasourceConfig> srcDataSources) {
this.srcDataSources = srcDataSources; this.srcDataSources = srcDataSources;

3
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java
View File

@@ -44,8 +44,7 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
while (!running) while (!running)
; ;
ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
int retry = canalClientConfig.getRetries() == null int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) { while (running) {

2
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -122,6 +122,8 @@ public class CanalAdapterLoader {
canalAdapter.getInstance(), canalAdapter.getInstance(),
group.getGroupId(), group.getGroupId(),
canalOuterAdapterGroups, canalOuterAdapterGroups,
canalClientConfig.getAccessKey(),
canalClientConfig.getSecretKey(),
canalClientConfig.getFlatMessage()); canalClientConfig.getFlatMessage());
canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
rocketMQWorker.start(); rocketMQWorker.start();

14
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java
View File

@@ -3,16 +3,12 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List; import java.util.List;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector; import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
/** /**
* rocketmq对应的client适配器工作线程 * rocketmq对应的client适配器工作线程
@@ -26,13 +22,14 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
private boolean flatMessage; private boolean flatMessage;
public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic, public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic,
String groupId, List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){ String groupId, List<List<OuterAdapter>> canalOuterAdapters, String accessKey,
String secretKey, boolean flatMessage){
super(canalOuterAdapters); super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig; this.canalClientConfig = canalClientConfig;
this.topic = topic; this.topic = topic;
this.flatMessage = flatMessage; this.flatMessage = flatMessage;
this.canalDestination = topic; this.canalDestination = topic;
this.connector = new RocketMQCanalConnector(nameServers, topic, groupId, flatMessage); this.connector = new RocketMQCanalConnector(nameServers, topic, groupId, accessKey, secretKey, flatMessage);
logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId); logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
} }
@@ -42,8 +39,7 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
; ;
ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
int retry = canalClientConfig.getRetries() == null int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) { while (running) {

2
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -21,6 +21,8 @@ canal.conf:
syncBatchSize: 1000 syncBatchSize: 1000
retries: 0 retries: 0
timeout: timeout:
accessKey:
secretKey:
mode: tcp # kafka rocketMQ mode: tcp # kafka rocketMQ
# srcDataSources: # srcDataSources:
# defaultDS: # defaultDS: