chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

= cold cleanup

Browse Source 
* using lambda if possible (we are already building in Java8)
  * using diamond when new generic instance
  * remove used improt
This commit is contained in:
zavakid
2020-09-17 10:48:15 +08:00
parent df60eda679
commit e7658879bc
133 changed files with 990 additions and 1558 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/common/Threads.java
View File

@@ -12,6 +12,6 @@ public class Threads {
pool, pool,
0L, 0L,
TimeUnit.MILLISECONDS, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<Runnable>(pool * 20), new ArrayBlockingQueue<>(pool * 20),
DaemonThreadFactory.daemonThreadFactory); DaemonThreadFactory.daemonThreadFactory);
} }

2
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/Pager.java
View File

@@ -9,7 +9,7 @@ public class Pager<T> implements Serializable {
private static final long serialVersionUID = -986577815091763517L; private static final long serialVersionUID = -986577815091763517L;
private Long count = 0L; private Long count = 0L;
private List<T> items = new ArrayList<T>(); private List<T> items = new ArrayList<>();
private Integer page = 1; private Integer page = 1;
private Integer size = 20; private Integer size = 20;
private Long offset = 0L; private Long offset = 0L;

2
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java
View File

@@ -140,7 +140,7 @@ public class YmlConfigBinder {
MutablePropertySources propertySources = new MutablePropertySources(); MutablePropertySources propertySources = new MutablePropertySources();
propertySources.addFirst(propertySource); propertySources.addFirst(propertySource);
PropertiesConfigurationFactory<Object> factory = new PropertiesConfigurationFactory<Object>(target); PropertiesConfigurationFactory<Object> factory = new PropertiesConfigurationFactory<>(target);
factory.setPropertySources(propertySources); factory.setPropertySources(propertySources);
factory.setIgnoreInvalidFields(true); factory.setIgnoreInvalidFields(true);
factory.setIgnoreUnknownFields(true); factory.setIgnoreUnknownFields(true);

2
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java
View File

@@ -25,7 +25,7 @@ class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher
} }
protected DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, String... names){ protected DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, String... names){
this(delimiters, ignoreCase, new HashSet<String>(Arrays.asList(names))); this(delimiters, ignoreCase, new HashSet<>(Arrays.asList(names)));
} }
DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, Set<String> names){ DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, Set<String> names){

4
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java
View File

@@ -277,7 +277,7 @@ public class PropertiesConfigurationFactory<T> implements FactoryBean<T>, Applic
} }
private Set<String> getNames(Iterable<String> prefixes) { private Set<String> getNames(Iterable<String> prefixes) {
Set<String> names = new LinkedHashSet<String>(); Set<String> names = new LinkedHashSet<>();
if (this.target != null) { if (this.target != null) {
PropertyDescriptor[] descriptors = BeanUtils.getPropertyDescriptors(this.target.getClass()); PropertyDescriptor[] descriptors = BeanUtils.getPropertyDescriptors(this.target.getClass());
for (PropertyDescriptor descriptor : descriptors) { for (PropertyDescriptor descriptor : descriptors) {
@@ -318,7 +318,7 @@ public class PropertiesConfigurationFactory<T> implements FactoryBean<T>, Applic
// We can filter properties to those starting with the target name, but // We can filter properties to those starting with the target name, but
// we can't do a complete filter since we need to trigger the // we can't do a complete filter since we need to trigger the
// unknown fields check // unknown fields check
Set<String> relaxedNames = new HashSet<String>(); Set<String> relaxedNames = new HashSet<>();
for (String relaxedTargetName : relaxedTargetNames) { for (String relaxedTargetName : relaxedTargetNames) {
relaxedNames.add(relaxedTargetName); relaxedNames.add(relaxedTargetName);
} }

18
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java
View File

@@ -9,23 +9,9 @@ package com.alibaba.otter.canal.client.adapter.config.bind;
*/ */
interface PropertyNamePatternsMatcher { interface PropertyNamePatternsMatcher {
PropertyNamePatternsMatcher ALL = new PropertyNamePatternsMatcher() { PropertyNamePatternsMatcher ALL = propertyName -> true;
@Override PropertyNamePatternsMatcher NONE = propertyName -> false;
public boolean matches(String propertyName) {
return true;
}
};
PropertyNamePatternsMatcher NONE = new PropertyNamePatternsMatcher() {
@Override
public boolean matches(String propertyName) {
return false;
}
};
/** /**
* Return {@code true} of the property name matches. * Return {@code true} of the property name matches.

4
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java
View File

@@ -37,9 +37,9 @@ public class PropertySourcesPropertyValues implements PropertyValues {
private final PropertyNamePatternsMatcher includes; private final PropertyNamePatternsMatcher includes;
private final Map<String, PropertyValue> propertyValues = new LinkedHashMap<String, PropertyValue>(); private final Map<String, PropertyValue> propertyValues = new LinkedHashMap<>();
private final ConcurrentHashMap<String, PropertySource<?>> collectionOwners = new ConcurrentHashMap<String, PropertySource<?>>(); private final ConcurrentHashMap<String, PropertySource<?>> collectionOwners = new ConcurrentHashMap<>();
private final boolean resolvePlaceholders; private final boolean resolvePlaceholders;

28
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java
View File

@@ -32,7 +32,7 @@ public class RelaxedDataBinder extends DataBinder {
private static final Set<Class<?>> EXCLUDED_EDITORS; private static final Set<Class<?>> EXCLUDED_EDITORS;
static { static {
Set<Class<?>> excluded = new HashSet<Class<?>>(); Set<Class<?>> excluded = new HashSet<>();
excluded.add(FileEditor.class); excluded.add(FileEditor.class);
EXCLUDED_EDITORS = Collections.unmodifiableSet(excluded); EXCLUDED_EDITORS = Collections.unmodifiableSet(excluded);
} }
@@ -43,7 +43,7 @@ public class RelaxedDataBinder extends DataBinder {
private boolean ignoreNestedProperties; private boolean ignoreNestedProperties;
private MultiValueMap<String, String> nameAliases = new LinkedMultiValueMap<String, String>(); private MultiValueMap<String, String> nameAliases = new LinkedMultiValueMap<>();
/** /**
* Create a new {@link RelaxedDataBinder} instance. * Create a new {@link RelaxedDataBinder} instance.
@@ -89,7 +89,7 @@ public class RelaxedDataBinder extends DataBinder {
* @param aliases a map of property name to aliases * @param aliases a map of property name to aliases
*/ */
public void setNameAliases(Map<String, List<String>> aliases) { public void setNameAliases(Map<String, List<String>> aliases) {
this.nameAliases = new LinkedMultiValueMap<String, String>(aliases); this.nameAliases = new LinkedMultiValueMap<>(aliases);
} }
/** /**
@@ -129,8 +129,8 @@ public class RelaxedDataBinder extends DataBinder {
BeanWrapper wrapper = new BeanWrapperImpl(target); BeanWrapper wrapper = new BeanWrapperImpl(target);
wrapper.setConversionService(new RelaxedConversionService(getConversionService())); wrapper.setConversionService(new RelaxedConversionService(getConversionService()));
wrapper.setAutoGrowNestedPaths(true); wrapper.setAutoGrowNestedPaths(true);
List<PropertyValue> sortedValues = new ArrayList<PropertyValue>(); List<PropertyValue> sortedValues = new ArrayList<>();
Set<String> modifiedNames = new HashSet<String>(); Set<String> modifiedNames = new HashSet<>();
List<String> sortedNames = getSortedPropertyNames(propertyValues); List<String> sortedNames = getSortedPropertyNames(propertyValues);
for (String name : sortedNames) { for (String name : sortedNames) {
PropertyValue propertyValue = propertyValues.getPropertyValue(name); PropertyValue propertyValue = propertyValues.getPropertyValue(name);
@@ -143,7 +143,7 @@ public class RelaxedDataBinder extends DataBinder {
} }
private List<String> getSortedPropertyNames(MutablePropertyValues propertyValues) { private List<String> getSortedPropertyNames(MutablePropertyValues propertyValues) {
List<String> names = new LinkedList<String>(); List<String> names = new LinkedList<>();
for (PropertyValue propertyValue : propertyValues.getPropertyValueList()) { for (PropertyValue propertyValue : propertyValues.getPropertyValueList()) {
names.add(propertyValue.getName()); names.add(propertyValue.getName());
} }
@@ -161,7 +161,7 @@ public class RelaxedDataBinder extends DataBinder {
* @param names the names to sort * @param names the names to sort
*/ */
private void sortPropertyNames(List<String> names) { private void sortPropertyNames(List<String> names) {
for (String name : new ArrayList<String>(names)) { for (String name : new ArrayList<>(names)) {
int propertyIndex = names.indexOf(name); int propertyIndex = names.indexOf(name);
RelaxedDataBinder.BeanPath path = new RelaxedDataBinder.BeanPath(name); RelaxedDataBinder.BeanPath path = new RelaxedDataBinder.BeanPath(name);
for (String prefix : path.prefixes()) { for (String prefix : path.prefixes()) {
@@ -319,7 +319,7 @@ public class RelaxedDataBinder extends DataBinder {
} }
Object extend = new LinkedHashMap<String, Object>(); Object extend = new LinkedHashMap<String, Object>();
if (!elementDescriptor.isMap() && path.isArrayIndex(index)) { if (!elementDescriptor.isMap() && path.isArrayIndex(index)) {
extend = new ArrayList<Object>(); extend = new ArrayList<>();
} }
wrapper.setPropertyValue(path.prefix(index + 1), extend); wrapper.setPropertyValue(path.prefix(index + 1), extend);
} }
@@ -347,7 +347,7 @@ public class RelaxedDataBinder extends DataBinder {
} }
Object extend = new LinkedHashMap<String, Object>(); Object extend = new LinkedHashMap<String, Object>();
if (descriptor.isCollection()) { if (descriptor.isCollection()) {
extend = new ArrayList<Object>(); extend = new ArrayList<>();
} }
if (descriptor.getType().equals(Object.class) && path.isLastNode(index)) { if (descriptor.getType().equals(Object.class) && path.isLastNode(index)) {
extend = BLANK; extend = BLANK;
@@ -411,7 +411,7 @@ public class RelaxedDataBinder extends DataBinder {
if (aliases == null) { if (aliases == null) {
return Collections.singleton(name); return Collections.singleton(name);
} }
List<String> nameAndAliases = new ArrayList<String>(aliases.size() + 1); List<String> nameAndAliases = new ArrayList<>(aliases.size() + 1);
nameAndAliases.add(name); nameAndAliases.add(name);
nameAndAliases.addAll(aliases); nameAndAliases.addAll(aliases);
return nameAndAliases; return nameAndAliases;
@@ -473,7 +473,7 @@ public class RelaxedDataBinder extends DataBinder {
} }
public List<String> prefixes() { public List<String> prefixes() {
List<String> prefixes = new ArrayList<String>(); List<String> prefixes = new ArrayList<>();
for (int index = 1; index < this.nodes.size(); index++) { for (int index = 1; index < this.nodes.size(); index++) {
prefixes.add(prefix(index)); prefixes.add(prefix(index));
} }
@@ -485,7 +485,7 @@ public class RelaxedDataBinder extends DataBinder {
} }
private List<PathNode> splitPath(String path) { private List<PathNode> splitPath(String path) {
List<PathNode> nodes = new ArrayList<PathNode>(); List<PathNode> nodes = new ArrayList<>();
String current = extractIndexedPaths(path, nodes); String current = extractIndexedPaths(path, nodes);
for (String name : StringUtils.delimitedListToStringArray(current, ".")) { for (String name : StringUtils.delimitedListToStringArray(current, ".")) {
if (StringUtils.hasText(name)) { if (StringUtils.hasText(name)) {
@@ -518,7 +518,7 @@ public class RelaxedDataBinder extends DataBinder {
} }
public void collapseKeys(int index) { public void collapseKeys(int index) {
List<PathNode> revised = new ArrayList<PathNode>(); List<PathNode> revised = new ArrayList<>();
for (int i = 0; i < index; i++) { for (int i = 0; i < index; i++) {
revised.add(this.nodes.get(i)); revised.add(this.nodes.get(i));
} }
@@ -668,7 +668,7 @@ public class RelaxedDataBinder extends DataBinder {
private static final Set<String> BENIGN_PROPERTY_SOURCE_NAMES; private static final Set<String> BENIGN_PROPERTY_SOURCE_NAMES;
static { static {
Set<String> names = new HashSet<String>(); Set<String> names = new HashSet<>();
names.add(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME); names.add(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME);
names.add(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME); names.add(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME);
BENIGN_PROPERTY_SOURCE_NAMES = Collections.unmodifiableSet(names); BENIGN_PROPERTY_SOURCE_NAMES = Collections.unmodifiableSet(names);

2
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java
View File

@@ -24,7 +24,7 @@ public final class RelaxedNames implements Iterable<String> {
private final String name; private final String name;
private final Set<String> values = new LinkedHashSet<String>(); private final Set<String> values = new LinkedHashSet<>();
/** /**
* Create a new {@link RelaxedNames} instance. * Create a new {@link RelaxedNames} instance.

6
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java
View File

@@ -22,7 +22,7 @@ import org.springframework.util.StringUtils;
*/ */
public class CompositePropertySource extends EnumerablePropertySource<Object> { public class CompositePropertySource extends EnumerablePropertySource<Object> {
private final Set<PropertySource<?>> propertySources = new LinkedHashSet<PropertySource<?>>(); private final Set<PropertySource<?>> propertySources = new LinkedHashSet<>();
/** /**
* Create a new {@code CompositePropertySource}. * Create a new {@code CompositePropertySource}.
@@ -56,7 +56,7 @@ public class CompositePropertySource extends EnumerablePropertySource<Object> {
@Override @Override
public String[] getPropertyNames() { public String[] getPropertyNames() {
Set<String> names = new LinkedHashSet<String>(); Set<String> names = new LinkedHashSet<>();
for (PropertySource<?> propertySource : this.propertySources) { for (PropertySource<?> propertySource : this.propertySources) {
if (!(propertySource instanceof EnumerablePropertySource)) { if (!(propertySource instanceof EnumerablePropertySource)) {
throw new IllegalStateException( throw new IllegalStateException(
@@ -83,7 +83,7 @@ public class CompositePropertySource extends EnumerablePropertySource<Object> {
* @since 4.1 * @since 4.1
*/ */
public void addFirstPropertySource(PropertySource<?> propertySource) { public void addFirstPropertySource(PropertySource<?> propertySource) {
List<PropertySource<?>> existing = new ArrayList<PropertySource<?>>(this.propertySources); List<PropertySource<?>> existing = new ArrayList<>(this.propertySources);
this.propertySources.clear(); this.propertySources.clear();
this.propertySources.add(propertySource); this.propertySources.add(propertySource);
this.propertySources.addAll(existing); this.propertySources.addAll(existing);

2
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java
View File

@@ -28,7 +28,7 @@ public class MutablePropertySources implements PropertySources {
private final Log logger; private final Log logger;
private final List<PropertySource<?>> propertySourceList = new CopyOnWriteArrayList<PropertySource<?>>(); private final List<PropertySource<?>> propertySourceList = new CopyOnWriteArrayList<>();
/** /**
* Create a new {@link MutablePropertySources} * Create a new {@link MutablePropertySources}

8
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java
View File

@@ -31,7 +31,7 @@ public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatch
} }
public void addActiveProfiles(String... profiles) { public void addActiveProfiles(String... profiles) {
LinkedHashSet<String> set = new LinkedHashSet<String>(Arrays.asList(this.activeProfiles)); LinkedHashSet<String> set = new LinkedHashSet<>(Arrays.asList(this.activeProfiles));
Collections.addAll(set, profiles); Collections.addAll(set, profiles);
this.activeProfiles = set.toArray(new String[set.size()]); this.activeProfiles = set.toArray(new String[set.size()]);
} }
@@ -67,14 +67,14 @@ public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatch
private ProfilesMatcher getProfilesMatcher() { private ProfilesMatcher getProfilesMatcher() {
return (this.activeProfiles.length != 0 ? new ActiveProfilesMatcher( return (this.activeProfiles.length != 0 ? new ActiveProfilesMatcher(
new HashSet<String>(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher()); new HashSet<>(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher());
} }
private Set<String> extractProfiles(List<String> profiles, ProfileType type) { private Set<String> extractProfiles(List<String> profiles, ProfileType type) {
if (CollectionUtils.isEmpty(profiles)) { if (CollectionUtils.isEmpty(profiles)) {
return null; return null;
} }
Set<String> extractedProfiles = new HashSet<String>(); Set<String> extractedProfiles = new HashSet<>();
for (String candidate : profiles) { for (String candidate : profiles) {
ProfileType candidateType = ProfileType.POSITIVE; ProfileType candidateType = ProfileType.POSITIVE;
if (candidate.startsWith("!")) { if (candidate.startsWith("!")) {
@@ -167,7 +167,7 @@ public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatch
*/ */
static class SpringProperties { static class SpringProperties {
private List<String> profiles = new ArrayList<String>(); private List<String> profiles = new ArrayList<>();
public List<String> getProfiles() { public List<String> getProfiles() {
return this.profiles; return this.profiles;

4
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java
View File

@@ -166,7 +166,7 @@ public abstract class YamlProcessor {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private Map<String, Object> asMap(Object object) { private Map<String, Object> asMap(Object object) {
// YAML can have numbers as keys // YAML can have numbers as keys
Map<String, Object> result = new LinkedHashMap<String, Object>(); Map<String, Object> result = new LinkedHashMap<>();
if (!(object instanceof Map)) { if (!(object instanceof Map)) {
// A document can be a text literal // A document can be a text literal
result.put("document", object); result.put("document", object);
@@ -249,7 +249,7 @@ public abstract class YamlProcessor {
* @since 4.1.3 * @since 4.1.3
*/ */
protected final Map<String, Object> getFlattenedMap(Map<String, Object> source) { protected final Map<String, Object> getFlattenedMap(Map<String, Object> source) {
Map<String, Object> result = new LinkedHashMap<String, Object>(); Map<String, Object> result = new LinkedHashMap<>();
buildFlattenedMap(result, source, null); buildFlattenedMap(result, source, null);
return result; return result;
} }

10
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java
View File

@@ -72,14 +72,8 @@ public class YamlPropertySourceLoader implements PropertySourceLoader {
} }
public Map<String, Object> process() { public Map<String, Object> process() {
final Map<String, Object> result = new LinkedHashMap<String, Object>(); final Map<String, Object> result = new LinkedHashMap<>();
process(new MatchCallback() { process((properties, map) -> result.putAll(getFlattenedMap(map)));
@Override
public void process(Properties properties, Map<String, Object> map) {
result.putAll(getFlattenedMap(map));
}
});
return result; return result;
} }

2
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java
View File

@@ -259,7 +259,7 @@ public class ExtensionLoader<T> {
} }
} }
Map<String, Class<?>> extensionClasses = new HashMap<String, Class<?>>(); Map<String, Class<?>> extensionClasses = new HashMap<>();
// 1. plugin folder，customized extension classLoader （jar_dir/plugin） // 1. plugin folder，customized extension classLoader （jar_dir/plugin）
String dir = File.separator + this.getJarDirectoryPath() + File.separator + "plugin"; String dir = File.separator + this.getJarDirectoryPath() + File.separator + "plugin";

5
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java
View File

@@ -4,7 +4,6 @@ import java.util.*;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage; import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
/** /**
@@ -20,7 +19,7 @@ public class MessageUtil {
return null; return null;
} }
List<CanalEntry.Entry> entries = message.getEntries(); List<CanalEntry.Entry> entries = message.getEntries();
List<Dml> dmls = new ArrayList<Dml>(entries.size()); List<Dml> dmls = new ArrayList<>(entries.size());
for (CanalEntry.Entry entry : entries) { for (CanalEntry.Entry entry : entries) {
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) { || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
@@ -136,7 +135,7 @@ public class MessageUtil {
if (commonMessages == null) { if (commonMessages == null) {
return new ArrayList<>(); return new ArrayList<>();
} }
List<Dml> dmls = new ArrayList<Dml>(commonMessages.size()); List<Dml> dmls = new ArrayList<>(commonMessages.size());
for (CommonMessage commonMessage : commonMessages) { for (CommonMessage commonMessage : commonMessages) {
Dml dml = flatMessage2Dml(destination, groupId, commonMessage); Dml dml = flatMessage2Dml(destination, groupId, commonMessage);
if (dml != null) { if (dml != null) {

11
client-adapter/common/src/test/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtilTest.java
View File

@@ -1,24 +1,13 @@
package com.alibaba.otter.canal.client.adapter.support; package com.alibaba.otter.canal.client.adapter.support;
import static org.mockito.AdditionalMatchers.or;
import static org.mockito.Matchers.isA;
import static org.mockito.Matchers.isNull;
import static org.powermock.api.mockito.PowerMockito.mockStatic;
import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil;
import com.alibaba.otter.canal.client.adapter.support.Util;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.ExpectedException; import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout; import org.junit.rules.Timeout;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunner;
import java.lang.reflect.Array;
@RunWith(PowerMockRunner.class) @RunWith(PowerMockRunner.class)
public class JdbcTypeUtilTest { public class JdbcTypeUtilTest {

2
client-adapter/common/src/test/java/com/alibaba/otter/canal/client/adapter/support/UtilTest.java
View File

@@ -3,9 +3,7 @@ package com.alibaba.otter.canal.client.adapter.support;
import static org.mockito.AdditionalMatchers.or; import static org.mockito.AdditionalMatchers.or;
import static org.mockito.Matchers.isA; import static org.mockito.Matchers.isA;
import static org.mockito.Matchers.isNull; import static org.mockito.Matchers.isNull;
import static org.powermock.api.mockito.PowerMockito.mockStatic;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.diffblue.deeptestutils.mock.DTUMemberMatcher; import com.diffblue.deeptestutils.mock.DTUMemberMatcher;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.junit.Assert; import org.junit.Assert;

15
client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java
View File

@@ -28,19 +28,14 @@ public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy {
private IZkChildListener childListener; // 监听所有的服务器列表 private IZkChildListener childListener; // 监听所有的服务器列表
private IZkDataListener dataListener; // 监听当前的工作节点 private IZkDataListener dataListener; // 监听当前的工作节点
private ZkClientx zkClient; private ZkClientx zkClient;
private volatile List<InetSocketAddress> currentAddress = new ArrayList<InetSocketAddress>(); private volatile List<InetSocketAddress> currentAddress = new ArrayList<>();
private volatile InetSocketAddress runningAddress = null; private volatile InetSocketAddress runningAddress = null;
public ClusterNodeAccessStrategy(String destination, ZkClientx zkClient){ public ClusterNodeAccessStrategy(String destination, ZkClientx zkClient){
this.destination = destination; this.destination = destination;
this.zkClient = zkClient; this.zkClient = zkClient;
childListener = new IZkChildListener() { // handleChildChange
childListener = (parentPath, currentChilds) -> initClusters(currentChilds);
public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception {
initClusters(currentChilds);
}
};
dataListener = new IZkDataListener() { dataListener = new IZkDataListener() {
@@ -80,9 +75,9 @@ public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy {
private void initClusters(List<String> currentChilds) { private void initClusters(List<String> currentChilds) {
if (currentChilds == null || currentChilds.isEmpty()) { if (currentChilds == null || currentChilds.isEmpty()) {
currentAddress = new ArrayList<InetSocketAddress>(); currentAddress = new ArrayList<>();
} else { } else {
List<InetSocketAddress> addresses = new ArrayList<InetSocketAddress>(); List<InetSocketAddress> addresses = new ArrayList<>();
for (String address : currentChilds) { for (String address : currentChilds) {
String[] strs = StringUtils.split(address, ":"); String[] strs = StringUtils.split(address, ":");
if (strs != null && strs.length == 2) { if (strs != null && strs.length == 2) {

2
client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java
View File

@@ -64,7 +64,7 @@ public class SimpleCanalConnector implements CanalConnector {
private SocketChannel channel; private SocketChannel channel;
private ReadableByteChannel readableChannel; private ReadableByteChannel readableChannel;
private WritableByteChannel writableChannel; private WritableByteChannel writableChannel;
private List<Compression> supportedCompressions = new ArrayList<Compression>(); private List<Compression> supportedCompressions = new ArrayList<>();
private ClientIdentity clientIdentity; private ClientIdentity clientIdentity;
private ClientRunningMonitor runningMonitor; // 运行控制 private ClientRunningMonitor runningMonitor; // 运行控制
private ZkClientx zkClientx; private ZkClientx zkClientx;

2
client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleNodeAccessStrategy.java
View File

@@ -14,7 +14,7 @@ import com.alibaba.otter.canal.client.CanalNodeAccessStrategy;
*/ */
public class SimpleNodeAccessStrategy implements CanalNodeAccessStrategy { public class SimpleNodeAccessStrategy implements CanalNodeAccessStrategy {
private List<SocketAddress> nodes = new ArrayList<SocketAddress>(); private List<SocketAddress> nodes = new ArrayList<>();
private int index = 0; private int index = 0;
public SimpleNodeAccessStrategy(List<? extends SocketAddress> nodes){ public SimpleNodeAccessStrategy(List<? extends SocketAddress> nodes){

7
client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java
View File

@@ -72,12 +72,7 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
initRunning(); initRunning();
} else { } else {
// 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作 // 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作
delayExector.schedule(new Runnable() { delayExector.schedule(() -> initRunning(), delayTime, TimeUnit.SECONDS);
public void run() {
initRunning();
}
}, delayTime, TimeUnit.SECONDS);
} }
} }

4
client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java
View File

@@ -80,11 +80,11 @@ public class KafkaCanalConnector implements CanalMQConnector {
connected = true; connected = true;
if (kafkaConsumer == null && !flatMessage) { if (kafkaConsumer == null && !flatMessage) {
kafkaConsumer = new KafkaConsumer<String, Message>(properties); kafkaConsumer = new KafkaConsumer<>(properties);
} }
if (kafkaConsumer2 == null && flatMessage) { if (kafkaConsumer2 == null && flatMessage) {
kafkaConsumer2 = new KafkaConsumer<String, String>(properties); kafkaConsumer2 = new KafkaConsumer<>(properties);
} }
} }

54
client/src/test/java/com/alibaba/otter/canal/client/running/ClientRunningTest.java
View File

@@ -59,49 +59,37 @@ public class ClientRunningTest extends AbstractZkTest {
final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089); final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089);
final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090); final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090);
final ExecutorService executor = Executors.newFixedThreadPool(3); final ExecutorService executor = Executors.newFixedThreadPool(3);
executor.submit(new Runnable() { executor.submit(() -> {
for (int i = 0; i < 10; i++) {
public void run() { if (!runningMonitor1.isStart()) {
for (int i = 0; i < 10; i++) { runningMonitor1.start();
if (!runningMonitor1.isStart()) {
runningMonitor1.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor1.stop();
sleep(2000L + RandomUtils.nextInt(500));
} }
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor1.stop();
sleep(2000L + RandomUtils.nextInt(500));
} }
}); });
executor.submit(new Runnable() { executor.submit(() -> {
for (int i = 0; i < 10; i++) {
public void run() { if (!runningMonitor2.isStart()) {
for (int i = 0; i < 10; i++) { runningMonitor2.start();
if (!runningMonitor2.isStart()) {
runningMonitor2.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor2.stop();
sleep(2000L + RandomUtils.nextInt(500));
} }
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor2.stop();
sleep(2000L + RandomUtils.nextInt(500));
} }
}); });
executor.submit(new Runnable() { executor.submit(() -> {
for (int i = 0; i < 10; i++) {
public void run() { if (!runningMonitor3.isStart()) {
for (int i = 0; i < 10; i++) { runningMonitor3.start();
if (!runningMonitor3.isStart()) {
runningMonitor3.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor3.stop();
sleep(2000L + RandomUtils.nextInt(500));
} }
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor3.stop();
sleep(2000L + RandomUtils.nextInt(500));
} }
}); });
sleep(30000L); sleep(30000L);

2
common/src/main/java/com/alibaba/otter/canal/common/utils/ExecutorTemplate.java
View File

@@ -34,7 +34,7 @@ public class ExecutorTemplate {
private volatile List<Future> futures = null; private volatile List<Future> futures = null;
public ExecutorTemplate(ThreadPoolExecutor executor){ public ExecutorTemplate(ThreadPoolExecutor executor){
this.futures = Collections.synchronizedList(new ArrayList<Future>()); this.futures = Collections.synchronizedList(new ArrayList<>());
this.executor = executor; this.executor = executor;
} }

8
common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java
View File

@@ -75,13 +75,7 @@ public class JsonUtils {
SerializeWriter out = new SerializeWriter(); SerializeWriter out = new SerializeWriter();
try { try {
JSONSerializer serializer = new JSONSerializer(out); JSONSerializer serializer = new JSONSerializer(out);
serializer.getPropertyFilters().add(new PropertyFilter() { serializer.getPropertyFilters().add((source, name, value) -> !propertyFliters.contains(name));
public boolean apply(Object source, String name, Object value) {
return !propertyFliters.contains(name);
}
});
serializer.write(obj); serializer.write(obj);
return out.toString(); return out.toString();
} finally { } finally {

18
common/src/main/java/com/alibaba/otter/canal/common/utils/NamedThreadFactory.java
View File

@@ -19,18 +19,14 @@ public class NamedThreadFactory implements ThreadFactory {
final private boolean daemon; final private boolean daemon;
final private ThreadGroup group; final private ThreadGroup group;
final private AtomicInteger threadNumber = new AtomicInteger(0); final private AtomicInteger threadNumber = new AtomicInteger(0);
final static UncaughtExceptionHandler uncaughtExceptionHandler = new UncaughtExceptionHandler() { final static UncaughtExceptionHandler uncaughtExceptionHandler = (t, e) -> {
if (e instanceof InterruptedException
|| (e.getCause() != null && e.getCause() instanceof InterruptedException)) {
return;
}
public void uncaughtException(Thread t, logger.error("from " + t.getName(), e);
Throwable e) { };
if (e instanceof InterruptedException
|| (e.getCause() != null && e.getCause() instanceof InterruptedException)) {
return;
}
logger.error("from " + t.getName(), e);
}
};
public NamedThreadFactory(){ public NamedThreadFactory(){
this(DEFAULT_NAME, true); this(DEFAULT_NAME, true);

2
common/src/main/java/com/alibaba/otter/canal/common/utils/UriUtils.java
View File

@@ -53,7 +53,7 @@ public final class UriUtils {
return Collections.EMPTY_MAP; return Collections.EMPTY_MAP;
} }
String query = uri.getRawQuery(); String query = uri.getRawQuery();
HashMap<String, String> params = new HashMap<String, String>(); HashMap<String, String> params = new HashMap<>();
@SuppressWarnings("resource") @SuppressWarnings("resource")
Scanner scan = new Scanner(query); Scanner scan = new Scanner(query);
scan.useDelimiter(SPLIT); scan.useDelimiter(SPLIT);

7
common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java
View File

@@ -23,12 +23,7 @@ import com.google.common.collect.MigrateMap;
public class ZkClientx extends ZkClient { public class ZkClientx extends ZkClient {
// 对于zkclient进行一次缓存，避免一个jvm内部使用多个zk connection // 对于zkclient进行一次缓存，避免一个jvm内部使用多个zk connection
private static Map<String, ZkClientx> clients = MigrateMap.makeComputingMap(new Function<String, ZkClientx>() { private static Map<String, ZkClientx> clients = MigrateMap.makeComputingMap(ZkClientx::new);
public ZkClientx apply(String servers) {
return new ZkClientx(servers);
}
});
public static ZkClientx getZkClient(String servers) { public static ZkClientx getZkClient(String servers) {
return clients.get(servers); return clients.get(servers);

7
common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java
View File

@@ -73,12 +73,7 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
initRunning(); initRunning();
} else { } else {
// 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作 // 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作
delayExector.schedule(new Runnable() { delayExector.schedule(() -> initRunning(), delayTime, TimeUnit.SECONDS);
public void run() {
initRunning();
}
}, delayTime, TimeUnit.SECONDS);
} }
} }

2
common/src/main/java/com/google/common/collect/MigrateMap.java
View File

@@ -24,7 +24,7 @@ public class MigrateMap {
} }
}); });
return new MigrateConcurrentMap<K, V>(computingCache); return new MigrateConcurrentMap<>(computingCache);
} }
public static <K, V> ConcurrentMap<K, V> makeComputingMap(Function<? super K, ? extends V> computingFunction) { public static <K, V> ConcurrentMap<K, V> makeComputingMap(Function<? super K, ? extends V> computingFunction) {

66
common/src/test/java/com/alibaba/otter/canal/common/ServerRunningTest.java
View File

@@ -57,55 +57,43 @@ public class ServerRunningTest extends AbstractZkTest {
final ServerRunningMonitor runningMonitor2 = buildServerRunning(countLatch, "127.0.0.1", 2089); final ServerRunningMonitor runningMonitor2 = buildServerRunning(countLatch, "127.0.0.1", 2089);
final ServerRunningMonitor runningMonitor3 = buildServerRunning(countLatch, "127.0.0.1", 2090); final ServerRunningMonitor runningMonitor3 = buildServerRunning(countLatch, "127.0.0.1", 2090);
final ExecutorService executor = Executors.newFixedThreadPool(3); final ExecutorService executor = Executors.newFixedThreadPool(3);
executor.submit(new Runnable() { executor.submit(() -> {
for (int i = 0; i < 10; i++) {
public void run() { if (!runningMonitor1.isStart()) {
for (int i = 0; i < 10; i++) { runningMonitor1.start();
if (!runningMonitor1.isStart()) {
runningMonitor1.start();
}
sleep(2000L + RandomUtils.nextInt(500));
if (runningMonitor1.check()) {
runningMonitor1.stop();
}
sleep(2000L + RandomUtils.nextInt(500));
} }
sleep(2000L + RandomUtils.nextInt(500));
if (runningMonitor1.check()) {
runningMonitor1.stop();
}
sleep(2000L + RandomUtils.nextInt(500));
} }
}); });
executor.submit(new Runnable() { executor.submit(() -> {
for (int i = 0; i < 10; i++) {
public void run() { if (!runningMonitor2.isStart()) {
for (int i = 0; i < 10; i++) { runningMonitor2.start();
if (!runningMonitor2.isStart()) {
runningMonitor2.start();
}
sleep(2000L + RandomUtils.nextInt(500));
if (runningMonitor2.check()) {
runningMonitor2.stop();
}
sleep(2000L + RandomUtils.nextInt(500));
} }
sleep(2000L + RandomUtils.nextInt(500));
if (runningMonitor2.check()) {
runningMonitor2.stop();
}
sleep(2000L + RandomUtils.nextInt(500));
} }
}); });
executor.submit(new Runnable() { executor.submit(() -> {
for (int i = 0; i < 10; i++) {
public void run() { if (!runningMonitor3.isStart()) {
for (int i = 0; i < 10; i++) { runningMonitor3.start();
if (!runningMonitor3.isStart()) {
runningMonitor3.start();
}
sleep(2000L + RandomUtils.nextInt(500));
if (runningMonitor3.check()) {
runningMonitor3.stop();
}
sleep(2000L + RandomUtils.nextInt(500));
} }
sleep(2000L + RandomUtils.nextInt(500));
if (runningMonitor3.check()) {
runningMonitor3.stop();
}
sleep(2000L + RandomUtils.nextInt(500));
} }
}); });
sleep(30000L); sleep(30000L);

6
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java
View File

@@ -42,7 +42,7 @@ public class AviaterRegexFilter {
this.defaultEmptyValue = defaultEmptyValue; this.defaultEmptyValue = defaultEmptyValue;
List<String> list = null; List<String> list = null;
if (StringUtils.isEmpty(pattern)) { if (StringUtils.isEmpty(pattern)) {
list = new ArrayList<String>(); list = new ArrayList<>();
} else { } else {
String[] ss = StringUtils.split(pattern, SPLIT); String[] ss = StringUtils.split(pattern, SPLIT);
list = Arrays.asList(ss); list = Arrays.asList(ss);
@@ -66,7 +66,7 @@ public class AviaterRegexFilter {
return defaultEmptyValue; return defaultEmptyValue;
} }
Map<String, Object> env = new HashMap<String, Object>(); Map<String, Object> env = new HashMap<>();
env.put("pattern", pattern); env.put("pattern", pattern);
env.put("target", filtered.toLowerCase()); env.put("target", filtered.toLowerCase());
return (Boolean) exp.execute(env); return (Boolean) exp.execute(env);
@@ -112,7 +112,7 @@ public class AviaterRegexFilter {
*/ */
private List<String> completionPattern(List<String> patterns) { private List<String> completionPattern(List<String> patterns) {
List<String> result = new ArrayList<String>(); List<String> result = new ArrayList<>();
for (String pattern : patterns) { for (String pattern : patterns) {
StringBuffer stringBuffer = new StringBuffer(); StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("^"); stringBuffer.append("^");

4
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/AbstractMQProducer.java
View File

@@ -35,7 +35,7 @@ public abstract class AbstractMQProducer implements CanalMQProducer {
parallelBuildThreadSize, parallelBuildThreadSize,
0, 0,
TimeUnit.SECONDS, TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(parallelBuildThreadSize * 2), new ArrayBlockingQueue<>(parallelBuildThreadSize * 2),
new NamedThreadFactory("MQ-Parallel-Builder"), new NamedThreadFactory("MQ-Parallel-Builder"),
new ThreadPoolExecutor.CallerRunsPolicy()); new ThreadPoolExecutor.CallerRunsPolicy());
@@ -44,7 +44,7 @@ public abstract class AbstractMQProducer implements CanalMQProducer {
parallelSendThreadSize, parallelSendThreadSize,
0, 0,
TimeUnit.SECONDS, TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(parallelSendThreadSize * 2), new ArrayBlockingQueue<>(parallelSendThreadSize * 2),
new NamedThreadFactory("MQ-Parallel-Sender"), new NamedThreadFactory("MQ-Parallel-Sender"),
new ThreadPoolExecutor.CallerRunsPolicy()); new ThreadPoolExecutor.CallerRunsPolicy());
} }

138
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java
View File

@@ -75,64 +75,58 @@ public class MQMessageUtils {
private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(CacheBuilder.newBuilder() private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(CacheBuilder.newBuilder()
.softValues(), .softValues(),
new Function<String, List<DynamicTopicData>>() { pkHashConfigs -> {
List<DynamicTopicData> datas = Lists.newArrayList();
String[] dynamicTopicArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
",",
";"),
";");
// schema.table
for (String dynamicTopic : dynamicTopicArray) {
DynamicTopicData data = new DynamicTopicData();
public List<DynamicTopicData> apply(String pkHashConfigs) { if (!isWildCard(dynamicTopic)) {
List<DynamicTopicData> datas = Lists.newArrayList(); data.simpleName = dynamicTopic;
String[] dynamicTopicArray = StringUtils.split(StringUtils.replace(pkHashConfigs, } else {
",", if (dynamicTopic.contains("\\.")) {
";"), data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
";"); } else {
// schema.table data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
for (String dynamicTopic : dynamicTopicArray) { }
DynamicTopicData data = new DynamicTopicData(); }
datas.add(data);
}
if (!isWildCard(dynamicTopic)) { return datas;
data.simpleName = dynamicTopic; });
} else {
if (dynamicTopic.contains("\\.")) {
data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
} else {
data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
}
}
datas.add(data);
}
return datas;
}
});
private static Map<String, List<TopicPartitionData>> topicPartitionDatas = MigrateMap.makeComputingMap(CacheBuilder.newBuilder() private static Map<String, List<TopicPartitionData>> topicPartitionDatas = MigrateMap.makeComputingMap(CacheBuilder.newBuilder()
.softValues(), .softValues(),
new Function<String, List<TopicPartitionData>>() { tPConfigs -> {
List<TopicPartitionData> datas = Lists.newArrayList();
public List<TopicPartitionData> apply(String tPConfigs) { String[] tPArray = StringUtils.split(StringUtils.replace(tPConfigs,
List<TopicPartitionData> datas = Lists.newArrayList(); ",",
String[] tPArray = StringUtils.split(StringUtils.replace(tPConfigs, ";"),
",", ";");
";"), for (String tPConfig : tPArray) {
";"); TopicPartitionData data = new TopicPartitionData();
for (String tPConfig : tPArray) { int i = tPConfig.lastIndexOf(":");
TopicPartitionData data = new TopicPartitionData(); if (i > 0) {
int i = tPConfig.lastIndexOf(":"); String tStr = tPConfig.substring(0, i);
if (i > 0) { String pStr = tPConfig.substring(i + 1);
String tStr = tPConfig.substring(0, i); if (!isWildCard(tStr)) {
String pStr = tPConfig.substring(i + 1); data.simpleName = tStr;
if (!isWildCard(tStr)) { } else {
data.simpleName = tStr; data.regexFilter = new AviaterRegexFilter(tStr);
} else {
data.regexFilter = new AviaterRegexFilter(tStr);
}
if (!StringUtils.isEmpty(pStr) && StringUtils.isNumeric(pStr)) {
data.partitionNum = Integer.valueOf(pStr);
}
datas.add(data);
} }
if (!StringUtils.isEmpty(pStr) && StringUtils.isNumeric(pStr)) {
data.partitionNum = Integer.valueOf(pStr);
}
datas.add(data);
} }
return datas;
} }
return datas;
}); });
/** /**
@@ -206,19 +200,15 @@ public class MQMessageUtils {
int i = 0; int i = 0;
for (ByteString byteString : rawEntries) { for (ByteString byteString : rawEntries) {
final int index = i; final int index = i;
template.submit(new Runnable() { template.submit(() -> {
try {
@Override Entry entry = Entry.parseFrom(byteString);
public void run() { RowChange rowChange = RowChange.parseFrom(entry.getStoreValue());
try { datas[index] = new EntryRowData();
Entry entry = Entry.parseFrom(byteString); datas[index].entry = entry;
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); datas[index].rowChange = rowChange;
datas[index] = new EntryRowData(); } catch (InvalidProtocolBufferException e) {
datas[index].entry = entry; throw new RuntimeException(e);
datas[index].rowChange = rowChange;
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
} }
}); });
@@ -232,18 +222,14 @@ public class MQMessageUtils {
int i = 0; int i = 0;
for (Entry entry : message.getEntries()) { for (Entry entry : message.getEntries()) {
final int index = i; final int index = i;
template.submit(new Runnable() { template.submit(() -> {
try {
@Override RowChange rowChange = RowChange.parseFrom(entry.getStoreValue());
public void run() { datas[index] = new EntryRowData();
try { datas[index].entry = entry;
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); datas[index].rowChange = rowChange;
datas[index] = new EntryRowData(); } catch (InvalidProtocolBufferException e) {
datas[index].entry = entry; throw new RuntimeException(e);
datas[index].rowChange = rowChange;
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
} }
}); });
@@ -679,7 +665,7 @@ public class MQMessageUtils {
CanalEntry.Entry entry) { CanalEntry.Entry entry) {
Message message = messageMap.get(topicName); Message message = messageMap.get(topicName);
if (message == null) { if (message == null) {
message = new Message(messageId, new ArrayList<CanalEntry.Entry>()); message = new Message(messageId, new ArrayList<>());
messageMap.put(topicName, message); messageMap.put(topicName, message);
} }
message.getEntries().add(entry); message.getEntries().add(entry);

8
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java
View File

@@ -120,13 +120,7 @@ public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQ
final String topicName = entry.getKey().replace('.', '_'); final String topicName = entry.getKey().replace('.', '_');
final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue(); final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
template.submit(new Runnable() { template.submit(() -> send(destination, topicName, messageSub));
@Override
public void run() {
send(destination, topicName, messageSub);
}
});
} }
template.waitForResult(); template.waitForResult();

2
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java
View File

@@ -37,7 +37,7 @@ public class JsonDiffConversion {
public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex, public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex,
String charsetName) { String charsetName) {
int position = buffer.position(); int position = buffer.position();
List<String> operation_names = new ArrayList<String>(); List<String> operation_names = new ArrayList<>();
while (buffer.hasRemaining()) { while (buffer.hasRemaining()) {
int operation_int = buffer.getUint8(); int operation_int = buffer.getUint8();
if (operation_int >= JSON_DIFF_OPERATION_COUNT) { if (operation_int >= JSON_DIFF_OPERATION_COUNT) {

2
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java
View File

@@ -16,7 +16,7 @@ import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
*/ */
public final class LogContext { public final class LogContext {
private final Map<Long, TableMapLogEvent> mapOfTable = new HashMap<Long, TableMapLogEvent>(); private final Map<Long, TableMapLogEvent> mapOfTable = new HashMap<>();
private FormatDescriptionLogEvent formatDescription; private FormatDescriptionLogEvent formatDescription;

12
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java
View File

@@ -601,7 +601,7 @@ public final class TableMapLogEvent extends LogEvent {
private void parse_signedness(LogBuffer buffer, int length) { private void parse_signedness(LogBuffer buffer, int length) {
// stores the signedness flags extracted from field // stores the signedness flags extracted from field
List<Boolean> datas = new ArrayList<Boolean>(); List<Boolean> datas = new ArrayList<>();
for (int i = 0; i < length; i++) { for (int i = 0; i < length; i++) {
int ut = buffer.getUint8(); int ut = buffer.getUint8();
for (int c = 0x80; c != 0; c >>= 1) { for (int c = 0x80; c != 0; c >>= 1) {
@@ -622,7 +622,7 @@ public final class TableMapLogEvent extends LogEvent {
// stores collation numbers extracted from field. // stores collation numbers extracted from field.
int limit = buffer.position() + length; int limit = buffer.position() + length;
this.default_charset = (int) buffer.getPackedLong(); this.default_charset = (int) buffer.getPackedLong();
List<TableMapLogEvent.Pair> datas = new ArrayList<TableMapLogEvent.Pair>(); List<TableMapLogEvent.Pair> datas = new ArrayList<>();
while (buffer.hasRemaining() && buffer.position() < limit) { while (buffer.hasRemaining() && buffer.position() < limit) {
int col_index = (int) buffer.getPackedLong(); int col_index = (int) buffer.getPackedLong();
int col_charset = (int) buffer.getPackedLong(); int col_charset = (int) buffer.getPackedLong();
@@ -639,7 +639,7 @@ public final class TableMapLogEvent extends LogEvent {
private List<Integer> parse_column_charset(LogBuffer buffer, int length) { private List<Integer> parse_column_charset(LogBuffer buffer, int length) {
// stores collation numbers extracted from field. // stores collation numbers extracted from field.
int limit = buffer.position() + length; int limit = buffer.position() + length;
List<Integer> datas = new ArrayList<Integer>(); List<Integer> datas = new ArrayList<>();
while (buffer.hasRemaining() && buffer.position() < limit) { while (buffer.hasRemaining() && buffer.position() < limit) {
int col_charset = (int) buffer.getPackedLong(); int col_charset = (int) buffer.getPackedLong();
datas.add(col_charset); datas.add(col_charset);
@@ -664,10 +664,10 @@ public final class TableMapLogEvent extends LogEvent {
// into a string separate vector. All of them are stored // into a string separate vector. All of them are stored
// in 'vec'. // in 'vec'.
int limit = buffer.position() + length; int limit = buffer.position() + length;
List<List<String>> datas = new ArrayList<List<String>>(); List<List<String>> datas = new ArrayList<>();
while (buffer.hasRemaining() && buffer.position() < limit) { while (buffer.hasRemaining() && buffer.position() < limit) {
int count = (int) buffer.getPackedLong(); int count = (int) buffer.getPackedLong();
List<String> data = new ArrayList<String>(count); List<String> data = new ArrayList<>(count);
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
int len1 = (int) buffer.getPackedLong(); int len1 = (int) buffer.getPackedLong();
data.add(buffer.getFixString(len1)); data.add(buffer.getFixString(len1));
@@ -694,7 +694,7 @@ public final class TableMapLogEvent extends LogEvent {
// stores geometry column's types extracted from field. // stores geometry column's types extracted from field.
int limit = buffer.position() + length; int limit = buffer.position() + length;
List<Integer> datas = new ArrayList<Integer>(); List<Integer> datas = new ArrayList<>();
while (buffer.hasRemaining() && buffer.position() < limit) { while (buffer.hasRemaining() && buffer.position() < limit) {
int col_type = (int) buffer.getPackedLong(); int col_type = (int) buffer.getPackedLong();
datas.add(col_type); datas.add(col_type);

2
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/event/LogHeaderTest.java
View File

@@ -30,7 +30,7 @@ public class LogHeaderTest {
final LogHeader objectUnderTest = new LogHeader(type); final LogHeader objectUnderTest = new LogHeader(type);
// Assert side effects // Assert side effects
final HashMap<String, String> hashMap = new HashMap<String, String>(); final HashMap<String, String> hashMap = new HashMap<>();
Assert.assertEquals(hashMap, objectUnderTest.gtidMap); Assert.assertEquals(hashMap, objectUnderTest.gtidMap);
} }

249
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
View File

@@ -74,12 +74,7 @@ public class CanalController {
} }
public CanalController(final Properties properties){ public CanalController(final Properties properties){
managerClients = MigrateMap.makeComputingMap(new Function<String, PlainCanalConfigClient>() { managerClients = MigrateMap.makeComputingMap(this::getManagerClient);
public PlainCanalConfigClient apply(String managerAddress) {
return getManagerClient(managerAddress);
}
});
// 初始化全局参数设置 // 初始化全局参数设置
globalInstanceConfig = initGlobalConfig(properties); globalInstanceConfig = initGlobalConfig(properties);
@@ -147,85 +142,82 @@ public class CanalController {
final ServerRunningData serverData = new ServerRunningData(registerIp + ":" + port); final ServerRunningData serverData = new ServerRunningData(registerIp + ":" + port);
ServerRunningMonitors.setServerData(serverData); ServerRunningMonitors.setServerData(serverData);
ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() { ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap((Function<String, ServerRunningMonitor>) destination -> {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
runningMonitor.setDestination(destination);
runningMonitor.setListener(new ServerRunningListener() {
public ServerRunningMonitor apply(final String destination) { public void processActiveEnter() {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData); try {
runningMonitor.setDestination(destination); MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
runningMonitor.setListener(new ServerRunningListener() { embededCanalServer.start(destination);
if (canalMQStarter != null) {
public void processActiveEnter() { canalMQStarter.startDestination(destination);
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.start(destination);
if (canalMQStarter != null) {
canalMQStarter.startDestination(destination);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
} }
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
} }
public void processActiveExit() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (canalMQStarter != null) {
canalMQStarter.stopDestination(destination);
}
embededCanalServer.stop(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
releaseCid(path);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
} }
// 触发创建一下cid节点
runningMonitor.init(); public void processActiveExit() {
return runningMonitor; try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (canalMQStarter != null) {
canalMQStarter.stopDestination(destination);
}
embededCanalServer.stop(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
releaseCid(path);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
} }
// 触发创建一下cid节点
runningMonitor.init();
return runningMonitor;
})); }));
// 初始化monitor机制 // 初始化monitor机制
@@ -303,40 +295,37 @@ public class CanalController {
} }
}; };
instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() { instanceConfigMonitors = MigrateMap.makeComputingMap(mode -> {
int scanInterval = Integer.valueOf(getProperty(properties,
CanalConstants.CANAL_AUTO_SCAN_INTERVAL,
"5"));
public InstanceConfigMonitor apply(InstanceMode mode) { if (mode.isSpring()) {
int scanInterval = Integer.valueOf(getProperty(properties, SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
CanalConstants.CANAL_AUTO_SCAN_INTERVAL, monitor.setScanIntervalInSecond(scanInterval);
"5")); monitor.setDefaultAction(defaultAction);
// 设置conf目录，默认是user.dir + conf目录组成
if (mode.isSpring()) { String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR);
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor(); if (StringUtils.isEmpty(rootDir)) {
monitor.setScanIntervalInSecond(scanInterval); rootDir = "../conf";
monitor.setDefaultAction(defaultAction);
// 设置conf目录，默认是user.dir + conf目录组成
String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR);
if (StringUtils.isEmpty(rootDir)) {
rootDir = "../conf";
}
if (StringUtils.equals("otter-canal", System.getProperty("appName"))) {
monitor.setRootConf(rootDir);
} else {
// eclipse debug模式
monitor.setRootConf("src/main/resources/");
}
return monitor;
} else if (mode.isManager()) {
ManagerInstanceConfigMonitor monitor = new ManagerInstanceConfigMonitor();
monitor.setScanIntervalInSecond(scanInterval);
monitor.setDefaultAction(defaultAction);
String managerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
monitor.setConfigClient(getManagerClient(managerAddress));
return monitor;
} else {
throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor");
} }
if (StringUtils.equals("otter-canal", System.getProperty("appName"))) {
monitor.setRootConf(rootDir);
} else {
// eclipse debug模式
monitor.setRootConf("src/main/resources/");
}
return monitor;
} else if (mode.isManager()) {
ManagerInstanceConfigMonitor monitor = new ManagerInstanceConfigMonitor();
monitor.setScanIntervalInSecond(scanInterval);
monitor.setDefaultAction(defaultAction);
String managerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
monitor.setConfigClient(getManagerClient(managerAddress));
return monitor;
} else {
throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor");
} }
}); });
} }
@@ -373,27 +362,23 @@ public class CanalController {
globalConfig.setSpringXml(springXml); globalConfig.setSpringXml(springXml);
} }
instanceGenerator = new CanalInstanceGenerator() { instanceGenerator = destination -> {
InstanceConfig config = instanceConfigs.get(destination);
public CanalInstance generate(String destination) { if (config == null) {
InstanceConfig config = instanceConfigs.get(destination); throw new CanalServerException("can't find destination:" + destination);
if (config == null) { }
throw new CanalServerException("can't find destination:" + destination);
}
if (config.getMode().isManager()) {
PlainCanalInstanceGenerator instanceGenerator = new PlainCanalInstanceGenerator(properties);
instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress()));
instanceGenerator.setSpringXml(config.getSpringXml());
return instanceGenerator.generate(destination);
} else if (config.getMode().isSpring()) {
SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator();
instanceGenerator.setSpringXml(config.getSpringXml());
return instanceGenerator.generate(destination);
} else {
throw new UnsupportedOperationException("unknow mode :" + config.getMode());
}
if (config.getMode().isManager()) {
PlainCanalInstanceGenerator instanceGenerator = new PlainCanalInstanceGenerator(properties);
instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress()));
instanceGenerator.setSpringXml(config.getSpringXml());
return instanceGenerator.generate(destination);
} else if (config.getMode().isSpring()) {
SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator();
instanceGenerator.setSpringXml(config.getSpringXml());
return instanceGenerator.generate(destination);
} else {
throw new UnsupportedOperationException("unknow mode :" + config.getMode());
} }
}; };

8
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
View File

@@ -121,13 +121,7 @@ public class CanalLauncher {
} }
private static void setGlobalUncaughtExceptionHandler() { private static void setGlobalUncaughtExceptionHandler() {
Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { Thread.setDefaultUncaughtExceptionHandler((t, e) -> logger.error("UnCaughtException", e));
@Override
public void uncaughtException(Thread t, Throwable e) {
logger.error("UnCaughtException", e);
}
});
} }
} }

24
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStarter.java
View File

@@ -88,21 +88,17 @@ public class CanalStarter {
controller = new CanalController(properties); controller = new CanalController(properties);
controller.start(); controller.start();
logger.info("## the canal server is running now ......"); logger.info("## the canal server is running now ......");
shutdownThread = new Thread() { shutdownThread = new Thread(() -> {
try {
public void run() { logger.info("## stop the canal server");
try { controller.stop();
logger.info("## stop the canal server"); CanalLauncher.runningLatch.countDown();
controller.stop(); } catch (Throwable e) {
CanalLauncher.runningLatch.countDown(); logger.warn("##something goes wrong when stopping canal Server:", e);
} catch (Throwable e) { } finally {
logger.warn("##something goes wrong when stopping canal Server:", e); logger.info("## canal server is down.");
} finally {
logger.info("## canal server is down.");
}
} }
});
};
Runtime.getRuntime().addShutdownHook(shutdownThread); Runtime.getRuntime().addShutdownHook(shutdownThread);
if (canalMQProducer != null) { if (canalMQProducer != null) {

2
deployer/src/main/java/com/alibaba/otter/canal/deployer/admin/CanalAdminController.java
View File

@@ -106,7 +106,7 @@ public class CanalAdminController implements CanalAdmin {
public String getRunningInstances() { public String getRunningInstances() {
try { try {
Map<String, CanalInstance> instances = CanalServerWithEmbedded.instance().getCanalInstances(); Map<String, CanalInstance> instances = CanalServerWithEmbedded.instance().getCanalInstances();
List<String> runningInstances = new ArrayList<String>(); List<String> runningInstances = new ArrayList<>();
instances.forEach((destination, instance) -> { instances.forEach((destination, instance) -> {
if (instance.isStart()) { if (instance.isStart()) {
runningInstances.add(destination); runningInstances.add(destination);

25
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/ManagerInstanceConfigMonitor.java
View File

@@ -32,12 +32,7 @@ public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle impleme
private long scanIntervalInSecond = 5; private long scanIntervalInSecond = 5;
private InstanceAction defaultAction = null; private InstanceAction defaultAction = null;
private Map<String, InstanceAction> actions = new MapMaker().makeMap(); private Map<String, InstanceAction> actions = new MapMaker().makeMap();
private Map<String, PlainCanal> configs = MigrateMap.makeComputingMap(new Function<String, PlainCanal>() { private Map<String, PlainCanal> configs = MigrateMap.makeComputingMap(destination -> new PlainCanal());
public PlainCanal apply(String destination) {
return new PlainCanal();
}
});
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
new NamedThreadFactory("canal-instance-scan")); new NamedThreadFactory("canal-instance-scan"));
@@ -46,19 +41,15 @@ public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle impleme
public void start() { public void start() {
super.start(); super.start();
executor.scheduleWithFixedDelay(new Runnable() { executor.scheduleWithFixedDelay(() -> {
try {
public void run() { scan();
try { if (isFirst) {
scan(); isFirst = false;
if (isFirst) {
isFirst = false;
}
} catch (Throwable e) {
logger.error("scan failed", e);
} }
} catch (Throwable e) {
logger.error("scan failed", e);
} }
}, 0, scanIntervalInSecond, TimeUnit.SECONDS); }, 0, scanIntervalInSecond, TimeUnit.SECONDS);
} }

58
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java
View File

@@ -39,12 +39,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
private long scanIntervalInSecond = 5; private long scanIntervalInSecond = 5;
private InstanceAction defaultAction = null; private InstanceAction defaultAction = null;
private Map<String, InstanceAction> actions = new MapMaker().makeMap(); private Map<String, InstanceAction> actions = new MapMaker().makeMap();
private Map<String, InstanceConfigFiles> lastFiles = MigrateMap.makeComputingMap(new Function<String, InstanceConfigFiles>() { private Map<String, InstanceConfigFiles> lastFiles = MigrateMap.makeComputingMap(InstanceConfigFiles::new);
public InstanceConfigFiles apply(String destination) {
return new InstanceConfigFiles(destination);
}
});
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
new NamedThreadFactory("canal-instance-scan")); new NamedThreadFactory("canal-instance-scan"));
@@ -58,19 +53,15 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
super.start(); super.start();
Assert.notNull(rootConf, "root conf dir is null!"); Assert.notNull(rootConf, "root conf dir is null!");
executor.scheduleWithFixedDelay(new Runnable() { executor.scheduleWithFixedDelay(() -> {
try {
public void run() { scan();
try { if (isFirst) {
scan(); isFirst = false;
if (isFirst) {
isFirst = false;
}
} catch (Throwable e) {
logger.error("scan failed", e);
} }
} catch (Throwable e) {
logger.error("scan failed", e);
} }
}, 0, scanIntervalInSecond, TimeUnit.SECONDS); }, 0, scanIntervalInSecond, TimeUnit.SECONDS);
} }
@@ -103,29 +94,22 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
return; return;
} }
File[] instanceDirs = rootdir.listFiles(new FileFilter() { File[] instanceDirs = rootdir.listFiles(pathname -> {
String filename = pathname.getName();
public boolean accept(File pathname) { return pathname.isDirectory() && !"spring".equalsIgnoreCase(filename);
String filename = pathname.getName();
return pathname.isDirectory() && !"spring".equalsIgnoreCase(filename);
}
}); });
// 扫描目录的新增 // 扫描目录的新增
Set<String> currentInstanceNames = new HashSet<String>(); Set<String> currentInstanceNames = new HashSet<>();
// 判断目录内文件的变化 // 判断目录内文件的变化
for (File instanceDir : instanceDirs) { for (File instanceDir : instanceDirs) {
String destination = instanceDir.getName(); String destination = instanceDir.getName();
currentInstanceNames.add(destination); currentInstanceNames.add(destination);
File[] instanceConfigs = instanceDir.listFiles(new FilenameFilter() { File[] instanceConfigs = instanceDir.listFiles((dir, name) -> {
// return !StringUtils.endsWithIgnoreCase(name, ".dat");
public boolean accept(File dir, String name) { // 限制一下，只针对instance.properties文件,避免因为.svn或者其他生成的临时文件导致出现reload
// return !StringUtils.endsWithIgnoreCase(name, ".dat"); return StringUtils.equalsIgnoreCase(name, "instance.properties");
// 限制一下，只针对instance.properties文件,避免因为.svn或者其他生成的临时文件导致出现reload
return StringUtils.equalsIgnoreCase(name, "instance.properties");
}
}); });
if (!actions.containsKey(destination) && instanceConfigs.length > 0) { if (!actions.containsKey(destination) && instanceConfigs.length > 0) {
@@ -150,7 +134,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
if (hasChanged || CollectionUtils.isEmpty(lastFile.getInstanceFiles())) { if (hasChanged || CollectionUtils.isEmpty(lastFile.getInstanceFiles())) {
// 更新内容 // 更新内容
List<FileInfo> newFileInfo = new ArrayList<FileInfo>(); List<FileInfo> newFileInfo = new ArrayList<>();
for (File instanceConfig : instanceConfigs) { for (File instanceConfig : instanceConfigs) {
newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified())); newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified()));
} }
@@ -163,7 +147,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
} }
// 判断目录是否删除 // 判断目录是否删除
Set<String> deleteInstanceNames = new HashSet<String>(); Set<String> deleteInstanceNames = new HashSet<>();
for (String destination : actions.keySet()) { for (String destination : actions.keySet()) {
if (!currentInstanceNames.contains(destination)) { if (!currentInstanceNames.contains(destination)) {
deleteInstanceNames.add(destination); deleteInstanceNames.add(destination);
@@ -181,7 +165,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
// 启动成功后记录配置文件信息 // 启动成功后记录配置文件信息
InstanceConfigFiles lastFile = lastFiles.get(destination); InstanceConfigFiles lastFile = lastFiles.get(destination);
List<FileInfo> newFileInfo = new ArrayList<FileInfo>(); List<FileInfo> newFileInfo = new ArrayList<>();
for (File instanceConfig : instanceConfigs) { for (File instanceConfig : instanceConfigs) {
newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified())); newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified()));
} }
@@ -243,10 +227,10 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
private String destination; // instance private String destination; // instance
// name // name
private List<FileInfo> springFile = new ArrayList<FileInfo>(); // spring的instance private List<FileInfo> springFile = new ArrayList<>(); // spring的instance
// xml // xml
private FileInfo rootFile; // canal.properties private FileInfo rootFile; // canal.properties
private List<FileInfo> instanceFiles = new ArrayList<FileInfo>(); // instance对应的配置 private List<FileInfo> instanceFiles = new ArrayList<>(); // instance对应的配置
public InstanceConfigFiles(String destination){ public InstanceConfigFiles(String destination){
this.destination = destination; this.destination = destination;

10
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java
View File

@@ -64,7 +64,7 @@ public class MysqlQueryExecutor {
ResultSetHeaderPacket rsHeader = new ResultSetHeaderPacket(); ResultSetHeaderPacket rsHeader = new ResultSetHeaderPacket();
rsHeader.fromBytes(body); rsHeader.fromBytes(body);
List<FieldPacket> fields = new ArrayList<FieldPacket>(); List<FieldPacket> fields = new ArrayList<>();
for (int i = 0; i < rsHeader.getColumnCount(); i++) { for (int i = 0; i < rsHeader.getColumnCount(); i++) {
FieldPacket fp = new FieldPacket(); FieldPacket fp = new FieldPacket();
fp.fromBytes(readNextPacket()); fp.fromBytes(readNextPacket());
@@ -73,7 +73,7 @@ public class MysqlQueryExecutor {
readEofPacket(); readEofPacket();
List<RowDataPacket> rowData = new ArrayList<RowDataPacket>(); List<RowDataPacket> rowData = new ArrayList<>();
while (true) { while (true) {
body = readNextPacket(); body = readNextPacket();
if (body[0] == -2) { if (body[0] == -2) {
@@ -99,7 +99,7 @@ public class MysqlQueryExecutor {
cmd.setQueryString(queryString); cmd.setQueryString(queryString);
byte[] bodyBytes = cmd.toBytes(); byte[] bodyBytes = cmd.toBytes();
PacketManager.writeBody(channel, bodyBytes); PacketManager.writeBody(channel, bodyBytes);
List<ResultSetPacket> resultSets = new ArrayList<ResultSetPacket>(); List<ResultSetPacket> resultSets = new ArrayList<>();
boolean moreResult = true; boolean moreResult = true;
while (moreResult) { while (moreResult) {
byte[] body = readNextPacket(); byte[] body = readNextPacket();
@@ -112,7 +112,7 @@ public class MysqlQueryExecutor {
ResultSetHeaderPacket rsHeader = new ResultSetHeaderPacket(); ResultSetHeaderPacket rsHeader = new ResultSetHeaderPacket();
rsHeader.fromBytes(body); rsHeader.fromBytes(body);
List<FieldPacket> fields = new ArrayList<FieldPacket>(); List<FieldPacket> fields = new ArrayList<>();
for (int i = 0; i < rsHeader.getColumnCount(); i++) { for (int i = 0; i < rsHeader.getColumnCount(); i++) {
FieldPacket fp = new FieldPacket(); FieldPacket fp = new FieldPacket();
fp.fromBytes(readNextPacket()); fp.fromBytes(readNextPacket());
@@ -121,7 +121,7 @@ public class MysqlQueryExecutor {
moreResult = readEofPacket(); moreResult = readEofPacket();
List<RowDataPacket> rowData = new ArrayList<RowDataPacket>(); List<RowDataPacket> rowData = new ArrayList<>();
while (true) { while (true) {
body = readNextPacket(); body = readNextPacket();
if (body[0] == -2) { if (body[0] == -2) {

4
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/MysqlGTIDSet.java
View File

@@ -84,11 +84,11 @@ public class MysqlGTIDSet implements GTIDSet {
Map<String, UUIDSet> m; Map<String, UUIDSet> m;
if (gtidData == null || gtidData.length() < 1) { if (gtidData == null || gtidData.length() < 1) {
m = new HashMap<String, UUIDSet>(); m = new HashMap<>();
} else { } else {
// 存在多个GTID时会有回车符 // 存在多个GTID时会有回车符
String[] uuidStrs = gtidData.replaceAll("\n", "").split(","); String[] uuidStrs = gtidData.replaceAll("\n", "").split(",");
m = new HashMap<String, UUIDSet>(uuidStrs.length); m = new HashMap<>(uuidStrs.length);
for (int i = 0; i < uuidStrs.length; i++) { for (int i = 0; i < uuidStrs.length; i++) {
UUIDSet uuidSet = UUIDSet.parse(uuidStrs[i]); UUIDSet uuidSet = UUIDSet.parse(uuidStrs[i]);
m.put(uuidSet.SID.toString(), uuidSet); m.put(uuidSet.SID.toString(), uuidSet);

4
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/UUIDSet.java
View File

@@ -105,7 +105,7 @@ public class UUIDSet {
throw new RuntimeException(String.format("parseUUIDSet failed due to wrong format: %s", str)); throw new RuntimeException(String.format("parseUUIDSet failed due to wrong format: %s", str));
} }
List<Interval> intervals = new ArrayList<Interval>(); List<Interval> intervals = new ArrayList<>();
for (int i = 1; i < ss.length; i++) { for (int i = 1; i < ss.length; i++) {
intervals.add(parseInterval(ss[i])); intervals.add(parseInterval(ss[i]));
} }
@@ -172,7 +172,7 @@ public class UUIDSet {
* @return * @return
*/ */
public static List<Interval> combine(List<Interval> intervals) { public static List<Interval> combine(List<Interval> intervals) {
List<Interval> combined = new ArrayList<Interval>(); List<Interval> combined = new ArrayList<>();
Collections.sort(intervals); Collections.sort(intervals);
int len = intervals.size(); int len = intervals.size();
for (int i = 0; i < len; i++) { for (int i = 0; i < len; i++) {

4
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ResultSetPacket.java
View File

@@ -7,8 +7,8 @@ import java.util.List;
public class ResultSetPacket { public class ResultSetPacket {
private SocketAddress sourceAddress; private SocketAddress sourceAddress;
private List<FieldPacket> fieldDescriptors = new ArrayList<FieldPacket>(); private List<FieldPacket> fieldDescriptors = new ArrayList<>();
private List<String> fieldValues = new ArrayList<String>(); private List<String> fieldValues = new ArrayList<>();
public void setFieldDescriptors(List<FieldPacket> fieldDescriptors) { public void setFieldDescriptors(List<FieldPacket> fieldDescriptors) {
this.fieldDescriptors = fieldDescriptors; this.fieldDescriptors = fieldDescriptors;

2
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/RowDataPacket.java
View File

@@ -9,7 +9,7 @@ import com.alibaba.otter.canal.parse.driver.mysql.utils.LengthCodedStringReader;
public class RowDataPacket extends PacketWithHeaderPacket { public class RowDataPacket extends PacketWithHeaderPacket {
private List<String> columns = new ArrayList<String>(); private List<String> columns = new ArrayList<>();
public void fromBytes(byte[] data) throws IOException { public void fromBytes(byte[] data) throws IOException {
int index = 0; int index = 0;

2
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/NettySocketChannelPool.java
View File

@@ -31,7 +31,7 @@ public abstract class NettySocketChannelPool {
private static EventLoopGroup group = new NioEventLoopGroup(); // 非阻塞IO线程组 private static EventLoopGroup group = new NioEventLoopGroup(); // 非阻塞IO线程组
private static Bootstrap boot = new Bootstrap(); // 主 private static Bootstrap boot = new Bootstrap(); // 主
private static Map<Channel, SocketChannel> chManager = new ConcurrentHashMap<Channel, SocketChannel>(); private static Map<Channel, SocketChannel> chManager = new ConcurrentHashMap<>();
private static final Logger logger = LoggerFactory.getLogger(NettySocketChannelPool.class); private static final Logger logger = LoggerFactory.getLogger(NettySocketChannelPool.class);
static { static {

2
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/CharsetUtil.java
View File

@@ -14,7 +14,7 @@ import org.apache.commons.lang.StringUtils;
public class CharsetUtil { public class CharsetUtil {
private static final String[] INDEX_TO_CHARSET = new String[2048]; private static final String[] INDEX_TO_CHARSET = new String[2048];
private static final Map<String, Integer> CHARSET_TO_INDEX = new HashMap<String, Integer>(); private static final Map<String, Integer> CHARSET_TO_INDEX = new HashMap<>();
static { static {
INDEX_TO_CHARSET[1] = "big5"; INDEX_TO_CHARSET[1] = "big5";
INDEX_TO_CHARSET[84] = "big5"; INDEX_TO_CHARSET[84] = "big5";

6
driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlGTIDSetTest.java
View File

@@ -38,7 +38,7 @@ public class MysqlGTIDSetTest {
@Test @Test
public void testParse() { public void testParse() {
Map<String, MysqlGTIDSet> cases = new HashMap<String, MysqlGTIDSet>(5); Map<String, MysqlGTIDSet> cases = new HashMap<>(5);
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1", cases.put("726757ad-4455-11e8-ae04-0242ac110002:1",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 2))); buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 2)));
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3", cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3",
@@ -89,12 +89,12 @@ public class MysqlGTIDSetTest {
} }
private MysqlGTIDSet buildForTest(List<Material> materials) { private MysqlGTIDSet buildForTest(List<Material> materials) {
Map<String, UUIDSet> sets = new HashMap<String, UUIDSet>(); Map<String, UUIDSet> sets = new HashMap<>();
for (Material a : materials) { for (Material a : materials) {
UUIDSet.Interval interval = new UUIDSet.Interval(); UUIDSet.Interval interval = new UUIDSet.Interval();
interval.start = a.start; interval.start = a.start;
interval.stop = a.stop; interval.stop = a.stop;
List<UUIDSet.Interval> intervals = new ArrayList<UUIDSet.Interval>(); List<UUIDSet.Interval> intervals = new ArrayList<>();
intervals.add(interval); intervals.add(interval);
if (a.start1 > 0 && a.stop1 > 0) { if (a.start1 > 0 && a.stop1 > 0) {

2
driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/UUIDSetTest.java
View File

@@ -16,7 +16,7 @@ public class UUIDSetTest {
@Test @Test
public void testToString() { public void testToString() {
Map<String, String> cases = new HashMap<String, String>(4); Map<String, String> cases = new HashMap<>(4);
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1", "726757ad-4455-11e8-ae04-0242ac110002:1"); cases.put("726757ad-4455-11e8-ae04-0242ac110002:1", "726757ad-4455-11e8-ae04-0242ac110002:1");
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3", "726757ad-4455-11e8-ae04-0242ac110002:1-3"); cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3", "726757ad-4455-11e8-ae04-0242ac110002:1-3");
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3:4-6", "726757ad-4455-11e8-ae04-0242ac110002:1-6"); cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3:4-6", "726757ad-4455-11e8-ae04-0242ac110002:1-6");

7
example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java
View File

@@ -25,12 +25,7 @@ public class AbstractCanalClientTest extends BaseCanalClientTest {
protected void start() { protected void start() {
Assert.notNull(connector, "connector is null"); Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() { thread = new Thread(this::process);
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler); thread.setUncaughtExceptionHandler(handler);
running = true; running = true;

7
example/src/main/java/com/alibaba/otter/canal/example/BaseCanalClientTest.java
View File

@@ -31,12 +31,7 @@ public class BaseCanalClientTest {
protected static final String SEP = SystemUtils.LINE_SEPARATOR; protected static final String SEP = SystemUtils.LINE_SEPARATOR;
protected static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; protected static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
protected volatile boolean running = false; protected volatile boolean running = false;
protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
protected Thread thread = null; protected Thread thread = null;
protected CanalConnector connector; protected CanalConnector connector;
protected static String context_format = null; protected static String context_format = null;

22
example/src/main/java/com/alibaba/otter/canal/example/ClusterCanalClientTest.java
View File

@@ -32,19 +32,15 @@ public class ClusterCanalClientTest extends AbstractCanalClientTest {
clientTest.setConnector(connector); clientTest.setConnector(connector);
clientTest.start(); clientTest.start();
Runtime.getRuntime().addShutdownHook(new Thread() { Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
public void run() { logger.info("## stop the canal client");
try { clientTest.stop();
logger.info("## stop the canal client"); } catch (Throwable e) {
clientTest.stop(); logger.warn("##something goes wrong when stopping canal:", e);
} catch (Throwable e) { } finally {
logger.warn("##something goes wrong when stopping canal:", e); logger.info("## canal client is down.");
} finally {
logger.info("## canal client is down.");
}
} }
}));
});
} }
} }

18
example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientPermanceTest.java
View File

@@ -20,23 +20,19 @@ public class SimpleCanalClientPermanceTest {
int perSum = 0; int perSum = 0;
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
long end = 0; long end = 0;
final ArrayBlockingQueue<Long> queue = new ArrayBlockingQueue<Long>(100); final ArrayBlockingQueue<Long> queue = new ArrayBlockingQueue<>(100);
try { try {
final CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, 11111), final CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, 11111),
destination, destination,
"canal", "canal",
"canal"); "canal");
Thread ackThread = new Thread(new Runnable() { Thread ackThread = new Thread(() -> {
while (true) {
@Override try {
public void run() { long batchId = queue.take();
while (true) { connector.ack(batchId);
try { } catch (InterruptedException e) {
long batchId = queue.take();
connector.ack(batchId);
} catch (InterruptedException e) {
}
} }
} }
}); });

22
example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientTest.java
View File

@@ -30,20 +30,16 @@ public class SimpleCanalClientTest extends AbstractCanalClientTest {
final SimpleCanalClientTest clientTest = new SimpleCanalClientTest(destination); final SimpleCanalClientTest clientTest = new SimpleCanalClientTest(destination);
clientTest.setConnector(connector); clientTest.setConnector(connector);
clientTest.start(); clientTest.start();
Runtime.getRuntime().addShutdownHook(new Thread() { Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
public void run() { logger.info("## stop the canal client");
try { clientTest.stop();
logger.info("## stop the canal client"); } catch (Throwable e) {
clientTest.stop(); logger.warn("##something goes wrong when stopping canal:", e);
} catch (Throwable e) { } finally {
logger.warn("##something goes wrong when stopping canal:", e); logger.info("## canal client is down.");
} finally {
logger.info("## canal client is down.");
}
} }
}));
});
} }
} }

36
example/src/main/java/com/alibaba/otter/canal/example/kafka/CanalKafkaClientExample.java
View File

@@ -26,12 +26,7 @@ public class CanalKafkaClientExample {
private Thread thread = null; private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { private Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId){ public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId){
connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false); connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false);
@@ -47,20 +42,16 @@ public class CanalKafkaClientExample {
logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId); logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
kafkaCanalClientExample.start(); kafkaCanalClientExample.start();
logger.info("## the canal kafka consumer is running now ......"); logger.info("## the canal kafka consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() { Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
public void run() { logger.info("## stop the kafka consumer");
try { kafkaCanalClientExample.stop();
logger.info("## stop the kafka consumer"); } catch (Throwable e) {
kafkaCanalClientExample.stop(); logger.warn("##something goes wrong when stopping kafka consumer:", e);
} catch (Throwable e) { } finally {
logger.warn("##something goes wrong when stopping kafka consumer:", e); logger.info("## kafka consumer is down.");
} finally {
logger.info("## kafka consumer is down.");
}
} }
}));
});
while (running) while (running)
; ;
} catch (Throwable e) { } catch (Throwable e) {
@@ -71,12 +62,7 @@ public class CanalKafkaClientExample {
public void start() { public void start() {
Assert.notNull(connector, "connector is null"); Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() { thread = new Thread(this::process);
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler); thread.setUncaughtExceptionHandler(handler);
thread.start(); thread.start();
running = true; running = true;

36
example/src/main/java/com/alibaba/otter/canal/example/kafka/CanalKafkaClientFlatMessageExample.java
View File

@@ -26,12 +26,7 @@ public class CanalKafkaClientFlatMessageExample {
private Thread thread = null; private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { private Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public CanalKafkaClientFlatMessageExample(String zkServers, String servers, String topic, Integer partition, public CanalKafkaClientFlatMessageExample(String zkServers, String servers, String topic, Integer partition,
String groupId){ String groupId){
@@ -48,20 +43,16 @@ public class CanalKafkaClientFlatMessageExample {
logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId); logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
kafkaCanalClientExample.start(); kafkaCanalClientExample.start();
logger.info("## the canal kafka consumer is running now ......"); logger.info("## the canal kafka consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() { Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
public void run() { logger.info("## stop the kafka consumer");
try { kafkaCanalClientExample.stop();
logger.info("## stop the kafka consumer"); } catch (Throwable e) {
kafkaCanalClientExample.stop(); logger.warn("##something goes wrong when stopping kafka consumer:", e);
} catch (Throwable e) { } finally {
logger.warn("##something goes wrong when stopping kafka consumer:", e); logger.info("## kafka consumer is down.");
} finally {
logger.info("## kafka consumer is down.");
}
} }
}));
});
while (running) while (running)
; ;
} catch (Throwable e) { } catch (Throwable e) {
@@ -72,12 +63,7 @@ public class CanalKafkaClientFlatMessageExample {
public void start() { public void start() {
Assert.notNull(connector, "connector is null"); Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() { thread = new Thread(this::process);
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler); thread.setUncaughtExceptionHandler(handler);
thread.start(); thread.start();
running = true; running = true;

36
example/src/main/java/com/alibaba/otter/canal/example/kafka/CanalKafkaOffsetClientExample.java
View File

@@ -35,12 +35,7 @@ public class CanalKafkaOffsetClientExample {
private Thread thread = null; private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { private Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public CanalKafkaOffsetClientExample(String servers, String topic, Integer partition, String groupId){ public CanalKafkaOffsetClientExample(String servers, String topic, Integer partition, String groupId){
connector = new KafkaOffsetCanalConnector(servers, topic, partition, groupId, false); connector = new KafkaOffsetCanalConnector(servers, topic, partition, groupId, false);
@@ -55,20 +50,16 @@ public class CanalKafkaOffsetClientExample {
logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId); logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
kafkaCanalClientExample.start(); kafkaCanalClientExample.start();
logger.info("## the canal kafka consumer is running now ......"); logger.info("## the canal kafka consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() { Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
public void run() { logger.info("## stop the kafka consumer");
try { kafkaCanalClientExample.stop();
logger.info("## stop the kafka consumer"); } catch (Throwable e) {
kafkaCanalClientExample.stop(); logger.warn("##something goes wrong when stopping kafka consumer:", e);
} catch (Throwable e) { } finally {
logger.warn("##something goes wrong when stopping kafka consumer:", e); logger.info("## kafka consumer is down.");
} finally {
logger.info("## kafka consumer is down.");
}
} }
}));
});
while (running) while (running)
; ;
} catch (Throwable e) { } catch (Throwable e) {
@@ -79,12 +70,7 @@ public class CanalKafkaOffsetClientExample {
public void start() { public void start() {
Assert.notNull(connector, "connector is null"); Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() { thread = new Thread(this::process);
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler); thread.setUncaughtExceptionHandler(handler);
thread.start(); thread.start();
running = true; running = true;

24
example/src/main/java/com/alibaba/otter/canal/example/kafka/KafkaClientRunningTest.java
View File

@@ -26,22 +26,18 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
public void testKafkaConsumer() { public void testKafkaConsumer() {
final ExecutorService executor = Executors.newFixedThreadPool(1); final ExecutorService executor = Executors.newFixedThreadPool(1);
final KafkaCanalConnector connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false); final KafkaCanalConnector connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false);
executor.submit(new Runnable() { executor.submit(() -> {
connector.connect();
@Override connector.subscribe();
public void run() { while (running) {
connector.connect(); List<Message> messages = connector.getList(3L, TimeUnit.SECONDS);
connector.subscribe(); if (messages != null) {
while (running) { System.out.println(messages);
List<Message> messages = connector.getList(3L, TimeUnit.SECONDS);
if (messages != null) {
System.out.println(messages);
}
connector.ack();
} }
connector.unsubscribe(); connector.ack();
connector.disconnect();
} }
connector.unsubscribe();
connector.disconnect();
}); });
sleep(60000); sleep(60000);

36
example/src/main/java/com/alibaba/otter/canal/example/rocketmq/CanalRocketMQClientExample.java
View File

@@ -24,12 +24,7 @@ public class CanalRocketMQClientExample extends AbstractRocektMQTest {
private Thread thread = null; private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { private Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public CanalRocketMQClientExample(String nameServers, String topic, String groupId) { public CanalRocketMQClientExample(String nameServers, String topic, String groupId) {
connector = new RocketMQCanalConnector(nameServers, topic, groupId, 500, false); connector = new RocketMQCanalConnector(nameServers, topic, groupId, 500, false);
@@ -55,20 +50,16 @@ public class CanalRocketMQClientExample extends AbstractRocektMQTest {
logger.info("## Start the rocketmq consumer: {}-{}", topic, groupId); logger.info("## Start the rocketmq consumer: {}-{}", topic, groupId);
rocketMQClientExample.start(); rocketMQClientExample.start();
logger.info("## The canal rocketmq consumer is running now ......"); logger.info("## The canal rocketmq consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() { Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
public void run() { logger.info("## Stop the rocketmq consumer");
try { rocketMQClientExample.stop();
logger.info("## Stop the rocketmq consumer"); } catch (Throwable e) {
rocketMQClientExample.stop(); logger.warn("## Something goes wrong when stopping rocketmq consumer:", e);
} catch (Throwable e) { } finally {
logger.warn("## Something goes wrong when stopping rocketmq consumer:", e); logger.info("## Rocketmq consumer is down.");
} finally {
logger.info("## Rocketmq consumer is down.");
}
} }
}));
});
while (running) while (running)
; ;
} catch (Throwable e) { } catch (Throwable e) {
@@ -79,12 +70,7 @@ public class CanalRocketMQClientExample extends AbstractRocektMQTest {
public void start() { public void start() {
Assert.notNull(connector, "connector is null"); Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() { thread = new Thread(this::process);
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler); thread.setUncaughtExceptionHandler(handler);
thread.start(); thread.start();
running = true; running = true;

36
example/src/main/java/com/alibaba/otter/canal/example/rocketmq/CanalRocketMQClientFlatMessageExample.java
View File

@@ -26,12 +26,7 @@ public class CanalRocketMQClientFlatMessageExample extends AbstractRocektMQTest
private Thread thread = null; private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { private Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public CanalRocketMQClientFlatMessageExample(String nameServers, String topic, String groupId){ public CanalRocketMQClientFlatMessageExample(String nameServers, String topic, String groupId){
connector = new RocketMQCanalConnector(nameServers, topic, groupId, 500, true); connector = new RocketMQCanalConnector(nameServers, topic, groupId, 500, true);
@@ -45,20 +40,16 @@ public class CanalRocketMQClientFlatMessageExample extends AbstractRocektMQTest
logger.info("## Start the rocketmq consumer: {}-{}", topic, groupId); logger.info("## Start the rocketmq consumer: {}-{}", topic, groupId);
rocketMQClientExample.start(); rocketMQClientExample.start();
logger.info("## The canal rocketmq consumer is running now ......"); logger.info("## The canal rocketmq consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() { Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
public void run() { logger.info("## Stop the rocketmq consumer");
try { rocketMQClientExample.stop();
logger.info("## Stop the rocketmq consumer"); } catch (Throwable e) {
rocketMQClientExample.stop(); logger.warn("## Something goes wrong when stopping rocketmq consumer:", e);
} catch (Throwable e) { } finally {
logger.warn("## Something goes wrong when stopping rocketmq consumer:", e); logger.info("## Rocketmq consumer is down.");
} finally {
logger.info("## Rocketmq consumer is down.");
}
} }
}));
});
while (running) while (running)
; ;
} catch (Throwable e) { } catch (Throwable e) {
@@ -69,12 +60,7 @@ public class CanalRocketMQClientFlatMessageExample extends AbstractRocektMQTest
public void start() { public void start() {
Assert.notNull(connector, "connector is null"); Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() { thread = new Thread(this::process);
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler); thread.setUncaughtExceptionHandler(handler);
thread.start(); thread.start();
running = true; running = true;

25
filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java
View File

@@ -14,20 +14,17 @@ import com.google.common.collect.MigrateMap;
public class PatternUtils { public class PatternUtils {
private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(CacheBuilder.newBuilder().softValues(), private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(CacheBuilder.newBuilder().softValues(),
new Function<String, Pattern>() { pattern -> {
try {
public Pattern apply(String pattern) { PatternCompiler pc = new Perl5Compiler();
try { return pc.compile(pattern,
PatternCompiler pc = new Perl5Compiler(); Perl5Compiler.CASE_INSENSITIVE_MASK
return pc.compile(pattern, | Perl5Compiler.READ_ONLY_MASK
Perl5Compiler.CASE_INSENSITIVE_MASK | Perl5Compiler.SINGLELINE_MASK);
| Perl5Compiler.READ_ONLY_MASK } catch (MalformedPatternException e) {
| Perl5Compiler.SINGLELINE_MASK); throw new RuntimeException(e);
} catch (MalformedPatternException e) { }
throw new RuntimeException(e); });
}
}
});
public static Pattern getPattern(String pattern) { public static Pattern getPattern(String pattern) {
return patterns.get(pattern); return patterns.get(pattern);

2
filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterELFilter.java
View File

@@ -29,7 +29,7 @@ public class AviaterELFilter implements CanalEventFilter<CanalEntry.Entry> {
return true; return true;
} }
Map<String, Object> env = new HashMap<String, Object>(); Map<String, Object> env = new HashMap<>();
env.put(ROOT_KEY, entry); env.put(ROOT_KEY, entry);
return (Boolean) AviatorEvaluator.execute(expression, env); return (Boolean) AviatorEvaluator.execute(expression, env);
} }

6
filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java
View File

@@ -44,7 +44,7 @@ public class AviaterRegexFilter implements CanalEventFilter<String> {
this.defaultEmptyValue = defaultEmptyValue; this.defaultEmptyValue = defaultEmptyValue;
List<String> list = null; List<String> list = null;
if (StringUtils.isEmpty(pattern)) { if (StringUtils.isEmpty(pattern)) {
list = new ArrayList<String>(); list = new ArrayList<>();
} else { } else {
String[] ss = StringUtils.split(pattern, SPLIT); String[] ss = StringUtils.split(pattern, SPLIT);
list = Arrays.asList(ss); list = Arrays.asList(ss);
@@ -68,7 +68,7 @@ public class AviaterRegexFilter implements CanalEventFilter<String> {
return defaultEmptyValue; return defaultEmptyValue;
} }
Map<String, Object> env = new HashMap<String, Object>(); Map<String, Object> env = new HashMap<>();
env.put("pattern", pattern); env.put("pattern", pattern);
env.put("target", filtered.toLowerCase()); env.put("target", filtered.toLowerCase());
return (Boolean) exp.execute(env); return (Boolean) exp.execute(env);
@@ -114,7 +114,7 @@ public class AviaterRegexFilter implements CanalEventFilter<String> {
*/ */
private List<String> completionPattern(List<String> patterns) { private List<String> completionPattern(List<String> patterns) {
List<String> result = new ArrayList<String>(); List<String> result = new ArrayList<>();
for (String pattern : patterns) { for (String pattern : patterns) {
StringBuffer stringBuffer = new StringBuffer(); StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("^"); stringBuffer.append("^");

4
filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterSimpleFilter.java
View File

@@ -30,7 +30,7 @@ public class AviaterSimpleFilter implements CanalEventFilter<String> {
public AviaterSimpleFilter(String filterExpression){ public AviaterSimpleFilter(String filterExpression){
if (StringUtils.isEmpty(filterExpression)) { if (StringUtils.isEmpty(filterExpression)) {
list = new ArrayList<String>(); list = new ArrayList<>();
} else { } else {
String[] ss = filterExpression.toLowerCase().split(SPLIT); String[] ss = filterExpression.toLowerCase().split(SPLIT);
list = Arrays.asList(ss); list = Arrays.asList(ss);
@@ -44,7 +44,7 @@ public class AviaterSimpleFilter implements CanalEventFilter<String> {
if (StringUtils.isEmpty(filtered)) { if (StringUtils.isEmpty(filtered)) {
return true; return true;
} }
Map<String, Object> env = new HashMap<String, Object>(); Map<String, Object> env = new HashMap<>();
env.put("list", list); env.put("list", list);
env.put("target", filtered.toLowerCase()); env.put("target", filtered.toLowerCase());
return (Boolean) exp.execute(env); return (Boolean) exp.execute(env);

27
filter/src/test/java/com/alibaba/otter/canal/filter/MutliAviaterFilterTest.java
View File

@@ -21,22 +21,19 @@ public class MutliAviaterFilterTest {
final CountDownLatch countDown = new CountDownLatch(count); final CountDownLatch countDown = new CountDownLatch(count);
final AtomicInteger successed = new AtomicInteger(0); final AtomicInteger successed = new AtomicInteger(0);
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
executor.submit(new Runnable() { executor.submit(() -> {
try {
public void run() { for (int i1 = 0; i1 < 100; i1++) {
try { doRegexTest();
for (int i = 0; i < 100; i++) { // try {
doRegexTest(); // Thread.sleep(10);
// try { // } catch (InterruptedException e) {
// Thread.sleep(10); // }
// } catch (InterruptedException e) {
// }
}
successed.incrementAndGet();
} finally {
countDown.countDown();
} }
successed.incrementAndGet();
} finally {
countDown.countDown();
} }
}); });
} }

16
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
View File

@@ -118,13 +118,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
} else { } else {
try { try {
File externalLibDir = new File(alarmHandlerPluginDir); File externalLibDir = new File(alarmHandlerPluginDir);
File[] jarFiles = externalLibDir.listFiles(new FilenameFilter() { File[] jarFiles = externalLibDir.listFiles((dir, name) -> name.endsWith(".jar"));
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".jar");
}
});
if (jarFiles == null || jarFiles.length == 0) { if (jarFiles == null || jarFiles.length == 0) {
throw new IllegalStateException(String.format("alarmHandlerPluginDir [%s] can't find any name endswith \".jar\" file.", throw new IllegalStateException(String.format("alarmHandlerPluginDir [%s] can't find any name endswith \".jar\" file.",
alarmHandlerPluginDir)); alarmHandlerPluginDir));
@@ -242,9 +236,9 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
List<List<DataSourcing>> groupDbAddresses = parameters.getGroupDbAddresses(); List<List<DataSourcing>> groupDbAddresses = parameters.getGroupDbAddresses();
if (!CollectionUtils.isEmpty(groupDbAddresses)) { if (!CollectionUtils.isEmpty(groupDbAddresses)) {
int size = groupDbAddresses.get(0).size();// 取第一个分组的数量，主备分组的数量必须一致 int size = groupDbAddresses.get(0).size();// 取第一个分组的数量，主备分组的数量必须一致
List<CanalEventParser> eventParsers = new ArrayList<CanalEventParser>(); List<CanalEventParser> eventParsers = new ArrayList<>();
for (int i = 0; i < size; i++) { for (int i = 0; i < size; i++) {
List<InetSocketAddress> dbAddress = new ArrayList<InetSocketAddress>(); List<InetSocketAddress> dbAddress = new ArrayList<>();
SourcingType lastType = null; SourcingType lastType = null;
for (List<DataSourcing> groupDbAddress : groupDbAddresses) { for (List<DataSourcing> groupDbAddress : groupDbAddresses) {
if (lastType != null && !lastType.equals(groupDbAddress.get(i).getType())) { if (lastType != null && !lastType.equals(groupDbAddress.get(i).getType())) {
@@ -270,7 +264,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
} }
} else { } else {
// 创建一个空数据库地址的parser，可能使用了tddl指定地址，启动的时候才会从tddl获取地址 // 创建一个空数据库地址的parser，可能使用了tddl指定地址，启动的时候才会从tddl获取地址
this.eventParser = doInitEventParser(type, new ArrayList<InetSocketAddress>()); this.eventParser = doInitEventParser(type, new ArrayList<>());
} }
logger.info("init eventParser end! \n\t load CanalEventParser:{}", eventParser.getClass().getName()); logger.info("init eventParser end! \n\t load CanalEventParser:{}", eventParser.getClass().getName());
@@ -494,7 +488,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
private synchronized ZkClientx getZkclientx() { private synchronized ZkClientx getZkclientx() {
// 做一下排序，保证相同的机器只使用同一个链接 // 做一下排序，保证相同的机器只使用同一个链接
List<String> zkClusters = new ArrayList<String>(parameters.getZkClusters()); List<String> zkClusters = new ArrayList<>(parameters.getZkClusters());
Collections.sort(zkClusters); Collections.sort(zkClusters);
return ZkClientx.getZkClient(StringUtils.join(zkClusters, ";")); return ZkClientx.getZkClient(StringUtils.join(zkClusters, ";"));

12
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
View File

@@ -661,7 +661,7 @@ public class CanalParameter implements Serializable {
public List<InetSocketAddress> getDbAddresses() { public List<InetSocketAddress> getDbAddresses() {
if (dbAddresses == null) { if (dbAddresses == null) {
dbAddresses = new ArrayList<InetSocketAddress>(); dbAddresses = new ArrayList<>();
if (masterAddress != null) { if (masterAddress != null) {
dbAddresses.add(masterAddress); dbAddresses.add(masterAddress);
} }
@@ -675,22 +675,22 @@ public class CanalParameter implements Serializable {
public List<List<DataSourcing>> getGroupDbAddresses() { public List<List<DataSourcing>> getGroupDbAddresses() {
if (groupDbAddresses == null) { if (groupDbAddresses == null) {
groupDbAddresses = new ArrayList<List<DataSourcing>>(); groupDbAddresses = new ArrayList<>();
if (dbAddresses != null) { if (dbAddresses != null) {
for (InetSocketAddress address : dbAddresses) { for (InetSocketAddress address : dbAddresses) {
List<DataSourcing> groupAddresses = new ArrayList<DataSourcing>(); List<DataSourcing> groupAddresses = new ArrayList<>();
groupAddresses.add(new DataSourcing(sourcingType, address)); groupAddresses.add(new DataSourcing(sourcingType, address));
groupDbAddresses.add(groupAddresses); groupDbAddresses.add(groupAddresses);
} }
} else { } else {
if (masterAddress != null) { if (masterAddress != null) {
List<DataSourcing> groupAddresses = new ArrayList<DataSourcing>(); List<DataSourcing> groupAddresses = new ArrayList<>();
groupAddresses.add(new DataSourcing(sourcingType, masterAddress)); groupAddresses.add(new DataSourcing(sourcingType, masterAddress));
groupDbAddresses.add(groupAddresses); groupDbAddresses.add(groupAddresses);
} }
if (standbyAddress != null) { if (standbyAddress != null) {
List<DataSourcing> groupAddresses = new ArrayList<DataSourcing>(); List<DataSourcing> groupAddresses = new ArrayList<>();
groupAddresses.add(new DataSourcing(sourcingType, standbyAddress)); groupAddresses.add(new DataSourcing(sourcingType, standbyAddress));
groupDbAddresses.add(groupAddresses); groupDbAddresses.add(groupAddresses);
} }
@@ -731,7 +731,7 @@ public class CanalParameter implements Serializable {
public List<String> getPositions() { public List<String> getPositions() {
if (positions == null) { if (positions == null) {
positions = new ArrayList<String>(); positions = new ArrayList<>();
String masterPosition = buildPosition(masterLogfileName, masterLogfileOffest, masterTimestamp); String masterPosition = buildPosition(masterLogfileName, masterLogfileOffest, masterTimestamp);
if (masterPosition != null) { if (masterPosition != null) {
positions.add(masterPosition); positions.add(masterPosition);

8
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/plain/HttpHelper.java
View File

@@ -57,13 +57,7 @@ public class HttpHelper {
// 创建支持忽略证书的https // 创建支持忽略证书的https
try { try {
SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() { SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, (x509Certificates, s) -> true).build();
@Override
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
}).build();
httpclient = HttpClientBuilder.create() httpclient = HttpClientBuilder.create()
.setSSLContext(sslContext) .setSSLContext(sslContext)

11
instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/PropertyPlaceholderConfigurer.java
View File

@@ -23,14 +23,7 @@ public class PropertyPlaceholderConfigurer extends org.springframework.beans.fac
private static final String PLACEHOLDER_PREFIX = "${"; private static final String PLACEHOLDER_PREFIX = "${";
private static final String PLACEHOLDER_SUFFIX = "}"; private static final String PLACEHOLDER_SUFFIX = "}";
public static ThreadLocal<Properties> propertiesLocal = new ThreadLocal<Properties>() { public static ThreadLocal<Properties> propertiesLocal = ThreadLocal.withInitial(Properties::new);
@Override
protected Properties initialValue() {
return new Properties();
}
};
private ResourceLoader loader; private ResourceLoader loader;
private String[] locationNames; private String[] locationNames;
@@ -57,7 +50,7 @@ public class PropertyPlaceholderConfigurer extends org.springframework.beans.fac
} }
if (locationNames != null) { if (locationNames != null) {
List<Resource> resources = new ArrayList<Resource>(locationNames.length); List<Resource> resources = new ArrayList<>(locationNames.length);
for (String location : locationNames) { for (String location : locationNames) {
location = trimToNull(location); location = trimToNull(location);

86
meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java
View File

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.meta;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
@@ -43,7 +44,7 @@ import com.google.common.collect.MigrateMap;
public class FileMixedMetaManager extends MemoryMetaManager implements CanalMetaManager { public class FileMixedMetaManager extends MemoryMetaManager implements CanalMetaManager {
private static final Logger logger = LoggerFactory.getLogger(FileMixedMetaManager.class); private static final Logger logger = LoggerFactory.getLogger(FileMixedMetaManager.class);
private static final Charset charset = Charset.forName("UTF-8"); private static final Charset charset = StandardCharsets.UTF_8;
private File dataDir; private File dataDir;
private String dataFileName = "meta.dat"; private String dataFileName = "meta.dat";
private Map<String, File> dataFileCaches; private Map<String, File> dataFileCaches;
@@ -69,58 +70,41 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write"); throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
} }
dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>() { dataFileCaches = MigrateMap.makeComputingMap(this::getDataFile);
public File apply(String destination) {
return getDataFile(destination);
}
});
executor = Executors.newScheduledThreadPool(1); executor = Executors.newScheduledThreadPool(1);
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() { destinations = MigrateMap.makeComputingMap(this::loadClientIdentity);
public List<ClientIdentity> apply(String destination) { cursors = MigrateMap.makeComputingMap(clientIdentity -> {
return loadClientIdentity(destination); Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
return position;
} }
}); });
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() { updateCursorTasks = Collections.synchronizedSet(new HashSet<>());
public Position apply(ClientIdentity clientIdentity) {
Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
return position;
}
}
});
updateCursorTasks = Collections.synchronizedSet(new HashSet<ClientIdentity>());
// 启动定时工作任务 // 启动定时工作任务
executor.scheduleAtFixedRate(new Runnable() { executor.scheduleAtFixedRate(() -> {
List<ClientIdentity> tasks = new ArrayList<>(updateCursorTasks);
public void run() { for (ClientIdentity clientIdentity : tasks) {
List<ClientIdentity> tasks = new ArrayList<ClientIdentity>(updateCursorTasks); MDC.put("destination", String.valueOf(clientIdentity.getDestination()));
for (ClientIdentity clientIdentity : tasks) { try {
MDC.put("destination", String.valueOf(clientIdentity.getDestination())); // 定时将内存中的最新值刷到file中，多次变更只刷一次
try { if (logger.isInfoEnabled()) {
// 定时将内存中的最新值刷到file中，多次变更只刷一次 LogPosition cursor = (LogPosition) getCursor(clientIdentity);
if (logger.isInfoEnabled()) { logger.info("clientId:{} cursor:[{},{},{},{},{}] address[{}]", clientIdentity.getClientId(), cursor.getPostion().getJournalName(),
LogPosition cursor = (LogPosition) getCursor(clientIdentity); cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(),
logger.info("clientId:{} cursor:[{},{},{},{},{}] address[{}]", new Object[] { cursor.getPostion().getServerId(), cursor.getPostion().getGtid(),
clientIdentity.getClientId(), cursor.getPostion().getJournalName(), cursor.getIdentity().getSourceAddress().toString());
cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(),
cursor.getPostion().getServerId(), cursor.getPostion().getGtid(),
cursor.getIdentity().getSourceAddress().toString() });
}
flushDataToFile(clientIdentity.getDestination());
updateCursorTasks.remove(clientIdentity);
} catch (Throwable e) {
// ignore
logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
} }
flushDataToFile(clientIdentity.getDestination());
updateCursorTasks.remove(clientIdentity);
} catch (Throwable e) {
// ignore
logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
} }
} }
}, },
@@ -142,24 +126,14 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
super.subscribe(clientIdentity); super.subscribe(clientIdentity);
// 订阅信息频率发生比较低，不需要做定时merge处理 // 订阅信息频率发生比较低，不需要做定时merge处理
executor.submit(new Runnable() { executor.submit(() -> flushDataToFile(clientIdentity.getDestination()));
public void run() {
flushDataToFile(clientIdentity.getDestination());
}
});
} }
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException { public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.unsubscribe(clientIdentity); super.unsubscribe(clientIdentity);
// 订阅信息频率发生比较低，不需要做定时merge处理 // 订阅信息频率发生比较低，不需要做定时merge处理
executor.submit(new Runnable() { executor.submit(() -> flushDataToFile(clientIdentity.getDestination()));
public void run() {
flushDataToFile(clientIdentity.getDestination());
}
});
} }
public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException { public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {

15
meta/src/main/java/com/alibaba/otter/canal/meta/MemoryMetaManager.java
View File

@@ -32,22 +32,11 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
public void start() { public void start() {
super.start(); super.start();
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() { batches = MigrateMap.makeComputingMap(MemoryClientIdentityBatch::create);
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
return MemoryClientIdentityBatch.create(clientIdentity);
}
});
cursors = new MapMaker().makeMap(); cursors = new MapMaker().makeMap();
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() { destinations = MigrateMap.makeComputingMap(destination -> Lists.newArrayList());
public List<ClientIdentity> apply(String destination) {
return Lists.newArrayList();
}
});
} }
public void stop() { public void stop() {

90
meta/src/main/java/com/alibaba/otter/canal/meta/MixedMetaManager.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.meta; package com.alibaba.otter.canal.meta;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
@@ -11,7 +10,6 @@ import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity; import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position; import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange; import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.base.Function;
import com.google.common.collect.MigrateMap; import com.google.common.collect.MigrateMap;
/** /**
@@ -37,36 +35,25 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
} }
executor = Executors.newFixedThreadPool(1); executor = Executors.newFixedThreadPool(1);
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() { destinations = MigrateMap.makeComputingMap(destination -> zooKeeperMetaManager.listAllSubscribeInfo(destination));
public List<ClientIdentity> apply(String destination) { cursors = MigrateMap.makeComputingMap(clientIdentity -> {
return zooKeeperMetaManager.listAllSubscribeInfo(destination); Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
return position;
} }
}); });
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() { batches = MigrateMap.makeComputingMap(clientIdentity -> {
// 读取一下zookeeper信息，初始化一次
public Position apply(ClientIdentity clientIdentity) { MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Position position = zooKeeperMetaManager.getCursor(clientIdentity); Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
if (position == null) { for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
return nullCursor; // 返回一个空对象标识，避免出现异常 batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
} else {
return position;
}
}
});
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
// 读取一下zookeeper信息，初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;
} }
return batches;
}); });
} }
@@ -85,23 +72,13 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException { public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.subscribe(clientIdentity); super.subscribe(clientIdentity);
executor.submit(new Runnable() { executor.submit(() -> zooKeeperMetaManager.subscribe(clientIdentity));
public void run() {
zooKeeperMetaManager.subscribe(clientIdentity);
}
});
} }
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException { public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.unsubscribe(clientIdentity); super.unsubscribe(clientIdentity);
executor.submit(new Runnable() { executor.submit(() -> zooKeeperMetaManager.unsubscribe(clientIdentity));
public void run() {
zooKeeperMetaManager.unsubscribe(clientIdentity);
}
});
} }
public void updateCursor(final ClientIdentity clientIdentity, final Position position) public void updateCursor(final ClientIdentity clientIdentity, final Position position)
@@ -109,12 +86,7 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
super.updateCursor(clientIdentity, position); super.updateCursor(clientIdentity, position);
// 异步刷新 // 异步刷新
executor.submit(new Runnable() { executor.submit(() -> zooKeeperMetaManager.updateCursor(clientIdentity, position));
public void run() {
zooKeeperMetaManager.updateCursor(clientIdentity, position);
}
});
} }
@Override @Override
@@ -131,12 +103,7 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
throws CanalMetaManagerException { throws CanalMetaManagerException {
final Long batchId = super.addBatch(clientIdentity, positionRange); final Long batchId = super.addBatch(clientIdentity, positionRange);
// 异步刷新 // 异步刷新
executor.submit(new Runnable() { executor.submit(() -> zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId));
public void run() {
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
}
});
return batchId; return batchId;
} }
@@ -144,23 +111,15 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
throws CanalMetaManagerException { throws CanalMetaManagerException {
super.addBatch(clientIdentity, positionRange, batchId); super.addBatch(clientIdentity, positionRange, batchId);
// 异步刷新 // 异步刷新
executor.submit(new Runnable() { executor.submit(() -> zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId));
public void run() {
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
}
});
} }
public PositionRange removeBatch(final ClientIdentity clientIdentity, final Long batchId) public PositionRange removeBatch(final ClientIdentity clientIdentity, final Long batchId)
throws CanalMetaManagerException { throws CanalMetaManagerException {
PositionRange positionRange = super.removeBatch(clientIdentity, batchId); PositionRange positionRange = super.removeBatch(clientIdentity, batchId);
// 异步刷新 // 异步刷新
executor.submit(new Runnable() { executor.submit(() -> {
zooKeeperMetaManager.removeBatch(clientIdentity, batchId);
public void run() {
zooKeeperMetaManager.removeBatch(clientIdentity, batchId);
}
}); });
return positionRange; return positionRange;
@@ -170,12 +129,7 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
super.clearAllBatchs(clientIdentity); super.clearAllBatchs(clientIdentity);
// 异步刷新 // 异步刷新
executor.submit(new Runnable() { executor.submit(() -> zooKeeperMetaManager.clearAllBatchs(clientIdentity));
public void run() {
zooKeeperMetaManager.clearAllBatchs(clientIdentity);
}
});
} }
// =============== setter / getter ================ // =============== setter / getter ================

79
meta/src/main/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManager.java
View File

@@ -18,7 +18,6 @@ import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity; import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position; import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange; import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.base.Function;
import com.google.common.collect.MigrateMap; import com.google.common.collect.MigrateMap;
/** /**
@@ -52,54 +51,40 @@ public class PeriodMixedMetaManager extends MemoryMetaManager implements CanalMe
} }
executor = Executors.newScheduledThreadPool(1); executor = Executors.newScheduledThreadPool(1);
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() { destinations = MigrateMap.makeComputingMap(destination -> zooKeeperMetaManager.listAllSubscribeInfo(destination));
public List<ClientIdentity> apply(String destination) { cursors = MigrateMap.makeComputingMap(clientIdentity -> {
return zooKeeperMetaManager.listAllSubscribeInfo(destination); Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
return position;
} }
}); });
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() { batches = MigrateMap.makeComputingMap(clientIdentity -> {
// 读取一下zookeeper信息，初始化一次
public Position apply(ClientIdentity clientIdentity) { MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Position position = zooKeeperMetaManager.getCursor(clientIdentity); Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
if (position == null) { for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
return nullCursor; // 返回一个空对象标识，避免出现异常 batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
} else {
return position;
}
} }
return batches;
}); });
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() { updateCursorTasks = Collections.synchronizedSet(new HashSet<>());
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
// 读取一下zookeeper信息，初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;
}
});
updateCursorTasks = Collections.synchronizedSet(new HashSet<ClientIdentity>());
// 启动定时工作任务 // 启动定时工作任务
executor.scheduleAtFixedRate(new Runnable() { executor.scheduleAtFixedRate(() -> {
List<ClientIdentity> tasks = new ArrayList<>(updateCursorTasks);
public void run() { for (ClientIdentity clientIdentity : tasks) {
List<ClientIdentity> tasks = new ArrayList<ClientIdentity>(updateCursorTasks); try {
for (ClientIdentity clientIdentity : tasks) { // 定时将内存中的最新值刷到zookeeper中，多次变更只刷一次
try { zooKeeperMetaManager.updateCursor(clientIdentity, getCursor(clientIdentity));
// 定时将内存中的最新值刷到zookeeper中，多次变更只刷一次 updateCursorTasks.remove(clientIdentity);
zooKeeperMetaManager.updateCursor(clientIdentity, getCursor(clientIdentity)); } catch (Throwable e) {
updateCursorTasks.remove(clientIdentity); // ignore
} catch (Throwable e) { logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
// ignore
logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
}
} }
} }
}, period, period, TimeUnit.MILLISECONDS); }, period, period, TimeUnit.MILLISECONDS);
@@ -121,24 +106,14 @@ public class PeriodMixedMetaManager extends MemoryMetaManager implements CanalMe
super.subscribe(clientIdentity); super.subscribe(clientIdentity);
// 订阅信息频率发生比较低，不需要做定时merge处理 // 订阅信息频率发生比较低，不需要做定时merge处理
executor.submit(new Runnable() { executor.submit(() -> zooKeeperMetaManager.subscribe(clientIdentity));
public void run() {
zooKeeperMetaManager.subscribe(clientIdentity);
}
});
} }
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException { public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.unsubscribe(clientIdentity); super.unsubscribe(clientIdentity);
// 订阅信息频率发生比较低，不需要做定时merge处理 // 订阅信息频率发生比较低，不需要做定时merge处理
executor.submit(new Runnable() { executor.submit(() -> zooKeeperMetaManager.unsubscribe(clientIdentity));
public void run() {
zooKeeperMetaManager.unsubscribe(clientIdentity);
}
});
} }
public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException { public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {

12
meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java
View File

@@ -103,7 +103,7 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
public List<ClientIdentity> listAllSubscribeInfo(String destination) throws CanalMetaManagerException { public List<ClientIdentity> listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
if (zkClientx == null) { //重新加载时可能为空 if (zkClientx == null) { //重新加载时可能为空
return new ArrayList<ClientIdentity>(); return new ArrayList<>();
} }
String path = ZookeeperPathUtils.getDestinationPath(destination); String path = ZookeeperPathUtils.getDestinationPath(destination);
List<String> childs = null; List<String> childs = null;
@@ -114,9 +114,9 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
} }
if (CollectionUtils.isEmpty(childs)) { if (CollectionUtils.isEmpty(childs)) {
return new ArrayList<ClientIdentity>(); return new ArrayList<>();
} }
List<Short> clientIds = new ArrayList<Short>(); List<Short> clientIds = new ArrayList<>();
for (String child : childs) { for (String child : childs) {
if (StringUtils.isNumeric(child)) { if (StringUtils.isNumeric(child)) {
clientIds.add(ZookeeperPathUtils.getClientId(child)); clientIds.add(ZookeeperPathUtils.getClientId(child));
@@ -252,7 +252,7 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
return null; return null;
} }
// 找到最大的Id // 找到最大的Id
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size()); ArrayList<Long> batchIds = new ArrayList<>(nodes.size());
for (String batchIdString : nodes) { for (String batchIdString : nodes) {
batchIds.add(Long.valueOf(batchIdString)); batchIds.add(Long.valueOf(batchIdString));
} }
@@ -279,7 +279,7 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
return null; return null;
} }
// 找到最小的Id // 找到最小的Id
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size()); ArrayList<Long> batchIds = new ArrayList<>(nodes.size());
for (String batchIdString : nodes) { for (String batchIdString : nodes) {
batchIds.add(Long.valueOf(batchIdString)); batchIds.add(Long.valueOf(batchIdString));
} }
@@ -306,7 +306,7 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
return Maps.newHashMap(); return Maps.newHashMap();
} }
// 找到最大的Id // 找到最大的Id
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size()); ArrayList<Long> batchIds = new ArrayList<>(nodes.size());
for (String batchIdString : nodes) { for (String batchIdString : nodes) {
batchIds.add(Long.valueOf(batchIdString)); batchIds.add(Long.valueOf(batchIdString));
} }

4
meta/src/test/java/com/alibaba/otter/canal/meta/AbstractMetaManagerTest.java
View File

@@ -20,7 +20,7 @@ import org.junit.Test;
public class AbstractMetaManagerTest extends AbstractZkTest { public class AbstractMetaManagerTest extends AbstractZkTest {
private static final String MYSQL_ADDRESS = "127.0.0.1"; private static final String MYSQL_ADDRESS = "127.0.0.1";
protected ClientIdentity clientIdentity = new ClientIdentity(destination, (short) 1); ; protected ClientIdentity clientIdentity = new ClientIdentity(destination, (short) 1);
@Test @Test
public void doSubscribeTest(CanalMetaManager metaManager) { public void doSubscribeTest(CanalMetaManager metaManager) {
@@ -116,6 +116,6 @@ public class AbstractMetaManagerTest extends AbstractZkTest {
LogPosition end = new LogPosition(); LogPosition end = new LogPosition();
end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L)); end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L)); end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
return new PositionRange<LogPosition>(start, end); return new PositionRange<>(start, end);
} }
} }

37
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
View File

@@ -80,14 +80,8 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
protected Thread parseThread = null; protected Thread parseThread = null;
protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error",
e);
public void uncaughtException(Thread t,
Throwable e) {
logger.error("parse events has an error",
e);
}
};
protected EventTransactionBuffer transactionBuffer; protected EventTransactionBuffer transactionBuffer;
protected int transactionSize = 1024; protected int transactionSize = 1024;
@@ -134,22 +128,19 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
public AbstractEventParser(){ public AbstractEventParser(){
// 初始化一下 // 初始化一下
transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() { transactionBuffer = new EventTransactionBuffer(transaction -> {
boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
if (!running) {
return;
}
public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException { if (!successed) {
boolean successed = consumeTheEventAndProfilingIfNecessary(transaction); throw new CanalParseException("consume failed!");
if (!running) { }
return;
}
if (!successed) { LogPosition position = buildLastTransactionPosition(transaction);
throw new CanalParseException("consume failed!"); if (position != null) { // 可能position为空
} logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
LogPosition position = buildLastTransactionPosition(transaction);
if (position != null) { // 可能position为空
logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
}
} }
}); });
} }
@@ -542,7 +533,7 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
*/ */
private Map<String, List<String>> parseFieldFilterMap(String config) { private Map<String, List<String>> parseFieldFilterMap(String config) {
Map<String, List<String>> map = new HashMap<String, List<String>>(); Map<String, List<String>> map = new HashMap<>();
if (StringUtils.isNotBlank(config)) { if (StringUtils.isNotBlank(config)) {
for (String filter : config.split(",")) { for (String filter : config.split(",")) {

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBuffer.java
View File

@@ -115,7 +115,7 @@ public class EventTransactionBuffer extends AbstractCanalLifeCycle {
long end = this.putSequence.get(); long end = this.putSequence.get();
if (start <= end) { if (start <= end) {
List<CanalEntry.Entry> transaction = new ArrayList<CanalEntry.Entry>(); List<CanalEntry.Entry> transaction = new ArrayList<>();
for (long next = start; next <= end; next++) { for (long next = start; next <= end; next++) {
transaction.add(this.entries[getIndex(next)]); transaction.add(this.entries[getIndex(next)]);
} }

4
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java
View File

@@ -23,7 +23,7 @@ public class TableMeta {
private String schema; private String schema;
private String table; private String table;
private List<FieldMeta> fields = new ArrayList<TableMeta.FieldMeta>(); private List<FieldMeta> fields = new ArrayList<>();
private String ddl; // 表结构的DDL语句 private String ddl; // 表结构的DDL语句
public TableMeta(){ public TableMeta(){
@@ -75,7 +75,7 @@ public class TableMeta {
} }
public List<FieldMeta> getPrimaryFields() { public List<FieldMeta> getPrimaryFields() {
List<FieldMeta> primarys = new ArrayList<TableMeta.FieldMeta>(); List<FieldMeta> primarys = new ArrayList<>();
for (FieldMeta meta : fields) { for (FieldMeta meta : fields) {
if (meta.isKey()) { if (meta.isKey()) {
primarys.add(meta); primarys.add(meta);

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventParser.java
View File

@@ -14,7 +14,7 @@ import com.alibaba.otter.canal.parse.CanalEventParser;
*/ */
public class GroupEventParser extends AbstractCanalLifeCycle implements CanalEventParser { public class GroupEventParser extends AbstractCanalLifeCycle implements CanalEventParser {
private List<CanalEventParser> eventParsers = new ArrayList<CanalEventParser>(); private List<CanalEventParser> eventParsers = new ArrayList<>();
public void start() { public void start() {
super.start(); super.start();

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
View File

@@ -714,7 +714,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
} }
int i = 0; int i = 0;
Map<String, String> maps = new HashMap<String, String>(names.size(), 1f); Map<String, String> maps = new HashMap<>(names.size(), 1f);
for (FieldPacket name : names) { for (FieldPacket name : names) {
maps.put(name.getName(), fields.get(i)); maps.put(name.getName(), fields.get(i));
i++; i++;

12
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
View File

@@ -98,9 +98,9 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
ExceptionHandler exceptionHandler = new SimpleFatalExceptionHandler(); ExceptionHandler exceptionHandler = new SimpleFatalExceptionHandler();
// stage 2 // stage 2
this.logContext = new LogContext(); this.logContext = new LogContext();
simpleParserStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer, simpleParserStage = new BatchEventProcessor<>(disruptorMsgBuffer,
sequenceBarrier, sequenceBarrier,
new SimpleParserStage(logContext)); new SimpleParserStage(logContext));
simpleParserStage.setExceptionHandler(exceptionHandler); simpleParserStage.setExceptionHandler(exceptionHandler);
disruptorMsgBuffer.addGatingSequences(simpleParserStage.getSequence()); disruptorMsgBuffer.addGatingSequences(simpleParserStage.getSequence());
@@ -119,9 +119,9 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
// stage 4 // stage 4
SequenceBarrier sinkSequenceBarrier = disruptorMsgBuffer.newBarrier(sequence); SequenceBarrier sinkSequenceBarrier = disruptorMsgBuffer.newBarrier(sequence);
sinkStoreStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer, sinkStoreStage = new BatchEventProcessor<>(disruptorMsgBuffer,
sinkSequenceBarrier, sinkSequenceBarrier,
new SinkStoreStage()); new SinkStoreStage());
sinkStoreStage.setExceptionHandler(exceptionHandler); sinkStoreStage.setExceptionHandler(exceptionHandler);
disruptorMsgBuffer.addGatingSequences(sinkStoreStage.getSequence()); disruptorMsgBuffer.addGatingSequences(sinkStoreStage.getSequence());

8
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
View File

@@ -88,8 +88,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
private volatile AviaterRegexFilter nameFilter; // 运行时引用可能会有变化，比如规则发生变化时 private volatile AviaterRegexFilter nameFilter; // 运行时引用可能会有变化，比如规则发生变化时
private volatile AviaterRegexFilter nameBlackFilter; private volatile AviaterRegexFilter nameBlackFilter;
private Map<String, List<String>> fieldFilterMap = new HashMap<String, List<String>>(); private Map<String, List<String>> fieldFilterMap = new HashMap<>();
private Map<String, List<String>> fieldBlackFilterMap = new HashMap<String, List<String>>(); private Map<String, List<String>> fieldBlackFilterMap = new HashMap<>();
private TableMetaCache tableMetaCache; private TableMetaCache tableMetaCache;
private Charset charset = Charset.defaultCharset(); private Charset charset = Charset.defaultCharset();
@@ -1040,7 +1040,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
if (fieldFilterMap != null) { if (fieldFilterMap != null) {
this.fieldFilterMap = fieldFilterMap; this.fieldFilterMap = fieldFilterMap;
} else { } else {
this.fieldFilterMap = new HashMap<String, List<String>>(); this.fieldFilterMap = new HashMap<>();
} }
for (Map.Entry<String, List<String>> entry : this.fieldFilterMap.entrySet()) { for (Map.Entry<String, List<String>> entry : this.fieldFilterMap.entrySet()) {
@@ -1052,7 +1052,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
if (fieldBlackFilterMap != null) { if (fieldBlackFilterMap != null) {
this.fieldBlackFilterMap = fieldBlackFilterMap; this.fieldBlackFilterMap = fieldBlackFilterMap;
} else { } else {
this.fieldBlackFilterMap = new HashMap<String, List<String>>(); this.fieldBlackFilterMap = new HashMap<>();
} }
for (Map.Entry<String, List<String>> entry : this.fieldBlackFilterMap.entrySet()) { for (Map.Entry<String, List<String>> entry : this.fieldBlackFilterMap.entrySet()) {

6
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
View File

@@ -105,7 +105,7 @@ public class TableMetaCache {
TableMeta tableMeta = memoryTableMeta.find(schema, table); TableMeta tableMeta = memoryTableMeta.find(schema, table);
return tableMeta.getFields(); return tableMeta.getFields();
} else { } else {
return new ArrayList<FieldMeta>(); return new ArrayList<>();
} }
} }
@@ -113,7 +113,7 @@ public class TableMetaCache {
* 处理desc table的结果 * 处理desc table的结果
*/ */
public static List<FieldMeta> parseTableMetaByDesc(ResultSetPacket packet) { public static List<FieldMeta> parseTableMetaByDesc(ResultSetPacket packet) {
Map<String, Integer> nameMaps = new HashMap<String, Integer>(6, 1f); Map<String, Integer> nameMaps = new HashMap<>(6, 1f);
int index = 0; int index = 0;
for (FieldPacket fieldPacket : packet.getFieldDescriptors()) { for (FieldPacket fieldPacket : packet.getFieldDescriptors()) {
nameMaps.put(fieldPacket.getOriginalName(), index++); nameMaps.put(fieldPacket.getOriginalName(), index++);
@@ -121,7 +121,7 @@ public class TableMetaCache {
int size = packet.getFieldDescriptors().size(); int size = packet.getFieldDescriptors().size();
int count = packet.getFieldValues().size() / packet.getFieldDescriptors().size(); int count = packet.getFieldValues().size() / packet.getFieldDescriptors().size();
List<FieldMeta> result = new ArrayList<FieldMeta>(); List<FieldMeta> result = new ArrayList<>();
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
FieldMeta meta = new FieldMeta(); FieldMeta meta = new FieldMeta();
// 做一个优化，使用String.intern()，共享String对象，减少内存使用 // 做一个优化，使用String.intern()，共享String对象，减少内存使用

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java
View File

@@ -56,7 +56,7 @@ public class DruidDdlParser {
return Arrays.asList(ddlResult); return Arrays.asList(ddlResult);
} }
List<DdlResult> ddlResults = new ArrayList<DdlResult>(); List<DdlResult> ddlResults = new ArrayList<>();
for (SQLStatement statement : stmtList) { for (SQLStatement statement : stmtList) {
if (statement instanceof SQLCreateTableStatement) { if (statement instanceof SQLCreateTableStatement) {
DdlResult ddlResult = new DdlResult(); DdlResult ddlResult = new DdlResult();

14
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BinLogFileQueue.java
View File

@@ -26,7 +26,7 @@ import com.alibaba.otter.canal.parse.exception.CanalParseException;
public class BinLogFileQueue { public class BinLogFileQueue {
private String baseName = "mysql-bin."; private String baseName = "mysql-bin.";
private List<File> binlogs = new ArrayList<File>(); private List<File> binlogs = new ArrayList<>();
private File directory; private File directory;
private ReentrantLock lock = new ReentrantLock(); private ReentrantLock lock = new ReentrantLock();
private Condition nextCondition = lock.newCondition(); private Condition nextCondition = lock.newCondition();
@@ -177,7 +177,7 @@ public class BinLogFileQueue {
* 获取当前所有binlog文件 * 获取当前所有binlog文件
*/ */
public List<File> currentBinlogs() { public List<File> currentBinlogs() {
return new ArrayList<File>(binlogs); return new ArrayList<>(binlogs);
} }
public void destory() { public void destory() {
@@ -216,7 +216,7 @@ public class BinLogFileQueue {
} }
private List<File> listBinlogFiles() { private List<File> listBinlogFiles() {
List<File> files = new ArrayList<File>(); List<File> files = new ArrayList<>();
files.addAll(FileUtils.listFiles(directory, new IOFileFilter() { files.addAll(FileUtils.listFiles(directory, new IOFileFilter() {
public boolean accept(File file) { public boolean accept(File file) {
@@ -230,13 +230,7 @@ public class BinLogFileQueue {
} }
}, null)); }, null));
// 排一下序列 // 排一下序列
Collections.sort(files, new Comparator<File>() { Collections.sort(files, Comparator.comparing(File::getName));
public int compare(File o1, File o2) {
return o1.getName().compareTo(o2.getName());
}
});
return files; return files;
} }

22
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java
View File

@@ -53,8 +53,8 @@ public class BinlogDownloadQueue {
private static final Logger logger = LoggerFactory.getLogger(BinlogDownloadQueue.class); private static final Logger logger = LoggerFactory.getLogger(BinlogDownloadQueue.class);
private static final int TIMEOUT = 10000; private static final int TIMEOUT = 10000;
private LinkedBlockingQueue<BinlogFile> downloadQueue = new LinkedBlockingQueue<BinlogFile>(); private LinkedBlockingQueue<BinlogFile> downloadQueue = new LinkedBlockingQueue<>();
private LinkedBlockingQueue<Runnable> taskQueue = new LinkedBlockingQueue<Runnable>(); private LinkedBlockingQueue<Runnable> taskQueue = new LinkedBlockingQueue<>();
private LinkedList<BinlogFile> binlogList; private LinkedList<BinlogFile> binlogList;
private final int batchFileSize; private final int batchFileSize;
private Thread downloadThread; private Thread downloadThread;
@@ -78,13 +78,7 @@ public class BinlogDownloadQueue {
String fileName = StringUtils.substringBetween(binlog.getDownloadLink(), "mysql-bin.", "?"); String fileName = StringUtils.substringBetween(binlog.getDownloadLink(), "mysql-bin.", "?");
binlog.setFileName(fileName); binlog.setFileName(fileName);
} }
Collections.sort(this.binlogList, new Comparator<BinlogFile>() { this.binlogList.sort(Comparator.comparing(BinlogFile::getFileName));
@Override
public int compare(BinlogFile o1, BinlogFile o2) {
return o1.getFileName().compareTo(o2.getFileName());
}
});
} }
public void cleanDir() throws IOException { public void cleanDir() throws IOException {
@@ -187,13 +181,9 @@ public class BinlogDownloadQueue {
builder.setMaxConnPerRoute(50); builder.setMaxConnPerRoute(50);
builder.setMaxConnTotal(100); builder.setMaxConnTotal(100);
// 创建支持忽略证书的https // 创建支持忽略证书的https
final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() { final SSLContext sslContext = new SSLContextBuilder()
.loadTrustMaterial(null, (x509Certificates, s) -> true)
@Override .build();
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
}).build();
httpClient = HttpClientBuilder.create() httpClient = HttpClientBuilder.create()
.setSSLContext(sslContext) .setSSLContext(sslContext)

20
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/HttpHelper.java
View File

@@ -140,13 +140,9 @@ public class HttpHelper {
CloseableHttpResponse response = null; CloseableHttpResponse response = null;
try { try {
// 创建支持忽略证书的https // 创建支持忽略证书的https
final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() { final SSLContext sslContext = new SSLContextBuilder()
.loadTrustMaterial(null, (x509Certificates, s) -> true)
@Override .build();
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
}).build();
CloseableHttpClient httpClient = HttpClientBuilder.create() CloseableHttpClient httpClient = HttpClientBuilder.create()
.setSSLContext(sslContext) .setSSLContext(sslContext)
@@ -205,13 +201,9 @@ public class HttpHelper {
CloseableHttpResponse response = null; CloseableHttpResponse response = null;
try { try {
// 创建支持忽略证书的https // 创建支持忽略证书的https
final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() { final SSLContext sslContext = new SSLContextBuilder()
.loadTrustMaterial(null, (x509Certificates, s) -> true)
@Override .build();
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
}).build();
CloseableHttpClient httpClient = HttpClientBuilder.create() CloseableHttpClient httpClient = HttpClientBuilder.create()
.setSSLContext(sslContext) .setSSLContext(sslContext)

85
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java
View File

@@ -30,16 +30,12 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
private int batchFileSize = 4; // 最多下载的binlog文件数量 private int batchFileSize = 4; // 最多下载的binlog文件数量
private RdsLocalBinlogEventParser rdsLocalBinlogEventParser = null; private RdsLocalBinlogEventParser rdsLocalBinlogEventParser = null;
private ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactory() { private ExecutorService executorService = Executors.newSingleThreadExecutor(r -> {
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r, Thread t = new Thread(r,
"rds-binlog-daemon-thread"); "rds-binlog-daemon-thread");
t.setDaemon(true); t.setDaemon(true);
return t; return t;
} });
});
@Override @Override
public void start() { public void start() {
@@ -79,32 +75,17 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
rdsLocalBinlogEventParser.setParallel(this.parallel); rdsLocalBinlogEventParser.setParallel(this.parallel);
rdsLocalBinlogEventParser.setParallelBufferSize(this.parallelBufferSize); rdsLocalBinlogEventParser.setParallelBufferSize(this.parallelBufferSize);
rdsLocalBinlogEventParser.setParallelThreadSize(this.parallelThreadSize); rdsLocalBinlogEventParser.setParallelThreadSize(this.parallelThreadSize);
rdsLocalBinlogEventParser.setFinishListener(new RdsLocalBinlogEventParser.ParseFinishListener() { rdsLocalBinlogEventParser.setFinishListener(() -> executorService.execute(() -> {
rdsLocalBinlogEventParser.stop();
@Override // empty the dump error count,or will go into local binlog mode again,with error
public void onFinish() { // position,never get out,fixed by bucketli
executorService.execute(new Runnable() { RdsBinlogEventParserProxy.this.setDumpErrorCount(0);
RdsBinlogEventParserProxy.this.start();
@Override }));
public void run() { this.setParserExceptionHandler(e -> {
rdsLocalBinlogEventParser.stop(); handleMysqlParserException(e);
// empty the dump error count,or will go into local binlog mode again,with error if (targetHandler != null) {
// position,never get out,fixed by bucketli targetHandler.handle(e);
RdsBinlogEventParserProxy.this.setDumpErrorCount(0);
RdsBinlogEventParserProxy.this.start();
}
});
}
});
this.setParserExceptionHandler(new ParserExceptionHandler() {
@Override
public void handle(Throwable e) {
handleMysqlParserException(e);
if (targetHandler != null) {
targetHandler.handle(e);
}
} }
}); });
} }
@@ -115,29 +96,25 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
private void handleMysqlParserException(Throwable throwable) { private void handleMysqlParserException(Throwable throwable) {
if (throwable instanceof PositionNotFoundException) { if (throwable instanceof PositionNotFoundException) {
logger.info("remove rds not found position, try download rds binlog!"); logger.info("remove rds not found position, try download rds binlog!");
executorService.execute(new Runnable() { executorService.execute(() -> {
try {
logger.info("stop mysql parser!");
RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this;
long serverId = rdsBinlogEventParserProxy.getServerId();
rdsLocalBinlogEventParser.setServerId(serverId);
rdsBinlogEventParserProxy.stop();
} catch (Throwable e) {
logger.info("handle exception failed", e);
}
@Override try {
public void run() { logger.info("start rds mysql binlog parser!");
try { rdsLocalBinlogEventParser.start();
logger.info("stop mysql parser!"); } catch (Throwable e) {
RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this; logger.info("handle exception failed", e);
long serverId = rdsBinlogEventParserProxy.getServerId(); rdsLocalBinlogEventParser.stop();
rdsLocalBinlogEventParser.setServerId(serverId); RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this;
rdsBinlogEventParserProxy.stop(); rdsBinlogEventParserProxy.start();// 继续重试
} catch (Throwable e) {
logger.info("handle exception failed", e);
}
try {
logger.info("start rds mysql binlog parser!");
rdsLocalBinlogEventParser.start();
} catch (Throwable e) {
logger.info("handle exception failed", e);
rdsLocalBinlogEventParser.stop();
RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this;
rdsBinlogEventParserProxy.start();// 继续重试
}
} }
}); });
} }

18
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java
View File

@@ -82,13 +82,7 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
logger.error("download binlog failed", e); logger.error("download binlog failed", e);
throw new CanalParseException(e); throw new CanalParseException(e);
} }
setParserExceptionHandler(new ParserExceptionHandler() { setParserExceptionHandler(this::handleMysqlParserException);
@Override
public void handle(Throwable e) {
handleMysqlParserException(e);
}
});
super.start(); super.start();
} }
@@ -105,13 +99,9 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
} }
try { try {
binlogDownloadQueue.execute(new Runnable() { binlogDownloadQueue.execute(() -> {
RdsLocalBinlogEventParser.super.stop();
@Override RdsLocalBinlogEventParser.super.start();
public void run() {
RdsLocalBinlogEventParser.super.stop();
RdsLocalBinlogEventParser.super.start();
}
}); });
} catch (InterruptedException e) { } catch (InterruptedException e) {
throw new RuntimeException(e); throw new RuntimeException(e);

10
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java
View File

@@ -177,7 +177,7 @@ public abstract class AbstractRequest<T> {
private String makeRequestString(Map<String, String> param) throws Exception { private String makeRequestString(Map<String, String> param) throws Exception {
fillCommonParam(param); fillCommonParam(param);
String sign = makeSignature(new TreeMap<String, String>(param)); String sign = makeSignature(new TreeMap<>(param));
StringBuilder builder = new StringBuilder(); StringBuilder builder = new StringBuilder();
for (Map.Entry<String, String> entry : param.entrySet()) { for (Map.Entry<String, String> entry : param.entrySet()) {
builder.append(encode(entry.getKey())).append("=").append(encode(entry.getValue())).append("&"); builder.append(encode(entry.getKey())).append("=").append(encode(entry.getValue())).append("&");
@@ -195,13 +195,7 @@ public abstract class AbstractRequest<T> {
*/ */
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private final HttpResponse executeHttpRequest(HttpGet getMethod, String host) throws Exception { private final HttpResponse executeHttpRequest(HttpGet getMethod, String host) throws Exception {
SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, (TrustStrategy) (arg0, arg1) -> true).build();
@Override
public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
return true;
}
}).build();
SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext,
new String[] { "TLSv1" }, new String[] { "TLSv1" },
null, null,

60
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
View File

@@ -52,16 +52,12 @@ public class DatabaseTableMeta implements TableMetaTSDB {
private static Logger logger = LoggerFactory.getLogger(DatabaseTableMeta.class); private static Logger logger = LoggerFactory.getLogger(DatabaseTableMeta.class);
private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'"); private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'");
private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation"); private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation");
private static ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { private static ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
Thread thread = new Thread(r,
@Override "[scheduler-table-meta-snapshot]");
public Thread newThread(Runnable r) { thread.setDaemon(true);
Thread thread = new Thread(r, return thread;
"[scheduler-table-meta-snapshot]"); });
thread.setDaemon(true);
return thread;
}
});
private ReadWriteLock lock = new ReentrantReadWriteLock(); private ReadWriteLock lock = new ReentrantReadWriteLock();
private AtomicBoolean initialized = new AtomicBoolean(false); private AtomicBoolean initialized = new AtomicBoolean(false);
private String destination; private String destination;
@@ -69,8 +65,8 @@ public class DatabaseTableMeta implements TableMetaTSDB {
private volatile MysqlConnection connection; // 查询meta信息的链接 private volatile MysqlConnection connection; // 查询meta信息的链接
private CanalEventFilter filter; private CanalEventFilter filter;
private CanalEventFilter blackFilter; private CanalEventFilter blackFilter;
private Map<String, List<String>> fieldFilterMap = new HashMap<String, List<String>>(); private Map<String, List<String>> fieldFilterMap = new HashMap<>();
private Map<String, List<String>> fieldBlackFilterMap = new HashMap<String, List<String>>(); private Map<String, List<String>> fieldBlackFilterMap = new HashMap<>();
private EntryPosition lastPosition; private EntryPosition lastPosition;
private boolean hasNewDdl; private boolean hasNewDdl;
private MetaHistoryDAO metaHistoryDAO; private MetaHistoryDAO metaHistoryDAO;
@@ -91,26 +87,22 @@ public class DatabaseTableMeta implements TableMetaTSDB {
// 24小时生成一份snapshot // 24小时生成一份snapshot
if (snapshotInterval > 0) { if (snapshotInterval > 0) {
scheduleSnapshotFuture = scheduler.scheduleWithFixedDelay(new Runnable() { scheduleSnapshotFuture = scheduler.scheduleWithFixedDelay(() -> {
boolean applyResult = false;
try {
MDC.put("destination", destination);
applyResult = applySnapshotToDB(lastPosition, false);
} catch (Throwable e) {
logger.error("scheudle applySnapshotToDB faield", e);
}
@Override try {
public void run() { MDC.put("destination", destination);
boolean applyResult = false; if (applyResult) {
try { snapshotExpire((int) TimeUnit.HOURS.toSeconds(snapshotExpire));
MDC.put("destination", destination);
applyResult = applySnapshotToDB(lastPosition, false);
} catch (Throwable e) {
logger.error("scheudle applySnapshotToDB faield", e);
}
try {
MDC.put("destination", destination);
if (applyResult) {
snapshotExpire((int) TimeUnit.HOURS.toSeconds(snapshotExpire));
}
} catch (Throwable e) {
logger.error("scheudle snapshotExpire faield", e);
} }
} catch (Throwable e) {
logger.error("scheudle snapshotExpire faield", e);
} }
}, snapshotInterval, snapshotInterval, TimeUnit.HOURS); }, snapshotInterval, snapshotInterval, TimeUnit.HOURS);
} }
@@ -199,7 +191,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
private boolean dumpTableMeta(MysqlConnection connection, final CanalEventFilter filter) { private boolean dumpTableMeta(MysqlConnection connection, final CanalEventFilter filter) {
try { try {
ResultSetPacket packet = connection.query("show databases"); ResultSetPacket packet = connection.query("show databases");
List<String> schemas = new ArrayList<String>(); List<String> schemas = new ArrayList<>();
for (String schema : packet.getFieldValues()) { for (String schema : packet.getFieldValues()) {
schemas.add(schema); schemas.add(schema);
} }
@@ -207,7 +199,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
for (String schema : schemas) { for (String schema : schemas) {
// filter views // filter views
packet = connection.query("show full tables from `" + schema + "` where Table_type = 'BASE TABLE'"); packet = connection.query("show full tables from `" + schema + "` where Table_type = 'BASE TABLE'");
List<String> tables = new ArrayList<String>(); List<String> tables = new ArrayList<>();
for (String table : packet.getFieldValues()) { for (String table : packet.getFieldValues()) {
if ("BASE TABLE".equalsIgnoreCase(table)) { if ("BASE TABLE".equalsIgnoreCase(table)) {
continue; continue;
@@ -245,7 +237,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
} }
private boolean applyHistoryToDB(EntryPosition position, String schema, String ddl, String extra) { private boolean applyHistoryToDB(EntryPosition position, String schema, String ddl, String extra) {
Map<String, String> content = new HashMap<String, String>(); Map<String, String> content = new HashMap<>();
content.put("destination", destination); content.put("destination", destination);
content.put("binlogFile", position.getJournalName()); content.put("binlogFile", position.getJournalName());
content.put("binlogOffest", String.valueOf(position.getPosition())); content.put("binlogOffest", String.valueOf(position.getPosition()));
@@ -326,7 +318,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
} }
if (compareAll) { if (compareAll) {
Map<String, String> content = new HashMap<String, String>(); Map<String, String> content = new HashMap<>();
content.put("destination", destination); content.put("destination", destination);
content.put("binlogFile", position.getJournalName()); content.put("binlogFile", position.getJournalName());
content.put("binlogOffest", String.valueOf(position.getPosition())); content.put("binlogOffest", String.valueOf(position.getPosition()));

4
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
View File

@@ -49,7 +49,7 @@ import com.alibaba.otter.canal.protocol.position.EntryPosition;
public class MemoryTableMeta implements TableMetaTSDB { public class MemoryTableMeta implements TableMetaTSDB {
private Logger logger = LoggerFactory.getLogger(MemoryTableMeta.class); private Logger logger = LoggerFactory.getLogger(MemoryTableMeta.class);
private Map<List<String>, TableMeta> tableMetas = new ConcurrentHashMap<List<String>, TableMeta>(); private Map<List<String>, TableMeta> tableMetas = new ConcurrentHashMap<>();
private SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL); private SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
public MemoryTableMeta(){ public MemoryTableMeta(){
@@ -143,7 +143,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
} }
public Map<String, String> snapshot() { public Map<String, String> snapshot() {
Map<String, String> schemaDdls = new HashMap<String, String>(); Map<String, String> schemaDdls = new HashMap<>();
for (Schema schema : repository.getSchemas()) { for (Schema schema : repository.getSchemas()) {
StringBuffer data = new StringBuffer(4 * 1024); StringBuffer data = new StringBuffer(4 * 1024);
for (String table : schema.showTables()) { for (String table : schema.showTables()) {

32
parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java
View File

@@ -68,15 +68,10 @@ public class FileMixedLogPositionManager extends AbstractLogPositionManager {
this.period = period; this.period = period;
this.memoryLogPositionManager = memoryLogPositionManager; this.memoryLogPositionManager = memoryLogPositionManager;
this.dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>() { this.dataFileCaches = MigrateMap.makeComputingMap(this::getDataFile);
public File apply(String destination) {
return getDataFile(destination);
}
});
this.executorService = Executors.newScheduledThreadPool(1); this.executorService = Executors.newScheduledThreadPool(1);
this.persistTasks = Collections.synchronizedSet(new HashSet<String>()); this.persistTasks = Collections.synchronizedSet(new HashSet<>());
} }
@Override @Override
@@ -100,19 +95,16 @@ public class FileMixedLogPositionManager extends AbstractLogPositionManager {
} }
// 启动定时工作任务 // 启动定时工作任务
executorService.scheduleAtFixedRate(new Runnable() { executorService.scheduleAtFixedRate(() -> {
List<String> tasks = new ArrayList<>(persistTasks);
public void run() { for (String destination : tasks) {
List<String> tasks = new ArrayList<String>(persistTasks); try {
for (String destination : tasks) { // 定时将内存中的最新值刷到file中，多次变更只刷一次
try { flushDataToFile(destination);
// 定时将内存中的最新值刷到file中，多次变更只刷一次 persistTasks.remove(destination);
flushDataToFile(destination); } catch (Throwable e) {
persistTasks.remove(destination); // ignore
} catch (Throwable e) { logger.error("period update" + destination + " curosr failed!", e);
// ignore
logger.error("period update" + destination + " curosr failed!", e);
}
} }
} }
}, period, period, TimeUnit.MILLISECONDS); }, period, period, TimeUnit.MILLISECONDS);

13
parse/src/main/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManager.java
View File

@@ -73,14 +73,11 @@ public class MixedLogPositionManager extends AbstractLogPositionManager {
@Override @Override
public void persistLogPosition(final String destination, final LogPosition logPosition) throws CanalParseException { public void persistLogPosition(final String destination, final LogPosition logPosition) throws CanalParseException {
memoryLogPositionManager.persistLogPosition(destination, logPosition); memoryLogPositionManager.persistLogPosition(destination, logPosition);
executor.submit(new Runnable() { executor.submit(() -> {
try {
public void run() { zooKeeperLogPositionManager.persistLogPosition(destination, logPosition);
try { } catch (Exception e) {
zooKeeperLogPositionManager.persistLogPosition(destination, logPosition); logger.error("ERROR # persist to zookeeper has an error", e);
} catch (Exception e) {
logger.error("ERROR # persist to zookeeper has an error", e);
}
} }
}); });
} }

Some files were not shown because too many files have changed in this diff Show More