chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge code and fixed issue #2755 #2756 #2735 #2714 #2711 #2710 #2616 #2602 #2585 #2565 #2935 #2434 #2623 #2418 #2387

Browse Source
This commit is contained in:
agapple
2020-08-22 13:07:24 +08:00
parent 90bf9e6de5
commit b79af46054
66 changed files with 459 additions and 477 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
admin/admin-ui/src/api/canalInstance.js
View File

@@ -59,13 +59,6 @@ export function instanceLog(id, nodeId) {
})
}
export function instanceMeta(id, nodeId) {
return request({
url: '/canal/instance/meta/' + id + '/' + nodeId,
method: 'get'
})
}
export function instanceStatus(id, option) {
return request({
url: '/canal/instance/status/' + id + '?option=' + option,

7
admin/admin-ui/src/router/index.js
View File

@@ -128,13 +128,6 @@ export const constantRoutes = [
component: () => import('@/views/canalServer/CanalInstanceLogDetail'),
meta: { title: 'Instance 日志' },
hidden: true
},
{
path: 'canalInstance/meta',
name: 'Instance meta',
component: () => import('@/views/canalServer/CanalInstanceMetaDetail'),
meta: { title: 'Instance Meta' },
hidden: true
}
]
},

9
admin/admin-ui/src/views/canalServer/CanalInstance.vue
View File

@@ -64,7 +64,6 @@
<el-dropdown-item @click.native="handleStart(scope.row)">启动</el-dropdown-item>
<el-dropdown-item @click.native="handleStop(scope.row)">停止</el-dropdown-item>
<el-dropdown-item @click.native="handleLog(scope.row)">日志</el-dropdown-item>
<el-dropdown-item @click.native="handleMeta(scope.row)">meta</el-dropdown-item>
</el-dropdown-menu>
</el-dropdown>
</template>
@@ -118,7 +117,6 @@ export default {
}
},
created() {
this.listQuery.name = this.$route.query.name
getClustersAndServers().then((res) => {
this.options = res.data
})
@@ -224,13 +222,6 @@ export default {
return
}
this.$router.push('canalInstance/log?id=' + row.id + '&nodeId=' + row.nodeServer.id)
},
handleMeta(row) {
if (row.nodeId === null) {
this.$message({ message: '当前Instance不是启动状态，无法查看meta', type: 'warning' })
return
}
this.$router.push('canalInstance/meta?id=' + row.id + '&nodeId=' + row.nodeServer.id)
}
}
}

53
admin/admin-ui/src/views/canalServer/CanalInstanceMetaDetail.vue
View File

@@ -1,53 +0,0 @@
<template>
<div>
<el-form ref="form" :model="form">
<div style="padding-left: 10px;padding-right: 10px;padding-top: 20px;">
<el-form-item>
{{ form.instance }}.meta&nbsp;&nbsp;&nbsp;&nbsp;
<el-button type="primary" @click="onRefresh">刷新</el-button>
<el-button type="info" @click="onBack(form.instance)">返回</el-button>
</el-form-item>
<el-input v-model="form.desc" :rows="35" :readonly="'readonly'" type="textarea" />
</div>
</el-form>
</div>
</template>
<script>
import { instanceMeta } from '@/api/canalInstance'
export default {
data() {
return {
form: {
instance: '',
desc: ''
}
}
},
created() {
this.fetchData()
},
methods: {
fetchData() {
instanceMeta(this.$route.query.id, this.$route.query.nodeId).then(res => {
this.form.instance = res.data.instance
this.form.desc = res.data.meta
})
},
onRefresh() {
this.fetchData()
},
onBack(instance) {
this.$router.push('/canalServer/canalInstances?name=' + instance)
}
}
}
</script>
<style scoped>
.line{
text-align: center;
}
</style>

8
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/connector/AdminConnector.java
View File

@@ -128,12 +128,4 @@ public interface AdminConnector {
*/
String instanceLog(String destination, String fileName, int lines);
/**
* meta
* @param destination
* @param fileName
* @return
*/
String instanceMeta(String destination, String fileName);
}

5
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/connector/SimpleAdminConnector.java
View File

@@ -206,11 +206,6 @@ public class SimpleAdminConnector implements AdminConnector {
return doLogAdmin("instance", "file", destination, fileName, lines);
}
@Override
public String instanceMeta(final String destination, final String fileName) {
return doLogAdmin("meta", "file", destination, fileName,100);
}
// ==================== helper method ====================
private String doServerAdmin(String action) {

14
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalInstanceController.java
View File

@@ -161,20 +161,6 @@ public class CanalInstanceController {
return BaseModel.getInstance(canalInstanceConfigService.remoteInstanceLog(id, nodeId));
}
/**
* 获取实例meta信息
*
* @param id
* @param nodeId
* @param env
* @return
*/
@GetMapping(value = "/instance/meta/{id}/{nodeId}")
public BaseModel<Map<String, String>> meta(@PathVariable Long id, @PathVariable Long nodeId,
@PathVariable String env) {
return BaseModel.getInstance(canalInstanceConfigService.remoteInstanceMeta(id, nodeId));
}
/**
* 通过Server id获取所有活动的Instance
*

2
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/CanalInstanceService.java
View File

@@ -26,8 +26,6 @@ public interface CanalInstanceService {
Map<String, String> remoteInstanceLog(Long id, Long nodeId);
Map<String, String> remoteInstanceMeta(Long id, Long nodeId);
boolean remoteOperation(Long id, Long nodeId, String option);
boolean instanceOperation(Long id, String option);

29
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/CanalInstanceServiceImpl.java
View File

@@ -1,10 +1,8 @@
package com.alibaba.otter.canal.admin.service.impl;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import io.ebean.Query;
import java.security.NoSuchAlgorithmException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -258,33 +256,6 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
return result;
}
@Override
public Map<String, String> remoteInstanceMeta(final Long id, final Long nodeId) {
Map<String, String> result = new HashMap<>();
NodeServer nodeServer = NodeServer.find.byId(nodeId);
if (nodeServer == null) {
return result;
}
CanalInstanceConfig canalInstanceConfig = CanalInstanceConfig.find.byId(id);
if (canalInstanceConfig == null) {
return result;
}
String meta;
if (nodeServer.getCanalCluster() != null) {
ZkClientx zkClientx = ZkClientx.getZkClient(nodeServer.getCanalCluster().getZkHosts());
String zkPath = MessageFormat.format("/{0}/{1}/{2}/{3}/{4}/{5}", "otter", "canal", "destinations", canalInstanceConfig.getName(), "1001", "cursor");
meta = new String((byte[]) zkClientx.readData(zkPath));
} else {
meta = SimpleAdminConnectors.execute(nodeServer.getIp(),
nodeServer.getAdminPort(),
adminConnector -> adminConnector.instanceMeta(canalInstanceConfig.getName(), "meta.dat"));
}
result.put("instance", canalInstanceConfig.getName());
result.put("meta", meta);
return result;
}
public boolean remoteOperation(Long id, Long nodeId, String option) {
NodeServer nodeServer = null;
if ("start".equals(option)) {

77
admin/admin-web/src/main/resources/canal-template.properties
View File

@@ -8,11 +8,11 @@ canal.register.ip =
canal.port = 11111
canal.metrics.pull.port = 11112
# canal instance user/passwd
canal.user = canal
canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
# canal.user = canal
# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
# canal admin config
canal.admin.manager = 127.0.0.1:8089
#canal.admin.manager = 127.0.0.1:8089
canal.admin.port = 11110
canal.admin.user = admin
canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
@@ -21,7 +21,7 @@ canal.zkServers =
# flush data to zk
canal.zookeeper.flush.period = 1000
canal.withoutNetty = false
# tcp, kafka, RocketMQ
# tcp, kafka, rocketMQ, rabbitMQ
canal.serverMode = tcp
# flush meta cursor/parse position to file
canal.file.data.dir = ${canal.conf.dir}
@@ -86,14 +86,10 @@ canal.instance.tsdb.snapshot.interval = 24
# purge snapshot expire , default 360 hour(15 days)
canal.instance.tsdb.snapshot.expire = 360
# aliyun ak/sk , support rds/mq
canal.aliyun.accessKey =
canal.aliyun.secretKey =
#################################################
######### destinations #############
#################################################
canal.destinations =
canal.destinations =
# conf root dir
canal.conf.dir = ../conf
# auto scan instance dir add/remove and start/stop instance
@@ -111,29 +107,56 @@ canal.instance.global.spring.xml = classpath:spring/file-instance.xml
#canal.instance.global.spring.xml = classpath:spring/default-instance.xml
##################################################
######### MQ #############
######### MQ Properties #############
##################################################
canal.mq.servers = 127.0.0.1:6667
canal.mq.retries = 0
canal.mq.batchSize = 16384
canal.mq.maxRequestSize = 1048576
canal.mq.lingerMs = 100
canal.mq.bufferMemory = 33554432
# aliyun ak/sk , support rds/mq
canal.aliyun.accessKey =
canal.aliyun.secretKey =
canal.aliyun.uid=
canal.mq.flatMessage = true
canal.mq.canalBatchSize = 50
canal.mq.canalGetTimeout = 100
canal.mq.flatMessage = true
canal.mq.compressionType = none
canal.mq.acks = all
#canal.mq.properties. =
canal.mq.producerGroup = test
# Set this value to "cloud", if you want open message trace feature in aliyun.
canal.mq.accessChannel = local
# aliyun mq namespace
#canal.mq.namespace =
canal.mq.database.hash = true
canal.mq.send.thread.size = 30
canal.mq.build.thread.size = 8
##################################################
######### Kafka Kerberos Info #############
######### Kafka #############
##################################################
canal.mq.kafka.kerberos.enable = false
canal.mq.kafka.kerberos.krb5FilePath = "../conf/kerberos/krb5.conf"
canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf"
kafka.bootstrap.servers = 127.0.0.1:6667
kafka.acks = all
kafka.compression.type = none
kafka.batch.size = 16384
kafka.linger.ms = 1
kafka.max.request.size = 1048576
kafka.buffer.memory = 33554432
kafka.max.in.flight.requests.per.connection = 1
kafka.retries = 0
kafka.kerberos.enable = false
kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
##################################################
######### RocketMQ #############
##################################################
rocketmq.producer.group = test
rocketmq.enable.message.trace = false
rocketmq.customized.trace.topic =
rocketmq.namespace =
rocketmq.namesrv.addr = 127.0.0.1:9876
rocketmq.retry.times.when.send.failed = 0
rocketmq.vip.channel.enabled = false
##################################################
######### RabbitMQ #############
##################################################
rabbitmq.host =
rabbitmq.virtual.host =
rabbitmq.exchange =
rabbitmq.username =
rabbitmq.password =

2
admin/pom.xml
View File

@@ -69,7 +69,7 @@
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.17</version>
<version>5.1.48</version>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>