diff --git a/canal-admin/canal-admin-server/src/main/resources/application.yml b/canal-admin/canal-admin-server/src/main/resources/application.yml index 76c62779..a4f3fc30 100644 --- a/canal-admin/canal-admin-server/src/main/resources/application.yml +++ b/canal-admin/canal-admin-server/src/main/resources/application.yml @@ -6,14 +6,16 @@ spring: time-zone: GMT+8 spring.datasource: - url: jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8&useSSL=false - username: root - password: 121212 + address: 127.0.0.1:3306 + database: canal_manager + username: canal + password: canal driver-class-name: com.mysql.jdbc.Driver + url: jdbc:mysql://${spring.datasource.address}/${spring.datasource.database}?useUnicode=true&characterEncoding=UTF-8&useSSL=false hikari: - maximum-pool-size: 10 + maximum-pool-size: 30 minimum-idle: 1 canal: adminUser: admin - adminPasswd: admin + adminPasswd: admin \ No newline at end of file diff --git a/canal-admin/canal-admin-server/src/main/resources/canal_manager.sql b/canal-admin/canal-admin-server/src/main/resources/canal_manager.sql index da748c39..673abac7 100644 --- a/canal-admin/canal-admin-server/src/main/resources/canal_manager.sql +++ b/canal-admin/canal-admin-server/src/main/resources/canal_manager.sql @@ -2,7 +2,7 @@ CREATE DATABASE /*!32312 IF NOT EXISTS*/ `canal_manager` /*!40100 DEFAULT CHARAC USE `canal_manager`; -SET NAMES utf8mb4; +SET NAMES utf8; SET FOREIGN_KEY_CHECKS = 0; -- ---------------------------- @@ -17,7 +17,7 @@ CREATE TABLE `canal_adapter_config` ( `content` text NOT NULL, `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; -- ---------------------------- -- Table structure for canal_cluster @@ -29,7 +29,7 @@ CREATE TABLE `canal_cluster` ( `zk_hosts` varchar(255) NOT NULL, `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; -- ---------------------------- -- Table structure for canal_config @@ -46,7 +46,7 @@ CREATE TABLE `canal_config` ( `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `sid_UNIQUE` (`server_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; -- ---------------------------- -- Table structure for canal_instance_config @@ -63,7 +63,7 @@ CREATE TABLE `canal_instance_config` ( `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `name_UNIQUE` (`name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; -- ---------------------------- -- Table structure for canal_node_server @@ -80,7 +80,7 @@ CREATE TABLE `canal_node_server` ( `status` varchar(45) DEFAULT NULL, `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; -- ---------------------------- -- Table structure for canal_user @@ -96,7 +96,7 @@ CREATE TABLE `canal_user` ( `avatar` varchar(255) DEFAULT NULL, `creation_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; SET FOREIGN_KEY_CHECKS = 1; diff --git a/deployer/src/main/bin/restart.sh b/deployer/src/main/bin/restart.sh index 63749ce5..e2499d51 100644 --- a/deployer/src/main/bin/restart.sh +++ b/deployer/src/main/bin/restart.sh @@ -1,5 +1,6 @@ #!/bin/bash -sh stop.sh +args=$@ -sh startup.sh +sh stop.sh $args +sh startup.sh $args \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 44655560..dd82544b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM canal/osbase:v1 +FROM canal/osbase:v2 MAINTAINER agapple (jianghang115@gmail.com) @@ -10,6 +10,7 @@ RUN \ cp -R /tmp/docker/alidata /alidata && \ chmod +x /alidata/bin/* && \ mkdir -p /home/admin && \ + cp -R /tmp/docker/app.sh /home/admin/ && \ cp -R /tmp/docker/admin/* /home/admin/ && \ /bin/cp -f alidata/bin/lark-wait /usr/bin/lark-wait && \ @@ -17,6 +18,9 @@ RUN \ tar -xzvf /home/admin/canal.deployer-*.tar.gz -C /home/admin/canal-server && \ /bin/rm -f /home/admin/canal.deployer-*.tar.gz && \ + tar zxvf /tmp/node_exporter.tar.gz -C /home/admin && \ + ln -s /home/admin/node_exporter-0.18.1.linux-arm64 /home/admin/node_exporter && \ + mkdir -p home/admin/canal-server/logs && \ chmod +x /home/admin/*.sh && \ chmod +x /home/admin/bin/*.sh && \ @@ -24,10 +28,10 @@ RUN \ yum clean all && \ true -# 2222 sys , 8000 debug , 11111 canal , 11112 metrics -EXPOSE 2222 11111 8000 11112 +# 11110 admin , 11111 canal , 11112 metrics, 9100 exporter +EXPOSE 11110 11111 11112 9100 WORKDIR /home/admin ENTRYPOINT [ "/alidata/bin/main.sh" ] -CMD [ "/home/admin/app.sh" ] +CMD [ "/home/admin/app.sh" ] \ No newline at end of file diff --git a/docker/Dockerfile_admin b/docker/Dockerfile_admin new file mode 100644 index 00000000..4707af97 --- /dev/null +++ b/docker/Dockerfile_admin @@ -0,0 +1,35 @@ +FROM canal/osadmin:v1 + +MAINTAINER agapple (jianghang115@gmail.com) + +# install canal +COPY image/ /tmp/docker/ +COPY canal.admin-*.tar.gz /home/admin/ + +RUN \ + cp -R /tmp/docker/alidata /alidata && \ + chmod +x /alidata/bin/* && \ + mkdir -p /home/admin && \ + mkdir -p /home/admin/bin/ && \ + cp -R /tmp/docker/app_admin.sh /home/admin/app.sh && \ + cp -R /tmp/docker/admin/* /home/admin/ && \ + /bin/cp -f alidata/bin/lark-wait /usr/bin/lark-wait && \ + + mkdir -p /home/admin/canal-admin && \ + tar -xzvf /home/admin/canal.admin-*.tar.gz -C /home/admin/canal-admin && \ + /bin/rm -f /home/admin/canal.admin-*.tar.gz && \ + + mkdir -p home/admin/canal-admin/logs && \ + chmod +x /home/admin/*.sh && \ + chmod +x /home/admin/bin/*.sh && \ + chown admin: -R /home/admin && \ + yum clean all && \ + true + +# 2222 sys , 8000 debug , 8089 web , 11110 admin , 11111 canal , 11112 metrics +EXPOSE 2222 11110 11111 8000 8089 11112 + +WORKDIR /home/admin + +ENTRYPOINT [ "/alidata/bin/main.sh" ] +CMD [ "/home/admin/app.sh" ] diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 010b97e0..1f31d0c7 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -4,6 +4,7 @@ MAINTAINER agapple (jianghang115@gmail.com) env DOWNLOAD_LINK="http://download.oracle.com/otn-pub/java/jdk/8u181-b13/96a7b8442fe848ef90c96a2fad6ed6d1/jdk-8u181-linux-x64.rpm" # install system + RUN \ /bin/cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ echo 'root:Hello1234' | chpasswd && \ @@ -32,7 +33,6 @@ RUN \ wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=xxx; oraclelicense=accept-securebackup-cookie" "$DOWNLOAD_LINK" -O /tmp/jdk-8-linux-x64.rpm && \ yum -y install /tmp/jdk-8-linux-x64.rpm && \ /bin/rm -f /tmp/jdk-8-linux-x64.rpm && \ - echo "export JAVA_HOME=/usr/java/latest" >> /etc/profile && \ echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile && \ yum clean all && \ diff --git a/docker/base/Dockerfile_admin b/docker/base/Dockerfile_admin new file mode 100644 index 00000000..bf730bd4 --- /dev/null +++ b/docker/base/Dockerfile_admin @@ -0,0 +1,11 @@ +FROM canal/osbase:v1 + +MAINTAINER agapple (jianghang115@gmail.com) + +RUN \ + groupadd -r mysql && useradd -r -g mysql mysql && \ + yum -y install wget mysql-server --nogpgcheck && \ + yum clean all && \ + true + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/base/Dockerfile_v2 b/docker/base/Dockerfile_v2 new file mode 100644 index 00000000..7de9ead1 --- /dev/null +++ b/docker/base/Dockerfile_v2 @@ -0,0 +1,11 @@ +FROM canal/osbase:v1 + +MAINTAINER agapple (jianghang115@gmail.com) + +env NODE_EPORTER_LINK="https://github.com/prometheus/node_exporter/releases/download/v0.18.1/node_exporter-0.18.1.linux-arm64.tar.gz" + +RUN \ + wget "$NODE_EPORTER_LINK" -O /tmp/node_exporter.tar.gz && \ + true + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/build.sh b/docker/build.sh index 4d838620..f49c3147 100644 --- a/docker/build.sh +++ b/docker/build.sh @@ -16,6 +16,15 @@ BASE=${bin_abs_path} if [ "$1" == "base" ] ; then docker build --no-cache -t canal/osbase $BASE/base +elif [ "$1" == "base_v2" ] ; then + docker build --no-cache -t canal/osbase $BASE/base -f $BASE/base/Dockerfile_v2 +elif [ "$1" == "base_admin" ] ; then + docker build --no-cache -t canal/osadmin $BASE/base -f $BASE/base/Dockerfile_admin +elif [ "$1" == "admin" ] ; then + rm -rf $BASE/canal.*.tar.gz ; + cd $BASE/../ && mvn clean package -Dmaven.test.skip -Denv=release && cd $current_path ; + cp $BASE/../target/canal.admin-*.tar.gz $BASE/ + docker build --no-cache -t canal/canal-admin $BASE/ -f $BASE/Dockerfile_admin else rm -rf $BASE/canal.*.tar.gz ; cd $BASE/../ && mvn clean package -Dmaven.test.skip -Denv=release && cd $current_path ; diff --git a/docker/image/admin/bin/clean_log.sh b/docker/image/admin/bin/clean_log.sh index 9d0a0bef..218483cc 100755 --- a/docker/image/admin/bin/clean_log.sh +++ b/docker/image/admin/bin/clean_log.sh @@ -26,6 +26,10 @@ baseClean(){ } CANAL_DIR="/home/admin/canal-server/logs" +if [ ! -d "$CANAL_DIR" ] + CANAL_DIR="/home/admin/canal-admin/logs" +fi + if [[ -d $CANAL_DIR ]]; then USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) if [[ $USAGE -ge 90 ]]; then diff --git a/docker/image/admin/health.sh b/docker/image/admin/health.sh index c48e6d9d..4c16240b 100755 --- a/docker/image/admin/health.sh +++ b/docker/image/admin/health.sh @@ -1,11 +1,23 @@ #!/bin/sh -metrics_port=`perl -le 'print $ENV{"canal.metrics.pull.port"}'` -if [ "$metrics_port" == "" ]; then - metrics_port="11112" + +if [ -d "/home/admin/canal-server" ]; then + serverPort=`perl -le 'print $ENV{"server.port"}'` + if [ -z "$serverPort" ] ; then + serverPort=8089 + fi + + CHECK_URL="http://127.0.0.1:$serverPort/index.html" + CHECK_POINT="Canal" +else + metrics_port=`perl -le 'print $ENV{"canal.metrics.pull.port"}'` + if [ "$metrics_port" == "" ]; then + metrics_port="11112" + fi + + CHECK_URL="http://127.0.0.1:$metrics_port/metrics" + CHECK_POINT="canal" fi -CHECK_URL="http://127.0.0.1:$metrics_port/metrics" -CHECK_POINT="canal" CHECK_COUNT=`curl -s --connect-timeout 7 --max-time 7 $CHECK_URL | grep -c $CHECK_POINT` if [ $CHECK_COUNT -eq 0 ]; then echo "[FAILED]" diff --git a/docker/image/admin/app.sh b/docker/image/app.sh similarity index 58% rename from docker/image/admin/app.sh rename to docker/image/app.sh index 413378c8..1465e8c0 100755 --- a/docker/image/admin/app.sh +++ b/docker/image/app.sh @@ -76,26 +76,41 @@ function checkStart() { function start_canal() { echo "start canal ..." - metricsPort=`perl -le 'print $ENV{"canal.metrics.pull.port"}'` - if [ -z "$metricsPort" ] ; then - metricsPort=11112 - fi + managerAddress=`perl -le 'print $ENV{"canal.admin.manager"}'` + if [ ! -z "$managerAddress" ] ; then + # canal_local.properties mode + adminPort=`perl -le 'print $ENV{"canal.admin.port"}'` + if [ -z "$adminPort" ] ; then + adminPort=11110 + fi - destination=`perl -le 'print $ENV{"canal.destinations"}'` - if [[ "$destination" =~ ',' ]]; then - echo "multi destination:$destination is not support" - exit 1; + su admin -c 'cd /home/admin/canal-server/bin/ && sh restart.sh local 1>>/tmp/start.log 2>&1' + sleep 5 + #check start + checkStart "canal" "nc 127.0.0.1 $metricsPort -w 1 -z | wc -l" 30 else - if [ "$destination" != "" ] && [ "$destination" != "example" ] ; then - if [ -d /home/admin/canal-server/conf/example ]; then - mv /home/admin/canal-server/conf/example /home/admin/canal-server/conf/$destination - fi - fi - fi - su admin -c 'cd /home/admin/canal-server/bin/ && sh restart.sh 1>>/tmp/start.log 2>&1' - sleep 5 - #check start - checkStart "canal" "nc 127.0.0.1 $metricsPort -w 1 -z | wc -l" 30 + metricsPort=`perl -le 'print $ENV{"canal.metrics.pull.port"}'` + if [ -z "$metricsPort" ] ; then + metricsPort=11112 + fi + + destination=`perl -le 'print $ENV{"canal.destinations"}'` + if [[ "$destination" =~ ',' ]]; then + echo "multi destination:$destination is not support" + exit 1; + else + if [ "$destination" != "" ] && [ "$destination" != "example" ] ; then + if [ -d /home/admin/canal-server/conf/example ]; then + mv /home/admin/canal-server/conf/example /home/admin/canal-server/conf/$destination + fi + fi + fi + + su admin -c 'cd /home/admin/canal-server/bin/ && sh restart.sh 1>>/tmp/start.log 2>&1' + sleep 5 + #check start + checkStart "canal" "nc 127.0.0.1 $metricsPort -w 1 -z | wc -l" 30 + fi } function stop_canal() { @@ -104,8 +119,17 @@ function stop_canal() { echo "stop canal successful ..." } +function start_exporter() { + su admin -c 'cd /home/admin/node_exporter && ./node_exporter 1>>/tmp/start.log 2>&1 &' +} + +function stop_exporter() { + su admin -c 'killall node_exporter' +} + echo "==> START ..." +start_exporter start_canal echo "==> START SUCCESSFUL ..." @@ -117,5 +141,6 @@ waitterm echo "==> STOP" stop_canal +start_exporter -echo "==> STOP SUCCESSFUL ..." +echo "==> STOP SUCCESSFUL ..." \ No newline at end of file diff --git a/docker/image/app_admin.sh b/docker/image/app_admin.sh new file mode 100755 index 00000000..156b9f0d --- /dev/null +++ b/docker/image/app_admin.sh @@ -0,0 +1,169 @@ +#!/bin/bash +set -e + +source /etc/profile +export JAVA_HOME=/usr/java/latest +export PATH=$JAVA_HOME/bin:$PATH +touch /tmp/start.log +chown admin: /tmp/start.log +chown -R admin: /home/admin/canal-admin +host=`hostname -i` + +MYSQL_USER_PASSWORD=`perl -le 'print $ENV{"spring.datasource.password"}'` +MYSQL_USER=`perl -le 'print $ENV{"spring.datasource.username"}'` +MYSQL_DATABASE=`perl -le 'print $ENV{"spring.datasource.database"}'` +MYSQL_ADDRESS=`perl -le 'print $ENV{"spring.datasource.address"}'` + +if [ -z "${MYSQL_USER_PASSWORD}" ]; then + MYSQL_USER_PASSWORD="canal" +fi +if [ -z "${MYSQL_USER}" ]; then + MYSQL_USER="canal" +fi +if [ -z "${MYSQL_DATABASE}" ]; then + MYSQL_DATABASE="canal_manager" +fi + +# waitterm +# wait TERM/INT signal. +# see: http://veithen.github.io/2014/11/16/sigterm-propagation.html +waitterm() { + local PID + # any process to block + tail -f /dev/null & + PID="$!" + # setup trap, could do nothing, or just kill the blocker + trap "kill -TERM ${PID}" TERM INT + # wait for signal, ignore wait exit code + wait "${PID}" || true + # clear trap + trap - TERM INT + # wait blocker, ignore blocker exit code + wait "${PID}" 2>/dev/null || true +} + +# waittermpid "${PIDFILE}". +# monitor process by pidfile && wait TERM/INT signal. +# if the process disappeared, return 1, means exit with ERROR. +# if TERM or INT signal received, return 0, means OK to exit. +waittermpid() { + local PIDFILE PID do_run error + PIDFILE="${1?}" + do_run=true + error=0 + trap "do_run=false" TERM INT + while "${do_run}" ; do + PID="$(cat "${PIDFILE}")" + if ! ps -p "${PID}" >/dev/null 2>&1 ; then + do_run=false + error=1 + else + sleep 1 + fi + done + trap - TERM INT + return "${error}" +} + + +function checkStart() { + local name=$1 + local cmd=$2 + local timeout=$3 + cost=5 + while [ $timeout -gt 0 ]; do + ST=`eval $cmd` + if [ "$ST" == "0" ]; then + sleep 1 + let timeout=timeout-1 + let cost=cost+1 + elif [ "$ST" == "" ]; then + sleep 1 + let timeout=timeout-1 + let cost=cost+1 + else + break + fi + done + echo "start $name successful" +} + +function start_mysql() { + echo "start mysql ..." + # start mysql + MYSQL_ROOT_PASSWORD=Hello1234 + # connect local mysql + if [ -z "$(ls -A /var/lib/mysql)" ]; then + mysql_install_db --user=mysql --datadir=/var/lib/mysql 1>>/tmp/start.log 2>&1 + # These statements _must_ be on individual lines, and _must_ end with + # semicolons (no line breaks or comments are permitted). + # TODO proper SQL escaping on ALL the things D: + TEMP_FILE='/tmp/init.sql' + echo "update mysql.user set password=password('${MYSQL_ROOT_PASSWORD}') where user='root';" >> $TEMP_FILE + echo "grant all privileges on *.* to 'root'@'%' WITH GRANT OPTION ;" >> $TEMP_FILE + echo "create database if not exists $MYSQL_DATABASE ;" >> $TEMP_FILE + echo "create user $MYSQL_USER identified by '$MYSQL_USER_PASSWORD' ;" >> $TEMP_FILE + echo "grant all privileges on $MYSQL_DATABASE.* to '$MYSQL_USER'@'%' identified by '$MYSQL_USER_PASSWORD' ;" >> $TEMP_FILE + echo "grant all privileges on $MYSQL_DATABASE.* to '$MYSQL_USER'@'localhost' identified by '$MYSQL_USER_PASSWORD' ;" >> $TEMP_FILE + echo "flush privileges;" >> $TEMP_FILE + service mysqld start + checkStart "mysql" "echo 'show status' | mysql -s -h127.0.0.1 -P3306 -uroot | grep -c Uptime" 30 + mysql -h127.0.0.1 -uroot -e "source $TEMP_FILE" 1>>/tmp/start.log 2>&1 + + cmd="mysql -h127.0.0.1 -u$MYSQL_USER -p$MYSQL_USER_PASSWORD $MYSQL_DATABASE -e 'source /home/admin/canal-admin/conf/canal_manager.sql' 1>>/tmp/start.log 2>&1" + eval $cmd + /bin/rm -f /home/admin/canal-admin/conf/canal_manager.sql + else + chown -R mysql:mysql /var/lib/mysql + service mysqld start + #check start + checkStart "mysql" "echo 'show status' | mysql -b -s -h127.0.0.1 -P3306 -uroot -p$MYSQL_ROOT_PASSWORD | grep -c Uptime" 30 + fi +} + +function stop_mysql() { + echo "stop mysql ..." + # stop mysql + service mysqld stop + echo "stop mysql successful ..." +} + +function start_admin() { + echo "start admin ..." + serverPort=`perl -le 'print $ENV{"server.port"}'` + if [ -z "$serverPort" ] ; then + serverPort=8089 + fi + su admin -c 'cd /home/admin/canal-admin/bin/ && sh restart.sh 1>>/tmp/start.log 2>&1' + sleep 5 + #check start + checkStart "canal" "nc 127.0.0.1 $serverPort -w 1 -z | wc -l" 30 +} + +function stop_admin() { + echo "stop admin" + su admin -c 'cd /home/admin/canal-admin/bin/ && sh stop.sh 1>>/tmp/start.log 2>&1' + echo "stop admin successful ..." +} + +echo "==> START ..." + +if [ -z "${MYSQL_ADDRESS}" ]; then + start_mysql +fi +start_admin + +echo "==> START SUCCESSFUL ..." + +tail -f /dev/null & +# wait TERM signal +waitterm + +echo "==> STOP" + +if [ -z "${MYSQL_ADDRESS}" ]; then + stop_admin +fi +stop_mysql + +echo "==> STOP SUCCESSFUL ..." diff --git a/docker/image/canal_manager.sql b/docker/image/canal_manager.sql new file mode 100644 index 00000000..673abac7 --- /dev/null +++ b/docker/image/canal_manager.sql @@ -0,0 +1,110 @@ +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `canal_manager` /*!40100 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin */; + +USE `canal_manager`; + +SET NAMES utf8; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for canal_adapter_config +-- ---------------------------- +DROP TABLE IF EXISTS `canal_adapter_config`; +CREATE TABLE `canal_adapter_config` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `category` varchar(45) NOT NULL, + `name` varchar(45) NOT NULL, + `status` varchar(45) DEFAULT NULL, + `content` text NOT NULL, + `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for canal_cluster +-- ---------------------------- +DROP TABLE IF EXISTS `canal_cluster`; +CREATE TABLE `canal_cluster` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(63) NOT NULL, + `zk_hosts` varchar(255) NOT NULL, + `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for canal_config +-- ---------------------------- +DROP TABLE IF EXISTS `canal_config`; +CREATE TABLE `canal_config` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `cluster_id` bigint(20) DEFAULT NULL, + `server_id` bigint(20) DEFAULT NULL, + `name` varchar(45) NOT NULL, + `status` varchar(45) DEFAULT NULL, + `content` text NOT NULL, + `content_md5` varchar(128) NOT NULL, + `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `sid_UNIQUE` (`server_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for canal_instance_config +-- ---------------------------- +DROP TABLE IF EXISTS `canal_instance_config`; +CREATE TABLE `canal_instance_config` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `cluster_id` bigint(20) DEFAULT NULL, + `server_id` bigint(20) DEFAULT NULL, + `name` varchar(45) NOT NULL, + `status` varchar(45) DEFAULT NULL, + `content` text NOT NULL, + `content_md5` varchar(128) DEFAULT NULL, + `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `name_UNIQUE` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for canal_node_server +-- ---------------------------- +DROP TABLE IF EXISTS `canal_node_server`; +CREATE TABLE `canal_node_server` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `cluster_id` bigint(20) DEFAULT NULL, + `name` varchar(63) NOT NULL, + `ip` varchar(63) NOT NULL, + `admin_port` int(11) DEFAULT NULL, + `tcp_port` int(11) DEFAULT NULL, + `metric_port` int(11) DEFAULT NULL, + `status` varchar(45) DEFAULT NULL, + `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for canal_user +-- ---------------------------- +DROP TABLE IF EXISTS `canal_user`; +CREATE TABLE `canal_user` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `username` varchar(31) NOT NULL, + `password` varchar(128) NOT NULL, + `name` varchar(31) NOT NULL, + `roles` varchar(31) NOT NULL, + `introduction` varchar(255) DEFAULT NULL, + `avatar` varchar(255) DEFAULT NULL, + `creation_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +SET FOREIGN_KEY_CHECKS = 1; + +-- ---------------------------- +-- Records of canal_user +-- ---------------------------- +BEGIN; +INSERT INTO `canal_user` VALUES (1, 'admin', '6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9', 'Canal Manager', 'admin', NULL, NULL, '2019-07-14 00:05:28'); +COMMIT; + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/docker/run.sh b/docker/run.sh index 7dbdb682..7772d212 100644 --- a/docker/run.sh +++ b/docker/run.sh @@ -3,7 +3,7 @@ function usage() { echo "Usage:" echo " run.sh [CONFIG]" - echo "example:" + echo "example 1 :" echo " run.sh -e canal.instance.master.address=127.0.0.1:3306 \\" echo " -e canal.instance.dbUsername=canal \\" echo " -e canal.instance.dbPassword=canal \\" @@ -11,6 +11,11 @@ function usage() { echo " -e canal.instance.tsdb.enable=true \\" echo " -e canal.instance.gtidon=false \\" echo " -e canal.instance.filter.regex=.*\\\\\\..* " + echo "example 2 :" + echo " run.sh -e canal.admin.manager=127.0.0.1:8089 \\" + echo " -e canal.admin.port=11110 \\" + echo " -e canal.admin.user=admin \\" + echo " -e canal.admin.passwd=4ACFE3202A5FF5CF467898FC58AAB1D615029441" exit } @@ -72,7 +77,7 @@ DATA="$BASE/data" mkdir -p $DATA CONFIG=${@:1} #VOLUMNS="-v $DATA:/home/admin/canal-server/logs" -PORTLIST="8000 2222 11111 11112" +PORTLIST="11110 11111 11112 9100" PORTS="" for PORT in $PORTLIST ; do #exist=`check_port $PORT` diff --git a/docker/run_admin.sh b/docker/run_admin.sh new file mode 100644 index 00000000..247c424a --- /dev/null +++ b/docker/run_admin.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +function usage() { + echo "Usage:" + echo " run_admin.sh [CONFIG]" + echo "example :" + echo " run_admin.sh -e server.port=8089 \\" + echo " -e canal.adminUser=admin \\" + echo " -e canal.adminPasswd=admin \\" + echo " -e spring.datasource.username=canal \\" + echo " -e spring.datasource.password=canal" + exit +} + +function check_port() { + local port=$1 + local TL=$(which telnet) + if [ -f $TL ]; then + data=`echo quit | telnet 127.0.0.1 $port| grep -ic connected` + echo $data + return + fi + + local NC=$(which nc) + if [ -f $NC ]; then + data=`nc -z -w 1 127.0.0.1 $port | grep -ic succeeded` + echo $data + return + fi + echo "0" + return +} + +function getMyIp() { + case "`uname`" in + Darwin) + myip=`echo "show State:/Network/Global/IPv4" | scutil | grep PrimaryInterface | awk '{print $3}' | xargs ifconfig | grep inet | grep -v inet6 | awk '{print $2}'` + ;; + *) + myip=`ip route get 1 | awk '{print $NF;exit}'` + ;; + esac + echo $myip +} + +NET_MODE="" +case "`uname`" in + Darwin) + bin_abs_path=`cd $(dirname $0); pwd` + ;; + Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + NET_MODE="--net=host" + ;; + *) + NET_MODE="--net=host" + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +BASE=${bin_abs_path} +if [ $# -eq 0 ]; then + usage +elif [ "$1" == "-h" ] ; then + usage +elif [ "$1" == "help" ] ; then + usage +fi + +DATA="$BASE/data" +mkdir -p $DATA +CONFIG=${@:1} +#VOLUMNS="-v $DATA:/home/admin/canal-admin/logs" +PORTLIST="8089" +PORTS="" +for PORT in $PORTLIST ; do + #exist=`check_port $PORT` + exist="0" + if [ "$exist" == "0" ]; then + PORTS="$PORTS -p $PORT:$PORT" + else + echo "port $PORT is used , pls check" + exit 1 + fi +done + +MEMORY="-m 1024m" +LOCALHOST=`getMyIp` +cmd="docker run -d -it -h $LOCALHOST $CONFIG --name=canal-admin $VOLUMNS $NET_MODE $PORTS $MEMORY canal/canal-admin" +echo $cmd +eval $cmd diff --git a/instance/manager/src/test/java/com/alibaba/otter/canal/instance/manager/PlainCanalConfigClientIntegration.java b/instance/manager/src/test/java/com/alibaba/otter/canal/instance/manager/PlainCanalConfigClientIntegration.java index 3e169e1e..b64aeab3 100644 --- a/instance/manager/src/test/java/com/alibaba/otter/canal/instance/manager/PlainCanalConfigClientIntegration.java +++ b/instance/manager/src/test/java/com/alibaba/otter/canal/instance/manager/PlainCanalConfigClientIntegration.java @@ -1,13 +1,11 @@ package com.alibaba.otter.canal.instance.manager; -import org.junit.Ignore; import org.junit.Test; import org.springframework.util.Assert; import com.alibaba.otter.canal.instance.manager.plain.PlainCanal; import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient; -@Ignore public class PlainCanalConfigClientIntegration { @Test @@ -27,8 +25,8 @@ public class PlainCanalConfigClientIntegration { String instances = client.findInstances(null); Assert.notNull(instances); - plain = client.findInstance("example", null); - Assert.notNull(plain); + plain = client.findInstance("example", null); + Assert.notNull(plain); plain = client.findInstance("example", plain.getMd5()); Assert.isNull(plain);