Nacos 生产集群部署实战：MySQL 主从 + Keepalived VIP-星海夜航

Nacos 负责微服务的发现、配置和元数据管理：服务通过 SDK、OpenAPI 或 Agent 注册后，消费方可以用 DNS 或 HTTP API 找到它；配置支持中心化、动态化管理，带版本跟踪、金丝雀发布、一键回滚；另外还有动态 DNS 的权重路由和服务元数据统计。官网 https://nacos.io/，项目在 https://github.com/alibaba/nacos。

架构规划

官方推荐的接入方式是「域名 + VIP」，可读性好、换 IP 方便，这里就按这个方案搭：

主机名	服务器IP	部署组件
Nacos-node1	192.168.66.15 / 192.168.66.100(VIP)	Nacos1.3.1、JDK1.8、MySQL5.7+、keepalived
Nacos-node2	192.168.66.16	Nacos1.3.1、JDK1.8、MySQL5.7+、keepalived
Nacos-node3	192.168.66.17	Nacos1.3.1、JDK1.8、MySQL5.7+、keepalived

环境初始化

三步基础工作：设主机名、写 hosts 解析、node1 免密登录其余节点并把 hosts 分发下去；关防火墙和 SELinux、装常用命令；最后是时间同步——node1 和公网时间服务器同步，node2/node3 以 node1 为源，统一用 chronyd。全部完成后三台机器用 RPM 装 JDK 1.8。

[root@localhost ~]# hostnamectl set-hostname nacos-node1
[root@localhost ~]# hostnamectl set-hostname nacos-node2
[root@localhost ~]# hostnamectl set-hostname nacos-node3

[root@nacos-node1 ~]# vi /etc/hosts
192.168.66.15   nacos-node1
192.168.66.16   nacos-node2
192.168.66.17   nacos-node3

[root@nacos-node1 ~]# ssh-keygen -t rsa
[root@nacos-node1 ~]# ssh-copy-id root@192.168.66.15
[root@nacos-node1 ~]# ssh-copy-id root@192.168.66.16
[root@nacos-node1 ~]# ssh-copy-id root@192.168.66.17

[root@nacos-node1 ~]# for i in 192.168.66.{16,17}; do echo ">>> $i";scp /etc/hosts root@$i:/etc/; done

[root@nacos-node1 ~]# ssh-copy-id root@nacos-node1
[root@nacos-node1 ~]# ssh-copy-id root@nacos-node2
[root@nacos-node1 ~]# ssh-copy-id root@nacos-node3

[root@nacos-node1 ~]# for i in 192.168.66.{15..17}; do echo ">>> $i";ssh root@$i "systemctl stop firewalld && systemctl disable firewalld && systemctl status firewalld"; done

[root@nacos-node1 ~]# for i in 192.168.66.{15..17}; do echo ">>> $i";ssh root@$i "sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config"; done

[root@nacos-node1 ~]# for i in 192.168.66.{15..17}; do echo ">>> $i";ssh root@$i "setenforce 0 && getenforce"; done

[root@nacos-node1 ~]# for i in 192.168.66.{15..17};do echo ">>> $i";ssh root@$i "yum -y install gcc gcc-c++ libaio make cmake zlib-devel openssl-devel pcre pcre-devel wget git curl lynx lftp mailx mutt rsync ntp net-tools vim lrzsz screen sysstat yum-plugin-security yum-utils createrepo bash-completion zip unzip bzip2 tree tmpwatch pinfo man-pages lshw pciutils gdisk system-storage-manager git  gdbm-devel sqlite-devel chrony";done

[root@nacos-node1 ~]# for i in 192.168.66.{15..17};do echo ">>> $i";ssh root@$i "yum install -y epel-release";done

[root@nacos-node1 ~]# systemctl start ntpd && systemctl enable ntpd
[root@nacos-node1 ~]# vim /etc/chrony.conf

[root@nacos-node1 ~]# systemctl start chronyd.service 
[root@nacos-node1 ~]# systemctl enable chronyd.service
[root@nacos-node1 ~]# systemctl status chronyd.service

[root@nacos-node1 ~]# timedatectl

[root@nacos-node1 ~]# for i in 192.168.66.{16,17};do echo ">>> $i";ssh root@$i "systemctl stop ntpd && systemctl disable ntpd && systemctl status ntpd";done

~]# vim /etc/chrony.conf
server 192.168.66.15 iburst

[root@nacos-node1 ~]# for i in 192.168.66.{16,17};do echo ">>> $i";ssh root@$i "systemctl restart chronyd.service && systemctl enable chronyd.service && systemctl status chronyd.service";done

[root@nacos-node1 ~]# for i in 192.168.66.{16,17};do echo ">>> $i";ssh root@$i "chronyc sources ";done

rpm -ivh jdk-8u231-linux-x64.rpm

MySQL 主从

Nacos 集群模式要求外接数据库，官方建议数据库做主备，这里搭一主两从。受资源限制，MySQL 和 Nacos 混部在同一批机器上。三台都按 RPM 装 5.7（包来自官方归档 https://downloads.mysql.com/archives/community/，需要 server / client / common / libs / devel / libs-compat 一组包），装完启动、重置 root 密码：

#卸载掉系统默认的mariadb
[root@nacos-node1 ~]# rpm -qa|grep mariadb|xargs rpm -e --nodeps

[root@nacos-node1 ~]# mkdir -p /opt/soft && cd /opt/soft
[root@nacos-node1 soft]# yum install mysql-community-*

[root@nacos-node1 ~]# systemctl start mysqld
[root@nacos-node1 ~]# systemctl enable mysqld
[root@nacos-node1 ~]# systemctl status mysqld

[root@nacos-node1 ~]# cat /var/log/mysqld.log |grep password
[root@nacos-node1 ~]# mysql -uroot -p
Enter password:

#修改密码策略
mysql> set global validate_password_policy=0;
Query OK, 0 rows affected (0.00 sec)

mysql> set global validate_password_mixed_case_count=0;
Query OK, 0 rows affected (0.00 sec)

mysql> set global validate_password_number_count=3;
Query OK, 0 rows affected (0.00 sec)

mysql> set global validate_password_special_char_count=0;
Query OK, 0 rows affected (0.00 sec)

mysql> set global validate_password_length=3;

mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY 'dqz123456';

主从基于 GTID，my.cnf 里 server-id、gtid-mode、enforce-gtid-consistency 这些参数三台都要按下面配置改好再重启：

#------------node1节点------------#
[root@nacos-node1 ~]# vim /etc/my.cnf
#添加如下信息
#gtid
server-id = 1
gtid-mode = ON
enforce-gtid-consistency = ON
log-slave-updates = ON
#binlog
expire_logs_days = 30
max_binlog_size  = 100M
log_bin = /data/mysql/mysql-bin
binlog-format = Row
#elay log
skip_slave_start= ON

#------------node2节点------------#
[root@nacos-node2 ~]# vim /etc/my.cnf
#添加如下信息
#gtid
server-id = 2
gtid-mode = ON
enforce-gtid-consistency = ON
log-slave-updates = ON
#binlog
expire_logs_days = 30
max_binlog_size  = 100M
log_bin = /data/mysql/mysql-bin
binlog-format = Row
#elay log
skip_slave_start= ON

#------------node3节点------------#
[root@nacos-node3 ~]# vim /etc/my.cnf
#添加如下信息
#gtid
server-id = 3
gtid-mode = ON
enforce-gtid-consistency = ON
log-slave-updates = ON
#binlog
expire_logs_days = 30
max_binlog_size  = 100M
log_bin = /data/mysql/mysql-bin
binlog-format = Row
#elay log
skip_slave_start= ON


#所有节点创建binlog日志存储目录并授权
mkdir -p /data/mysql
chown -R mysql. /data/

#所有节点重启MySQL服务
systemctl restart mysqld

主库建同步账号。基于 GTID 的复制会重放事务，不要在从库上建同名账号，否则复制链路容易错乱：

[root@nacos-node1 ~]# mysql -uroot -p

#修改密码策略
mysql> set global validate_password_policy=0;
mysql> set global validate_password_mixed_case_count=0;
mysql> set global validate_password_number_count=3;
mysql> set global validate_password_special_char_count=0;
mysql> set global validate_password_length=3;


mysql> grant replication slave on *.* to 'slave'@'192.168.66.%' identified by 'dqz123456';
mysql> flush privileges;

mysql> show variables like "%gtid%";
+----------------------------------+-----------+
| Variable_name                    | Value     |
+----------------------------------+-----------+
| binlog_gtid_simple_recovery      | ON        |
| enforce_gtid_consistency         | ON        |
| gtid_executed_compression_period | 1000      |
| gtid_mode                        | ON        |
| gtid_next                        | AUTOMATIC |
| gtid_owned                       |           |
| gtid_purged                      |           |
| session_track_gtids              | OFF       |
+----------------------------------+-----------+
8 rows in set (0.00 sec)

mysql> show variables like '%gtid_next%';
+---------------+-----------+
| Variable_name | Value     |
+---------------+-----------+
| gtid_next     | AUTOMATIC |
+---------------+-----------+
1 row in set (0.00 sec)

mysql> show master status;
+------------------+----------+--------------+------------------+------------------------------------------+
| File             | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set                        |
+------------------+----------+--------------+------------------+------------------------------------------+
| mysql-bin.000001 |      601 |              |                  | a4c5734e-d16e-11ea-b7c1-000c29b0226c:1-2 |
+------------------+----------+--------------+------------------+------------------------------------------+
1 row in set (0.00 sec)

[root@nacos2 ~]# mysql -uroot -p
Enter password:

mysql> CHANGE MASTER TO MASTER_HOST='192.168.66.15',MASTER_USER='slave',MASTER_PASSWORD='dqz123456',MASTER_AUTO_POSITION=1;

mysql> START SLAVE;

mysql> SHOW SLAVE STATUSG

mysql> SHOW SLAVE HOSTS;

验证 GTID：主库随便建个库，看从库是否自动跟过来。

#主库创建新库
mysql> CREATE DATABASE TESTDB;

#从库查看复制情况
mysql> SHOW DATABASES;

部署 Nacos 集群

从官方 release 页 https://github.com/alibaba/nacos/releases 下载 zip 包，三台机器都上传到 /opt/soft 解压：

[root@nacos-node1 ~]# cd /opt/soft/
[root@nacos-node1 soft]# unzip nacos-server-1.3.1.zip -d /usr/local/

集群配置文件里填三个节点自己的 IP，Nacos 默认服务端口是 8848：

[root@nacos-node1 soft]# cd /usr/local/nacos/conf

[root@nacos-node1 conf]# vim cluster.conf
# ip:port
192.168.66.15:8848
192.168.66.16:8848
192.168.66.17:8848

在主库建 Nacos 用的用户和库（库名后面连接配置要用），再导入安装目录 conf 下的初始化 SQL：

[root@nacos-node1 conf]# mysql -uroot -p
Enter password:

mysql> CREATE DATABASE `nacos_config` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;

mysql> CREATE USER 'nacos'@'%' IDENTIFIED BY 'dqz123456';

mysql> grant all privileges on nacos_config.* to nacos@'%';

mysql> flush privileges;

mysql> use nacos_config
mysql> source /usr/local/nacos/conf/nacos-mysql.sql;

各节点改数据源配置：db.num 是数据库实例数量，db.url.0（多个实例就继续 db.url.1）指向主库：

[root@nacos-node1 ~]# cd /usr/local/nacos/conf/
[root@nacos-node1 conf]# vim application.properties

#拷贝给其他节点
[root@nacos-node1 conf]# scp application.properties root@192.168.66.16:/usr/local/nacos/conf/

[root@nacos-node1 conf]# scp application.properties root@192.168.66.17:/usr/local/nacos/conf/

三台节点以集群模式启动，看 start.out 确认无报错：

[root@nacos-node1 ~]# cd /usr/local/nacos/bin/
[root@nacos-node1 bin]# ./startup.sh

tailf /usr/local/nacos/logs/start.out

浏览器访问 http://192.168.66.15:8848/nacos/#/login，默认账号密码都是 nacos / nacos：

Keepalived + VIP 接入

Nacos 前面挂 keepalived 做虚拟 IP，对外只暴露 192.168.66.100。keepalived 在 node1 编译安装后把二进制传给另外两台（官网 https://www.keepalived.org/download.html）：

[root@nacos-node1 ~]# cd /opt/soft/
[root@nacos-node1 soft]# wget https://www.keepalived.org/software/keepalived-2.0.20.tar.gz


#所有节点都需要安装依赖
[root@nacos-node1 script]# for i in 192.168.66.{15..17}; do echo ">>> $i"; ssh root@$i "yum -y install libnl libnl-devel libnfnetlink-devel openssl-devel"; done

[root@nacos-node1 soft]# tar -xf keepalived-2.0.20.tar.gz
[root@nacos-node1 soft]# cd keepalived-2.0.20
[root@nacos-node1 keepalived-2.0.20]# ./configure --prefix=/usr/local/keepalived

[root@nacos-node1 keepalived-2.0.20]# make && make install

源码安装默认读不到配置，把配置文件放到 keepalived 默认路径 /etc/keepalived/keepalived.conf：

[root@nacos-node1 ~]# cd /usr/local/keepalived/etc/keepalived/
[root@nacos-node1 keepalived]# cp keepalived.conf{,_bak}
[root@nacos-node1 keepalived]# vim keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id nacos-node1		//唯一值，一般为所在服务器主机名
   vrrp_skip_check_adv_addr
   # vrrp_strict         //注释该参数
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

#注意此处添加nacos服务健康检查
vrrp_script check_nacos {
    script "/bin/bash /script/check_nacos.sh"	#nacos健康检查脚本路径
    interval 5		#健康检查周期，每隔5秒执行一次脚本     
}

vrrp_instance VI_1 {
state MASTER
    interface ens33	#绑定的网卡名称
    virtual_router_id 51		#主备ID必须保证一致
    priority 100			#优先级
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
	192.168.66.100/24	#VIP地址，虚拟IP
    }
    track_script {   #调用脚本
       check_nacos   #这里的名称必须与vrrp_scrip写的保证一致
    }
}

[root@nacos-node1 ~]# mkdir /etc/keepalived/
[root@nacos-node1 ~]# cp /usr/local/keepalived/etc/keepalived/keepalived.conf /etc/keepalived/

健康检查脚本检测到 nacos 进程不在就停掉本机 keepalived，让 VIP 漂走，同时调用推送脚本发告警。脚本的路径和名称必须和配置文件里写的检测脚本一致：

[root@nacos-node1 ~]# mkdir /script
[root@nacos-node1 ~]# vim /script/check_nacos.sh
#!/bin/bash
network=`ifconfig ens33 |grep -w inet |awk '{print $2}'`
nacos=`netstat -tnlp|grep 8848|wc -l`
if [ "$nacos" -eq 0 ];then
    #这里调用消息通知脚本，并配置需要发送的消息
    /script/qywx_nacos.py "主机：$(hostname)--主机IP：$network：nacos服务心跳检测异常,>>> 请登入服务器查看;VIP已切换,请注意"
    systemctl stop keepalived.service
fi

[root@nacos-node1 ~]# vim /script/qywx_nacos.py
#!/usr/bin/python
# -*- coding: utf-8 -*-
import requests
import json
import sys
import os

headers = {'Content-Type': 'application/json;charset=utf-8'}
api_url = "企业微信机器人或者钉钉机器人的webhook地址"
def msg(text):
    json_text= {
     "msgtype": "text",
        "text": {
            "content": text
        },
    }
    print requests.post(api_url,json.dumps(json_text),headers=headers).content

if __name__ == '__main__':
    text = sys.argv[1]
msg(text)

推送脚本依赖 pip 和 requests，所有节点装好；再给两个脚本加执行权限：

[root@nacos-node1 ~]# curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py

[root@nacos-node1 ~]# for i in 192.168.66.{16,17}; do echo ">>> $i";scp /root/get-pip.py root@$i:/root/; done

[root@nacos-node1 ~]# for i in 192.168.66.{15..17}; do echo ">>> $i"; ssh root@$i python /root/get-pip.py; done

[root@nacos-node1 ~]# for i in 192.168.66.{15..17}; do echo ">>> $i"; ssh root@$i pip install requests; done

[root@nacos-node1 ~]# chmod +x /script/check_nacos.sh /script/qywx_nacos.py

停掉 nacos 试一次，机器人能收到消息说明链路通了：

把编译产物和目录拷给另外两台，备节点分别改配置：router_id 三台不能重复（建议直接用主机名），virtual_router_id 必须一致，priority 按 node1=100、node2=90、node3=80 递减：

[root@nacos-node1 ~]# for i in 192.168.66.{16,17}; do echo ">>> $i"; scp -r /usr/local/keepalived root@$i:/usr/local/ && scp -r /script root@$i:/ && scp -r /etc/keepalived root@$i:/etc/ && scp /usr/lib/systemd/system/keepalived.service root@$i:/usr/lib/systemd/system/; done

[root@nacos-node2 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id nacos-node2
   vrrp_skip_check_adv_addr
   # vrrp_strict      
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_script check_nacos {
    script "/bin/bash /script/check_nacos.sh"
    interval 5
}

vrrp_instance VI_1 {
state BACKUP
    interface ens33
    virtual_router_id 51
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
	192.168.66.100/24
    }
    track_script {
       check_nacos
    }
}

[root@nacos-node3 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id nacos-node3
   vrrp_skip_check_adv_addr
   # vrrp_strict      
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_script check_nacos {
    script "/bin/bash /script/check_nacos.sh"
    interval 5
}

vrrp_instance VI_1 {
state BACKUP
    interface ens33
    virtual_router_id 51
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
	192.168.66.100/24
    }
    track_script {
       check_nacos
    }
}

三台启动 keepalived，VIP 应落在 node1：

[root@nacos-node1 ~]# for i in 192.168.66.{15..17}; do echo ">>> $i"; ssh root@$i systemctl daemon-reload && systemctl start keepalived && systemctl status keepalived|grep Active; done

[root@nacos-node1 ~]# for i in 192.168.66.{15..17}; do echo ">>> $i"; ssh root@$i "ip a show ens33"; done

之后统一走 VIP 访问：http://192.168.66.100:8848/nacos/#/login：

踩坑备忘

时间不同步会让 Nacos 集群节点之间选举和日志对不上，初始化阶段最容易被跳过；GTID 主从的账号一定要只在主库建；keepalived 起不来但没有报错时，先检查健康检查脚本指向的服务是否真的存在——服务不在，脚本会把 keepalived 自己停掉，看起来就像“起不来的假象”。

文章版权归作者所有，未经允许请勿转载。

THE END

Nacos 生产集群部署实战：MySQL 主从 + Keepalived VIP