Info

参考 # 自动主从切换快速开始
根据次文档改成 docker-compose 部署，并且使用三个NameServer 节点，可是部署到生产环境，根据需要添加broker

Ip 角色
192.168.10.67 NameServer 1，broker
192.168.10.68 NameServer 2，broker
192.168.10.69 NameServer 3

1 配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
目录结构
tree
.
├── hosts.ini
└── rocketmq
    ├── broker-1
    │   ├── broker
    │   │   ├── broker.conf # broker配置文件
    │   │   └── plain_acl.yml # 权限控制
    │   ├── docker-compose.yml
    │   └── namesrv
    │       └── namesrv.conf 
    ├── broker-2
    │   ├── broker
    │   │   ├── broker.conf # 与broker-1 配置一样
    │   │   └── plain_acl.yml
    │   ├── docker-compose.yml
    │   └── namesrv
    │       └── namesrv.conf # NameServer配置
    └── broker-3
        ├── docker-compose.yml
        ├── namesrv
        │   └── namesrv.conf
        └── users.properties # dashboard，账户配置

2 配置

2.1 NameServer

2.1.1 n0

1
2
3
4
5
6
7
8
9
cat rocketmq/broker-1/namesrv/namesrv.conf 
listenPort = 9876
enableControllerInNamesrv = true
controllerDLegerGroup = group1
controllerDLegerPeers = n0-192.168.10.67:9877;n1-192.168.10.68:9877;n2-192.168.10.69:9877
controllerDLegerSelfId = n0
controllerStorePath = /home/rocketmq/store/controller
enableElectUncleanMaster = false
notifyBrokerRoleChanged  =  true

2.1.2 n1

1
2
3
4
5
6
7
8
9
10
cat rocketmq/broker-2/namesrv/namesrv.conf 
listenPort = 9876
enableControllerInNamesrv = true
enableController = true
controllerDLegerGroup = group1
controllerDLegerPeers = n0-192.168.10.67:9877;n1-192.168.10.68:9877;n2-192.168.10.69:9877
controllerDLegerSelfId = n1
controllerStorePath = /home/rocketmq/store/controller
enableElectUncleanMaster = false
notifyBrokerRoleChanged  =  true

2.1.3 n2

1
2
3
4
5
6
7
8
9
10
cat rocketmq/broker-3/namesrv/namesrv.conf 
listenPort = 9876
enableController = true
enableControllerInNamesrv = true
controllerDLegerGroup = group1
controllerDLegerPeers = n0-192.168.10.67:9877;n1-192.168.10.68:9877;n2-192.168.10.69:9877
controllerDLegerSelfId = n2
controllerStorePath = /home/rocketmq/store/controller
enableElectUncleanMaster = false
notifyBrokerRoleChanged  =  true

2.2 Broker

2.2.1 a

两个节点配置一样

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
cat rocketmq/broker-1/broker/broker.conf
cat rocketmq/broker-2/broker/broker.conf
brokerClusterName=DefaultCluster
brokerName=broker-a
brokerId=-1
brokerRole=SLAVE
listenPort=10911
allAckInSyncStateSet=true
deleteWhen=04
fileReservedTime=48
flushDiskType=ASYNC_FLUSH
enableControllerMode = true
controllerAddr = 192.168.10.67:9877;192.168.10.68:9877;192.168.10.69:9877
namesrvAddr = 192.168.10.67:9876;192.168.10.68:9876;192.168.10.69:9876
storePathRootDir=/home/rocketmq/store
storePathCommitLog=/home/rocketmq/store/commitlog

2.2.2 b

这里只是展示下配置多个broker 怎么写，此文档演示不会部署

1
2
3
4
5
6
7
8
9
10
11
12
13
14
brokerClusterName=DefaultCluster
brokerName=broker-b # 只需要修改这里
brokerId=-1
brokerRole=SLAVE
listenPort=10911
allAckInSyncStateSet=true
deleteWhen=04
fileReservedTime=48
flushDiskType=ASYNC_FLUSH
enableControllerMode = true
controllerAddr = 192.168.10.67:9877;192.168.10.68:9877;192.168.10.69:9877
namesrvAddr = 192.168.10.67:9876;192.168.10.68:9876;192.168.10.69:9876
storePathRootDir=/home/rocketmq/store
storePathCommitLog=/home/rocketmq/store/commitlog

2.3 docker-compose 配置

2.3.1 第一个节点

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
services:
  namesrv-1:
    image: apache/rocketmq:5.3.0
    container_name: rmqnamesrv-1
    network_mode: host
    environment:
      - MAX_POSSIBLE_HEAP=100000000
      - JAVA_OPT_EXT=-Xms512M -Xmx512M -Xmn128m -Drocketmq.namesrv.kvConfigPath=/home/rocketmq/store/kvConfig.json
    volumes:
      - ./namesrv/namesrv.conf:/home/rocketmq/rocketmq-5.3.0/conf/namesrv.conf
      - ./namesrv/store:/home/rocketmq/store
      - /etc/localtime:/etc/localtime
    expose:
      - 9877
      - 9877
    command: sh mqnamesrv -c /home/rocketmq/rocketmq-5.3.0/conf/namesrv.conf
  broker-a:
    image: apache/rocketmq:5.3.0
    container_name: rmqbroker-a
    network_mode: host
    environment:
      - JAVA_OPT_EXT=-Xms2048M -Xmx2048M -Xmn512m
      - JAVA_OPTS=-Duser.timezone=Asia/Shanghai
    expose:
      - 10909
      - 10911
      - 10912
    volumes:
      - ./broker/broker.conf:/home/rocketmq/rocketmq-5.3.0/conf/broker.conf
      - ./broker/plain_acl.yml:/home/rocketmq/rocketmq-5.3.0/conf/acl/plain_acl.yml
      - ./broker/logs:/home/rocketmq/logs
      - ./broker/store:/opt/rmqstore/store
      - /etc/localtime:/etc/localtime:ro
    command: sh mqbroker -c /home/rocketmq/rocketmq-5.3.0/conf/broker.conf
    restart: unless-stopped

2.3.2 第二个节点

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
cat rocketmq/broker-2/docker-compose.yml 
services:
  namesrv-2:
    image: apache/rocketmq:5.3.0
    container_name: rmqnamesrv-2
    network_mode: host
    environment:
      - MAX_POSSIBLE_HEAP=100000000
      - JAVA_OPT_EXT=-Xms512M -Xmx512M -Xmn128m -Drocketmq.namesrv.kvConfigPath=/home/rocketmq/store/kvConfig.json
    volumes:
      - ./namesrv/namesrv.conf:/home/rocketmq/rocketmq-5.3.0/conf/namesrv.conf
      - ./namesrv/store:/home/rocketmq/store
      - /etc/localtime:/etc/localtime
    expose:
      - 9877
    command: sh mqnamesrv -c /home/rocketmq/rocketmq-5.3.0/conf/namesrv.conf
  broker-a:
    image: apache/rocketmq:5.3.0
    container_name: rmqbroker-a
    network_mode: host
    environment:
      - JAVA_OPT_EXT=-Xms2048M -Xmx2048M -Xmn512m
      - JAVA_OPTS=-Duser.timezone=Asia/Shanghai
    expose:
      - 10909
      - 10911
      - 10912
    volumes:
      - ./broker/broker.conf:/home/rocketmq/rocketmq-5.3.0/conf/broker.conf
      - ./broker/plain_acl.yml:/home/rocketmq/rocketmq-5.3.0/conf/acl/plain_acl.yml
      - ./broker/logs:/home/rocketmq/logs
      - ./broker/store:/home/rocketmq/store
      - /etc/localtime:/etc/localtime:ro
    command: sh mqbroker -c  /home/rocketmq/rocketmq-5.3.0/conf/broker.conf

2.3.3 第三个节点

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
cat rocketmq/broker-3/docker-compose.yml 
services:
  namesrv-3:
    image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/rocketmq:5.3.0
    container_name: rmqnamesrv-3
    network_mode: host
    environment:
      - MAX_POSSIBLE_HEAP=100000000
      - JAVA_OPT_EXT=-Xms512M -Xmx512M -Xmn128m -Drocketmq.namesrv.kvConfigPath=/home/rocketmq/store/kvConfig.json
    volumes:
      - ./namesrv/namesrv.conf:/home/rocketmq/rocketmq-5.3.0/conf/namesrv.conf
      - ./namesrv/store:/home/rocketmq/store
      - /etc/localtime:/etc/localtime
    expose:
      - 9877
    command: sh mqnamesrv -c /home/rocketmq/rocketmq-5.3.0/conf/namesrv.conf
  rocketmq-dashboard:
    image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apacherocketmq/rocketmq-dashboard:latest
    container_name: rocketmq-dashboard
    environment:
      - JAVA_OPTS=-Drocketmq.namesrv.addr=192.168.10.67:9876;192.168.10.68:9876;192.168.10.69:9876 -Duser.timezone='Asia/Shanghai'
      - JAVA_OPT_EXT=-Xms512M -Xmx512M -Xmn128m
      - rocketmq.config.loginRequired=true
      - rocketmq.config.accessKey=vldRocketMQ
      - rocketmq.config.secretKey=ZKdyFW3AeknPDtz6cjDA
      - rocketmq.config.dataPath=/rocketmq-console-data
    volumes:
      - /etc/localtime:/etc/localtime
      - ./users.properties:/rocketmq-console-data/users.properties
    ports:
      - 8080:8080

3 启动服务

Attention

这里使用 ansible 部署，也可以手动上次包

3.1 Hosts 文件

已经配置免密，没有的可以取消注释修改

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
cat hosts.ini 
[rocketmq]
192.168.10.67
192.168.10.68
192.168.10.69

[all:children]
rocketmq
[all:vars]
ansible_become=true
ansible_become_method=sudo
; ansible_ssh_user='csdn'
; ansible_ssh_pass="DSJzx@1234"
; ansible_become_pass="DSJzx@1234"
; ansible_ssh_port=22

3.2 批量启动服务

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# 修改ip
find rocketmq -type f -exec sed -i 's/192.168.10.67/192.168.10.67/g' {} +
find rocketmq -type f -exec sed -i 's/192.168.10.68/192.168.10.68/g' {} +
find rocketmq -type f -exec sed -i 's/192.168.10.69/192.168.10.69/g' {} +
手动替换plain_acl.yml网段192.168.10.*
修改jvm配置

# 创建目录并设置权限
ansible rocketmq -i hosts.ini -m file -a "path=/data/server/rocketmq/ state=directory owner=3000 group=3000 mode=0755" --become
# 拷贝文件
ansible 192.168.10.67 -i hosts.ini -m copy -a "src=rocketmq/broker-1/ dest=/data/server/rocketmq/" --become
ansible 192.168.10.68 -i hosts.ini -m copy -a "src=rocketmq/broker-2/ dest=/data/server/rocketmq/" --become
ansible 192.168.10.69 -i hosts.ini -m copy -a "src=rocketmq/broker-3/ dest=/data/server/rocketmq/" --become

# 启动docker-compose服务
ansible rocketmq -i hosts.ini -m shell -a "cd /data/server/rocketmq/ && docker-compose up -d" --become
ansible rocketmq -i hosts.ini -m shell -a "cd /data/server/rocketmq/ && docker-compose down" --become
ansible rocketmq -i hosts.ini -m file -a "path=/data/server/rocketmq/ state=directory owner=3000 group=3000 recurse=yes" --become  # 没有这部会导致没有权限启动不起来
ansible rocketmq -i hosts.ini -m shell -a "cd /data/server/rocketmq/ && docker-compose up -d" --become

4 查看服务

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# 查看 Controller 集群
docker exec -it rmqnamesrv-1 ./mqadmin getControllerMetaData -a 192.168.10.68:9877
#ControllerGroup	group1
#ControllerLeaderId	n0
#ControllerLeaderAddress	192.168.10.67:9877
#Peer:	n0:192.168.10.67:9877
#Peer:	n1:192.168.10.68:9877
#Peer:	n2:192.168.10.69:9877

# 查看 broker 状态
docker exec -it rmqnamesrv-1 ./mqadmin getSyncStateSet -a 192.168.10.67:9877 -b broker-a
#brokerName	broker-a
#MasterBrokerId	2
#MasterAddr	192.168.10.68:10911
#MasterEpoch	2
#SyncStateSetEpoch	4
#SyncStateSetNums	2
InSyncReplica:	ReplicaIdentity{brokerName='broker-a', brokerId=2, brokerAddress='192.168.10.68:10911', alive=true}
InSyncReplica:	ReplicaIdentity{brokerName='broker-a', brokerId=3, brokerAddress='192.168.10.67:10911', alive=true}
NotInSyncReplica:	ReplicaIdentity{brokerName='broker-a', brokerId=1, brokerAddress='192.168.10.67:10911', alive=false}

5 遇到的问题

5.1 broker 节点同步不了，改成单网卡

节点多网卡，broker 使用不能通信的网卡，如

1
2
3
4
5
# 日志报错
AutoSwitchHAClient connect to master 10.0.32.192:10912 failed
# 手动在容器里查看，正确的ip应该是192.168.10.67，但访问的是10.0.32.192
hostname -I 
192.168.10.67 172.17.0.1 10.0.32.192 172.18.0.1