服务器信息

IP Hostname module
172.16.1.12 ServerOne nfs,rpcbind，两块或以上硬盘，keepalived，rsync+inotify
172.16.1.13 ServerTwo nfs,rpcbind,keepalived
172.16.1.14 client centos

部署NFS

请参考前面的文档,不用配置客户端

部署keepalived主备

1
2
3
4
# ServerOne和ServerTwo都执行
[root@ServerOne ~]# yum -y install keepalived
[root@ServerOne ~]# cd /etc/keepalived/
[root@ServerOne keepalived]# cp keepalived.conf keepalived.conf_bak

ServerOne配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
[root@ServerOne keepalived]# vi keepalived.conf

! Configuration File for keepalived
global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id MASTER		# 主 
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}
vrrp_instance VI_1 {
    state MASTER	# 主
    interface ens33
    virtual_router_id 51
    priority 100	# 优先级
    nopreempt	# 非抢占
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        172.16.1.78		# VIP
    }
}
[root@ServerOne keepalived]# scp keepalived.conf root@172.16.1.13:/etc/keepalived/

ServerTwo配置

1
2
3
4
5
[root@ServerTwokeepalived]# vi keepalived.conf
修改：
router_id BACKUP
state BACKUP
priority 90

Server主备放行防火墙

1
2
3
4
5
6
7
vi /etc/firewalld/zones/public.xml
  <rule family="ipv4">
    <protocol value="vrrp"/>
    <accept/>
  </rule>
  
firewall-cmd --reload

主备启动服务

1
2
3
4
5
systemctl start keepalived
# 查看VIP
[root@ServerOne ~]# ip a | grep 1.78
[root@ServerTwo ~]# ip a | grep 1.78
    inet 172.16.1.78/32 scope global ens33

编写keepalived监控脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
[root@ServerOne ~]# vi /etc/keepalived/check_keepalived.sh
#!/bin/bash
nfsstart=`ps -ef | grep '\[nfsd\]' | wc -l`
if [ $nfsstart -eq 0 ];
    then
        systemctl restart nfs
        retry=`ps -ef | grep '\[nfsd\]' | wc -l`
        if [ $retry -eq 0 ];
            then
                systemctl stop keepalived
        fi
fi
[root@ServerOne ~]# scp /etc/keepalived/check_keepalived.sh root@172.16.1.13:/etc/keepalived/

配置免密登录

ServerOne 和 ServerTwo都做

1
2
3
4
# ssh-keygen
ssh-copy -i root@172.16.1.12
ssh-copy -i root@172.16.1.13
ssh-copy -i root@172.16.1.14

部署rsync+inotify

1
2
3
4
[root@ServerOne ~]# tar zxf /inotify-tools-3.14.tar.gz -C /usr/src
[root@ServerOne ~]# cd /usr/src/inotify-tools-3.14/
[root@ServerOne inotify-tools-3.14]# ./configure --prefix=/usr/local/inotify && make && make install
[root@ServerOne inotify-tools-3.14]# ln -s /usr/local/inotify/bin/* /usr/local/bin/

编写实时同步脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
[root@ServerOne ~]# cd /etc/keepalived/
[root@ServerOne keepalive]# vi rsync.sh
#!/bin/bash
SrcPath=/datanfs/
DestPath=/datanfs/
BackupServerone=172.16.1.13
#BackupServertwo=172.16.1.100
inotifywait -mrq --format %w%f -e create,delete,close_write $SrcPath | while read line;
  do
     if [ -f  $line ];
        then
            rsync -a --delete $SrcPath root@${BackupServerone}:$DestPath
     else
            rsync -a --delete $SrcPath root@${BackupServerone}:$DestPath
     fi
  done

运行脚本测试

1
2
3
4
5
6
7
8
9
[root@ServerOne keepalive]# sh /etc/keepalived/rsync.sh &
[root@ServerOne keepalive]# ps -ef | grep rsync.sh
root      47252 112299  0 14:50 pts/1    00:00:00 sh /etc/keepalived/rsync.sh
[root@ServerOne datanfs]# echo "2021/02/08 test rsync" > 20210208test.txt
[root@ServerOne datanfs]# ls
20210208test.txt
# 备机验证
[root@ServerTwo datanfs]# ls
20210208test.txt

编写检查rsync同步脚本

1
2
3
4
5
6
[root@server1 keepalived]# vi /etc/keepalived/check_rsync.sh
CheckRsync=`ps -ef | grep 'sh /etc/keepalived/rsync.sh' | grep -v auto | wc -l`
if [ $CheckRsync -eq 0 ];
    then
        sh /etc/keepalived/rsync.sh >& /etc/keepalived/rsync.log
fi

将检测脚本写入到任务定时器中

1
2
3
4
5
6
7
8
# ServerOne
[root@ServerOne ~]# crontab -e
* * * * * /bin/bash /etc/keepalived/check_keepalived.sh &
* * * * * /bin/bash /etc/keepalived/check_rsync &

# ServerTwo
[root@ServerTwo ~]# crontab -e
* * * * * /bin/bash /etc/keepalived/check_keepalived.sh

客户端验证

创建存储目录

1
2
3
4
[root@client~]# mkdir /datanfs_test
[root@client~]# mount -t nfs 172.16.1.78:/datanfs /datanfs_test/
[root@client~]# df -hT | grep datanfs
172.16.1.78:/datanfs nfs4       20G   33M   20G    1% /datanfs_test

Server端写入数据验证

1
2
3
4
5
6
7
[root@ServerOne datanfs]# echo "Client test" > Client_test.txt
[root@ServerOne datanfs]# ls
20210208test.txt  Client_test.txt
# client查看
[root@client ~]# cd /datanfs_test/
[root@client datanfs_test]# ls
Client_test.txt

模拟故障

关闭ServerTwo节点keepalived服务

VIP地址会漂移到ServerOne节点上

1
2
3
4
[root@ServerTwo ~]# systemctl stop keepalived
[root@ServerTwo ~]# ps -ef | grep keepalived | grep -v auto
[root@ServerTwo ~]# ip a | grep 1.78
# VIP地址已不存在

查看VIP地址是否漂移到ServerOne节点上

1
2
[root@ServerOne ~]# ip a| grep 1.78
    inet 172.16.1.78/32 scope global ens33

Client节点

1
2
# 卸载挂载点
[root@client ~]# umount -l /datanfs_test

umount不加参数卸不掉，需要加参数 -l
-l :立即断开文件系统

重新挂载

1
2
3
4
5
6
[root@client ~]# mount -t nfs 172.16.1.78:/datanfs /datanfs_test/
[root@client ~]# df -hT | grep datanfs
172.16.1.78:/datanfs nfs4       20G   33M   20G    1% /datanfs_test
[root@client datanfs_test]# ls
Client_test.txt
# 数据正常存在！

附带

Server端keepalived主备切换之后，Client端需要重新挂载nfs共享存储！使用以下脚本可以解决手动卸载与挂载的问题

client自动卸载与挂载共享目录脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/bin/bash
#!/usr/bin/expect
check_kp=`ps -ef | grep 'keepalived -D' | grep -v auto | wc -l`
if [ $check_kp -eq 0 ];
    then
        expect -c "
        spawn ssh root@172.16.1.14
        expect \"#\"
        send \"umount -l /datanfs_test\r\"
        expect \"#\"
        send \"mount -t nfs 172.16.1.78:/datanfs /datanfs_test\r\"
        expect \"#\"
        send \"exit\r\"
        interact"
fi