Linux-keepalive-lvs双机高可用负载均衡

基于Centos5.7测试通过

一、下载

# 建议安装稳定版本1.2.1
wget http://www.keepalived.org/software/keepalived-1.2.1.tar.gz

二、安装LVS管理工具

yum -y install ipvsadm

三、编译安装

# 依赖包
yum -y install popt-devel openssl-devel libnl-devel

# 处理Use IPVS Framework       : No问题
# 仍然报错则需要升级kernel，因为kernel和kernel-headers不匹配，注意：此方法需要重启服务器
yum -y install kernel-headers kernel-devel
# 查看是否匹配，软链接不存在的话，则有问题
ll /lib/modules/$(uname -r)/build

# 不重启线上服务安装对应版本kernel-devel，rpm -e可以删除指定包，不会删除依赖包：
# rpm -qa|grep kernel
# rpm -e kernel-devel-2.6.32-573.7.1.el6.x86_64  # 版本过高
# 下载/lib/modules/$(uname -r)/build对应的版本并安装
# rpm -ivh kernel-devel-2.6.32-279.el6.x86_64.rpm

# 安装
tar xzf keepalived-1.2.1.tar.gz 
cd keepalived-1.2.1
./configure --prefix=/usr/local/keepalived --with-kernel-dir=/lib/modules/$(uname -r)/build
make && make install
install -o root -g root -m 500 /usr/local/keepalived/etc/rc.d/init.d/keepalived /etc/init.d/keepalived
install -o root -g root -m 644 /usr/local/keepalived/etc/sysconfig/keepalived /etc/sysconfig/keepalived
install -o root -g root -d -m 755 /etc/keepalived
ln -s /usr/local/keepalived/sbin/keepalived /usr/sbin/

# 配置文件参考：/usr/local/keepalived/etc/keepalived目录下

四、修改防火墙

# 官方建议关闭防火墙，如果LVS主机都配置了防火墙，则需要配置以下访问权限
iptables -A INPUT -i eth0 -d 224.0.0.0/8 -j ACCEPT
iptables -A INPUT -i eth0 -p vrrp -j ACCEPT

五、配置双机高可用负载均衡

1、技术选择

# 配置virtual_server有三种方式，只有借助iptable的mark才能用于双机高可用案例
virtual_server IP port
virtual_server fwmark int  # 使用此方法(firewall mark)
virtual_server group string

2、通过防火墙为每个包打标签

# 主上面(MAC为备的MAC，标记为3)
# iptables -t mangle -I PREROUTING -d $VIP -p tcp -m tcp --dport $VPORT -m mac \ ! --mac-source $MAC_Director2 -j MARK --set-mark 0x3
iptables -t mangle -I PREROUTING -d 192.168.100.60 -p tcp -m tcp --dport 80 -m mac ! --mac-source 48:5B:39:12:52:11 -j MARK --set-mark 0x3

# 防火墙mangle链实例
#   # Generated by iptables-save v1.3.5 on Fri May 22 14:27:21 2015
#   *mangle
#   :PREROUTING ACCEPT [8501667:1563640067]
#   :INPUT ACCEPT [8501361:1563623487]
#   :FORWARD ACCEPT [0:0]
#   :OUTPUT ACCEPT [9047544:1223155323]
#   :POSTROUTING ACCEPT [9047544:1223155323]
#   -A PREROUTING -d 192.168.100.60 -p tcp -m tcp --dport 80 -m mac ! --mac-source 48:5B:39:12:52:11 -j MARK --set-mark 0x3
#   COMMIT
#   # Completed on Fri May 22 14:27:21 2015
#   # Generated by iptables-save v1.3.5 on Fri May 22 14:27:21 2015

# 备上面(MAC为主的MAC，标记为4)
# iptables -t mangle -I PREROUTING -d $VIP -p tcp -m tcp --dport $VPORT -m mac \ ! --mac-source $MAC_Director1 -j MARK --set-mark 0x4
iptables -t mangle -I PREROUTING -d 192.168.100.60 -p tcp -m tcp --dport 80 -m mac ! --mac-source 00:21:5E:70:AB:C6 -j MARK --set-mark 0x4

3、master配置文件

# /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   # diff router_id
   router_id LVS_1
}

vrrp_sync_group G1 { 
   group { 
        VI_1 
   } 
}

vrrp_instance VI_1 {
    # diff state
    state MASTER
    interface eth1
    # 同一网段包含多组vrrp_instance(或者keepalived)则需要修改virtual_router_id值,且唯一
    virtual_router_id 51
    # MASTER bigger priority
    priority 150
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass pMP8NsHBsCdF
    }
    virtual_ipaddress {
        192.168.100.60
    }

    notify_master "/oper/script/check_lvs.sh master"
    notify_backup "/oper/script/check_lvs.sh backup"
}
# fwmark int
virtual_server fwmark 3 80 {
    delay_loop 6
    lb_algo wlc
    lb_kind DR
    nat_mask 255.255.255.224
    persistence_timeout 50
    protocol TCP

    real_server 192.168.100.47 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

    real_server 192.168.100.59 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

4、backup配置文件

# /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   # diff router_id
   router_id LVS_2
}

vrrp_sync_group G1 { 
   group { 
        VI_1 
   } 
}

vrrp_instance VI_1 {
    # diff state
    state BACKUP
    interface eth1
    # 同一网段包含多组vrrp_instance(或者keepalived)则需要修改virtual_router_id值,且唯一
    virtual_router_id 51
    # MASTER bigger priority
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass pMP8NsHBsCdF
    }
    virtual_ipaddress {
        192.168.100.60
    }

    notify_master "/oper/script/check_lvs.sh master"
    notify_backup "/oper/script/check_lvs.sh backup"
}

virtual_server fwmark 4 80 {
    delay_loop 6
    lb_algo wlc
    lb_kind DR
    nat_mask 255.255.255.224
    persistence_timeout 50
    protocol TCP

    real_server 192.168.100.47 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

    real_server 192.168.100.59 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

五、启动

# 运行ipvsadm，解决keepalive启动报错(IPVS: Can't initialize ipvs: Protocol not available)
ipvsadm
/etc/init.d/keepalived start
# 启动keepalived，默认主从均配置好ipvsadm路由策略，通过下面命令查看(只是VIP接管问题)
ipvsadm
# ActiveConn是活动连接数,也就是tcp连接状态的ESTABLISHED;InActConn是指除了ESTABLISHED以外的,所有的其它状态的tcp连接

六、后端服务器配置

1、后端脚本

重要： 没有这个脚本会导致后端服务器无法访问，后端只需要添加这个脚本即可，建议路径：/etc/init.d/lvs_realserver

原理：active router收到vip的请求后，将mac地址改成real server的mac地址，并发送给real server ，real server 的链路层收到请求，上传到ip层，这时ip层需要验证目标ip，所以real server 需要一个配置一个vip，否则会拒绝掉这个包,这里使用lo:0,因为eth0:1会和局域网中的vip发生冲突。然后由real server直接回复客户端。

#!/bin/bash
# description : start realserver
VIP=192.168.100.60
. /etc/rc.d/init.d/functions

case "$1" in
start)
    echo " start LVS of REALServer"
    /sbin/ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
    echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
    echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
    echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
    echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
;;
stop)
    /sbin/ifconfig lo:0 down
    echo "close LVS Directorserver"
    echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
    echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
    echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
    echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
;;
*)
    echo "Usage: $0 {start|stop}"
    exit 1
esac

2、授权、启动

chmod 500 /etc/init.d/lvs_realserver
/etc/init.d/lvs_realserver start

七、报警脚本

默认的监控报警需要搭建email服务器，但是通常这种邮件服务器会被其它邮件服务器拒绝掉，建议使用邮件客户端设置报警或者短信接口报警，下面为测试脚本示例：

1、服务端配置文件

# 主从vrrp_instance下均添加下面两行配置文件
# chmod 500 /oper/script/check_lvs.sh
notify_master "/oper/script/check_lvs.sh master"
notify_backup "/oper/script/check_lvs.sh backup"

2、脚本示例

#!/bin/bash
ctime=`date +"%F %H:%M:%S"`
ip=192.168.100.47
msg="${ctime}: ${ip} change $1"
logfile=/var/log/lvs.log

# 邮件
smtp='smtp.163.com'
emailu='user'
emailp='passwd'
senderemail='user@163.com'
toemail='aaa@163.com bbb@163.com'

#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
# 标题 消息 收件人
# sendm "${ctime}：keepalive通知" ${scan_result} safe@163.com
# http://caspian.dotconf.net/menu/Software/SendEmail/sendEmail-v1.56.tar.gz
sendm(){
    SUBJECT="$1"
    local msg="$2"
    toemail="$3"
    /usr/local/bin/sendEmail -f $senderemail -t $toemail -s $smtp -u "$SUBJECT" -xu $emailu -xp $emailp -o message-charset=gbk -m ${msg}
}

sendm "keepalive通知" "${msg}" "${toemail}"
echo "${msg}" >> ${logfile}

八、测试

1、确认脑裂方法

a、查看日志(tailf /var/log/messages)，均声明为MASTER
b、查看虚拟IP(ip a)，两台机器均包含VIP

2、测试相关包是否打标记

 # 主上面查看，远程telnet
telnet 192.168.100.60 80
iptables -t mangle -nvL

# 第一列表示为处理的数据包数，访问没有计数的话，说明没有包打过标记，有问题
 pkts bytes target     prot opt in     out     source               destination         
 100K   12M MARK       tcp  --  *      *       0.0.0.0/0            192.168.100.60      tcp dpt:80 MAC ! 48:5B:39:12:52:11 MARK set 0x3

3、测试后端服务器掉线踢出

[root@test-39 keepalived]# ipvsadm
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.100.60:http rr persistent 50
  -> 192.168.100.59:http          Route   1      0          0         
  -> 192.168.100.47:http          Route   1      0          0         
[root@test-39 keepalived]# ipvsadm     # 关闭59的real_server相关进程后，LVS自动删除相关路由
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.100.60:http rr persistent 50
  -> 192.168.100.47:http          Route   1      0          0

4、测试后端服务器的VIP是否OK

[root@test-47 public]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet 192.168.100.60/32 brd 192.168.100.60 scope global lo:0  # 此配置用于后端服务器，注意lo下面必须包含了相关虚拟VIP才能正常路由访问
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 00:21:5e:70:ab:c4 brd ff:ff:ff:ff:ff:ff
    inet 192.168.3.47/24 brd 192.168.3.255 scope global eth0
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 00:21:5e:70:ab:c6 brd ff:ff:ff:ff:ff:ff
    inet 192.168.100.47/27 brd 192.168.100.63 scope global eth1
    inet 192.168.100.60/32 scope global eth1    # 此配置用于keepalive，MASTER上面才有

5、ipvsadm显示详细负载连接

ipvsadm -Lcn

文章目录

Linux-keepalive-lvs双机高可用负载均衡

一、下载

二、安装LVS管理工具

三、编译安装

四、修改防火墙

五、配置双机高可用负载均衡

1、技术选择

2、通过防火墙为每个包打标签

3、master配置文件

4、backup配置文件

五、启动

六、后端服务器配置

1、后端脚本

2、授权、启动

七、报警脚本

1、服务端配置文件

2、脚本示例

八、测试

1、确认脑裂方法

2、测试相关包是否打标记

3、测试后端服务器掉线踢出

4、测试后端服务器的VIP是否OK

5、ipvsadm显示详细负载连接

参考文档

See Also

最近文章

分类

标签

友情链接

其它