LVS and HAProxy

[topology]

1. For LVS-NAT

2. For LVS-DR

3. For HAProxy

[test-lvs-nat-rr.py]

#!/usr/bin/env python

from mininet.net import Containernet

from mininet.cli import CLI

from mininet.link import Link,TCLink,Intf

from mininet.log import setLogLevel

from mininet.node import Docker

from time import sleep

if '__main__' == __name__:

setLogLevel('info')

net = Containernet(link=TCLink)

h1 = net.addHost('h1',ip="10.0.1.1/24")

r = net.addHost('r')

h2 = net.addDocker('h2', mac = '00:00:00:00:02:02', ip="10.0.2.2/24", dimage="apache-php-mysql:v7",cpu_period=50000, cpu_quota=1000)

h3 = net.addDocker('h3', mac = '00:00:00:00:03:03', ip="10.0.3.3/24", dimage="apache-php-mysql:v7",cpu_period=50000, cpu_quota=1000)

net.addLink(h1, r, cls=TCLink, bw=10)

net.addLink(h2, r, cls=TCLink, bw=10)

net.addLink(h3, r, cls=TCLink, bw=10)

net.start()

h1,h2,h3,r=net.get('h1','h2','h3','r')

r.cmd("ifconfig r-eth0 0")

r.cmd("ifconfig r-eth1 0")

r.cmd("ifconfig r-eth2 0")

r.cmd("echo 1 > /proc/sys/net/ipv4/ip_forward")

r.cmd("ip addr add 10.0.1.254/24 brd + dev r-eth0")

r.cmd("ip addr add 10.0.2.254/24 brd + dev r-eth1")

r.cmd("ip addr add 10.0.3.254/24 brd + dev r-eth2")

r.cmd("iptables -t nat -A POSTROUTING -s 10.0.2.0/24 -o r-eth0 -j MASQUERADE")

r.cmd("iptables -t nat -A POSTROUTING -s 10.0.3.0/24 -o r-eth0 -j MASQUERADE")

r.cmd("ipvsadm -A -t 10.0.1.254:80 -s rr")

r.cmd("ipvsadm -a -t 10.0.1.254:80 -r 10.0.2.2:80 -m")

r.cmd("ipvsadm -a -t 10.0.1.254:80 -r 10.0.3.3:80 -m")

h1.cmd("ip route add default via 10.0.1.254")

h2.cmd("ip route del default")

h2.cmd("ip route add default via 10.0.2.254")

h2.cmd("cd /var/www/html; echo h2 > a.htm ; python -m SimpleHTTPServer 80 &")

h3.cmd("ip route del default")

h3.cmd("ip route add default via 10.0.3.254")

h3.cmd("cd /var/www/html; echo h3 > a.htm ; python -m SimpleHTTPServer 80 &")

CLI(net)

net.stop()

[test-lvs-nat-wrr.py]

#!/usr/bin/env python

from mininet.net import Containernet

from mininet.cli import CLI

from mininet.link import Link,TCLink,Intf

from mininet.log import setLogLevel

from mininet.node import Docker

from time import sleep

if '__main__' == __name__:

setLogLevel('info')

net = Containernet(link=TCLink)

h1 = net.addHost('h1',ip="10.0.1.1/24")

r = net.addHost('r')

h2 = net.addDocker('h2', mac = '00:00:00:00:02:02', ip="10.0.2.2/24", dimage="apache-php-mysql:v7",cpu_period=50000, cpu_quota=1000)

h3 = net.addDocker('h3', mac = '00:00:00:00:03:03', ip="10.0.3.3/24", dimage="apache-php-mysql:v7",cpu_period=50000, cpu_quota=1000)

net.addLink(h1, r, cls=TCLink, bw=10)

net.addLink(h2, r, cls=TCLink, bw=10)

net.addLink(h3, r, cls=TCLink, bw=10)

net.start()

h1,h2,h3,r=net.get('h1','h2','h3','r')

r.cmd("ifconfig r-eth0 0")

r.cmd("ifconfig r-eth1 0")

r.cmd("ifconfig r-eth2 0")

r.cmd("echo 1 > /proc/sys/net/ipv4/ip_forward")

r.cmd("ip addr add 10.0.1.254/24 brd + dev r-eth0")

r.cmd("ip addr add 10.0.2.254/24 brd + dev r-eth1")

r.cmd("ip addr add 10.0.3.254/24 brd + dev r-eth2")

r.cmd("iptables -t nat -A POSTROUTING -s 10.0.2.0/24 -o r-eth0 -j MASQUERADE")

r.cmd("iptables -t nat -A POSTROUTING -s 10.0.3.0/24 -o r-eth0 -j MASQUERADE")

r.cmd("ipvsadm -A -t 10.0.1.254:80 -s wrr")

r.cmd("ipvsadm -a -t 10.0.1.254:80 -r 10.0.2.2:80 -m -w 1")

r.cmd("ipvsadm -a -t 10.0.1.254:80 -r 10.0.3.3:80 -m -w 2")

h1.cmd("ip route add default via 10.0.1.254")

h2.cmd("ip route del default")

h2.cmd("ip route add default via 10.0.2.254")

h2.cmd("cd /var/www/html; echo h2 > a.htm ; python -m SimpleHTTPServer 80 &")

h3.cmd("ip route del default")

h3.cmd("ip route add default via 10.0.3.254")

h3.cmd("cd /var/www/html; echo h3 > a.htm ; python -m SimpleHTTPServer 80 &")

CLI(net)

net.stop()

[test-lvs-dr.py]

#!/usr/bin/env python

from mininet.net import Containernet

from mininet.cli import CLI

from mininet.link import Link,TCLink,Intf

from mininet.log import setLogLevel

from mininet.node import Docker

from time import sleep

if '__main__' == __name__:

setLogLevel('info')

net = Containernet(link=TCLink)

h1 = net.addHost('h1',ip="10.0.1.1/24")

r = net.addHost('r')

br = net.addHost('br')

lb = net.addHost('lb', mac = '00:00:00:00:01:01', ip="10.0.2.1/24")

h2 = net.addDocker('h2', mac = '00:00:00:00:02:02', ip="10.0.2.2/24", dimage="apache-php-mysql:v7",cpu_period=50000, cpu_quota=1000)

h3 = net.addDocker('h3', mac = '00:00:00:00:03:03', ip="10.0.2.3/24", dimage="apache-php-mysql:v7",cpu_period=50000, cpu_quota=1000)

net.addLink(h1, r, cls=TCLink, bw=10)

net.addLink(br, r, cls=TCLink, bw=10)

net.addLink(lb, br, cls=TCLink, bw=10)

net.addLink(h2, br, cls=TCLink, bw=10)

net.addLink(h3, br, cls=TCLink, bw=10)

net.start()

h1,h2,h3,lb,br,r=net.get('h1','h2','h3','lb','br','r')

r.cmd("ifconfig r-eth0 0")

r.cmd("ifconfig r-eth1 0")

r.cmd("echo 1 > /proc/sys/net/ipv4/ip_forward")

r.cmd("ip addr add 10.0.1.254/24 brd + dev r-eth0")

r.cmd("ip addr add 10.0.2.254/24 brd + dev r-eth1")

br.cmd("ifconfig br-eth0 0")

br.cmd("ifconfig br-eth1 0")

br.cmd("ifconfig br-eth2 0")

br.cmd("ifconfig br-eth3 0")

br.cmd("brctl addbr br0")

br.cmd("brctl addif br0 br-eth0")

br.cmd("brctl addif br0 br-eth1")

br.cmd("brctl addif br0 br-eth2")

br.cmd("brctl addif br0 br-eth3")

br.cmd("ifconfig br0 up")

lb.cmd("ifconfig lb-eth0:0 10.0.2.100/24")

lb.cmd("ip route add default via 10.0.2.254")

#lb.cmd("ipvsadm -A -t 10.0.2.100:80 -s rr")

#lb.cmd("ipvsadm -a -t 10.0.2.100:80 -r 10.0.2.2 -g")

#lb.cmd("ipvsadm -a -t 10.0.2.100:80 -r 10.0.2.3 -g")

h1.cmd("ip route add default via 10.0.1.254")

h2.cmd("ip route del default")

h2.cmd("ip route add default via 10.0.2.254")

h2.cmd("cd /var/www/html; echo h2 > a.htm ; python -m SimpleHTTPServer 80 &")

h2.cmd("echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce")

h2.cmd("echo 2 > /proc/sys/net/ipv4/conf/h2-eth0/arp_announce")

h2.cmd("echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore")

h2.cmd("echo 1 > /proc/sys/net/ipv4/conf/h2-eth0/arp_ignore")

h2.cmd("ifconfig lo:0 10.0.2.100 netmask 255.255.255.255 broadcast 10.0.2.100")

h2.cmd("route add -host 10.0.2.100 dev lo:0")

h3.cmd("ip route del default")

h3.cmd("ip route add default via 10.0.2.254")

h3.cmd("cd /var/www/html; echo h3 > a.htm ; python -m SimpleHTTPServer 80 &")

h3.cmd("echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce")

h3.cmd("echo 2 > /proc/sys/net/ipv4/conf/h2-eth0/arp_announce")

h3.cmd("echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore")

h3.cmd("echo 1 > /proc/sys/net/ipv4/conf/h2-eth0/arp_ignore")

h3.cmd("ifconfig lo:0 10.0.2.100 netmask 255.255.255.255 broadcast 10.0.2.100")

h3.cmd("route add -host 10.0.2.100 dev lo:0")

CLI(net)

net.stop()

[test-haproxy.py]

#!/usr/bin/env python

from mininet.net import Containernet

from mininet.cli import CLI

from mininet.link import Link,TCLink,Intf

from mininet.log import setLogLevel

from mininet.node import Docker

from time import sleep

from datetime import datetime

import time

if '__main__' == __name__:

setLogLevel('info')

net = Containernet (link=TCLink)

h1 = net.addHost('h1',ip="10.0.1.1/24")

r = net.addHost('r')

h2 = net.addDocker('h2', mac = '00:00:00:00:02:02', ip="10.0.2.2/24", dimage="apache-php-mysql:v7",cpu_period=50000, cpu_quota=1000)

h3 = net.addDocker('h3', mac = '00:00:00:00:03:03', ip="10.0.3.3/24", dimage="apache-php-mysql:v7",cpu_period=50000, cpu_quota=1000)

Link(h1, r)

Link(r, h2)

Link(r, h3)

net.build()

h1,h2,h3,r,=net.get('h1','h2','h3','r')

r.cmd("ifconfig r-eth0 0")

r.cmd("ifconfig r-eth1 0")

r.cmd("ifconfig r-eth2 0")

r.cmd("echo 1 > /proc/sys/net/ipv4/ip_forward")

r.cmd("ip addr add 10.0.1.254/24 brd + dev r-eth0")

r.cmd("ip addr add 10.0.2.254/24 brd + dev r-eth1")

r.cmd("ip addr add 10.0.3.254/24 brd + dev r-eth2")

r.cmd("iptables -t nat -A POSTROUTING -s 10.0.2.0/24 -o r-eth0 -j MASQUERADE")

r.cmd("iptables -t nat -A POSTROUTING -s 10.0.3.0/24 -o r-eth0 -j MASQUERADE")

h1.cmd("ip route add default via 10.0.1.254")

h2.cmd("ip route del default")

h2.cmd("ip route add default via 10.0.2.254")

h2.cmd("cd /var/www/html; echo h2 > a.htm ; python -m SimpleHTTPServer 80 &")

h3.cmd("ip route del default")

h3.cmd("ip route add default via 10.0.3.254")

h3.cmd("cd /var/www/html; echo h3 > a.htm ; python -m SimpleHTTPServer 80 &")

CLI(net)

net.stop()

[haproxy.cfg]

listen test

bind 10.0.1.254:80

mode tcp

balance static-rr

balance roundrobin

server web1 10.0.2.2:80 check weight 1 check inter 1s

server web2 10.0.3.3:80 check weight 1 check inter 1s

[Execution]

For LVS-NAT-RR (We can see that h1 gets the web pages from h3 or h2 in a round-robin way.)

For LVS-NAT-WRR (We can see that h1 gets the web pages from h3 or h2 in a weighted round-robin way. The ratio for h2:h3 is 1:2.)

Open another terminal to log into h3. Terminate the web service

Go back to mininet terminal to get the webpage from h1. We can see some executions of curl program fail. Because lvs itself does not have health check function. (LVS needs “keepalived” to do health check job.)

For LVS-DR

For HAProxy

Open another terminal to log into h3. Terminate the web service. We can see only the web page can be obtained from h2. No curl program fail.

[References]

https://blog.51cto.com/lansgg/1229421

Dr. Chih-Heng Ke (smallko@gmail.com)

Department of Computer Science and Information Engineering,

National Quemoy University, Kinmen, Taiwan.