High Availability

$B%$%s%?!<%M%C%H$G>&MQ%"%W%j%1!<%7%g%s$,=EBg$K$J$l$P$J$k$[$I!"Lr$KN)$D%5!<%S%9$rDs6!$9$k$3$H$OHs>o$K=EMW$K$J$k!#%/%i%9%?%j%s%0%7%9%F%`$NMxE@$O%O!<%I%&%'%"$H%=%U%H%&%'%"$N>iD9@-$r;}$?$;$k$3$H$G$"$k!#;E;v$NJ,C4$r%/%i%9%?!<$,;D$C$?%N!<%I$K$h$C$F!"E,3N$K0z$-7Q$2$k$h$&$K9b$$2DMQ@-$O<:GT$7$?%N!<%I$d%G!<%b%s$K$h$C$F%7%9%F%`$r:F9=@.$9$k!#(B

$B;v1. mon+heartbeat+fake+coda $B$K$h$k2r7hK!(B

$B%P!<%A%c%k%5!<%P!<$N9b$$M-8z@-$O(B "mon"$B$d(B"heartbeat" $B$d(B"fake"$B$d(B"coda" $B$N%=%U%H%&%'%"$r;H$C$FDs6!$9$k!#(B "mon"$B$OHFMQ%j%=!<%9$r%b%K%?%j%s%0$9$k%7%9%F%`$G%M%C%H%o!<%/%5!<%S%9$NM-8z@-$H%5!<%P!<%N!<%I$r%b%K%?!<$9$k$?$a$K;H$o$l$k!#(B "heartbeat"$B$O8=:_#2$D$N%N!<%I4V$G%7%j%"%k%i%$%s$d(BUDP $B?4Go$r;H$C$FDs6!$7$^$9!#(B "fake"$B$O(BARP$B$r$@$^$7$F;H$&$3$H$K$h$C$F!"(BIP$B$r$N$C$H$C$?%=%U%H%&%'%"$N$3$H$G$9!#(B Linux Virtual Server $B$N9b$$M-8z@-$O

$B%5!<%P!<$N%U%'%$%k%*!<%P!<$Oe$GF0:n$9$k!#(B fping.monitor $B$OKhIC%5!<%P!<%N!<%I$,F0$$$F$$$k$+$I$&$+8!=P$9$k$?$a$K9=@.$5$l$F$$$F!"4XO"@-$N$"$k%5!<%S%9%b%K%?!<$b(B m $BIC$4$H$K$9$Y$F$N%N!<%I>e$N%5!<%S%9%G!<%b%s$r8!=P$9$k$?$a$K9=@.$5$l$F$$$k!#Nc$($P!"(Bhttp.monitor $B$O(B http $B%5!<%S%9$r%A%'%C%/$9$k$?$a$K;H$o$l!"(B ftp.monitor $B$O(B FTP $B%5!<%S%9$N$?$a$N$b$N$G$"$k!#(B $B%5!<%P!<%N!<%I$+%G!<%b%s$,>e$,$C$F$$$k$+!"Mn$A$F$$$k$+$r8!=P$9$k4V!"(Blvs$B%F!<%V%k$N%k!<%k$rDI2C!":o=|$9$k$?$a$K7YJs$,=q$-9~$^$l$k!#(B $B$=$N7k2L!"%m!<%I%P%i%s%5!<$O%5!<%S%9%G!<%b%s$d%5!<%P!<$,Mn$A$?;~$=$l$r<+F0E*$K1#$9$3$H$,$G$-!"@5>o$J>uBV$KLa$C$?;~!"$=$l$i$N%5!<%S%9$rLa$i$;$k$3$H$,$G$-$k!#(B

$B$=$7$F%m!<%I%P%i%s%5!<$,A4BN$N%7%9%F%`$GC10l8N>c$7$?$H$9$k!#0lHV=EMW$J%m!<%I%P%i%s%5!<$N8N>c$r1#$9$?$a$K!";d$?$A$O%m!<%I%P%i%s%5!<$N%P%C%/%"%C%W%5!<%P!<$r%;%C%H%"%C%W$9$kI,MW$,$"$k!#(B"fake"$B$H$$$&%=%U%H%&%'%"$O%m!<%I%P%i%s%5!<$,Mn$A$?;~!"%m!<%I%P%i%s%5!<$N(BIP$B%"%I%l%9$r0z$-7Q$0$?$a$N%P%C%/%"%C%W$H$7$F;H$o$l$k!#(B $B$^$?!"(B"heartbeat"$B$O%P%C%/%"%C%W%5!<%P!<$N(B"fake"$B$,5/F0$9$k$+!"$7$J$$$+$rD4$Y$k$?$a$K%m!<%I%P%i%s%5!<$N>u67$r8!=P$9$k$?$a$K;H$o$l$k!#(B $B#2$D$N(B"heartbeat"$B%G!<%b%s$O%W%i%$%^%j$H%P%C%/%"%C%W$GF0:n$7!"%7%j%"%k%i%$%s>e$G!"<~4|E*$K(B"I'm alive"$B$N$h$&$J%a%C%;!<%8$r$d$j$H$j$7$F$$$k!#(B $B%P%C%/%"%C%W$N(B"heartcode"$B%G!<%b%s$,0lDj;~4VFb$G%W%i%$%^%j$+$i(B"I'm alive"$B%a%C%;!<%8$rJ9$/$3$H$,$G$-$J$+$C$?$i!"%P!<%A%c%k(BIP$B%"%I%l%9$r0z$-7Q$.!"%m!<%I%P%i%s%7%s%0%5!<%S%9$r6!5k$9$k$?$a$K(B"fake"$B$r5/F0$9$k!#(B $B8e$G!"%W%i%$%^%j$+$i(B"I'm alive"$B%a%C%;!<%8$r$B$7$+$7!"%W%i%$%^%j%m!<%I%P%i%s%5!<$N%U%'%$%k%*!<%P!<$d%F%$%/%*!<%P!<$O%O%C%7%e%F!<%V%k$K$"$k8=:_ Coda $B$O>iD9@-$N$"$kJ,;6%U%!%$%k%7%9%F%`$G(BAndrew$B%U%!%$%k%7%9%F%`$N;RB9$G$"$k!#(B $B%U%!%$%k$rM-8z$K4JC1$K4IM}$9$k$?$a$K!"%5!<%P!<$N%3%s%F%s%D$O(BCoda$B$r;H$C$FJ]B8$9$k!#(B

$B@_DjNc(B

$B$B%j%"%k%5!<%P!<$N%U%'%$%k%*!<%P!<(B

"mon"$B$O%/%i%9%?!<$G%5!<%S%9%G!<%b%s$H%5!<%P!<%N!<%I$r4F;k$9$k$?$a$K;H$o$l$k!#(B $BNc$($P!"(Bfping.monitor $B$O%5!<%P!<%N!<%I$r4F;k$9$k$?$a$K;H$o$l!"(Bhttp.monitor $B$O(B http $B%5!<%S%9$r%A%'%C%/$9$k$?$a$K;H$o$l!"(B ftp.monitor $B$O(B FTP $B%5!<%S%9$N$?$a$N$b$N$G$"$k!#(B $B$@$+$i;d$?$A$O!"%5!<%P!<%N!<%I$+%G!<%b%s$,>e$,$C$F$$$k$+!"Mn$A$F$$$k$+$r8!=P$9$k4V!"(Blvs$B%F!<%V%k$N%k!<%k$rDI2C!":o=|$9$k$?$a$K7YJs$r=q$-9~$`I,MW$,$"$k!#(B $B$3$l$O(B lvs.aler$B$H8F$P$l$kNc$@!#$3$l$O%j%"%k%5!<%P!<$N%Q%i%a!<%?$H$7$F(BIP$B$H%]!<%HHV9f$r;H$&!#(B 
#!/usr/bin/perl 
# 
# lvs.alert - Linux Virtual Server alert for mon 
# 
# It can be activated by mon to remove a real server when the 
# service is down, or add the server when the service is up. 
# 
# "mon"$B$K$h$C$F!"%5!<%S%9$,Mn$A$?;~%j%"%k%5!<%P!<$r%O%C%7%e%F!<%V%k$+$i:o=|$7!"(B # $B%5!<%S%9$,La$C$?$i%j%"%k%5!<%P!<$r%O%C%7%e%F!<%V%k$KDI2C$9$k$?$a$K5/F0$5$l$k(B
# 

use Getopt::Std;
getopts ("s:g:h:t:l:P:V:R:W:F:u");

$ipvsadm = "/sbin/ipvsadm";
$protocol = $opt_P;
$virtual_service = $opt_V; 
$remote = $opt_R;

if ($opt_u) {
    $weight = $opt_W;
    if ($opt_F eq "nat") {
	$forwarding = "-m";
    } elsif ($opt_F eq "tun") {
	$forwarding = "-i";
    } else {
	 $forwarding = "-g";
    }

    if ($protocol eq "tcp") {
        system("$ipvsadm -a -t $virtual_service -r $remote -w $weight $forwarding");
    } else {
	system("$ipvsadm -a -u $virtual_service -r $remote -w $weight $forwarding");
    }
} else { 
    if ($protocol eq "tcp") {
	system("$ipvsadm -d -t $virtual_service -r $remote");
    } else {
	system("$ipvsadm -d -u $virtual_service -r $remote");
    }
};

lvs.alert $B$O(B /usr/lib/mon/alert.d $B%G%#%l%/%H%j$KCV$+$l$k!#(B"mon"$B$N@_Dj%U%!%$%k$O(B(/etc/mon/mon.cf $BKt$O(B /etc/mon.cf)$B 

#
# The mon.cf file
#
#
# global options
#
cfbasedir   = /etc/mon 
alertdir   = /usr/lib/mon/alert.d
mondir     = /usr/lib/mon/mon.d
maxprocs    = 20
histlength = 100
randstart = 30s

#
# group definitions (hostnames or IP addresses)
#
hostgroup www1 www1.domain.com 

hostgroup www2 www2.domain.com

#
# Web server 1
# 
watch www1
    service http 
       	interval 10s 
	monitor http.monitor 
	period wd {Sun-Sat} 
	    alert mail.alert wensong 
	    upalert mail.alert wensong 
	    alert lvs.alert -P tcp -V 10.0.0.3:80 -R 192.168.0.1 -W 5 -F dr
	    upalert lvs.alert -P tcp -V 10.0.0.3:80 -R 192.168.0.1 -W 5 -F dr

#
# Web server 2
# 
watch www2 
    service http 
        interval 10s 
	monitor http.monitor 
	period wd {Sun-Sat} 
	    alert mail.alert wensong 
	    upalert mail.alert wensong 
	    alert lvs.alert -P tcp -V 10.0.0.3:80 -R 192.168.0.2 -W 5 -F dr
	    upalert lvs.alert -P tcp -V 10.0.0.3:80 -R 192.168.0.2 -W 5 -F dr

$B$b$7(B LVS/NAT $B$G(B destination port $B$,0c$C$F$$$?$i!"(B "lvs.alert -V 10.0.0.3:80 -R 192.168.0.3:8080" $B$N$h$&$K@_Dj$7$J$$$GE,59<+J,$N4D6-$K9g$o$;$FBe$($F$/$@$5$$(B

$B%m!<%I%P%i%s%5!<$O%5!<%S%9%G!<%b%s$d%5!<%P!<$,Mn$A$?$3$H$r<+F0E*$K1#$9$3$H$,$G$-!"Mn$A$?%5!<%P!<$,I|5"$7$?;~!"%5!<%S%9$r85$N%5!<%P!<$KLa$9$3$H$,$G$-$^$9!#(B

$B%m!<%I%P%i%s%5!<$N%U%'%$%k%*!<%P!<(B

$B%m!<%I%P%i%s%5!<$,A4BN$N%7%9%F%`$GC10l8N>c$9$k$N$rKI$0$?$a$K!";d$?$A$O%m!<%I%P%i%s%5!<$N%P%C%/%"%C%W$r%;%C%H%"%C%W$9$kI,MW$,$"$j!"$*8_$$$r<~4|E*$K(B"heartbeat"$B$5$;$kI,MW$,$"$k!#(B"heartbeat"$B%Q%C%1!<%8$r4^$`(B GettingStandard $B%I%-%e%a%s%H$rFI$s$GM_$7$$!#$=$l$r;H$($P4JC1$K#2$D$N%N!<%I$N(B "heartbeat" $B%7%9%F%`$r%;%C%H%"%C%W$9$k$3$H$,$G$-$k!#(B

$BNc$($P!"#2$D$N%m!<%I%P%i%s%5!<$,0J2<$N$h$&$J(BIP $B%"%I%l%9$r;}$C$F$$$?$H$9$k!#(B
lvs1.domain.com (primary) 10.0.0.1
lvs2.domain.com (backup) 10.0.0.2
www.domain.com (VIP) 10.0.0.3

lvs1.domain.com $B$H(B lvs2.domain.com $B$NN>J}$K(B "heartbeat" $B$r%$%s%9%H!<%k$7$?8e!" 

#
#       keepalive: heartbeats $B$N4V3V$O2?IC$K$9$k$+(B
#
keepalive 2
#
#       deadtime: $BMn$A$?%[%9%H$r@k8@$9$k4V3V$O2?IC$K$9$k$+(B
#
deadtime 10
#       hopfudge: $B@_Dj%U%!%$%kCf$G:GBg%[%C%W?t$+$i%N!<%I$N?t$r0z$/(B
hopfudge 1
#
#       udpport: UDP $B%]!<%H$O2?HV!)(B
#
udpport 1001
#       udp: heartbeat $B$K;H$&$N$O$I$N%$%s%?!<%U%'%$%9!)(B
udp     eth0
#
#       logfacility: syslog $B$N$?$a$K;H$&5!G=$O!)(B
#
logfacility     local0
#
#       $B%/%i%9%?!<$K$"$k%^%7%s$O!)(B
#	"uname -n" $B$G8+$D$+$i$J$$$HBLL\$G$9(B
#       $B=q<0(B : node    nodename ...
    
node    lvs1.domain.com
node    lvs2.domain.com

/etc/ha.d/haresources $B$K$"$k%U%!%$%k$O 

lvs1.domain.com 10.0.0.3 lvs mon

/etc/rc.d/init.d/lvs $B$K$"$k%U%!%$%k$O 

#!/bin/sh

#
# $B$"$J$?$O%m!<%+%k%U%!%$%k%7%9%F%`$K$@$14^$^$l$k%Q%9$r%;%C%H$7$?$$$@$m$&(B 
#
PATH=/bin:/usr/bin:/sbin:/usr/sbin
export PATH

IPVSADM=/sbin/ipvsadm

case "$1" in
    start)
	if [ -x $IPVSADM ]
	then
            echo 1 > /proc/sys/net/ipv4/ip_forward
	    $IPVSADM -A -t 10.0.0.3:80
	    $IPVSADM -a -t 10.0.0.3:80 -r 192.168.0.1 -w 5 -g
	    $IPVSADM -a -t 10.0.0.3:80 -r 192.168.0.2 -w 5 -g
	fi
	;;
    stop)
	if [ -x $IPVSADM ]
	then
	    $IPVSADM -C
	fi
	;;
    *)
    	echo "Usage: lvs {start|stop}"
	exit 1
esac

exit 0

$B:G8e$K!"(Blvs1 $B$H(B lvs2 $B$NN>J}$N%N!<%I$G$9$Y$F$N%U%!%$%k$,:n$i$l$F$$$k$+$I$&$+3NG'$7!"$"$J$?<+?H$N@_Dj$rEA$($F$+$i!"(Bheartbeat $B%G!<%b%s$r#2$D$N%N!<%I$G%9%?!<%H$5$;$J$5$$!#(B

"fake"($BM>7W$J(B ARP $B$K$h$k(B IP $B$N%F%$%/%*!<%P!7W$J(B ARP $B$K$h$C$F(B IP$B%"%I%l%9(B 10.0.0.3 $B$N(BHA $B%j%=!<%9$r0z$-7Q$.!"(B/etc/rc.d/init.d/lvs $B$H(B /etc/rc.d/init.d/mon $B$N%9%/%j%W%H$r%9%?!<%H$5$;$k!#(Blvs1.domain.com $B$,@5>o$KLa$C$?$i!"(Blvs2.domain.com $B$O(B HA $B%j%=!<%9$r2rJ|$7!"@)8f$r(B lvs1.domain.com $B$KLa$9!#(B

2. The ldirectord $B$H(B heartbeat $B2r7hK!(B

ldirectord (Linux Director Daemon)$B$O(BJacob Rief$B$K$h$C$F=q$+$l$?%j%"%k%5!<%P!heartbeat$B$H0l=o$KF0:n$7$^$9!#(B ldirectord $B%W%m%0%i%`$O(B ipvs $B$N(B TAR-BALL $B$K$"$k(B contrib $B%G%#%l%/%H%j$N2<$d!":G?7$N(B heartbeat $B$N(B CVS $B%l%]%8%H%j$r%A%'%C%/$9$l$P8+$D$1$k$3$H$,$G$-$^$9!#(B ldirectord $B$K$D$$$F$N$9$Y$F$N>pJs$O(B 'perldoc ldirectord' $B$r8+$F$/$@$5$$!#(B $B$3$N0NBg$J%W%m%0%i%`$r=q$$$F$/$l$?(BJacob Rief$B$K46mon $B$r;H$C$?(B ldirectord $B$NMxE@$O

  • ldirectord $B$O(B LVS $B$r4F;k$9$k$?$a$KFCJL$K:n$i$l$^$7$?(B
    ldirectord $B$O(B /etc/ha.d/xxx.cf $B$N$h$&$K(B $B@_Dj%U%!%$%k$rFI$_=P$7$^$9!#(B $B$3$l$O(B IPVS $B%k!<%F%#%s%0%F!<%V%k@_Dj$K$D$$$F$N$9$Y$F$N>pJs$r4^$s$G$$$^$9!#(B ldirectord $B$,>e$,$C$?;~!"(BIPVS $B%k!<%F%#%s%0%F!<%V%k$OE,@Z$K@_Dj$5$l$^$9!#(B $BB>$N%5!<%S%9$r;_$a$J$$$G$"$k%5!<%S%9$N%Q%i%a!<%?$r=$@5$9$k$3$H$r2DG=$K$9$k$?$a$KJ#?t$N@_Dj%U%!%$%k$G0[$J$k%P!<%A%c%k%5!<%S%9@_Dj$rJ]B8$9$k$3$H$b$G$-$k!#(B
  • ldirecotored $B$O(B heartbeat $B$K$h$C$F4JC1$K%9%?!<%H!"%9%H%C%W$5$;$k$3$H$,$G$-$k!#(B
    ldirectord $B$r(B /etc/ha.d/resource.d/ $B$N2<$KCV$/$3$H$G!" node1 IPaddr::10.0.0.3 ldirectord::www ldirectord::mail

    $B$$$:$l$K$7$F$b!"(B ldirectord $B$b$B@_DjNc(B

    mon $B$H(B heartbeat $B$H(B fake $B$H(B coda $B$N2r7hK!$K$h$kF3F~Nc$H$7$F!" 

    #
# ldirectord $B$N$?$a$N(B /etc/ha.d/www.cf
#

# timeout:  $B%j%"%k%5!<%P!<$,Mn$A$?$H@k8@$5$l$k$^$G$NIC?t(B
timeout = 10

# checkinterval: $B%5!<%P!<$,%A%'%C%/$9$k$^$G$NIC?t(B
checkinterval = 10

#
# virtual = x.y.z.w:p
#     protocol = tcp|udp
#     scheduler = rr|wrr|lc|wlc
#     real = x.y.z.w:p gate|masq|ipip [weight]
#     ...
#     

virtual = 10.0.0.3:80
     protocol = tcp
     scheduler = wlc
     real = 192.168.0.1:80 gate 5
     real = 192.168.0.2:80 gate 5
     request = "/.testpage"
     receive = "test page"

    /etc/ha.d/haresources $B%U%!%$%k$O 

        lvs1.domain.com IPaddr::10.0.0.3 ldirectord::www

    WEB$B%5!<%P!<$K#1$D!"%I%-%e%a%s%H%k!<%H$K(B .testpage $B$r:n$kI,MW$,$"$k!#(B 

        echo "test page" > .testpage

    $B%W%i%$%^%j$H%P%C%/%"%C%W$G(B heartbeat $B%G!<%b%s$rF0$+$7$J$5$$!#$b$7@_Dj$,4V0c$C$F$$$?$i!"$=$l$>$l$N(B /var/log/ha-log $B$H(B /var/log/ldirector.log $B$rD4$Y$J$5$$!#(B

    $Id: HighAvailability.html,v 1.4 2000/03/21 12:55:00 wensong Exp $
    Created on: 1998/12/5