Difference between revisions of "Install Xen 4.1.4 with Remus and DRBD on Ubuntu 12.10"

From Xen
(Created page with "This example uses two servers, "left" (server #1), and "right" (server #2). =Install a new Ubuntu 12.10 system= <pre> Choose 'Use entire disk with LVM', but only use like 15GB f…")
 
Line 2: Line 2:
   
 
=Install a new Ubuntu 12.10 system=
 
=Install a new Ubuntu 12.10 system=
  +
Both Servers
 
<pre>
 
<pre>
 
Choose 'Use entire disk with LVM', but only use like 15GB for the system to leave the rest for VMs
 
Choose 'Use entire disk with LVM', but only use like 15GB for the system to leave the rest for VMs
Line 36: Line 37:
   
 
==Install Xen Pre-requisites==
 
==Install Xen Pre-requisites==
  +
Server #1
  +
 
Short-cut install for dependencies
 
Short-cut install for dependencies
 
<pre>
 
<pre>
Line 100: Line 103:
   
 
==Optional: Test Xen==
 
==Optional: Test Xen==
  +
Either server
 
<pre>
 
<pre>
 
pvs
 
pvs
Line 141: Line 145:
 
=Install DRBD=
 
=Install DRBD=
   
Note: Installs /etc/xen/scripts/block-drbd for xen drbd block support as well
+
Note: Installs /etc/xen/scripts/block-drbd for xen drbd block support as well
   
  +
Server #1
 
<pre>
 
<pre>
 
apt-get install autoconf build-essential
 
apt-get install autoconf build-essential
</pre>
 
 
Server #1
 
<pre>
 
 
wget http://remusha.wikidot.com/local--files/configuring-and-installing-remus/drbd-8.3.11-remus.tar.gz
 
wget http://remusha.wikidot.com/local--files/configuring-and-installing-remus/drbd-8.3.11-remus.tar.gz
 
cd ./drbd-8.3.11
 
cd ./drbd-8.3.11
Line 217: Line 218:
 
</pre>
 
</pre>
   
Watch the progress of the transfer
+
Watch the progress of the transfer (either server)
 
<pre>
 
<pre>
 
cat /proc/drbd
 
cat /proc/drbd
Line 223: Line 224:
   
 
Once it's completed, make the secondary a primary also:
 
Once it's completed, make the secondary a primary also:
  +
  +
Server #2
 
<pre>
 
<pre>
 
drbdadm primary drbd-vm
 
drbdadm primary drbd-vm
Line 234: Line 237:
   
 
===Setup Remus===
 
===Setup Remus===
  +
Both servers
 
<pre>
 
<pre>
 
vi /etc/xen/xend-config.sxp
 
vi /etc/xen/xend-config.sxp
Line 257: Line 261:
   
 
==Test VM with Remus==
 
==Test VM with Remus==
  +
Server #1
 
<pre>
 
<pre>
 
mkdir -p /var/lib/xen/images/ubuntu-netboot
 
mkdir -p /var/lib/xen/images/ubuntu-netboot

Revision as of 02:11, 19 February 2013

This example uses two servers, "left" (server #1), and "right" (server #2).

Install a new Ubuntu 12.10 system

Both Servers

Choose 'Use entire disk with LVM', but only use like 15GB for the system to leave the rest for VMs

Network setup

apt-get install bridge-utils
vi /etc/network/interfaces
#
auto xenbr0
iface xenbr0 inet static
address 10.0.1.50
netmask 255.255.255.0
gateway 10.0.1.1
dns-nameservers 10.0.1.1
bridge_ports eth0

auto eth0
iface eth0 inet manual
#
/etc/init.d/networking restart
vi /etc/hosts
#
10.0.1.50 left
10.0.1.52 right
#

Install Xen

Install Xen Pre-requisites

Server #1

Short-cut install for dependencies

apt-get build-dep xen
apt-get install libc6-dev libglib2.0-dev libyajl-dev yajl-tools libbz2-dev bison flex zlib1g-dev git-core texinfo debhelper debconf-utils debootstrap fakeroot

If you're missing something, here's the full list

apt-get install libjpeg-turbo8-dev
apt-get install libvncserver-dev
apt-get install git-core mercurial screen tcpdump minicom ntp ntpdate tree debootstrap bcc bin86 gawk bridge-utils
apt-get install iproute libcurl3 libcurl4-openssl-dev bzip2 module-init-tools transfig tgif texinfo pciutils-dev build-essential make gcc 
apt-get install libc6-dev zlib1g-dev python python-dev python-twisted libncurses5-dev patch libvncserver-dev iasl libbz2-dev 
apt-get install e2fslibs-dev uuid-dev libtext-template-perl autoconf debhelper debconf-utils docbook-xml docbook-xsl dpatch xsltproc rcconf bison flex 
apt-get install gcc-multilib ocaml-findlib libyajl-dev yajl-tools libglib2.0-dev libsdl-ttf2.0-0 libsdl-ttf2.0-dev

Install Xen 4.1.4

Server #1

apt-get install transfig
http://bits.xensource.com/oss-xen/release/4.1.4/xen-4.1.4.tar.gz
tar zxvf xen-4.1.4.tar.gz
cd xen-4.1.4
make world
mv ./dist/install/usr/lib64/* ./dist/install/usr/lib/
mv ./dist/install/usr/lib64/xen/bin/* ./dist/install/usr/lib/xen/bin/

tar zcvf xen-dist.tgz ./xen-4.1.4/dist
scp ./xen-dist.tgz user@10.0.1.52:.

Server #2

tar zxvf xen-dist.tgz

Both servers

cd /home/user/xen-4.1.4/dist
./install.sh

update-grub

ls -al /boot/xen*
-rw-r--r-- 1 root root   737577 Feb 18 10:53 /boot/xen-4.1.4.gz
lrwxrwxrwx 1 root root       12 Feb 18 10:53 /boot/xen-4.1.gz -> xen-4.1.4.gz
lrwxrwxrwx 1 root root       12 Feb 18 10:53 /boot/xen-4.gz -> xen-4.1.4.gz
lrwxrwxrwx 1 root root       12 Feb 18 10:53 /boot/xen.gz -> xen-4.1.4.gz
-rw-r--r-- 1 root root 14003510 Feb 18 10:53 /boot/xen-syms-4.1.4

update-rc.d xencommons defaults 19 18
update-rc.d xend defaults 20 21
update-rc.d xendomains defaults 21 20
update-rc.d xen-watchdog defaults 22 23

grep Xen /boot/grub/grub.cfg
# You should see "Ubuntu GNU\/Linux, with Xen hypervisor" in there, that's where I got the below from
sed -i 's/GRUB_DEFAULT=.*\+/GRUB_DEFAULT="Ubuntu GNU\/Linux, with Xen hypervisor"/' /etc/default/grub
update-grub
reboot

Optional: Test Xen

Either server

pvs
lvcreate -L 8G -n ubuntu right
mkdir -p /var/lib/xen/images/ubuntu-netboot
cd /var/lib/xen/images/ubuntu-netboot
wget http://ubuntu.c3sl.ufpr.br/ubuntu/dists/precise/main/installer-amd64/current/images/netboot/xen/initrd.gz
wget http://ubuntu.c3sl.ufpr.br/ubuntu/dists/precise/main/installer-amd64/current/images/netboot/xen/vmlinuz

vi /etc/xen/ubuntu.cfg
#
name = "ubuntu"

memory = 256

disk = ['phy:/dev/left/ubuntu,xvda,w']
vif = [' ']

kernel = "/var/lib/xen/images/ubuntu-netboot/vmlinuz"
ramdisk = "/var/lib/xen/images/ubuntu-netboot/initrd.gz"
extra = "debian-installer/exit/always_halt=true -- console=hvc0"
#

xm create /etc/xen/ubuntu.cfg -c
# run the install
vi  /etc/xen/ubuntu.cfg
# 
bootloader = "pygrub"
# comment out 'kernel' 'ramdisk' and 'extra'
#

# Press Ctrl+] to exit the console view
xm create /etc/xen/ubuntu.cfg -c

# see it running
xm list

Hint: Tapping escape while booting should display grub menu if needed for debugging

Install DRBD

Note: Installs /etc/xen/scripts/block-drbd for xen drbd block support as well

Server #1

apt-get install autoconf build-essential
wget http://remusha.wikidot.com/local--files/configuring-and-installing-remus/drbd-8.3.11-remus.tar.gz
cd ./drbd-8.3.11
./autogen.sh
./configure --prefix=/usr --localstatedir=/var --sysconfdir=/etc --with-km
make
make install
cd drbd
make clean all
make install
vi /etc/modules
# add drbd
#
tar zcvf drbd.tgz ./drbd-8.3.11
scp ./drbd.tgz user@10.0.1.52:.

Server #2

# on 1.50
tar zxvf ./drbd.tgz
cd ./drbd-8.3.11
make install
cd drbd
make install
vi /etc/modules
# add drbd
#

Both Servers

cp /home/user/drbd-8.3.11/scripts/global_common.conf.protoD /etc/drbd.d/global_common.conf
cp /home/user/drbd-8.3.11/scripts/testvms_protoD.res /etc/drbd.d/SystemHA_protoD.res
uname -a # center
lvcreate -n test -L 10G center
vi /etc/drbd.d/SystemHA_protoD.res
#
resource drbd-vm {
        device /dev/drbd1;
        disk /dev/center/test;
        meta-disk internal;
        on left {
                address 10.0.1.50:7791;
        }
        on center {
                address 10.0.1.51:7791;
        }
}
#

#Create the meta-data for the SystemHA-disk and then bring up the resource. Do this on both machines
drbdadm create-md drbd-vm
###answer y or yes for all questions, in the above command
drbdadm up drbd-vm

###Sanity check. You should see something like this in the output
root@null:~$ cat /proc/drbd 
version: 8.3.9 (api:88/proto:86-95)
GIT-hash: 3ba4fc581d6215744597b3d4c525db276ce000ee build by root@null, 2011-05-14 19:33:56

 1: cs:Connected ro:Secondary/Secondary ds:Inconsistent/Inconsistent D r-----

Server #1 - this will override all of the DRBD data on server #2

drbdadm -- --overwrite-data-of-peer primary drbd-vm

Watch the progress of the transfer (either server)

cat /proc/drbd

Once it's completed, make the secondary a primary also:

Server #2

drbdadm primary drbd-vm
Note:When configuring your VM, use drbd:drbd-vm for the disk. not phy:/… or any other format
Note:It's one VM per DRBD disk like this, so you have to create a new one for each VM
Note:You should do some research on DRBD and dual primary setups. They're potentially hard to manage, so forewarning you now.

Setup Remus

Both servers

vi /etc/xen/xend-config.sxp
# Ensure the following are uncommented
(xend-relocation-server yes)
(xend-relocation-port 8002)
(xend-relocation-address '')
(xend-relocation-hosts-allow '')
#

vi /etc/modules
#
sch_plug
sch_prio
sch_ingress
cls_basic
cls_tcindex
cls_u32
act_mirred
ifb
#

Test VM with Remus

Server #1

mkdir -p /var/lib/xen/images/ubuntu-netboot
cd /var/lib/xen/images/ubuntu-netboot
wget http://ubuntu.c3sl.ufpr.br/ubuntu/dists/precise/main/installer-amd64/current/images/netboot/xen/initrd.gz
wget http://ubuntu.c3sl.ufpr.br/ubuntu/dists/precise/main/installer-amd64/current/images/netboot/xen/vmlinuz

vi /etc/xen/rt.cfg
#
name = "rt"

memory = 256

disk = [ 'drbd:drbd-vm,xvda,w' ]
vif = [' ']

kernel = "/var/lib/xen/images/ubuntu-netboot/vmlinuz"
ramdisk = "/var/lib/xen/images/ubuntu-netboot/initrd.gz"
extra = "debian-installer/exit/always_halt=true -- console=hvc0"
#

xm create /etc/xen/rt.cfg -c
# run the install
sudo ln -s /usr/lib/xen-4.1/bin/pygrub /usr/bin/pygrub
vi  /etc/xen/rt.cfg
# 
bootloader = "pygrub"
# comment out 'kernel' 'ramdisk' and 'extra'
#

# Press Ctrl+] to exit the console view
xm create /etc/xen/rt.cfg -c

# see it running
xm list

# test live migration, if you can't live migrate, then Remus isn't going to work either
xm migrate rt right

# try remus ("rt" is the domU, and "right" is the other server, assuming we're running from "left" here)
remus -i 40 rt right

# You should get a whole bunch of these messages:
#
PROF: flushed memory at 1361239602.516625
PROF: suspending at 1361239602.554033
#

Now if you kill the server in some way, the VM will come up automatically on the secondary.