Difference between revisions of "Install Xen 4.2.1 with Remus and DRBD on Ubuntu 12.10"

From Xen
(Install DRBD: corrected drbd notes)
m (Install DRBD: html fix)
Line 313: Line 313:
Now if you kill the server in some way, the VM will come up automatically on the secondary.
Now if you kill the server in some way, the VM will come up automatically on the secondary.
[[Category:Xen]] [[Category:Users]] [[Category:HowTo]] [[Category:Ubuntu]]
[[Category:Xen]] [[Category:Users]] [[Category:HowTo]] [[Category:Ubuntu]]

Revision as of 12:44, 19 February 2013

This example uses two servers, "left" (server #1), and "right" (server #2).

Install a new Ubuntu 12.10 system

Both Servers

Choose 'Use entire disk with LVM', but only use like 15GB for the system to leave the rest for VMs

Network setup

apt-get install bridge-utils
vi /etc/network/interfaces
auto xenbr0
iface xenbr0 inet static
bridge_ports eth0

auto eth0
iface eth0 inet manual
/etc/init.d/networking restart
vi /etc/hosts
# left right

Install Xen

Install Xen Pre-requisites

Server #1

Short-cut install for dependencies

apt-get build-dep xen
apt-get install libc6-dev libglib2.0-dev libyajl-dev yajl-tools libbz2-dev bison flex zlib1g-dev git-core texinfo debhelper debconf-utils debootstrap fakeroot

If you're missing something, here's the full list

apt-get install libjpeg-turbo8-dev
apt-get install libvncserver-dev
apt-get install git-core mercurial screen tcpdump minicom ntp ntpdate tree debootstrap bcc bin86 gawk bridge-utils
apt-get install iproute libcurl3 libcurl4-openssl-dev bzip2 module-init-tools transfig tgif texinfo pciutils-dev build-essential make gcc 
apt-get install libc6-dev zlib1g-dev python python-dev python-twisted libncurses5-dev patch libvncserver-dev iasl libbz2-dev 
apt-get install e2fslibs-dev uuid-dev libtext-template-perl autoconf debhelper debconf-utils docbook-xml docbook-xsl dpatch xsltproc rcconf bison flex 
apt-get install gcc-multilib ocaml-findlib libyajl-dev yajl-tools libglib2.0-dev libsdl-ttf2.0-0 libsdl-ttf2.0-dev

Install Xen 4.2.1

Server #1

apt-get install transfig libpixman-1-dev
apt-get install git-core
git clone git://xenbits.xen.org/xen.git
wget http://bits.xensource.com/oss-xen/release/4.2.1/xen-4.2.1.tar.gz
tar zxvf xen-4.2.1.tar.gz
cd xen-4.2.1
wget http://download.locatrix.com/xen/remus-qdisc-py.patch
patch -p1 < remus-qdisc-py.patch
vi ./.config
make world
make deb

tar zcvf xen-dist.tgz ./xen-4.2.1/dist
scp ./xen-dist.tgz eross@

Server #2

tar zxvf xen-dist.tgz

Both servers

cd /home/eross/xen-4.2.1/dist


ls -al /boot/xen*
-rw-r--r-- 1 root root   802303 Jan 30 20:57 /boot/xen-4.2.1.gz
lrwxrwxrwx 1 root root       12 Jan 30 20:57 /boot/xen-4.2.gz -> xen-4.2.1.gz
lrwxrwxrwx 1 root root       12 Jan 30 20:57 /boot/xen-4.gz -> xen-4.2.1.gz
lrwxrwxrwx 1 root root       12 Jan 30 20:57 /boot/xen.gz -> xen-4.2.1.gz
-rw-r--r-- 1 root root 15388780 Jan 30 20:57 /boot/xen-syms-4.2.1

update-rc.d xencommons defaults 19 18
update-rc.d xendomains defaults 21 20
update-rc.d xen-watchdog defaults 22 23

grep Xen /boot/grub/grub.cfg
# You should see "Ubuntu GNU\/Linux, with Xen hypervisor" in there, that's where I got the below from
sed -i 's/GRUB_DEFAULT=.*\+/GRUB_DEFAULT="Ubuntu GNU\/Linux, with Xen hypervisor"/' /etc/default/grub

# xen-tools gives you some handy scripts for creating guests
# http://packages.ubuntu.com/quantal/all/xen-tools/filelist
apt-get install libtext-template-perl libconfig-inifiles-perl libfile-slurp-perl liblist-moreutils-perl
wget http://mirror.pnl.gov/ubuntu/pool/universe/x/xen-tools/xen-tools_4.3.1-1_all.deb
dpkg -i xen-tools_4.3.1-1_all.deb


Optional: Test Xen

Either server

xen-create-image --hostname=ubuntu \
  --memory=512mb \
  --vcpus=2 \
  --lvm=right \
  --dhcp \
  --pygrub \

# This may take a few minutes, it's running things and downloading in the background
# See what's going on here in another window:
tail -f /etc/xen/ubuntu.cfg

# Hint: Press Ctrl+] to exit the console view
xl create /etc/xen/ubuntu.cfg -c

# see it running
xl list

# reconnect
xl console ubuntu

Hint: Tapping escape while booting should display grub menu if needed for debugging

Install DRBD

Note: Installs /etc/xen/scripts/block-drbd for xen drbd block support as well

Server #1

apt-get install autoconf build-essential
wget http://remusha.wikidot.com/local--files/configuring-and-installing-remus/drbd-8.3.11-remus.tar.gz
cd ./drbd-8.3.11
./configure --prefix=/usr --localstatedir=/var --sysconfdir=/etc --with-km
make install
cd drbd
make clean all
make install
vi /etc/modules
# add drbd
tar zcvf drbd.tgz ./drbd-8.3.11
scp ./drbd.tgz user@

Server #2

# NOTE: Server #2 must have exactly the same kernel version for you to use the binaries from server #1
# Double-check uname -a on both servers to confirm
tar zxvf ./drbd.tgz
cd ./drbd-8.3.11
make install
vi /etc/modules
# add drbd

Both Servers

cp /home/user/drbd-8.3.11/scripts/global_common.conf.protoD /etc/drbd.d/global_common.conf
cp /home/user/drbd-8.3.11/scripts/testvms_protoD.res /etc/drbd.d/SystemHA_protoD.res
uname -a # right
lvcreate -n test -L 10G right
vi /etc/drbd.d/SystemHA_protoD.res
resource drbd-vm {
        device /dev/drbd1;
        disk /dev/right/test;
        meta-disk internal;
        on left {
        on right{

#Create the meta-data for the SystemHA-disk and then bring up the resource. Do this on both machines
drbdadm create-md drbd-vm
###answer y or yes for all questions, in the above command
drbdadm up drbd-vm

###Sanity check. You should see something like this in the output
root@null:~$ cat /proc/drbd 
version: 8.3.9 (api:88/proto:86-95)
GIT-hash: 3ba4fc581d6215744597b3d4c525db276ce000ee build by root@null, 2011-05-14 19:33:56

 1: cs:Connected ro:Secondary/Secondary ds:Inconsistent/Inconsistent D r-----

Server #1 - this will override all of the DRBD data on server #2

drbdadm -- --overwrite-data-of-peer primary drbd-vm

Watch the progress of the transfer (either server)

cat /proc/drbd

Once completed you can continue on.


  • You don't need to set the secondary to primary manually. This will be done by the /etc/xen/scripts/block-drbd script. When Remus is running you can do 'cat /proc/drbd' and you'll notice the status will be Primary/Primary
  • When configuring your VM, use drbd:drbd-vm for the disk. not phy:/… or any other format
  • It's one VM per DRBD disk like this, so you have to create a new one for each VM
  • You should do some research on DRBD and dual primary setups. They're potentially hard to manage, so forewarning you now.

Setup Remus

Both servers

vi /etc/xen/xend-config.sxp
# Ensure the following are uncommented
(xend-relocation-server yes)
(xend-relocation-port 8002)
(xend-relocation-address '')
(xend-relocation-hosts-allow '')

vi /etc/modules

Test VM with Remus

Server #1

pvcreate /dev/drbd1
vgcreate drbdtest /dev/drbd1

xen-create-image --hostname=ubuntu \
  --memory=512mb \
  --vcpus=2 \
  --lvm=drbdtest \
  --dhcp \
  --pygrub \

vi /etc/xen/rt.cfg
name = "rt"

memory = 256

disk = [ 'drbd:drbd-vm,xvda,w' ]
vif = [' ']

kernel = "/var/lib/xen/images/ubuntu-netboot/vmlinuz"
ramdisk = "/var/lib/xen/images/ubuntu-netboot/initrd.gz"
extra = "debian-installer/exit/always_halt=true -- console=hvc0"

xm create /etc/xen/rt.cfg -c
# run the install
sudo ln -s /usr/lib/xen-4.1/bin/pygrub /usr/bin/pygrub
vi  /etc/xen/rt.cfg
bootloader = "pygrub"
# comment out 'kernel' 'ramdisk' and 'extra'

# Press Ctrl+] to exit the console view
xl create /etc/xen/rt.cfg -c

# see it running
xl list

# test live migration, if you can't live migrate, then Remus isn't going to work either
xl migrate --live rt right

# try remus ("rt" is the domU, and "right" is the other server, assuming we're running from "left" here)
remus -i 40 rt right

# You should get a whole bunch of these messages:
PROF: flushed memory at 1361239602.516625
PROF: suspending at 1361239602.554033

Now if you kill the server in some way, the VM will come up automatically on the secondary.