Merge pull request #143 from cloudbuilders/fix_races
Fix NBD race conditions
This commit is contained in:
commit
037d3bdea4
5 changed files with 227 additions and 69 deletions
|
@ -12,6 +12,27 @@ if [ ! "oneiric" = "$UBUNTU_VERSION" ]; then
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Clean up any resources that may be in use
|
||||||
|
cleanup() {
|
||||||
|
set +o errexit
|
||||||
|
unmount_images
|
||||||
|
|
||||||
|
if [ -n "$ROOTFS" ]; then
|
||||||
|
umount $ROOTFS/dev
|
||||||
|
umount $ROOTFS
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Release NBD devices
|
||||||
|
if [ -n "$NBD" ]; then
|
||||||
|
qemu-nbd -d $NBD
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Kill ourselves to signal any calling process
|
||||||
|
trap 2; kill -2 $$
|
||||||
|
}
|
||||||
|
|
||||||
|
trap cleanup SIGHUP SIGINT SIGTERM
|
||||||
|
|
||||||
# Echo commands
|
# Echo commands
|
||||||
set -o xtrace
|
set -o xtrace
|
||||||
|
|
||||||
|
@ -100,9 +121,6 @@ function kill_unmount() {
|
||||||
# Install deps if needed
|
# Install deps if needed
|
||||||
dpkg -l kvm libvirt-bin kpartx || apt-get install -y --force-yes kvm libvirt-bin kpartx
|
dpkg -l kvm libvirt-bin kpartx || apt-get install -y --force-yes kvm libvirt-bin kpartx
|
||||||
|
|
||||||
# Let Ctrl-c kill tail and exit
|
|
||||||
trap kill_unmount SIGINT
|
|
||||||
|
|
||||||
# Where Openstack code will live in image
|
# Where Openstack code will live in image
|
||||||
DEST=${DEST:-/opt/stack}
|
DEST=${DEST:-/opt/stack}
|
||||||
|
|
||||||
|
@ -239,26 +257,35 @@ rm -f $VM_DIR/disk
|
||||||
# Create our instance fs
|
# Create our instance fs
|
||||||
qemu-img create -f qcow2 -b $VM_IMAGE disk
|
qemu-img create -f qcow2 -b $VM_IMAGE disk
|
||||||
|
|
||||||
|
# Finds the next available NBD device
|
||||||
|
# Exits script if error connecting or none free
|
||||||
|
# map_nbd image
|
||||||
|
# returns full nbd device path
|
||||||
|
function map_nbd {
|
||||||
|
for i in `seq 0 15`; do
|
||||||
|
if [ ! -e /sys/block/nbd$i/pid ]; then
|
||||||
|
NBD=/dev/nbd$i
|
||||||
|
# Connect to nbd and wait till it is ready
|
||||||
|
qemu-nbd -c $NBD $1
|
||||||
|
if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then
|
||||||
|
echo "Couldn't connect $NBD"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ -z "$NBD" ]; then
|
||||||
|
echo "No free NBD slots"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo $NBD
|
||||||
|
}
|
||||||
|
|
||||||
# Make sure we have nbd-ness
|
# Make sure we have nbd-ness
|
||||||
modprobe nbd max_part=63
|
modprobe nbd max_part=63
|
||||||
|
|
||||||
# Set up nbd
|
# Set up nbd
|
||||||
for i in `seq 0 15`; do
|
NBD=`map_nbd disk`
|
||||||
if [ ! -e /sys/block/nbd$i/pid ]; then
|
|
||||||
NBD=/dev/nbd$i
|
|
||||||
# Connect to nbd and wait till it is ready
|
|
||||||
qemu-nbd -c $NBD disk
|
|
||||||
if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then
|
|
||||||
echo "Couldn't connect $NBD"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
if [ -z "$NBD" ]; then
|
|
||||||
echo "No free NBD slots"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
NBD_DEV=`basename $NBD`
|
NBD_DEV=`basename $NBD`
|
||||||
|
|
||||||
# Mount the instance
|
# Mount the instance
|
||||||
|
@ -381,7 +408,9 @@ sed -e 's/^PasswordAuthentication.*$/PasswordAuthentication yes/' -i $ROOTFS/etc
|
||||||
|
|
||||||
# Unmount
|
# Unmount
|
||||||
umount $ROOTFS || echo 'ok'
|
umount $ROOTFS || echo 'ok'
|
||||||
|
ROOTFS=""
|
||||||
qemu-nbd -d $NBD
|
qemu-nbd -d $NBD
|
||||||
|
NBD=""
|
||||||
|
|
||||||
# Create the instance
|
# Create the instance
|
||||||
cd $VM_DIR && virsh create libvirt.xml
|
cd $VM_DIR && virsh create libvirt.xml
|
||||||
|
|
|
@ -11,6 +11,22 @@ PXEDIR=${PXEDIR:-/var/cache/devstack/pxe}
|
||||||
OPWD=`pwd`
|
OPWD=`pwd`
|
||||||
PROGDIR=`dirname $0`
|
PROGDIR=`dirname $0`
|
||||||
|
|
||||||
|
# Clean up any resources that may be in use
|
||||||
|
cleanup() {
|
||||||
|
set +o errexit
|
||||||
|
|
||||||
|
# Mop up temporary files
|
||||||
|
if [ -n "$MNTDIR" -a -d "$MNTDIR" ]; then
|
||||||
|
umount $MNTDIR
|
||||||
|
rmdir $MNTDIR
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Kill ourselves to signal any calling process
|
||||||
|
trap 2; kill -2 $$
|
||||||
|
}
|
||||||
|
|
||||||
|
trap cleanup SIGHUP SIGINT SIGTERM
|
||||||
|
|
||||||
mkdir -p $DEST_DIR/pxelinux.cfg
|
mkdir -p $DEST_DIR/pxelinux.cfg
|
||||||
cd $DEST_DIR
|
cd $DEST_DIR
|
||||||
for i in memdisk menu.c32 pxelinux.0; do
|
for i in memdisk menu.c32 pxelinux.0; do
|
||||||
|
|
|
@ -1,55 +1,114 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# build_ramdisk.sh - Build RAM disk images
|
# build_ramdisk.sh - Build RAM disk images
|
||||||
|
|
||||||
|
# exit on error to stop unexpected errors
|
||||||
|
set -o errexit
|
||||||
|
|
||||||
if [ ! "$#" -eq "1" ]; then
|
if [ ! "$#" -eq "1" ]; then
|
||||||
echo "$0 builds a gziped natty openstack install"
|
echo "$0 builds a gziped Ubuntu OpenStack install"
|
||||||
echo "usage: $0 dest"
|
echo "usage: $0 dest"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Clean up any resources that may be in use
|
||||||
|
cleanup() {
|
||||||
|
set +o errexit
|
||||||
|
|
||||||
|
# Mop up temporary files
|
||||||
|
if [ -n "$MNTDIR" -a -d "$MNTDIR" ]; then
|
||||||
|
umount $MNTDIR
|
||||||
|
rmdir $MNTDIR
|
||||||
|
fi
|
||||||
|
if [ -n "$DEV_FILE_TMP" -a -e "$DEV_FILE_TMP "]; then
|
||||||
|
rm -f $DEV_FILE_TMP
|
||||||
|
fi
|
||||||
|
if [ -n "$IMG_FILE_TMP" -a -e "$IMG_FILE_TMP" ]; then
|
||||||
|
rm -f $IMG_FILE_TMP
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Release NBD devices
|
||||||
|
if [ -n "$NBD" ]; then
|
||||||
|
qemu-nbd -d $NBD
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Kill ourselves to signal any calling process
|
||||||
|
trap 2; kill -2 $$
|
||||||
|
}
|
||||||
|
|
||||||
|
trap cleanup SIGHUP SIGINT SIGTERM
|
||||||
|
|
||||||
|
# Set up nbd
|
||||||
|
modprobe nbd max_part=63
|
||||||
|
|
||||||
|
# Echo commands
|
||||||
|
set -o xtrace
|
||||||
|
|
||||||
IMG_FILE=$1
|
IMG_FILE=$1
|
||||||
|
|
||||||
PROGDIR=`dirname $0`
|
# Keep track of the current directory
|
||||||
CHROOTCACHE=${CHROOTCACHE:-/var/cache/devstack}
|
TOOLS_DIR=$(cd $(dirname "$0") && pwd)
|
||||||
|
TOP_DIR=`cd $TOOLS_DIR/..; pwd`
|
||||||
# Source params
|
|
||||||
source ./stackrc
|
|
||||||
|
|
||||||
# Store cwd
|
# Store cwd
|
||||||
CWD=`pwd`
|
CWD=`pwd`
|
||||||
|
|
||||||
|
cd $TOP_DIR
|
||||||
|
|
||||||
|
# Source params
|
||||||
|
source ./stackrc
|
||||||
|
|
||||||
|
CACHEDIR=${CACHEDIR:-/var/cache/devstack}
|
||||||
|
|
||||||
DEST=${DEST:-/opt/stack}
|
DEST=${DEST:-/opt/stack}
|
||||||
|
|
||||||
|
# Configure the root password of the vm to be the same as ``ADMIN_PASSWORD``
|
||||||
|
ROOT_PASSWORD=${ADMIN_PASSWORD:-password}
|
||||||
|
|
||||||
|
# Base image (natty by default)
|
||||||
|
DIST_NAME=${DIST_NAME:-natty}
|
||||||
|
|
||||||
# Param string to pass to stack.sh. Like "EC2_DMZ_HOST=192.168.1.1 MYSQL_USER=nova"
|
# Param string to pass to stack.sh. Like "EC2_DMZ_HOST=192.168.1.1 MYSQL_USER=nova"
|
||||||
STACKSH_PARAMS=${STACKSH_PARAMS:-}
|
STACKSH_PARAMS=${STACKSH_PARAMS:-}
|
||||||
|
|
||||||
# Option to use the version of devstack on which we are currently working
|
# Option to use the version of devstack on which we are currently working
|
||||||
USE_CURRENT_DEVSTACK=${USE_CURRENT_DEVSTACK:-1}
|
USE_CURRENT_DEVSTACK=${USE_CURRENT_DEVSTACK:-1}
|
||||||
|
|
||||||
# Set up nbd
|
# clean install
|
||||||
modprobe nbd max_part=63
|
if [ ! -r $CACHEDIR/$DIST_NAME-base.img ]; then
|
||||||
NBD=${NBD:-/dev/nbd9}
|
$TOOLS_DIR/get_uec_image.sh $DIST_NAME $CACHEDIR/$DIST_NAME-base.img
|
||||||
NBD_DEV=`basename $NBD`
|
|
||||||
|
|
||||||
# clean install of natty
|
|
||||||
if [ ! -r $CHROOTCACHE/natty-base.img ]; then
|
|
||||||
$PROGDIR/get_uec_image.sh natty $CHROOTCACHE/natty-base.img
|
|
||||||
# # copy kernel modules...
|
|
||||||
# # NOTE(ja): is there a better way to do this?
|
|
||||||
# cp -pr /lib/modules/`uname -r` $CHROOTCACHE/natty-base/lib/modules
|
|
||||||
# # a simple password - pass
|
|
||||||
# echo root:pass | chroot $CHROOTCACHE/natty-base chpasswd
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# prime natty with as many apt/pips as we can
|
# Finds the next available NBD device
|
||||||
if [ ! -r $CHROOTCACHE/natty-dev.img ]; then
|
# Exits script if error connecting or none free
|
||||||
cp -p $CHROOTCACHE/natty-base.img $CHROOTCACHE/natty-dev.img
|
# map_nbd image
|
||||||
|
# returns full nbd device path
|
||||||
qemu-nbd -c $NBD $CHROOTCACHE/natty-dev.img
|
function map_nbd {
|
||||||
if ! timeout 60 sh -c "while ! [ -e /sys/block/$NBD_DEV/pid ]; do sleep 1; done"; then
|
for i in `seq 0 15`; do
|
||||||
echo "Couldn't connect $NBD"
|
if [ ! -e /sys/block/nbd$i/pid ]; then
|
||||||
|
NBD=/dev/nbd$i
|
||||||
|
# Connect to nbd and wait till it is ready
|
||||||
|
qemu-nbd -c $NBD $1
|
||||||
|
if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then
|
||||||
|
echo "Couldn't connect $NBD"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ -z "$NBD" ]; then
|
||||||
|
echo "No free NBD slots"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
echo $NBD
|
||||||
|
}
|
||||||
|
|
||||||
|
# prime image with as many apt/pips as we can
|
||||||
|
DEV_FILE=$CACHEDIR/$DIST_NAME-dev.img
|
||||||
|
DEV_FILE_TMP=`mktemp $DEV_FILE.XXXXXX`
|
||||||
|
if [ ! -r $DEV_FILE ]; then
|
||||||
|
cp -p $CACHEDIR/$DIST_NAME-base.img $DEV_FILE_TMP
|
||||||
|
|
||||||
|
NBD=`map_nbd $DEV_FILE_TMP`
|
||||||
MNTDIR=`mktemp -d --tmpdir mntXXXXXXXX`
|
MNTDIR=`mktemp -d --tmpdir mntXXXXXXXX`
|
||||||
mount -t ext4 ${NBD}p1 $MNTDIR
|
mount -t ext4 ${NBD}p1 $MNTDIR
|
||||||
cp -p /etc/resolv.conf $MNTDIR/etc/resolv.conf
|
cp -p /etc/resolv.conf $MNTDIR/etc/resolv.conf
|
||||||
|
@ -66,6 +125,7 @@ if [ ! -r $CHROOTCACHE/natty-dev.img ]; then
|
||||||
|
|
||||||
# a simple password - pass
|
# a simple password - pass
|
||||||
echo stack:pass | chroot $MNTDIR chpasswd
|
echo stack:pass | chroot $MNTDIR chpasswd
|
||||||
|
echo root:$ROOT_PASSWORD | chroot $MNTDIR chpasswd
|
||||||
|
|
||||||
# and has sudo ability (in the future this should be limited to only what
|
# and has sudo ability (in the future this should be limited to only what
|
||||||
# stack requires)
|
# stack requires)
|
||||||
|
@ -74,27 +134,31 @@ if [ ! -r $CHROOTCACHE/natty-dev.img ]; then
|
||||||
umount $MNTDIR
|
umount $MNTDIR
|
||||||
rmdir $MNTDIR
|
rmdir $MNTDIR
|
||||||
qemu-nbd -d $NBD
|
qemu-nbd -d $NBD
|
||||||
|
NBD=""
|
||||||
|
mv $DEV_FILE_TMP $DEV_FILE
|
||||||
fi
|
fi
|
||||||
|
rm -f $DEV_FILE_TMP
|
||||||
|
|
||||||
# clone git repositories onto the system
|
# clone git repositories onto the system
|
||||||
# ======================================
|
# ======================================
|
||||||
|
|
||||||
|
IMG_FILE_TMP=`mktemp $IMG_FILE.XXXXXX`
|
||||||
|
|
||||||
if [ ! -r $IMG_FILE ]; then
|
if [ ! -r $IMG_FILE ]; then
|
||||||
qemu-nbd -c $NBD $CHROOTCACHE/natty-dev.img
|
NBD=`map_nbd $DEV_FILE`
|
||||||
if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then
|
|
||||||
echo "Couldn't connect $NBD"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Pre-create the image file
|
# Pre-create the image file
|
||||||
# FIXME(dt): This should really get the partition size to
|
# FIXME(dt): This should really get the partition size to
|
||||||
# pre-create the image file
|
# pre-create the image file
|
||||||
dd if=/dev/zero of=$IMG_FILE bs=1 count=1 seek=$((2*1024*1024*1024))
|
dd if=/dev/zero of=$IMG_FILE_TMP bs=1 count=1 seek=$((2*1024*1024*1024))
|
||||||
# Create filesystem image for RAM disk
|
# Create filesystem image for RAM disk
|
||||||
dd if=${NBD}p1 of=$IMG_FILE bs=1M
|
dd if=${NBD}p1 of=$IMG_FILE_TMP bs=1M
|
||||||
|
|
||||||
qemu-nbd -d $NBD
|
qemu-nbd -d $NBD
|
||||||
|
NBD=""
|
||||||
|
mv $IMG_FILE_TMP $IMG_FILE
|
||||||
fi
|
fi
|
||||||
|
rm -f $IMG_FILE_TMP
|
||||||
|
|
||||||
MNTDIR=`mktemp -d --tmpdir mntXXXXXXXX`
|
MNTDIR=`mktemp -d --tmpdir mntXXXXXXXX`
|
||||||
mount -t ext4 -o loop $IMG_FILE $MNTDIR
|
mount -t ext4 -o loop $IMG_FILE $MNTDIR
|
||||||
|
|
|
@ -11,6 +11,26 @@ PXEDIR=${PXEDIR:-/var/cache/devstack/pxe}
|
||||||
OPWD=`pwd`
|
OPWD=`pwd`
|
||||||
PROGDIR=`dirname $0`
|
PROGDIR=`dirname $0`
|
||||||
|
|
||||||
|
# Clean up any resources that may be in use
|
||||||
|
cleanup() {
|
||||||
|
set +o errexit
|
||||||
|
|
||||||
|
# Mop up temporary files
|
||||||
|
if [ -n "$DEST_DEV" ]; then
|
||||||
|
umount $DEST_DIR
|
||||||
|
rmdir $DEST_DIR
|
||||||
|
fi
|
||||||
|
if [ -n "$MNTDIR" -a -d "$MNTDIR" ]; then
|
||||||
|
umount $MNTDIR
|
||||||
|
rmdir $MNTDIR
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Kill ourselves to signal any calling process
|
||||||
|
trap 2; kill -2 $$
|
||||||
|
}
|
||||||
|
|
||||||
|
trap cleanup SIGHUP SIGINT SIGTERM
|
||||||
|
|
||||||
if [ -b $DEST_DIR ]; then
|
if [ -b $DEST_DIR ]; then
|
||||||
# We have a block device, install syslinux and mount it
|
# We have a block device, install syslinux and mount it
|
||||||
DEST_DEV=$DEST_DIR
|
DEST_DEV=$DEST_DIR
|
||||||
|
|
|
@ -26,6 +26,24 @@ usage() {
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Clean up any resources that may be in use
|
||||||
|
cleanup() {
|
||||||
|
set +o errexit
|
||||||
|
|
||||||
|
# Mop up temporary files
|
||||||
|
if [ -n "$IMG_FILE_TMP" -a -e "$IMG_FILE_TMP" ]; then
|
||||||
|
rm -f $IMG_FILE_TMP
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Release NBD devices
|
||||||
|
if [ -n "$NBD" ]; then
|
||||||
|
qemu-nbd -d $NBD
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Kill ourselves to signal any calling process
|
||||||
|
trap 2; kill -2 $$
|
||||||
|
}
|
||||||
|
|
||||||
while getopts f:hmr: c; do
|
while getopts f:hmr: c; do
|
||||||
case $c in
|
case $c in
|
||||||
f) FORMAT=$OPTARG
|
f) FORMAT=$OPTARG
|
||||||
|
@ -89,6 +107,8 @@ case $DIST_NAME in
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
trap cleanup SIGHUP SIGINT SIGTERM
|
||||||
|
|
||||||
# Prepare the base image
|
# Prepare the base image
|
||||||
|
|
||||||
# Get the UEC image
|
# Get the UEC image
|
||||||
|
@ -111,25 +131,33 @@ if [ $ROOTSIZE -gt 2000 ]; then
|
||||||
qemu-img resize $IMG_FILE_TMP +$((ROOTSIZE - 2000))M
|
qemu-img resize $IMG_FILE_TMP +$((ROOTSIZE - 2000))M
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Finds the next available NBD device
|
||||||
|
# Exits script if error connecting or none free
|
||||||
|
# map_nbd image
|
||||||
|
# returns full nbd device path
|
||||||
|
function map_nbd {
|
||||||
|
for i in `seq 0 15`; do
|
||||||
|
if [ ! -e /sys/block/nbd$i/pid ]; then
|
||||||
|
NBD=/dev/nbd$i
|
||||||
|
# Connect to nbd and wait till it is ready
|
||||||
|
qemu-nbd -c $NBD $1
|
||||||
|
if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then
|
||||||
|
echo "Couldn't connect $NBD"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ -z "$NBD" ]; then
|
||||||
|
echo "No free NBD slots"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo $NBD
|
||||||
|
}
|
||||||
|
|
||||||
# Set up nbd
|
# Set up nbd
|
||||||
modprobe nbd max_part=63
|
modprobe nbd max_part=63
|
||||||
for i in `seq 1 15`; do
|
NBD=`map_nbd $IMG_FILE_TMP`
|
||||||
if [ ! -e /sys/block/nbd$i/pid ]; then
|
|
||||||
NBD=/dev/nbd$i
|
|
||||||
# Connect to nbd and wait till it is ready
|
|
||||||
qemu-nbd -c $NBD $IMG_FILE_TMP
|
|
||||||
if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then
|
|
||||||
echo "Couldn't connect $NBD"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
if [ -z "$NBD" ]; then
|
|
||||||
echo "No free NBD slots"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
NBD_DEV=`basename $NBD`
|
|
||||||
|
|
||||||
# Resize partition 1 to full size of the disk image
|
# Resize partition 1 to full size of the disk image
|
||||||
echo "d
|
echo "d
|
||||||
|
@ -162,5 +190,6 @@ rm -f $MNTDIR/etc/resolv.conf
|
||||||
umount $MNTDIR
|
umount $MNTDIR
|
||||||
rmdir $MNTDIR
|
rmdir $MNTDIR
|
||||||
qemu-nbd -d $NBD
|
qemu-nbd -d $NBD
|
||||||
|
NBD=""
|
||||||
|
|
||||||
mv $IMG_FILE_TMP $IMG_FILE
|
mv $IMG_FILE_TMP $IMG_FILE
|
||||||
|
|
Loading…
Reference in a new issue