diff --git a/tools/build_ci_config.sh b/tools/build_ci_config.sh index 9112471..8eed8ec 100755 --- a/tools/build_ci_config.sh +++ b/tools/build_ci_config.sh @@ -6,7 +6,7 @@ function usage { echo "$0 - Build config.ini for openstack-integration-tests" echo "" - echo "Usage: $0 configfile" + echo "Usage: $0 configdir" exit 1 } @@ -14,22 +14,27 @@ if [ ! "$#" -eq "1" ]; then usage fi -CONFIG_FILE=$1 +CONFIG_DIR=$1 +CONFIG_CONF=$CONFIG_DIR/storm.conf +CONFIG_INI=$CONFIG_DIR/config.ini # Clean up any resources that may be in use cleanup() { set +o errexit # Mop up temporary files - if [ -n "$CONFIG_FILE_TMP" -a -e "$CONFIG_FILE_TMP" ]; then - rm -f $CONFIG_FILE_TMP + if [ -n "$CONFIG_CONF_TMP" -a -e "$CONFIG_CONF_TMP" ]; then + rm -f $CONFIG_CONF_TMP + fi + if [ -n "$CONFIG_INI_TMP" -a -e "$CONFIG_INI_TMP" ]; then + rm -f $CONFIG_INI_TMP fi # Kill ourselves to signal any calling process trap 2; kill -2 $$ } -trap cleanup SIGHUP SIGINT SIGTERM +trap cleanup SIGHUP SIGINT SIGTERM SIGQUIT EXIT # Keep track of the current directory TOOLS_DIR=$(cd $(dirname "$0") && pwd) @@ -50,17 +55,19 @@ DEST=${DEST:-/opt/stack} DIST_NAME=${DIST_NAME:-oneiric} -# Process network configuration vars -GUEST_NETWORK=${GUEST_NETWORK:-1} -GUEST_RECREATE_NET=${GUEST_RECREATE_NET:-yes} +if [ ! -f $DEST/.ramdisk ]; then + # Process network configuration vars + GUEST_NETWORK=${GUEST_NETWORK:-1} + GUEST_RECREATE_NET=${GUEST_RECREATE_NET:-yes} -GUEST_IP=${GUEST_IP:-192.168.$GUEST_NETWORK.50} -GUEST_CIDR=${GUEST_CIDR:-$GUEST_IP/24} -GUEST_NETMASK=${GUEST_NETMASK:-255.255.255.0} -GUEST_GATEWAY=${GUEST_GATEWAY:-192.168.$GUEST_NETWORK.1} -GUEST_MAC=${GUEST_MAC:-"02:16:3e:07:69:`printf '%02X' $GUEST_NETWORK`"} -GUEST_RAM=${GUEST_RAM:-1524288} -GUEST_CORES=${GUEST_CORES:-1} + GUEST_IP=${GUEST_IP:-192.168.$GUEST_NETWORK.50} + GUEST_CIDR=${GUEST_CIDR:-$GUEST_IP/24} + GUEST_NETMASK=${GUEST_NETMASK:-255.255.255.0} + GUEST_GATEWAY=${GUEST_GATEWAY:-192.168.$GUEST_NETWORK.1} + GUEST_MAC=${GUEST_MAC:-"02:16:3e:07:69:`printf '%02X' $GUEST_NETWORK`"} + GUEST_RAM=${GUEST_RAM:-1524288} + GUEST_CORES=${GUEST_CORES:-1} +fi # Use the GUEST_IP unless an explicit IP is set by ``HOST_IP`` HOST_IP=${HOST_IP:-$GUEST_IP} @@ -77,34 +84,75 @@ set `echo $GLANCE_HOSTPORT | tr ':' ' '` GLANCE_HOST=$1 GLANCE_PORT=$2 -CONFIG_FILE_TMP=$(mktemp $CONFIG_FILE.XXXXXX) +# Create storm.conf + +CONFIG_CONF_TMP=$(mktemp $CONFIG_CONF.XXXXXX) + cat >$CONFIG_CONF_TMP <$CONFIG_FILE_TMP <$CONFIG_INI_TMP <$CONFIG_FILE_TMP <$CONFIG_INI_TMP <>$CONFIG_FILE_TMP <>$CONFIG_INI_TMP < $NET_XML < - devstack-$GUEST_NETWORK - - - - -EOF - -if [[ "$GUEST_RECREATE_NET" == "yes" ]]; then - virsh net-destroy devstack-$GUEST_NETWORK || true - virsh net-create $VM_DIR/net.xml -fi - -# libvirt.xml configuration -LIBVIRT_XML=$VM_DIR/libvirt.xml -cat > $LIBVIRT_XML < - $GUEST_NAME - $GUEST_RAM - - hvm - - - - - - $GUEST_CORES - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -EOF - -# Mount point for instance fs -ROOTFS=$VM_DIR/root -mkdir -p $ROOTFS - -# Clean up from previous runs -umount $ROOTFS || echo 'ok' - -# Clean up old runs -cd $VM_DIR -rm -f $VM_DIR/disk - -# Create our instance fs -qemu-img create -f qcow2 -b $VM_IMAGE disk - -# Finds the next available NBD device -# Exits script if error connecting or none free -# map_nbd image -# returns full nbd device path -function map_nbd { - for i in `seq 0 15`; do - if [ ! -e /sys/block/nbd$i/pid ]; then - NBD=/dev/nbd$i - # Connect to nbd and wait till it is ready - qemu-nbd -c $NBD $1 - if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then - echo "Couldn't connect $NBD" - exit 1 - fi - break - fi - done - if [ -z "$NBD" ]; then - echo "No free NBD slots" - exit 1 - fi - echo $NBD -} - -# Make sure we have nbd-ness -modprobe nbd max_part=63 - -# Set up nbd -NBD=`map_nbd disk` -NBD_DEV=`basename $NBD` - -# Mount the instance -mount ${NBD}p1 $ROOTFS - -# Configure instance network -INTERFACES=$ROOTFS/etc/network/interfaces -cat > $INTERFACES <> $ROOTFS/etc/sudoers - -# Gracefully cp only if source file/dir exists -function cp_it { - if [ -e $1 ] || [ -d $1 ]; then - cp -pRL $1 $2 - fi -} - -# Copy over your ssh keys and env if desired -COPYENV=${COPYENV:-1} -if [ "$COPYENV" = "1" ]; then - cp_it ~/.ssh $ROOTFS/$DEST/.ssh - cp_it ~/.ssh/id_rsa.pub $ROOTFS/$DEST/.ssh/authorized_keys - cp_it ~/.gitconfig $ROOTFS/$DEST/.gitconfig - cp_it ~/.vimrc $ROOTFS/$DEST/.vimrc - cp_it ~/.bashrc $ROOTFS/$DEST/.bashrc -fi - -# pre-cache uec images -for image_url in ${IMAGE_URLS//,/ }; do - IMAGE_FNAME=`basename "$image_url"` - if [ ! -f $IMAGES_DIR/$IMAGE_FNAME ]; then - wget -c $image_url -O $IMAGES_DIR/$IMAGE_FNAME - fi - cp $IMAGES_DIR/$IMAGE_FNAME $ROOTFS/$DEST/devstack/files -done - -# Configure the runner -RUN_SH=$ROOTFS/$DEST/run.sh -cat > $RUN_SH < /$DEST/run.sh.log -echo >> /$DEST/run.sh.log -echo >> /$DEST/run.sh.log -echo "All done! Time to start clicking." >> /$DEST/run.sh.log -cat $DEST/run.sh.log -EOF -chmod 755 $RUN_SH - -# Make runner launch on boot -RC_LOCAL=$ROOTFS/etc/init.d/zlocal -cat > $RC_LOCAL < /etc/hostname -hostname $GUEST_NAME -su -c "$DEST/run.sh" stack -EOF -chmod +x $RC_LOCAL -chroot $ROOTFS sudo update-rc.d zlocal defaults 99 - -# Make our ip address hostnames look nice at the command prompt -echo "export PS1='${debian_chroot:+($debian_chroot)}\\u@\\H:\\w\\$ '" >> $ROOTFS/$DEST/.bashrc -echo "export PS1='${debian_chroot:+($debian_chroot)}\\u@\\H:\\w\\$ '" >> $ROOTFS/etc/profile - -# Give stack ownership over $DEST so it may do the work needed -chroot $ROOTFS chown -R stack $DEST - -# Set the hostname -echo $GUEST_NAME > $ROOTFS/etc/hostname - -# We need the hostname to resolve for rabbit to launch -if ! grep -q $GUEST_NAME $ROOTFS/etc/hosts; then - echo "$GUEST_IP $GUEST_NAME" >> $ROOTFS/etc/hosts -fi - -# GRUB 2 wants to see /dev -mount -o bind /dev $ROOTFS/dev - -# Change boot params so that we get a console log -G_DEV_UUID=`blkid -t LABEL=cloudimg-rootfs -s UUID -o value | head -1` -sed -e "s/GRUB_TIMEOUT=.*$/GRUB_TIMEOUT=3/" -i $ROOTFS/etc/default/grub -sed -e "s,GRUB_CMDLINE_LINUX_DEFAULT=.*$,GRUB_CMDLINE_LINUX_DEFAULT=\"console=ttyS0 console=tty0 ds=nocloud ubuntu-pass=pass\",g" -i $ROOTFS/etc/default/grub -sed -e 's/[#]*GRUB_TERMINAL=.*$/GRUB_TERMINAL="serial console"/' -i $ROOTFS/etc/default/grub -echo 'GRUB_SERIAL_COMMAND="serial --unit=0"' >>$ROOTFS/etc/default/grub -echo 'GRUB_DISABLE_OS_PROBER=true' >>$ROOTFS/etc/default/grub -echo "GRUB_DEVICE_UUID=$G_DEV_UUID" >>$ROOTFS/etc/default/grub - -chroot $ROOTFS update-grub - -# Pre-generate ssh host keys and allow password login -chroot $ROOTFS dpkg-reconfigure openssh-server -sed -e 's/^PasswordAuthentication.*$/PasswordAuthentication yes/' -i $ROOTFS/etc/ssh/sshd_config - -# Unmount -umount $ROOTFS/dev -umount $ROOTFS || echo 'ok' -ROOTFS="" -qemu-nbd -d $NBD -NBD="" - -trap - SIGHUP SIGINT SIGTERM SIGQUIT EXIT - -# Create the instance -cd $VM_DIR && virsh create libvirt.xml - -# Tail the console log till we are done -WAIT_TILL_LAUNCH=${WAIT_TILL_LAUNCH:-1} -if [ "$WAIT_TILL_LAUNCH" = "1" ]; then - # Done creating the container, let's tail the log - echo - echo "=============================================================" - echo " -- YAY! --" - echo "=============================================================" - echo - echo "We're done launching the vm, about to start tailing the" - echo "stack.sh log. It will take a second or two to start." - echo - echo "Just CTRL-C at any time to stop tailing." - - while [ ! -e "$VM_DIR/console.log" ]; do - sleep 1 - done - - tail -F $VM_DIR/console.log & - - TAIL_PID=$! - - function kill_tail() { - kill $TAIL_PID - exit 1 - } - - # Let Ctrl-c kill tail and exit - trap kill_tail SIGINT - - set +o xtrace - - echo "Waiting stack.sh to finish..." - while ! cat $VM_DIR/console.log | grep -q 'All done' ; do - sleep 1 - done - - set -o xtrace - - kill $TAIL_PID - - if ! grep -q "^stack.sh completed in" $VM_DIR/console.log; then - exit 1 - fi - echo "" - echo "Finished - Zip-a-dee Doo-dah!" -fi diff --git a/tools/build_lxc.sh b/tools/build_lxc.sh deleted file mode 100755 index c5957b2..0000000 --- a/tools/build_lxc.sh +++ /dev/null @@ -1,324 +0,0 @@ -#!/usr/bin/env bash - -# Debug stuff -set -o errexit -set -o xtrace - -# Sanity check -if [ "$EUID" -ne "0" ]; then - echo "This script must be run with root privileges." - exit 1 -fi - -# Keep track of ubuntu version -UBUNTU_VERSION=`cat /etc/lsb-release | grep CODENAME | sed 's/.*=//g'` - -# Move to top devstack dir -cd .. - -# Abort if localrc is not set -if [ ! -e ./localrc ]; then - echo "You must have a localrc with ALL necessary passwords defined before proceeding." - echo "See stack.sh for required passwords." - exit 1 -fi - -# Source params -source ./stackrc - -# Store cwd -CWD=`pwd` - -# Configurable params -BRIDGE=${BRIDGE:-br0} -GUEST_NAME=${GUEST_NAME:-STACK} -GUEST_IP=${GUEST_IP:-192.168.1.50} -GUEST_CIDR=${GUEST_CIDR:-$GUEST_IP/24} -GUEST_NETMASK=${GUEST_NETMASK:-255.255.255.0} -GUEST_GATEWAY=${GUEST_GATEWAY:-192.168.1.1} -NAMESERVER=${NAMESERVER:-`cat /etc/resolv.conf | grep nameserver | head -1 | cut -d " " -f2`} -COPYENV=${COPYENV:-1} -DEST=${DEST:-/opt/stack} -WAIT_TILL_LAUNCH=${WAIT_TILL_LAUNCH:-1} - -# Param string to pass to stack.sh. Like "EC2_DMZ_HOST=192.168.1.1 MYSQL_USER=nova" -# By default, n-vol is disabled for lxc, as iscsitarget doesn't work properly in lxc -STACKSH_PARAMS=${STACKSH_PARAMS:-"ENABLED_SERVICES=g-api,g-reg,key,n-api,n-cpu,n-net,n-sch,n-vnc,horizon,mysql,rabbit"} - -# Option to use the version of devstack on which we are currently working -USE_CURRENT_DEVSTACK=${USE_CURRENT_DEVSTACK:-1} - - -# Install deps -apt-get install -y lxc debootstrap - -# Install cgroup-bin from source, since the packaging is buggy and possibly incompatible with our setup -if ! which cgdelete | grep -q cgdelete; then - apt-get install -y g++ bison flex libpam0g-dev make - wget http://sourceforge.net/projects/libcg/files/libcgroup/v0.37.1/libcgroup-0.37.1.tar.bz2/download -O /tmp/libcgroup-0.37.1.tar.bz2 - cd /tmp && bunzip2 libcgroup-0.37.1.tar.bz2 && tar xfv libcgroup-0.37.1.tar - cd libcgroup-0.37.1 - ./configure - make install - ldconfig -fi - -# Create lxc configuration -LXC_CONF=/tmp/$GUEST_NAME.conf -cat > $LXC_CONF <> $ROOTFS/etc/sudoers - -# Copy kernel modules -mkdir -p $ROOTFS/lib/modules/`uname -r`/kernel -cp -p /lib/modules/`uname -r`/modules.dep $ROOTFS/lib/modules/`uname -r`/ -cp -pR /lib/modules/`uname -r`/kernel/net $ROOTFS/lib/modules/`uname -r`/kernel/ - -# Gracefully cp only if source file/dir exists -function cp_it { - if [ -e $1 ] || [ -d $1 ]; then - cp -pRL $1 $2 - fi -} - -# Copy over your ssh keys and env if desired -if [ "$COPYENV" = "1" ]; then - cp_it ~/.ssh $ROOTFS/$DEST/.ssh - cp_it ~/.ssh/id_rsa.pub $ROOTFS/$DEST/.ssh/authorized_keys - cp_it ~/.gitconfig $ROOTFS/$DEST/.gitconfig - cp_it ~/.vimrc $ROOTFS/$DEST/.vimrc - cp_it ~/.bashrc $ROOTFS/$DEST/.bashrc -fi - -# Make our ip address hostnames look nice at the command prompt -echo "export PS1='${debian_chroot:+($debian_chroot)}\\u@\\H:\\w\\$ '" >> $ROOTFS/$DEST/.bashrc -echo "export PS1='${debian_chroot:+($debian_chroot)}\\u@\\H:\\w\\$ '" >> $ROOTFS/etc/profile - -# Give stack ownership over $DEST so it may do the work needed -chroot $ROOTFS chown -R stack $DEST - -# Configure instance network -INTERFACES=$ROOTFS/etc/network/interfaces -cat > $INTERFACES < $RUN_SH < /$DEST/run.sh.log -echo >> /$DEST/run.sh.log -echo >> /$DEST/run.sh.log -echo "All done! Time to start clicking." >> /$DEST/run.sh.log -EOF - -# Make the run.sh executable -chmod 755 $RUN_SH - -# Make runner launch on boot -RC_LOCAL=$ROOTFS/etc/init.d/local -cat > $RC_LOCAL <> $CHROOTCACHE/natty-dev/etc/sudoers -fi - -# clone git repositories onto the system -# ====================================== - -if [ ! -d $CHROOTCACHE/natty-stack ]; then - rsync -azH $CHROOTCACHE/natty-dev/ $CHROOTCACHE/natty-stack/ -fi - -# git clone only if directory doesn't exist already. Since ``DEST`` might not -# be owned by the installation user, we create the directory and change the -# ownership to the proper user. -function git_clone { - - # clone new copy or fetch latest changes - CHECKOUT=$CHROOTCACHE/natty-stack$2 - if [ ! -d $CHECKOUT ]; then - mkdir -p $CHECKOUT - git clone $1 $CHECKOUT - else - pushd $CHECKOUT - git fetch - popd - fi - - # FIXME(ja): checkout specified version (should works for branches and tags) - - pushd $CHECKOUT - # checkout the proper branch/tag - git checkout $3 - # force our local version to be the same as the remote version - git reset --hard origin/$3 - popd - - # give ownership to the stack user - chroot $CHROOTCACHE/natty-stack/ chown -R stack $2 -} - -git_clone $NOVA_REPO $DEST/nova $NOVA_BRANCH -git_clone $GLANCE_REPO $DEST/glance $GLANCE_BRANCH -git_clone $KEYSTONE_REPO $DEST/keystone $KEYSTONE_BRANCH -git_clone $NOVNC_REPO $DEST/novnc $NOVNC_BRANCH -git_clone $HORIZON_REPO $DEST/horizon $HORIZON_BRANCH $HORIZON_TAG -git_clone $NOVACLIENT_REPO $DEST/python-novaclient $NOVACLIENT_BRANCH -git_clone $OPENSTACKX_REPO $DEST/openstackx $OPENSTACKX_BRANCH - -chroot $CHROOTCACHE/natty-stack mkdir -p $DEST/files -wget -c http://images.ansolabs.com/tty.tgz -O $CHROOTCACHE/natty-stack$DEST/files/tty.tgz - -# Use this version of devstack? -if [ "$USE_CURRENT_DEVSTACK" = "1" ]; then - rm -rf $CHROOTCACHE/natty-stack/$DEST/devstack - cp -pr $CWD $CHROOTCACHE/natty-stack/$DEST/devstack -fi - -cp -pr $CHROOTCACHE/natty-stack $NFSDIR - -# set hostname -echo $NAME > $NFSDIR/etc/hostname -echo "127.0.0.1 localhost $NAME" > $NFSDIR/etc/hosts - -# injecting root's public ssh key if it exists -if [ -f /root/.ssh/id_rsa.pub ]; then - mkdir $NFSDIR/root/.ssh - chmod 700 $NFSDIR/root/.ssh - cp /root/.ssh/id_rsa.pub $NFSDIR/root/.ssh/authorized_keys -fi diff --git a/tools/build_pxe_env.sh b/tools/build_pxe_env.sh index 1ab51f8..d01dad0 100755 --- a/tools/build_pxe_env.sh +++ b/tools/build_pxe_env.sh @@ -10,8 +10,7 @@ dpkg -l syslinux || apt-get install -y syslinux DEST_DIR=${1:-/tmp}/tftpboot -PXEDIR=${PXEDIR:-/var/cache/devstack/pxe} -OPWD=`pwd` +PXEDIR=${PXEDIR:-/opt/ramstack/pxe} PROGDIR=`dirname $0` # Clean up any resources that may be in use @@ -28,7 +27,11 @@ cleanup() { trap 2; kill -2 $$ } -trap cleanup SIGHUP SIGINT SIGTERM +trap cleanup SIGHUP SIGINT SIGTERM SIGQUIT EXIT + +# Keep track of the current directory +TOOLS_DIR=$(cd $(dirname "$0") && pwd) +TOP_DIR=`cd $TOOLS_DIR/..; pwd` mkdir -p $DEST_DIR/pxelinux.cfg cd $DEST_DIR @@ -42,7 +45,7 @@ default menu.c32 prompt 0 timeout 0 -MENU TITLE PXE Boot Menu +MENU TITLE devstack PXE Boot Menu EOF @@ -54,7 +57,7 @@ fi # Get image into place if [ ! -r $PXEDIR/stack-initrd.img ]; then - cd $OPWD + cd $TOP_DIR $PROGDIR/build_ramdisk.sh $PXEDIR/stack-initrd.img fi if [ ! -r $PXEDIR/stack-initrd.gz ]; then @@ -110,3 +113,5 @@ LABEL local MENU LABEL ^Local disk LOCALBOOT 0 EOF + +trap cleanup SIGHUP SIGINT SIGTERM SIGQUIT EXIT diff --git a/tools/build_ramdisk.sh b/tools/build_ramdisk.sh index 2c914dc..feaa8a9 100755 --- a/tools/build_ramdisk.sh +++ b/tools/build_ramdisk.sh @@ -57,7 +57,7 @@ cd $TOP_DIR # Source params source ./stackrc -CACHEDIR=${CACHEDIR:-/var/cache/devstack} +CACHEDIR=${CACHEDIR:-/opt/stack/cache} DEST=${DEST:-/opt/stack} diff --git a/tools/build_uec.sh b/tools/build_uec.sh index 2a578fc..8167105 100755 --- a/tools/build_uec.sh +++ b/tools/build_uec.sh @@ -37,27 +37,34 @@ DEPS="kvm libvirt-bin kpartx cloud-utils curl" apt-get install -y --force-yes $DEPS || true # allow this to fail gracefully for concurrent builds # Where to store files and instances -WORK_DIR=${WORK_DIR:-/opt/kvmstack} +WORK_DIR=${WORK_DIR:-/opt/uecstack} # Where to store images image_dir=$WORK_DIR/images/$DIST_NAME mkdir -p $image_dir -# Original version of built image -uec_url=http://uec-images.ubuntu.com/$DIST_NAME/current/$DIST_NAME-server-cloudimg-amd64.tar.gz -tarball=$image_dir/$(basename $uec_url) - -# download the base uec image if we haven't already -if [ ! -f $tarball ]; then - curl $uec_url -o $tarball - (cd $image_dir && tar -Sxvzf $tarball) - resize-part-image $image_dir/*.img $GUEST_SIZE $image_dir/disk - cp $image_dir/*-vmlinuz-virtual $image_dir/kernel +# Start over with a clean base image, if desired +if [ $CLEAN_BASE ]; then + rm -f $image_dir/disk fi +# Get the base image if it does not yet exist +if [ ! -e $image_dir/disk ]; then + $TOOLS_DIR/get_uec_image.sh -r $GUEST_SIZE $DIST_NAME $image_dir/disk $image_dir/kernel +fi -# Configure the root password of the vm to be the same as ``ADMIN_PASSWORD`` -ROOT_PASSWORD=${ADMIN_PASSWORD:-password} +# Copy over dev environment if COPY_ENV is set. +# This will also copy over your current devstack. +if [ $COPY_ENV ]; then + cd $TOOLS_DIR + ./copy_dev_environment_to_uec.sh $image_dir/disk +fi + +# Option to warm the base image with software requirements. +if [ $WARM_CACHE ]; then + cd $TOOLS_DIR + ./warm_apts_and_pips_for_uec.sh $image_dir/disk +fi # Name of our instance, used by libvirt GUEST_NAME=${GUEST_NAME:-devstack} @@ -178,22 +185,23 @@ cat > $vm_dir/uec/user-data< localrc < localrc <> $vm_dir/uec/user-data< $MNT_DIR/etc/network/interfaces <$MNT_DIR/etc/hostname +echo "127.0.0.1 localhost ramstack" >$MNT_DIR/etc/hosts + +# Configure the runner +RUN_SH=$MNT_DIR/$DEST/run.sh +cat > $RUN_SH < $DEST/run.sh.log +echo >> $DEST/run.sh.log +echo >> $DEST/run.sh.log +echo "All done! Time to start clicking." >> $DEST/run.sh.log +EOF + +# Make the run.sh executable +chmod 755 $RUN_SH +chroot $MNT_DIR chown stack $DEST/run.sh + +umount $MNT_DIR/dev +umount $MNT_DIR +rmdir $MNT_DIR +mv $DEST_FILE_TMP $DEST_FILE +rm -f $DEST_FILE_TMP + +trap - SIGHUP SIGINT SIGTERM SIGQUIT EXIT diff --git a/tools/build_usb_boot.sh b/tools/build_usb_boot.sh index e4dabc0..cca2a68 100755 --- a/tools/build_usb_boot.sh +++ b/tools/build_usb_boot.sh @@ -7,8 +7,7 @@ # Needs to run as root DEST_DIR=${1:-/tmp/syslinux-boot} -PXEDIR=${PXEDIR:-/var/cache/devstack/pxe} -OPWD=`pwd` +PXEDIR=${PXEDIR:-/opt/ramstack/pxe} PROGDIR=`dirname $0` # Clean up any resources that may be in use @@ -29,7 +28,11 @@ cleanup() { trap 2; kill -2 $$ } -trap cleanup SIGHUP SIGINT SIGTERM +trap cleanup SIGHUP SIGINT SIGTERM SIGQUIT EXIT + +# Keep track of the current directory +TOOLS_DIR=$(cd $(dirname "$0") && pwd) +TOP_DIR=`cd $TOOLS_DIR/..; pwd` if [ -b $DEST_DIR ]; then # We have a block device, install syslinux and mount it @@ -62,7 +65,7 @@ default /syslinux/menu.c32 prompt 0 timeout 0 -MENU TITLE Boot Menu +MENU TITLE devstack Boot Menu EOF @@ -74,8 +77,8 @@ fi # Get image into place if [ ! -r $PXEDIR/stack-initrd.img ]; then - cd $OPWD - $PROGDIR/build_ramdisk.sh $PXEDIR/stack-initrd.img + cd $TOP_DIR + $PROGDIR/build_uec_ramdisk.sh $PXEDIR/stack-initrd.img fi if [ ! -r $PXEDIR/stack-initrd.gz ]; then gzip -1 -c $PXEDIR/stack-initrd.img >$PXEDIR/stack-initrd.gz @@ -139,3 +142,5 @@ if [ -n "$DEST_DEV" ]; then umount $DEST_DIR rmdir $DEST_DIR fi + +trap - SIGHUP SIGINT SIGTERM SIGQUIT EXIT diff --git a/tools/setup_stack_user.sh b/tools/copy_dev_environment_to_uec.sh similarity index 86% rename from tools/setup_stack_user.sh rename to tools/copy_dev_environment_to_uec.sh index fcb9733..c949b32 100755 --- a/tools/setup_stack_user.sh +++ b/tools/copy_dev_environment_to_uec.sh @@ -44,10 +44,8 @@ chroot $STAGING_DIR useradd stack -s /bin/bash -d $DEST -G libvirtd || true echo stack:pass | chroot $STAGING_DIR chpasswd # Configure sudo -grep -q "^#includedir.*/etc/sudoers.d" $STAGING_DIR/etc/sudoers || - echo "#includedir /etc/sudoers.d" | sudo tee -a $STAGING_DIR/etc/sudoers -cp $TOP_DIR/files/sudo/* $STAGING_DIR/etc/sudoers.d/ -sed -e "s,%USER%,$USER,g" -i $STAGING_DIR/etc/sudoers.d/* +( umask 226 && echo "stack ALL=(ALL) NOPASSWD:ALL" \ + > $STAGING_DIR/etc/sudoers.d/50_stack_sh ) # Gracefully cp only if source file/dir exists function cp_it { @@ -63,6 +61,10 @@ cp_it ~/.gitconfig $STAGING_DIR/$DEST/.gitconfig cp_it ~/.vimrc $STAGING_DIR/$DEST/.vimrc cp_it ~/.bashrc $STAGING_DIR/$DEST/.bashrc +# Copy devstack +rm -rf $STAGING_DIR/$DEST/devstack +cp_it . $STAGING_DIR/$DEST/devstack + # Give stack ownership over $DEST so it may do the work needed chroot $STAGING_DIR chown -R stack $DEST diff --git a/tools/get_uec_image.sh b/tools/get_uec_image.sh index 7b95aab..7fa920e 100755 --- a/tools/get_uec_image.sh +++ b/tools/get_uec_image.sh @@ -1,14 +1,8 @@ #!/bin/bash -# get_uec_image.sh - Prepare Ubuntu images in various formats -# -# Supported formats: qcow (kvm), vmdk (vmserver), vdi (vbox), vhd (vpc), raw -# -# Required to run as root +# get_uec_image.sh - Prepare Ubuntu UEC images -CACHEDIR=${CACHEDIR:-/var/cache/devstack} -FORMAT=${FORMAT:-qcow2} +CACHEDIR=${CACHEDIR:-/opt/stack/cache} ROOTSIZE=${ROOTSIZE:-2000} -MIN_PKGS=${MIN_PKGS:-"apt-utils gpgv openssh-server"} # Keep track of the current directory TOOLS_DIR=$(cd $(dirname "$0") && pwd) @@ -18,14 +12,14 @@ TOP_DIR=`cd $TOOLS_DIR/..; pwd` set -o errexit usage() { - echo "Usage: $0 - Prepare Ubuntu images" + echo "Usage: $0 - Fetch and prepare Ubuntu images" echo "" - echo "$0 [-f format] [-r rootsize] release imagefile" + echo "$0 [-r rootsize] release imagefile [kernel]" echo "" - echo "-f format - image format: qcow2 (default), vmdk, vdi, vhd, xen, raw, fs" - echo "-r size - root fs size in MB (min 2000MB)" + echo "-r size - root fs size (min 2000MB)" echo "release - Ubuntu release: jaunty - oneric" echo "imagefile - output image file" + echo "kernel - output kernel" exit 1 } @@ -38,42 +32,21 @@ cleanup() { rm -f $IMG_FILE_TMP fi - # Release NBD devices - if [ -n "$NBD" ]; then - qemu-nbd -d $NBD - fi - # Kill ourselves to signal any calling process trap 2; kill -2 $$ } -# apt-get wrapper to just get arguments set correctly -function apt_get() { - local sudo="sudo" - [ "$(id -u)" = "0" ] && sudo="env" - $sudo DEBIAN_FRONTEND=noninteractive apt-get \ - --option "Dpkg::Options::=--force-confold" --assume-yes "$@" -} - -while getopts f:hmr: c; do +while getopts hr: c; do case $c in - f) FORMAT=$OPTARG - ;; h) usage ;; - m) MINIMAL=1 - ;; r) ROOTSIZE=$OPTARG - if [[ $ROOTSIZE < 2000 ]]; then - echo "root size must be greater than 2000MB" - exit 1 - fi ;; esac done shift `expr $OPTIND - 1` -if [ ! "$#" -eq "2" ]; then +if [[ ! "$#" -eq "2" && ! "$#" -eq "3" ]]; then usage fi @@ -81,134 +54,49 @@ fi DIST_NAME=$1 IMG_FILE=$2 IMG_FILE_TMP=`mktemp $IMG_FILE.XXXXXX` - -case $FORMAT in - kvm|qcow2) FORMAT=qcow2 - QFORMAT=qcow2 - ;; - vmserver|vmdk) - FORMAT=vmdk - QFORMAT=vmdk - ;; - vbox|vdi) FORMAT=vdi - QFORMAT=vdi - ;; - vhd|vpc) FORMAT=vhd - QFORMAT=vpc - ;; - xen) FORMAT=raw - QFORMAT=raw - ;; - raw) FORMAT=raw - QFORMAT=raw - ;; - *) echo "Unknown format: $FORMAT" - usage -esac +KERNEL=$3 case $DIST_NAME in oneiric) ;; natty) ;; maverick) ;; lucid) ;; - karmic) ;; - jaunty) ;; *) echo "Unknown release: $DIST_NAME" usage ;; esac -trap cleanup SIGHUP SIGINT SIGTERM SIGQUIT +trap cleanup SIGHUP SIGINT SIGTERM SIGQUIT EXIT -# Check for dependencies - -if [ ! -x "`which qemu-img`" -o ! -x "`which qemu-nbd`" ]; then +# Check dependencies +if [ ! -x "`which qemu-img`" -o -z "`dpkg -l | grep cloud-utils`" ]; then # Missing KVM? - apt_get install qemu-kvm + apt_get install qemu-kvm cloud-utils fi -# Prepare the base image +# Find resize script +RESIZE=`which resize-part-image || which uec-resize-image` +if [ -z "$RESIZE" ]; then + echo "resize tool from cloud-utils not found" + exit 1 +fi # Get the UEC image UEC_NAME=$DIST_NAME-server-cloudimg-amd64 -if [ ! -e $CACHEDIR/$UEC_NAME-disk1.img ]; then - mkdir -p $CACHEDIR - (cd $CACHEDIR && wget -N http://uec-images.ubuntu.com/$DIST_NAME/current/$UEC_NAME-disk1.img) +if [ ! -d $CACHEDIR ]; then + mkdir -p $CACHEDIR/$DIST_NAME +fi +if [ ! -e $CACHEDIR/$DIST_NAME/$UEC_NAME.tar.gz ]; then + (cd $CACHEDIR/$DIST_NAME && wget -N http://uec-images.ubuntu.com/$DIST_NAME/current/$UEC_NAME.tar.gz) + (cd $CACHEDIR/$DIST_NAME && tar Sxvzf $UEC_NAME.tar.gz) fi -if [ "$FORMAT" = "qcow2" ]; then - # Just copy image - cp -p $CACHEDIR/$UEC_NAME-disk1.img $IMG_FILE_TMP -else - # Convert image - qemu-img convert -O $QFORMAT $CACHEDIR/$UEC_NAME-disk1.img $IMG_FILE_TMP -fi - -# Resize the image if necessary -if [ $ROOTSIZE -gt 2000 ]; then - # Resize the container - qemu-img resize $IMG_FILE_TMP +$((ROOTSIZE - 2000))M -fi - -# Finds the next available NBD device -# Exits script if error connecting or none free -# map_nbd image -# returns full nbd device path -function map_nbd { - for i in `seq 0 15`; do - if [ ! -e /sys/block/nbd$i/pid ]; then - NBD=/dev/nbd$i - # Connect to nbd and wait till it is ready - qemu-nbd -c $NBD $1 - if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then - echo "Couldn't connect $NBD" - exit 1 - fi - break - fi - done - if [ -z "$NBD" ]; then - echo "No free NBD slots" - exit 1 - fi - echo $NBD -} - -# Set up nbd -modprobe nbd max_part=63 -NBD=`map_nbd $IMG_FILE_TMP` - -# Resize partition 1 to full size of the disk image -echo "d -n -p -1 -2 - -t -83 -a -1 -w -" | fdisk $NBD -e2fsck -f -p ${NBD}p1 -resize2fs ${NBD}p1 - -# Do some preliminary installs -MNTDIR=`mktemp -d mntXXXXXXXX` -mount -t ext4 ${NBD}p1 $MNTDIR - -# Install our required packages -cp -p files/sources.list $MNTDIR/etc/apt/sources.list -sed -e "s,%DIST%,$DIST_NAME,g" -i $MNTDIR/etc/apt/sources.list -cp -p /etc/resolv.conf $MNTDIR/etc/resolv.conf -chroot $MNTDIR apt-get update -chroot $MNTDIR apt-get install -y $MIN_PKGS -rm -f $MNTDIR/etc/resolv.conf - -umount $MNTDIR -rmdir $MNTDIR -qemu-nbd -d $NBD -NBD="" - +$RESIZE $CACHEDIR/$DIST_NAME/$UEC_NAME.img ${ROOTSIZE} $IMG_FILE_TMP mv $IMG_FILE_TMP $IMG_FILE + +# Copy kernel to destination +if [ -n "$KERNEL" ]; then + cp -p $CACHEDIR/$DIST_NAME/*-vmlinuz-virtual $KERNEL +fi + +trap - SIGHUP SIGINT SIGTERM SIGQUIT EXIT diff --git a/tools/install_openvpn.sh b/tools/install_openvpn.sh old mode 100644 new mode 100755 diff --git a/tools/jenkins/configurations/kvm.sh b/tools/jenkins/configurations/kvm.sh index 5a9df47..727b42a 100755 --- a/tools/jenkins/configurations/kvm.sh +++ b/tools/jenkins/configurations/kvm.sh @@ -1,5 +1,9 @@ #!/bin/bash +# exit on error to stop unexpected errors +set -o errexit +set -o xtrace + EXECUTOR_NUMBER=$1 CONFIGURATION=$2 ADAPTER=$3 @@ -25,7 +29,7 @@ cd ../../.. TOP_DIR=$(pwd) # Deps -apt-get install -y --force-yes libvirt-bin +apt-get install -y --force-yes libvirt-bin || true # Name test instance based on executor BASE_NAME=executor-`printf "%02d" $EXECUTOR_NUMBER` diff --git a/tools/lxc_network_hostonlyplusnat.sh b/tools/lxc_network_hostonlyplusnat.sh deleted file mode 100755 index 4e29ed7..0000000 --- a/tools/lxc_network_hostonlyplusnat.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash - -# Print some usage info -function usage { - echo "Usage: $0 [OPTION] [host_ip]" - echo "Set up temporary networking for LXC" - echo "" - echo " -n, --dry-run Just print the commands that would execute." - echo " -h, --help Print this usage message." - echo "" - exit -} - -# Allow passing the ip address on the command line. -function process_option { - case "$1" in - -h|--help) usage;; - -n|--dry-run) dry_run=1;; - *) host_ip="$1" - esac -} - -# Set up some defaults -host_ip= -dry_run=0 -bridge=br0 -DRIER= - -# Process the args -for arg in "$@"; do - process_option $arg -done - -if [ $dry_run ]; then - DRIER=echo -fi - -if [ "$UID" -ne "0" ]; then - echo "This script must be run with root privileges." - exit 1 -fi - -# Check for bridge-utils. -BRCTL=`which brctl` -if [ ! -x "$BRCTL" ]; then - echo "This script requires you to install bridge-utils." - echo "Try: sudo apt-get install bridge-utils." - exit 1 -fi - -# Scare off the nubs. -echo "=====================================================" -echo -echo "WARNING" -echo -echo "This script will modify your current network setup," -echo "this can be a scary thing and it is recommended that" -echo "you have something equivalent to physical access to" -echo "this machine before continuing in case your network" -echo "gets all funky." -echo -echo "If you don't want to continue, hit CTRL-C now." - -if [ -z "$host_ip" ]; -then - echo "Otherwise, please type in your host's ip address and" - echo "hit enter." - echo - echo "=====================================================" - read host_ip -else - echo "Otherwise hit enter." - echo - echo "=====================================================" - read accept -fi - - -# Add a bridge interface, this will choke if there is already -# a bridge named $bridge -$DRIER $BRCTL addbr $bridge -$DRIER ip addr add 192.168.1.1/24 dev $bridge -if [ $dry_run ]; then - echo "echo 1 > /proc/sys/net/ipv4/ip_forward" -else - echo 1 > /proc/sys/net/ipv4/ip_forward -fi -$DRIER ifconfig $bridge up - -# Set up the NAT for the instances -$DRIER iptables -t nat -A POSTROUTING -s 192.168.1.0/24 -j SNAT --to-source $host_ip -$DRIER iptables -I FORWARD -s 192.168.1.0/24 -j ACCEPT - diff --git a/tools/make_image.sh b/tools/make_image.sh deleted file mode 100755 index a69f5e3..0000000 --- a/tools/make_image.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/bin/bash -# make_image.sh - Create Ubuntu images in various formats -# -# Supported formats: qcow (kvm), vmdk (vmserver), vdi (vbox), vhd (vpc), raw -# -# Requires sudo to root - -ROOTSIZE=${ROOTSIZE:-8192} -SWAPSIZE=${SWAPSIZE:-1024} -MIN_PKGS=${MIN_PKGS:-"apt-utils gpgv openssh-server"} - -usage() { - echo "Usage: $0 - Create Ubuntu images" - echo "" - echo "$0 [-m] [-r rootsize] [-s swapsize] release format" - echo "$0 -C [-m] release chrootdir" - echo "$0 -I [-r rootsize] [-s swapsize] chrootdir format" - echo "" - echo "-C - Create the initial chroot dir" - echo "-I - Create the final image from a chroot" - echo "-m - minimal installation" - echo "-r size - root fs size in MB" - echo "-s size - swap fs size in MB" - echo "release - Ubuntu release: jaunty - oneric" - echo "format - image format: qcow2, vmdk, vdi, vhd, xen, raw, fs" - exit 1 -} - -while getopts CIhmr:s: c; do - case $c in - C) CHROOTONLY=1 - ;; - I) IMAGEONLY=1 - ;; - h) usage - ;; - m) MINIMAL=1 - ;; - r) ROOTSIZE=$OPTARG - ;; - s) SWAPSIZE=$OPTARG - ;; - esac -done -shift `expr $OPTIND - 1` - -if [ ! "$#" -eq "2" -o -n "$CHROOTONLY" -a -n "$IMAGEONLY" ]; then - usage -fi - -# Default args -RELEASE=$1 -FORMAT=$2 -CHROOTDIR="" - -if [ -n "$CHROOTONLY" ]; then - RELEASE=$1 - CHROOTDIR=$2 - FORMAT="pass" -fi - -if [ -n "$IMAGEONLY" ]; then - CHROOTDIR=$1 - FORMAT=$2 - RELEASE="pass" -fi - -# Make sure that we have the proper version of ubuntu -UBUNTU_VERSION=`cat /etc/lsb-release | grep CODENAME | sed 's/.*=//g'` -if [ "$UBUNTU_VERSION" = "natty" -a "$RELEASE" = "oneiric" ]; then - echo "natty installs can't build oneiric images" - exit 1 -fi - -case $FORMAT in - kvm|qcow2) FORMAT=qcow2 - QFORMAT=qcow2 - HYPER=kvm - ;; - vmserver|vmdk) - FORMAT=vmdk - QFORMAT=vmdk - HYPER=vmserver - ;; - vbox|vdi) FORMAT=vdi - QFORMAT=vdi - HYPER=kvm - ;; - vhd|vpc) FORMAT=vhd - QFORMAT=vpc - HYPER=kvm - ;; - xen) FORMAT=raw - QFORMAT=raw - HYPER=xen - ;; - raw) FORMAT=raw - QFORMAT=raw - HYPER=kvm - ;; - pass) ;; - *) echo "Unknown format: $FORMAT" - usage -esac - -case $RELEASE in - oneiric) ;; - natty) ;; - maverick) ;; - lucid) ;; - karmic) ;; - jaunty) ;; - pass) ;; - *) echo "Unknown release: $RELEASE" - usage - ;; -esac - -# Install stuff if necessary -if [ -z `which vmbuilder` ]; then - sudo apt-get install -y ubuntu-vm-builder -fi - -if [ -n "$CHROOTONLY" ]; then - # Build a chroot directory - HYPER=kvm - if [ "$MINIMAL" = 1 ]; then - ARGS="--variant=minbase" - for i in $MIN_PKGS; do - ARGS="$ARGS --addpkg=$i" - done - fi - sudo vmbuilder $HYPER ubuntu $ARGS \ - --suite $RELEASE \ - --only-chroot \ - --chroot-dir=$CHROOTDIR \ - --overwrite \ - --addpkg=$MIN_PKGS \ - - sudo cp -p files/sources.list $CHROOTDIR/etc/apt/sources.list - sed -e "s,%DIST%,$RELEASE,g" -i $CHROOTDIR/etc/apt/sources.list - sudo chroot $CHROOTDIR apt-get update - - exit 0 -fi - -# Build the image -TMPDIR=tmp -TMPDISK=`mktemp imgXXXXXXXX` -SIZE=$[$ROOTSIZE+$SWAPSIZE+1] -dd if=/dev/null of=$TMPDISK bs=1M seek=$SIZE count=1 - -if [ -n "$IMAGEONLY" ]; then - # Build image from chroot - sudo vmbuilder $HYPER ubuntu $ARGS \ - --existing-chroot=$CHROOTDIR \ - --overwrite \ - --rootsize=$ROOTSIZE \ - --swapsize=$SWAPSIZE \ - --tmpfs - \ - --raw=$TMPDISK \ - -else - # Do the whole shebang in one pass - ARGS="--variant=minbase" - for i in $MIN_PKGS; do - ARGS="$ARGS --addpkg=$i" - done - sudo vmbuilder $HYPER ubuntu $ARGS \ - --suite $RELEASE \ - --overwrite \ - --rootsize=$ROOTSIZE \ - --swapsize=$SWAPSIZE \ - --tmpfs - \ - --raw=$TMPDISK \ - -fi - -if [ "$FORMAT" = "raw" ]; then - # Get image - mv $TMPDISK $RELEASE.$FORMAT -else - # Convert image - qemu-img convert -O $QFORMAT $TMPDISK $RELEASE.$FORMAT - rm $TMPDISK -fi -rm -rf ubuntu-$HYPER diff --git a/tools/upload_image.sh b/tools/upload_image.sh deleted file mode 100755 index da73f16..0000000 --- a/tools/upload_image.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -# upload_image.sh - Upload Ubuntu images (create if necessary) in various formats -# Supported formats: qcow (kvm), vmdk (vmserver), vdi (vbox), vhd (vpc) -# Requires sudo to root - -usage() { - echo "$0 - Upload images to OpenStack" - echo "" - echo "$0 [-h host] [-p port] release format" - exit 1 -} - -HOST=${HOST:-localhost} -PORT=${PORT:-9292} -DEST=${DEST:-/opt/stack} - -while getopts h:p: c; do - case $c in - h) HOST=$OPTARG - ;; - p) PORT=$OPTARG - ;; - esac -done -shift `expr $OPTIND - 1` - -RELEASE=$1 -FORMAT=$2 - -case $FORMAT in - kvm|qcow2) FORMAT=qcow2 - TARGET=kvm - ;; - vmserver|vmdk) - FORMAT=vmdk - TARGET=vmserver - ;; - vbox|vdi) TARGET=kvm - FORMAT=vdi - ;; - vhd|vpc) TARGET=kvm - FORMAT=vhd - ;; - *) echo "Unknown format: $FORMAT" - usage -esac - -case $RELEASE in - natty) ;; - maverick) ;; - lucid) ;; - karmic) ;; - jaunty) ;; - *) if [ ! -r $RELEASE.$FORMAT ]; then - echo "Unknown release: $RELEASE" - usage - fi - ;; -esac - -GLANCE=`which glance` -if [ -z "$GLANCE" ]; then - if [ -x "$DEST/glance/bin/glance" ]; then - # Look for stack.sh's install - GLANCE="$DEST/glance/bin/glance" - else - # Install Glance client in $DEST - echo "Glance not found, must install client" - OWD=`pwd` - cd $DEST - sudo apt-get install python-pip python-eventlet python-routes python-greenlet python-argparse python-sqlalchemy python-wsgiref python-pastedeploy python-xattr - sudo pip install kombu - sudo git clone https://github.com/cloudbuilders/glance.git - cd glance - sudo python setup.py develop - cd $OWD - GLANCE=`which glance` - fi -fi - -# Create image if it doesn't exist -if [ ! -r $RELEASE.$FORMAT ]; then - DIR=`dirname $0` - echo "$RELEASE.$FORMAT not found, creating..." - $DIR/make_image.sh $RELEASE $FORMAT -fi - -# Upload the image -echo "Uploading image $RELEASE.$FORMAT to $HOST" -$GLANCE add name=$RELEASE.$FORMAT is_public=true disk_format=$FORMAT --host $HOST --port $PORT <$RELEASE.$FORMAT diff --git a/tools/warm_apts_and_pips.sh b/tools/warm_apts_and_pips_for_uec.sh similarity index 100% rename from tools/warm_apts_and_pips.sh rename to tools/warm_apts_and_pips_for_uec.sh