From eeec0206285e200f34b6005459c4d79a588e7f1d Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 18:42:11 -0700 Subject: [PATCH 01/16] another attempt at reclone which preserves existing git object dir --- stack.sh | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/stack.sh b/stack.sh index 18206cd..74cb383 100755 --- a/stack.sh +++ b/stack.sh @@ -302,20 +302,27 @@ sudo PIP_DOWNLOAD_CACHE=/var/cache/pip pip install `cat $FILES/pips/*` # be owned by the installation user, we create the directory and change the # ownership to the proper user. function git_clone { - # if there is an existing checkout, move it out of the way - if [[ "$RECLONE" == "yes" ]]; then - # FIXME(ja): if we were smarter we could speed up RECLONE by - # using the old git repo as the basis of our new clone... - if [ -d $2 ]; then - mv $2 /tmp/stack.`date +%s` - fi - fi + # do a full clone only if the directory doesn't exist if [ ! -d $2 ]; then git clone $1 $2 cd $2 # This checkout syntax works for both branches and tags git checkout $3 + elif [[ "$RECLONE" == "yes" ]]; then + # if it does exist then simulate what clone does if asked to RECLONE + cd $2 + # set the url to pull from and fetch + git remote set-url origin $1 + git fetch origin + # if we don't delete the local content, then our system has pyc files + # from the previous branch leading to breakage (due to the py files + # having older timestamps than our pyc, so python thinks the pyc files + # are correct using them) + rm -rf * + git checkout -f origin/$3 + git branch -D $3 + git checkout -b $3 fi } From 917c66584f9c4d596fdf651330e48cd5219a4436 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 18:47:06 -0700 Subject: [PATCH 02/16] use variable names for git_clone function --- stack.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/stack.sh b/stack.sh index 74cb383..ed5dc36 100755 --- a/stack.sh +++ b/stack.sh @@ -303,26 +303,31 @@ sudo PIP_DOWNLOAD_CACHE=/var/cache/pip pip install `cat $FILES/pips/*` # ownership to the proper user. function git_clone { + GIT_REMOTE=$1 + GIT_DEST=$2 + GIT_BRANCH=$3 + # do a full clone only if the directory doesn't exist - if [ ! -d $2 ]; then - git clone $1 $2 + if [ ! -d $GIT_DEST ]; then + git clone $GIT_REMOTE $GIT_DEST cd $2 # This checkout syntax works for both branches and tags - git checkout $3 + git checkout $GIT_BRANCH elif [[ "$RECLONE" == "yes" ]]; then # if it does exist then simulate what clone does if asked to RECLONE - cd $2 + cd $GIT_BRANCH # set the url to pull from and fetch - git remote set-url origin $1 + git remote set-url origin $GIT_REMOTE git fetch origin # if we don't delete the local content, then our system has pyc files # from the previous branch leading to breakage (due to the py files # having older timestamps than our pyc, so python thinks the pyc files # are correct using them) rm -rf * - git checkout -f origin/$3 - git branch -D $3 - git checkout -b $3 + git checkout -f origin/$GIT_BRANCH + # a local branch might not exist for $3 + git branch -D $GIT_BRANCH || true + git checkout -b $GIT_BRANCH fi } From b9b3ad49a6fc318b51c3746036721918924b523e Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 18:52:13 -0700 Subject: [PATCH 03/16] some of the files are owned by root --- stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.sh b/stack.sh index ed5dc36..8dbb9e9 100755 --- a/stack.sh +++ b/stack.sh @@ -323,7 +323,7 @@ function git_clone { # from the previous branch leading to breakage (due to the py files # having older timestamps than our pyc, so python thinks the pyc files # are correct using them) - rm -rf * + sudo rm -rf * git checkout -f origin/$GIT_BRANCH # a local branch might not exist for $3 git branch -D $GIT_BRANCH || true From 480644bd7461f514f66b41e87ff727c6d86f7fdb Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 18:52:58 -0700 Subject: [PATCH 04/16] error in conversion --- stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.sh b/stack.sh index 8dbb9e9..e3f8a94 100755 --- a/stack.sh +++ b/stack.sh @@ -315,7 +315,7 @@ function git_clone { git checkout $GIT_BRANCH elif [[ "$RECLONE" == "yes" ]]; then # if it does exist then simulate what clone does if asked to RECLONE - cd $GIT_BRANCH + cd $GIT_DEST # set the url to pull from and fetch git remote set-url origin $GIT_REMOTE git fetch origin From 9fef844fd9adf0d1a2a68890d5b4d92244f95874 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 19:06:46 -0700 Subject: [PATCH 05/16] use git clean - thanks lundy --- stack.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/stack.sh b/stack.sh index e3f8a94..4b73102 100755 --- a/stack.sh +++ b/stack.sh @@ -319,11 +319,10 @@ function git_clone { # set the url to pull from and fetch git remote set-url origin $GIT_REMOTE git fetch origin - # if we don't delete the local content, then our system has pyc files - # from the previous branch leading to breakage (due to the py files - # having older timestamps than our pyc, so python thinks the pyc files - # are correct using them) - sudo rm -rf * + # remove the existing ignored files (like pyc) as they cause breakage + # (due to the py files having older timestamps than our pyc, so python + # thinks the pyc files are correct using them) + sudo git clean -f -d git checkout -f origin/$GIT_BRANCH # a local branch might not exist for $3 git branch -D $GIT_BRANCH || true From e09a6e4a82debdc5d8d06b0b74312d1266208b76 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 19:09:52 -0700 Subject: [PATCH 06/16] update comment --- stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.sh b/stack.sh index 4b73102..6c7dcdf 100755 --- a/stack.sh +++ b/stack.sh @@ -324,7 +324,7 @@ function git_clone { # thinks the pyc files are correct using them) sudo git clean -f -d git checkout -f origin/$GIT_BRANCH - # a local branch might not exist for $3 + # a local branch might not exist git branch -D $GIT_BRANCH || true git checkout -b $GIT_BRANCH fi From f70569e33d4484fbe66dad4399d9887c0185de22 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 21:56:25 -0700 Subject: [PATCH 07/16] don't modify the paste.ini inplace - copy to nova, then modify --- stack.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/stack.sh b/stack.sh index 18206cd..b54b486 100755 --- a/stack.sh +++ b/stack.sh @@ -470,11 +470,15 @@ fi # Nova # ---- -# We are going to use the sample http middleware configuration from the keystone -# project to launch nova. This paste config adds the configuration required -# for nova to validate keystone tokens - except we need to switch the config -# to use our service token instead (instead of the invalid token 999888777666). -sudo sed -e "s,999888777666,$SERVICE_TOKEN,g" -i $KEYSTONE_DIR/examples/paste/nova-api-paste.ini +if [[ "$ENABLED_SERVICES" =~ "n-api" ]]; then + # We are going to use the sample http middleware configuration from the + # keystone project to launch nova. This paste config adds the configuration + # required for nova to validate keystone tokens - except we need to switch + # the config to use our service token instead (instead of the invalid token + # 999888777666). + cp $KEYSTONE_DIR/examples/paste/nova-api-paste.ini $NOVA_DIR/bin + sed -e "s,999888777666,$SERVICE_TOKEN,g" -i $NOVA_DIR/bin/nova-api-paste.ini +fi if [[ "$ENABLED_SERVICES" =~ "n-cpu" ]]; then @@ -591,7 +595,7 @@ add_nova_flag "--libvirt_type=$LIBVIRT_TYPE" add_nova_flag "--osapi_extensions_path=$OPENSTACKX_DIR/extensions" add_nova_flag "--vncproxy_url=http://$HOST_IP:6080" add_nova_flag "--vncproxy_wwwroot=$NOVNC_DIR/" -add_nova_flag "--api_paste_config=$KEYSTONE_DIR/examples/paste/nova-api-paste.ini" +add_nova_flag "--api_paste_config=$NOVA_DIR/bint/nova-api-paste.ini" add_nova_flag "--image_service=nova.image.glance.GlanceImageService" add_nova_flag "--ec2_dmz_host=$EC2_DMZ_HOST" add_nova_flag "--rabbit_host=$RABBIT_HOST" From 42dc9a77156df998d5663b01b0d63ea27ec1930f Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 22:03:11 -0700 Subject: [PATCH 08/16] this is why we do a test run before merging --- stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.sh b/stack.sh index b54b486..b07b2bc 100755 --- a/stack.sh +++ b/stack.sh @@ -595,7 +595,7 @@ add_nova_flag "--libvirt_type=$LIBVIRT_TYPE" add_nova_flag "--osapi_extensions_path=$OPENSTACKX_DIR/extensions" add_nova_flag "--vncproxy_url=http://$HOST_IP:6080" add_nova_flag "--vncproxy_wwwroot=$NOVNC_DIR/" -add_nova_flag "--api_paste_config=$NOVA_DIR/bint/nova-api-paste.ini" +add_nova_flag "--api_paste_config=$NOVA_DIR/bin/nova-api-paste.ini" add_nova_flag "--image_service=nova.image.glance.GlanceImageService" add_nova_flag "--ec2_dmz_host=$EC2_DMZ_HOST" add_nova_flag "--rabbit_host=$RABBIT_HOST" From e753fdf42c45a7319159f401158b717960aa99ba Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Tue, 25 Oct 2011 15:45:26 -0500 Subject: [PATCH 09/16] Add get_uec_image.sh --- tools/get_uec_image.sh | 159 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100755 tools/get_uec_image.sh diff --git a/tools/get_uec_image.sh b/tools/get_uec_image.sh new file mode 100755 index 0000000..fde53d2 --- /dev/null +++ b/tools/get_uec_image.sh @@ -0,0 +1,159 @@ +#!/bin/bash +# get_uec_image.sh - Prepare Ubuntu images in various formats +# +# Supported formats: qcow (kvm), vmdk (vmserver), vdi (vbox), vhd (vpc), raw +# +# Requires to run as root + +CACHEDIR=${CACHEDIR:-/var/cache/devstack} +FORMAT=${FORMAT:-qcow2} +ROOTSIZE=${ROOTSIZE:-2000} +MIN_PKGS=${MIN_PKGS:-"apt-utils gpgv openssh-server"} + +usage() { + echo "Usage: $0 - Prepare Ubuntu images" + echo "" + echo "$0 [-f format] [-r rootsize] release imagefile" + echo "" + echo "-f format - image format: qcow2 (default), vmdk, vdi, vhd, xen, raw, fs" + echo "-r size - root fs size in MB (min 2000MB)" + echo "release - Ubuntu release: jaunty - oneric" + echo "imagefile - output image file + exit 1 +} + +while getopts f:hmr: c; do + case $c in + f) FORMAT=$OPTARG + ;; + h) usage + ;; + m) MINIMAL=1 + ;; + r) ROOTSIZE=$OPTARG + if $(( ROOTSIZE < 2000 )); then + echo "root size must be greater than 2000MB" + exit 1 + fi + ;; + esac +done +shift `expr $OPTIND - 1` + +if [ ! "$#" -eq "2" ]; then + usage +fi + +# Default args +DIST_NAME=$1 +IMG_FILE=$2 + +case $FORMAT in + kvm|qcow2) FORMAT=qcow2 + QFORMAT=qcow2 + ;; + vmserver|vmdk) + FORMAT=vmdk + QFORMAT=vmdk + ;; + vbox|vdi) FORMAT=vdi + QFORMAT=vdi + ;; + vhd|vpc) FORMAT=vhd + QFORMAT=vpc + ;; + xen) FORMAT=raw + QFORMAT=raw + ;; + raw) FORMAT=raw + QFORMAT=raw + ;; + *) echo "Unknown format: $FORMAT" + usage +esac + +case $DIST_NAME in + oneiric) ;; + natty) ;; + maverick) ;; + lucid) ;; + karmic) ;; + jaunty) ;; + *) echo "Unknown release: $DIST_NAME" + usage + ;; +esac + +# Set up nbd +modprobe nbd max_part=63 +NBD=${NBD:-/dev/nbd9} +NBD_DEV=`basename $NBD` + +# Prepare the base image + +# Get the UEC image +UEC_NAME=$DIST_NAME-server-cloudimg-amd64 +if [ ! -e $CACHEDIR/$UEC_NAME-disk1.img ]; then + (cd $CACHEDIR; wget -N http://uec-images.ubuntu.com/$DIST_NAME/current/$UEC_NAME-disk1.img) + + + # Connect to nbd and wait till it is ready + qemu-nbd -d $NBD + qemu-nbd -c $NBD $CACHEDIR/$UEC_NAME-disk1.img + if ! timeout 60 sh -c "while ! [ -e /sys/block/$NBD_DEV/pid ]; do sleep 1; done"; then + echo "Couldn't connect $NBD" + exit 1 + fi + MNTDIR=`mktemp -d mntXXXXXXXX` + mount -t ext4 ${NBD}p1 $MNTDIR + + # Install our required packages + cp -p files/sources.list $MNTDIR/etc/apt/sources.list + cp -p /etc/resolv.conf $MNTDIR/etc/resolv.conf + chroot $MNTDIR apt-get update + chroot $MNTDIR apt-get install -y $MIN_PKGS + rm -f $MNTDIR/etc/resolv.conf + + umount $MNTDIR + rmdir $MNTDIR + qemu-nbd -d $NBD +fi + +if [ "$FORMAT" = "qcow2" ]; then + # Just copy image + cp -p $CACHEDIR/$UEC_NAME-disk1.img $IMG_FILE +else + # Convert image + qemu-img convert -O $QFORMAT $CACHEDIR/$UEC_NAME-disk1.img $IMG_FILE +fi + +# Resize the image if necessary +if [ $ROOTSIZE -gt 2000 ]; then + # Resize the container + qemu-img resize $IMG_FILE +$((ROOTSIZE - 2000))M + + # Connect to nbd and wait till it is ready + qemu-nbd -c $NBD $IMG_FILE + if ! timeout 60 sh -c "while ! [ -e /sys/block/$NBD_DEV/pid ]; do sleep 1; done"; then + echo "Couldn't connect $NBD" + exit 1 + fi + + # Resize partition 1 to full size of the disk image + echo "d +n +p +1 +2 + +t +83 +a +1 +w +" | fdisk $NBD + fsck -t ext4 -f ${NBD}p1 + resize2fs ${NBD}p1 + + qemu-nbd -d $NBD +fi From ecc5a909292db82648d48cc7ae0e873f7f9d66ac Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Tue, 25 Oct 2011 15:49:47 -0500 Subject: [PATCH 10/16] Add check for version compatibility --- tools/make_image.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/make_image.sh b/tools/make_image.sh index d81cef5..e957c75 100755 --- a/tools/make_image.sh +++ b/tools/make_image.sh @@ -65,6 +65,13 @@ if [ -n "$IMAGEONLY" ]; then RELEASE="pass" fi +# Make sure that we have the proper version of ubuntu +UBUNTU_VERSION=`cat /etc/lsb-release | grep CODENAME | sed 's/.*=//g'` +if [ "$UBUNTU_VERSION" = "natty" -a "$RELEASE" = "oneiric" ]; then + echo "natty installs can't build oneiric images" + exit 1 +fi + case $FORMAT in kvm|qcow2) FORMAT=qcow2 QFORMAT=qcow2 From a03b99dc997d7fefaea88162fc93a05e086d518b Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Tue, 25 Oct 2011 16:28:49 -0500 Subject: [PATCH 11/16] Fix typos --- tools/get_uec_image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/get_uec_image.sh b/tools/get_uec_image.sh index fde53d2..e9d708d 100755 --- a/tools/get_uec_image.sh +++ b/tools/get_uec_image.sh @@ -3,7 +3,7 @@ # # Supported formats: qcow (kvm), vmdk (vmserver), vdi (vbox), vhd (vpc), raw # -# Requires to run as root +# Required to run as root CACHEDIR=${CACHEDIR:-/var/cache/devstack} FORMAT=${FORMAT:-qcow2} @@ -18,7 +18,7 @@ usage() { echo "-f format - image format: qcow2 (default), vmdk, vdi, vhd, xen, raw, fs" echo "-r size - root fs size in MB (min 2000MB)" echo "release - Ubuntu release: jaunty - oneric" - echo "imagefile - output image file + echo "imagefile - output image file" exit 1 } From 6065772feca1862844efd6388b4782354154a6fc Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 25 Oct 2011 23:41:59 -0700 Subject: [PATCH 12/16] cache the images --- tools/build_kvm.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/build_kvm.sh b/tools/build_kvm.sh index 32c7bf6..d804f42 100755 --- a/tools/build_kvm.sh +++ b/tools/build_kvm.sh @@ -284,6 +284,15 @@ if [ "$COPYENV" = "1" ]; then cp_it ~/.bashrc $ROOTFS/$DEST/.bashrc fi +# pre-cache uec images +for image_url in ${IMAGE_URLS//,/ }; do + IMAGE_FNAME=`basename "$image_url"` + if [ ! -f $IMAGES_DIR/$IMAGE_FNAME ]; then + wget -c $image_url -O $IMAGES_DIR/$IMAGE_FNAME + fi + cp $IMAGES_DIR/$IMAGE_FNAME $ROOTFS/$DEST/devstack/files/images +fi + # Configure the runner RUN_SH=$ROOTFS/$DEST/run.sh cat > $RUN_SH < Date: Tue, 25 Oct 2011 23:43:03 -0700 Subject: [PATCH 13/16] typo --- tools/build_kvm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build_kvm.sh b/tools/build_kvm.sh index d804f42..4ca1f1a 100755 --- a/tools/build_kvm.sh +++ b/tools/build_kvm.sh @@ -291,7 +291,7 @@ for image_url in ${IMAGE_URLS//,/ }; do wget -c $image_url -O $IMAGES_DIR/$IMAGE_FNAME fi cp $IMAGES_DIR/$IMAGE_FNAME $ROOTFS/$DEST/devstack/files/images -fi +done # Configure the runner RUN_SH=$ROOTFS/$DEST/run.sh From 244655f0c57f5453b701c5e9c55608cea55b6a97 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 25 Oct 2011 23:48:39 -0700 Subject: [PATCH 14/16] copy to a directory not file --- tools/build_kvm.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/build_kvm.sh b/tools/build_kvm.sh index 4ca1f1a..3f32598 100755 --- a/tools/build_kvm.sh +++ b/tools/build_kvm.sh @@ -290,6 +290,7 @@ for image_url in ${IMAGE_URLS//,/ }; do if [ ! -f $IMAGES_DIR/$IMAGE_FNAME ]; then wget -c $image_url -O $IMAGES_DIR/$IMAGE_FNAME fi + mkdir -p $ROOTFS/$DEST/devstack/files/images cp $IMAGES_DIR/$IMAGE_FNAME $ROOTFS/$DEST/devstack/files/images done From 00dcc067f7f4e2c2faaba9aeb3cc23a1981eaf12 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 25 Oct 2011 23:51:43 -0700 Subject: [PATCH 15/16] put them in the right place --- tools/build_kvm.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/build_kvm.sh b/tools/build_kvm.sh index 3f32598..97260e2 100755 --- a/tools/build_kvm.sh +++ b/tools/build_kvm.sh @@ -290,8 +290,7 @@ for image_url in ${IMAGE_URLS//,/ }; do if [ ! -f $IMAGES_DIR/$IMAGE_FNAME ]; then wget -c $image_url -O $IMAGES_DIR/$IMAGE_FNAME fi - mkdir -p $ROOTFS/$DEST/devstack/files/images - cp $IMAGES_DIR/$IMAGE_FNAME $ROOTFS/$DEST/devstack/files/images + cp $IMAGES_DIR/$IMAGE_FNAME $ROOTFS/$DEST/devstack/files done # Configure the runner From 56a505fc2d3b95c310f2330b4e4d45d8dfe8f0e1 Mon Sep 17 00:00:00 2001 From: Justin Shepherd Date: Wed, 26 Oct 2011 10:45:02 -0500 Subject: [PATCH 16/16] bug fix.. resulted in following error message: ./exercise.sh: line 96: [: missing `]' --- exercise.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exercise.sh b/exercise.sh index b73d6aa..3f2c94e 100755 --- a/exercise.sh +++ b/exercise.sh @@ -93,7 +93,7 @@ IP=`nova show $NAME | grep "private network" | cut -d"|" -f3` # for single node deployments, we can ping private ips MULTI_HOST=${MULTI_HOST:-0} -if [ "$MULTI_HOST" = "0"]; then +if [ "$MULTI_HOST" = "0" ]; then # ping it once (timeout of a second) ping -c1 -w1 $IP || true