From eeec0206285e200f34b6005459c4d79a588e7f1d Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 18:42:11 -0700 Subject: [PATCH 01/15] another attempt at reclone which preserves existing git object dir --- stack.sh | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/stack.sh b/stack.sh index 18206cd..74cb383 100755 --- a/stack.sh +++ b/stack.sh @@ -302,20 +302,27 @@ sudo PIP_DOWNLOAD_CACHE=/var/cache/pip pip install `cat $FILES/pips/*` # be owned by the installation user, we create the directory and change the # ownership to the proper user. function git_clone { - # if there is an existing checkout, move it out of the way - if [[ "$RECLONE" == "yes" ]]; then - # FIXME(ja): if we were smarter we could speed up RECLONE by - # using the old git repo as the basis of our new clone... - if [ -d $2 ]; then - mv $2 /tmp/stack.`date +%s` - fi - fi + # do a full clone only if the directory doesn't exist if [ ! -d $2 ]; then git clone $1 $2 cd $2 # This checkout syntax works for both branches and tags git checkout $3 + elif [[ "$RECLONE" == "yes" ]]; then + # if it does exist then simulate what clone does if asked to RECLONE + cd $2 + # set the url to pull from and fetch + git remote set-url origin $1 + git fetch origin + # if we don't delete the local content, then our system has pyc files + # from the previous branch leading to breakage (due to the py files + # having older timestamps than our pyc, so python thinks the pyc files + # are correct using them) + rm -rf * + git checkout -f origin/$3 + git branch -D $3 + git checkout -b $3 fi } From 917c66584f9c4d596fdf651330e48cd5219a4436 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 18:47:06 -0700 Subject: [PATCH 02/15] use variable names for git_clone function --- stack.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/stack.sh b/stack.sh index 74cb383..ed5dc36 100755 --- a/stack.sh +++ b/stack.sh @@ -303,26 +303,31 @@ sudo PIP_DOWNLOAD_CACHE=/var/cache/pip pip install `cat $FILES/pips/*` # ownership to the proper user. function git_clone { + GIT_REMOTE=$1 + GIT_DEST=$2 + GIT_BRANCH=$3 + # do a full clone only if the directory doesn't exist - if [ ! -d $2 ]; then - git clone $1 $2 + if [ ! -d $GIT_DEST ]; then + git clone $GIT_REMOTE $GIT_DEST cd $2 # This checkout syntax works for both branches and tags - git checkout $3 + git checkout $GIT_BRANCH elif [[ "$RECLONE" == "yes" ]]; then # if it does exist then simulate what clone does if asked to RECLONE - cd $2 + cd $GIT_BRANCH # set the url to pull from and fetch - git remote set-url origin $1 + git remote set-url origin $GIT_REMOTE git fetch origin # if we don't delete the local content, then our system has pyc files # from the previous branch leading to breakage (due to the py files # having older timestamps than our pyc, so python thinks the pyc files # are correct using them) rm -rf * - git checkout -f origin/$3 - git branch -D $3 - git checkout -b $3 + git checkout -f origin/$GIT_BRANCH + # a local branch might not exist for $3 + git branch -D $GIT_BRANCH || true + git checkout -b $GIT_BRANCH fi } From b9b3ad49a6fc318b51c3746036721918924b523e Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 18:52:13 -0700 Subject: [PATCH 03/15] some of the files are owned by root --- stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.sh b/stack.sh index ed5dc36..8dbb9e9 100755 --- a/stack.sh +++ b/stack.sh @@ -323,7 +323,7 @@ function git_clone { # from the previous branch leading to breakage (due to the py files # having older timestamps than our pyc, so python thinks the pyc files # are correct using them) - rm -rf * + sudo rm -rf * git checkout -f origin/$GIT_BRANCH # a local branch might not exist for $3 git branch -D $GIT_BRANCH || true From 480644bd7461f514f66b41e87ff727c6d86f7fdb Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 18:52:58 -0700 Subject: [PATCH 04/15] error in conversion --- stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.sh b/stack.sh index 8dbb9e9..e3f8a94 100755 --- a/stack.sh +++ b/stack.sh @@ -315,7 +315,7 @@ function git_clone { git checkout $GIT_BRANCH elif [[ "$RECLONE" == "yes" ]]; then # if it does exist then simulate what clone does if asked to RECLONE - cd $GIT_BRANCH + cd $GIT_DEST # set the url to pull from and fetch git remote set-url origin $GIT_REMOTE git fetch origin From 9fef844fd9adf0d1a2a68890d5b4d92244f95874 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 19:06:46 -0700 Subject: [PATCH 05/15] use git clean - thanks lundy --- stack.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/stack.sh b/stack.sh index e3f8a94..4b73102 100755 --- a/stack.sh +++ b/stack.sh @@ -319,11 +319,10 @@ function git_clone { # set the url to pull from and fetch git remote set-url origin $GIT_REMOTE git fetch origin - # if we don't delete the local content, then our system has pyc files - # from the previous branch leading to breakage (due to the py files - # having older timestamps than our pyc, so python thinks the pyc files - # are correct using them) - sudo rm -rf * + # remove the existing ignored files (like pyc) as they cause breakage + # (due to the py files having older timestamps than our pyc, so python + # thinks the pyc files are correct using them) + sudo git clean -f -d git checkout -f origin/$GIT_BRANCH # a local branch might not exist for $3 git branch -D $GIT_BRANCH || true From e09a6e4a82debdc5d8d06b0b74312d1266208b76 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 19:09:52 -0700 Subject: [PATCH 06/15] update comment --- stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.sh b/stack.sh index 4b73102..6c7dcdf 100755 --- a/stack.sh +++ b/stack.sh @@ -324,7 +324,7 @@ function git_clone { # thinks the pyc files are correct using them) sudo git clean -f -d git checkout -f origin/$GIT_BRANCH - # a local branch might not exist for $3 + # a local branch might not exist git branch -D $GIT_BRANCH || true git checkout -b $GIT_BRANCH fi From f70569e33d4484fbe66dad4399d9887c0185de22 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 21:56:25 -0700 Subject: [PATCH 07/15] don't modify the paste.ini inplace - copy to nova, then modify --- stack.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/stack.sh b/stack.sh index 18206cd..b54b486 100755 --- a/stack.sh +++ b/stack.sh @@ -470,11 +470,15 @@ fi # Nova # ---- -# We are going to use the sample http middleware configuration from the keystone -# project to launch nova. This paste config adds the configuration required -# for nova to validate keystone tokens - except we need to switch the config -# to use our service token instead (instead of the invalid token 999888777666). -sudo sed -e "s,999888777666,$SERVICE_TOKEN,g" -i $KEYSTONE_DIR/examples/paste/nova-api-paste.ini +if [[ "$ENABLED_SERVICES" =~ "n-api" ]]; then + # We are going to use the sample http middleware configuration from the + # keystone project to launch nova. This paste config adds the configuration + # required for nova to validate keystone tokens - except we need to switch + # the config to use our service token instead (instead of the invalid token + # 999888777666). + cp $KEYSTONE_DIR/examples/paste/nova-api-paste.ini $NOVA_DIR/bin + sed -e "s,999888777666,$SERVICE_TOKEN,g" -i $NOVA_DIR/bin/nova-api-paste.ini +fi if [[ "$ENABLED_SERVICES" =~ "n-cpu" ]]; then @@ -591,7 +595,7 @@ add_nova_flag "--libvirt_type=$LIBVIRT_TYPE" add_nova_flag "--osapi_extensions_path=$OPENSTACKX_DIR/extensions" add_nova_flag "--vncproxy_url=http://$HOST_IP:6080" add_nova_flag "--vncproxy_wwwroot=$NOVNC_DIR/" -add_nova_flag "--api_paste_config=$KEYSTONE_DIR/examples/paste/nova-api-paste.ini" +add_nova_flag "--api_paste_config=$NOVA_DIR/bint/nova-api-paste.ini" add_nova_flag "--image_service=nova.image.glance.GlanceImageService" add_nova_flag "--ec2_dmz_host=$EC2_DMZ_HOST" add_nova_flag "--rabbit_host=$RABBIT_HOST" From 42dc9a77156df998d5663b01b0d63ea27ec1930f Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 24 Oct 2011 22:03:11 -0700 Subject: [PATCH 08/15] this is why we do a test run before merging --- stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.sh b/stack.sh index b54b486..b07b2bc 100755 --- a/stack.sh +++ b/stack.sh @@ -595,7 +595,7 @@ add_nova_flag "--libvirt_type=$LIBVIRT_TYPE" add_nova_flag "--osapi_extensions_path=$OPENSTACKX_DIR/extensions" add_nova_flag "--vncproxy_url=http://$HOST_IP:6080" add_nova_flag "--vncproxy_wwwroot=$NOVNC_DIR/" -add_nova_flag "--api_paste_config=$NOVA_DIR/bint/nova-api-paste.ini" +add_nova_flag "--api_paste_config=$NOVA_DIR/bin/nova-api-paste.ini" add_nova_flag "--image_service=nova.image.glance.GlanceImageService" add_nova_flag "--ec2_dmz_host=$EC2_DMZ_HOST" add_nova_flag "--rabbit_host=$RABBIT_HOST" From 6065772feca1862844efd6388b4782354154a6fc Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 25 Oct 2011 23:41:59 -0700 Subject: [PATCH 09/15] cache the images --- tools/build_kvm.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/build_kvm.sh b/tools/build_kvm.sh index 32c7bf6..d804f42 100755 --- a/tools/build_kvm.sh +++ b/tools/build_kvm.sh @@ -284,6 +284,15 @@ if [ "$COPYENV" = "1" ]; then cp_it ~/.bashrc $ROOTFS/$DEST/.bashrc fi +# pre-cache uec images +for image_url in ${IMAGE_URLS//,/ }; do + IMAGE_FNAME=`basename "$image_url"` + if [ ! -f $IMAGES_DIR/$IMAGE_FNAME ]; then + wget -c $image_url -O $IMAGES_DIR/$IMAGE_FNAME + fi + cp $IMAGES_DIR/$IMAGE_FNAME $ROOTFS/$DEST/devstack/files/images +fi + # Configure the runner RUN_SH=$ROOTFS/$DEST/run.sh cat > $RUN_SH < Date: Tue, 25 Oct 2011 23:43:03 -0700 Subject: [PATCH 10/15] typo --- tools/build_kvm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build_kvm.sh b/tools/build_kvm.sh index d804f42..4ca1f1a 100755 --- a/tools/build_kvm.sh +++ b/tools/build_kvm.sh @@ -291,7 +291,7 @@ for image_url in ${IMAGE_URLS//,/ }; do wget -c $image_url -O $IMAGES_DIR/$IMAGE_FNAME fi cp $IMAGES_DIR/$IMAGE_FNAME $ROOTFS/$DEST/devstack/files/images -fi +done # Configure the runner RUN_SH=$ROOTFS/$DEST/run.sh From 244655f0c57f5453b701c5e9c55608cea55b6a97 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 25 Oct 2011 23:48:39 -0700 Subject: [PATCH 11/15] copy to a directory not file --- tools/build_kvm.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/build_kvm.sh b/tools/build_kvm.sh index 4ca1f1a..3f32598 100755 --- a/tools/build_kvm.sh +++ b/tools/build_kvm.sh @@ -290,6 +290,7 @@ for image_url in ${IMAGE_URLS//,/ }; do if [ ! -f $IMAGES_DIR/$IMAGE_FNAME ]; then wget -c $image_url -O $IMAGES_DIR/$IMAGE_FNAME fi + mkdir -p $ROOTFS/$DEST/devstack/files/images cp $IMAGES_DIR/$IMAGE_FNAME $ROOTFS/$DEST/devstack/files/images done From 00dcc067f7f4e2c2faaba9aeb3cc23a1981eaf12 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 25 Oct 2011 23:51:43 -0700 Subject: [PATCH 12/15] put them in the right place --- tools/build_kvm.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/build_kvm.sh b/tools/build_kvm.sh index 3f32598..97260e2 100755 --- a/tools/build_kvm.sh +++ b/tools/build_kvm.sh @@ -290,8 +290,7 @@ for image_url in ${IMAGE_URLS//,/ }; do if [ ! -f $IMAGES_DIR/$IMAGE_FNAME ]; then wget -c $image_url -O $IMAGES_DIR/$IMAGE_FNAME fi - mkdir -p $ROOTFS/$DEST/devstack/files/images - cp $IMAGES_DIR/$IMAGE_FNAME $ROOTFS/$DEST/devstack/files/images + cp $IMAGES_DIR/$IMAGE_FNAME $ROOTFS/$DEST/devstack/files done # Configure the runner From 56a505fc2d3b95c310f2330b4e4d45d8dfe8f0e1 Mon Sep 17 00:00:00 2001 From: Justin Shepherd Date: Wed, 26 Oct 2011 10:45:02 -0500 Subject: [PATCH 13/15] bug fix.. resulted in following error message: ./exercise.sh: line 96: [: missing `]' --- exercise.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exercise.sh b/exercise.sh index b73d6aa..3f2c94e 100755 --- a/exercise.sh +++ b/exercise.sh @@ -93,7 +93,7 @@ IP=`nova show $NAME | grep "private network" | cut -d"|" -f3` # for single node deployments, we can ping private ips MULTI_HOST=${MULTI_HOST:-0} -if [ "$MULTI_HOST" = "0"]; then +if [ "$MULTI_HOST" = "0" ]; then # ping it once (timeout of a second) ping -c1 -w1 $IP || true From 7d45a0fa601fab0e4e7237f8920e049744a94c4a Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Wed, 26 Oct 2011 08:51:15 -0700 Subject: [PATCH 14/15] default the root password to admin password and don't fail if group already exists --- tools/build_kvm.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/build_kvm.sh b/tools/build_kvm.sh index 32c7bf6..ea135f6 100755 --- a/tools/build_kvm.sh +++ b/tools/build_kvm.sh @@ -19,9 +19,6 @@ set -o xtrace TOOLS_DIR=$(cd $(dirname "$0") && pwd) TOP_DIR=$TOOLS_DIR/.. -# Configure the root password of the vm -ROOT_PASSWORD=${ROOT_PASSWORD:-password} - # Where to store files and instances KVMSTACK_DIR=${KVMSTACK_DIR:-/opt/kvmstack} @@ -44,6 +41,10 @@ fi # Source params source ./stackrc +# Configure the root password of the vm to be the same as ``ADMIN_PASSWORD`` +ROOT_PASSWORD=${ADMIN_PASSWORD:-password} + + # Base image (natty by default) DIST_NAME=${DIST_NAME:-natty} IMAGE_FNAME=$DIST_NAME.raw @@ -260,7 +261,7 @@ iface eth0 inet static EOF # User configuration for the instance -chroot $ROOTFS groupadd libvirtd +chroot $ROOTFS groupadd libvirtd || true chroot $ROOTFS useradd stack -s /bin/bash -d $DEST -G libvirtd cp -pr $TOOLS_DIR/.. $ROOTFS/$DEST/devstack echo "root:$ROOT_PASSWORD" | chroot $ROOTFS chpasswd From ad21d1a78e83ea8db1c87b65e646215d9cce8198 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Wed, 26 Oct 2011 12:39:00 -0700 Subject: [PATCH 15/15] raise an error if stack fails in lxc --- tools/build_lxc.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/build_lxc.sh b/tools/build_lxc.sh index 9cbbedf..b629183 100755 --- a/tools/build_lxc.sh +++ b/tools/build_lxc.sh @@ -301,6 +301,11 @@ if [ "$WAIT_TILL_LAUNCH" = "1" ]; then done kill $TAIL_PID + + if grep -q "stack.sh failed" $ROOTFS/$DEST/run.sh.log; then + exit 1 + fi + echo "" echo "Finished - Zip-a-dee Doo-dah!" fi