From e7fa90934d3d0f1dc0a89fbf6f498e927f041d39 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 7 Nov 2011 16:10:59 -0600 Subject: [PATCH 1/6] script to warm apts/pips on a base image, to speed up performace of build_ scripts --- tools/warm_apts_and_pips.sh | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tools/warm_apts_and_pips.sh diff --git a/tools/warm_apts_and_pips.sh b/tools/warm_apts_and_pips.sh new file mode 100644 index 0000000..c0b02b9 --- /dev/null +++ b/tools/warm_apts_and_pips.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Keep track of the current directory +TOOLS_DIR=$(cd $(dirname "$0") && pwd) +TOP_DIR=`cd $TOOLS_DIR/..; pwd` + +# cd to top of devstack +cd $TOP_DIR + +# Echo usage +usage() { + echo "Cache OpenStack dependencies on a uec image to speed up performance." + echo "" + echo "Usage: $0 [full path to raw uec base image]" +} + +# Make sure this is a raw image +if ! qemu-img info $1 | grep -q "file format: raw"; then + usage + exit 1 +fi + +# Mount the image +STAGING_DIR=`mktemp -d uec.XXXXXXXXXX` +mkdir -p $STAGING_DIR +mount -t ext4 -o loop $1 $STAGING_DIR + +# Make sure that base requirements are installed +cp /etc/resolv.conf $STAGING_DIR/etc/resolv.conf + +# Perform caching on the base image to speed up subsequent runs +chroot $STAGING_DIR apt-get update +chroot $STAGING_DIR apt-get install -y --download-only `cat files/apts/* | grep NOPRIME | cut -d\# -f1` +chroot $STAGING_DIR apt-get install -y --force-yes `cat files/apts/* | grep -v NOPRIME | cut -d\# -f1` +chroot $STAGING_DIR pip install `cat files/pips/*` +umount $STAGING_DIR && rm -rf $STAGING_DIR From 069f2f7a534b8ace5e9a6e68143c7951b65d4046 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 7 Nov 2011 16:13:03 -0600 Subject: [PATCH 2/6] +x --- tools/warm_apts_and_pips.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/warm_apts_and_pips.sh diff --git a/tools/warm_apts_and_pips.sh b/tools/warm_apts_and_pips.sh old mode 100644 new mode 100755 From 8655bf0e6efdd3d87a5ed149be2ee5e3aa8473db Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 7 Nov 2011 16:37:00 -0600 Subject: [PATCH 3/6] more checks to make sure script is run as intended --- tools/warm_apts_and_pips.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/warm_apts_and_pips.sh b/tools/warm_apts_and_pips.sh index c0b02b9..b20519f 100755 --- a/tools/warm_apts_and_pips.sh +++ b/tools/warm_apts_and_pips.sh @@ -1,5 +1,11 @@ #!/usr/bin/env bash +# echo commands +set -o xtrace + +# exit on error to stop unexpected errors +set -o errexit + # Keep track of the current directory TOOLS_DIR=$(cd $(dirname "$0") && pwd) TOP_DIR=`cd $TOOLS_DIR/..; pwd` @@ -20,6 +26,12 @@ if ! qemu-img info $1 | grep -q "file format: raw"; then exit 1 fi +# Make sure we are in the correct dir +if [ ! -d files/apts ]; then + echo "Please run this script from devstack/tools/" + exit 1 +fi + # Mount the image STAGING_DIR=`mktemp -d uec.XXXXXXXXXX` mkdir -p $STAGING_DIR From 208ae2f6aa0bf5dbc669c6fd4f2e4649c04ed039 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 7 Nov 2011 16:38:03 -0600 Subject: [PATCH 4/6] fix some comments --- tools/warm_apts_and_pips.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/warm_apts_and_pips.sh b/tools/warm_apts_and_pips.sh index b20519f..10bd4af 100755 --- a/tools/warm_apts_and_pips.sh +++ b/tools/warm_apts_and_pips.sh @@ -1,16 +1,16 @@ #!/usr/bin/env bash -# echo commands +# Echo commands set -o xtrace -# exit on error to stop unexpected errors +# Exit on error to stop unexpected errors set -o errexit # Keep track of the current directory TOOLS_DIR=$(cd $(dirname "$0") && pwd) TOP_DIR=`cd $TOOLS_DIR/..; pwd` -# cd to top of devstack +# Change dir to top of devstack cd $TOP_DIR # Echo usage From c1024d8987a9b35ef41f5a5615f9827d4b4e4468 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Wed, 9 Nov 2011 18:58:07 -0800 Subject: [PATCH 5/6] tweaks to warm script, add script to configure stack user --- tools/setup_stack_user.sh | 66 +++++++++++++++++++++++++++++++++++++ tools/warm_apts_and_pips.sh | 11 ++++--- 2 files changed, 73 insertions(+), 4 deletions(-) create mode 100755 tools/setup_stack_user.sh diff --git a/tools/setup_stack_user.sh b/tools/setup_stack_user.sh new file mode 100755 index 0000000..85d418e --- /dev/null +++ b/tools/setup_stack_user.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# Echo commands +set -o xtrace + +# Exit on error to stop unexpected errors +set -o errexit + +# Keep track of the current directory +TOOLS_DIR=$(cd $(dirname "$0") && pwd) +TOP_DIR=`cd $TOOLS_DIR/..; pwd` + +# Change dir to top of devstack +cd $TOP_DIR + +# Echo usage +usage() { + echo "Add stack user and keys" + echo "" + echo "Usage: $0 [full path to raw uec base image]" +} + +# Make sure this is a raw image +if ! qemu-img info $1 | grep -q "file format: raw"; then + usage + exit 1 +fi + +# Mount the image +DEST=/opt/stack +STAGING_DIR=/tmp/`echo $1 | sed "s/\//_/g"`.stage.user +mkdir -p $STAGING_DIR +umount $STAGING_DIR || true +sleep 1 +mount -t ext4 -o loop $1 $STAGING_DIR +mkdir -p $STAGING_DIR/$DEST + +# Create a stack user that is a member of the libvirtd group so that stack +# is able to interact with libvirt. +chroot $STAGING_DIR groupadd libvirtd || true +chroot $STAGING_DIR useradd stack -s /bin/bash -d $DEST -G libvirtd || true + +# a simple password - pass +echo stack:pass | chroot $STAGING_DIR chpasswd + +# and has sudo ability (in the future this should be limited to only what +# stack requires) +echo "stack ALL=(ALL) NOPASSWD: ALL" >> $STAGING_DIR/etc/sudoers + +# Gracefully cp only if source file/dir exists +function cp_it { + if [ -e $1 ] || [ -d $1 ]; then + cp -pRL $1 $2 + fi +} + +# Copy over your ssh keys and env if desired +cp_it ~/.ssh $STAGING_DIR/$DEST/.ssh +cp_it ~/.ssh/id_rsa.pub $STAGING_DIR/$DEST/.ssh/authorized_keys +cp_it ~/.gitconfig $STAGING_DIR/$DEST/.gitconfig +cp_it ~/.vimrc $STAGING_DIR/$DEST/.vimrc +cp_it ~/.bashrc $STAGING_DIR/$DEST/.bashrc + +# Give stack ownership over $DEST so it may do the work needed +chroot $STAGING_DIR chown -R stack $DEST + diff --git a/tools/warm_apts_and_pips.sh b/tools/warm_apts_and_pips.sh index 10bd4af..854a938 100755 --- a/tools/warm_apts_and_pips.sh +++ b/tools/warm_apts_and_pips.sh @@ -33,8 +33,10 @@ if [ ! -d files/apts ]; then fi # Mount the image -STAGING_DIR=`mktemp -d uec.XXXXXXXXXX` +STAGING_DIR=/tmp/`echo $1 | sed "s/\//_/g"`.stage mkdir -p $STAGING_DIR +umount $STAGING_DIR || true +sleep 1 mount -t ext4 -o loop $1 $STAGING_DIR # Make sure that base requirements are installed @@ -43,6 +45,7 @@ cp /etc/resolv.conf $STAGING_DIR/etc/resolv.conf # Perform caching on the base image to speed up subsequent runs chroot $STAGING_DIR apt-get update chroot $STAGING_DIR apt-get install -y --download-only `cat files/apts/* | grep NOPRIME | cut -d\# -f1` -chroot $STAGING_DIR apt-get install -y --force-yes `cat files/apts/* | grep -v NOPRIME | cut -d\# -f1` -chroot $STAGING_DIR pip install `cat files/pips/*` -umount $STAGING_DIR && rm -rf $STAGING_DIR +chroot $STAGING_DIR apt-get install -y --force-yes `cat files/apts/* | grep -v NOPRIME | cut -d\# -f1` || true +mkdir -p $STAGING_DIR/var/cache/pip +PIP_DOWNLOAD_CACHE=/var/cache/pip chroot $STAGING_DIR pip install `cat files/pips/*` || true +umount $STAGING_DIR From e228093ecda9300428a399600758580eff3b44fe Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Wed, 9 Nov 2011 22:29:22 -0800 Subject: [PATCH 6/6] some cleanup for utility scripts --- tools/setup_stack_user.sh | 10 +++++++++- tools/warm_apts_and_pips.sh | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/setup_stack_user.sh b/tools/setup_stack_user.sh index 85d418e..231a20f 100755 --- a/tools/setup_stack_user.sh +++ b/tools/setup_stack_user.sh @@ -40,9 +40,15 @@ mkdir -p $STAGING_DIR/$DEST chroot $STAGING_DIR groupadd libvirtd || true chroot $STAGING_DIR useradd stack -s /bin/bash -d $DEST -G libvirtd || true -# a simple password - pass +# Add a simple password - pass echo stack:pass | chroot $STAGING_DIR chpasswd +# Configure sudo +grep -q "^#includedir.*/etc/sudoers.d" $STAGING_DIR/etc/sudoers || + echo "#includedir /etc/sudoers.d" | sudo tee -a $STAGING_DIR/etc/sudoers +cp $TOP_DIR/files/sudo/* $STAGING_DIR/etc/sudoers.d/ +sed -e "s,%USER%,$USER,g" -i $STAGING_DIR/etc/sudoers.d/* + # and has sudo ability (in the future this should be limited to only what # stack requires) echo "stack ALL=(ALL) NOPASSWD: ALL" >> $STAGING_DIR/etc/sudoers @@ -64,3 +70,5 @@ cp_it ~/.bashrc $STAGING_DIR/$DEST/.bashrc # Give stack ownership over $DEST so it may do the work needed chroot $STAGING_DIR chown -R stack $DEST +# Unmount +umount $STAGING_DIR diff --git a/tools/warm_apts_and_pips.sh b/tools/warm_apts_and_pips.sh index 854a938..ec7e916 100755 --- a/tools/warm_apts_and_pips.sh +++ b/tools/warm_apts_and_pips.sh @@ -48,4 +48,6 @@ chroot $STAGING_DIR apt-get install -y --download-only `cat files/apts/* | grep chroot $STAGING_DIR apt-get install -y --force-yes `cat files/apts/* | grep -v NOPRIME | cut -d\# -f1` || true mkdir -p $STAGING_DIR/var/cache/pip PIP_DOWNLOAD_CACHE=/var/cache/pip chroot $STAGING_DIR pip install `cat files/pips/*` || true + +# Unmount umount $STAGING_DIR