From dd6636be060edb782e09a86364ad3b7db63c7d63 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Tue, 11 Oct 2011 20:28:39 -0500 Subject: [PATCH 01/37] Cleanup PXE boot config --- tools/build_pxe_boot.sh | 46 ++++++----------------------------------- 1 file changed, 6 insertions(+), 40 deletions(-) diff --git a/tools/build_pxe_boot.sh b/tools/build_pxe_boot.sh index 84aa43b..0653664 100755 --- a/tools/build_pxe_boot.sh +++ b/tools/build_pxe_boot.sh @@ -4,15 +4,8 @@ # build_pxe_boot.sh [-k kernel-version] destdir # # Assumes syslinux is installed -# Assumes devstack files are in `pwd`/pxe # Only needs to run as root if the destdir permissions require it -UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu/dists/natty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64 - -MEMTEST_VER=4.10 -MEMTEST_BIN=memtest86+-${MEMTEST_VER}.bin -MEMTEST_URL=http://www.memtest.org/download/${MEMTEST_VER}/ - KVER=`uname -r` if [ "$1" = "-k" ]; then KVER=$2 @@ -30,8 +23,8 @@ for i in memdisk menu.c32 pxelinux.0; do cp -p /usr/lib/syslinux/$i $DEST_DIR done -DEFAULT=$DEST_DIR/pxelinux.cfg/default -cat >$DEFAULT <$CFG <>$DEFAULT <>$CFG <>$DEFAULT <>$CFG <>$DEFAULT <>$DEFAULT <>$DEFAULT <>$CFG < Date: Tue, 11 Oct 2011 20:32:07 -0500 Subject: [PATCH 02/37] Add build_usb_boot.sh --- tools/build_usb_boot.sh | 105 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100755 tools/build_usb_boot.sh diff --git a/tools/build_usb_boot.sh b/tools/build_usb_boot.sh new file mode 100755 index 0000000..b4205b7 --- /dev/null +++ b/tools/build_usb_boot.sh @@ -0,0 +1,105 @@ +#!/bin/bash -e +# build_usb_boot.sh - Create a syslinux boot environment +# +# build_usb_boot.sh [-k kernel-version] destdev +# +# Assumes syslinux is installed +# Assumes devstack files are in `pwd`/pxe +# Needs to run as root + +KVER=`uname -r` +if [ "$1" = "-k" ]; then + KVER=$2 + shift;shift +fi + +DEST_DIR=${1:-/tmp/syslinux-boot} +PXEDIR=${PXEDIR:-/var/cache/devstack/pxe} +OPWD=`pwd` +PROGDIR=`dirname $0` + +if [ -b $DEST_DIR ]; then + # We have a block device, install syslinux and mount it + DEST_DEV=$DEST_DIR + DEST_DIR=`mktemp -d mntXXXXXX` + + # Install syslinux on the device + syslinux --install --directory syslinux $DEST_DEV + + mount $DEST_DEV $DEST_DIR +else + # We have a directory (for sanity checking output) + DEST_DEV="" + if [ ! -d $DEST_DIR/syslinux ]; then + mkdir -p $DEST_DIR/syslinux + fi +fi + +# Get some more stuff from syslinux +for i in memdisk menu.c32; do + cp -p /usr/lib/syslinux/$i $DEST_DIR/syslinux +done + +CFG=$DEST_DIR/syslinux/syslinux.cfg +cat >$CFG <>$CFG <>$CFG <>$CFG < Date: Tue, 11 Oct 2011 20:34:07 -0500 Subject: [PATCH 03/37] Cleanups --- tools/build_usb_boot.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/build_usb_boot.sh b/tools/build_usb_boot.sh index b4205b7..ac49848 100755 --- a/tools/build_usb_boot.sh +++ b/tools/build_usb_boot.sh @@ -4,7 +4,6 @@ # build_usb_boot.sh [-k kernel-version] destdev # # Assumes syslinux is installed -# Assumes devstack files are in `pwd`/pxe # Needs to run as root KVER=`uname -r` @@ -95,7 +94,6 @@ cat >>$CFG < Date: Wed, 12 Oct 2011 07:13:13 +0000 Subject: [PATCH 04/37] Prompt users for passwords, and write those passwords to localrc --- stack.sh | 76 ++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/stack.sh b/stack.sh index f6bf534..cdeb8a0 100755 --- a/stack.sh +++ b/stack.sh @@ -91,14 +91,14 @@ set -o xtrace # This script is customizable through setting environment variables. If you # want to override a setting you can either:: # -# export MYSQL_PASS=anothersecret +# export MYSQL_PASSWORD=anothersecret # ./stack.sh # -# You can also pass options on a single line ``MYSQL_PASS=simple ./stack.sh`` +# You can also pass options on a single line ``MYSQL_PASSWORD=simple ./stack.sh`` # # Additionally, you can put any local variables into a ``localrc`` file, like:: # -# MYSQL_PASS=anothersecret +# MYSQL_PASSWORD=anothersecret # MYSQL_USER=hellaroot # # We try to have sensible defaults, so you should be able to run ``./stack.sh`` @@ -111,7 +111,7 @@ set -o xtrace # # If ``localrc`` exists, then ``stackrc`` will load those settings. This is # useful for changing a branch or repostiory to test other versions. Also you -# can store your other settings like **MYSQL_PASS** or **ADMIN_PASSWORD** instead +# can store your other settings like **MYSQL_PASSWORD** or **ADMIN_PASSWORD** instead # of letting devstack generate random ones for you. source ./stackrc @@ -146,6 +146,43 @@ if [ ! -n "$HOST_IP" ]; then HOST_IP=`LC_ALL=C /sbin/ifconfig | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'` fi +# Generic helper to configure passwords +function read_password { + set +o xtrace + var=$1; msg=$2 + pw=${!var} + + # If the password is not defined yet, proceed to prompt user for a password. + if [ ! $pw ]; then + # If there is no localrc file, create one + if [ ! -e localrc ]; then + touch localrc + fi + + # Presumably if we got this far it can only be that our localrc is missing + # the required password. Prompt user for a password and write to localrc. + if ! grep -q $1 localrc; then + echo '' + echo '################################################################################' + echo $msg + echo '################################################################################' + echo "This value will be written to your localrc file." + echo "It is probably best to avoid spaces and weird characters." + echo "If you leave this blank, a random default value will be used." + echo "Enter a password now:" + read $var + pw=${!var} + if [ ! $pw ]; then + pw=`openssl rand -hex 10` + fi + eval "$var=$pw" + echo "$var=$pw" >> localrc + fi + fi + set -o xtrace +} + + # Nova Network Configuration # -------------------------- @@ -194,31 +231,32 @@ FLAT_INTERFACE=${FLAT_INTERFACE:-eth0} # By default this script will install and configure MySQL. If you want to # use an existing server, you can pass in the user/password/host parameters. -# You will need to send the same ``MYSQL_PASS`` to every host if you are doing +# You will need to send the same ``MYSQL_PASSWORD`` to every host if you are doing # a multi-node devstack installation. MYSQL_USER=${MYSQL_USER:-root} -MYSQL_PASS=${MYSQL_PASS:-`openssl rand -hex 12`} +read_password MYSQL_PASSWORD "ENTER A PASSWORD TO USE FOR MYSQL." MYSQL_HOST=${MYSQL_HOST:-localhost} # don't specify /db in this string, so we can use it for multiple services -BASE_SQL_CONN=${BASE_SQL_CONN:-mysql://$MYSQL_USER:$MYSQL_PASS@$MYSQL_HOST} +BASE_SQL_CONN=${BASE_SQL_CONN:-mysql://$MYSQL_USER:$MYSQL_PASSWORD@$MYSQL_HOST} # Rabbit connection info RABBIT_HOST=${RABBIT_HOST:-localhost} RABBIT_PASSWORD=${RABBIT_PASSWORD:-`openssl rand -hex 12`} +read_password RABBIT_PASSWORD "ENTER A PASSWORD TO USE FOR RABBIT." # Glance connection info. Note the port must be specified. GLANCE_HOSTPORT=${GLANCE_HOSTPORT:-$HOST_IP:9292} + # Keystone # -------- # Service Token - Openstack components need to have an admin token # to validate user tokens. -SERVICE_TOKEN=${SERVICE_TOKEN:-`openssl rand -hex 12`} +read_password SERVICE_TOKEN "ENTER A SERVICE_TOKEN TO USE FOR THE SERVICE ADMIN TOKEN." # Dash currently truncates usernames and passwords at 20 characters -# so use 10 bytes -ADMIN_PASSWORD=${ADMIN_PASSWORD:-`openssl rand -hex 10`} +read_password ADMIN_PASSWORD "ENTER A PASSWORD TO USE FOR DASH AND KEYSTONE (20 CHARS OR LESS)." # Install Packages @@ -301,15 +339,15 @@ if [[ "$ENABLED_SERVICES" =~ "mysql" ]]; then # Seed configuration with mysql password so that apt-get install doesn't # prompt us for a password upon install. cat < Date: Wed, 12 Oct 2011 07:17:11 +0000 Subject: [PATCH 05/37] use hard path to localrc --- stack.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stack.sh b/stack.sh index cdeb8a0..32b7a0c 100755 --- a/stack.sh +++ b/stack.sh @@ -43,6 +43,9 @@ if [ ! -d $FILES ]; then exit 1 fi +# Keep track of the current working directory. +CWD=`pwd` + # OpenStack is designed to be run as a regular user (Dashboard will fail to run # as root, since apache refused to startup serve content from root user). If # stack.sh is run as root, it automatically creates a stack user with @@ -152,6 +155,8 @@ function read_password { var=$1; msg=$2 pw=${!var} + localrc=$CWD/localrc + # If the password is not defined yet, proceed to prompt user for a password. if [ ! $pw ]; then # If there is no localrc file, create one From 66b8bbcbd85cefb044a3be54cd84d72fcb6b63be Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Wed, 12 Oct 2011 07:21:41 +0000 Subject: [PATCH 06/37] update msg --- stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.sh b/stack.sh index 32b7a0c..68e5afa 100755 --- a/stack.sh +++ b/stack.sh @@ -171,7 +171,7 @@ function read_password { echo '################################################################################' echo $msg echo '################################################################################' - echo "This value will be written to your localrc file." + echo "This value will be written to your localrc file so you don't have to enter it again." echo "It is probably best to avoid spaces and weird characters." echo "If you leave this blank, a random default value will be used." echo "Enter a password now:" From c4e47ab8586069dea52b86fa939ad8a8deeacc2e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 7 Oct 2011 13:29:29 +0000 Subject: [PATCH 07/37] simplify script duration output Bash contains a variable 'SECONDS' that indicates how long the current shell has been alive. It seems sane to just use that to indicate to the user how long the script took. --- stack.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/stack.sh b/stack.sh index f6bf534..fbc75b5 100755 --- a/stack.sh +++ b/stack.sh @@ -20,9 +20,6 @@ # Sanity Check # ============ -# Record the start time. This allows us to print how long this script takes to run. -START_TIME=`python -c "import time; print time.time()"` - # Warn users who aren't on natty, but allow them to override check and attempt # installation with ``FORCE=yes ./stack`` if ! grep -q natty /etc/lsb-release; then @@ -666,7 +663,5 @@ fi # Fin # === -# End our timer and give a timing summary -END_TIME=`python -c "import time; print time.time()"` -ELAPSED=`python -c "print $END_TIME - $START_TIME"` -echo "stack.sh completed in $ELAPSED seconds." +# indicate how long this took to run (bash maintained variable 'SECONDS') +echo "stack.sh completed in $SECONDS seconds." From 7c481189bdeda3c79029b1b47d8edd1bf8bd50fa Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 7 Oct 2011 13:50:21 +0000 Subject: [PATCH 08/37] log output of stack.sh to logfile in current dir use the variable LOGFILE to log stack.sh output for debugging --- stack.sh | 105 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 62 insertions(+), 43 deletions(-) diff --git a/stack.sh b/stack.sh index fbc75b5..2e823ea 100755 --- a/stack.sh +++ b/stack.sh @@ -40,48 +40,6 @@ if [ ! -d $FILES ]; then exit 1 fi -# OpenStack is designed to be run as a regular user (Dashboard will fail to run -# as root, since apache refused to startup serve content from root user). If -# stack.sh is run as root, it automatically creates a stack user with -# sudo privileges and runs as that user. - -if [[ $EUID -eq 0 ]]; then - echo "You are running this script as root." - - # since this script runs as a normal user, we need to give that user - # ability to run sudo - apt-get update - apt-get install -qqy sudo - - if ! getent passwd | grep -q stack; then - echo "Creating a user called stack" - useradd -U -G sudo -s /bin/bash -m stack - fi - echo "Giving stack user passwordless sudo priviledges" - echo "stack ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers - - echo "Copying files to stack user" - cp -r -f `pwd` /home/stack/ - THIS_DIR=$(basename $(dirname $(readlink -f $0))) - chown -R stack /home/stack/$THIS_DIR - echo "Running the script as stack in 3 seconds..." - sleep 3 - if [[ "$SHELL_AFTER_RUN" != "no" ]]; then - exec su -c "cd /home/stack/$THIS_DIR/; bash stack.sh; bash" stack - else - exec su -c "cd /home/stack/$THIS_DIR/; bash stack.sh" stack - fi - exit 0 -fi - -# So that errors don't compound we exit on any errors so you see only the -# first error that occured. -set -o errexit - -# Print the commands being run so that we can see the command that triggers -# an error. It is also useful for following allowing as the install occurs. -set -o xtrace - # Settings # ======== @@ -112,6 +70,56 @@ set -o xtrace # of letting devstack generate random ones for you. source ./stackrc +LOGFILE=${LOGFILE:-"$PWD/stack.sh.$$.log"} +( +# So that errors don't compound we exit on any errors so you see only the +# first error that occured. +trap failed ERR +failed() { + local r=$? + set +o xtrace + [ -n "$LOGFILE" ] && echo "${0##*/} failed: full log in $LOGFILE" + exit $r +} + +# Print the commands being run so that we can see the command that triggers +# an error. It is also useful for following along as the install occurs. +set -o xtrace + +# OpenStack is designed to be run as a regular user (Dashboard will fail to run +# as root, since apache refused to startup serve content from root user). If +# stack.sh is run as root, it automatically creates a stack user with +# sudo privileges and runs as that user. + +if [[ $EUID -eq 0 ]]; then + echo "You are running this script as root." + + # since this script runs as a normal user, we need to give that user + # ability to run sudo + apt-get update + apt-get install -y sudo + + if ! getent passwd | grep -q stack; then + echo "Creating a user called stack" + useradd -U -G sudo -s /bin/bash -m stack + fi + echo "Giving stack user passwordless sudo priviledges" + echo "stack ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + + echo "Copying files to stack user" + cp -r -f `pwd` /home/stack/ + THIS_DIR=$(basename $(dirname $(readlink -f $0))) + chown -R stack /home/stack/$THIS_DIR + echo "Running the script as stack in 3 seconds..." + sleep 3 + if [[ "$SHELL_AFTER_RUN" != "no" ]]; then + exec su -c "cd /home/stack/$THIS_DIR/; bash stack.sh; bash" stack + else + exec su -c "cd /home/stack/$THIS_DIR/; bash stack.sh" stack + fi + exit 0 +fi + # Destination path for installation ``DEST`` DEST=${DEST:-/opt/stack} sudo mkdir -p $DEST @@ -663,5 +671,16 @@ fi # Fin # === + +) 2>&1 | tee "${LOGFILE}" + +# because of the way pipes work, the left side of the pipe may +# have failed, but 'tee' will succeed. We need to check that. +for ret in "${PIPESTATUS[@]}"; do + [ $ret -eq 0 ] || exit $ret +done + # indicate how long this took to run (bash maintained variable 'SECONDS') -echo "stack.sh completed in $SECONDS seconds." +echo "stack.sh completed in $SECONDS seconds." | tee -a "${LOGFILE}" + +exit 0 From 0d2145a0f37da869fcc4754e41b1b8e0618679ea Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 7 Oct 2011 15:18:10 +0000 Subject: [PATCH 09/37] some improvements to 'create stack user' path * Increase the timeout from 3 to 10, so user has a bigger chance to kill the script if being run interactively before a 'stack' user is created. * explicitly ask 'getent' for the stack user rather than getting all users and grepping (which would match an "openstack" user, or a user named "bob.stack") * use $PWD rather than `pwd` * create file in sudoers.d rather than modifying /etc/sudoers. --- stack.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/stack.sh b/stack.sh index 2e823ea..9881ede 100755 --- a/stack.sh +++ b/stack.sh @@ -93,29 +93,31 @@ set -o xtrace if [[ $EUID -eq 0 ]]; then echo "You are running this script as root." + echo "In 10 seconds, we will create a user 'stack' and run as that user" + sleep 10 # since this script runs as a normal user, we need to give that user # ability to run sudo apt-get update apt-get install -y sudo - if ! getent passwd | grep -q stack; then + if ! getent passwd stack >/dev/null; then echo "Creating a user called stack" useradd -U -G sudo -s /bin/bash -m stack fi + echo "Giving stack user passwordless sudo priviledges" - echo "stack ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + ( umask 226 && echo "stack ALL=(ALL) NOPASSWD: ALL" \ + >> /etc/sudoers.d/50_stack_sh ) echo "Copying files to stack user" - cp -r -f `pwd` /home/stack/ - THIS_DIR=$(basename $(dirname $(readlink -f $0))) - chown -R stack /home/stack/$THIS_DIR - echo "Running the script as stack in 3 seconds..." - sleep 3 + STACK_DIR="/home/stack/${PWD%/*}" + cp -r -f "$PWD" "$STACK_DIR" + chown -R stack "$STACK_DIR" if [[ "$SHELL_AFTER_RUN" != "no" ]]; then - exec su -c "cd /home/stack/$THIS_DIR/; bash stack.sh; bash" stack + exec su -ec "cd $STACK_DIR; bash stack.sh; bash" stack else - exec su -c "cd /home/stack/$THIS_DIR/; bash stack.sh" stack + exec su -ec "cd $STACK_DIR; bash stack.sh" stack fi exit 0 fi From d67a18bb0716b6dca1cfcfced39c65ac3b41a6e3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 7 Oct 2011 14:44:38 +0000 Subject: [PATCH 10/37] make sure apt-get update is run Previously, the only path that would only ensure that 'apt-get update' was run was when the stack user was created. Archives can be out of date, apt-get should be run. --- stack.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/stack.sh b/stack.sh index 9881ede..045bcc9 100755 --- a/stack.sh +++ b/stack.sh @@ -235,6 +235,7 @@ ADMIN_PASSWORD=${ADMIN_PASSWORD:-`openssl rand -hex 10`} # install apt requirements +sudo apt-get update sudo apt-get install -qqy `cat $FILES/apts/* | cut -d\# -f1 | grep -Ev "mysql-server|rabbitmq-server"` # install python requirements From b94f4bf35b9c2acd469f839515a96f9c30a45331 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 7 Oct 2011 14:51:07 +0000 Subject: [PATCH 11/37] move 'how to use cloud' info outside of xtrace make the successful run output end with more clear messages. By moving them outside of the "do it all" path, the xtrace output will not mix with echo statements. --- stack.sh | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/stack.sh b/stack.sh index 045bcc9..5f13671 100755 --- a/stack.sh +++ b/stack.sh @@ -654,6 +654,16 @@ if [[ "$ENABLED_SERVICES" =~ "g-reg" ]]; then fi +# Fin +# === + + +) 2>&1 | tee "${LOGFILE}" + +# Check that the left side of the above pipe succeeded +for ret in "${PIPESTATUS[@]}"; do [ $ret -eq 0 ] || exit $ret; done + +( # Using the cloud # =============== @@ -671,19 +681,7 @@ if [[ "$ENABLED_SERVICES" =~ "key" ]]; then echo "the password: $ADMIN_PASSWORD" fi -# Fin -# === - - -) 2>&1 | tee "${LOGFILE}" - -# because of the way pipes work, the left side of the pipe may -# have failed, but 'tee' will succeed. We need to check that. -for ret in "${PIPESTATUS[@]}"; do - [ $ret -eq 0 ] || exit $ret -done - # indicate how long this took to run (bash maintained variable 'SECONDS') -echo "stack.sh completed in $SECONDS seconds." | tee -a "${LOGFILE}" +echo "stack.sh completed in $SECONDS seconds." -exit 0 +) | tee -a "$LOGFILE" From 8ab1ade42ed3b4d4bda00c3dc5a6110c05c201e4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 7 Oct 2011 21:03:16 -0400 Subject: [PATCH 12/37] instead of 'newgrp' and a pipe, use 'sg' sg and newgrp basically do the same thing, but sg takes a command to execute rather than just executing sh. This just seems cleaner. --- stack.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stack.sh b/stack.sh index 5f13671..516c920 100755 --- a/stack.sh +++ b/stack.sh @@ -592,9 +592,8 @@ fi # have to do a little more than that in our script. Since we add the group # ``libvirtd`` to our user in this script, when nova-compute is run it is # within the context of our original shell (so our groups won't be updated). -# We can send the command nova-compute to the ``newgrp`` command to execute -# in a specific context. -screen_it n-cpu "cd $NOVA_DIR && echo $NOVA_DIR/bin/nova-compute | newgrp libvirtd" +# Use 'sg' to execute nova-compute as a member of the libvirtd group. +screen_it n-cpu "cd $NOVA_DIR && sg libvirtd $NOVA_DIR/bin/nova-compute" screen_it n-net "cd $NOVA_DIR && $NOVA_DIR/bin/nova-network" screen_it n-sch "cd $NOVA_DIR && $NOVA_DIR/bin/nova-scheduler" screen_it n-vnc "cd $NOVNC_DIR && ./utils/nova-wsproxy.py 6080 --web . --flagfile=../nova/bin/nova.conf" From f9da5081222d2c353e577cb83c538912bca26972 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 7 Oct 2011 21:28:00 -0400 Subject: [PATCH 13/37] move setting variables and run-as-stack-user code Change the over all flow so that the default settings are filled in in the outside shell (not inside the '()' and 'tee'). This way, those variables pass through to the subshell, but they're also available in the parent shell at the end for outputting status. Also, exit failure rather than success if 'exec' to run as stack user failed. --- stack.sh | 99 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/stack.sh b/stack.sh index 516c920..a4cbc2e 100755 --- a/stack.sh +++ b/stack.sh @@ -40,52 +40,6 @@ if [ ! -d $FILES ]; then exit 1 fi -# Settings -# ======== - -# This script is customizable through setting environment variables. If you -# want to override a setting you can either:: -# -# export MYSQL_PASS=anothersecret -# ./stack.sh -# -# You can also pass options on a single line ``MYSQL_PASS=simple ./stack.sh`` -# -# Additionally, you can put any local variables into a ``localrc`` file, like:: -# -# MYSQL_PASS=anothersecret -# MYSQL_USER=hellaroot -# -# We try to have sensible defaults, so you should be able to run ``./stack.sh`` -# in most cases. -# -# We our settings from ``stackrc``. This file is distributed with devstack and -# contains locations for what repositories to use. If you want to use other -# repositories and branches, you can add your own settings with another file -# called ``localrc`` -# -# If ``localrc`` exists, then ``stackrc`` will load those settings. This is -# useful for changing a branch or repostiory to test other versions. Also you -# can store your other settings like **MYSQL_PASS** or **ADMIN_PASSWORD** instead -# of letting devstack generate random ones for you. -source ./stackrc - -LOGFILE=${LOGFILE:-"$PWD/stack.sh.$$.log"} -( -# So that errors don't compound we exit on any errors so you see only the -# first error that occured. -trap failed ERR -failed() { - local r=$? - set +o xtrace - [ -n "$LOGFILE" ] && echo "${0##*/} failed: full log in $LOGFILE" - exit $r -} - -# Print the commands being run so that we can see the command that triggers -# an error. It is also useful for following along as the install occurs. -set -o xtrace - # OpenStack is designed to be run as a regular user (Dashboard will fail to run # as root, since apache refused to startup serve content from root user). If # stack.sh is run as root, it automatically creates a stack user with @@ -119,13 +73,42 @@ if [[ $EUID -eq 0 ]]; then else exec su -ec "cd $STACK_DIR; bash stack.sh" stack fi - exit 0 + exit 1 fi + +# Settings +# ======== + +# This script is customizable through setting environment variables. If you +# want to override a setting you can either:: +# +# export MYSQL_PASS=anothersecret +# ./stack.sh +# +# You can also pass options on a single line ``MYSQL_PASS=simple ./stack.sh`` +# +# Additionally, you can put any local variables into a ``localrc`` file, like:: +# +# MYSQL_PASS=anothersecret +# MYSQL_USER=hellaroot +# +# We try to have sensible defaults, so you should be able to run ``./stack.sh`` +# in most cases. +# +# We our settings from ``stackrc``. This file is distributed with devstack and +# contains locations for what repositories to use. If you want to use other +# repositories and branches, you can add your own settings with another file +# called ``localrc`` +# +# If ``localrc`` exists, then ``stackrc`` will load those settings. This is +# useful for changing a branch or repostiory to test other versions. Also you +# can store your other settings like **MYSQL_PASS** or **ADMIN_PASSWORD** instead +# of letting devstack generate random ones for you. +source ./stackrc + # Destination path for installation ``DEST`` DEST=${DEST:-/opt/stack} -sudo mkdir -p $DEST -sudo chown `whoami` $DEST # Set the destination directories for openstack projects NOVA_DIR=$DEST/nova @@ -227,6 +210,24 @@ SERVICE_TOKEN=${SERVICE_TOKEN:-`openssl rand -hex 12`} # so use 10 bytes ADMIN_PASSWORD=${ADMIN_PASSWORD:-`openssl rand -hex 10`} +LOGFILE=${LOGFILE:-"$PWD/stack.sh.$$.log"} +( +# So that errors don't compound we exit on any errors so you see only the +# first error that occured. +trap failed ERR +failed() { + local r=$? + set +o xtrace + [ -n "$LOGFILE" ] && echo "${0##*/} failed: full log in $LOGFILE" + exit $r +} + +# Print the commands being run so that we can see the command that triggers +# an error. It is also useful for following along as the install occurs. +set -o xtrace + +sudo mkdir -p $DEST +sudo chown `whoami` $DEST # Install Packages # ================ From b4db225494a2c8d19db523c48fc48cbaee0402a0 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Wed, 12 Oct 2011 14:08:08 -0700 Subject: [PATCH 14/37] No need to check localrc for password. Also use TOP_DIR as described by smoser. --- stack.sh | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/stack.sh b/stack.sh index 68e5afa..4fbfa28 100755 --- a/stack.sh +++ b/stack.sh @@ -43,8 +43,8 @@ if [ ! -d $FILES ]; then exit 1 fi -# Keep track of the current working directory. -CWD=`pwd` +# Keep track of the current devstack directory. +TOP_DIR=$(cd $(dirname "$0") && pwd) # OpenStack is designed to be run as a regular user (Dashboard will fail to run # as root, since apache refused to startup serve content from root user). If @@ -155,34 +155,32 @@ function read_password { var=$1; msg=$2 pw=${!var} - localrc=$CWD/localrc + localrc=$TOP_DIR/localrc # If the password is not defined yet, proceed to prompt user for a password. if [ ! $pw ]; then # If there is no localrc file, create one - if [ ! -e localrc ]; then - touch localrc + if [ ! -e $localrc ]; then + touch $localrc fi # Presumably if we got this far it can only be that our localrc is missing # the required password. Prompt user for a password and write to localrc. - if ! grep -q $1 localrc; then - echo '' - echo '################################################################################' - echo $msg - echo '################################################################################' - echo "This value will be written to your localrc file so you don't have to enter it again." - echo "It is probably best to avoid spaces and weird characters." - echo "If you leave this blank, a random default value will be used." - echo "Enter a password now:" - read $var - pw=${!var} - if [ ! $pw ]; then - pw=`openssl rand -hex 10` - fi - eval "$var=$pw" - echo "$var=$pw" >> localrc + echo '' + echo '################################################################################' + echo $msg + echo '################################################################################' + echo "This value will be written to your localrc file so you don't have to enter it again." + echo "It is probably best to avoid spaces and weird characters." + echo "If you leave this blank, a random default value will be used." + echo "Enter a password now:" + read $var + pw=${!var} + if [ ! $pw ]; then + pw=`openssl rand -hex 10` fi + eval "$var=$pw" + echo "$var=$pw" >> $localrc fi set -o xtrace } From 102e440e2b4908051e314f7d88730270bc89f1fd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Oct 2011 20:00:34 -0400 Subject: [PATCH 15/37] fix bad usage of 'su'. instead use 'set -e'. I had had tried to use -e argument to su, thinking that it would go through to sh, but it does not. instead, just use 'set -e' --- stack.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stack.sh b/stack.sh index a4cbc2e..2d76875 100755 --- a/stack.sh +++ b/stack.sh @@ -69,9 +69,9 @@ if [[ $EUID -eq 0 ]]; then cp -r -f "$PWD" "$STACK_DIR" chown -R stack "$STACK_DIR" if [[ "$SHELL_AFTER_RUN" != "no" ]]; then - exec su -ec "cd $STACK_DIR; bash stack.sh; bash" stack + exec su -c "set -e; cd $STACK_DIR; bash stack.sh; bash" stack else - exec su -ec "cd $STACK_DIR; bash stack.sh" stack + exec su -c "set -e; cd $STACK_DIR; bash stack.sh" stack fi exit 1 fi From df47967da46462e5f0b444a48d9019988449f94b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Oct 2011 20:15:50 -0400 Subject: [PATCH 16/37] fix STACK_DIR variable --- stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.sh b/stack.sh index 2d76875..7ef13a5 100755 --- a/stack.sh +++ b/stack.sh @@ -65,7 +65,7 @@ if [[ $EUID -eq 0 ]]; then >> /etc/sudoers.d/50_stack_sh ) echo "Copying files to stack user" - STACK_DIR="/home/stack/${PWD%/*}" + STACK_DIR="/home/stack/${PWD##*/}" cp -r -f "$PWD" "$STACK_DIR" chown -R stack "$STACK_DIR" if [[ "$SHELL_AFTER_RUN" != "no" ]]; then From 55c9d3f13dac9e4073d941cdbe55532f3f3f5279 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Oct 2011 20:19:46 -0400 Subject: [PATCH 17/37] fix writing of 50_stack_sh in sudoers --- stack.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stack.sh b/stack.sh index 7ef13a5..b43f3cc 100755 --- a/stack.sh +++ b/stack.sh @@ -61,8 +61,8 @@ if [[ $EUID -eq 0 ]]; then fi echo "Giving stack user passwordless sudo priviledges" - ( umask 226 && echo "stack ALL=(ALL) NOPASSWD: ALL" \ - >> /etc/sudoers.d/50_stack_sh ) + ( umask 226 && echo "stack ALL=(ALL) NOPASSWD:ALL" \ + > /etc/sudoers.d/50_stack_sh ) echo "Copying files to stack user" STACK_DIR="/home/stack/${PWD##*/}" From 4bec581e4e5a15c204f1042f9ecd4701c33ac4ca Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Oct 2011 20:32:16 -0400 Subject: [PATCH 18/37] add includedir entry to sudoers if its not there --- stack.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stack.sh b/stack.sh index b43f3cc..38560e6 100755 --- a/stack.sh +++ b/stack.sh @@ -61,6 +61,9 @@ if [[ $EUID -eq 0 ]]; then fi echo "Giving stack user passwordless sudo priviledges" + # natty uec images sudoers does not have a '#includedir'. add one. + grep -q "^#includedir.*/etc/sudoers.d" /etc/sudoers || + echo "#includedir /etc/sudoers.d" >> /etc/sudoers ( umask 226 && echo "stack ALL=(ALL) NOPASSWD:ALL" \ > /etc/sudoers.d/50_stack_sh ) From b69d6ce529be9e498dfc6d6b982713de4bd3bd4a Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 13 Oct 2011 10:36:00 -0700 Subject: [PATCH 19/37] update readme with note about gh-pages --- README | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README b/README index 61119e9..82ef7fb 100644 --- a/README +++ b/README @@ -7,6 +7,8 @@ Tool to quickly deploy openstack dev environments. * To make it easier for developers to dive into openstack so that they can productively contribute without having to understand every part of the system at once * To make it easy to prototype cross-project features +Read more at http://devstack.org (built from the gh-pages branch) + Be sure to carefully read these scripts before you run them as they install software and may alter your networking configuration. # To start a dev cloud on your local machine (installing on a dedicated vm is safer!): @@ -32,7 +34,6 @@ You can tweak environment variables by creating file name 'localrc' should you n * Add python-novaclient cli support * syslog -* allow rabbit connection to be specified via environment variables with sensible defaults * Add volume support * Add quantum support From 906d586bb082df17da99213d479f1e0f02139e28 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 13 Oct 2011 10:37:45 -0700 Subject: [PATCH 20/37] move readme to markdown --- README => README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README => README.md (100%) diff --git a/README b/README.md similarity index 100% rename from README rename to README.md From d5d5b680f8ebcc17b9cf2cc9e32af62396a04406 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Thu, 13 Oct 2011 18:45:42 +0100 Subject: [PATCH 21/37] Set sensible default for mysql client Using $HOME/.my.cnf to make easier for user to connect to mysql client. --- stack.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/stack.sh b/stack.sh index 38560e6..8c5b7ee 100755 --- a/stack.sh +++ b/stack.sh @@ -318,6 +318,16 @@ mysql-server-5.1 mysql-server/root_password_again password $MYSQL_PASS mysql-server-5.1 mysql-server/start_on_boot boolean true MYSQL_PRESEED + if [[ ! -e $HOME/.my.cnf ]]; then + cat <$HOME/.my.cnf +[client] +user=$MYSQL_USER +password=$MYSQL_PASS +host=$MYSQL_HOST +EOF + chmod 0600 $HOME/.my.cnf + fi + # Install and start mysql-server sudo apt-get -y -q install mysql-server # Update the DB to give user ‘$MYSQL_USER’@’%’ full control of the all databases: From 2e536a3faf3d699ef255b74793b4591779738624 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 13 Oct 2011 11:40:16 -0700 Subject: [PATCH 22/37] add some docs --- stack.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stack.sh b/stack.sh index 8c5b7ee..0b34790 100755 --- a/stack.sh +++ b/stack.sh @@ -318,6 +318,9 @@ mysql-server-5.1 mysql-server/root_password_again password $MYSQL_PASS mysql-server-5.1 mysql-server/start_on_boot boolean true MYSQL_PRESEED + # while ``.my.cnf`` is not needed for openstack to function, it is useful + # as it allows you to access the mysql databases via ``mysql nova`` instead + # of having to specify the username/password each time. if [[ ! -e $HOME/.my.cnf ]]; then cat <$HOME/.my.cnf [client] From cf145b77e4dbe675108c2c7b788e7505d5818fed Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Thu, 13 Oct 2011 15:07:36 -0700 Subject: [PATCH 23/37] merge trunk, and also make sure localrc exists before running build_lxc.sh --- build_lxc.sh | 9 ++++++++- stack.sh | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/build_lxc.sh b/build_lxc.sh index 643da7e..8ca3c41 100755 --- a/build_lxc.sh +++ b/build_lxc.sh @@ -11,6 +11,13 @@ if ! grep -q natty /etc/lsb-release; then echo "WARNING: this script has only been tested on natty" fi +# Abort if localrc is not set +if [ ! -e ./localrc ]; then + echo "You must have a localrc with ALL necessary passwords defined before proceeding." + echo "See stack.sh for required passwords." + exit 1 +fi + # Source params source ./stackrc @@ -248,4 +255,4 @@ while [ ! -e "$ROOTFS/$DEST/run.sh.log" ]; do sleep 1 done -tail -F $ROOTFS/$DEST/run.sh.log +tail -F $ROOTFS/$DEST/devstack/stack.sh.*.log diff --git a/stack.sh b/stack.sh index 97c39f8..dc151a6 100755 --- a/stack.sh +++ b/stack.sh @@ -366,7 +366,7 @@ MYSQL_PRESEED cat <$HOME/.my.cnf [client] user=$MYSQL_USER -password=$MYSQL_PASS +password=$MYSQL_PASSWORD host=$MYSQL_HOST EOF chmod 0600 $HOME/.my.cnf From d1b1cdb04c4ba4e892f4eda722cf3f1daa260865 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Thu, 13 Oct 2011 15:25:38 -0700 Subject: [PATCH 24/37] tail run.sh.log instead of the stack.sh.log --- build_lxc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_lxc.sh b/build_lxc.sh index 8ca3c41..65a06cb 100755 --- a/build_lxc.sh +++ b/build_lxc.sh @@ -255,4 +255,4 @@ while [ ! -e "$ROOTFS/$DEST/run.sh.log" ]; do sleep 1 done -tail -F $ROOTFS/$DEST/devstack/stack.sh.*.log +tail -F $ROOTFS/$DEST/run.sh.log From f2b5f8856f98f72fbdaac536387c2c3672dd5e64 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Thu, 13 Oct 2011 16:44:42 -0700 Subject: [PATCH 25/37] starting work to extract image configuration so it isn't hard coded --- stack.sh | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/stack.sh b/stack.sh index 0b34790..3330342 100755 --- a/stack.sh +++ b/stack.sh @@ -637,36 +637,19 @@ if [[ "$ENABLED_SERVICES" =~ "g-reg" ]]; then # Downloads the image (ami/aki/ari style), then extracts it. Upon extraction # we upload to glance with the glance cli tool. TTY is a stripped down # version of ubuntu. - if [ ! -f $FILES/tty.tgz ]; then - wget -c http://images.ansolabs.com/tty.tgz -O $FILES/tty.tgz + IMAGE_URL=http://smoser.brickies.net/ubuntu/ttylinux-uec/ttylinux-uec-amd64-11.2_2.6.35-15_1.tar.gz + IMAGE_FNAME=`echo "$IMAGE_URL" | python -c "import sys; print sys.stdin.read().split('/')[-1]"` + IMAGE_NAME=`echo "$IMAGE_FNAME" | python -c "import sys; print sys.stdin.read().split('.tar.gz')[0].split('.tgz')[0]"` + if [ ! -f $FILES/$IMAGE_FNAME ]; then + wget -c $IMAGE_URL -O $FILES/$IMAGE_FNAME fi # extract ami-tty/image, aki-tty/image & ari-tty/image - tar -zxf $FILES/tty.tgz -C $FILES/images + tar -zxf $FILES/$IMAGE_FNAME -C $FILES/images - # Use glance client to add the kernel, ramdisk and finally the root - # filesystem. We parse the results of the uploads to get glance IDs of the - # ramdisk and kernel and use them for the root filesystem. - RVAL=`glance add -A $SERVICE_TOKEN name="tty-kernel" is_public=true container_format=aki disk_format=aki < $FILES/images/aki-tty/image` + RVAL=`glance add -A $SERVICE_TOKEN name="$IMAGE_NAME-kernel" is_public=true container_format=aki disk_format=aki < $FILES/images/$IMAGE_NAME-vmlinuz*` KERNEL_ID=`echo $RVAL | cut -d":" -f2 | tr -d " "` - RVAL=`glance add -A $SERVICE_TOKEN name="tty-ramdisk" is_public=true container_format=ari disk_format=ari < $FILES/images/ari-tty/image` - RAMDISK_ID=`echo $RVAL | cut -d":" -f2 | tr -d " "` - glance add -A $SERVICE_TOKEN name="tty" is_public=true container_format=ami disk_format=ami kernel_id=$KERNEL_ID ramdisk_id=$RAMDISK_ID < $FILES/images/ami-tty/image - - # Ubuntu 11.04 aka Natty - # ---------------------- - - # Downloaded from ubuntu enterprise cloud images. This - # image doesn't use the ramdisk functionality - if [ ! -f $FILES/natty.tgz ]; then - wget -c http://uec-images.ubuntu.com/natty/current/natty-server-cloudimg-amd64.tar.gz -O $FILES/natty.tgz - fi - - tar -zxf $FILES/natty.tgz -C $FILES/images - - RVAL=`glance add -A $SERVICE_TOKEN name="uec-natty-kernel" is_public=true container_format=aki disk_format=aki < $FILES/images/natty-server-cloudimg-amd64-vmlinuz-virtual` - KERNEL_ID=`echo $RVAL | cut -d":" -f2 | tr -d " "` - glance add -A $SERVICE_TOKEN name="uec-natty" is_public=true container_format=ami disk_format=ami kernel_id=$KERNEL_ID < $FILES/images/natty-server-cloudimg-amd64.img + glance add -A $SERVICE_TOKEN name="$IMAGE_NAME" is_public=true container_format=ami disk_format=ami kernel_id=$KERNEL_ID < $FILES/images/$IMAGE_NAME.img fi From 0ab1d46e05c310a9977aebd3156b5269dffb91fc Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Thu, 13 Oct 2011 23:03:23 -0700 Subject: [PATCH 26/37] Allow user to download and install arbitrary images --- stack.sh | 29 +++++++++++++---------------- stackrc | 3 +++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/stack.sh b/stack.sh index 870811b..29f1377 100755 --- a/stack.sh +++ b/stack.sh @@ -660,38 +660,35 @@ screen_it dash "cd $DASH_DIR && sudo /etc/init.d/apache2 restart; sudo tail -f / # Install Images # ============== -# Upload a couple images to glance. **TTY** is a simple small image that use the -# lets you login to it with username/password of user/password. TTY is useful -# for basic functionality. We all include an Ubuntu cloud build of **Natty**. -# Natty uses cloud-init, supporting login via keypair and sending scripts as -# userdata. +# Upload an image to glance. # -# Read more about cloud-init at https://help.ubuntu.com/community/CloudInit +# The default image is a small ***TTY*** testing image, which lets you login +# the username/password of root/password. +# +# TTY also uses cloud-init, supporting login via keypair and sending scripts as +# userdata. See https://help.ubuntu.com/community/CloudInit for more on cloud-init +# +# Override IMAGE_URL if you would to launch a different image. Some other options: +# natty: http://uec-images.ubuntu.com/natty/current/natty-server-cloudimg-amd64.tar.gz if [[ "$ENABLED_SERVICES" =~ "g-reg" ]]; then - # create a directory for the downloadedthe images tarballs. + # Create a directory for the downloadedthe images tarballs. mkdir -p $FILES/images - # Debug Image (TTY) - # ----------------- - - # Downloads the image (ami/aki/ari style), then extracts it. Upon extraction - # we upload to glance with the glance cli tool. TTY is a stripped down - # version of ubuntu. - IMAGE_URL=http://smoser.brickies.net/ubuntu/ttylinux-uec/ttylinux-uec-amd64-11.2_2.6.35-15_1.tar.gz + # Downloads the image (uec ami+aki style), then extracts it. IMAGE_FNAME=`echo "$IMAGE_URL" | python -c "import sys; print sys.stdin.read().split('/')[-1]"` IMAGE_NAME=`echo "$IMAGE_FNAME" | python -c "import sys; print sys.stdin.read().split('.tar.gz')[0].split('.tgz')[0]"` if [ ! -f $FILES/$IMAGE_FNAME ]; then wget -c $IMAGE_URL -O $FILES/$IMAGE_FNAME fi - # extract ami-tty/image, aki-tty/image & ari-tty/image + # Extract ami and aki files tar -zxf $FILES/$IMAGE_FNAME -C $FILES/images + # Upload to glance with the glance cli tool. RVAL=`glance add -A $SERVICE_TOKEN name="$IMAGE_NAME-kernel" is_public=true container_format=aki disk_format=aki < $FILES/images/$IMAGE_NAME-vmlinuz*` KERNEL_ID=`echo $RVAL | cut -d":" -f2 | tr -d " "` glance add -A $SERVICE_TOKEN name="$IMAGE_NAME" is_public=true container_format=ami disk_format=ami kernel_id=$KERNEL_ID < $FILES/images/$IMAGE_NAME.img - fi # Fin diff --git a/stackrc b/stackrc index 15d73ce..9a94721 100644 --- a/stackrc +++ b/stackrc @@ -27,6 +27,9 @@ NOVACLIENT_BRANCH=master OPENSTACKX_REPO=https://github.com/cloudbuilders/openstackx.git OPENSTACKX_BRANCH=diablo +# Specify a uec image to download and install into glance. +IMAGE_URL=http://smoser.brickies.net/ubuntu/ttylinux-uec/ttylinux-uec-amd64-11.2_2.6.35-15_1.tar.gz + # allow local overrides of env variables if [ -f ./localrc ]; then source ./localrc From 93947e59be5566391a3fa0a0b32906ce8c4ceffa Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Thu, 13 Oct 2011 23:07:56 -0700 Subject: [PATCH 27/37] improve docs --- stack.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stack.sh b/stack.sh index 29f1377..ea4db93 100755 --- a/stack.sh +++ b/stack.sh @@ -685,7 +685,9 @@ if [[ "$ENABLED_SERVICES" =~ "g-reg" ]]; then # Extract ami and aki files tar -zxf $FILES/$IMAGE_FNAME -C $FILES/images - # Upload to glance with the glance cli tool. + # Use glance client to add the kernel the root filesystem. + # We parse the results of the first upload to get the glance ID of the + # kernel for use when uploading the root filesystem. RVAL=`glance add -A $SERVICE_TOKEN name="$IMAGE_NAME-kernel" is_public=true container_format=aki disk_format=aki < $FILES/images/$IMAGE_NAME-vmlinuz*` KERNEL_ID=`echo $RVAL | cut -d":" -f2 | tr -d " "` glance add -A $SERVICE_TOKEN name="$IMAGE_NAME" is_public=true container_format=ami disk_format=ami kernel_id=$KERNEL_ID < $FILES/images/$IMAGE_NAME.img From 120f4860cda1a946ebdba1acc03fc09caf2790de Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Fri, 14 Oct 2011 09:31:09 -0700 Subject: [PATCH 28/37] allow multiple images --- stack.sh | 34 +++++++++++++++++++--------------- stackrc | 4 ++-- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/stack.sh b/stack.sh index ea4db93..b580a4a 100755 --- a/stack.sh +++ b/stack.sh @@ -668,29 +668,33 @@ screen_it dash "cd $DASH_DIR && sudo /etc/init.d/apache2 restart; sudo tail -f / # TTY also uses cloud-init, supporting login via keypair and sending scripts as # userdata. See https://help.ubuntu.com/community/CloudInit for more on cloud-init # -# Override IMAGE_URL if you would to launch a different image. Some other options: +# Override IMAGE_URLS if you would to launch a different image(s). +# Specify IMAGE_URLS as a comma-separated list of uec urls. Some other options include: # natty: http://uec-images.ubuntu.com/natty/current/natty-server-cloudimg-amd64.tar.gz +# oneiric: http://uec-images.ubuntu.com/oneiric/current/oneiric-server-cloudimg-amd64.tar.gz if [[ "$ENABLED_SERVICES" =~ "g-reg" ]]; then # Create a directory for the downloadedthe images tarballs. mkdir -p $FILES/images - # Downloads the image (uec ami+aki style), then extracts it. - IMAGE_FNAME=`echo "$IMAGE_URL" | python -c "import sys; print sys.stdin.read().split('/')[-1]"` - IMAGE_NAME=`echo "$IMAGE_FNAME" | python -c "import sys; print sys.stdin.read().split('.tar.gz')[0].split('.tgz')[0]"` - if [ ! -f $FILES/$IMAGE_FNAME ]; then - wget -c $IMAGE_URL -O $FILES/$IMAGE_FNAME - fi + for image_url in ${IMAGE_URLS//,/ }; do + # Downloads the image (uec ami+aki style), then extracts it. + IMAGE_FNAME=`echo "$image_url" | python -c "import sys; print sys.stdin.read().split('/')[-1]"` + IMAGE_NAME=`echo "$IMAGE_FNAME" | python -c "import sys; print sys.stdin.read().split('.tar.gz')[0].split('.tgz')[0]"` + if [ ! -f $FILES/$IMAGE_FNAME ]; then + wget -c $image_url -O $FILES/$IMAGE_FNAME + fi - # Extract ami and aki files - tar -zxf $FILES/$IMAGE_FNAME -C $FILES/images + # Extract ami and aki files + tar -zxf $FILES/$IMAGE_FNAME -C $FILES/images - # Use glance client to add the kernel the root filesystem. - # We parse the results of the first upload to get the glance ID of the - # kernel for use when uploading the root filesystem. - RVAL=`glance add -A $SERVICE_TOKEN name="$IMAGE_NAME-kernel" is_public=true container_format=aki disk_format=aki < $FILES/images/$IMAGE_NAME-vmlinuz*` - KERNEL_ID=`echo $RVAL | cut -d":" -f2 | tr -d " "` - glance add -A $SERVICE_TOKEN name="$IMAGE_NAME" is_public=true container_format=ami disk_format=ami kernel_id=$KERNEL_ID < $FILES/images/$IMAGE_NAME.img + # Use glance client to add the kernel the root filesystem. + # We parse the results of the first upload to get the glance ID of the + # kernel for use when uploading the root filesystem. + RVAL=`glance add -A $SERVICE_TOKEN name="$IMAGE_NAME-kernel" is_public=true container_format=aki disk_format=aki < $FILES/images/$IMAGE_NAME-vmlinuz*` + KERNEL_ID=`echo $RVAL | cut -d":" -f2 | tr -d " "` + glance add -A $SERVICE_TOKEN name="$IMAGE_NAME" is_public=true container_format=ami disk_format=ami kernel_id=$KERNEL_ID < $FILES/images/$IMAGE_NAME.img + done fi # Fin diff --git a/stackrc b/stackrc index 9a94721..e1b65c3 100644 --- a/stackrc +++ b/stackrc @@ -27,8 +27,8 @@ NOVACLIENT_BRANCH=master OPENSTACKX_REPO=https://github.com/cloudbuilders/openstackx.git OPENSTACKX_BRANCH=diablo -# Specify a uec image to download and install into glance. -IMAGE_URL=http://smoser.brickies.net/ubuntu/ttylinux-uec/ttylinux-uec-amd64-11.2_2.6.35-15_1.tar.gz +# Specify a comma-separated list of uec images to download and install into glance. +IMAGE_URLS=http://smoser.brickies.net/ubuntu/ttylinux-uec/ttylinux-uec-amd64-11.2_2.6.35-15_1.tar.gz # allow local overrides of env variables if [ -f ./localrc ]; then From 7a8989e9711ec0d3f3401f46a20bdc55c75fd8e9 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Fri, 14 Oct 2011 10:20:30 -0700 Subject: [PATCH 29/37] fix doc --- stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.sh b/stack.sh index b580a4a..a7eacbf 100755 --- a/stack.sh +++ b/stack.sh @@ -674,7 +674,7 @@ screen_it dash "cd $DASH_DIR && sudo /etc/init.d/apache2 restart; sudo tail -f / # oneiric: http://uec-images.ubuntu.com/oneiric/current/oneiric-server-cloudimg-amd64.tar.gz if [[ "$ENABLED_SERVICES" =~ "g-reg" ]]; then - # Create a directory for the downloadedthe images tarballs. + # Create a directory for the downloaded image tarballs. mkdir -p $FILES/images for image_url in ${IMAGE_URLS//,/ }; do From e8efef79809f8a88910926f4efeb3eb8d6b0ddcd Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Fri, 14 Oct 2011 12:00:50 -0700 Subject: [PATCH 30/37] move lxc to tools --- build_lxc.sh => tools/build_lxc.sh | 3 +++ build_lxc_multi.sh => tools/build_lxc_multi.sh | 0 .../lxc_network_hostonlyplusnat.sh | 0 3 files changed, 3 insertions(+) rename build_lxc.sh => tools/build_lxc.sh (99%) rename build_lxc_multi.sh => tools/build_lxc_multi.sh (100%) rename lxc_network_hostonlyplusnat.sh => tools/lxc_network_hostonlyplusnat.sh (100%) diff --git a/build_lxc.sh b/tools/build_lxc.sh similarity index 99% rename from build_lxc.sh rename to tools/build_lxc.sh index 65a06cb..d4df344 100755 --- a/build_lxc.sh +++ b/tools/build_lxc.sh @@ -11,6 +11,9 @@ if ! grep -q natty /etc/lsb-release; then echo "WARNING: this script has only been tested on natty" fi +# Move to top devstack dir +cd .. + # Abort if localrc is not set if [ ! -e ./localrc ]; then echo "You must have a localrc with ALL necessary passwords defined before proceeding." diff --git a/build_lxc_multi.sh b/tools/build_lxc_multi.sh similarity index 100% rename from build_lxc_multi.sh rename to tools/build_lxc_multi.sh diff --git a/lxc_network_hostonlyplusnat.sh b/tools/lxc_network_hostonlyplusnat.sh similarity index 100% rename from lxc_network_hostonlyplusnat.sh rename to tools/lxc_network_hostonlyplusnat.sh From 9ff71acb08e4885d7da8cb44aedb2ff79f4e0909 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Fri, 14 Oct 2011 15:02:20 -0700 Subject: [PATCH 31/37] make build_lxc.sh work on oneiric --- tools/build_lxc.sh | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/tools/build_lxc.sh b/tools/build_lxc.sh index d4df344..507f93e 100755 --- a/tools/build_lxc.sh +++ b/tools/build_lxc.sh @@ -6,10 +6,8 @@ if [ "$EUID" -ne "0" ]; then exit 1 fi -# Warn users who aren't on natty -if ! grep -q natty /etc/lsb-release; then - echo "WARNING: this script has only been tested on natty" -fi +# Keep track of ubuntu version +UBUNTU_VERSION=`cat /etc/lsb-release | grep CODENAME | sed 's/.*=//g'` # Move to top devstack dir cd .. @@ -92,8 +90,21 @@ function git_clone { fi } +# Helper to create the container +function create_lxc { + if [ "natty" = "$UBUNTU_VERSION" ]; then + lxc-create -n $CONTAINER -t natty -f $LXC_CONF + else + lxc-create -n $CONTAINER -t ubuntu -f $LXC_CONF + fi +} + # Location of the base image directory -CACHEDIR=/var/cache/lxc/natty/rootfs-amd64 +if [ "natty" = "$UBUNTU_VERSION" ]; then + CACHEDIR=/var/cache/lxc/natty/rootfs-amd64 +else + CACHEDIR=/var/cache/lxc/oneiric/rootfs-amd64 +fi # Provide option to do totally clean install if [ "$CLEAR_LXC_CACHE" = "1" ]; then @@ -106,7 +117,7 @@ if [ ! -f $CACHEDIR/bootstrapped ]; then # lazy and doesn't do anything if a container already exists) lxc-destroy -n $CONTAINER # trigger the initial debootstrap - lxc-create -n $CONTAINER -t natty -f $LXC_CONF + create_lxc chroot $CACHEDIR apt-get update chroot $CACHEDIR apt-get install -y --force-yes `cat files/apts/* | cut -d\# -f1 | egrep -v "(rabbitmq|libvirt-bin|mysql-server)"` chroot $CACHEDIR pip install `cat files/pips/*` @@ -143,7 +154,7 @@ if [ "$TERMINATE" = "1" ]; then fi # Create the container -lxc-create -n $CONTAINER -t natty -f $LXC_CONF +create_lxc # Specify where our container rootfs lives ROOTFS=/var/lib/lxc/$CONTAINER/rootfs/ @@ -207,6 +218,10 @@ cat > $RUN_SH < /$DEST/run.sh.log +cd $DEST/devstack && $STACKSH_PARAMS FORCE=yes ./stack.sh > /$DEST/run.sh.log echo >> /$DEST/run.sh.log echo >> /$DEST/run.sh.log echo "All done! Time to start clicking." >> /$DEST/run.sh.log @@ -228,11 +243,13 @@ EOF chmod 755 $RUN_SH # Make runner launch on boot -RC_LOCAL=$ROOTFS/etc/rc.local +RC_LOCAL=$ROOTFS/etc/init.d/local cat > $RC_LOCAL < Date: Fri, 14 Oct 2011 16:50:27 -0700 Subject: [PATCH 32/37] make tail stop when stack.sh is finished --- tools/build_lxc.sh | 49 +++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/tools/build_lxc.sh b/tools/build_lxc.sh index 507f93e..42bd328 100755 --- a/tools/build_lxc.sh +++ b/tools/build_lxc.sh @@ -35,6 +35,7 @@ CONTAINER_GATEWAY=${CONTAINER_GATEWAY:-192.168.1.1} NAMESERVER=${NAMESERVER:-$CONTAINER_GATEWAY} COPYENV=${COPYENV:-1} DEST=${DEST:-/opt/stack} +WAIT_TILL_LAUNCH=${WAIT_TILL_LAUNCH:-1} # Param string to pass to stack.sh. Like "EC2_DMZ_HOST=192.168.1.1 MYSQL_USER=nova" STACKSH_PARAMS=${STACKSH_PARAMS:-} @@ -260,19 +261,39 @@ fi # Start our container lxc-start -d -n $CONTAINER -# Done creating the container, let's tail the log -echo -echo "=============================================================" -echo " -- YAY! --" -echo "=============================================================" -echo -echo "We're done creating the container, about to start tailing the" -echo "stack.sh log. It will take a second or two to start." -echo -echo "Just CTRL-C at any time to stop tailing." +if [ "$WAIT_TILL_LAUNCH" = "1" ]; then + # Done creating the container, let's tail the log + echo + echo "=============================================================" + echo " -- YAY! --" + echo "=============================================================" + echo + echo "We're done creating the container, about to start tailing the" + echo "stack.sh log. It will take a second or two to start." + echo + echo "Just CTRL-C at any time to stop tailing." -while [ ! -e "$ROOTFS/$DEST/run.sh.log" ]; do - sleep 1 -done + while [ ! -e "$ROOTFS/$DEST/run.sh.log" ]; do + sleep 1 + done -tail -F $ROOTFS/$DEST/run.sh.log + tail -F $ROOTFS/$DEST/run.sh.log & + + TAIL_PID=$! + + function kill_tail() { + exit 1 + } + + # Let Ctrl-c kill tail and exit + trap kill_tail SIGINT + + echo "Waiting stack.sh to finish..." + while ! cat $ROOTFS/$DEST/run.sh.log | grep -q 'All done' ; do + sleep 5 + done + + kill $TAIL_PID + echo "" + echo "Finished - Zip-a-dee Doo-dah!" +fi From f43307d408924977a2603d5e4f4aa30f41dfe4fc Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sat, 15 Oct 2011 17:40:38 -0700 Subject: [PATCH 33/37] exercise.sh was using wrong variable to generate token --- exercise.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exercise.sh b/exercise.sh index d9d4c0a..c791601 100755 --- a/exercise.sh +++ b/exercise.sh @@ -55,7 +55,7 @@ export NOVA_REGION_NAME=RegionOne # Get a token for clients that don't support service catalog # ========================================================== -SERVICE_TOKEN=`curl -s -d "{\"auth\":{\"passwordCredentials\": {\"username\": \"$NOVA_PROJECT_ID\", \"password\": \"$NOVA_API_KEY\"}}}" -H "Content-type: application/json" http://$HOST:5000/v2.0/tokens | python -c "import sys; import json; tok = json.loads(sys.stdin.read()); print tok['access']['token']['id'];"` +SERVICE_TOKEN=`curl -s -d "{\"auth\":{\"passwordCredentials\": {\"username\": \"$NOVA_USERNAME\", \"password\": \"$NOVA_API_KEY\"}}}" -H "Content-type: application/json" http://$HOST:5000/v2.0/tokens | python -c "import sys; import json; tok = json.loads(sys.stdin.read()); print tok['access']['token']['id'];"` # Launching a server # ================== From b9c77d69a6be25bca73780df671ab7538997e1ac Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sat, 15 Oct 2011 18:37:25 -0700 Subject: [PATCH 34/37] add debug logging and notes about generating token --- exercise.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/exercise.sh b/exercise.sh index c791601..f8c82d0 100755 --- a/exercise.sh +++ b/exercise.sh @@ -52,10 +52,17 @@ export NOVA_VERSION=1.1 # FIXME - why does this need to be specified? export NOVA_REGION_NAME=RegionOne +# set log level to DEBUG (helps debug issues) +export NOVACLIENT_DEBUG=1 # Get a token for clients that don't support service catalog # ========================================================== -SERVICE_TOKEN=`curl -s -d "{\"auth\":{\"passwordCredentials\": {\"username\": \"$NOVA_USERNAME\", \"password\": \"$NOVA_API_KEY\"}}}" -H "Content-type: application/json" http://$HOST:5000/v2.0/tokens | python -c "import sys; import json; tok = json.loads(sys.stdin.read()); print tok['access']['token']['id'];"` + +# manually create a token by querying keystone (sending JSON data). Keystone +# returns a token and catalog of endpoints. We use python to parse the token +# and save it. + +TOKEN=`curl -s -d "{\"auth\":{\"passwordCredentials\": {\"username\": \"$NOVA_USERNAME\", \"password\": \"$NOVA_API_KEY\"}}}" -H "Content-type: application/json" http://$HOST:5000/v2.0/tokens | python -c "import sys; import json; tok = json.loads(sys.stdin.read()); print tok['access']['token']['id'];"` # Launching a server # ================== From 4e8847cd6a8bb4dce9c3a7c66636d61ba51b519c Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sat, 15 Oct 2011 19:29:55 -0700 Subject: [PATCH 35/37] update glance command in exercise.sh --- exercise.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exercise.sh b/exercise.sh index f8c82d0..30f0d7d 100755 --- a/exercise.sh +++ b/exercise.sh @@ -80,7 +80,7 @@ nova flavor-list nova image-list # But we recommend using glance directly -glance -A $SERVICE_TOKEN index +glance -A $TOKEN index # show details of the active servers:: # From d888e1cdcb7eadf6c6b3634631400a4fe4c97769 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sat, 15 Oct 2011 20:01:12 -0700 Subject: [PATCH 36/37] update exercise to boot a server, ping it, then destroy it --- exercise.sh | 43 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/exercise.sh b/exercise.sh index 30f0d7d..84679f2 100755 --- a/exercise.sh +++ b/exercise.sh @@ -70,9 +70,6 @@ TOKEN=`curl -s -d "{\"auth\":{\"passwordCredentials\": {\"username\": \"$NOVA_U # List servers for tenant: nova list -# List of flavors: -nova flavor-list - # Images # ------ @@ -82,8 +79,38 @@ nova image-list # But we recommend using glance directly glance -A $TOKEN index -# show details of the active servers:: -# -# nova show 1234 -# -nova list | grep ACTIVE | cut -d \| -f2 | xargs -n1 nova show +# Let's grab the id of the first AMI image to launch +IMAGE=`glance -A $TOKEN index | egrep ami | cut -d" " -f1` + + +# Flavors +# ------- + +# List of flavors: +nova flavor-list + +# and grab the first flavor in the list to launch +FLAVOR=`nova flavor-list | head -n 4 | tail -n 1 | cut -d"|" -f2` + +NAME="firstpost" + +nova boot --flavor $FLAVOR --image $IMAGE $NAME + +# let's give it 10 seconds to launch +sleep 10 + +# check that the status is active +nova show $NAME | grep status | grep -q ACTIVE + +# get the IP of the server +IP=`nova show $NAME | grep "private network" | cut -d"|" -f3` + +# ping it once (timeout of a second) +ping -c1 -w1 $IP + +# shutdown the server +nova delete $NAME + +# FIXME: validate shutdown within 5 seconds +# (nova show $NAME returns 1 or status != ACTIVE)? + From da89268c9272e5e649d7d2332c86bf2fe68f57ee Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sat, 15 Oct 2011 20:14:07 -0700 Subject: [PATCH 37/37] update the ping to ping, wait 5 second, really ping --- exercise.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/exercise.sh b/exercise.sh index 84679f2..dc8163f 100755 --- a/exercise.sh +++ b/exercise.sh @@ -106,7 +106,13 @@ nova show $NAME | grep status | grep -q ACTIVE IP=`nova show $NAME | grep "private network" | cut -d"|" -f3` # ping it once (timeout of a second) -ping -c1 -w1 $IP +ping -c1 -w1 $IP || true + +# sometimes the first ping fails (10 seconds isn't enough time for the VM's +# network to respond?), so let's wait 5 seconds and really test ping +sleep 5 + +ping -c1 -w1 $IP # shutdown the server nova delete $NAME