From 2f140207569d61c9a44a2337c8007da064fd6659 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 26 Sep 2011 13:02:40 -0700 Subject: [PATCH 1/6] implement stackrc for variable configuration (esp repos) --- build_lxc.sh | 37 +++++++++++++++++++++++++++---------- build_nfs.sh | 19 ++++++++++++------- stack.sh | 35 +++++++++++++++++------------------ 3 files changed, 56 insertions(+), 35 deletions(-) diff --git a/build_lxc.sh b/build_lxc.sh index e8612e6..15af51d 100755 --- a/build_lxc.sh +++ b/build_lxc.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# Source params +source ./stackrc + # Configurable params BRIDGE=${BRIDGE:-br0} CONTAINER=${CONTAINER:-STACK} @@ -51,10 +54,23 @@ if [ -d /cgroup/$CONTAINER ]; then cgdelete -r cpu,net_cls:$CONTAINER fi +# git clone only if directory doesn't exist already. Since ``DEST`` might not +# be owned by the installation user, we create the directory and change the +# ownership to the proper user. +function git_clone { + if [ ! -d $2 ]; then + sudo mkdir $2 + sudo chown `whoami` $2 + git clone $1 $2 + cd $2 + # This checkout syntax works for both branches and tags + git checkout $3 + fi +} # Warm the base image on first install CACHEDIR=/var/cache/lxc/natty/rootfs-amd64 -if [ ! -d $CACHEDIR ]; then +if [ ! -d $CACHEDIR/opt/nova ]; then # by deleting the container, we force lxc-create to re-bootstrap (lxc is # lazy and doesn't do anything if a container already exists) lxc-destroy -n $CONTAINER @@ -63,15 +79,16 @@ if [ ! -d $CACHEDIR ]; then chroot $CACHEDIR apt-get update chroot $CACHEDIR apt-get install -y --force-yes `cat files/apts/* | cut -d\# -f1 | egrep -v "(rabbitmq|libvirt-bin|mysql-server)"` chroot $CACHEDIR pip install `cat files/pips/*` - # FIXME (anthony) - provide ability to vary source locations - #git clone https://github.com/cloudbuilders/nova.git $CACHEDIR/opt/nova - bzr clone lp:~hudson-openstack/nova/milestone-proposed/ $CACHEDIR/opt/nova - git clone https://github.com/cloudbuilders/openstackx.git $CACHEDIR/opt/openstackx - git clone https://github.com/cloudbuilders/noVNC.git $CACHEDIR/opt/noVNC - git clone https://github.com/cloudbuilders/openstack-dashboard.git $CACHEDIR/opt/dash - git clone https://github.com/cloudbuilders/python-novaclient.git $CACHEDIR/opt/python-novaclient - git clone https://github.com/cloudbuilders/keystone.git $CACHEDIR/opt/keystone - git clone https://github.com/cloudbuilders/glance.git $CACHEDIR/opt/glance + + git_clone $NOVA_REPO $CACHEDIR/opt/nova $NOVA_BRANCH + git_clone $GLANCE_REPO $CACHEDIR/opt/glance $GLANCE_BRANCH + git_clone $KEYSTONE_REPO $CACHEDIR/opt/keystone $KEYSTONE_BRANCH + git_clone $NOVNC_REPO $CACHEDIR/opt/novnc $NOVNC_BRANCH + git_clone $DASH_REPO $CACHEDIR/opt/dash $DASH_BRANCH $DASH_TAG + git_clone $NIXON_REPO $CACHEDIR/opt/nixon $NIXON_BRANCH + git_clone $NOVACLIENT_REPO $CACHEDIR/opt/python-novaclient $NOVACLIENT_BRANCH + git_clone $OPENSTACKX_REPO $CACHEDIR/opt/openstackx $OPENSTACKX_BRANCH + git_clone $MUNIN_REPO $CACHEDIR/opt/openstack-munin $MUNIN_BRANCH fi # Destroy the old container diff --git a/build_nfs.sh b/build_nfs.sh index 59a3609..fa3e3ef 100755 --- a/build_nfs.sh +++ b/build_nfs.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Source params +source ./stackrc + # TODO: make dest not hardcoded NAME=$1 @@ -15,13 +18,15 @@ if [ ! -d proto ]; then chroot proto apt-get update chroot proto apt-get install -y `cat files/apts/* | cut -d\# -f1 | egrep -v "(rabbitmq|libvirt-bin|mysql-server)"` chroot proto pip install `cat files/pips/*` - git clone https://github.com/cloudbuilders/nova.git proto/opt/nova - git clone https://github.com/cloudbuilders/openstackx.git proto/opt/openstackx - git clone https://github.com/cloudbuilders/noVNC.git proto/opt/noVNC - git clone https://github.com/cloudbuilders/openstack-dashboard.git proto/opt/dash - git clone https://github.com/cloudbuilders/python-novaclient.git proto/opt/python-novaclient - git clone https://github.com/cloudbuilders/keystone.git proto/opt/keystone - git clone https://github.com/cloudbuilders/glance.git proto/opt/glance + git_clone $NOVA_REPO proto/opt/nova $NOVA_BRANCH + git_clone $GLANCE_REPO proto/opt/glance $GLANCE_BRANCH + git_clone $KEYSTONE_REPO proto/opt/keystone $KEYSTONE_BRANCH + git_clone $NOVNC_REPO proto/opt/novnc $NOVNC_BRANCH + git_clone $DASH_REPO proto/opt/dash $DASH_BRANCH $DASH_TAG + git_clone $NIXON_REPO proto/opt/nixon $NIXON_BRANCH + git_clone $NOVACLIENT_REPO proto/opt/python-novaclient $NOVACLIENT_BRANCH + git_clone $OPENSTACKX_REPO proto/opt/openstackx $OPENSTACKX_BRANCH + git_clone $MUNIN_REPO proto/opt/openstack-munin $MUNIN_BRANCH chroot proto mkdir -p /opt/files wget -c http://images.ansolabs.com/tty.tgz -O proto/opt/files/tty.tgz fi diff --git a/stack.sh b/stack.sh index 19c8f90..c7c2e80 100755 --- a/stack.sh +++ b/stack.sh @@ -59,6 +59,9 @@ set -o errexit # an error. It is also useful for following allowing as the install occurs. set -o xtrace +# Import variables +source ./stackrc + # Destination path for installation ``DEST`` DEST=${DEST:-/opt} @@ -69,7 +72,7 @@ NIXON_DIR=$DEST/dash/openstack-dashboard/dashboard/nixon GLANCE_DIR=$DEST/glance KEYSTONE_DIR=$DEST/keystone NOVACLIENT_DIR=$DEST/python-novaclient -API_DIR=$DEST/openstackx +OPENSTACKX_DIR=$DEST/openstackx NOVNC_DIR=$DEST/noVNC MUNIN_DIR=$DEST/openstack-munin @@ -140,34 +143,30 @@ function git_clone { sudo mkdir $2 sudo chown `whoami` $2 git clone $1 $2 + cd $2 + git checkout $3 fi } # compute service -# FIXME - need to factor out these repositories -# git_clone https://github.com/cloudbuilders/nova.git $NOVA_DIR -if [ ! -d $NOVA_DIR ]; then - bzr clone lp:~hudson-openstack/nova/milestone-proposed/ $NOVA_DIR -fi +git_clone $NOVA_REPO $NOVA_DIR $NOVA_BRANCH # image catalog service -git_clone https://github.com/cloudbuilders/glance.git $GLANCE_DIR +git_clone $GLANCE_REPO $GLANCE_DIR $GLANCE_BRANCH # unified auth system (manages accounts/tokens) -git_clone https://github.com/cloudbuilders/keystone.git $KEYSTONE_DIR +git_clone $KEYSTONE_REPO $KEYSTONE_DIR $KEYSTONE_BRANCH # a websockets/html5 or flash powered VNC console for vm instances -git_clone https://github.com/cloudbuilders/noVNC.git $NOVNC_DIR +git_clone $NOVNC_REPO $NOVNC_DIR $NOVNC_BRANCH # django powered web control panel for openstack -git_clone https://github.com/cloudbuilders/openstack-dashboard.git $DASH_DIR -# FIXME - need to factor out logic like this -cd $DASH_DIR && sudo git fetch && sudo git checkout origin/keystone_diablo +git_clone $DASH_REPO $DASH_DIR $DASH_BRANCH $DASH_TAG # add nixon, will use this to show munin graphs in dashboard -git_clone https://github.com/cloudbuilders/nixon.git $NIXON_DIR +git_clone $NIXON_REPO $NIXON_DIR $NIXON_BRANCH # python client library to nova that dashboard (and others) use -git_clone https://github.com/cloudbuilders/python-novaclient.git $NOVACLIENT_DIR +git_clone $NOVACLIENT_REPO $NOVACLIENT_DIR $NOVACLIENT_BRANCH # openstackx is a collection of extensions to openstack.compute & nova # that is *deprecated*. The code is being moved into python-novaclient & nova. -git_clone https://github.com/cloudbuilders/openstackx.git $API_DIR +git_clone $OPENSTACKX_REPO $OPENSTACKX_DIR $OPENSTACKX_BRANCH # openstack-munin is a collection of munin plugins for monitoring the stack -git_clone https://github.com/cloudbuilders/openstack-munin.git $MUNIN_DIR +git_clone $MUNIN_REPO $MUNIN_DIR $MUNIN_BRANCH # Initialization # ============== @@ -179,7 +178,7 @@ cd $NOVA_DIR; sudo python setup.py develop cd $NOVACLIENT_DIR; sudo python setup.py develop cd $KEYSTONE_DIR; sudo python setup.py develop cd $GLANCE_DIR; sudo python setup.py develop -cd $API_DIR; sudo python setup.py develop +cd $OPENSTACKX_DIR; sudo python setup.py develop cd $DASH_DIR/django-openstack; sudo python setup.py develop cd $DASH_DIR/openstack-dashboard; sudo python setup.py develop @@ -355,7 +354,7 @@ add_nova_flag "--public_interface=$PUBLIC_INTERFACE" add_nova_flag "--vlan_interface=$VLAN_INTERFACE" add_nova_flag "--sql_connection=$BASE_SQL_CONN/nova" add_nova_flag "--libvirt_type=$LIBVIRT_TYPE" -add_nova_flag "--osapi_extensions_path=$API_DIR/extensions" +add_nova_flag "--osapi_extensions_path=$OPENSTACKX_DIR/extensions" add_nova_flag "--vncproxy_url=http://$HOST_IP:6080" add_nova_flag "--vncproxy_wwwroot=$NOVNC_DIR/" add_nova_flag "--api_paste_config=$KEYSTONE_DIR/examples/paste/nova-api-paste.ini" From 303233e2f57259bc809230aa60b99655f5650853 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 26 Sep 2011 13:12:57 -0700 Subject: [PATCH 2/6] tweaks to comments and cache organization --- build_lxc.sh | 23 ++++++++++++----------- stack.sh | 1 + 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/build_lxc.sh b/build_lxc.sh index 15af51d..c24539e 100755 --- a/build_lxc.sh +++ b/build_lxc.sh @@ -70,7 +70,7 @@ function git_clone { # Warm the base image on first install CACHEDIR=/var/cache/lxc/natty/rootfs-amd64 -if [ ! -d $CACHEDIR/opt/nova ]; then +if [ ! -d $CACHEDIR/ ]; then # by deleting the container, we force lxc-create to re-bootstrap (lxc is # lazy and doesn't do anything if a container already exists) lxc-destroy -n $CONTAINER @@ -79,18 +79,19 @@ if [ ! -d $CACHEDIR/opt/nova ]; then chroot $CACHEDIR apt-get update chroot $CACHEDIR apt-get install -y --force-yes `cat files/apts/* | cut -d\# -f1 | egrep -v "(rabbitmq|libvirt-bin|mysql-server)"` chroot $CACHEDIR pip install `cat files/pips/*` - - git_clone $NOVA_REPO $CACHEDIR/opt/nova $NOVA_BRANCH - git_clone $GLANCE_REPO $CACHEDIR/opt/glance $GLANCE_BRANCH - git_clone $KEYSTONE_REPO $CACHEDIR/opt/keystone $KEYSTONE_BRANCH - git_clone $NOVNC_REPO $CACHEDIR/opt/novnc $NOVNC_BRANCH - git_clone $DASH_REPO $CACHEDIR/opt/dash $DASH_BRANCH $DASH_TAG - git_clone $NIXON_REPO $CACHEDIR/opt/nixon $NIXON_BRANCH - git_clone $NOVACLIENT_REPO $CACHEDIR/opt/python-novaclient $NOVACLIENT_BRANCH - git_clone $OPENSTACKX_REPO $CACHEDIR/opt/openstackx $OPENSTACKX_BRANCH - git_clone $MUNIN_REPO $CACHEDIR/opt/openstack-munin $MUNIN_BRANCH fi +# Cache openstack code +git_clone $NOVA_REPO $CACHEDIR/opt/nova $NOVA_BRANCH +git_clone $GLANCE_REPO $CACHEDIR/opt/glance $GLANCE_BRANCH +git_clone $KEYSTONE_REPO $CACHEDIR/opt/keystone $KEYSTONE_BRANCH +git_clone $NOVNC_REPO $CACHEDIR/opt/novnc $NOVNC_BRANCH +git_clone $DASH_REPO $CACHEDIR/opt/dash $DASH_BRANCH $DASH_TAG +git_clone $NIXON_REPO $CACHEDIR/opt/nixon $NIXON_BRANCH +git_clone $NOVACLIENT_REPO $CACHEDIR/opt/python-novaclient $NOVACLIENT_BRANCH +git_clone $OPENSTACKX_REPO $CACHEDIR/opt/openstackx $OPENSTACKX_BRANCH +git_clone $MUNIN_REPO $CACHEDIR/opt/openstack-munin $MUNIN_BRANCH + # Destroy the old container lxc-destroy -n $CONTAINER diff --git a/stack.sh b/stack.sh index c7c2e80..85b7474 100755 --- a/stack.sh +++ b/stack.sh @@ -144,6 +144,7 @@ function git_clone { sudo chown `whoami` $2 git clone $1 $2 cd $2 + # This checkout syntax works for both branches and tags git checkout $3 fi } From 096fb5a86a3d198d8f731b24347ce49196190e1d Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 26 Sep 2011 13:14:46 -0700 Subject: [PATCH 3/6] add stackrc file --- stackrc | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 stackrc diff --git a/stackrc b/stackrc new file mode 100644 index 0000000..2d265d3 --- /dev/null +++ b/stackrc @@ -0,0 +1,36 @@ +# compute service +NOVA_REPO=https://github.com/openstack/nova.git +NOVA_BRANCH=2011.3 + +# image catalog service +GLANCE_REPO=https://github.com/cloudbuilders/glance.git +GLANCE_BRANCH=diablo + +# unified auth system (manages accounts/tokens) +KEYSTONE_REPO=https://github.com/cloudbuilders/keystone.git +KEYSTONE_BRANCH=diablo + +# a websockets/html5 or flash powered VNC console for vm instances +NOVNC_REPO=https://github.com/cloudbuilders/noVNC.git +NOVNC_BRANCH=master + +# django powered web control panel for openstack +DASH_REPO=https://github.com/cloudbuilders/openstack-dashboard.git +DASH_BRANCH=master + +# add nixon, will use this to show munin graphs in dashboard +NIXON_REPO=https://github.com/cloudbuilders/nixon.git +NIXON_BRANCH=diablo + +# python client library to nova that dashboard (and others) use +NOVACLIENT_REPO=https://github.com/cloudbuilders/python-novaclient.git +NOVACLIENT_BRANCH=master + +# openstackx is a collection of extensions to openstack.compute & nova +# that is *deprecated*. The code is being moved into python-novaclient & nova. +OPENSTACKX_REPO=https://github.com/cloudbuilders/openstackx.git +OPENSTACKX_BRANCH=diablo + +# openstack-munin is a collection of munin plugins for monitoring the stack +MUNIN_REPO=https://github.com/cloudbuilders/openstack-munin.git +MUNIN_BRANCH=master From 01ad91a32f1c01c446cbffde96eb32057f6f5694 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 26 Sep 2011 13:25:11 -0700 Subject: [PATCH 4/6] remove trailing slash --- build_lxc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_lxc.sh b/build_lxc.sh index c24539e..3d0ad89 100755 --- a/build_lxc.sh +++ b/build_lxc.sh @@ -70,7 +70,7 @@ function git_clone { # Warm the base image on first install CACHEDIR=/var/cache/lxc/natty/rootfs-amd64 -if [ ! -d $CACHEDIR/ ]; then +if [ ! -d $CACHEDIR ]; then # by deleting the container, we force lxc-create to re-bootstrap (lxc is # lazy and doesn't do anything if a container already exists) lxc-destroy -n $CONTAINER From d8c259af5ae8a5eba10bf1d8fbf368251c028caf Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 26 Sep 2011 14:04:13 -0700 Subject: [PATCH 5/6] use stackrc.example, and prompt user before copying --- build_lxc.sh | 7 +++++++ build_nfs.sh | 7 +++++++ stack.sh | 7 +++++++ stackrc => stackrc.example | 0 4 files changed, 21 insertions(+) rename stackrc => stackrc.example (100%) diff --git a/build_lxc.sh b/build_lxc.sh index 3d0ad89..4c624aa 100755 --- a/build_lxc.sh +++ b/build_lxc.sh @@ -1,5 +1,12 @@ #!/usr/bin/env bash +# Use stackrc.example if stackrc is missing +if [ ! -e ./stackrc ]; then + read -n1 -p "No stackrc present. Copy stackrc.example to stackrc? (y/n) " + echo + [[ $REPLY = [yY] ]] && cp stackrc.example stackrc|| { echo "Aborting: Missing stackrc"; exit 1; } +fi + # Source params source ./stackrc diff --git a/build_nfs.sh b/build_nfs.sh index fa3e3ef..c0fa733 100755 --- a/build_nfs.sh +++ b/build_nfs.sh @@ -1,5 +1,12 @@ #!/bin/bash +# Use stackrc.example if stackrc is missing +if [ ! -e ./stackrc ]; then + read -n1 -p "No stackrc present. Copy stackrc.example to stackrc? (y/n) " + echo + [[ $REPLY = [yY] ]] && cp stackrc.example stackrc|| { echo "Aborting: Missing stackrc"; exit 1; } +fi + # Source params source ./stackrc diff --git a/stack.sh b/stack.sh index 85b7474..54d777a 100755 --- a/stack.sh +++ b/stack.sh @@ -59,6 +59,13 @@ set -o errexit # an error. It is also useful for following allowing as the install occurs. set -o xtrace +# Use stackrc.example if stackrc is missing +if [ ! -e ./stackrc ]; then + read -n1 -p "No stackrc present. Copy stackrc.example to stackrc? (y/n) " + echo + [[ $REPLY = [yY] ]] && cp stackrc.example stackrc|| { echo "Aborting: Missing stackrc"; exit 1; } +fi + # Import variables source ./stackrc diff --git a/stackrc b/stackrc.example similarity index 100% rename from stackrc rename to stackrc.example From a34b695aadb6db1aedd5543604dac0400cd90b34 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 26 Sep 2011 15:24:59 -0700 Subject: [PATCH 6/6] by default, use the current version/directory of devstack for child lxc containers --- build_lxc.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/build_lxc.sh b/build_lxc.sh index 4c624aa..674742c 100755 --- a/build_lxc.sh +++ b/build_lxc.sh @@ -23,6 +23,9 @@ COPYENV=${COPYENV:-1} # Param string to pass to stack.sh. Like "EC2_DMZ_HOST=192.168.1.1 MYSQL_USER=nova" STACKSH_PARAMS=${STACKSH_PARAMS:-} +# Option to use the version of devstack on which we are currently working +USE_CURRENT_DEVSTACK=${USE_CURRENT_DEVSTACK:-1} + # Warn users who aren't on natty if ! grep -q natty /etc/lsb-release; then echo "WARNING: this script has only been tested on natty" @@ -99,6 +102,12 @@ git_clone $NOVACLIENT_REPO $CACHEDIR/opt/python-novaclient $NOVACLIENT_BRANCH git_clone $OPENSTACKX_REPO $CACHEDIR/opt/openstackx $OPENSTACKX_BRANCH git_clone $MUNIN_REPO $CACHEDIR/opt/openstack-munin $MUNIN_BRANCH +# Use this version of devstack? +if [ "$USE_CURRENT_DEVSTACK" = "1" ]; then + rm -rf $CACHEDIR/opt/devstack + cp -pr . $CACHEDIR/opt/devstack +fi + # Destroy the old container lxc-destroy -n $CONTAINER