diff --git a/tools/build_uec.sh b/tools/build_uec.sh index d95ba77..39c0d17 100755 --- a/tools/build_uec.sh +++ b/tools/build_uec.sh @@ -90,9 +90,10 @@ GUEST_CORES=${GUEST_CORES:-1} # libvirt.xml configuration NET_XML=$vm_dir/net.xml +NET_NAME=${NET_NAME:-devstack-$GUEST_NETWORK} cat > $NET_XML < - devstack-$GUEST_NETWORK + $NET_NAME @@ -104,9 +105,9 @@ cat > $NET_XML < $LIBVIRT_XML < - + @@ -170,7 +171,7 @@ instance-type: m1.ignore local-hostname: $GUEST_NAME.local EOF -# set metadata +# set user-data cat > $vm_dir/uec/user-data< localrc <> $vm_dir/uec/user-data< /opt/stack/.ssh/authorized_keys +chown -R stack /opt/stack +chmod 700 /opt/stack/.ssh +chmod 600 /opt/stack/.ssh/authorized_keys + +grep -q "^#includedir.*/etc/sudoers.d" /etc/sudoers || + echo "#includedir /etc/sudoers.d" >> /etc/sudoers +( umask 226 && echo "stack ALL=(ALL) NOPASSWD:ALL" \ + > /etc/sudoers.d/50_stack_sh ) +EOF +fi + +# Run stack.sh +cat >> $vm_dir/uec/user-data<localrc +RECLONE=yes +GUEST_NETWORK=$EXECUTOR_NUMBER +GUEST_NAME=$GUEST_NAME +FLOATING_RANGE=192.168.$EXECUTOR_NUMBER.128/27 +GUEST_CORES=1 +GUEST_RAM=12574720 +MYSQL_PASSWORD=chicken +RABBIT_PASSWORD=chicken +SERVICE_TOKEN=chicken +ADMIN_PASSWORD=chicken +USERNAME=admin +TENANT=admin +NET_NAME=$BASE_NAME +ACTIVE_TIMEOUT=45 +BOOT_TIMEOUT=45 +$RC +EOF +cd tools +sudo ./build_uec.sh + +# Make the address of the instances available to test runners +echo HEAD=`cat /var/lib/libvirt/dnsmasq/$BASE_NAME.leases | cut -d " " -f3` > $TOP_DIR/addresses diff --git a/tools/jenkins/configurations/xs.sh b/tools/jenkins/configurations/xs.sh new file mode 100755 index 0000000..864f949 --- /dev/null +++ b/tools/jenkins/configurations/xs.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -o errexit +set -o xtrace + + +EXECUTOR_NUMBER=$1 +CONFIGURATION=$2 +ADAPTER=$3 +RC=$4 + +function usage() { + echo "Usage: $0 - Build a test configuration" + echo "" + echo "$0 [EXECUTOR_NUMBER] [CONFIGURATION] [ADAPTER] [RC (optional)]" + exit 1 +} + +# Validate inputs +if [[ "$EXECUTOR_NUMBER" = "" || "$CONFIGURATION" = "" || "$ADAPTER" = "" ]]; then + usage +fi + +# Configuration of xenrc +XENRC=/var/lib/jenkins/xenrc +if [ ! -e $XENRC ]; then + echo "/var/lib/jenkins/xenrc is not present! See README.md" + exit 1 +fi + +# Move to top of devstack +cd ../../.. + +# Use xenrc as the start of our localrc +cp $XENRC localrc + +# Set the PUB_IP +PUB_IP=192.168.1.1$EXECUTOR_NUMBER +echo "PUB_IP=$PUB_IP" >> localrc + +# Overrides +echo "$RC" >> localrc + +# Source localrc +. localrc + +# Make host ip available to tester +echo "HEAD=$PUB_IP" > addresses + +# Build configuration +REMOTE_DEVSTACK=/root/devstack +ssh root@$XEN_IP "rm -rf $REMOTE_DEVSTACK" +scp -pr . root@$XEN_IP:$REMOTE_DEVSTACK +ssh root@$XEN_IP "cd $REMOTE_DEVSTACK/tools/xen && ./build_domU.sh" diff --git a/tools/jenkins/jenkins_home/.gitignore b/tools/jenkins/jenkins_home/.gitignore new file mode 100644 index 0000000..d831d01 --- /dev/null +++ b/tools/jenkins/jenkins_home/.gitignore @@ -0,0 +1,3 @@ +builds +workspace +*.sw* diff --git a/tools/jenkins/jenkins_home/build_jenkins.sh b/tools/jenkins/jenkins_home/build_jenkins.sh new file mode 100755 index 0000000..7d68679 --- /dev/null +++ b/tools/jenkins/jenkins_home/build_jenkins.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +# Echo commands, exit on error +set -o xtrace +set -o errexit + +# Make sure only root can run our script +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" + exit 1 +fi + +# Make sure user has configured an ssh pubkey +if [ ! -e /root/.ssh/id_rsa.pub ]; then + echo "Public key is missing. This is used to ssh into your instances." + echo "Please run ssh-keygen before proceeding" + exit 1 +fi + +# This directory +CUR_DIR=$(cd $(dirname "$0") && pwd) + +# Configure trunk jenkins! +echo "deb http://pkg.jenkins-ci.org/debian binary/" > /etc/apt/sources.list.d/jenkins.list +wget -q -O - http://pkg.jenkins-ci.org/debian/jenkins-ci.org.key | sudo apt-key add - +apt-get update + + +# Clean out old jenkins - useful if you are having issues upgrading +CLEAN_JENKINS=${CLEAN_JENKINS:-no} +if [ "$CLEAN_JENKINS" = "yes" ] then; + apt-get remove jenkins jenkins-common +fi + +# Install software +DEPS="jenkins cloud-utils" +apt-get install -y --force-yes $DEPS + +# Install jenkins +if [ ! -e /var/lib/jenkins ]; then + echo "Jenkins installation failed" + exit 1 +fi + +# Setup sudo +JENKINS_SUDO=/etc/sudoers.d/jenkins +cat > $JENKINS_SUDO < $JENKINS_GITCONF < + + 4 + Jenkins + jenkins@rcb.me + +EOF + +# Add build numbers +JOBS=`ls jobs` +for job in ${JOBS// / }; do + if [ ! -e jobs/$job/nextBuildNumber ]; then + echo 1 > jobs/$job/nextBuildNumber + fi +done + +# Set ownership to jenkins +chown -R jenkins $CUR_DIR + +# Make sure this directory is accessible to jenkins +if ! su -c "ls $CUR_DIR" jenkins; then + echo "Your devstack directory is not accessible by jenkins." + echo "There is a decent chance you are trying to run this from a directory in /root." + echo "If so, try moving devstack elsewhere (eg. /opt/devstack)." + exit 1 +fi + +# Move aside old jobs, if present +if [ ! -h /var/lib/jenkins/jobs ]; then + echo "Installing jobs symlink" + if [ -d /var/lib/jenkins/jobs ]; then + mv /var/lib/jenkins/jobs /var/lib/jenkins/jobs.old + fi +fi + +# Set up jobs symlink +rm -f /var/lib/jenkins/jobs +ln -s $CUR_DIR/jobs /var/lib/jenkins/jobs + +# List of plugins +PLUGINS=http://hudson-ci.org/downloads/plugins/build-timeout/1.6/build-timeout.hpi,http://mirrors.jenkins-ci.org/plugins/git/1.1.12/git.hpi,http://hudson-ci.org/downloads/plugins/global-build-stats/1.2/global-build-stats.hpi,http://hudson-ci.org/downloads/plugins/greenballs/1.10/greenballs.hpi,http://download.hudson-labs.org/plugins/console-column-plugin/1.0/console-column-plugin.hpi + +# Configure plugins +for plugin in ${PLUGINS//,/ }; do + name=`basename $plugin` + dest=/var/lib/jenkins/plugins/$name + if [ ! -e $dest ]; then + curl -L $plugin -o $dest + fi +done + +# Restart jenkins +/etc/init.d/jenkins stop || true +/etc/init.d/jenkins start diff --git a/tools/jenkins/jenkins_home/clean.sh b/tools/jenkins/jenkins_home/clean.sh new file mode 100755 index 0000000..eb03022 --- /dev/null +++ b/tools/jenkins/jenkins_home/clean.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# This script is not yet for general consumption. + +set -o errexit + +if [ ! "$FORCE" = "yes" ]; then + echo "FORCE not set to 'yes'. Make sure this is something you really want to do. Exiting." + exit 1 +fi + +virsh list | cut -d " " -f1 | grep -v "-" | egrep -e "[0-9]" | xargs -n 1 virsh destroy || true +virsh net-list | grep active | cut -d " " -f1 | xargs -n 1 virsh net-destroy || true +killall dnsmasq || true +if [ "$CLEAN" = "yes" ]; then + rm -rf jobs +fi +rm /var/lib/jenkins/jobs +git checkout -f +git fetch +git merge origin/jenkins +./build_jenkins.sh diff --git a/tools/jenkins/jenkins_home/jobs/diablo-kvm_ha/config.xml b/tools/jenkins/jenkins_home/jobs/diablo-kvm_ha/config.xml new file mode 100644 index 0000000..820e9d6 --- /dev/null +++ b/tools/jenkins/jenkins_home/jobs/diablo-kvm_ha/config.xml @@ -0,0 +1,82 @@ + + + + + false + + + + + RC + + + + + + + + 2 + + + origin + +refs/heads/*:refs/remotes/origin/* + git://github.com/cloudbuilders/devstack.git + + + + + jenkins + + + false + false + false + false + false + false + false + + Default + + + + + + + false + + + true + false + false + false + + false + + + ADAPTER + + euca + floating_ips + + + + + + sed -i 's/) 2>&1 | tee "${LOGFILE}"/)/' stack.sh + + + set -o errexit +cd tools/jenkins +sudo ./build_configuration.sh $EXECUTOR_NUMBER kvm $ADAPTER "$RC" + + + set -o errexit +cd tools/jenkins +sudo ./run_test.sh $EXECUTOR_NUMBER $ADAPTER $RC "$RC" + + + + + false + \ No newline at end of file diff --git a/tools/jenkins/jenkins_home/jobs/diablo-kvm_ha/configurations/axis-ADAPTER/euca/config.xml b/tools/jenkins/jenkins_home/jobs/diablo-kvm_ha/configurations/axis-ADAPTER/euca/config.xml new file mode 100644 index 0000000..0be70a5 --- /dev/null +++ b/tools/jenkins/jenkins_home/jobs/diablo-kvm_ha/configurations/axis-ADAPTER/euca/config.xml @@ -0,0 +1,15 @@ + + + false + + + false + false + false + false + + false + + + + \ No newline at end of file diff --git a/tools/jenkins/jenkins_home/jobs/diablo-kvm_ha/configurations/axis-ADAPTER/floatingips/config.xml b/tools/jenkins/jenkins_home/jobs/diablo-kvm_ha/configurations/axis-ADAPTER/floatingips/config.xml new file mode 100644 index 0000000..0be70a5 --- /dev/null +++ b/tools/jenkins/jenkins_home/jobs/diablo-kvm_ha/configurations/axis-ADAPTER/floatingips/config.xml @@ -0,0 +1,15 @@ + + + false + + + false + false + false + false + + false + + + + \ No newline at end of file diff --git a/tools/jenkins/jenkins_home/jobs/diablo-xs_ha/config.xml b/tools/jenkins/jenkins_home/jobs/diablo-xs_ha/config.xml new file mode 100644 index 0000000..21cd496 --- /dev/null +++ b/tools/jenkins/jenkins_home/jobs/diablo-xs_ha/config.xml @@ -0,0 +1,88 @@ + + + + In order for this to work, you must create a /var/lib/jenkins/xenrc file as described in README.md + false + + + + + RC + + + + + + + + 2 + + + origin + +refs/heads/*:refs/remotes/origin/* + git://github.com/cloudbuilders/devstack.git + + + + + jenkins + + + false + false + false + false + false + false + false + + Default + + + + + + + false + + + true + false + false + false + + false + + + ADAPTER + + euca + floating_ips + + + + + + sed -i 's/) 2>&1 | tee "${LOGFILE}"/)/' stack.sh + + + set -o errexit +cd tools/jenkins +sudo ./build_configuration.sh $EXECUTOR_NUMBER xs $ADAPTER "$RC" + + + #!/bin/bash +set -o errexit +set -o xtrace + +. localrc + +# Unlike kvm, ssh to the xen host to run tests, in case the test instance is launch with a host only network +ssh root@$XEN_IP "cd devstack && . localrc && cd tools/jenkins && ./run_test.sh $EXECUTOR_NUMBER $ADAPTER '$RC'" + + + + + + true + \ No newline at end of file diff --git a/tools/jenkins/jenkins_home/print_summary.py b/tools/jenkins/jenkins_home/print_summary.py new file mode 100755 index 0000000..1d71a4a --- /dev/null +++ b/tools/jenkins/jenkins_home/print_summary.py @@ -0,0 +1,45 @@ +#!/usr/bin/python +import urllib +import json +import sys + + +def print_usage(): + print "Usage: %s [jenkins_url (eg. http://50.56.12.202:8080/)]"\ + % sys.argv[0] + sys.exit() + + +def fetch_blob(url): + return json.loads(urllib.urlopen(url + '/api/json').read()) + + +if len(sys.argv) < 2: + print_usage() + +BASE_URL = sys.argv[1] + +root = fetch_blob(BASE_URL) +results = {} +for job_url in root['jobs']: + job = fetch_blob(job_url['url']) + if job.get('activeConfigurations'): + (tag, name) = job['name'].split('-') + if not results.get(tag): + results[tag] = {} + if not results[tag].get(name): + results[tag][name] = [] + + for config_url in job['activeConfigurations']: + config = fetch_blob(config_url['url']) + + log_url = '' + if config.get('lastBuild'): + log_url = config['lastBuild']['url'] + 'console' + + results[tag][name].append({'test': config['displayName'], + 'status': config['color'], + 'logUrl': log_url, + 'healthReport': config['healthReport']}) + +print json.dumps(results) diff --git a/tools/jenkins/run_test.sh b/tools/jenkins/run_test.sh new file mode 100755 index 0000000..4649563 --- /dev/null +++ b/tools/jenkins/run_test.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +EXECUTOR_NUMBER=$1 +ADAPTER=$2 +RC=$3 + +function usage() { + echo "Usage: $0 - Run a test" + echo "" + echo "$0 [EXECUTOR_NUMBER] [ADAPTER] [RC (optional)]" + exit 1 +} + +# Validate inputs +if [[ "$EXECUTOR_NUMBER" = "" || "$ADAPTER" = "" ]]; then + usage +fi + +# Execute configuration script +cd adapters && ./$ADAPTER.sh $EXECUTOR_NUMBER $ADAPTER "$RC" diff --git a/tools/setup_stack_user.sh b/tools/setup_stack_user.sh index 231a20f..fcb9733 100755 --- a/tools/setup_stack_user.sh +++ b/tools/setup_stack_user.sh @@ -49,10 +49,6 @@ grep -q "^#includedir.*/etc/sudoers.d" $STAGING_DIR/etc/sudoers || cp $TOP_DIR/files/sudo/* $STAGING_DIR/etc/sudoers.d/ sed -e "s,%USER%,$USER,g" -i $STAGING_DIR/etc/sudoers.d/* -# and has sudo ability (in the future this should be limited to only what -# stack requires) -echo "stack ALL=(ALL) NOPASSWD: ALL" >> $STAGING_DIR/etc/sudoers - # Gracefully cp only if source file/dir exists function cp_it { if [ -e $1 ] || [ -d $1 ]; then diff --git a/tools/xen/build_domU.sh b/tools/xen/build_domU.sh index 6362849..d79d5c3 100755 --- a/tools/xen/build_domU.sh +++ b/tools/xen/build_domU.sh @@ -240,6 +240,11 @@ if [ "$DO_SHUTDOWN" = "1" ]; then xe vm-shutdown uuid=$uuid xe vm-destroy uuid=$uuid done + + # Destroy orphaned vdis + for uuid in `xe vdi-list | grep -1 Glance | grep uuid | sed "s/.*\: //g"`; do + xe vdi-destroy uuid=$uuid + done fi # Path to head xva. By default keep overwriting the same one to save space