From 63a64e2f277471101a0b5502cfdce8956a8fb31e Mon Sep 17 00:00:00 2001 From: Sebastian Rieger Date: Sat, 5 Dec 2015 23:45:50 +0100 Subject: [PATCH] - initial version of virl-benchmark to automatically start multiple simulations, measure the delay until the simulations are active and the delay until the telnet ports of all nodes can be used --- virl-benchmark/check-reachability | 64 +++++++++++ virl-benchmark/ports-vms.py | 127 +++++++++++++++++++++ virl-benchmark/start-virl-topology | 126 ++++++++++++++++++++ virl-benchmark/stop-virl-benchmark | 27 +++++ virl-benchmark/test-virl-telnet-connection | 17 +++ 5 files changed, 361 insertions(+) create mode 100644 virl-benchmark/check-reachability create mode 100644 virl-benchmark/ports-vms.py create mode 100644 virl-benchmark/start-virl-topology create mode 100644 virl-benchmark/stop-virl-benchmark create mode 100644 virl-benchmark/test-virl-telnet-connection diff --git a/virl-benchmark/check-reachability b/virl-benchmark/check-reachability new file mode 100644 index 0000000..7be31fc --- /dev/null +++ b/virl-benchmark/check-reachability @@ -0,0 +1,64 @@ +#!/bin/bash + +# Check reachability of all running VIRL VMs by logging in on the telnet console port and expect a prompt that includes the host name + +# usage +if [ ! $# -gt 2 ] ; then + echo -e "usage: $0 [debug level], e.g.,\n" + echo "$0 guest VIRLBENCH-223721@smb-WaZLhH 5" + exit -1 +fi + +# get the IP address of eth0 +IP=$(ip addr show | grep eth0 | grep inet | tr -s " " | cut -d " " -f 3 | cut -d "/" -f 1) + +RUNNING=true + +USERNAME=$1 +TIMEOUT=$2 +SIMULATION=$3 +if [ $4 ]; then + DEBUG=$4 +else + DEBUG=0 +fi + +while $RUNNING = true +do +# Check if all nodes are ready-to-use using expect + + # get the telnet ports of all nodes in all simulations + VM_PORTS=$(./ports-vms.py $USERNAME $SIMULATION) + + # connect to every telnet port of each node and expect the hostname in the prompt + VMS_UNUSABLE=0 + for VM_PORT in $VM_PORTS + do + VM_TELNET_PORT=$(echo $VM_PORT | cut -d "=" -f 2) + VM_NAME=$(echo $VM_PORT | cut -d "=" -f 1) + # connect to every telnet port and check whether it can be used by pressing return + # twice and expecting the hostname to appear in the resulting prompt each time + if [ $DEBUG -lt 2 ]; then + ./test-virl-telnet-connection $IP $VM_TELNET_PORT $VM_NAME $TIMEOUT >/dev/null + else + ./test-virl-telnet-connection $IP $VM_TELNET_PORT $VM_NAME $TIMEOUT + fi + EXPECT_EXITCODE=$? + if [ $EXPECT_EXITCODE -eq 5 ] ; then + VMS_UNUSABLE=$(expr $VMS_UNUSABLE + 1) + if [ $DEBUG -gt 0 ]; then echo "$VM_NAME ($VM_TELNET_PORT) still unusable"; fi + fi + done + + if [ $VMS_UNUSABLE -gt 0 ]; then + if [ $DEBUG -gt 0 ]; then echo "$VMS_UNUSABLE VMs are still unusable"; fi + else + RUNNING=false + fi + + sleep 1 + +done + +DATE=$(date) +echo "Finished at $DATE" diff --git a/virl-benchmark/ports-vms.py b/virl-benchmark/ports-vms.py new file mode 100644 index 0000000..6f61b0b --- /dev/null +++ b/virl-benchmark/ports-vms.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 +# +# Show all ports of all VM instances for +# all users and topologies. +# Like the console port and the VNC port +# +# rschmied@cisco.com +# +# modified by sebastian.rieger@informatik.hs-fulda.de to display telnet console ports of running nodes in a topology +# + + +import os, libvirt, re, sys +from keystoneclient import client as keystone +from novaclient import client as nova +from xml.dom.minidom import parseString + +""" + +In [14]: m.group(0) +Out[14]: '---' + +In [15]: m.group(1) +Out[15]: '/guest/endpoint' + +In [16]: m.group(2) +Out[16]: 'Sample_Project' + +In [16]: m.group(3) +Out[16]: 'asa-test-topo' + +In [17]: m.group(4) +Out[17]: 'iosv-2' + +In [18]: m.group(5) +Out[18]: 'Multipoint Connection-1' + +will not match jumphost ports! +not interested in these, anyway + +""" + +class KeystoneV3NovaAuthPlugin(object): + def __init__(self, keystone_client): + self.keystone_client = keystone_client + + def authenticate(self, client, fake_auth_url): + client.auth_url = fake_auth_url + client.service_catalog = self.keystone_client.service_catalog + client.auth_token = self.keystone_client.auth_token + client.tenant_id = self.keystone_client.tenant_id + client.management_url = self.keystone_client.service_catalog.url_for( + attr='region', + filter_value=client.region_name, + endpoint_type=client.endpoint_type, + service_type=client.service_type, + service_name=client.service_name).rstrip('/') + +def getports(user,simulation): + # Sample output / field mapping + # -- + # USER PROJECT TOPOLOGY NODE + + # -- + # USER TOPOLOGY NODE + prog=re.compile(r'-<(.*)-[_0-9a-z]{6}>-<(.*)>', re.IGNORECASE) + + # table=list() + try: + libvirt_uri = os.environ['LIBVIRT_DEFAULT_URI'] + except: + libvirt_uri = "qemu:///system" + print "LIBVIRT_DEFAULT_URI env not set!" + print "Using default '" + libvirt_uri + "'" + conn=libvirt.openReadOnly(libvirt_uri) + + kc = keystone.Client(auth_url=os.environ['OS_AUTH_URL'], + username=os.environ['OS_USERNAME'], password=os.environ['OS_PASSWORD'], + project_name=os.environ['OS_TENANT_NAME']) + kc.session.auth = kc + kc.authenticate() + nc=nova.Client('2', os.environ['OS_USERNAME'], os.environ['OS_PASSWORD'], + os.environ['OS_TENANT_NAME'], auth_system='keystonev3', auth_plugin=KeystoneV3NovaAuthPlugin(kc), auth_url='http://fake/v2.0') + + for server in nc.servers.list(search_opts={'all_tenants': True}): + m=prog.match(server.name) + if m: + try: + domain=conn.lookupByUUIDString(server.id) + except: + print "Domain not found / not running" + return 1 + else: + doc=parseString(domain.XMLDesc(flags=0)) + # get the VNC port + #port=doc.getElementsByTagName('graphics')[0].getAttribute('port') + # get the serial console TCP port + for i in doc.getElementsByTagName('source'): + if i.parentNode.nodeName == u'console': + console=i.getAttribute('service') + # get the instance name + name=doc.getElementsByTagName('name')[0].childNodes[0].nodeValue + # print info + if simulation == "*": + if m.group(1) == user: + print m.group(3) + "=" + console + else: + if m.group(1) == user and server.name.find(simulation) != -1: + print m.group(3) + "=" + console + + +def main(): + if len(sys.argv) != 3: + sys.stdout.write(str(sys.argv[0])) + print ": username and simulation (e.g., project name or session-id) needed as argument! bailing out" + return 1 + else: + user = str(sys.argv[1]).strip() + simulation = str(sys.argv[2]).strip() + getports(user,simulation) + return 0 + +if __name__ == '__main__': + sys.exit(main()) + diff --git a/virl-benchmark/start-virl-topology b/virl-benchmark/start-virl-topology new file mode 100644 index 0000000..68f2314 --- /dev/null +++ b/virl-benchmark/start-virl-topology @@ -0,0 +1,126 @@ +#!/bin/bash +# start-virl-topology-benchmark.sh +# HS-Fulda - sebastian.rieger@informatik.hs-fulda.de +# +# changelog: +# V1.0 initial version + +# usage +if [ ! $# -gt 4 ] ; then + echo -e "usage: $0 [debug level], e.g.,\n" + echo "$0 /home/virl/git-virl-hs-fulda/GIT-VIRL-HS-Fulda/advcompnet-lab2-dcn-fabricpath.virl 6 guest password" + exit -1 +fi + +TOPOLOGY=$1 +RUNCOUNT=$2 +USERNAME=$3 +PASSWORD=$4 +TIMEOUT=$5 +if [ $6 ]; then + DEBUG=$6 +else + DEBUG=0 +fi + +TOPOLOGYFILENAME=$(basename $TOPOLOGY) +TIMESTAMP=$(date +%H%M%S) +FILENAME="VIRLBENCH-$TIMESTAMP@$TOPOLOGYFILENAME" + +DATE=$(date) +echo "Script started at $DATE" + +# start the simulations +RUN=0 +while [ $RUN -lt $RUNCOUNT ] +do + # use simengine-launch to start the simulation + if [ $DEBUG -lt 2 ]; then + virl_std_client --username $USERNAME --password $PASSWORD simengine-launch \ + --virl-file $TOPOLOGY --file-name $FILENAME >/dev/null 2>&1 + else + virl_std_client --username $USERNAME --password $PASSWORD simengine-launch \ + --virl-file $TOPOLOGY --file-name $FILENAME + fi + RUN=$(expr $RUN + 1) +done + +DATE=$(date) +echo "Started at $DATE" + +RUNNING=true + +# wait for simulations to start +sleep 5 + +# get all running benchmark simulations +SIMS=$(virl_std_client --username $USERNAME --password $PASSWORD --quiet --json simengine-list 2>&1 | egrep -o -e "VIRLBENCH-$TIMESTAMP@(.*)-[_a-zA-Z0-9]{6}") +if [ $DEBUG -gt 1 ]; then + echo "Running simulations:" + for SIM in $SIMS; do + echo $SIM + done + echo +fi + +# count nodes in the running benchmark simulations +NODE_COUNT=0 +for SIM in $SIMS; do + SIM_NODE_COUNT=$(virl_std_client --username $USERNAME --password $PASSWORD simengine-nodes --session-id $SIM 2>&1 | grep -c state) + if [ $DEBUG -gt 1 ]; then + echo "$SIM has $SIM_NODE_COUNT nodes" + fi + NODE_COUNT=$(expr $NODE_COUNT + $SIM_NODE_COUNT) +done +if [ $DEBUG -gt 0 ] ; then echo "$NODE_COUNT nodes"; fi + +# continuously count all ACTIVE nodes in running benchmark simulations until all nodes are ACTIVE +ACTIVE_COUNT=0 +while [ $ACTIVE_COUNT -lt $NODE_COUNT ] +do + ACTIVE_COUNT=0 + for SIM in $SIMS; do + SIM_NODE_ACTIVE_COUNT=$(virl_std_client --username $USERNAME --password $PASSWORD simengine-nodes --session-id $SIM 2>&1 | grep -c ACTIVE) + if [ $DEBUG -gt 1 ]; then + echo "$SIM has $SIM_NODE_ACTIVE_COUNT active nodes" + fi + ACTIVE_COUNT=$(expr $ACTIVE_COUNT + $SIM_NODE_ACTIVE_COUNT) + done + + if [ $DEBUG -gt 0 ] ; then echo "$ACTIVE_COUNT of $NODE_COUNT are ACTIVE"; fi + sleep 1 +done + +DATE=$(date) +echo "Active at $DATE" + +# Check if the nodes are ready-for-use using expect +./check-reachability $USERNAME $TIMEOUT "VIRLBENCH-$TIMESTAMP" $DEBUG + +# Check using load + +# Alternative to check if nodes are ready, use CPU load threshold for VMs (IOSv and IOSvL2 have a high CPU load during initial boot) +#THRESHOLD="70.0" + +#while $RUNNING = true +#do +# VM_LOADS=$(top -n 1 | grep kvm.real | tr -s " " | cut -d " " -f 9) +# +# VMS_WITH_HIGH_CPU_LOAD=0 +# for VM_LOAD in $VM_LOADS +# do +# if [ $(echo "$VM_LOAD > $THRESHOLD" | bc) -eq 1 ] ; then +# VMS_WITH_HIGH_CPU_LOAD=$(expr $VMS_WITH_HIGH_CPU_LOAD + 1) +# echo "$VM_LOAD > 6x0.0" +# fi +# done +# +# if [ $VMS_WITH_HIGH_CPU_LOAD -gt 0 ]; then +# echo "cpu load = high" +# else +# RUNNING=false +# fi +#done + +DATE=$(date) +echo "Finished at $DATE" diff --git a/virl-benchmark/stop-virl-benchmark b/virl-benchmark/stop-virl-benchmark new file mode 100644 index 0000000..3c38f70 --- /dev/null +++ b/virl-benchmark/stop-virl-benchmark @@ -0,0 +1,27 @@ +#!/bin/bash +# stop-virl-topology-benchmark.sh +# HS-Fulda - sebastian.rieger@informatik.hs-fulda.de +# +# changelog: +# V1.0 initial version + +# usage +if [ ! $# -eq 2 ] ; then + echo -e "usage: $0 , e.g.,\n" + echo "$0 guest password" + exit -1 +fi + +USERNAME=$1 +PASSWORD=$2 + +# get all running benchmark simulations +RUNNING_BENCHMARKS=$(virl_std_client --username $USERNAME --password $PASSWORD --quiet --json \ + simengine-list | egrep -o -e "VIRLBENCH(.*)@(.*)-[_a-zA-Z0-9]{6}") + +# stop each benchmark simulation +for BENCHMARK in $RUNNING_BENCHMARKS; do + echo "Stopping $BENCHMARK" + virl_std_client --username $USERNAME --password $PASSWORD --quiet \ + simengine-stop --session-id $BENCHMARK +done \ No newline at end of file diff --git a/virl-benchmark/test-virl-telnet-connection b/virl-benchmark/test-virl-telnet-connection new file mode 100644 index 0000000..71f4a39 --- /dev/null +++ b/virl-benchmark/test-virl-telnet-connection @@ -0,0 +1,17 @@ +#!/usr/bin/expect + +# connect to telnet port, press return and expect argv 2 (hostname) in +# prompt, press return again and expect to receive argv 2 again + +set timeout [lindex $argv 3] +spawn telnet [lindex $argv 0] [lindex $argv 1] +send "\r" +expect { + [lindex $argv 2] +} +send "\r" +expect { + [lindex $argv 2] + { exit 0 } +} +exit 5