- initial version of virl-benchmark to automatically start multiple

simulations, measure the delay until the simulations are active and the delay until the telnet ports of all nodes can be used
9 years ago · 63a64e2f27
5 changed files with 361 additions and 0 deletions
--- a/virl-benchmark/check-reachability
+++ b/virl-benchmark/check-reachability
@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Check reachability of all running VIRL VMs by logging in on the telnet console port and expect a prompt that includes the host name
+
+# usage
+if [ ! $# -gt 2 ] ; then
+  echo -e "usage: $0 <username> <timeout for telnet connection to each node> <simulation> [debug level], e.g.,\n"
+  echo "$0 guest VIRLBENCH-223721@smb-WaZLhH 5"
+  exit -1
+fi
+
+# get the IP address of eth0
+IP=$(ip addr show | grep eth0 | grep inet | tr -s " " | cut -d " " -f 3 | cut -d "/" -f 1)
+
+RUNNING=true
+
+USERNAME=$1
+TIMEOUT=$2
+SIMULATION=$3
+if [ $4 ]; then 
+  DEBUG=$4
+else
+  DEBUG=0
+fi
+
+while $RUNNING = true
+do
+# Check if all nodes are ready-to-use using expect
+
+  # get the telnet ports of all nodes in all simulations
+  VM_PORTS=$(./ports-vms.py $USERNAME $SIMULATION)
+  
+  # connect to every telnet port of each node and expect the hostname in the prompt
+  VMS_UNUSABLE=0
+  for VM_PORT in $VM_PORTS
+  do
+    VM_TELNET_PORT=$(echo $VM_PORT | cut -d "=" -f 2)
+    VM_NAME=$(echo $VM_PORT | cut -d "=" -f 1)
+    # connect to every telnet port and check whether it can be used by pressing return
+    # twice and expecting the hostname to appear in the resulting prompt each time
+    if [ $DEBUG -lt 2 ]; then
+    	./test-virl-telnet-connection $IP $VM_TELNET_PORT $VM_NAME $TIMEOUT >/dev/null
+    else
+    	./test-virl-telnet-connection $IP $VM_TELNET_PORT $VM_NAME $TIMEOUT
+    fi
+    EXPECT_EXITCODE=$?
+    if [ $EXPECT_EXITCODE -eq 5 ] ; then
+      VMS_UNUSABLE=$(expr $VMS_UNUSABLE + 1)
+      if [ $DEBUG -gt 0 ]; then echo "$VM_NAME ($VM_TELNET_PORT) still unusable"; fi
+    fi
+  done
+  
+  if [ $VMS_UNUSABLE -gt 0 ]; then
+    if [ $DEBUG -gt 0 ]; then echo "$VMS_UNUSABLE VMs are still unusable"; fi
+  else
+    RUNNING=false
+  fi
+
+  sleep 1
+
+done
+
+DATE=$(date)
+echo "Finished at $DATE"
--- a/virl-benchmark/ports-vms.py
+++ b/virl-benchmark/ports-vms.py
@ -0,0 +1,127 @@
+#!/usr/bin/env python
+#
+# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
+#
+# Show all ports of all VM instances for
+# all users and topologies.
+# Like the console port and the VNC port
+#
+# rschmied@cisco.com
+#
+# modified by sebastian.rieger@informatik.hs-fulda.de to display telnet console ports of running nodes in a topology
+#
+
+
+import os, libvirt, re, sys
+from keystoneclient import client as keystone
+from novaclient import client as nova
+from xml.dom.minidom import parseString
+
+"""
+
+In [14]: m.group(0)
+Out[14]: '</guest/endpoint>-<Sample_Project@asa-test-topo-yJHnoK>-<iosv-2>-<Multipoint Connection-1>'
+
+In [15]: m.group(1)
+Out[15]: '/guest/endpoint'
+
+In [16]: m.group(2)
+Out[16]: 'Sample_Project'
+
+In [16]: m.group(3)
+Out[16]: 'asa-test-topo'
+
+In [17]: m.group(4)
+Out[17]: 'iosv-2'
+
+In [18]: m.group(5)
+Out[18]: 'Multipoint Connection-1'
+
+will not match jumphost ports!
+not interested in these, anyway
+
+"""
+
+class KeystoneV3NovaAuthPlugin(object):
+    def __init__(self, keystone_client):
+        self.keystone_client = keystone_client
+
+    def authenticate(self, client, fake_auth_url):
+        client.auth_url = fake_auth_url
+        client.service_catalog = self.keystone_client.service_catalog
+        client.auth_token = self.keystone_client.auth_token
+        client.tenant_id = self.keystone_client.tenant_id
+        client.management_url = self.keystone_client.service_catalog.url_for(
+            attr='region',
+            filter_value=client.region_name,
+            endpoint_type=client.endpoint_type,
+            service_type=client.service_type,
+            service_name=client.service_name).rstrip('/')
+
+def getports(user,simulation):
+	# Sample output / field mapping
+	# </guest/endpoint>-<Sample_Topologies@single-server-WO9N_h>-<csr1000v-1>
+	#         USER            PROJECT          TOPOLOGY             NODE
+	
+	# </advcompnet/endpoint>-<advcompnet-lab1-dcn-scenario1-ymaMSJ>-<veos-4>
+	#   USER                  TOPOLOGY                               NODE
+	prog=re.compile(r'</(.*)/endpoint>-<(.*)-[_0-9a-z]{6}>-<(.*)>', re.IGNORECASE)
+
+	# table=list()
+	try:
+		libvirt_uri = os.environ['LIBVIRT_DEFAULT_URI']
+	except:
+		libvirt_uri = "qemu:///system"
+		print "LIBVIRT_DEFAULT_URI env not set!"
+		print "Using default '" + libvirt_uri + "'"
+	conn=libvirt.openReadOnly(libvirt_uri)
+	
+	kc = keystone.Client(auth_url=os.environ['OS_AUTH_URL'],
+                     username=os.environ['OS_USERNAME'], password=os.environ['OS_PASSWORD'],
+                     project_name=os.environ['OS_TENANT_NAME'])
+	kc.session.auth = kc
+	kc.authenticate()
+	nc=nova.Client('2', os.environ['OS_USERNAME'], os.environ['OS_PASSWORD'], 
+		os.environ['OS_TENANT_NAME'], auth_system='keystonev3', auth_plugin=KeystoneV3NovaAuthPlugin(kc), auth_url='http://fake/v2.0')
+
+	for server in nc.servers.list(search_opts={'all_tenants': True}):
+		m=prog.match(server.name)
+		if m:
+			try:
+				domain=conn.lookupByUUIDString(server.id)
+			except:
+				print "Domain not found / not running"
+				return 1
+			else:
+				doc=parseString(domain.XMLDesc(flags=0))
+			# get the VNC port
+			#port=doc.getElementsByTagName('graphics')[0].getAttribute('port')
+			# get the serial console TCP port
+			for i in doc.getElementsByTagName('source'):
+				if i.parentNode.nodeName == u'console':
+					console=i.getAttribute('service')
+			# get the instance name
+			name=doc.getElementsByTagName('name')[0].childNodes[0].nodeValue
+			# print info
+			if simulation == "*":
+			        if m.group(1) == user:
+			                print m.group(3) + "=" + console
+                        else:
+			        if m.group(1) == user and server.name.find(simulation) != -1:
+			                print m.group(3) + "=" + console
+                       
+
+def main():
+        if len(sys.argv) != 3:
+                sys.stdout.write(str(sys.argv[0]))
+                print ": username and simulation (e.g., project name or session-id) needed as argument! bailing out"
+                return 1
+        else:
+                user = str(sys.argv[1]).strip()
+                simulation = str(sys.argv[2]).strip()
+                getports(user,simulation)
+                return 0
+
+if __name__ == '__main__':
+	sys.exit(main())
+
--- a/virl-benchmark/start-virl-topology
+++ b/virl-benchmark/start-virl-topology
@ -0,0 +1,126 @@
+#!/bin/bash
+# start-virl-topology-benchmark.sh
+# HS-Fulda - sebastian.rieger@informatik.hs-fulda.de
+#
+# changelog:
+# V1.0   initial version
+
+# usage
+if [ ! $# -gt 4 ] ; then
+  echo -e "usage: $0 <my-topology.virl> <number of concurrent simulations> <username> <password> <timeout> [debug level], e.g.,\n"
+  echo "$0 /home/virl/git-virl-hs-fulda/GIT-VIRL-HS-Fulda/advcompnet-lab2-dcn-fabricpath.virl 6 guest password"
+  exit -1
+fi
+
+TOPOLOGY=$1
+RUNCOUNT=$2
+USERNAME=$3
+PASSWORD=$4
+TIMEOUT=$5
+if [ $6 ]; then
+  DEBUG=$6
+else
+  DEBUG=0
+fi
+
+TOPOLOGYFILENAME=$(basename $TOPOLOGY)
+TIMESTAMP=$(date +%H%M%S)
+FILENAME="VIRLBENCH-$TIMESTAMP@$TOPOLOGYFILENAME"
+
+DATE=$(date)
+echo "Script started at $DATE"
+
+# start the simulations
+RUN=0
+while [ $RUN -lt $RUNCOUNT ]
+do
+  # use simengine-launch to start the simulation
+  if [ $DEBUG -lt 2 ]; then
+    virl_std_client --username $USERNAME --password $PASSWORD simengine-launch \
+      --virl-file $TOPOLOGY --file-name $FILENAME >/dev/null 2>&1
+  else
+    virl_std_client --username $USERNAME --password $PASSWORD simengine-launch \
+      --virl-file $TOPOLOGY --file-name $FILENAME
+  fi
+  RUN=$(expr $RUN + 1)
+done
+
+DATE=$(date)
+echo "Started at $DATE"
+
+RUNNING=true
+
+# wait for simulations to start
+sleep 5
+
+# get all running benchmark simulations
+SIMS=$(virl_std_client --username $USERNAME --password $PASSWORD --quiet --json simengine-list 2>&1 | egrep -o -e "VIRLBENCH-$TIMESTAMP@(.*)-[_a-zA-Z0-9]{6}")
+if [ $DEBUG -gt 1 ]; then
+  echo "Running simulations:"
+  for SIM in $SIMS; do
+    echo $SIM
+  done
+  echo
+fi
+        
+# count nodes in the running benchmark simulations
+NODE_COUNT=0
+for SIM in $SIMS; do
+  SIM_NODE_COUNT=$(virl_std_client --username $USERNAME --password $PASSWORD simengine-nodes --session-id $SIM 2>&1 | grep -c state)
+  if [ $DEBUG -gt 1 ]; then
+    echo "$SIM has $SIM_NODE_COUNT nodes"
+  fi
+  NODE_COUNT=$(expr $NODE_COUNT + $SIM_NODE_COUNT)
+done
+if [ $DEBUG -gt 0 ] ; then echo "$NODE_COUNT nodes"; fi
+
+# continuously count all ACTIVE nodes in running benchmark simulations until all nodes are ACTIVE
+ACTIVE_COUNT=0
+while [ $ACTIVE_COUNT -lt $NODE_COUNT ]
+do
+  ACTIVE_COUNT=0
+  for SIM in $SIMS; do
+    SIM_NODE_ACTIVE_COUNT=$(virl_std_client --username $USERNAME --password $PASSWORD simengine-nodes --session-id $SIM 2>&1 | grep -c ACTIVE)
+    if [ $DEBUG -gt 1 ]; then
+      echo "$SIM has $SIM_NODE_ACTIVE_COUNT active nodes"
+    fi
+    ACTIVE_COUNT=$(expr $ACTIVE_COUNT + $SIM_NODE_ACTIVE_COUNT)
+  done
+
+  if [ $DEBUG -gt 0 ] ; then echo "$ACTIVE_COUNT of $NODE_COUNT are ACTIVE"; fi
+  sleep 1
+done
+
+DATE=$(date)
+echo "Active at $DATE"
+
+# Check if the nodes are ready-for-use using expect
+./check-reachability $USERNAME $TIMEOUT "VIRLBENCH-$TIMESTAMP" $DEBUG
+
+# Check using load
+
+# Alternative to check if nodes are ready, use CPU load threshold for VMs (IOSv and IOSvL2 have a high CPU load during initial boot)
+#THRESHOLD="70.0"
+
+#while $RUNNING = true
+#do
+#  VM_LOADS=$(top -n 1 | grep kvm.real | tr -s " " | cut -d " " -f 9)
+#
+#  VMS_WITH_HIGH_CPU_LOAD=0
+#  for VM_LOAD in $VM_LOADS
+#  do
+#    if [ $(echo "$VM_LOAD > $THRESHOLD" | bc) -eq 1 ] ; then 
+#      VMS_WITH_HIGH_CPU_LOAD=$(expr $VMS_WITH_HIGH_CPU_LOAD + 1)
+#      echo "$VM_LOAD > 6x0.0"
+#    fi
+#  done
+#  
+#  if [ $VMS_WITH_HIGH_CPU_LOAD -gt 0 ]; then
+#    echo "cpu load = high"
+#  else
+#    RUNNING=false
+#  fi
+#done
+
+DATE=$(date)
+echo "Finished at $DATE"
--- a/virl-benchmark/stop-virl-benchmark
+++ b/virl-benchmark/stop-virl-benchmark
@ -0,0 +1,27 @@
+#!/bin/bash
+# stop-virl-topology-benchmark.sh
+# HS-Fulda - sebastian.rieger@informatik.hs-fulda.de
+#
+# changelog:
+# V1.0   initial version
+
+# usage
+if [ ! $# -eq 2 ] ; then
+  echo -e "usage: $0 <username> <password>, e.g.,\n"
+  echo "$0 guest password"
+  exit -1
+fi
+
+USERNAME=$1
+PASSWORD=$2
+
+# get all running benchmark simulations
+RUNNING_BENCHMARKS=$(virl_std_client --username $USERNAME --password $PASSWORD --quiet --json \
+  simengine-list | egrep -o -e "VIRLBENCH(.*)@(.*)-[_a-zA-Z0-9]{6}")
+
+# stop each benchmark simulation
+for BENCHMARK in $RUNNING_BENCHMARKS; do
+  echo "Stopping $BENCHMARK"
+  virl_std_client --username $USERNAME --password $PASSWORD --quiet \
+    simengine-stop --session-id $BENCHMARK
+done
--- a/virl-benchmark/test-virl-telnet-connection
+++ b/virl-benchmark/test-virl-telnet-connection
@ -0,0 +1,17 @@
+#!/usr/bin/expect
+
+# connect to telnet port, press return and expect argv 2 (hostname) in
+# prompt, press return again and expect to receive argv 2 again 
+
+set timeout [lindex $argv 3]
+spawn telnet [lindex $argv 0] [lindex $argv 1]
+send "\r"
+expect { 
+  [lindex $argv 2]
+}
+send "\r"
+expect { 
+  [lindex $argv 2]
+  { exit 0 }
+}
+exit 5