From 2e978098c3e9ceddf3c98c7549d87dc4393ada34 Mon Sep 17 00:00:00 2001 From: Sebastian Rieger Date: Sat, 30 Dec 2017 23:06:18 +0100 Subject: [PATCH] added global timeout to detect otherwise endless running checks, when nodes crash or resources are depleted --- gns3-bench.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/gns3-bench.py b/gns3-bench.py index 83ce769..8de8789 100644 --- a/gns3-bench.py +++ b/gns3-bench.py @@ -27,6 +27,7 @@ PORT = 13080 PASSWORD = sys.argv[4] TIMEOUT = int(sys.argv[5]) +GLOBAL_TIMEOUT = 1000 PROJECT_NAME = sys.argv[1] # PROJECT_ID = "69e293ac-a339-4d1d-b2e3-047dc2da8566" @@ -141,6 +142,30 @@ for run in range(1, NUMBER_OF_BENCHMARK_RUNS + 1): # check if nodes are usable usable_nodes = [] while len(nodes) > 0: + + # perf output + # print(os.popen("top -b -n 1 | head -20 >> gns3bench-perf.log")) + + time_globalTimeout = time.time() + # if usability of nodes is checked longer than global timeout, exit + # this can happen if nodes get stuck, e.g., due to high load, missing resources etc. + if time_globalTimeout - time_active > GLOBAL_TIMEOUT: + date_usable = 0 + durationUsable = 0 + avgConsoleDelay = 0 + durationStart = time_started - time_start + durationActive = time_active - time_start + results.append("%s;%i;%i;%s;%s;%s;%s;%i;%i;%i;%f" % ( + (PROJECT_NAME + ": TIMEOUT"), NUMBER_OF_CONCURRENT_BENCHMARK_PROJECTS, nodeCount, date_start, + date_started, date_active, date_usable, durationStart, durationActive, durationUsable, avgConsoleDelay)) + results.append(" Occurred while still checking nodes: " + str(nodes)) + + f = open("gns3bench-error.log", "a") + f.writelines(results) + f.write("\r\n") + f.close() + sys.exit("global timeout") + for entry in nodes: print("Checking node %s in %s..." % (entry["name"], entry["project_id"])) try: