added global timeout to detect otherwise endless running checks, when nodes crash or resources are depleted

gns3-bench.py

@@ -27,6 +27,7 @@ PORT = 13080
 PASSWORD = sys.argv[4]
 
 TIMEOUT = int(sys.argv[5])
+GLOBAL_TIMEOUT = 1000
 
 PROJECT_NAME = sys.argv[1]
 # PROJECT_ID = "69e293ac-a339-4d1d-b2e3-047dc2da8566"
@@ -141,6 +142,30 @@ for run in range(1, NUMBER_OF_BENCHMARK_RUNS + 1):
     # check if nodes are usable
     usable_nodes = []
     while len(nodes) > 0:
+
+        # perf output
+        # print(os.popen("top -b -n 1 | head -20 >> gns3bench-perf.log"))
+
+        time_globalTimeout = time.time()
+        # if usability of nodes is checked longer than global timeout, exit
+        # this can happen if nodes get stuck, e.g., due to high load, missing resources etc.
+        if time_globalTimeout - time_active > GLOBAL_TIMEOUT:
+            date_usable = 0
+            durationUsable = 0
+            avgConsoleDelay = 0
+            durationStart = time_started - time_start
+            durationActive = time_active - time_start
+            results.append("%s;%i;%i;%s;%s;%s;%s;%i;%i;%i;%f" % (
+                (PROJECT_NAME + ": TIMEOUT"), NUMBER_OF_CONCURRENT_BENCHMARK_PROJECTS, nodeCount, date_start,
+                date_started, date_active, date_usable, durationStart, durationActive, durationUsable, avgConsoleDelay))
+            results.append("  Occurred while still checking nodes: " + str(nodes))
+
+            f = open("gns3bench-error.log", "a")
+            f.writelines(results)
+            f.write("\r\n")
+            f.close()
+            sys.exit("global timeout")
+
         for entry in nodes:
             print("Checking node %s in %s..." % (entry["name"], entry["project_id"]))
             try: