Add timeout handler when running on the bots
If there are no tests finishing in 18 minutes (at least, upper bound
36 minutes) we print the jstack of all java processes
Change-Id: I1132ffbb243e7c6a418d79ffac1637a161b8afbb
diff --git a/build.gradle b/build.gradle
index ec00f11..64f09d7 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1343,6 +1343,9 @@
println "Start executing test ${desc.name} [${desc.className}]"
}
afterTest { desc, result ->
+ if (project.hasProperty('update_test_timestamp')) {
+ file(project.getProperty('update_test_timestamp')).text = new Date().getTime()
+ }
println "Done executing test ${desc.name} [${desc.className}] with result: ${result.resultType}"
}
}
diff --git a/tools/archive.py b/tools/archive.py
index ad58bca..013ff15 100755
--- a/tools/archive.py
+++ b/tools/archive.py
@@ -79,7 +79,7 @@
return GetVersionDestination('http://storage.googleapis.com/', '', is_master)
def Main():
- if not 'BUILDBOT_BUILDERNAME' in os.environ:
+ if utils.is_bot():
raise Exception('You are not a bot, don\'t archive builds')
# Generate an r8-ed build without dependencies.
diff --git a/tools/test.py b/tools/test.py
index e4433b3..d9dca9f 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -12,6 +12,8 @@
import optparse
import subprocess
import sys
+import thread
+import time
import utils
import notify
@@ -25,6 +27,14 @@
"4.4.4",
"4.0.4"]
+# How often do we check for progress on the bots:
+# Should be long enough that a normal run would always have med progress
+# Should be short enough that we ensure that two calls are close enough
+# to happen before bot times out.
+# A false positiv, i.e., printing the stacks of non hanging processes
+# is not a problem, no harm done except some logging in the stdout.
+TIMEOUT_HANDLER_PERIOD = 60 * 18
+
def ParseOptions():
result = optparse.OptionParser()
result.add_option('--no-internal', '--no_internal',
@@ -95,7 +105,7 @@
def Main():
(options, args) = ParseOptions()
- if 'BUILDBOT_BUILDERNAME' in os.environ:
+ if utils.is_bot():
gradle.RunGradle(['clean'])
# Build R8lib with dependencies for bootstrapping tests before adding test sources
@@ -171,6 +181,12 @@
'%s.tar.gz' % sha1)
utils.unpack_archive('%s.tar.gz' % sha1)
+ if utils.is_bot() and not utils.IsWindows():
+ timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
+ if os.path.exists(timestamp_file):
+ os.remove(timestamp_file)
+ gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
+ thread.start_new_thread(timeout_handler, (timestamp_file,))
# Now run tests on selected runtime(s).
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
@@ -193,6 +209,45 @@
return 0
+
+def print_jstacks():
+ processes = subprocess.check_output(['ps', 'aux'])
+ for l in processes.splitlines():
+ if 'java' in l and 'openjdk' in l:
+ # Example line:
+ # ricow 184313 2.6 0.0 36839068 31808 ? Sl 09:53 0:00 /us..
+ columns = l.split()
+ pid = columns[1]
+ return_value = subprocess.call(['jstack', pid])
+ if return_value:
+ print('Could not jstack %s' % l)
+ print('----') # May be eaten by gradle prints.
+ print('----') # May be eaten by gradle prints.
+
+def get_time_from_file(timestamp_file):
+ if os.path.exists(timestamp_file):
+ timestamp = os.stat(timestamp_file).st_mtime
+ print('TIMEOUT HANDLER timestamp: %s' % (timestamp))
+ print('---') # May be eaten by gradle prints.
+ print('---') # May be eaten by gradle prints.
+ sys.stdout.flush()
+ return timestamp
+ else:
+ print('TIMEOUT HANDLER no timestamp file yet')
+ print('---') # May be eaten by gradle prints.
+ print('---') # May be eaten by gradle prints.
+ sys.stdout.flush()
+ return None
+
+def timeout_handler(timestamp_file):
+ last_timestamp = None
+ while True:
+ time.sleep(TIMEOUT_HANDLER_PERIOD)
+ new_timestamp = get_time_from_file(timestamp_file)
+ if last_timestamp and new_timestamp == last_timestamp:
+ print_jstacks()
+ last_timestamp = new_timestamp
+
if __name__ == '__main__':
return_code = Main()
if return_code != 0:
diff --git a/tools/utils.py b/tools/utils.py
index d2c4935..5355ba8 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -313,3 +313,6 @@
def get_android_jar(api):
return os.path.join(REPO_ROOT, ANDROID_JAR.format(api=api))
+
+def is_bot():
+ return 'BUILDBOT_BUILDERNAME' in os.environ
