Merge "Add timeout handler when running on the bots"
diff --git a/build.gradle b/build.gradle
index ec00f11..64f09d7 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1343,6 +1343,9 @@
println "Start executing test ${desc.name} [${desc.className}]"
}
afterTest { desc, result ->
+ if (project.hasProperty('update_test_timestamp')) {
+ file(project.getProperty('update_test_timestamp')).text = new Date().getTime()
+ }
println "Done executing test ${desc.name} [${desc.className}] with result: ${result.resultType}"
}
}
diff --git a/tools/archive.py b/tools/archive.py
index ad58bca..013ff15 100755
--- a/tools/archive.py
+++ b/tools/archive.py
@@ -79,7 +79,7 @@
return GetVersionDestination('http://storage.googleapis.com/', '', is_master)
def Main():
- if not 'BUILDBOT_BUILDERNAME' in os.environ:
+ if utils.is_bot():
raise Exception('You are not a bot, don\'t archive builds')
# Generate an r8-ed build without dependencies.
diff --git a/tools/test.py b/tools/test.py
index e4433b3..d9dca9f 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -12,6 +12,8 @@
import optparse
import subprocess
import sys
+import thread
+import time
import utils
import notify
@@ -25,6 +27,14 @@
"4.4.4",
"4.0.4"]
+# How often do we check for progress on the bots:
+# Should be long enough that a normal run would always have med progress
+# Should be short enough that we ensure that two calls are close enough
+# to happen before bot times out.
+# A false positiv, i.e., printing the stacks of non hanging processes
+# is not a problem, no harm done except some logging in the stdout.
+TIMEOUT_HANDLER_PERIOD = 60 * 18
+
def ParseOptions():
result = optparse.OptionParser()
result.add_option('--no-internal', '--no_internal',
@@ -95,7 +105,7 @@
def Main():
(options, args) = ParseOptions()
- if 'BUILDBOT_BUILDERNAME' in os.environ:
+ if utils.is_bot():
gradle.RunGradle(['clean'])
# Build R8lib with dependencies for bootstrapping tests before adding test sources
@@ -171,6 +181,12 @@
'%s.tar.gz' % sha1)
utils.unpack_archive('%s.tar.gz' % sha1)
+ if utils.is_bot() and not utils.IsWindows():
+ timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
+ if os.path.exists(timestamp_file):
+ os.remove(timestamp_file)
+ gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
+ thread.start_new_thread(timeout_handler, (timestamp_file,))
# Now run tests on selected runtime(s).
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
@@ -193,6 +209,45 @@
return 0
+
+def print_jstacks():
+ processes = subprocess.check_output(['ps', 'aux'])
+ for l in processes.splitlines():
+ if 'java' in l and 'openjdk' in l:
+ # Example line:
+ # ricow 184313 2.6 0.0 36839068 31808 ? Sl 09:53 0:00 /us..
+ columns = l.split()
+ pid = columns[1]
+ return_value = subprocess.call(['jstack', pid])
+ if return_value:
+ print('Could not jstack %s' % l)
+ print('----') # May be eaten by gradle prints.
+ print('----') # May be eaten by gradle prints.
+
+def get_time_from_file(timestamp_file):
+ if os.path.exists(timestamp_file):
+ timestamp = os.stat(timestamp_file).st_mtime
+ print('TIMEOUT HANDLER timestamp: %s' % (timestamp))
+ print('---') # May be eaten by gradle prints.
+ print('---') # May be eaten by gradle prints.
+ sys.stdout.flush()
+ return timestamp
+ else:
+ print('TIMEOUT HANDLER no timestamp file yet')
+ print('---') # May be eaten by gradle prints.
+ print('---') # May be eaten by gradle prints.
+ sys.stdout.flush()
+ return None
+
+def timeout_handler(timestamp_file):
+ last_timestamp = None
+ while True:
+ time.sleep(TIMEOUT_HANDLER_PERIOD)
+ new_timestamp = get_time_from_file(timestamp_file)
+ if last_timestamp and new_timestamp == last_timestamp:
+ print_jstacks()
+ last_timestamp = new_timestamp
+
if __name__ == '__main__':
return_code = Main()
if return_code != 0:
diff --git a/tools/utils.py b/tools/utils.py
index d2c4935..5355ba8 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -313,3 +313,6 @@
def get_android_jar(api):
return os.path.join(REPO_ROOT, ANDROID_JAR.format(api=api))
+
+def is_bot():
+ return 'BUILDBOT_BUILDERNAME' in os.environ
