blob: 733d190db830915d305dcc25106355761e9982bd [file] [log] [blame]
Mads Ager418d1ca2017-05-22 09:35:49 +02001# Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file
2# for details. All rights reserved. Use of this source code is governed by a
3# BSD-style license that can be found in the LICENSE file.
4
5# Different utility functions used accross scripts
6
7import hashlib
Søren Gjesse6e5e5842019-09-03 08:48:30 +02008import json
Mads Ager418d1ca2017-05-22 09:35:49 +02009import os
Tamas Kenez82efeb52017-06-12 13:56:22 +020010import re
Mads Ager418d1ca2017-05-22 09:35:49 +020011import shutil
12import subprocess
13import sys
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +020014import tarfile
Mads Ager418d1ca2017-05-22 09:35:49 +020015import tempfile
Morten Krogh-Jespersen16e925d2019-01-25 14:40:38 +010016import zipfile
Mads Ager418d1ca2017-05-22 09:35:49 +020017
Ian Zerny37097652019-04-11 13:13:27 +020018import defines
19import jdk
20
Christoffer Quist Adamsen17879c12019-01-22 16:13:54 +010021ANDROID_JAR_DIR = 'third_party/android_jar/lib-v{api}'
22ANDROID_JAR = os.path.join(ANDROID_JAR_DIR, 'android.jar')
Ian Zerny3f54e222019-02-12 10:51:17 +010023TOOLS_DIR = defines.TOOLS_DIR
24REPO_ROOT = defines.REPO_ROOT
25THIRD_PARTY = defines.THIRD_PARTY
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +010026ANDROID_SDK = os.path.join(THIRD_PARTY, 'android_sdk')
Tamas Kenezfc34cd82017-07-13 12:43:57 +020027MEMORY_USE_TMP_FILE = 'memory_use.tmp'
Tamas Kenez02bff032017-07-18 12:13:58 +020028DEX_SEGMENTS_RESULT_PATTERN = re.compile('- ([^:]+): ([0-9]+)')
Mads Ager12a56bc2017-11-27 11:51:25 +010029BUILD = os.path.join(REPO_ROOT, 'build')
Ian Zerny0f5fc732018-11-15 14:34:41 +010030BUILD_DEPS_DIR = os.path.join(BUILD, 'deps')
31BUILD_MAIN_DIR = os.path.join(BUILD, 'classes', 'main')
32BUILD_TEST_DIR = os.path.join(BUILD, 'classes', 'test')
Mads Ager12a56bc2017-11-27 11:51:25 +010033LIBS = os.path.join(BUILD, 'libs')
34GENERATED_LICENSE_DIR = os.path.join(BUILD, 'generatedLicense')
Mads Agera4911eb2017-11-22 13:19:36 +010035SRC_ROOT = os.path.join(REPO_ROOT, 'src', 'main', 'java')
Søren Gjesse6e5e5842019-09-03 08:48:30 +020036TEST_ROOT = os.path.join(REPO_ROOT, 'src', 'test', 'java')
Ian Zerny59dfa4c2019-10-25 10:34:36 +020037REPO_SOURCE = 'https://r8.googlesource.com/r8'
Søren Gjessedc9d8a22017-10-12 12:40:59 +020038
39D8 = 'd8'
40R8 = 'r8'
Tamas Kenez03ab76f2018-12-07 14:33:25 +010041R8LIB = 'r8lib'
Morten Krogh-Jespersene28db462019-01-09 13:32:15 +010042R8LIB_NO_DEPS = 'r8LibNoDeps'
Mads Agerb10c07f2017-11-27 13:25:52 +010043R8_SRC = 'sourceJar'
Søren Gjessedc9d8a22017-10-12 12:40:59 +020044COMPATDX = 'compatdx'
Tamas Kenez03ab76f2018-12-07 14:33:25 +010045COMPATDXLIB = 'compatdxlib'
Søren Gjessedc9d8a22017-10-12 12:40:59 +020046COMPATPROGUARD = 'compatproguard'
Tamas Kenez03ab76f2018-12-07 14:33:25 +010047COMPATPROGUARDLIB = 'compatproguardlib'
Søren Gjessedc9d8a22017-10-12 12:40:59 +020048
Rico Wind74fab302017-10-02 07:25:33 +020049D8_JAR = os.path.join(LIBS, 'd8.jar')
50R8_JAR = os.path.join(LIBS, 'r8.jar')
Tamas Kenez03ab76f2018-12-07 14:33:25 +010051R8LIB_JAR = os.path.join(LIBS, 'r8lib.jar')
Mads Agerb10c07f2017-11-27 13:25:52 +010052R8_SRC_JAR = os.path.join(LIBS, 'r8-src.jar')
Tamas Kenez03ab76f2018-12-07 14:33:25 +010053R8LIB_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8lib-exclude-deps.jar')
Tamas Kenez180be092018-12-05 15:23:06 +010054R8_FULL_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8-full-exclude-deps.jar')
Rico Wind74fab302017-10-02 07:25:33 +020055COMPATDX_JAR = os.path.join(LIBS, 'compatdx.jar')
Tamas Kenez03ab76f2018-12-07 14:33:25 +010056COMPATDXLIB_JAR = os.path.join(LIBS, 'compatdxlib.jar')
Søren Gjessedc9d8a22017-10-12 12:40:59 +020057COMPATPROGUARD_JAR = os.path.join(LIBS, 'compatproguard.jar')
Tamas Kenez03ab76f2018-12-07 14:33:25 +010058COMPATPROGUARDLIB_JAR = os.path.join(LIBS, 'compatproguardlib.jar')
Mads Ager12a56bc2017-11-27 11:51:25 +010059MAVEN_ZIP = os.path.join(LIBS, 'r8.zip')
Rico Wind8fc8bfa2019-03-22 09:57:36 +010060MAVEN_ZIP_LIB = os.path.join(LIBS, 'r8lib.zip')
Søren Gjesse6e5e5842019-09-03 08:48:30 +020061# TODO(b/134732760): The JSON configuration should be moved.
62DESUGAR_CONFIGURATION = os.path.join(
63 TEST_ROOT,
64 'com',
65 'android',
66 'tools',
67 'r8',
68 'desugar',
69 'corelib',
70 'desugar_jdk_libs.json')
71DESUGAR_CONFIGURATION_MAVEN_ZIP = os.path.join(
72 LIBS, 'desugar_jdk_libs_configuration.zip')
Mads Ager12a56bc2017-11-27 11:51:25 +010073GENERATED_LICENSE = os.path.join(GENERATED_LICENSE_DIR, 'LICENSE')
Mathias Rav3fb4a3a2018-05-29 15:41:36 +020074RT_JAR = os.path.join(REPO_ROOT, 'third_party/openjdk/openjdk-rt-1.8/rt.jar')
Mathias Ravb46dc002018-06-06 09:37:11 +020075R8LIB_KEEP_RULES = os.path.join(REPO_ROOT, 'src/main/keep.txt')
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +010076RETRACE_JAR = os.path.join(
77 THIRD_PARTY,
78 'proguard',
79 'proguard6.0.1',
80 'lib',
81 'retrace.jar')
Ian Zerny37097652019-04-11 13:13:27 +020082PROGUARD_JAR = os.path.join(
83 THIRD_PARTY,
84 'proguard',
85 'proguard6.0.1',
86 'lib',
87 'proguard.jar')
Morten Krogh-Jespersen480784d2019-02-05 08:10:46 +010088CF_SEGMENTS_TOOL = os.path.join(THIRD_PARTY, 'cf_segments')
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +010089PINNED_R8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8.jar')
90PINNED_PGR8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8-pg6.0.1.jar')
Ian Zernyfbb1f7a2019-05-02 14:34:13 +020091SAMPLE_LIBRARIES_SHA_FILE = os.path.join(
92 THIRD_PARTY, 'sample_libraries.tar.gz.sha1')
Morten Krogh-Jespersen64740202019-02-07 13:06:04 +010093OPENSOURCE_APPS_SHA_FILE = os.path.join(
Rico Windec283b52019-04-03 15:16:55 +020094 THIRD_PARTY, 'opensource_apps.tar.gz.sha1')
95OPENSOURCE_APPS_FOLDER = os.path.join(THIRD_PARTY, 'opensource_apps')
Søren Gjesse1c115b52019-08-14 12:43:57 +020096BAZEL_SHA_FILE = os.path.join(THIRD_PARTY, 'bazel.tar.gz.sha1')
97BAZEL_TOOL = os.path.join(THIRD_PARTY, 'bazel')
Søren Gjesse699f6362019-10-09 14:56:33 +020098JAVA8_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'jdk8', 'linux-x86.tar.gz.sha1')
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +010099
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100100ANDROID_HOME_ENVIROMENT_NAME = "ANDROID_HOME"
101ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME = "ANDROID_TOOLS_VERSION"
Morten Krogh-Jespersenc8efedd2019-01-28 11:36:17 +0100102USER_HOME = os.path.expanduser('~')
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100103
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200104R8_TEST_RESULTS_BUCKET = 'r8-test-results'
105
106def archive_file(name, gs_dir, src_file):
107 gs_file = '%s/%s' % (gs_dir, name)
108 upload_file_to_cloud_storage(src_file, gs_file, public_read=False)
109
110def archive_value(name, gs_dir, value):
111 with TempDir() as temp:
112 tempfile = os.path.join(temp, name);
113 with open(tempfile, 'w') as f:
114 f.write(str(value))
115 archive_file(name, gs_dir, tempfile)
116
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100117def getAndroidHome():
118 return os.environ.get(
119 ANDROID_HOME_ENVIROMENT_NAME, os.path.join(USER_HOME, 'Android', 'Sdk'))
120
121def getAndroidBuildTools():
122 version = os.environ.get(ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME, '28.0.3')
123 return os.path.join(getAndroidHome(), 'build-tools', version)
Morten Krogh-Jespersenc8efedd2019-01-28 11:36:17 +0100124
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100125def Print(s, quiet=False):
126 if quiet:
127 return
128 print(s)
129
130def Warn(message):
131 CRED = '\033[91m'
132 CEND = '\033[0m'
133 print(CRED + message + CEND)
134
135def PrintCmd(cmd, env=None, quiet=False):
136 if quiet:
137 return
138 if type(cmd) is list:
139 cmd = ' '.join(cmd)
140 if env:
141 env = ' '.join(['{}=\"{}\"'.format(x, y) for x, y in env.iteritems()])
142 print('Running: {} {}'.format(env, cmd))
143 else:
144 print('Running: {}'.format(cmd))
Mads Ager418d1ca2017-05-22 09:35:49 +0200145 # I know this will hit os on windows eventually if we don't do this.
146 sys.stdout.flush()
147
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100148class ProgressLogger(object):
149 CLEAR_LINE = '\033[K'
150 UP = '\033[F'
151
152 def __init__(self, quiet=False):
153 self._count = 0
154 self._has_printed = False
155 self._quiet = quiet
156
157 def log(self, text):
158 if self._quiet:
159 if self._has_printed:
160 sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
161 if len(text) > 140:
162 text = text[0:140] + '...'
163 print(text)
164 self._has_printed = True
165
166 def done(self):
167 if self._quiet and self._has_printed:
168 sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
169 print('')
170 sys.stdout.write(ProgressLogger.UP)
171
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100172def RunCmd(cmd, env_vars=None, quiet=False, fail=True, logging=True):
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100173 PrintCmd(cmd, env=env_vars, quiet=quiet)
174 env = os.environ.copy()
175 if env_vars:
176 env.update(env_vars)
177 process = subprocess.Popen(
178 cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
179 stdout = []
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100180 logger = ProgressLogger(quiet=quiet) if logging else None
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100181 failed = False
182 while True:
183 line = process.stdout.readline()
184 if line != b'':
185 stripped = line.rstrip()
186 stdout.append(stripped)
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100187 if logger:
188 logger.log(stripped)
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100189
190 # TODO(christofferqa): r8 should fail with non-zero exit code.
Morten Krogh-Jespersen121c47b2019-01-25 09:57:21 +0100191 if ('AssertionError:' in stripped
192 or 'CompilationError:' in stripped
193 or 'CompilationFailedException:' in stripped
Morten Krogh-Jespersen5d02a6b2019-10-29 14:48:56 +0100194 or 'Compilation failed' in stripped
195 or 'FAILURE:' in stripped
196 or 'org.gradle.api.ProjectConfigurationException' in stripped
197 or 'BUILD FAILED' in stripped):
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100198 failed = True
199 else:
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100200 if logger:
201 logger.done()
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100202 exit_code = process.poll()
203 if exit_code or failed:
204 for line in stdout:
205 Warn(line)
Christoffer Quist Adamsen7cf4c562019-03-07 10:57:33 +0100206 if fail:
207 raise subprocess.CalledProcessError(
208 exit_code or -1, cmd, output='\n'.join(stdout))
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100209 return stdout
210
Christoffer Quist Adamsen7cf4c562019-03-07 10:57:33 +0100211def RunGradlew(
212 args, clean=True, stacktrace=True, use_daemon=False, env_vars=None,
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100213 quiet=False, fail=True, logging=True):
Christoffer Quist Adamsen7cf4c562019-03-07 10:57:33 +0100214 cmd = ['./gradlew']
215 if clean:
216 assert 'clean' not in args
217 cmd.append('clean')
218 if stacktrace:
219 assert '--stacktrace' not in args
220 cmd.append('--stacktrace')
221 if not use_daemon:
222 assert '--no-daemon' not in args
223 cmd.append('--no-daemon')
224 cmd.extend(args)
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100225 return RunCmd(cmd, env_vars=env_vars, quiet=quiet, fail=fail, logging=logging)
Christoffer Quist Adamsen7cf4c562019-03-07 10:57:33 +0100226
Rico Windf80f5a22017-06-16 09:15:57 +0200227def IsWindows():
Ian Zerny3f54e222019-02-12 10:51:17 +0100228 return defines.IsWindows()
Ian Zerny5fffb0a2019-02-11 13:54:22 +0100229
230def IsLinux():
Ian Zerny3f54e222019-02-12 10:51:17 +0100231 return defines.IsLinux()
Ian Zerny5fffb0a2019-02-11 13:54:22 +0100232
233def IsOsX():
Ian Zerny3f54e222019-02-12 10:51:17 +0100234 return defines.IsOsX()
Ian Zerny5fffb0a2019-02-11 13:54:22 +0100235
236def EnsureDepFromGoogleCloudStorage(dep, tgz, sha1, msg):
237 if not os.path.exists(dep) or os.path.getmtime(tgz) < os.path.getmtime(sha1):
238 DownloadFromGoogleCloudStorage(sha1)
239 # Update the mtime of the tar file to make sure we do not run again unless
240 # there is an update.
241 os.utime(tgz, None)
242 else:
243 print 'Ensure cloud dependency:', msg, 'present'
Rico Windf80f5a22017-06-16 09:15:57 +0200244
Jean-Marie Henaffe4e36d12018-04-05 10:33:50 +0200245def DownloadFromX20(sha1_file):
246 download_script = os.path.join(REPO_ROOT, 'tools', 'download_from_x20.py')
247 cmd = [download_script, sha1_file]
248 PrintCmd(cmd)
249 subprocess.check_call(cmd)
250
Rico Wind533e3ce2019-04-04 10:26:12 +0200251def DownloadFromGoogleCloudStorage(sha1_file, bucket='r8-deps', auth=False):
Rico Windf80f5a22017-06-16 09:15:57 +0200252 suffix = '.bat' if IsWindows() else ''
253 download_script = 'download_from_google_storage%s' % suffix
Rico Wind533e3ce2019-04-04 10:26:12 +0200254 cmd = [download_script]
255 if not auth:
256 cmd.append('-n')
257 cmd.extend(['-b', bucket, '-u', '-s', sha1_file])
Mads Ager418d1ca2017-05-22 09:35:49 +0200258 PrintCmd(cmd)
259 subprocess.check_call(cmd)
260
261def get_sha1(filename):
262 sha1 = hashlib.sha1()
263 with open(filename, 'rb') as f:
264 while True:
265 chunk = f.read(1024*1024)
266 if not chunk:
267 break
268 sha1.update(chunk)
269 return sha1.hexdigest()
270
Rico Wind1b09c562019-01-17 08:53:09 +0100271def is_master():
272 remotes = subprocess.check_output(['git', 'branch', '-r', '--contains',
273 'HEAD'])
274 return 'origin/master' in remotes
275
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200276def get_HEAD_sha1():
Morten Krogh-Jespersene0ce6a32019-02-07 11:44:45 +0100277 return get_HEAD_sha1_for_checkout(REPO_ROOT)
278
279def get_HEAD_sha1_for_checkout(checkout):
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200280 cmd = ['git', 'rev-parse', 'HEAD']
281 PrintCmd(cmd)
Morten Krogh-Jespersene0ce6a32019-02-07 11:44:45 +0100282 with ChangedWorkingDirectory(checkout):
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200283 return subprocess.check_output(cmd).strip()
284
Tamas Kenez971eec62017-05-24 11:08:40 +0200285def makedirs_if_needed(path):
286 try:
287 os.makedirs(path)
288 except OSError:
289 if not os.path.isdir(path):
290 raise
291
Rico Wind800fd712018-09-24 11:29:33 +0200292def upload_dir_to_cloud_storage(directory, destination, is_html=False, public_read=True):
Rico Winda94f01c2017-06-27 10:32:34 +0200293 # Upload and make the content encoding right for viewing directly
Rico Windd55ce272019-03-26 08:02:09 +0100294 cmd = ['gsutil.py', '-m', 'cp']
Rico Windd0d88cf2018-02-09 09:46:11 +0100295 if is_html:
296 cmd += ['-z', 'html']
Rico Wind800fd712018-09-24 11:29:33 +0200297 if public_read:
298 cmd += ['-a', 'public-read']
299 cmd += ['-R', directory, destination]
Rico Winda94f01c2017-06-27 10:32:34 +0200300 PrintCmd(cmd)
301 subprocess.check_call(cmd)
302
Rico Wind800fd712018-09-24 11:29:33 +0200303def upload_file_to_cloud_storage(source, destination, public_read=True):
304 cmd = ['gsutil.py', 'cp']
305 if public_read:
306 cmd += ['-a', 'public-read']
307 cmd += [source, destination]
Rico Windb4621c12017-08-28 12:48:53 +0200308 PrintCmd(cmd)
309 subprocess.check_call(cmd)
310
Rico Wind139eece2018-09-25 09:42:09 +0200311def delete_file_from_cloud_storage(destination):
312 cmd = ['gsutil.py', 'rm', destination]
313 PrintCmd(cmd)
314 subprocess.check_call(cmd)
315
Rico Wind4fd2dda2018-09-26 17:41:45 +0200316def ls_files_on_cloud_storage(destination):
317 cmd = ['gsutil.py', 'ls', destination]
318 PrintCmd(cmd)
319 return subprocess.check_output(cmd)
320
Rico Wind139eece2018-09-25 09:42:09 +0200321def cat_file_on_cloud_storage(destination, ignore_errors=False):
322 cmd = ['gsutil.py', 'cat', destination]
323 PrintCmd(cmd)
324 try:
325 return subprocess.check_output(cmd)
326 except subprocess.CalledProcessError as e:
327 if ignore_errors:
328 return ''
329 else:
330 raise e
331
332def file_exists_on_cloud_storage(destination):
333 cmd = ['gsutil.py', 'ls', destination]
334 PrintCmd(cmd)
335 return subprocess.call(cmd) == 0
336
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200337def download_file_from_cloud_storage(source, destination):
338 cmd = ['gsutil.py', 'cp', source, destination]
339 PrintCmd(cmd)
340 subprocess.check_call(cmd)
341
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100342def create_archive(name, sources=None):
343 if not sources:
344 sources = [name]
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200345 tarname = '%s.tar.gz' % name
346 with tarfile.open(tarname, 'w:gz') as tar:
Morten Krogh-Jespersen54090862019-02-19 11:31:10 +0100347 for source in sources:
348 tar.add(source)
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200349 return tarname
350
351def extract_dir(filename):
352 return filename[0:len(filename) - len('.tar.gz')]
353
354def unpack_archive(filename):
355 dest_dir = extract_dir(filename)
356 if os.path.exists(dest_dir):
357 print 'Deleting existing dir %s' % dest_dir
358 shutil.rmtree(dest_dir)
359 dirname = os.path.dirname(os.path.abspath(filename))
360 with tarfile.open(filename, 'r:gz') as tar:
361 tar.extractall(path=dirname)
362
Morten Krogh-Jespersen54090862019-02-19 11:31:10 +0100363def check_prodacces():
364 subprocess.check_call(['prodaccess'])
365
Rico Wind1a29c4f2018-01-25 08:43:08 +0100366# Note that gcs is eventually consistent with regards to list operations.
367# This is not a problem in our case, but don't ever use this method
368# for synchronization.
369def cloud_storage_exists(destination):
370 cmd = ['gsutil.py', 'ls', destination]
371 PrintCmd(cmd)
372 exit_code = subprocess.call(cmd)
373 return exit_code == 0
374
Mads Ager418d1ca2017-05-22 09:35:49 +0200375class TempDir(object):
Søren Gjessecbeae782019-05-21 14:14:25 +0200376 def __init__(self, prefix='', delete=True):
Mads Ager418d1ca2017-05-22 09:35:49 +0200377 self._temp_dir = None
378 self._prefix = prefix
Søren Gjessecbeae782019-05-21 14:14:25 +0200379 self._delete = delete
Mads Ager418d1ca2017-05-22 09:35:49 +0200380
381 def __enter__(self):
382 self._temp_dir = tempfile.mkdtemp(self._prefix)
383 return self._temp_dir
384
385 def __exit__(self, *_):
Søren Gjessecbeae782019-05-21 14:14:25 +0200386 if self._delete:
387 shutil.rmtree(self._temp_dir, ignore_errors=True)
Mads Ager418d1ca2017-05-22 09:35:49 +0200388
389class ChangedWorkingDirectory(object):
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100390 def __init__(self, working_directory, quiet=False):
391 self._quiet = quiet
Mads Ager418d1ca2017-05-22 09:35:49 +0200392 self._working_directory = working_directory
393
394 def __enter__(self):
395 self._old_cwd = os.getcwd()
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100396 if not self._quiet:
397 print 'Enter directory:', self._working_directory
Mads Ager418d1ca2017-05-22 09:35:49 +0200398 os.chdir(self._working_directory)
399
400 def __exit__(self, *_):
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100401 if not self._quiet:
402 print 'Enter directory:', self._old_cwd
Mads Ager418d1ca2017-05-22 09:35:49 +0200403 os.chdir(self._old_cwd)
Tamas Kenez82efeb52017-06-12 13:56:22 +0200404
405# Reading Android CTS test_result.xml
406
407class CtsModule(object):
408 def __init__(self, module_name):
409 self.name = module_name
410
411class CtsTestCase(object):
412 def __init__(self, test_case_name):
413 self.name = test_case_name
414
415class CtsTest(object):
416 def __init__(self, test_name, outcome):
417 self.name = test_name
418 self.outcome = outcome
419
420# Generator yielding CtsModule, CtsTestCase or CtsTest from
421# reading through a CTS test_result.xml file.
422def read_cts_test_result(file_xml):
423 re_module = re.compile('<Module name="([^"]*)"')
424 re_test_case = re.compile('<TestCase name="([^"]*)"')
425 re_test = re.compile('<Test result="(pass|fail)" name="([^"]*)"')
426 with open(file_xml) as f:
427 for line in f:
428 m = re_module.search(line)
429 if m:
430 yield CtsModule(m.groups()[0])
431 continue
432 m = re_test_case.search(line)
433 if m:
434 yield CtsTestCase(m.groups()[0])
435 continue
436 m = re_test.search(line)
437 if m:
438 outcome = m.groups()[0]
Rico Windf80f5a22017-06-16 09:15:57 +0200439 assert outcome in ['fail', 'pass']
Tamas Kenez82efeb52017-06-12 13:56:22 +0200440 yield CtsTest(m.groups()[1], outcome == 'pass')
Tamas Kenezfc34cd82017-07-13 12:43:57 +0200441
442def grep_memoryuse(logfile):
443 re_vmhwm = re.compile('^VmHWM:[ \t]*([0-9]+)[ \t]*([a-zA-Z]*)')
444 result = None
445 with open(logfile) as f:
446 for line in f:
447 m = re_vmhwm.search(line)
448 if m:
449 groups = m.groups()
450 s = len(groups)
451 if s >= 1:
452 result = int(groups[0])
453 if s >= 2:
454 unit = groups[1]
455 if unit == 'kB':
456 result *= 1024
457 elif unit != '':
458 raise Exception('Unrecognized unit in memory usage log: {}'
459 .format(unit))
460 if result is None:
461 raise Exception('No memory usage found in log: {}'.format(logfile))
Tamas Kenez02bff032017-07-18 12:13:58 +0200462 return result
463
464# Return a dictionary: {segment_name -> segments_size}
465def getDexSegmentSizes(dex_files):
466 assert len(dex_files) > 0
Ian Zerny3f54e222019-02-12 10:51:17 +0100467 cmd = [jdk.GetJavaExecutable(), '-jar', R8_JAR, 'dexsegments']
Tamas Kenez02bff032017-07-18 12:13:58 +0200468 cmd.extend(dex_files)
469 PrintCmd(cmd)
470 output = subprocess.check_output(cmd)
471
472 matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)
473
474 if matches is None or len(matches) == 0:
475 raise Exception('DexSegments failed to return any output for' \
476 ' these files: {}'.format(dex_files))
477
478 result = {}
479
480 for match in matches:
481 result[match[0]] = int(match[1])
482
483 return result
484
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100485# Return a dictionary: {segment_name -> segments_size}
486def getCfSegmentSizes(cfFile):
Ian Zerny3f54e222019-02-12 10:51:17 +0100487 cmd = [jdk.GetJavaExecutable(),
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100488 '-cp',
Morten Krogh-Jespersen480784d2019-02-05 08:10:46 +0100489 CF_SEGMENTS_TOOL,
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100490 'com.android.tools.r8.cf_segments.MeasureLib',
491 cfFile]
492 PrintCmd(cmd)
493 output = subprocess.check_output(cmd)
494
495 matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)
496
497 if matches is None or len(matches) == 0:
498 raise Exception('CfSegments failed to return any output for' \
499 ' the file: ' + cfFile)
500
501 result = {}
502
503 for match in matches:
504 result[match[0]] = int(match[1])
505
506 return result
507
Søren Gjesse1c115b52019-08-14 12:43:57 +0200508def get_maven_path(artifact, version):
509 return os.path.join('com', 'android', 'tools', artifact, version)
Rico Windc0b16382018-05-17 13:23:43 +0200510
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100511def print_cfsegments(prefix, cf_files):
512 for cf_file in cf_files:
513 for segment_name, size in getCfSegmentSizes(cf_file).items():
514 print('{}-{}(CodeSize): {}'
515 .format(prefix, segment_name, size))
516
Tamas Kenez02bff032017-07-18 12:13:58 +0200517def print_dexsegments(prefix, dex_files):
518 for segment_name, size in getDexSegmentSizes(dex_files).items():
519 print('{}-{}(CodeSize): {}'
520 .format(prefix, segment_name, size))
Tamas Kenez2cf47cf2017-07-25 10:22:52 +0200521
Mads Agerbc7b2ce2018-02-05 11:28:47 +0100522# Ensure that we are not benchmarking with a google jvm.
Tamas Kenez2cf47cf2017-07-25 10:22:52 +0200523def check_java_version():
Ian Zerny3f54e222019-02-12 10:51:17 +0100524 cmd= [jdk.GetJavaExecutable(), '-version']
Tamas Kenez2cf47cf2017-07-25 10:22:52 +0200525 output = subprocess.check_output(cmd, stderr = subprocess.STDOUT)
526 m = re.search('openjdk version "([^"]*)"', output)
527 if m is None:
528 raise Exception("Can't check java version: no version string in output"
529 " of 'java -version': '{}'".format(output))
530 version = m.groups(0)[0]
Mads Agerbc7b2ce2018-02-05 11:28:47 +0100531 m = re.search('google', version)
532 if m is not None:
533 raise Exception("Do not use google JVM for benchmarking: " + version)
Tamas Kenez0cad51c2017-08-21 14:42:01 +0200534
Christoffer Quist Adamsen17879c12019-01-22 16:13:54 +0100535def get_android_jar_dir(api):
536 return os.path.join(REPO_ROOT, ANDROID_JAR_DIR.format(api=api))
537
Rico Wind9d70f612018-08-31 09:17:43 +0200538def get_android_jar(api):
539 return os.path.join(REPO_ROOT, ANDROID_JAR.format(api=api))
Rico Windda6836e2018-12-07 12:32:03 +0100540
Christoffer Quist Adamsen17879c12019-01-22 16:13:54 +0100541def get_android_optional_jars(api):
542 android_optional_jars_dir = os.path.join(get_android_jar_dir(api), 'optional')
543 android_optional_jars = [
544 os.path.join(android_optional_jars_dir, 'android.test.base.jar'),
545 os.path.join(android_optional_jars_dir, 'android.test.mock.jar'),
546 os.path.join(android_optional_jars_dir, 'android.test.runner.jar'),
547 os.path.join(android_optional_jars_dir, 'org.apache.http.legacy.jar')
548 ]
549 return [
550 android_optional_jar for android_optional_jar in android_optional_jars
551 if os.path.isfile(android_optional_jar)]
552
Rico Windfaaac012019-02-25 11:24:05 +0100553def is_bot():
Rico Winde11f4392019-03-18 07:59:37 +0100554 return 'SWARMING_BOT_ID' in os.environ
Morten Krogh-Jespersen16e925d2019-01-25 14:40:38 +0100555
556def uncompressed_size(path):
557 return sum(z.file_size for z in zipfile.ZipFile(path).infolist())
Morten Krogh-Jespersen0de13732019-03-01 08:56:39 +0100558
559def getR8Version(path):
560 cmd = [jdk.GetJavaExecutable(), '-cp', path, 'com.android.tools.r8.R8',
561 '--version']
562 output = subprocess.check_output(cmd, stderr = subprocess.STDOUT)
563 # output is on form 'R8 <version>' so we just strip of 'R8 '.
564 return output.splitlines()[0][3:]
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200565
566def desugar_configuration_version():
567 with open(DESUGAR_CONFIGURATION, 'r') as f:
568 configuration_json = json.loads(f.read())
569 configuration_format_version = \
570 configuration_json.get('configuration_format_version')
571 if (configuration_format_version != 1):
572 raise Exception(
573 'Unsupported "configuration_format_version" '
574 + configuration_format_version)
575 version = configuration_json.get('version')
576 if not version:
577 raise Exception(
578 'No "version" found in ' + utils.DESUGAR_CONFIGURATION)
Søren Gjesse1e171532019-09-03 09:44:22 +0200579 check_basic_semver_version(version, 'in ' + DESUGAR_CONFIGURATION)
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200580 return version
Søren Gjesse1e171532019-09-03 09:44:22 +0200581
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100582class SemanticVersion:
583 def __init__(self, major, minor, patch):
584 self.major = major
585 self.minor = minor
586 self.patch = patch
587 # Build metadata currently not suppported
588
589 def larger_than(self, other):
590 if self.major > other.major:
591 return True
592 if self.major == other.major and self.minor > other.minor:
593 return True
594 if self.patch:
595 return (self.major == other.major
596 and self.minor == other.minor
597 and self.patch > other.patch)
598 else:
599 return False
600
601
Søren Gjesse1e171532019-09-03 09:44:22 +0200602# Check that the passed string is formatted as a basic semver version (x.y.z)
603# See https://semver.org/.
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100604def check_basic_semver_version(version, error_context = '', components = 3):
605 regexp = '^'
606 for x in range(components):
607 regexp += '([0-9]+)'
608 if x < components - 1:
609 regexp += '\\.'
610 regexp += '$'
611 reg = re.compile(regexp)
612 match = reg.match(version)
613 if not match:
Søren Gjesse1e171532019-09-03 09:44:22 +0200614 raise Exception("Invalid version '"
615 + version
616 + "'"
617 + (' ' + error_context) if len(error_context) > 0 else '')
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100618 if components == 2:
619 return SemanticVersion(int(match.group(1)), int(match.group(2)), None)
620 elif components == 3:
621 return SemanticVersion(
622 int(match.group(1)), int(match.group(2)), int(match.group(3)))
623 else:
624 raise Exception('Argument "components" must be 2 or 3')