diff options
| -rwxr-xr-x | compilation_stats.py | 101 | ||||
| -rwxr-xr-x | run.py | 5 | ||||
| -rwxr-xr-x | tools/benchmarks/run.py | 45 | ||||
| -rwxr-xr-x | tools/compilation_statistics/run.py | 315 | ||||
| -rw-r--r-- | tools/utils.py | 43 |
5 files changed, 179 insertions, 330 deletions
diff --git a/compilation_stats.py b/compilation_stats.py new file mode 100755 index 0000000..78dea5f --- /dev/null +++ b/compilation_stats.py @@ -0,0 +1,101 @@ +#! /usr/bin/env python3 + +# Copyright (C) 2021 Linaro Limited. All rights received. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import sys +import json +import tempfile +import shutil + +from collections import OrderedDict + +dir_benchs = os.path.dirname(os.path.realpath(__file__)) +dir_tools = os.path.join(dir_benchs, '..') +sys.path.insert(0, dir_tools) + +from tools import utils, utils_adb, utils_print, utils_stats + +def BuildOptions(): + parser = argparse.ArgumentParser( + description = "Collect compilation statistics.", + # Print default values. + formatter_class = argparse.ArgumentDefaultsHelpFormatter) + utils.AddCommonRunOptions(parser) + utils.AddOutputFormatOptions(parser, utils.default_output_formats) + args = parser.parse_args() + return args + +def SaveAndPrintResults(apk, + compilation_times, + section_sizes, + output_json_filename): + output_obj = utils.ReadJSON(output_json_filename) + + apk_basename = os.path.basename(apk) + output_obj[apk_basename] = dict() + + output_obj[apk_basename].update(compilation_times) + print("Compilation times (seconds):") + utils.PrintData(compilation_times) + + output_obj[apk_basename]["Executable size"] = section_sizes + print("Executable sizes (bytes):") + utils.PrintData(section_sizes) + + with open(output_json_filename, "w") as fp: + json.dump(output_obj, fp, indent=2) + +if __name__ == "__main__": + # create temp directory to pull executables from the device + # to measure their size + work_dir = tempfile.mkdtemp() + try: + args = BuildOptions() + apk = args.add_pathname[0] + apk_name = utils.TargetPathJoin(args.target_copy_path, apk) + # command is used to call a shell script using chroot + # this script calls dex2oat on a given APK and prints + # before/after timestamps + # after command is executed we pull the executable from the device + # and measure its size + if 'ART_COMMAND' in os.environ: + command = os.getenv('ART_COMMAND') + else: + utils.Error("ART_COMMAND is not set.") + format_data = {'workdir': os.path.dirname(apk_name)} + command = command.format(**format_data) + + compilation_times = [] + for i in range(args.iterations): + print("Compiling APK") + results = utils_adb.shell(command, args.target, exit_on_error=False) + lines = results[1] + compilation_time = utils.ExtractCompilationTimeFromOutput(lines) + compilation_times += [compilation_time] + print("Compilation took {:.2f}s\n".format(compilation_time)) + + # Pull the executable and get its size + local_oat = os.path.join(work_dir, apk + '.oat') + utils_adb.pull(args.output_oat, local_oat, args.target) + section_sizes = utils.GetSectionSizes(local_oat) + + compile_time_dict = OrderedDict([("Time", compilation_times)]) + SaveAndPrintResults(apk, compile_time_dict, section_sizes, args.output_json) + + finally: + shutil.rmtree(work_dir) + @@ -23,7 +23,6 @@ from collections import OrderedDict from tools import utils from tools import utils_stats from tools.benchmarks.run import GetAndPrintBenchmarkResults -from tools.compilation_statistics.run import GetAndPrintCompilationStatisticsResults def BuildOptions(): parser = argparse.ArgumentParser( @@ -52,9 +51,5 @@ if __name__ == "__main__": result = OrderedDict() result[utils.benchmarks_label] = GetAndPrintBenchmarkResults(args) - if args.target: - result[utils.compilation_statistics_label] = \ - GetAndPrintCompilationStatisticsResults(args) - utils.OutputObject(result, 'pkl', args.output_pkl) utils.OutputObject(result, 'json', args.output_json) diff --git a/tools/benchmarks/run.py b/tools/benchmarks/run.py index edfb3a6..844d484 100755 --- a/tools/benchmarks/run.py +++ b/tools/benchmarks/run.py @@ -18,9 +18,11 @@ import argparse import csv import os +import shutil import subprocess import sys import time +import tempfile from collections import OrderedDict @@ -34,8 +36,6 @@ import utils_stats bench_runner_main = 'org.linaro.bench.RunBench' -# Options - def BuildOptions(): parser = argparse.ArgumentParser( description = "Run java benchmarks.", @@ -160,13 +160,18 @@ def RunBench(apk, classname, try: for line in outerr.rstrip().splitlines(): - if not line.startswith('benchmarks/'): - continue - name = line.split(":")[0].rstrip() - score = float(line.split(":")[1].strip().split(" ")[0].strip()) - if name not in result: - result[name] = list() - result[name].append(score) + if line.startswith('benchmarks/'): + name = line.split(":")[0].rstrip() + score = float(line.split(":")[1].strip().split(" ")[0].strip()) + if name not in result: + result[name] = list() + result[name].append(score) + duration = utils.ExtractCompilationTimeFromOutput(outerr) + if duration > 0: + if utils.compilation_times_label not in result: + result[utils.compilation_times_label] = list() + result[utils.compilation_times_label].append(duration) + except Exception as e: utils.Warning(str(e) + "\n \-> Error parsing output from %s", e) rc += 1 @@ -178,6 +183,7 @@ def RunBench(apk, classname, def RunBenchs(apk, bench_names, target, + output_oat, auto_calibrate, iterations=utils.default_n_iterations, mode=utils.default_mode, @@ -198,6 +204,16 @@ def RunBenchs(apk, bench_names, android_root = android_root, target = target, cpuset = cpuset) + if output_oat is not None: + try: + work_dir = tempfile.mkdtemp() + local_oat = os.path.join(work_dir, 'bench.oat') + utils_adb.pull(output_oat, local_oat, target) + section_sizes = utils.GetSectionSizes(local_oat) + result[utils.compilation_statistics_label] = section_sizes + finally: + shutil.rmtree(work_dir) + return rc @@ -253,6 +269,7 @@ def GetBenchmarkResults(args): rc = RunBenchs(remote_apk, benchmarks, args.target, + args.output_oat, not args.no_auto_calibrate, args.iterations, args.mode, @@ -269,7 +286,15 @@ def GetBenchmarkResults(args): def GetAndPrintBenchmarkResults(args): results = GetBenchmarkResults(args) utils.PrintData(results) - unflattened_results = utils.Unflatten(results) + + # remove compilation statistics from the results, it was + # already printed + filtered_results = dict() + for (k,v) in results.items(): + if utils.compilation_statistics_label not in k: + filtered_results[k] = v + + unflattened_results = utils.Unflatten(filtered_results) utils_stats.ComputeAndPrintGeomeanWithRelativeDiff(unflattened_results) print('') return results diff --git a/tools/compilation_statistics/run.py b/tools/compilation_statistics/run.py deleted file mode 100755 index 23fa91b..0000000 --- a/tools/compilation_statistics/run.py +++ /dev/null @@ -1,315 +0,0 @@ -#! /usr/bin/env python3 - -# Copyright (C) 2015 Linaro Limited. All rights received. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import glob -import json -import os -import pickle -import re -import shutil -import subprocess -import sys -import tempfile -import time - -from collections import OrderedDict - -dir_compilation_statistics = os.path.dirname(os.path.realpath(__file__)) -dir_tools = os.path.join(dir_compilation_statistics, '..') -sys.path.insert(0, dir_tools) - -import utils -import utils_adb -import utils_stats - -memory_unit_prefixes = {'' : 1, 'G' : 2 ** 30, 'K' : 2 ** 10, 'M' : 2 ** 20} -sections = set(['.bss', '.rodata', '.text', 'Total']) - -def BuildOptions(): - parser = argparse.ArgumentParser( - description = '''Collect statistics about the APK compilation process on a target - adb device: Compilation time, memory usage by the compiler - (arena, Java, and native allocations, and free native memory), - and size of the generated executable (total, .bss, .rodata, and - .text section sizes).''', - # Print default values. - formatter_class = argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('pathnames', - nargs = '+', - help='''Path containing APK files or a file name for which - compilation statistics should be collected.''') - utils.AddCommonRunOptions(parser) - utils.AddOutputFormatOptions(parser, utils.default_output_formats) - - # TODO: Support running on host? - # For now override the default value for the `--target`. - parser.set_defaults(target=utils.adb_default_target_string) - - args = parser.parse_args() - - # This cannot fire for now since this script always runs on target, but - # eventually we may want to run on host as well. - utils.ValidateCommonRunOptions(args) - - return args - -def GetStats(apk, - target, - isa, - compiler_mode, - android_root, - target_copy_path, - iterations, - cpuset, - work_dir, - boot_oat_file): - path, env, runtime_param = utils.GetAndroidRootConfiguration(android_root, isa.endswith('64')) - dex2oat = utils.TargetPathJoin(path, 'dex2oat') - - if boot_oat_file: - oat = utils.TargetPathJoin(target_copy_path, 'boot.' + isa + '.oat') - art = utils.TargetPathJoin(target_copy_path, 'boot.' + isa + '.art') - - # Check if dump file exists. - dump_oat_file_location = utils.TargetPathJoin(target_copy_path, 'boot.oat.' + isa + '.txt') - dump_exists_command = "if [ -f %s ] ; then echo found; fi; exit 0" \ - % (dump_oat_file_location) - # Since we are interested in whether the dump file exists or not, we can't simply execute - # [ -f file_name ] since newer versions of adb return the error code of the command that's - # being executed. Therefore, if the file is found we output a string and at the end we - # always return error code 0, so that we can get an error only if the failure was due - # to adb not executing properly. - rc, out = utils_adb.shell(dump_exists_command, target) - # The command prints an extra new line as well. - if out.strip() != "found": - # Dump the oat file the first time, keeping only the parts we are interested in. - dump_command = 'oatdump --oat-file=%s | grep "dex2oat-" > %s' % (boot_oat_file, \ - dump_oat_file_location) - utils_adb.shell(dump_command, target) - # Read dex2oat-host from dump file. - dex2oat_host_command = 'grep "dex2oat-host" %s' % (dump_oat_file_location) - rc, out = utils_adb.shell(dex2oat_host_command, target) - if rc: - utils.Error("Dump file doesn't contain dex2oat-host.") - if out.strip() == 'dex2oat-host = x86-64': - utils.Error("boot.oat was built on a x84-64 machine, which is most likely the" \ - " host: %s \nWe want it to be built on the target instead. Have you" \ - " configured the device with WITH_DEXPREOPT=false ?" % out) - - # Read command. - dex2oat_cmdline_command = 'grep "dex2oat-cmdline" %s' % (dump_oat_file_location) - rc, out = utils_adb.shell(dex2oat_cmdline_command, target) - if rc: - utils.Error("Dump file doesn't contain dex2oat-cmldine.") - command = out.strip() - # Replace destination: --oat-file, fix beginning of command. - command = re.sub("--oat-file=(.+?) --", "--oat-file=%s --" % oat, command) - command = re.sub("--image=(.+?) --", "--image=%s --" % art, command) - command = re.sub("dex2oat-cmdline +=", dex2oat, command) - # Force 1 thread only - we want compilation times to be as stable as possible and we are - # interested in single thread performance, not multi-thread (throughput). - command = re.sub(" -j\d+ ", " -j1 ", command) - # Remove newline at end. - command = re.sub("\n$", "", command) - command = '(echo $BASHPID && ' - - if cpuset: - command += 'echo $BASHPID > /dev/cpuset/' + cpuset + '/tasks && ' - - command += env + ' exec ' + command + ') | head -n1' - else: - runtime_arguments = ' --runtime-arg -Xnorelocate ' - - for param in runtime_param: - runtime_arguments += '--runtime-arg ' + param + ' ' - - apk_path = utils.TargetPathJoin(target_copy_path, apk) - oat = apk_path + '.' + isa + '.oat' - dex2oat_options = utils.GetDex2oatOptions(compiler_mode) - # Only the output of the first command is necessary; execute in a subshell - # to guarantee PID value; only one thread is used for compilation to reduce - # measurement noise. - command = '(echo $BASHPID && ' - - if cpuset: - command += 'echo $BASHPID > /dev/cpuset/' + cpuset + '/tasks && ' - - command += env + ' exec ' + dex2oat + \ - ' -j1' + runtime_arguments + ' '.join(dex2oat_options) + \ - ' --dex-file=' + apk_path + ' --oat-file=' + oat - command += ' --instruction-set=' + isa + ') | head -n1' - - linux_target = os.getenv('ART_TARGET_LINUX', 'false') == 'true' - dex2oat_time_regex = '.*?took (?P<value>.*?)(?P<unit>[mnu]{,1})s.*?\)' - compilation_times = [] - for i in range(iterations): - rc, stdout = utils_adb.shell(command, target) - if linux_target: - # On Linux, dex2oat writes to stdout, and output of compilation time is likely last - for out in reversed(stdout.splitlines()): - compile_time = re.match(dex2oat_time_regex, out) - if compile_time: - break - else: - # To simplify parsing, assume that PID values are rarely recycled by the system. - stats_command = 'logcat -dsv process dex2oat | grep "^I([[:space:]]*' + \ - stdout.rstrip() + ').*took" | tail -n1' - rc, out = utils_adb.shell(stats_command, target) - compile_time = re.match(dex2oat_time_regex, out) - - if not compile_time: - utils.Error('dex2oat failed; check adb logcat.') - - value = float(compile_time.group('value')) * \ - utils.si_unit_prefixes[compile_time.group('unit')] - compilation_times.append(value) - - # The rest of the statistics are deterministic, so there is no need to run several - # iterations; just get the values from the last run. - out = out[compile_time.end():] - # Newer versions of dex2oat also have number of threads output, that we need to get rid of - out = re.sub('\(threads:\s+[0-9]+\) ', '', out) - memory_stats = OrderedDict() - byte_size = True - - for m in re.findall(' (.*?)=([0-9]+)([GKM]?)B( \(([0-9]+)B\))?', out): - # Old versions of dex2oat do not show the exact memory usage values in bytes, so - # try to parse the output in the new format first, and if that fails, fall back - # to the legacy one. - if m[4]: - value = int(m[4]) - else: - value = int(m[1]) * memory_unit_prefixes[m[2]] - - if m[2]: - byte_size = False - - memory_stats[m[0]] = [value] - - if not byte_size: - utils.Warning('Memory usage values have been rounded down, so they might be ' - 'inaccurate.') - - if boot_oat_file: - local_oat = os.path.join(utils.dir_root, work_dir, "boot.%s.oat" % isa) - else: - local_oat = os.path.join(utils.dir_root, work_dir, apk + '.oat') - utils_adb.pull(oat, local_oat, target) - command = ['size', '-A', '-d', local_oat] - rc, outerr = utils.Command(command) - section_sizes = OrderedDict((s[0], [int(s[1])]) for s - in re.findall('(\S+)\s+([0-9]+).*', outerr) - if s[0] in sections) - return OrderedDict([(utils.compilation_times_label, compilation_times), - (utils.memory_stats_label, memory_stats), - (utils.oat_size_label, section_sizes)]) - - -def GetCompilationStatisticsResults(args): - utils.CheckDependencies(['adb', 'size']) - isa = utils_adb.GetISA(args.target, args.mode) - res = OrderedDict() - work_dir = tempfile.mkdtemp() - apk_list = set() - boot_oat_file = None - - for pathname in args.pathnames: - if pathname == "boot.oat": - # Check if multiple boot.oat parameters have been passed. - if boot_oat_file: - continue - - # Get ISA list to check that the environment is in a good state. - isa_list = utils_adb.GetISAList(args.target) - # The oat cache is accessible only to root. - utils_adb.root(args.target) - # Find oat file on device. - find_command = 'find / -type d \( -name proc -o -name sys \) -prune -o ' \ - '-name "*boot.oat" -print 2>/dev/null' - rc, out = utils_adb.shell(find_command, args.target) - boot_oat_files = out.splitlines()[:-1] - - if len(boot_oat_files) != len(isa_list): - utils.Error("Number of architectures different from number of boot.oat files. " \ - "The list of boot.oat files is here:\n\n %s\n\nMake sure there are " \ - "no stale boot.oat files in %s or some other directory. " \ - "Another possibility is that you didn't build Android with " \ - "`WITH_DEXPREOPT=false`. Do a `lunch` and then `WITH_DEXPREOPT=false " \ - "make -j$(nproc)`." % (boot_oat_files, args.target_copy_path)) - # Order both lists. Now, as long as both oat files have the same parent dir, order - # should match. - isa_list.sort() - boot_oat_files.sort() - # Remove leading dot and trailing whitespace. - boot_oat_file = boot_oat_files[isa_list.index(isa)][1:].strip() - apk_list.add("boot.oat " + isa) - elif os.path.isfile(pathname): - apk_list.add(pathname) - else: - dentries = [dentry for dentry in glob.glob(os.path.join(pathname, '*.apk')) - if os.path.isfile(dentry)] - - for d in dentries: - apk_list.add(d) - - for apk in sorted(apk_list): - # pathname just contains boot.oat. - if apk[:8] == "boot.oat": - res[apk] = GetStats(apk, args.target, isa, args.compiler_mode, args.android_root, - args.target_copy_path, args.iterations, args.cpuset, work_dir, - boot_oat_file) - # This is a local path for boot.oat. We get stats locally without compiling on target. - elif apk[-8:] == "boot.oat": - res["boot.oat"] = GetLocalOatSizeStats(apk) - else: - utils_adb.push(apk, args.target_copy_path, args.target) - apk_name = os.path.basename(apk) - res[apk_name] = GetStats(apk_name, args.target, isa, args.compiler_mode, - args.android_root, args.target_copy_path, - args.iterations, args.cpuset, work_dir, None) - - shutil.rmtree(work_dir) - return res - -def GetAndPrintCompilationStatisticsResults(args): - results = GetCompilationStatisticsResults(args) - utils.PrintData(results) - print('') - return results - -def GetLocalOatSizeStats(oat_path): - command = ['size', '-A', '-d', oat_path] - rc, outerr = utils.Command(command) - section_sizes = OrderedDict((s[0], [int(s[1])]) for s - in re.findall('(\S+)\s+([0-9]+).*', outerr) - if s[0] in sections) - # Add total file size in bytes. - command = ['stat', '-c', '%s', oat_path] - rc, outerr = utils.Command(command) - section_sizes['FileSize'] = [int(outerr)] - return OrderedDict([(utils.oat_size_label, section_sizes)]) - -if __name__ == "__main__": - # TODO: Mac OS support - if os.uname().sysname != 'Linux': - utils.Error('Running this script is supported only on Linux.') - - args = BuildOptions() - stats = GetAndPrintCompilationStatisticsResults(args) - - utils.OutputObject(stats, 'pkl', args.output_pkl) - utils.OutputObject(stats, 'json', args.output_json) diff --git a/tools/utils.py b/tools/utils.py index 6a0ab5f..b2e5e3d 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -18,6 +18,7 @@ import fnmatch import json import os import pickle +import re import subprocess import sys import time @@ -55,6 +56,13 @@ default_mode = '' default_compiler_mode = None default_n_iterations = 1 +# used in shell scripts for compile stats +before_timestamp_key = "Before: " +after_timestamp_key = "After: " + +sections = set(['.bss', '.rodata', '.text', 'Total']) + + # TODO: Use python's logging and warning capabilities instead! def Info(message): print('INFO: ' + message) @@ -181,6 +189,9 @@ def AddCommonRunOptions(parser): nargs='+', default = None, help = 'Add pathnames to be considered for compilation statistics.') + opts.add_argument('--output-oat', + default=None, + help = 'Full name of the compiled executable file') def ValidateCommonRunOptions(args): options_requiring_target_mode = ['mode', 'compiler-mode'] @@ -469,3 +480,35 @@ def TargetPathJoin(path, *paths): path = path.replace(os.sep, '/') return path + +def ReadJSON(json_filename): + output_obj = dict() + try: + with open(json_filename, "r") as fp: + output_obj = json.load(fp) + except IOError: + pass + return output_obj + +def ExtractCompilationTimeFromOutput(lines): + timestamp_before = 0 + timestamp_after = 0 + # Extracting the size and timestamps (before and after compilation) + # cmdline.sh script is expected to print lines with + # two timestamps: before (before_timestamp_key) + # and after (after_timestamp_key) compilation + for line in lines.rstrip().splitlines(): + if (line.startswith(after_timestamp_key)): + timestamp_after = float(line[len(after_timestamp_key):]) + if (line.startswith(before_timestamp_key)): + timestamp_before = float(line[len(before_timestamp_key):]) + return timestamp_after - timestamp_before + +def GetSectionSizes(path_to_executable): + command = ['size', '-A', '-d', path_to_executable] + rc, outerr = Command(command) + section_sizes = OrderedDict( + (section_line[0], [int(section_line[1])]) for section_line + in re.findall('(\S+)\s+([0-9]+).*', outerr) + if section_line[0] in sections) + return section_sizes |