ART: Add compilation_stats_target.sh to iterate over different device

configurations and collect compilation stats for them

Currently we have compilation metrics when running benchmarks,
but we don't have tools to measure the impact of our changes
on real world applications. This script adds support of this and
allows us to collect compilation time and executable size
for already existing APKs.

Test: ./scripts/benchmarks/compilation_stats_target.sh
./benchmarks/apks/another.music.player_5870.apk --iterations 1

Change-Id: I0c214810c9ac2976d6913814f8b41444c05c12b8

2 files changed, 278 insertions, 30 deletions

diff --git a/benchmarks/benchmarks_run_target.sh b/benchmarks/benchmarks_run_target.sh
index 56c3664c..b5eb24d3 100755
--- a/benchmarks/benchmarks_run_target.sh
+++ b/benchmarks/benchmarks_run_target.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2016-2017, Linaro Ltd.
+# Copyright (c) 2016-2021, Linaro Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -167,17 +167,14 @@ list_devices() {
 }
 
 # Arguments:
-#   ${1}: bitness
+#   ${1}: isa
 create_target_cmdline_sh() {
-  local isa="arm"
-  if [[ "$1" == "64" ]]; then
-    isa+="64"
-  fi
+  local isa="${1}"
 
   local dump_cfg=""
   if ${options["dump-cfg"]}; then
-    target_cfg_file="/data/local/tmp/bench.arm$1.cfg"
-    dump_cfg="-j1 --dump-cfg=${target_cfg_file}"
+    target_cfg_file="/data/local/tmp/bench.${1}.cfg"
+    dump_cfg="--dump-cfg=${target_cfg_file}"
     local verbose_methods="${options["dump-cfg-methods"]}"
     if [[ "${options["dump-cfg-methods"]}" == "benchmarks" ]]; then
       verbose_methods=""
@@ -205,26 +202,31 @@ create_target_cmdline_sh() {
   local isa_features=${options["isa-features"]}
 
   read -r -d '' target_script << EOM
-$(get_target_art_test_env_vars_exports)
-cd /data/local/tmp
-if [[ ! -d /data/local/tmp/oat/${isa} ]]; then
-  mkdir -p /data/local/tmp/oat/${isa}
-  $(get_target_dex2oat_cmd \
-      "${isa}" \
-      "/data/local/tmp/bench.apk" \
-      "${isa_features}") \
-    $(get_target_art_test_bootclasspath_for_dex2oat) ${dump_cfg}
-
-  if [[ \$? -ne 0 ]]; then
+    $(get_target_art_test_env_vars_exports)
+    cd /data/local/tmp
+
     rm -rf /data/local/tmp/oat/${isa}
-    exit 1
-  fi
-fi
+    mkdir -p /data/local/tmp/oat/${isa}
+    before=\$EPOCHREALTIME
+    $(get_target_dex2oat_cmd \
+        "${isa}" \
+        "/data/local/tmp/bench.apk" \
+        "${isa_features}") \
+        $(get_target_art_test_bootclasspath_for_dex2oat) -j1 ${dump_cfg}
+    after=\$EPOCHREALTIME
+
+    echo "Before: "\$before
+    echo "After: "\$after
+
+    if [[ \$? -ne 0 ]]; then
+      rm -rf /data/local/tmp/oat/${isa}
+      exit 1
+    fi
 
-$(get_target_art_test_dalvikvm_cmd \
-    "/data/local/tmp" \
-    "/data/local/tmp/bench.apk" \
-    "${isa_features}") \$@
+    $(get_target_art_test_dalvikvm_cmd \
+        "/data/local/tmp" \
+        "/data/local/tmp/bench.apk" \
+        "${isa_features}") \$@
 EOM
 
   safe adb_shell "mkdir -p ${ART_TEST_CHROOT}/data/local/tmp/"
@@ -239,7 +241,12 @@ copy_cfg_file_from_device() {
 }
 
 run_benchmarks() {
-  create_target_cmdline_sh "$1"
+  local isa="arm"
+  if [[ "$1" == "64" ]]; then
+    isa+="64"
+  fi
+
+  create_target_cmdline_sh "${isa}"
   export ART_COMMAND="chroot ${ART_TEST_CHROOT} sh /data/local/tmp/cmdline.sh "
 
   local run_cmd_options=()
@@ -248,9 +255,8 @@ run_benchmarks() {
   run_cmd_options+=("--mode" "$1")
   run_cmd_options+=("--output-json" "${LOG_DIRECTORY}/$2_$1_$3_aot.json")
   run_cmd_options+=("--target-copy-path" "${ART_TEST_CHROOT}/data/local/tmp")
-  if is_full_source_tree; then
-    run_cmd_options+=("--add-pathname" "$(get_boot_oat_location "$1")")
-  fi
+  run_cmd_options+=("--output-oat" "${ART_TEST_CHROOT}/data/local/tmp/oat/${isa}/bench.odex")
+
   for benchmark in "${benchmarks[@]}"; do
     run_cmd_options+=("--filter" "${benchmark}")
   done
diff --git a/benchmarks/compilation_stats_target.sh b/benchmarks/compilation_stats_target.sh
new file mode 100755
index 00000000..8a093819
--- /dev/null
+++ b/benchmarks/compilation_stats_target.sh
@@ -0,0 +1,242 @@
+#!/bin/bash
+#
+# Copyright (c) 2021, Linaro Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+readonly local_path=$(dirname "$0")
+source "${local_path}/../utils/utils.sh"
+source "${local_path}/../utils/utils_test.sh"
+source "${local_path}/../utils/utils_android.sh"
+source "${local_path}/../utils/utils_android_root.sh"
+source "${local_path}/../utils/utils_benchmarks.sh"
+source "${local_path}/../devices/cpu_freq_utils.sh"
+
+readonly LOG_DIRECTORY="$(get_workspace)"
+
+readonly default_iterations=10
+
+# shellcheck disable=SC2034
+declare -A options_format=(
+  ["help"]="p:usage()"
+  ["h"]="r:&help"
+  ["verbose"]="p:enable_verbose()"
+  ["v"]="r:&verbose"
+  ["iterations"]="${default_iterations}"
+  ["mode"]="all"
+  ["linux"]="false"
+  ["x86"]="false"
+  ["sudo"]="false"
+  ["cpu"]="all"
+  ["skip-build"]="false"
+  ["skip-run"]="false"
+  ["list-devices"]="p:list_devices()"
+  ["target-device"]=""
+  ["dump-cfg"]="false"
+)
+declare -A options=()
+declare -a benchmarks=()
+
+validate_apk_names() {
+  local apk
+  for apk in "${benchmarks[@]}"; do
+    if [[ ! -f "${apk}" ]]; then
+      log E "APK file ${apk} does not exist"
+      exit 1
+    fi
+  done
+}
+
+usage() {
+  log I "Usage: $0 [OPTION]... [APK]..."
+  log I ""
+  log I "This script should be used for getting compile stats for APK(s) on a target device."
+  log I "The script expects a device to be connected. Use --list-devices to show supported"
+  log I "devices."
+  log I ""
+  log I "-------------------------------------------"
+  log I " -h, --help                - help"
+  log I " -v, --verbose             - verbose"
+  log I " --list-devices            - List the devices supported by this script."
+  log I " --mode <all|32|64>        - Get compile stats for the specified mode(s)."
+  log I "                             (default: all)"
+  log I " --cpu <all|big|little|default> - CPU mode."
+  log I "                             \"big\": Run with only big cores and pin their frequency"
+  log I "                             \"little\": Run with only little cores and pin their"
+  log I "                             frequency"
+  log I "                             \"all\": With big.LITTLE devices:"
+  log I "                             Run consecutively with only little cores enabled and"
+  log I "                             pinned,"
+  log I "                             and then with only big cores enabled and pinned."
+  log I "                             For devices without big.LITTLE, all cores are enabled and"
+  log I "                             pinned"
+  log I "                             \"default\": Run with unaltered default CPU configuration"
+  log I "                             (no pinning)."
+  log I "                             (default: all)"
+  log I " --iterations <n>          - The number of compilations for each APK."
+  log I "                             (default: $default_iterations)"
+  log I " --target-device <device>  - Use specific lunch target and configuration for Arm "
+  log I "                             target platform".
+  log I "                             (default: \"\" - which will cause a default target arch"
+  log I "                             to be used."
+  log I " --skip-build <false|true> - Skips the build step and compiles APKs with prebuilt"
+  log I "                             artifacts from \$OUT directory."
+  log I "                             This option is mainly used for automation."
+  log I "                             Make sure \`adb\` and \`dx\` are in your PATH when using"
+  log I "                             this option."
+  log I "                             (default: false)"
+  log I " --skip-run <false|true>   - Skips the running benchmark but just builds the artifacts"
+  log I "                             (default: false)"
+  log I " --dump-cfg                - Dump control-flow graphs. Depending on the mode they will be"
+  log I "                             either in bench.arm64.cfg or bench.arm32.cfg."
+  log I "-------------------------------------------"
+  log I "Default Configuration:"
+  log I " --default                 - Default benchmark configuration, equivalent to"
+  log I "                             \`--mode all --cpu all --iterations $default_iterations\`."
+  log I "-------------------------------------------"
+  exit 0
+}
+
+list_devices() {
+  list_devices_from_config "${local_path}/../devices/config/*"
+  exit 0
+}
+
+# Arguments:
+#   ${1}: isa
+#   ${2}: APK name
+create_target_cmdline_sh() {
+  local isa="${1}"
+
+  apk="${2}"
+  apk_basename=$(basename "${apk}" ".apk")
+
+  local dump_cfg=""
+  if ${options["dump-cfg"]}; then
+    target_cfg_file="/data/local/tmp/${apk_basename}.${1}.cfg"
+    dump_cfg="--dump-cfg=${target_cfg_file}"
+  fi
+
+  # use --no-watch-dog option to bypass 9.5 minutes timeout in dex2oat
+  read -r -d '' target_script << EOM
+    $(get_target_art_test_env_vars_exports)
+    rm -rf /data/local/tmp/oat/${isa}
+    mkdir -p /data/local/tmp/oat/${isa}
+
+    before=\$EPOCHREALTIME
+    $(get_target_dex2oat_cmd "${isa}" "/data/local/tmp/${apk_basename}.apk") \
+      $(get_target_art_test_bootclasspath_for_dex2oat) -j1 ${dump_cfg} --no-watch-dog
+    after=\$EPOCHREALTIME
+
+    echo "Before: "\$before
+    echo "After: "\$after
+    \$@
+EOM
+
+  safe adb_shell "mkdir -p ${ART_TEST_CHROOT}/data/local/tmp/"
+  safe adb_shell "echo '${target_script}' > ${ART_TEST_CHROOT}/data/local/tmp/cmdline.sh"
+}
+
+# Arguments:
+#   ${1}: bitness
+#   ${2}: device
+#   ${3}: CPU mode
+run_apks() {
+  local compilation_stats_file="${LOG_DIRECTORY}/${2}_${1}_${3}_compilation_stats.json"
+  if [[ -f "${compilation_stats_file}" ]]; then
+    safe rm "${compilation_stats_file}"
+  fi
+
+  local isa="arm"
+  if [[ "${1}" == "64" ]]; then
+    isa+="64"
+  fi
+
+  start_adb_section "run_benchmarks_${2}_${1}_${3}"
+  for apk in "${benchmarks[@]}"; do
+    log I "Processing $apk"
+    adb push "${apk}" "${ART_TEST_CHROOT}/data/local/tmp"
+    apk_name=$(basename "${apk}")
+    apk_basename=$(basename "${apk_name}" ".apk")
+    output_oat="${ART_TEST_CHROOT}/data/local/tmp/oat/${isa}/${apk_basename}.odex"
+
+    create_target_cmdline_sh "${isa}" "${apk_name}"
+    export ART_COMMAND="chroot ${ART_TEST_CHROOT} sh /data/local/tmp/cmdline.sh "
+
+    local run_cmd_options=()
+    run_cmd_options+=("--target")
+    run_cmd_options+=("--iterations" "${options["iterations"]}")
+    run_cmd_options+=("--mode" "${1}")
+    run_cmd_options+=("--output-json" "${compilation_stats_file}")
+    run_cmd_options+=("--output-oat" "${output_oat}")
+    run_cmd_options+=("--target-copy-path" "${ART_TEST_CHROOT}/data/local/tmp")
+    run_cmd_options+=("--add-pathname" "${apk_name}")
+    ./benchmarks/compilation_stats.py "${run_cmd_options[@]}"
+  done
+  end_adb_section "run_benchmarks_${2}_${1}_${3}" "$?"
+}
+
+# Arguments:
+#   ${1}: bitness
+run_all_apks() {
+  local -r bitness="${1}"
+  local -r target_device=$(safe adb_shell getprop ro.product.device)
+  local -r cpu="${options["cpu"]}"
+  local -r path_to_devices="${local_path}/../devices"
+  set_freq_and_run run_apks "${bitness}" "${cpu}" "${target_device}" "${path_to_devices}"
+}
+
+main() {
+  exit_on_failure arguments_parser options_format options benchmarks -- "$@"
+
+  if [[ -z "${benchmarks[@]}" ]]; then
+    log E "Empty list of APKs to compile"
+    exit 1
+  fi
+
+  validate_apk_names
+
+  readonly options
+  dump_options
+
+  local -r mode="${options["mode"]}"
+  local -r skip_build="${options["skip-build"]}"
+
+  if android_build_already_setup; then
+    log E "This test does not support environment targets. Please re-run in a clean environment."
+    exit 1
+  fi
+
+  for bits in 32 64; do
+    if [[ "$mode" != "all" && "$mode" != "$bits" ]]; then
+      log I "Skipping ${bits}bit benchmarks."
+      continue
+    fi
+    log I "Starting ${bits}bit benchmarks."
+
+    # Set environment variables.
+    set_environment_for_benchmark_run options
+    if [[ "${skip_build}" == "false" ]]; then
+      # Build target.
+      build_target "${bits}"
+    else
+      setup_android_target_from_bits "${bits}"
+    fi
+
+    buildbot_device_prepare "${bits}"
+    run_all_apks "${bits}"
+  done
+}
+
+main "$@"