Merge "ART: Add compilation_stats_target.sh to iterate over different device configurations and collect compilation stats for them"

2 files changed, 278 insertions, 30 deletions

diff --git a/benchmarks/benchmarks_run_target.sh b/benchmarks/benchmarks_run_target.sh
index 56c3664c..b5eb24d3 100755
--- a/benchmarks/benchmarks_run_target.sh
+++ b/benchmarks/benchmarks_run_target.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2016-2017, Linaro Ltd.
+# Copyright (c) 2016-2021, Linaro Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -167,17 +167,14 @@ list_devices() {
 }
 
 # Arguments:
-#   ${1}: bitness
+#   ${1}: isa
 create_target_cmdline_sh() {
-  local isa="arm"
-  if [[ "$1" == "64" ]]; then
-    isa+="64"
-  fi
+  local isa="${1}"
 
   local dump_cfg=""
   if ${options["dump-cfg"]}; then
-    target_cfg_file="/data/local/tmp/bench.arm$1.cfg"
-    dump_cfg="-j1 --dump-cfg=${target_cfg_file}"
+    target_cfg_file="/data/local/tmp/bench.${1}.cfg"
+    dump_cfg="--dump-cfg=${target_cfg_file}"
     local verbose_methods="${options["dump-cfg-methods"]}"
     if [[ "${options["dump-cfg-methods"]}" == "benchmarks" ]]; then
       verbose_methods=""
@@ -205,26 +202,31 @@ create_target_cmdline_sh() {
   local isa_features=${options["isa-features"]}
 
   read -r -d '' target_script << EOM
-$(get_target_art_test_env_vars_exports)
-cd /data/local/tmp
-if [[ ! -d /data/local/tmp/oat/${isa} ]]; then
-  mkdir -p /data/local/tmp/oat/${isa}
-  $(get_target_dex2oat_cmd \
-      "${isa}" \
-      "/data/local/tmp/bench.apk" \
-      "${isa_features}") \
-    $(get_target_art_test_bootclasspath_for_dex2oat) ${dump_cfg}
-
-  if [[ \$? -ne 0 ]]; then
+    $(get_target_art_test_env_vars_exports)
+    cd /data/local/tmp
+
     rm -rf /data/local/tmp/oat/${isa}
-    exit 1
-  fi
-fi
+    mkdir -p /data/local/tmp/oat/${isa}
+    before=\$EPOCHREALTIME
+    $(get_target_dex2oat_cmd \
+        "${isa}" \
+        "/data/local/tmp/bench.apk" \
+        "${isa_features}") \
+        $(get_target_art_test_bootclasspath_for_dex2oat) -j1 ${dump_cfg}
+    after=\$EPOCHREALTIME
+
+    echo "Before: "\$before
+    echo "After: "\$after
+
+    if [[ \$? -ne 0 ]]; then
+      rm -rf /data/local/tmp/oat/${isa}
+      exit 1
+    fi
 
-$(get_target_art_test_dalvikvm_cmd \
-    "/data/local/tmp" \
-    "/data/local/tmp/bench.apk" \
-    "${isa_features}") \$@
+    $(get_target_art_test_dalvikvm_cmd \
+        "/data/local/tmp" \
+        "/data/local/tmp/bench.apk" \
+        "${isa_features}") \$@
 EOM
 
   safe adb_shell "mkdir -p ${ART_TEST_CHROOT}/data/local/tmp/"
@@ -239,7 +241,12 @@ copy_cfg_file_from_device() {
 }
 
 run_benchmarks() {
-  create_target_cmdline_sh "$1"
+  local isa="arm"
+  if [[ "$1" == "64" ]]; then
+    isa+="64"
+  fi
+
+  create_target_cmdline_sh "${isa}"
   export ART_COMMAND="chroot ${ART_TEST_CHROOT} sh /data/local/tmp/cmdline.sh "
 
   local run_cmd_options=()
@@ -248,9 +255,8 @@ run_benchmarks() {
   run_cmd_options+=("--mode" "$1")
   run_cmd_options+=("--output-json" "${LOG_DIRECTORY}/$2_$1_$3_aot.json")
   run_cmd_options+=("--target-copy-path" "${ART_TEST_CHROOT}/data/local/tmp")
-  if is_full_source_tree; then
-    run_cmd_options+=("--add-pathname" "$(get_boot_oat_location "$1")")
-  fi
+  run_cmd_options+=("--output-oat" "${ART_TEST_CHROOT}/data/local/tmp/oat/${isa}/bench.odex")
+
   for benchmark in "${benchmarks[@]}"; do
     run_cmd_options+=("--filter" "${benchmark}")
   done
diff --git a/benchmarks/compilation_stats_target.sh b/benchmarks/compilation_stats_target.sh
new file mode 100755
index 00000000..8a093819
--- /dev/null
+++ b/benchmarks/compilation_stats_target.sh
@@ -0,0 +1,242 @@
+#!/bin/bash
+#
+# Copyright (c) 2021, Linaro Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+readonly local_path=$(dirname "$0")
+source "${local_path}/../utils/utils.sh"
+source "${local_path}/../utils/utils_test.sh"
+source "${local_path}/../utils/utils_android.sh"
+source "${local_path}/../utils/utils_android_root.sh"
+source "${local_path}/../utils/utils_benchmarks.sh"
+source "${local_path}/../devices/cpu_freq_utils.sh"
+
+readonly LOG_DIRECTORY="$(get_workspace)"
+
+readonly default_iterations=10
+
+# shellcheck disable=SC2034
+declare -A options_format=(
+  ["help"]="p:usage()"
+  ["h"]="r:&help"
+  ["verbose"]="p:enable_verbose()"
+  ["v"]="r:&verbose"
+  ["iterations"]="${default_iterations}"
+  ["mode"]="all"
+  ["linux"]="false"
+  ["x86"]="false"
+  ["sudo"]="false"
+  ["cpu"]="all"
+  ["skip-build"]="false"
+  ["skip-run"]="false"
+  ["list-devices"]="p:list_devices()"
+  ["target-device"]=""
+  ["dump-cfg"]="false"
+)
+declare -A options=()
+declare -a benchmarks=()
+
+validate_apk_names() {
+  local apk
+  for apk in "${benchmarks[@]}"; do
+    if [[ ! -f "${apk}" ]]; then
+      log E "APK file ${apk} does not exist"
+      exit 1
+    fi
+  done
+}
+
+usage() {
+  log I "Usage: $0 [OPTION]... [APK]..."
+  log I ""
+  log I "This script should be used for getting compile stats for APK(s) on a target device."
+  log I "The script expects a device to be connected. Use --list-devices to show supported"
+  log I "devices."
+  log I ""
+  log I "-------------------------------------------"
+  log I " -h, --help                - help"
+  log I " -v, --verbose             - verbose"
+  log I " --list-devices            - List the devices supported by this script."
+  log I " --mode <all|32|64>        - Get compile stats for the specified mode(s)."
+  log I "                             (default: all)"
+  log I " --cpu <all|big|little|default> - CPU mode."
+  log I "                             \"big\": Run with only big cores and pin their frequency"
+  log I "                             \"little\": Run with only little cores and pin their"
+  log I "                             frequency"
+  log I "                             \"all\": With big.LITTLE devices:"
+  log I "                             Run consecutively with only little cores enabled and"
+  log I "                             pinned,"
+  log I "                             and then with only big cores enabled and pinned."
+  log I "                             For devices without big.LITTLE, all cores are enabled and"
+  log I "                             pinned"
+  log I "                             \"default\": Run with unaltered default CPU configuration"
+  log I "                             (no pinning)."
+  log I "                             (default: all)"
+  log I " --iterations <n>          - The number of compilations for each APK."
+  log I "                             (default: $default_iterations)"
+  log I " --target-device <device>  - Use specific lunch target and configuration for Arm "
+  log I "                             target platform".
+  log I "                             (default: \"\" - which will cause a default target arch"
+  log I "                             to be used."
+  log I " --skip-build <false|true> - Skips the build step and compiles APKs with prebuilt"
+  log I "                             artifacts from \$OUT directory."
+  log I "                             This option is mainly used for automation."
+  log I "                             Make sure \`adb\` and \`dx\` are in your PATH when using"
+  log I "                             this option."
+  log I "                             (default: false)"
+  log I " --skip-run <false|true>   - Skips the running benchmark but just builds the artifacts"
+  log I "                             (default: false)"
+  log I " --dump-cfg                - Dump control-flow graphs. Depending on the mode they will be"
+  log I "                             either in bench.arm64.cfg or bench.arm32.cfg."
+  log I "-------------------------------------------"
+  log I "Default Configuration:"
+  log I " --default                 - Default benchmark configuration, equivalent to"
+  log I "                             \`--mode all --cpu all --iterations $default_iterations\`."
+  log I "-------------------------------------------"
+  exit 0
+}
+
+list_devices() {
+  list_devices_from_config "${local_path}/../devices/config/*"
+  exit 0
+}
+
+# Arguments:
+#   ${1}: isa
+#   ${2}: APK name
+create_target_cmdline_sh() {
+  local isa="${1}"
+
+  apk="${2}"
+  apk_basename=$(basename "${apk}" ".apk")
+
+  local dump_cfg=""
+  if ${options["dump-cfg"]}; then
+    target_cfg_file="/data/local/tmp/${apk_basename}.${1}.cfg"
+    dump_cfg="--dump-cfg=${target_cfg_file}"
+  fi
+
+  # use --no-watch-dog option to bypass 9.5 minutes timeout in dex2oat
+  read -r -d '' target_script << EOM
+    $(get_target_art_test_env_vars_exports)
+    rm -rf /data/local/tmp/oat/${isa}
+    mkdir -p /data/local/tmp/oat/${isa}
+
+    before=\$EPOCHREALTIME
+    $(get_target_dex2oat_cmd "${isa}" "/data/local/tmp/${apk_basename}.apk") \
+      $(get_target_art_test_bootclasspath_for_dex2oat) -j1 ${dump_cfg} --no-watch-dog
+    after=\$EPOCHREALTIME
+
+    echo "Before: "\$before
+    echo "After: "\$after
+    \$@
+EOM
+
+  safe adb_shell "mkdir -p ${ART_TEST_CHROOT}/data/local/tmp/"
+  safe adb_shell "echo '${target_script}' > ${ART_TEST_CHROOT}/data/local/tmp/cmdline.sh"
+}
+
+# Arguments:
+#   ${1}: bitness
+#   ${2}: device
+#   ${3}: CPU mode
+run_apks() {
+  local compilation_stats_file="${LOG_DIRECTORY}/${2}_${1}_${3}_compilation_stats.json"
+  if [[ -f "${compilation_stats_file}" ]]; then
+    safe rm "${compilation_stats_file}"
+  fi
+
+  local isa="arm"
+  if [[ "${1}" == "64" ]]; then
+    isa+="64"
+  fi
+
+  start_adb_section "run_benchmarks_${2}_${1}_${3}"
+  for apk in "${benchmarks[@]}"; do
+    log I "Processing $apk"
+    adb push "${apk}" "${ART_TEST_CHROOT}/data/local/tmp"
+    apk_name=$(basename "${apk}")
+    apk_basename=$(basename "${apk_name}" ".apk")
+    output_oat="${ART_TEST_CHROOT}/data/local/tmp/oat/${isa}/${apk_basename}.odex"
+
+    create_target_cmdline_sh "${isa}" "${apk_name}"
+    export ART_COMMAND="chroot ${ART_TEST_CHROOT} sh /data/local/tmp/cmdline.sh "
+
+    local run_cmd_options=()
+    run_cmd_options+=("--target")
+    run_cmd_options+=("--iterations" "${options["iterations"]}")
+    run_cmd_options+=("--mode" "${1}")
+    run_cmd_options+=("--output-json" "${compilation_stats_file}")
+    run_cmd_options+=("--output-oat" "${output_oat}")
+    run_cmd_options+=("--target-copy-path" "${ART_TEST_CHROOT}/data/local/tmp")
+    run_cmd_options+=("--add-pathname" "${apk_name}")
+    ./benchmarks/compilation_stats.py "${run_cmd_options[@]}"
+  done
+  end_adb_section "run_benchmarks_${2}_${1}_${3}" "$?"
+}
+
+# Arguments:
+#   ${1}: bitness
+run_all_apks() {
+  local -r bitness="${1}"
+  local -r target_device=$(safe adb_shell getprop ro.product.device)
+  local -r cpu="${options["cpu"]}"
+  local -r path_to_devices="${local_path}/../devices"
+  set_freq_and_run run_apks "${bitness}" "${cpu}" "${target_device}" "${path_to_devices}"
+}
+
+main() {
+  exit_on_failure arguments_parser options_format options benchmarks -- "$@"
+
+  if [[ -z "${benchmarks[@]}" ]]; then
+    log E "Empty list of APKs to compile"
+    exit 1
+  fi
+
+  validate_apk_names
+
+  readonly options
+  dump_options
+
+  local -r mode="${options["mode"]}"
+  local -r skip_build="${options["skip-build"]}"
+
+  if android_build_already_setup; then
+    log E "This test does not support environment targets. Please re-run in a clean environment."
+    exit 1
+  fi
+
+  for bits in 32 64; do
+    if [[ "$mode" != "all" && "$mode" != "$bits" ]]; then
+      log I "Skipping ${bits}bit benchmarks."
+      continue
+    fi
+    log I "Starting ${bits}bit benchmarks."
+
+    # Set environment variables.
+    set_environment_for_benchmark_run options
+    if [[ "${skip_build}" == "false" ]]; then
+      # Build target.
+      build_target "${bits}"
+    else
+      setup_android_target_from_bits "${bits}"
+    fi
+
+    buildbot_device_prepare "${bits}"
+    run_all_apks "${bits}"
+  done
+}
+
+main "$@"