Merge changes I2ba9e433,I78785a1e

* changes:
  tools: Add a tool to compare Janbench results
  experiments: Add a jankbench experiment

2 files changed, 271 insertions, 0 deletions

diff --git a/experiments/run_jankbench.py b/experiments/run_jankbench.py
new file mode 100755
index 0000000..efb648b
--- /dev/null
+++ b/experiments/run_jankbench.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright (C) 2017, ARM Limited, Google, and contributors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+
+from conf import LisaLogging
+LisaLogging.setup()
+import json
+import os
+import devlib
+from env import TestEnv
+from android import Screen, Workload, System
+from trace import Trace
+import trappy
+import pandas as pd
+import sqlite3
+import argparse
+import shutil
+
+parser = argparse.ArgumentParser(description='Jankbench tests')
+
+parser.add_argument('--out_prefix', dest='out_prefix', action='store', default='default',
+                    help='prefix for out directory')
+
+parser.add_argument('--collect', dest='collect', action='store', default='systrace',
+                    help='what to collect (default systrace)')
+
+parser.add_argument('--test', dest='test_name', action='store',
+                    default='overdraw',
+                    help='which test to run')
+
+parser.add_argument('--iterations', dest='iterations', action='store',
+                    default=10, type=int,
+                    help='Number of times to repeat the tests per run (default 10)')
+
+parser.add_argument('--serial', dest='serial', action='store',
+                    help='Serial number of device to test')
+
+parser.add_argument('--all', dest='run_all', action='store_true',
+                    help='Run all tests')
+
+parser.add_argument('--reinstall', dest='reinstall', action='store_true',
+                    help='Rebuild and reinstall test apks')
+
+parser.add_argument('--reimage', dest='reimage', action='store',
+                    default='',
+                    help='Flag to reimage target device (kernel-update kernel image | all-update complete android image)')
+
+parser.add_argument('--kernel_path', dest='kernel_path', action='store',
+                    default='',
+                    help='Path to kernel source directory. Required if reimage option is used')
+
+args = parser.parse_args()
+
+def make_dir(outdir):
+    try:
+        shutil.rmtree(outdir)
+    except:
+        print "couldn't remove " + outdir
+        pass
+    os.makedirs(outdir)
+
+def print_results(te, test_outdir, test_name, wload):
+    res_df = wload.get_results(test_outdir).describe(percentiles=[0.25,0.5,0.75,0.95,0.99])
+    stats_df = res_df['total_duration']
+    te._log.info("==============================================")
+    te._log.info("JANKBENCH TEST RESULT: Frame stats: {}".format(test_name))
+    stats_str = stats_df.to_string().split('\n')
+    for s in stats_str:
+        te._log.info(s)
+    te._log.info("==============================================")
+
+def experiment():
+    def run_test(outdir, test_name):
+        te._log.info("Running test {}".format(test_name))
+        wload.run(outdir, test_name=test_name, iterations=args.iterations, collect=args.collect)
+
+    if args.reimage:
+        System.reimage(te, args.kernel_path, args.reimage)
+
+    # Get workload
+    wload = Workload.getInstance(te, 'Jankbench', args.reinstall)
+
+    outdir=te.res_dir + '_' + args.out_prefix
+    make_dir(outdir)
+
+    # Run Jankbench
+    if args.run_all:
+        te._log.info("Running all tests: {}".format(wload.test_list))
+        for test in wload.get_test_list():
+            test_outdir = os.path.join(outdir, test)
+            make_dir(test_outdir)
+            run_test(test_outdir, test)
+            print_results(te, test_outdir, test, wload)
+    else:
+        test_outdir = os.path.join(outdir, args.test_name)
+        make_dir(test_outdir)
+        run_test(test_outdir, args.test_name)
+        print_results(te, test_outdir, args.test_name, wload)
+
+    # Dump platform descriptor
+    te.platform_dump(te.res_dir)
+
+    te._log.info('RESULTS are in out directory: {}'.format(outdir))
+
+# Setup target configuration
+my_conf = {
+
+    # Target platform and board
+    "platform"     : 'android',
+
+    # Useful for reading names of little/big cluster
+    # and energy model info, its device specific and use
+    # only if needed for analysis
+    # "board"        : 'pixel',
+
+    # Device
+    # By default the device connected is detected, but if more than 1
+    # device, override the following to get a specific device.
+    # "device"       : "HT6880200489",
+
+    # Folder where all the results will be collected
+    "results_dir" : "Jankbench",
+
+    # Define devlib modules to load
+    "modules"     : [
+        'cpufreq',      # enable CPUFreq support
+        'cpuidle',      # enable cpuidle support
+        # 'cgroups'     # Enable for cgroup support
+    ],
+
+    "emeter" : {
+        'instrument': 'monsoon',
+        'conf': { }
+    },
+
+    # Tools required by the experiments
+    "tools"   : [ 'taskset'],
+
+    "skip_nrg_model" : True,
+}
+
+if args.serial:
+    my_conf["device"] = args.serial
+
+# Initialize a test environment using:
+te = TestEnv(my_conf, wipe=False)
+target = te.target
+
+results = experiment()
diff --git a/tools/scripts/compare_janbench.py b/tools/scripts/compare_janbench.py
new file mode 100755
index 0000000..61718ad
--- /dev/null
+++ b/tools/scripts/compare_janbench.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright (C) 2017, ARM Limited, Google, and contributors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import pandas as pd
+import sqlite3
+import argparse
+import shutil
+
+# Run a comparison experiment between 2 Jankbench result directories
+# containing multiple subtests. This makes it easy to compare results
+# between 2 jankbench runs.
+#
+# Sample run:
+# ./compare_jankbench.py --baseline='./results/Jankbench_baseline'
+# --compare-with='./results/Jankbench_kernelchange'
+#
+# The output will be something like (only showing 25% and 50%):
+#                       25% compare  25%_diff     50%_compare 50%_diff
+#  test_name
+#  image_list_view       2.11249      0.0178108    5.7952  0.0242445
+#  list_view             2.02227      -3.65839     5.74957  -0.095421
+#  shadow_grid           6.00877      -0.000898    6.23746 -0.0057695
+#  high_hitrate_text     5.81625      0.0264913    6.03504  0.0017795
+#
+# (Note that baseline_df is only used for calculations.
+
+JANKBENCH_DB_NAME = 'BenchmarkResults'
+
+def get_results(out_dir):
+    """
+    Extract data from results db and return as a pandas dataframe
+
+    :param out_dir: Output directory for a run of the Jankbench workload
+    :type out_dir: str
+    """
+    path = os.path.join(out_dir, JANKBENCH_DB_NAME)
+    columns = ['_id', 'name', 'run_id', 'iteration', 'total_duration', 'jank_frame']
+    data = []
+    conn = sqlite3.connect(path)
+    for row in conn.execute('SELECT {} FROM ui_results'.format(','.join(columns))):
+        data.append(row)
+    return pd.DataFrame(data, columns=columns)
+
+def build_stats_df(test_outdir):
+    """
+    Build a .describe() df with statistics
+    """
+    stats_dfs = []
+    for t in tests:
+        test_dir = os.path.join(test_outdir, t)
+        res_df = get_results(test_dir).describe(percentiles=[0.25,0.5,0.75,0.9,0.95,0.99])
+        stats_df = res_df['total_duration']
+        stats_df['test_name'] = t
+        stats_dfs.append(stats_df)
+    fdf = pd.concat(stats_dfs, axis = 1).T
+    fdf.set_index('test_name', inplace=True)
+    return fdf
+
+
+parser = argparse.ArgumentParser(description='Jankbench comparisons')
+
+parser.add_argument('--baseline', dest='baseline', action='store', default='default',
+                    required=True, help='baseline out directory')
+
+parser.add_argument('--compare-with', dest='compare_with', action='store', default='default',
+                    required=True, help='out directory to compare with baseline')
+
+args = parser.parse_args()
+
+# Get list of Jankbench tests available
+tests = os.listdir(args.baseline)
+tests = [t for t in tests if os.path.isdir(os.path.join(args.baseline, t))]
+
+# Build a baseline df (experiment baseline - say without kernel change)
+#  compare df (experiment with change)
+#  diff (difference in stats between compare and baseline)
+
+baseline_df = build_stats_df(args.baseline)
+compare_df = build_stats_df(args.compare_with)
+diff = compare_df - baseline_df
+
+diff.columns = [str(col) + '_diff' for col in diff.columns]
+baseline_df.columns = [str(col) + '_baseline' for col in baseline_df.columns]
+compare_df.columns  = [str(col) + '_compare' for col in compare_df.columns]
+
+final_df = pd.concat([compare_df, diff], axis=1)
+final_df = final_df.reindex_axis(sorted(final_df.columns), axis=1)
+
+# Print the results
+print final_df