summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDan Albert <danalbert@google.com>2015-11-17 13:47:16 -0800
committerDan Albert <danalbert@google.com>2015-11-17 13:47:16 -0800
commita14a4add9d8e72c10f68f50adde4d73c299f186f (patch)
treec19277787d3fdf604f54d73b571c0fc848ced12c
parentf9a912f6cfbc0c8e9419cafb0c7b1698926dce5f (diff)
parent691026242b81aceac4805322e2a51cb5ccbb6766 (diff)
downloadchromite-a14a4add9d8e72c10f68f50adde4d73c299f186f.tar.gz
Merge to upstream 691026242b81aceac4805322e2a51cb5ccbb6766HEADndk-r11cndk-r11bndk-r11ndk-r11-releasemastermainbrillo-m10-releasebrillo-m10-dev
-rw-r--r--bootstrap/scripts/brillo.py59
-rw-r--r--bootstrap/scripts/brillo_unittest.py176
-rw-r--r--cbuildbot/archive_lib.py3
-rw-r--r--cbuildbot/archive_lib_unittest.py8
-rw-r--r--cbuildbot/binhost_test.py2
-rw-r--r--cbuildbot/builders/generic_builders.py7
l---------cbuildbot/builders/init_unittest (renamed from bootstrap/scripts/brillo_unittest)0
-rw-r--r--cbuildbot/builders/init_unittest.py (renamed from cbuildbot/builders/builders_unittest.py)0
-rw-r--r--cbuildbot/builders/simple_builders.py2
-rw-r--r--cbuildbot/cbuildbot_run.py1
-rw-r--r--cbuildbot/chromeos_config.py163
-rw-r--r--cbuildbot/commands.py105
-rw-r--r--cbuildbot/commands_unittest.py132
-rw-r--r--cbuildbot/config_dump.json1914
-rw-r--r--cbuildbot/config_lib.py12
-rw-r--r--cbuildbot/constants.py7
-rw-r--r--cbuildbot/lkgm_manager.py8
-rw-r--r--cbuildbot/manifest_version.py154
-rw-r--r--cbuildbot/manifest_version_unittest.py239
l---------cbuildbot/results_lib_unittest (renamed from bootstrap/brillo)0
-rw-r--r--cbuildbot/results_lib_unittest.py (renamed from cbuildbot/stages/stage_results_unittest.py)0
-rw-r--r--cbuildbot/run_tests.py94
-rw-r--r--cbuildbot/run_tests_unittest.py32
-rw-r--r--cbuildbot/stages/artifact_stages.py45
-rw-r--r--cbuildbot/stages/build_stages.py44
-rw-r--r--cbuildbot/stages/build_stages_unittest.py26
-rw-r--r--cbuildbot/stages/chrome_stages_unittest.py3
-rw-r--r--cbuildbot/stages/completion_stages.py30
-rw-r--r--cbuildbot/stages/report_stages.py79
-rw-r--r--cbuildbot/stages/report_stages_unittest.py38
-rw-r--r--cbuildbot/stages/sdk_stages_unittest.py3
-rw-r--r--cbuildbot/stages/sync_stages.py41
-rw-r--r--cbuildbot/stages/test_stages.py28
-rw-r--r--cbuildbot/stages/test_stages_unittest.py19
-rw-r--r--cbuildbot/swarming_lib.py53
-rw-r--r--cbuildbot/triage_lib.py21
-rw-r--r--cbuildbot/triage_lib_unittest.py28
-rw-r--r--cbuildbot/update_binhost_json.py2
-rw-r--r--cbuildbot/validation_pool.py39
-rw-r--r--cbuildbot/validation_pool_unittest.py81
-rw-r--r--cidb/developer.readme9
-rw-r--r--cidb/migrations/00042_create_build_message_table.sql15
-rw-r--r--cidb/migrations/00043_alter_build_table_add_important.sql5
-rw-r--r--cidb/migrations/00044_alter_views_add_important.sql32
-rw-r--r--cidb/migrations/00045_alter_build_message_table_add_board.sql6
-rw-r--r--cidb/schema.dump31
-rw-r--r--cidb/schema.dump.readme12
-rw-r--r--cli/command.py25
-rw-r--r--cli/command_unittest.py47
-rw-r--r--cli/cros/cros_build.py40
-rw-r--r--cli/cros/cros_chrome_sdk.py6
-rw-r--r--cli/cros/cros_payload.py78
-rw-r--r--cli/cros/cros_payload_unittest.py165
-rw-r--r--cli/cros/lint.py23
-rw-r--r--cli/cros/lint_unittest.py25
-rw-r--r--cli/flash_unittest.py3
-rw-r--r--compute/bot_constants.py6
-rw-r--r--compute/compute_configs.py5
-rw-r--r--compute/setup_bot.py3
-rw-r--r--lib/blueprint_lib.py159
l---------lib/blueprint_lib_unittest1
-rw-r--r--lib/blueprint_lib_unittest.py121
-rw-r--r--lib/bootstrap_lib.py101
l---------lib/bootstrap_lib_unittest1
-rw-r--r--lib/bootstrap_lib_unittest.py128
-rw-r--r--lib/brick_lib.py273
l---------lib/brick_lib_unittest1
-rw-r--r--lib/brick_lib_unittest.py217
-rw-r--r--lib/chrome_util.py5
-rw-r--r--lib/chroot_util.py29
-rw-r--r--lib/cidb.py163
-rw-r--r--lib/cidb_integration_test.py81
l---------lib/cidb_setup_unittest1
-rw-r--r--lib/cidb_setup_unittest.py120
-rw-r--r--lib/cidb_unittest.py110
-rw-r--r--lib/cipd.py115
-rw-r--r--lib/commandline.py94
-rw-r--r--lib/commandline_unittest.py75
-rw-r--r--lib/cros_build_lib.py21
-rw-r--r--lib/cros_build_lib_unittest.py20
-rw-r--r--lib/cros_test_lib.py126
-rw-r--r--lib/cros_test_lib_unittest.py86
-rw-r--r--lib/factory_unittest.py2
-rw-r--r--lib/fake_cidb.py16
-rw-r--r--lib/gerrit.py4
-rw-r--r--lib/git.py4
-rw-r--r--lib/gob_util.py2
-rw-r--r--lib/graphite_lib/elasticsearch_mock_unittest.py1
-rw-r--r--lib/gs.py10
-rw-r--r--lib/gs_unittest.py21
-rw-r--r--lib/launch_control/__init__.py0
l---------lib/launch_control/launch_control (renamed from cbuildbot/builders/builders_unittest)0
-rw-r--r--lib/launch_control/launch_control.py142
-rw-r--r--lib/launch_control/processed_builds.py84
l---------lib/launch_control/processed_builds_unittest (renamed from cbuildbot/stages/stage_results_unittest)0
-rw-r--r--lib/launch_control/processed_builds_unittest.py84
-rw-r--r--lib/operation.py20
-rw-r--r--lib/operation_unittest.py76
-rw-r--r--lib/patch.py15
-rw-r--r--lib/patch_unittest.py86
-rw-r--r--lib/path_util.py57
-rw-r--r--lib/path_util_unittest.py56
-rw-r--r--lib/paygen/gslock_unittest.py178
-rw-r--r--lib/paygen/gspaths.py12
-rw-r--r--lib/paygen/gspaths_unittest.py12
-rw-r--r--lib/paygen/paygen_build_lib.py481
-rw-r--r--lib/paygen/paygen_build_lib_unittest.py575
-rw-r--r--lib/paygen/paygen_payload_lib.py47
-rw-r--r--lib/paygen/paygen_payload_lib_unittest.py45
-rw-r--r--lib/portage_util.py30
-rw-r--r--lib/portage_util_unittest.py28
-rw-r--r--lib/project_sdk.py160
l---------lib/project_sdk_unittest1
-rw-r--r--lib/project_sdk_unittest.py168
-rw-r--r--lib/sysroot_lib.py25
-rw-r--r--lib/toolchain.py15
-rw-r--r--lib/toolchain_list.py31
-rw-r--r--lib/toolchain_unittest.py71
-rw-r--r--lib/workspace_lib.py329
l---------lib/workspace_lib_unittest1
-rw-r--r--lib/workspace_lib_unittest.py229
-rwxr-xr-xmobmonitor/scripts/mobmonitor.py8
-rw-r--r--mobmonitor/static/js/main.js2
-rw-r--r--mobmonitor/util/__init__.py0
-rw-r--r--mobmonitor/util/collect_logs.py40
-rw-r--r--scripts/cidb_admin.py2
-rw-r--r--scripts/cros_best_revision.py2
-rw-r--r--scripts/cros_brick_utils.py40
-rw-r--r--scripts/cros_generate_breakpad_symbols.py16
-rw-r--r--scripts/cros_generate_breakpad_symbols_unittest.py43
-rw-r--r--scripts/cros_list_modified_packages.py20
-rw-r--r--scripts/cros_list_overlays.py33
-rw-r--r--scripts/cros_mark_as_stable.py22
-rw-r--r--scripts/cros_mark_as_stable_unittest.py18
-rw-r--r--scripts/cros_setup_toolchains.py30
-rw-r--r--scripts/cros_show_waterfall_layout.py2
-rw-r--r--scripts/cros_sysroot_utils.py8
-rw-r--r--scripts/cros_unittest.py2
-rw-r--r--scripts/cros_workon.py11
-rw-r--r--scripts/gerrit.py2
-rw-r--r--scripts/parallel_emerge.py5
-rw-r--r--scripts/pushimage.py256
-rw-r--r--scripts/pushimage_unittest.py117
-rw-r--r--scripts/summarize_build_stats.py93
-rw-r--r--scripts/summarize_build_stats_unittest.py1
-rw-r--r--scripts/upload_symbols.py50
-rw-r--r--scripts/upload_symbols_unittest.py22
-rwxr-xr-xscripts/wrapper.py52
-rw-r--r--signing/signer_instructions/README76
-rw-r--r--signing/signer_instructions/test.multi.instructions20
150 files changed, 5480 insertions, 4913 deletions
diff --git a/bootstrap/scripts/brillo.py b/bootstrap/scripts/brillo.py
deleted file mode 100644
index 1841a4771..000000000
--- a/bootstrap/scripts/brillo.py
+++ /dev/null
@@ -1,59 +0,0 @@
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Bootstrap wrapper for 'brillo' command.
-
-For most commands of the form "brillo XYZ", we reinvoke
-REPO_DIR/chromite/bin/brillo XYZ, after detecting REPO_DIR based on the CWD.
-
-For the "brillo sdk" command, we reinvoke "../bin/brillo sdk" from the current
-git repository. This allows the SDK command to be run, even if there is no repo
-checkout.
-"""
-
-from __future__ import print_function
-
-import os
-
-from chromite.lib import bootstrap_lib
-from chromite.lib import cros_build_lib
-from chromite.lib import git
-from chromite.lib import workspace_lib
-
-
-def LocateBrilloCommand(args):
- bootstrap_path = bootstrap_lib.FindBootstrapPath(save_to_env=True)
-
- if len(args) >= 1 and args[0] == 'sdk':
- if not bootstrap_path:
- cros_build_lib.Die(
- 'You are bootstrapping chromite from a repo checkout.\n'
- 'You must use a git clone. (brbug.com/580: link docs)')
-
- # Run 'brillo sdk' from the repository containing this command.
- return os.path.join(bootstrap_path, 'bin', 'brillo')
-
- # If we are in a workspace, and the workspace has an associated SDK, use it.
- workspace_path = workspace_lib.WorkspacePath()
- if workspace_path:
- sdk_path = bootstrap_lib.GetActiveSdkPath(bootstrap_path, workspace_path)
- if not sdk_path:
- cros_build_lib.Die(
- 'The current workspace has no valid SDK.\n'
- 'Please run "brillo sdk --update" (brbug.com/580: link docs)')
-
- # Use SDK associated with workspace, or nothing.
- return os.path.join(sdk_path, 'chromite', 'bin', 'brillo')
-
- # Run all other commands from 'brillo' wrapper in repo detected via CWD.
- repo_path = git.FindRepoCheckoutRoot(os.getcwd())
- if repo_path:
- return os.path.join(repo_path, 'chromite', 'bin', 'brillo')
-
- # Couldn't find the real brillo command to run.
- cros_build_lib.Die('Unable to detect which SDK you want to use.')
-
-def main(args):
- bin_cmd = LocateBrilloCommand(args)
- os.execv(bin_cmd, [bin_cmd] + args)
diff --git a/bootstrap/scripts/brillo_unittest.py b/bootstrap/scripts/brillo_unittest.py
deleted file mode 100644
index c4c522672..000000000
--- a/bootstrap/scripts/brillo_unittest.py
+++ /dev/null
@@ -1,176 +0,0 @@
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Test the bootstrap brillo command."""
-
-from __future__ import print_function
-
-import mock
-import os
-
-from chromite.lib import cros_build_lib
-from chromite.lib import cros_test_lib
-from chromite.lib import git
-
-from chromite.bootstrap.scripts import brillo
-
-
-class TestBootstrapBrilloCmd(cros_test_lib.WorkspaceTestCase):
- """Tests for the bootstrap brillo command."""
-
- def setUp(self):
- # Make certain we never exec anything.
- self.mock_exec = self.PatchObject(os, 'execv', autospec=True)
-
- self.mock_repo_root = self.PatchObject(
- git, 'FindRepoCheckoutRoot', autospec=True)
-
- def _verifyLocateBrilloCommand(self, expected):
- self.assertEqual(expected,
- brillo.LocateBrilloCommand(['flash']))
- self.assertEqual(expected,
- brillo.LocateBrilloCommand(['flash', '--help']))
-
- def _verifyLocateBrilloCommandSdkHandling(self, expected):
- self.assertEqual(expected,
- brillo.LocateBrilloCommand(['sdk']))
- self.assertEqual(expected,
- brillo.LocateBrilloCommand(['sdk', '--help']))
-
- def _verifyLocateBrilloCommandFail(self):
- with self.assertRaises(cros_build_lib.DieSystemExit):
- brillo.LocateBrilloCommand(['flash'])
-
- def _verifyLocateBrilloCommandSdkFail(self):
- with self.assertRaises(cros_build_lib.DieSystemExit):
- brillo.LocateBrilloCommand(['sdk'])
-
- def testCommandLookupActiveWorkspace(self):
- """Test that sdk commands are run in the Git Repository."""
- self.CreateBootstrap('1.2.3')
- self.CreateWorkspace('1.2.3')
-
- sdk_wrapper = os.path.join(
- self.bootstrap_path, 'sdk_checkouts/1.2.3/chromite/bin/brillo')
- bootstrap_wrapper = os.path.join(self.bootstrap_path, 'bin/brillo')
-
- # We are not inside a repo.
- self.mock_repo_root.return_value = None
-
- self._verifyLocateBrilloCommand(sdk_wrapper)
- self._verifyLocateBrilloCommandSdkHandling(bootstrap_wrapper)
-
- # We are inside a repo, shouldn't affect the result.
- self.mock_repo_root.return_value = '/repo'
-
- self._verifyLocateBrilloCommand(sdk_wrapper)
- self._verifyLocateBrilloCommandSdkHandling(bootstrap_wrapper)
-
- def testCommandLookupInactiveWorkspace(self):
- """Test that sdk commands are run in the Git Repository."""
- self.CreateBootstrap()
- self.CreateWorkspace()
- self.mock_repo_root.return_value = None
-
- bootstrap_wrapper = os.path.join(self.bootstrap_path, 'bin/brillo')
-
- self._verifyLocateBrilloCommandFail()
- self._verifyLocateBrilloCommandSdkHandling(bootstrap_wrapper)
-
- # Having a repo root shouldn't affect the result.
- self.mock_repo_root.return_value = '/repo'
-
- self._verifyLocateBrilloCommandFail()
- self._verifyLocateBrilloCommandSdkHandling(bootstrap_wrapper)
-
- def testCommandLookupRepoFromBootstrap(self):
- """Test that sdk commands are run in the Git Repository."""
- self.CreateBootstrap('1.2.3')
- self.CreateWorkspace()
- self.mock_workspace_path.return_value = None
- self.mock_repo_root.return_value = '/repo'
-
- bootstrap_wrapper = os.path.join(self.bootstrap_path, 'bin/brillo')
- repo_wrapper = '/repo/chromite/bin/brillo'
-
- self._verifyLocateBrilloCommand(repo_wrapper)
- self._verifyLocateBrilloCommandSdkHandling(bootstrap_wrapper)
-
- def testCommandLookupBootstrapOnly(self):
- """Test that sdk commands are run in the Git Repository."""
- self.CreateBootstrap('1.2.3')
- self.CreateWorkspace()
- self.mock_workspace_path.return_value = None
- self.mock_repo_root.return_value = None
-
- bootstrap_wrapper = os.path.join(self.bootstrap_path, 'bin/brillo')
-
- self._verifyLocateBrilloCommandFail()
- self._verifyLocateBrilloCommandSdkHandling(bootstrap_wrapper)
-
- def testCommandLookupRepoOnly(self):
- """Test that sdk commands are run in the Git Repository."""
- self.CreateBootstrap('1.2.3')
- self.CreateWorkspace()
- self.mock_bootstrap_path.return_value = None
- self.mock_workspace_path.return_value = None
- self.mock_repo_root.return_value = '/repo'
-
- repo_wrapper = '/repo/chromite/bin/brillo'
-
- self._verifyLocateBrilloCommand(repo_wrapper)
- self._verifyLocateBrilloCommandSdkFail()
-
- def testMainInActiveWorkspace(self):
- self.CreateBootstrap('1.2.3')
- self.CreateWorkspace('1.2.3')
- self.mock_repo_root.return_value = None
-
- brillo.main(['flash', '--help'])
-
- expected_cmd = os.path.join(
- self.bootstrap_path, 'sdk_checkouts/1.2.3/chromite/bin/brillo')
-
- self.assertEqual(
- [mock.call(expected_cmd, [expected_cmd, 'flash', '--help'])],
- self.mock_exec.call_args_list)
-
- def testMainInRepo(self):
- self.CreateBootstrap('1.2.3')
- self.CreateWorkspace('1.2.3')
- self.mock_workspace_path.return_value = None
- self.mock_repo_root.return_value = '/repo'
-
- brillo.main(['flash', '--help'])
-
- expected_cmd = '/repo/chromite/bin/brillo'
-
- self.assertEqual(
- [mock.call(expected_cmd, [expected_cmd, 'flash', '--help'])],
- self.mock_exec.call_args_list)
-
- def testMainNoCmd(self):
- self.CreateBootstrap('1.2.3')
- self.CreateWorkspace('1.2.3')
- self.mock_workspace_path.return_value = None
- self.mock_repo_root.return_value = None
-
- with self.assertRaises(cros_build_lib.DieSystemExit):
- brillo.main(['flash', '--help'])
-
- self.assertEqual([], self.mock_exec.call_args_list)
-
- def testMainSdkCmd(self):
- self.CreateBootstrap('1.2.3')
- self.CreateWorkspace('1.2.3')
- self.mock_workspace_path.return_value = None
- self.mock_repo_root.return_value = None
-
- brillo.main(['sdk', '--help'])
-
- expected_cmd = os.path.join(self.bootstrap_path, 'bin/brillo')
-
- self.assertEqual(
- [mock.call(expected_cmd, [expected_cmd, 'sdk', '--help'])],
- self.mock_exec.call_args_list)
diff --git a/cbuildbot/archive_lib.py b/cbuildbot/archive_lib.py
index a4cef3521..cf8d08014 100644
--- a/cbuildbot/archive_lib.py
+++ b/cbuildbot/archive_lib.py
@@ -10,7 +10,6 @@ import os
from chromite.cbuildbot import commands
from chromite.cbuildbot import config_lib
-from chromite.cbuildbot import constants
from chromite.lib import cros_logging as logging
from chromite.lib import gs
@@ -46,7 +45,7 @@ def GetBaseUploadURI(config, archive_base=None, bot_id=None,
if archive_base:
return '%s/%s' % (archive_base, bot_id)
elif remote_trybot or config.gs_path == config_lib.GS_PATH_DEFAULT:
- return '%s/%s' % (constants.DEFAULT_ARCHIVE_BUCKET, bot_id)
+ return '%s/%s' % (config_lib.GetConfig().params.ARCHIVE_URL, bot_id)
else:
return config.gs_path
diff --git a/cbuildbot/archive_lib_unittest.py b/cbuildbot/archive_lib_unittest.py
index 96dc2c7f6..691e7dfe8 100644
--- a/cbuildbot/archive_lib_unittest.py
+++ b/cbuildbot/archive_lib_unittest.py
@@ -121,14 +121,14 @@ class GetBaseUploadURITest(cros_test_lib.TestCase):
def testRemoteTrybotTrue(self):
"""Test GetBaseUploadURI with no archive base but remote_trybot is True."""
expected_result = ('%s/trybot-%s' %
- (archive_lib.constants.DEFAULT_ARCHIVE_BUCKET,
+ (config_lib.GetConfig().params.ARCHIVE_URL,
DEFAULT_BOT_NAME))
result = self._GetBaseUploadURI(remote_trybot=True)
self.assertEqual(expected_result, result)
def testBotIdRemoteTrybotTrue(self):
expected_result = ('%s/%s' %
- (archive_lib.constants.DEFAULT_ARCHIVE_BUCKET,
+ (config_lib.GetConfig().params.ARCHIVE_URL,
self.BOT_ID))
result = self._GetBaseUploadURI(bot_id=self.BOT_ID, remote_trybot=True)
self.assertEqual(expected_result, result)
@@ -139,14 +139,14 @@ class GetBaseUploadURITest(cros_test_lib.TestCase):
# Test without bot_id.
expected_result = ('%s/%s' %
- (archive_lib.constants.DEFAULT_ARCHIVE_BUCKET,
+ (config_lib.GetConfig().params.ARCHIVE_URL,
DEFAULT_BOT_NAME))
result = self._GetBaseUploadURI(remote_trybot=False)
self.assertEqual(expected_result, result)
# Test with bot_id.
expected_result = ('%s/%s' %
- (archive_lib.constants.DEFAULT_ARCHIVE_BUCKET,
+ (config_lib.GetConfig().params.ARCHIVE_URL,
self.BOT_ID))
result = self._GetBaseUploadURI(bot_id=self.BOT_ID, remote_trybot=False)
self.assertEqual(expected_result, result)
diff --git a/cbuildbot/binhost_test.py b/cbuildbot/binhost_test.py
index 0bab437b6..eb19707a5 100644
--- a/cbuildbot/binhost_test.py
+++ b/cbuildbot/binhost_test.py
@@ -40,7 +40,7 @@ class PrebuiltCompatibilityTest(cros_test_lib.TestCase):
# TODO(davidjames): Empty this list.
BOARDS_WITHOUT_CHROMIUM_PFQS = ['rush_ryu', 'smaug']
- site_config = config_lib.LoadConfigFromFile()
+ site_config = config_lib.GetConfig()
@classmethod
def setUpClass(cls):
diff --git a/cbuildbot/builders/generic_builders.py b/cbuildbot/builders/generic_builders.py
index 5614774b8..a8823108b 100644
--- a/cbuildbot/builders/generic_builders.py
+++ b/cbuildbot/builders/generic_builders.py
@@ -14,6 +14,7 @@ import traceback
from chromite.cbuildbot import constants
from chromite.cbuildbot import failures_lib
+from chromite.cbuildbot import manifest_version
from chromite.cbuildbot import results_lib
from chromite.cbuildbot import trybot_patch_pool
from chromite.cbuildbot.stages import build_stages
@@ -152,9 +153,11 @@ class Builder(object):
def GetVersionInfo(self):
"""Returns a manifest_version.VersionInfo object for this build.
- Subclasses must override this method.
+ Chrome OS Subclasses must override this method. Site specific builds which
+ don't use Chrome OS versioning should leave this alone.
"""
- raise NotImplementedError()
+ # Placeholder version for non-Chrome OS builds.
+ return manifest_version.VersionInfo('1.0.0')
def GetSyncInstance(self):
"""Returns an instance of a SyncStage that should be run.
diff --git a/bootstrap/scripts/brillo_unittest b/cbuildbot/builders/init_unittest
index ef3e37b67..ef3e37b67 120000
--- a/bootstrap/scripts/brillo_unittest
+++ b/cbuildbot/builders/init_unittest
diff --git a/cbuildbot/builders/builders_unittest.py b/cbuildbot/builders/init_unittest.py
index 084365137..084365137 100644
--- a/cbuildbot/builders/builders_unittest.py
+++ b/cbuildbot/builders/init_unittest.py
diff --git a/cbuildbot/builders/simple_builders.py b/cbuildbot/builders/simple_builders.py
index ce6ce1971..4a5f02fa8 100644
--- a/cbuildbot/builders/simple_builders.py
+++ b/cbuildbot/builders/simple_builders.py
@@ -390,6 +390,8 @@ class DistributedBuilder(SimpleBuilder):
not self._run.config.afdo_generate_min):
self._RunStage(afdo_stages.AFDOUpdateEbuildStage)
finally:
+ if self._run.config.master:
+ self._RunStage(report_stages.SlaveFailureSummaryStage)
if self._run.config.push_overlays:
publish = (was_build_successful and completion_successful and
build_finished)
diff --git a/cbuildbot/cbuildbot_run.py b/cbuildbot/cbuildbot_run.py
index ee2d67b58..ef81864ce 100644
--- a/cbuildbot/cbuildbot_run.py
+++ b/cbuildbot/cbuildbot_run.py
@@ -169,7 +169,6 @@ class RunAttributes(object):
'instruction_urls_per_channel', # Set by ArchiveStage
'success', # Set by cbuildbot.py:Builder
'packages_under_test', # Set by BuildPackagesStage.
- 'gce_tarball_generated', # Set by ArchiveStage.
))
# Attributes that need to be set by stages that can run in parallel
diff --git a/cbuildbot/chromeos_config.py b/cbuildbot/chromeos_config.py
index 77131d0ae..883510c93 100644
--- a/cbuildbot/chromeos_config.py
+++ b/cbuildbot/chromeos_config.py
@@ -123,8 +123,6 @@ class HWTestList(object):
async_kwargs = kwargs.copy()
async_kwargs.update(async_dict)
async_kwargs['priority'] = constants.HWTEST_POST_BUILD_PRIORITY
- async_kwargs['retry'] = False
- async_kwargs['max_retries'] = None
async_kwargs['async'] = True
async_kwargs['suite_min_duts'] = 1
@@ -359,12 +357,12 @@ _arm_internal_release_boards = frozenset([
'veyron_mickey',
'veyron_mighty',
'veyron_minnie',
+ 'veyron_minnie-cheets',
'veyron_pinky',
'veyron_rialto',
'veyron_romy',
'veyron_shark',
'veyron_speedy',
- 'veyron_thea',
'whirlwind',
])
@@ -386,10 +384,12 @@ _x86_internal_release_boards = frozenset([
'butterfly',
'candy',
'celes',
+ 'chell',
'cid',
'clapper',
'cranky',
'cyan',
+ 'cyan-cheets',
'enguarde',
'expresso',
'falco',
@@ -406,6 +406,8 @@ _x86_internal_release_boards = frozenset([
'kunimitsu',
'lakitu',
'lakitu_mobbuild',
+ 'lakitu_next',
+ 'lars',
'leon',
'link',
'lulu',
@@ -423,6 +425,7 @@ _x86_internal_release_boards = frozenset([
'peppy',
'quawks',
'rambi',
+ 'reks',
'rikku',
'samus',
'slippy',
@@ -433,6 +436,7 @@ _x86_internal_release_boards = frozenset([
'stumpy_moblab',
'sumo',
'swanky',
+ 'terra',
'tidus',
'tricky',
'ultima',
@@ -495,14 +499,18 @@ _brillo_boards = frozenset([
'arkham',
'gizmo',
'kayle',
- 'lakitu',
- 'lakitu_mobbuild',
'panther_embedded',
'purin',
'storm',
'whirlwind',
])
+_lakitu_boards = frozenset([
+ 'lakitu',
+ 'lakitu_mobbuild',
+ 'lakitu_next',
+])
+
_moblab_boards = frozenset([
'stumpy_moblab',
'panther_moblab',
@@ -522,35 +530,25 @@ _toolchains_from_source = frozenset([
'x32-generic',
])
-_noimagetest_boards = frozenset([
- 'lakitu',
- 'lakitu_mobbuild',
-])
+_noimagetest_boards = _lakitu_boards
-_nohwqual_boards = frozenset([
+_nohwqual_boards = _lakitu_boards | frozenset([
'kayle',
- 'lakitu',
- 'lakitu_mobbuild',
])
_norootfs_verification_boards = frozenset([
])
-_base_layout_boards = frozenset([
- 'lakitu',
- 'lakitu_mobbuild',
-])
+_base_layout_boards = _lakitu_boards
_no_unittest_boards = frozenset((
))
-_upload_gce_images_boards = frozenset([
- 'lakitu',
- 'lakitu_mobbuild',
-])
-
-_no_vmtest_boards = _arm_boards | _brillo_boards
+_upload_gce_images_boards = _lakitu_boards
+_no_vmtest_boards = _arm_boards | _brillo_boards | frozenset((
+ 'cyan-cheets',
+))
# This is a list of configs that should be included on the main waterfall, but
# aren't included by default (see IsDefaultMainWaterfall). This loosely
@@ -586,18 +584,24 @@ _waterfall_config_map = {
constants.WATERFALL_INTERNAL: frozenset([
# Experimental Paladins.
+ 'lakitu_next-paladin',
'panther_moblab-paladin',
'stumpy_moblab-paladin',
+ 'veyron_mighty-paladin',
+ 'veyron_speedy-paladin',
# Experimental Canaries (Group)
+ 'kunimitsu-release-group',
'storm-release-group',
'strago-release-group',
- 'veyron-c-release-group',
+ 'strago-b-release-group',
+ 'veyron-d-release-group',
# Experimental Canaries
'bobcat-release',
'daisy_winter-release',
'kayle-release',
+ 'lakitu_next-release',
'nyan_freon-release',
'panther_moblab-release',
'rush_ryu-release',
@@ -607,12 +611,12 @@ _waterfall_config_map = {
# Incremental Builders.
'mario-incremental',
'lakitu-incremental',
+ 'lakitu_next-incremental',
# Firmware Builders.
'link-depthcharge-full-firmware',
# Toolchain Builders.
- 'internal-toolchain-major',
'internal-toolchain-minor',
]),
@@ -883,6 +887,15 @@ def GetConfig():
dev_installer_prebuilts=False,
# TODO(gauravsh): crbug.com/356414 Start running tests on Brillo configs.
vm_tests=[],
+ )
+
+ lakitu = config_lib.BuildConfig(
+ sync_chrome=False,
+ chrome_sdk=False,
+ afdo_use=False,
+ dev_installer_prebuilts=False,
+ vm_tests=[],
+ vm_tests_override=None,
hw_tests=[],
)
@@ -1006,7 +1019,6 @@ def GetConfig():
chrome_try = config_lib.BuildConfig(
build_type=constants.CHROME_PFQ_TYPE,
chrome_rev=constants.CHROME_REV_TOT,
- use_lkgm=True,
important=False,
manifest_version=False,
)
@@ -1036,7 +1048,6 @@ def GetConfig():
'perf_v2', pool=constants.HWTEST_CHROME_PERF_POOL,
timeout=90 * 60, critical=True, num=1)],
use_chrome_lkgm=True,
- use_lkgm=False,
useflags=append_useflags(['-cros-debug']),
)
@@ -1054,6 +1065,8 @@ def GetConfig():
base.update(manifest=constants.OFFICIAL_MANIFEST)
if board in _brillo_boards:
base.update(brillo)
+ if board in _lakitu_boards:
+ base.update(lakitu)
if board in _moblab_boards:
base.update(moblab)
if board in _minimal_profile_boards:
@@ -1235,16 +1248,6 @@ def GetConfig():
_CreateConfigsForBoards(telemetry, _telemetry_boards, 'telemetry')
- _toolchain_major = site_config.AddConfigWithoutTemplate(
- 'toolchain-major',
- _cros_sdk,
- latest_toolchain=True,
- prebuilts=False,
- trybot_list=False,
- gcc_githash='svn-mirror/google/main',
- description='Test next major toolchain revision',
- )
-
_toolchain_minor = site_config.AddConfigWithoutTemplate(
'toolchain-minor',
_cros_sdk,
@@ -1297,7 +1300,6 @@ def GetConfig():
trybot_list=True,
)
-
site_config.AddConfig(
tot_asan_info, 'amd64-generic-tot-asan-informational',
boards=['amd64-generic'],
@@ -1372,7 +1374,6 @@ def GetConfig():
_CreateConfigsForBoards(chrome_perf, _chrome_perf_boards, 'chrome-perf',
trybot_list=True)
-
_CreateConfigsForBoards(chromium_info,
['x86-generic', 'amd64-generic'],
'telem-chromium-pfq-informational',
@@ -1606,6 +1607,7 @@ def GetConfig():
'nyan_freon',
'falco',
'gizmo',
+ 'glados',
'guado_moblab',
'kayle',
'lakitu',
@@ -1630,7 +1632,6 @@ def GetConfig():
'stumpy',
'tricky',
'veyron_pinky',
- 'whirlwind',
'wolf',
'x86-alex',
'x86-generic',
@@ -1667,11 +1668,19 @@ def GetConfig():
'peach_pit',
'peppy',
'stumpy',
+ 'veyron_mighty',
+ 'veyron_speedy',
'wolf',
'x86-alex',
'x86-zgb',
])
+
+ # Jetstream devices run unique hw tests
+ _paladin_jetstream_hwtest_boards = frozenset([
+ 'whirlwind',
+ ])
+
_paladin_moblab_hwtest_boards = frozenset([
'guado_moblab',
])
@@ -1702,6 +1711,13 @@ def GetConfig():
blocking=True, num=1, timeout=120*60,
pool=constants.HWTEST_PALADIN_POOL)
])
+ if board in _paladin_jetstream_hwtest_boards:
+ customizations.update(
+ hw_tests=[
+ config_lib.HWTestConfig(
+ constants.HWTEST_JETSTREAM_COMMIT_SUITE,
+ pool=constants.HWTEST_PALADIN_POOL)
+ ])
if board not in _paladin_important_boards:
customizations.update(important=False)
if board in _paladin_chroot_replace_boards:
@@ -1745,10 +1761,8 @@ def GetConfig():
customizations,
base_config)
-
_CreatePaladinConfigs()
-
site_config.AddConfig(
internal_paladin, 'lumpy-incremental-paladin',
boards=['lumpy'],
@@ -1758,8 +1772,8 @@ def GetConfig():
unittests=False,
)
- ### Paladins (CQ builders) which do not run VM or Unit tests on the builder
- ### itself.
+ # Paladins (CQ builders) which do not run VM or Unit tests on the builder
+ # itself.
external_brillo_paladin = paladin.derive(brillo)
site_config.AddConfig(
@@ -1816,15 +1830,15 @@ def GetConfig():
ShardHWTestsBetweenBuilders('x86-zgb-paladin', 'x86-alex-paladin')
ShardHWTestsBetweenBuilders('wolf-paladin', 'peppy-paladin')
ShardHWTestsBetweenBuilders('daisy_skate-paladin', 'peach_pit-paladin')
+ ShardHWTestsBetweenBuilders('veyron_mighty-paladin', 'veyron_speedy-paladin')
ShardHWTestsBetweenBuilders('lumpy-paladin', 'stumpy-paladin')
# Add a pre-cq config for every board.
_CreateConfigsForBoards(pre_cq, _all_boards, 'pre-cq')
- # Override 'lakitu-pre-cq' - it's in _brillo_boards, but should run vmtests.
site_config.AddConfig(
pre_cq, 'lakitu-pre-cq',
_base_configs['lakitu'],
- vm_tests=[constants.SMOKE_SUITE_TEST_TYPE],
+ vm_tests=[constants.SMOKE_SUITE_TEST_TYPE, constants.GCE_VM_TEST_TYPE],
)
_CreateConfigsForBoards(no_vmtest_pre_cq, _all_boards, 'no-vmtest-pre-cq')
@@ -1913,15 +1927,13 @@ def GetConfig():
site_config.AddConfig(
internal_incremental, 'lakitu-incremental',
_base_configs['lakitu'],
- vm_tests=[constants.SMOKE_SUITE_TEST_TYPE],
+ vm_tests=[constants.SMOKE_SUITE_TEST_TYPE, constants.GCE_VM_TEST_TYPE],
)
- site_config.AddConfigWithoutTemplate(
- 'internal-toolchain-major',
- _toolchain_major, internal, official,
- boards=['x86-alex', 'stumpy', 'daisy', 'lakitu'],
- build_tests=True,
- description=_toolchain_major['description'] + ' (internal)',
+ site_config.AddConfig(
+ internal_incremental, 'lakitu_next-incremental',
+ _base_configs['lakitu_next'],
+ vm_tests=[constants.SMOKE_SUITE_TEST_TYPE, constants.GCE_VM_TEST_TYPE],
)
site_config.AddConfigWithoutTemplate(
@@ -1945,6 +1957,7 @@ def GetConfig():
manifest=constants.OFFICIAL_MANIFEST,
manifest_version=True,
images=['base', 'recovery', 'test', 'factory_install'],
+ sign_types=['recovery'],
push_image=True,
upload_symbols=True,
binhost_bucket='gs://chromeos-dev-installer',
@@ -2159,6 +2172,13 @@ def GetConfig():
chrome_sdk=False,
)
+ site_config.AddConfig(
+ _release, 'smaug-release',
+ _base_configs['smaug'],
+ images=['base', 'recovery', 'test'],
+ sign_types=['nv_lp0_firmware'],
+ )
+
# Now generate generic release configs if we haven't created anything more
# specific above already.
def _AddReleaseConfigs():
@@ -2303,18 +2323,26 @@ def GetConfig():
site_config.AddConfig(
_release, 'lakitu-release',
_base_configs['lakitu'],
- vm_tests=[constants.SMOKE_SUITE_TEST_TYPE],
+ vm_tests=[constants.SMOKE_SUITE_TEST_TYPE, constants.GCE_VM_TEST_TYPE],
+ sign_types=['base'],
important=True,
)
site_config.AddConfig(
_release, 'lakitu_mobbuild-release',
_base_configs['lakitu_mobbuild'],
- vm_tests=[constants.SMOKE_SUITE_TEST_TYPE],
+ vm_tests=[constants.SMOKE_SUITE_TEST_TYPE, constants.GCE_VM_TEST_TYPE],
signer_tests=False,
important=True,
)
+ site_config.AddConfig(
+ _release, 'lakitu_next-release',
+ _base_configs['lakitu_next'],
+ vm_tests=[constants.SMOKE_SUITE_TEST_TYPE, constants.GCE_VM_TEST_TYPE],
+ signer_tests=False,
+ )
+
_wificell_pre_cq = site_config.AddTemplate(
'wificell-pre-cq',
pre_cq,
@@ -2480,7 +2508,6 @@ def GetConfig():
_AddGroupConfig(
'rambi-e', 'orco', (
'heli',
- 'wizpig',
),
)
@@ -2543,16 +2570,16 @@ def GetConfig():
_AddGroupConfig(
'veyron-c', 'veyron_brain', (
'veyron_danger',
- 'veyron_thea',
- 'veyron_shark',
+ 'veyron_mickey',
),
- important=False,
)
_AddGroupConfig(
- 'veyron-d', 'veyron_mickey', (
+ 'veyron-d', 'veyron_shark', (
'veyron_romy',
+ 'veyron_minnie-cheets',
),
+ important=False,
)
# jecht-based boards
@@ -2574,6 +2601,15 @@ def GetConfig():
important=False,
)
+ _AddGroupConfig(
+ 'strago-b', 'reks', (
+ 'cyan-cheets',
+ 'wizpig',
+ 'terra',
+ ),
+ important=False,
+ )
+
# oak-based boards
_AddGroupConfig(
'oak', 'oak', (
@@ -2583,6 +2619,7 @@ def GetConfig():
# glados-based boards
_AddGroupConfig(
'glados', 'glados', (
+ 'chell',
),
)
@@ -2598,7 +2635,9 @@ def GetConfig():
# kunimitsu-based boards
_AddGroupConfig(
'kunimitsu', 'kunimitsu', (
+ 'lars',
),
+ important=False,
)
# Factory and Firmware releases much inherit from these classes.
@@ -2617,6 +2656,7 @@ def GetConfig():
description='Factory Builds',
paygen=False,
afdo_use=False,
+ sign_types=['factory'],
)
_firmware = config_lib.BuildConfig(
@@ -2638,6 +2678,7 @@ def GetConfig():
trybot_list=False,
paygen=False,
image_test=False,
+ sign_types=['firmware'],
)
_firmware_release = site_config.AddTemplate(
@@ -2670,6 +2711,7 @@ def GetConfig():
'beltino',
'butterfly',
'candy',
+ 'chell',
'clapper',
'cyan',
'daisy',
@@ -2682,6 +2724,7 @@ def GetConfig():
'gnawty',
'jecht',
'kip',
+ 'lars',
'leon',
'link',
'lumpy',
@@ -2696,6 +2739,7 @@ def GetConfig():
'peppy',
'quawks',
'rambi',
+ 'reks',
'rikku',
'samus',
'slippy',
@@ -2707,6 +2751,7 @@ def GetConfig():
'stumpy',
'sumo',
'swanky',
+ 'terra',
'winky',
'wolf',
'x86-mario',
diff --git a/cbuildbot/commands.py b/cbuildbot/commands.py
index 84e6d9622..520294490 100644
--- a/cbuildbot/commands.py
+++ b/cbuildbot/commands.py
@@ -62,6 +62,7 @@ _DEFAULT_HWTEST_TIMEOUT_MINS = 1440
_SWARMING_EXPIRATION = 20 * 60
_RUN_SUITE_PATH = '/usr/local/autotest/site_utils/run_suite.py'
_ABORT_SUITE_PATH = '/usr/local/autotest/site_utils/abort_suite.py'
+_MAX_HWTEST_CMD_RETRY = 10
# =========================== Command Helpers =================================
@@ -176,14 +177,16 @@ def BuildRootGitCleanup(buildroot):
try:
if os.path.isdir(cwd):
git.CleanAndDetachHead(cwd)
- git.GarbageCollection(cwd)
+
+ if os.path.isdir(repo_git_store):
+ git.GarbageCollection(repo_git_store)
except cros_build_lib.RunCommandError as e:
result = e.result
logging.PrintBuildbotStepWarnings()
logging.warning('\n%s', result.error)
# If there's no repository corruption, just delete the index.
- corrupted = git.IsGitRepositoryCorrupted(cwd)
+ corrupted = git.IsGitRepositoryCorrupted(repo_git_store)
lock.write_lock()
logging.warning('Deleting %s because %s failed', cwd, result.cmd)
osutils.RmDir(cwd, ignore_missing=True)
@@ -904,11 +907,13 @@ def RunHWTestSuite(build, suite, board, pool=None, num=None, file_bugs=None,
try:
cmd = [_RUN_SUITE_PATH]
cmd += _GetRunSuiteArgs(build, suite, board, pool, num, file_bugs,
- wait_for_results, priority, timeout_mins, retry,
- max_retries, minimum_duts, suite_min_duts,
- offload_failures_only, subsystems)
+ priority, timeout_mins, retry, max_retries,
+ minimum_duts, suite_min_duts, offload_failures_only,
+ subsystems)
swarming_args = _CreateSwarmingArgs(build, suite, timeout_mins)
- HWTestCreateAndWait(cmd, swarming_args, debug)
+ job_id = _HWTestCreate(cmd, swarming_args, debug)
+ if wait_for_results and job_id:
+ _HWTestWait(cmd, job_id, swarming_args)
except cros_build_lib.RunCommandError as e:
result = e.result
if not result.task_summary_json:
@@ -960,8 +965,7 @@ def RunHWTestSuite(build, suite, board, pool=None, num=None, file_bugs=None,
# pylint: disable=docstring-missing-args
def _GetRunSuiteArgs(build, suite, board, pool=None, num=None,
- file_bugs=None, wait_for_results=None,
- priority=None, timeout_mins=None,
+ file_bugs=None, priority=None, timeout_mins=None,
retry=None, max_retries=None, minimum_duts=0,
suite_min_duts=0, offload_failures_only=None,
subsystems=None):
@@ -990,9 +994,6 @@ def _GetRunSuiteArgs(build, suite, board, pool=None, num=None,
if file_bugs is not None:
args += ['--file_bugs', str(file_bugs)]
- if wait_for_results is not None:
- args += ['--no_wait', str(not wait_for_results)]
-
if priority is not None:
args += ['--priority', priority]
@@ -1064,21 +1065,23 @@ def _CreateSwarmingArgs(build, suite, timeout_mins=None):
return swarming_args
-def HWTestCreateAndWait(cmd, swarming_args, debug=False):
- """Start and wait on HWTest suite in the lab.
+def _HWTestCreate(cmd, swarming_args, debug=False):
+ """Start a suite in the HWTest lab, and return its id.
- This method first run a command to create the suite.
- And then run a second command to wait for the suite result.
- Since we are using swarming client, which contiuously send
- request to swarming server to poll task result, there is
- no need to retry on any network related failures.
+ This method runs a command to create the suite. Since we are using
+ swarming client, which contiuously send request to swarming server
+ to poll task result, there is no need to retry on any network
+ related failures.
Args:
cmd: Proxied run_suite command.
debug: If True, log command rather than running it.
swarming_args: A dictionary of args to passed to RunSwarmingCommand.
+
+ Returns:
+ Job id of created suite. Returned id will be None if no job id was created.
"""
- # Start the suite
+ # Start the suite.
start_cmd = list(cmd) + ['-c']
if debug:
@@ -1096,15 +1099,27 @@ def HWTestCreateAndWait(cmd, swarming_args, debug=False):
m = re.search(r'Created suite job:.*object_id=(?P<job_id>\d*)',
result.output)
if m:
- job_id = m.group('job_id')
- # Wait on the suite
- wait_cmd = list(cmd) + ['-m', str(job_id)]
- result = swarming_lib.RunSwarmingCommand(
- wait_cmd, capture_output=True, combine_stdout_stderr=True,
- **swarming_args)
- for output in result.task_summary_json['shards'][0]['outputs']:
- sys.stdout.write(output)
- sys.stdout.flush()
+ return m.group('job_id')
+ return None
+
+def _HWTestWait(cmd, job_id, swarming_args):
+ """Wait for HWTest suite to complete.
+
+ Args:
+ cmd: Proxied run_suite command.
+ job_id: The job id of the suite that was created.
+ swarming_args: A dictionary of args to passed to RunSwarmingCommand.
+ """
+ # Wait on the suite
+ wait_cmd = list(cmd) + ['-m', str(job_id)]
+ result = swarming_lib.RunSwarmingCommandWithRetries(
+ max_retry=_MAX_HWTEST_CMD_RETRY,
+ error_check=swarming_lib.SwarmingRetriableErrorCheck,
+ cmd=wait_cmd, capture_output=True, combine_stdout_stderr=True,
+ **swarming_args)
+ for output in result.task_summary_json['shards'][0]['outputs']:
+ sys.stdout.write(output)
+ sys.stdout.flush()
def AbortHWTests(config_type_or_name, version, debug, suite=''):
@@ -1562,21 +1577,33 @@ def UploadArchivedFile(archive_dir, upload_urls, filename, debug,
_UploadPathToGS(uploaded_file_path, upload_urls, debug, timeout)
-def UploadSymbols(buildroot, board, official, cnt, failed_list):
+def UploadSymbols(buildroot, board=None, official=False, cnt=None,
+ failed_list=None, breakpad_root=None, product_name=None,
+ error_code_ok=True):
"""Upload debug symbols for this build."""
- cmd = ['upload_symbols', '--yes', '--board', board,
- '--root', os.path.join(buildroot, constants.DEFAULT_CHROOT_DIR)]
- if failed_list is not None:
- cmd += ['--failed-list', str(failed_list)]
+ cmd = ['upload_symbols', '--yes']
+
+ if board is not None:
+ # Board requires both root and board to be set to be useful.
+ cmd += [
+ '--root', os.path.join(buildroot, constants.DEFAULT_CHROOT_DIR),
+ '--board', board]
if official:
cmd.append('--official_build')
if cnt is not None:
cmd += ['--upload-limit', str(cnt)]
+ if failed_list is not None:
+ cmd += ['--failed-list', str(failed_list)]
+ if breakpad_root is not None:
+ cmd += ['--breakpad_root', breakpad_root]
+ if product_name is not None:
+ cmd += ['--product_name', product_name]
# We don't want to import upload_symbols directly because it uses the
# swarming module which itself imports a _lot_ of stuff. It has also
# been known to hang. We want to keep cbuildbot isolated & robust.
- ret = RunBuildScript(buildroot, cmd, chromite_cmd=True, error_code_ok=True)
+ ret = RunBuildScript(buildroot, cmd, chromite_cmd=True,
+ error_code_ok=error_code_ok)
if ret.returncode:
# TODO(davidjames): Convert this to a fatal error.
# See http://crbug.com/212437
@@ -1959,7 +1986,15 @@ def BuildStandaloneArchive(archive_dir, image_dir, artifact_info):
A KeyError is a required field is missing from artifact_info.
"""
if 'archive' not in artifact_info:
- # Nothing to do, just return the list as-is.
+ # Copy the file in 'paths' as is to the archive directory.
+ if len(artifact_info['paths']) > 1:
+ raise ValueError('default archive type does not support multiple inputs')
+ src_image = os.path.join(image_dir, artifact_info['paths'][0])
+ tgt_image = os.path.join(archive_dir, artifact_info['paths'][0])
+ if not os.path.exists(tgt_image):
+ # The image may have already been copied into place. If so, overwriting it
+ # can affect parallel processes.
+ shutil.copy(src_image, tgt_image)
return artifact_info['paths']
inputs = artifact_info['paths']
diff --git a/cbuildbot/commands_unittest.py b/cbuildbot/commands_unittest.py
index da89b7bb8..2aeb7a8e9 100644
--- a/cbuildbot/commands_unittest.py
+++ b/cbuildbot/commands_unittest.py
@@ -211,6 +211,11 @@ Autotest instance: cautotest
02-23-2015 [06:26:53] Created suite job: http://cautotest.corp.google.com/afe/#tab_id=view_job&object_id=26960110
@@@STEP_LINK@Suite created@http://cautotest.corp.google.com/afe/#tab_id=view_job&object_id=26960110@@@
'''
+
+ WAIT_RETRY_OUTPUT = '''
+ERROR: Encountered swarming internal error
+'''
+
WAIT_OUTPUT = '''
The suite job has another 3:09:50.012887 till timeout.
The suite job has another 2:39:39.789250 till timeout.
@@ -228,7 +233,7 @@ The suite job has another 2:39:39.789250 till timeout.
self._pool = 'test-pool'
self._num = 42
self._file_bugs = True
- self._wait_for_results = False
+ self._wait_for_results = True
self._priority = 'test-priority'
self._timeout_mins = 23
self._retry = False
@@ -238,6 +243,11 @@ The suite job has another 2:39:39.789250 till timeout.
self.create_cmd = None
self.wait_cmd = None
self.temp_json_path = os.path.join(self.tempdir, 'temp_summary.json')
+ # Bot died
+ self.retriable_swarming_code = 80
+ self.internal_failure_exit_code = 1
+ # A random code that's not retriable.
+ self.swarming_code = 10
topology.FetchTopologyFromCIDB(None)
def RunHWTestSuite(self, *args, **kwargs):
@@ -250,7 +260,8 @@ The suite job has another 2:39:39.789250 till timeout.
finally:
print(logs.messages)
- def SetCmdResults(self, create_return_code=0, wait_return_code=0, args=(),
+ def SetCmdResults(self, create_return_code=0, wait_return_code=0,
+ wait_retry=False, args=(),
swarming_timeout_secs=SWARMING_TIMEOUT_DEFAULT,
swarming_io_timeout_secs=SWARMING_TIMEOUT_DEFAULT,
swarming_hard_timeout_secs=SWARMING_TIMEOUT_DEFAULT,
@@ -260,6 +271,7 @@ The suite job has another 2:39:39.789250 till timeout.
Args:
create_return_code: Return code from create command.
wait_return_code: Return code from wait command.
+ wait_retry: Boolean, if wait command should be retried.
args: Additional args to pass to create and wait commands.
swarming_timeout_secs: swarming client timeout.
swarming_io_timeout_secs: swarming client io timeout.
@@ -292,12 +304,21 @@ The suite job has another 2:39:39.789250 till timeout.
self.create_cmd,
side_effect=lambda *args, **kwargs: create_results.next(),
)
+ wait_results_list = []
+ if wait_retry:
+ r = self.rc.CmdResult(
+ returncode=self.internal_failure_exit_code,
+ output=self.WAIT_RETRY_OUTPUT,
+ error='')
+ wait_results_list.append(r)
+
+ wait_results_list.append(
+ self.rc.CmdResult(
+ returncode=wait_return_code, output=self.WAIT_OUTPUT,
+ error='')
+ )
+ wait_results = iter(wait_results_list)
- wait_results = iter([
- self.rc.CmdResult(returncode=wait_return_code,
- output=self.WAIT_OUTPUT,
- error=''),
- ])
self.rc.AddCmdResult(
self.wait_cmd,
side_effect=lambda *args, **kwargs: wait_results.next(),
@@ -309,8 +330,11 @@ The suite job has another 2:39:39.789250 till timeout.
Args:
task_outputs: A list of tuple, the first element is the value of 'outputs'
field in the json dictionary, the second is a boolean
- indicating whether there is an internal failure.
- ('some output', True)
+ indicating whether there is an internal failure,
+ the third is a state code for the internal failure.
+ e.g.
+ ('some output', True, 80)
+ ('some output', False, None)
"""
orig_func = commands._CreateSwarmingArgs
@@ -327,6 +351,7 @@ The suite job has another 2:39:39.789250 till timeout.
j = {'shards':[{'name': 'fake_name', 'bot_id': 'chromeos-server990',
'created_ts': '2015-06-12 12:00:00',
'internal_failure': s[1],
+ 'state': s[2],
'outputs': [s[0]]}]}
return_values.append(j)
return_values_iter = iter(return_values)
@@ -339,16 +364,15 @@ The suite job has another 2:39:39.789250 till timeout.
def testRunHWTestSuiteMinimal(self):
"""Test RunHWTestSuite without optional arguments."""
self.SetCmdResults()
- self.PatchJson([(self.JOB_ID_OUTPUT, False), (self.WAIT_OUTPUT, False)])
+ # When run without optional arguments, wait_for_result default to None.
+ # the wait cmd will not run.
+ self.PatchJson([(self.JOB_ID_OUTPUT, False, None)])
with self.OutputCapturer() as output:
self.RunHWTestSuite()
self.assertCommandCalled(self.create_cmd, capture_output=True,
combine_stdout_stderr=True)
- self.assertCommandCalled(self.wait_cmd, capture_output=True,
- combine_stdout_stderr=True)
self.assertIn(self.JOB_ID_OUTPUT, '\n'.join(output.GetStdoutLines()))
- self.assertIn(self.WAIT_OUTPUT, '\n'.join(output.GetStdoutLines()))
def testRunHWTestSuiteMaximal(self):
"""Test RunHWTestSuite with all arguments."""
@@ -357,7 +381,7 @@ The suite job has another 2:39:39.789250 till timeout.
self.SetCmdResults(
args=[
'--pool', 'test-pool', '--num', '42',
- '--file_bugs', 'True', '--no_wait', 'True',
+ '--file_bugs', 'True',
'--priority', 'test-priority', '--timeout_mins', '23',
'--retry', 'False', '--max_retries', '3', '--minimum_duts', '2',
'--suite_min_duts', '2'
@@ -366,7 +390,8 @@ The suite job has another 2:39:39.789250 till timeout.
swarming_io_timeout_secs=swarming_timeout,
swarming_hard_timeout_secs=swarming_timeout)
- self.PatchJson([(self.JOB_ID_OUTPUT, False), (self.WAIT_OUTPUT, False)])
+ self.PatchJson([(self.JOB_ID_OUTPUT, False, None),
+ (self.WAIT_OUTPUT, False, None)])
with self.OutputCapturer() as output:
self.RunHWTestSuite(self._pool, self._num, self._file_bugs,
self._wait_for_results, self._priority,
@@ -382,35 +407,35 @@ The suite job has another 2:39:39.789250 till timeout.
def testRunHWTestSuiteFailure(self):
"""Test RunHWTestSuite when ERROR is returned."""
- self.PatchJson([(self.JOB_ID_OUTPUT, False)])
+ self.PatchJson([(self.JOB_ID_OUTPUT, False, None)])
self.rc.SetDefaultCmdResult(returncode=1, output=self.JOB_ID_OUTPUT)
with self.OutputCapturer():
self.assertRaises(failures_lib.TestFailure, self.RunHWTestSuite)
def testRunHWTestSuiteTimedOut(self):
"""Test RunHWTestSuite when SUITE_TIMEOUT is returned."""
- self.PatchJson([(self.JOB_ID_OUTPUT, False)])
+ self.PatchJson([(self.JOB_ID_OUTPUT, False, None)])
self.rc.SetDefaultCmdResult(returncode=4, output=self.JOB_ID_OUTPUT)
with self.OutputCapturer():
self.assertRaises(failures_lib.SuiteTimedOut, self.RunHWTestSuite)
def testRunHWTestSuiteInfraFail(self):
"""Test RunHWTestSuite when INFRA_FAILURE is returned."""
- self.PatchJson([(self.JOB_ID_OUTPUT, False)])
+ self.PatchJson([(self.JOB_ID_OUTPUT, False, None)])
self.rc.SetDefaultCmdResult(returncode=3, output=self.JOB_ID_OUTPUT)
with self.OutputCapturer():
self.assertRaises(failures_lib.TestLabFailure, self.RunHWTestSuite)
def testRunHWTestBoardNotAvailable(self):
"""Test RunHWTestSuite when BOARD_NOT_AVAILABLE is returned."""
- self.PatchJson([(self.JOB_ID_OUTPUT, False)])
+ self.PatchJson([(self.JOB_ID_OUTPUT, False, None)])
self.rc.SetDefaultCmdResult(returncode=5, output=self.JOB_ID_OUTPUT)
with self.OutputCapturer():
self.assertRaises(failures_lib.BoardNotAvailable, self.RunHWTestSuite)
def testRunHWTestTestWarning(self):
"""Test RunHWTestSuite when WARNING is returned."""
- self.PatchJson([(self.JOB_ID_OUTPUT, False)])
+ self.PatchJson([(self.JOB_ID_OUTPUT, False, None)])
self.rc.SetDefaultCmdResult(returncode=2, output=self.JOB_ID_OUTPUT)
with self.OutputCapturer():
self.assertRaises(failures_lib.TestWarning, self.RunHWTestSuite)
@@ -425,15 +450,35 @@ The suite job has another 2:39:39.789250 till timeout.
self.assertIn(unknown_failure, '\n'.join(output.GetStdoutLines()))
def testRunHWTestTestSwarmingClientInternalFailure(self):
- """Test RunHWTestSuite when no summary file is generated."""
+ """Test RunHWTestSuite when swarming encounters internal failure."""
unknown_failure = 'Unknown failure'
- self.PatchJson(task_outputs=[(self.JOB_ID_OUTPUT, True)])
+ self.PatchJson(
+ task_outputs=[(self.JOB_ID_OUTPUT, True, self.swarming_code)])
self.rc.SetDefaultCmdResult(returncode=1, output=unknown_failure)
with self.OutputCapturer() as output:
self.assertRaises(failures_lib.SwarmingProxyFailure, self.RunHWTestSuite)
self.assertIn(unknown_failure, '\n'.join(output.GetStdoutLines()))
self.assertIn('summary json content', '\n'.join(output.GetStdoutLines()))
+ def testRunHWTestTestSwarmingClientWithRetires(self):
+ """Test RunHWTestSuite with retries."""
+ self.SetCmdResults(wait_retry=True)
+ self.PatchJson(
+ [(self.JOB_ID_OUTPUT, False, None),
+ (self.WAIT_RETRY_OUTPUT, True, self.retriable_swarming_code),
+ (self.WAIT_OUTPUT, False, None),
+ ])
+ with self.OutputCapturer() as output:
+ self.RunHWTestSuite(wait_for_results=self._wait_for_results)
+ self.assertCommandCalled(self.create_cmd, capture_output=True,
+ combine_stdout_stderr=True)
+ self.assertCommandCalled(self.wait_cmd, capture_output=True,
+ combine_stdout_stderr=True)
+ self.assertIn(self.WAIT_RETRY_OUTPUT.strip(),
+ '\n'.join(output.GetStdoutLines()))
+ self.assertIn(self.WAIT_OUTPUT, '\n'.join(output.GetStdoutLines()))
+ self.assertIn(self.JOB_ID_OUTPUT, '\n'.join(output.GetStdoutLines()))
+
def testGetRunSuiteArgsWithSubsystems(self):
"""Test _GetRunSuiteArgs when subsystems is specified."""
result_1 = commands._GetRunSuiteArgs(build=self._build,
@@ -570,27 +615,44 @@ f6b0b80d5f2d9a2fb41ebb6e2cee7ad8 *./updater4.sh
commands.GenerateBreakpadSymbols(self.tempdir, self._board, False)
self.assertCommandContains(['--board=%s' % self._board])
- def testUploadSymbols(self, official=False, cnt=None):
- """Test UploadSymbols Command."""
- commands.UploadSymbols(self.tempdir, self._board, official, cnt, None)
- self.assertCommandContains(['--board', self._board])
- self.assertCommandContains(['--official_build'], expected=official)
- self.assertCommandContains(['--upload-limit'], expected=cnt is not None)
- self.assertCommandContains(['--failed-list'], expected=False)
+ def testUploadSymbolsMinimal(self):
+ """Test uploading symbols for official builds"""
+ commands.UploadSymbols('/buildroot', 'MyBoard')
+ self.assertCommandContains(
+ ['/buildroot/chromite/bin/upload_symbols', '--yes',
+ '--root', '/buildroot/chroot',
+ '--board', 'MyBoard'])
- def testOfficialUploadSymbols(self):
+ def testUploadSymbolsMinimalNoneChromeOS(self):
"""Test uploading symbols for official builds"""
- self.testUploadSymbols(official=True)
+ commands.UploadSymbols(
+ '/buildroot', breakpad_root='/breakpad', product_name='CoolProduct')
+ self.assertCommandContains(
+ ['/buildroot/chromite/bin/upload_symbols', '--yes',
+ '--breakpad_root', '/breakpad',
+ '--product_name', 'CoolProduct'])
- def testLimitUploadSymbols(self):
- """Test uploading a limited number of symbols"""
- self.testUploadSymbols(cnt=10)
+ def testUploadSymbolsMaximal(self):
+ """Test uploading symbols for official builds"""
+ commands.UploadSymbols(
+ '/buildroot', 'MyBoard', official=True, cnt=55,
+ failed_list='/failed_list.txt', breakpad_root='/breakpad',
+ product_name='CoolProduct')
+ self.assertCommandContains(
+ ['/buildroot/chromite/bin/upload_symbols', '--yes',
+ '--root', '/buildroot/chroot',
+ '--board', 'MyBoard',
+ '--official_build',
+ '--upload-limit', '55',
+ '--failed-list', '/failed_list.txt',
+ '--breakpad_root', '/breakpad',
+ '--product_name', 'CoolProduct'])
def testFailedUploadSymbols(self):
"""Test when uploading fails"""
self.rc.SetDefaultCmdResult(returncode=1, error='i am sad')
# This should not throw an exception.
- commands.UploadSymbols(self.tempdir, self._board, None, None, None)
+ commands.UploadSymbols(self.tempdir)
def testPushImages(self):
"""Test PushImages Command."""
diff --git a/cbuildbot/config_dump.json b/cbuildbot/config_dump.json
index 83d8cb75c..fac792771 100644
--- a/cbuildbot/config_dump.json
+++ b/cbuildbot/config_dump.json
@@ -84,6 +84,7 @@
"sanity_check_slaves": null,
"separate_debug_symbols": true,
"shared_user_password": null,
+ "sign_types": null,
"signer_tests": false,
"sync_chrome": null,
"trybot_list": false,
@@ -192,8 +193,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"profile": "asan",
"vm_tests": [
@@ -214,8 +215,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"important": false,
"internal": true,
@@ -225,7 +226,6 @@
"unittests": false,
"uprev": false,
"use_chrome_lkgm": true,
- "use_lkgm": false,
"useflags": [
"-cros-debug",
"chrome_internal"
@@ -244,8 +244,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"important": true,
"internal": true,
@@ -273,8 +273,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"important": false,
"internal": true,
@@ -282,7 +282,6 @@
"manifest_version": false,
"overlays": "both",
"uprev": false,
- "use_lkgm": true,
"useflags": [
"chrome_internal"
],
@@ -301,8 +300,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"important": true,
"manifest_version": true,
@@ -323,14 +322,13 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"important": false,
"manifest_version": false,
"overlays": "public",
"uprev": false,
- "use_lkgm": true,
"vm_tests": [
"smoke_suite",
"pfq_suite"
@@ -357,8 +355,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"image_test": true,
"images": [
@@ -396,8 +394,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"hwqual": true,
"image_test": false,
@@ -413,6 +411,9 @@
],
"paygen": false,
"push_image": true,
+ "sign_types": [
+ "firmware"
+ ],
"signer_tests": false,
"sync_chrome": false,
"trybot_list": false,
@@ -442,8 +443,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"image_test": false,
"images": [],
@@ -455,6 +456,9 @@
"chromeos-base/autotest-all"
],
"paygen": false,
+ "sign_types": [
+ "firmware"
+ ],
"signer_tests": false,
"sync_chrome": false,
"trybot_list": false,
@@ -488,8 +492,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"hwqual": true,
"image_test": true,
@@ -506,6 +510,9 @@
"overlays": "both",
"paygen": false,
"push_image": true,
+ "sign_types": [
+ "factory"
+ ],
"signer_tests": true,
"trybot_list": true,
"upload_hw_test_artifacts": false,
@@ -541,8 +548,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"hwqual": true,
"image_test": false,
@@ -558,6 +565,9 @@
],
"paygen": false,
"push_image": true,
+ "sign_types": [
+ "firmware"
+ ],
"signer_tests": false,
"sync_chrome": false,
"trybot_list": false,
@@ -582,8 +592,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"image_test": true,
"images": [
@@ -604,8 +614,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"overlays": "public",
"uprev": false
@@ -616,8 +626,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"profile": "llvm"
},
@@ -644,8 +654,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"hwqual": true,
"image_test": true,
@@ -664,6 +674,9 @@
"paygen_skip_delta_payloads": true,
"paygen_skip_testing": true,
"push_image": true,
+ "sign_types": [
+ "recovery"
+ ],
"signer_tests": false,
"trybot_list": true,
"upload_symbols": true,
@@ -697,8 +710,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"image_test": true,
"images": [
@@ -726,8 +739,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"image_test": true,
"images": [
@@ -785,8 +798,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"image_test": true,
"images": [
@@ -852,14 +865,14 @@
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": true,\n \"max_retries\": 10,\n \"minimum_duts\": 1,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"bvt\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"sanity\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": true,\n \"max_retries\": 10,\n \"minimum_duts\": 4,\n \"num\": 6,\n \"offload_failures_only\": false,\n \"pool\": \"bvt\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 6,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": true,\n \"max_retries\": 10,\n \"minimum_duts\": 4,\n \"num\": 2,\n \"offload_failures_only\": false,\n \"pool\": \"bvt\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 6,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": true,\n \"max_retries\": null,\n \"minimum_duts\": 4,\n \"num\": 2,\n \"offload_failures_only\": false,\n \"pool\": \"bvt\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": true,\n \"max_retries\": null,\n \"minimum_duts\": 4,\n \"num\": 2,\n \"offload_failures_only\": false,\n \"pool\": \"bvt\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": true,\n \"max_retries\": 10,\n \"minimum_duts\": 4,\n \"num\": 2,\n \"offload_failures_only\": false,\n \"pool\": \"bvt\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": true,\n \"max_retries\": 10,\n \"minimum_duts\": 4,\n \"num\": 2,\n \"offload_failures_only\": false,\n \"pool\": \"bvt\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"hwqual": true,
"image_test": true,
@@ -876,6 +889,9 @@
"overlays": "both",
"paygen": true,
"push_image": true,
+ "sign_types": [
+ "recovery"
+ ],
"signer_tests": true,
"trybot_list": true,
"upload_symbols": true,
@@ -928,6 +944,9 @@
"overlays": "both",
"paygen": false,
"push_image": false,
+ "sign_types": [
+ "recovery"
+ ],
"signer_tests": true,
"trybot_list": false,
"upload_symbols": true,
@@ -960,15 +979,15 @@
"hw_tests": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 4,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": false,\n \"suite\": \"perf_v2\",\n \"suite_min_duts\": 0,\n \"timeout\": 7200,\n \"warn_only\": false\n}"
],
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"hwqual": true,
"image_test": true,
@@ -985,6 +1004,9 @@
"overlays": "both",
"paygen": false,
"push_image": false,
+ "sign_types": [
+ "recovery"
+ ],
"signer_tests": true,
"trybot_list": false,
"upload_symbols": true,
@@ -1006,8 +1028,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"overlays": "public",
"uprev": false,
@@ -1021,8 +1043,8 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"internal": true,
"manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
@@ -1041,15 +1063,14 @@
"hw_tests_override": [
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 3,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
"{\n \"async\": false,\n \"blocking\": true,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"au\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
- "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": null,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": false,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}",
+ "{\n \"async\": true,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 1,\n \"offload_failures_only\": false,\n \"pool\": \"suites\",\n \"priority\": \"PostBuild\",\n \"retry\": true,\n \"suite\": \"bvt-perbuild\",\n \"suite_min_duts\": 1,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
],
"important": false,
"manifest_version": false,
"overlays": "public",
"profile": "asan",
"uprev": false,
- "use_lkgm": true,
"vm_tests": [
"smoke_suite"
],
@@ -1428,7 +1449,6 @@
],
"chrome_sdk": false,
"dev_installer_prebuilts": false,
- "hw_tests": [],
"sync_chrome": false,
"vm_tests": []
},
@@ -2853,7 +2873,6 @@
],
"chrome_sdk": false,
"dev_installer_prebuilts": false,
- "hw_tests": [],
"image_test": false,
"images": [
"base",
@@ -2923,7 +2942,6 @@
"chrome_sdk": false,
"dev_installer_prebuilts": false,
"grouped": true,
- "hw_tests": [],
"image_test": false,
"images": [
"base",
@@ -2947,7 +2965,6 @@
"chrome_sdk_build_chrome": false,
"dev_installer_prebuilts": false,
"grouped": true,
- "hw_tests": [],
"image_test": false,
"images": [
"base",
@@ -2965,7 +2982,6 @@
],
"chrome_sdk": false,
"dev_installer_prebuilts": false,
- "hw_tests": [],
"image_test": false,
"images": [
"base",
@@ -3102,7 +3118,6 @@
],
"chrome_sdk": false,
"dev_installer_prebuilts": false,
- "hw_tests": [],
"image_test": false,
"images": [
"base",
@@ -4384,6 +4399,171 @@
"-chrome_internal"
]
},
+ "chell-chrome-pfq": {
+ "_template": "chrome-pfq",
+ "boards": [
+ "chell"
+ ],
+ "important": false,
+ "manifest": "official.xml"
+ },
+ "chell-chromium-pfq": {
+ "_template": "chromium-pfq",
+ "boards": [
+ "chell"
+ ],
+ "description": "Preflight Chromium Uprev & Build (internal)",
+ "doc": "http://www.chromium.org/chromium-os/build/builder-overview#TOC-Chrome-PFQ",
+ "important": false,
+ "internal": true,
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
+ "chell-compile-only-pre-cq": {
+ "_template": "compile-only-pre-cq",
+ "boards": [
+ "chell"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "chell-firmware": {
+ "_template": "firmware",
+ "boards": [
+ "chell"
+ ],
+ "manifest": "official.xml",
+ "useflags": [
+ "chrome_internal",
+ "chromeless_tty"
+ ]
+ },
+ "chell-full": {
+ "_template": "full",
+ "boards": [
+ "chell"
+ ],
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
+ "chell-no-vmtest-pre-cq": {
+ "_template": "no-vmtest-pre-cq",
+ "boards": [
+ "chell"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "chell-paladin": {
+ "_template": "paladin",
+ "boards": [
+ "chell"
+ ],
+ "description": "Commit Queue (internal)",
+ "important": false,
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "private",
+ "useflags": [
+ "chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "chell-payloads": {
+ "_template": "payloads",
+ "boards": [
+ "chell"
+ ]
+ },
+ "chell-pre-cq": {
+ "_template": "pre-cq",
+ "boards": [
+ "chell"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "chell-release": {
+ "_template": "release",
+ "boards": [
+ "chell"
+ ]
+ },
+ "chell-release-afdo": {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "chell"
+ ],
+ "child_configs": [
+ {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "chell"
+ ],
+ "grouped": true,
+ "name": "chell-release-afdo-generate"
+ },
+ {
+ "_template": "release-afdo-use",
+ "boards": [
+ "chell"
+ ],
+ "grouped": true,
+ "name": "chell-release-afdo-use"
+ }
+ ]
+ },
+ "chell-release-afdo-generate": {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "chell"
+ ]
+ },
+ "chell-release-afdo-use": {
+ "_template": "release-afdo-use",
+ "boards": [
+ "chell"
+ ]
+ },
+ "chell-tot-chrome-pfq-informational": {
+ "_template": "chrome-pfq-informational",
+ "boards": [
+ "chell"
+ ],
+ "manifest": "official.xml"
+ },
+ "chell-tot-chromium-pfq-informational": {
+ "_template": "chromium-pfq-informational",
+ "boards": [
+ "chell"
+ ],
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
"chromiumos-sdk": {
"active_waterfall": "chromiumos",
"archive_build_debug": true,
@@ -4908,6 +5088,173 @@
"-chrome_internal"
]
},
+ "cyan-cheets-chrome-pfq": {
+ "_template": "chrome-pfq",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "important": false,
+ "manifest": "official.xml",
+ "vm_tests": []
+ },
+ "cyan-cheets-chromium-pfq": {
+ "_template": "chromium-pfq",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "description": "Preflight Chromium Uprev & Build (internal)",
+ "doc": "http://www.chromium.org/chromium-os/build/builder-overview#TOC-Chrome-PFQ",
+ "important": false,
+ "internal": true,
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "cyan-cheets-compile-only-pre-cq": {
+ "_template": "compile-only-pre-cq",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "cyan-cheets-full": {
+ "_template": "full",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "cyan-cheets-no-vmtest-pre-cq": {
+ "_template": "no-vmtest-pre-cq",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "cyan-cheets-paladin": {
+ "_template": "paladin",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "description": "Commit Queue (internal)",
+ "important": false,
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "private",
+ "useflags": [
+ "chrome_internal"
+ ],
+ "vm_tests": [],
+ "vm_tests_override": null
+ },
+ "cyan-cheets-payloads": {
+ "_template": "payloads",
+ "boards": [
+ "cyan-cheets"
+ ]
+ },
+ "cyan-cheets-pre-cq": {
+ "_template": "pre-cq",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "cyan-cheets-release": {
+ "_template": "release",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "vm_tests": []
+ },
+ "cyan-cheets-release-afdo": {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "child_configs": [
+ {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "grouped": true,
+ "name": "cyan-cheets-release-afdo-generate",
+ "vm_tests": []
+ },
+ {
+ "_template": "release-afdo-use",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "grouped": true,
+ "name": "cyan-cheets-release-afdo-use",
+ "vm_tests": []
+ }
+ ],
+ "vm_tests": []
+ },
+ "cyan-cheets-release-afdo-generate": {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "vm_tests": []
+ },
+ "cyan-cheets-release-afdo-use": {
+ "_template": "release-afdo-use",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "vm_tests": []
+ },
+ "cyan-cheets-tot-chrome-pfq-informational": {
+ "_template": "chrome-pfq-informational",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "manifest": "official.xml",
+ "vm_tests": []
+ },
+ "cyan-cheets-tot-chromium-pfq-informational": {
+ "_template": "chromium-pfq-informational",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
+ },
"cyan-chrome-pfq": {
"_template": "chrome-pfq",
"boards": [
@@ -6981,7 +7328,6 @@
],
"chrome_sdk": false,
"dev_installer_prebuilts": false,
- "hw_tests": [],
"important": true,
"paygen": false,
"signer_tests": false,
@@ -7087,9 +7433,25 @@
"useflags": [
"-chrome_internal"
]
+ },
+ {
+ "_template": "full",
+ "boards": [
+ "chell"
+ ],
+ "build_packages_in_background": true,
+ "chrome_sdk_build_chrome": false,
+ "grouped": true,
+ "name": "chell-full",
+ "prebuilts": "public",
+ "unittests": null,
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
}
],
- "description": "Full Builds; Group config (boards: glados)",
+ "description": "Full Builds; Group config (boards: glados, chell)",
"prebuilts": "public",
"useflags": [
"-chrome_internal"
@@ -7110,11 +7472,11 @@
},
"glados-paladin": {
"_template": "paladin",
+ "active_waterfall": "chromeos",
"boards": [
"glados"
],
"description": "Commit Queue (internal)",
- "important": false,
"internal": true,
"manifest": "official.xml",
"manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
@@ -7200,9 +7562,21 @@
],
"grouped": true,
"name": "glados-release"
+ },
+ {
+ "_template": "release",
+ "boards": [
+ "chell"
+ ],
+ "build_packages_in_background": true,
+ "chrome_sdk_build_chrome": false,
+ "grouped": true,
+ "name": "chell-release",
+ "unittests": null,
+ "vm_tests": []
}
],
- "description": "Release Builds (canary) (internal); Group config (boards: glados)",
+ "description": "Release Builds (canary) (internal); Group config (boards: glados, chell)",
"important": true
},
"glados-tot-chrome-pfq-informational": {
@@ -8112,41 +8486,6 @@
"-chrome_internal"
]
},
- "internal-toolchain-major": {
- "active_waterfall": "chromeos",
- "archive_build_debug": true,
- "boards": [
- "x86-alex",
- "stumpy",
- "daisy",
- "lakitu"
- ],
- "build_type": "chroot",
- "builder_class_name": "sdk_builders.ChrootSdkBuilder",
- "chrome_sdk": true,
- "chromeos_official": true,
- "description": "Test next major toolchain revision (internal)",
- "doc": "http://www.chromium.org/chromium-os/build/builder-overview#TOC-Continuous",
- "gcc_githash": "svn-mirror/google/main",
- "git_sync": true,
- "hw_tests_override": [],
- "image_test": true,
- "images": [
- "base",
- "recovery",
- "test",
- "factory_install"
- ],
- "internal": true,
- "latest_toolchain": true,
- "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
- "overlays": "both",
- "use_sdk": false,
- "useflags": [
- "chrome_internal"
- ],
- "usepkg_build_packages": false
- },
"internal-toolchain-minor": {
"active_waterfall": "chromeos",
"archive_build_debug": true,
@@ -8761,7 +9100,6 @@
"dev_manifest": "kayle.xml",
"factory_install_netboot": false,
"factory_toolkit": false,
- "hw_tests": [],
"hwqual": false,
"images": [
"base"
@@ -9182,9 +9520,25 @@
"useflags": [
"-chrome_internal"
]
+ },
+ {
+ "_template": "full",
+ "boards": [
+ "lars"
+ ],
+ "build_packages_in_background": true,
+ "chrome_sdk_build_chrome": false,
+ "grouped": true,
+ "name": "lars-full",
+ "prebuilts": "public",
+ "unittests": null,
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
}
],
- "description": "Full Builds; Group config (boards: kunimitsu)",
+ "description": "Full Builds; Group config (boards: kunimitsu, lars)",
"prebuilts": "public",
"useflags": [
"-chrome_internal"
@@ -9295,10 +9649,21 @@
],
"grouped": true,
"name": "kunimitsu-release"
+ },
+ {
+ "_template": "release",
+ "boards": [
+ "lars"
+ ],
+ "build_packages_in_background": true,
+ "chrome_sdk_build_chrome": false,
+ "grouped": true,
+ "name": "lars-release",
+ "unittests": null,
+ "vm_tests": []
}
],
- "description": "Release Builds (canary) (internal); Group config (boards: kunimitsu)",
- "important": true
+ "description": "Release Builds (canary) (internal); Group config (boards: kunimitsu, lars)"
},
"kunimitsu-tot-chrome-pfq-informational": {
"_template": "chrome-pfq-informational",
@@ -9401,7 +9766,8 @@
"chrome_internal"
],
"vm_tests": [
- "smoke_suite"
+ "smoke_suite",
+ "gce_vm_test"
]
},
"lakitu-no-vmtest-pre-cq": {
@@ -9465,6 +9831,10 @@
"upload_gce_images": true,
"useflags": [
"chrome_internal"
+ ],
+ "vm_tests": [
+ "smoke_suite",
+ "gce_vm_test"
]
},
"lakitu-release": {
@@ -9481,10 +9851,14 @@
"hwqual": false,
"image_test": false,
"important": true,
+ "sign_types": [
+ "base"
+ ],
"sync_chrome": false,
"upload_gce_images": true,
"vm_tests": [
- "smoke_suite"
+ "smoke_suite",
+ "gce_vm_test"
]
},
"lakitu-tot-chrome-pfq-informational": {
@@ -9660,7 +10034,8 @@
"sync_chrome": false,
"upload_gce_images": true,
"vm_tests": [
- "smoke_suite"
+ "smoke_suite",
+ "gce_vm_test"
]
},
"lakitu_mobbuild-tot-chrome-pfq-informational": {
@@ -9687,6 +10062,370 @@
],
"vm_tests": []
},
+ "lakitu_next-chrome-pfq": {
+ "_template": "chrome-pfq",
+ "boards": [
+ "lakitu_next"
+ ],
+ "chrome_sdk": false,
+ "disk_layout": "base",
+ "important": false,
+ "manifest": "official.xml",
+ "sync_chrome": false,
+ "upload_gce_images": true,
+ "vm_tests": []
+ },
+ "lakitu_next-chromium-pfq": {
+ "_template": "chromium-pfq",
+ "boards": [
+ "lakitu_next"
+ ],
+ "chrome_sdk": false,
+ "description": "Preflight Chromium Uprev & Build (internal)",
+ "disk_layout": "base",
+ "doc": "http://www.chromium.org/chromium-os/build/builder-overview#TOC-Chrome-PFQ",
+ "important": false,
+ "internal": true,
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "public",
+ "sync_chrome": false,
+ "upload_gce_images": true,
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "lakitu_next-compile-only-pre-cq": {
+ "_template": "compile-only-pre-cq",
+ "boards": [
+ "lakitu_next"
+ ],
+ "disk_layout": "base",
+ "image_test": false,
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "sync_chrome": false,
+ "upload_gce_images": true,
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "lakitu_next-full": {
+ "_template": "full",
+ "boards": [
+ "lakitu_next"
+ ],
+ "chrome_sdk": false,
+ "disk_layout": "base",
+ "image_test": false,
+ "prebuilts": "public",
+ "sync_chrome": false,
+ "upload_gce_images": true,
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "lakitu_next-incremental": {
+ "_template": "incremental",
+ "active_waterfall": "chromeos",
+ "boards": [
+ "lakitu_next"
+ ],
+ "description": "Incremental Builds (internal)",
+ "disk_layout": "base",
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "sync_chrome": false,
+ "upload_gce_images": true,
+ "useflags": [
+ "chrome_internal"
+ ],
+ "vm_tests": [
+ "smoke_suite",
+ "gce_vm_test"
+ ]
+ },
+ "lakitu_next-no-vmtest-pre-cq": {
+ "_template": "no-vmtest-pre-cq",
+ "boards": [
+ "lakitu_next"
+ ],
+ "disk_layout": "base",
+ "image_test": false,
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "sync_chrome": false,
+ "upload_gce_images": true,
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "lakitu_next-paladin": {
+ "_template": "paladin",
+ "active_waterfall": "chromeos",
+ "boards": [
+ "lakitu_next"
+ ],
+ "chrome_sdk": false,
+ "description": "Commit Queue (internal)",
+ "disk_layout": "base",
+ "image_test": false,
+ "important": false,
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "private",
+ "sync_chrome": false,
+ "upload_gce_images": true,
+ "useflags": [
+ "chrome_internal"
+ ],
+ "vm_tests": [],
+ "vm_tests_override": null
+ },
+ "lakitu_next-payloads": {
+ "_template": "payloads",
+ "boards": [
+ "lakitu_next"
+ ]
+ },
+ "lakitu_next-pre-cq": {
+ "_template": "pre-cq",
+ "boards": [
+ "lakitu_next"
+ ],
+ "disk_layout": "base",
+ "image_test": false,
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "sync_chrome": false,
+ "upload_gce_images": true,
+ "useflags": [
+ "chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "lakitu_next-release": {
+ "_template": "release",
+ "active_waterfall": "chromeos",
+ "afdo_use": false,
+ "boards": [
+ "lakitu_next"
+ ],
+ "chrome_sdk": false,
+ "dev_installer_prebuilts": false,
+ "disk_layout": "base",
+ "hw_tests": [],
+ "hwqual": false,
+ "image_test": false,
+ "signer_tests": false,
+ "sync_chrome": false,
+ "upload_gce_images": true,
+ "vm_tests": [
+ "smoke_suite",
+ "gce_vm_test"
+ ]
+ },
+ "lakitu_next-tot-chrome-pfq-informational": {
+ "_template": "chrome-pfq-informational",
+ "boards": [
+ "lakitu_next"
+ ],
+ "disk_layout": "base",
+ "manifest": "official.xml",
+ "sync_chrome": false,
+ "upload_gce_images": true,
+ "vm_tests": []
+ },
+ "lakitu_next-tot-chromium-pfq-informational": {
+ "_template": "chromium-pfq-informational",
+ "boards": [
+ "lakitu_next"
+ ],
+ "disk_layout": "base",
+ "sync_chrome": false,
+ "upload_gce_images": true,
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "lars-chrome-pfq": {
+ "_template": "chrome-pfq",
+ "boards": [
+ "lars"
+ ],
+ "important": false,
+ "manifest": "official.xml"
+ },
+ "lars-chromium-pfq": {
+ "_template": "chromium-pfq",
+ "boards": [
+ "lars"
+ ],
+ "description": "Preflight Chromium Uprev & Build (internal)",
+ "doc": "http://www.chromium.org/chromium-os/build/builder-overview#TOC-Chrome-PFQ",
+ "important": false,
+ "internal": true,
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
+ "lars-compile-only-pre-cq": {
+ "_template": "compile-only-pre-cq",
+ "boards": [
+ "lars"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "lars-firmware": {
+ "_template": "firmware",
+ "boards": [
+ "lars"
+ ],
+ "manifest": "official.xml",
+ "useflags": [
+ "chrome_internal",
+ "chromeless_tty"
+ ]
+ },
+ "lars-full": {
+ "_template": "full",
+ "boards": [
+ "lars"
+ ],
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
+ "lars-no-vmtest-pre-cq": {
+ "_template": "no-vmtest-pre-cq",
+ "boards": [
+ "lars"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "lars-paladin": {
+ "_template": "paladin",
+ "boards": [
+ "lars"
+ ],
+ "description": "Commit Queue (internal)",
+ "important": false,
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "private",
+ "useflags": [
+ "chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "lars-payloads": {
+ "_template": "payloads",
+ "boards": [
+ "lars"
+ ]
+ },
+ "lars-pre-cq": {
+ "_template": "pre-cq",
+ "boards": [
+ "lars"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "lars-release": {
+ "_template": "release",
+ "boards": [
+ "lars"
+ ]
+ },
+ "lars-release-afdo": {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "lars"
+ ],
+ "child_configs": [
+ {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "lars"
+ ],
+ "grouped": true,
+ "name": "lars-release-afdo-generate"
+ },
+ {
+ "_template": "release-afdo-use",
+ "boards": [
+ "lars"
+ ],
+ "grouped": true,
+ "name": "lars-release-afdo-use"
+ }
+ ]
+ },
+ "lars-release-afdo-generate": {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "lars"
+ ]
+ },
+ "lars-release-afdo-use": {
+ "_template": "release-afdo-use",
+ "boards": [
+ "lars"
+ ]
+ },
+ "lars-tot-chrome-pfq-informational": {
+ "_template": "chrome-pfq-informational",
+ "boards": [
+ "lars"
+ ],
+ "manifest": "official.xml"
+ },
+ "lars-tot-chromium-pfq-informational": {
+ "_template": "chromium-pfq-informational",
+ "boards": [
+ "lars"
+ ],
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
"leon-chrome-pfq": {
"_template": "chrome-pfq",
"boards": [
@@ -12870,7 +13609,6 @@
],
"chrome_sdk": false,
"dev_installer_prebuilts": false,
- "hw_tests": [],
"important": true,
"paygen": false,
"profile": "minimal",
@@ -12941,7 +13679,6 @@
],
"chrome_sdk": false,
"dev_installer_prebuilts": false,
- "hw_tests": [],
"sync_chrome": false,
"vm_tests": []
},
@@ -14595,7 +15332,6 @@
],
"chrome_sdk": false,
"dev_installer_prebuilts": false,
- "hw_tests": [],
"sync_chrome": false,
"vm_tests": []
},
@@ -15386,25 +16122,9 @@
"-chrome_internal"
],
"vm_tests": []
- },
- {
- "_template": "full",
- "boards": [
- "wizpig"
- ],
- "build_packages_in_background": true,
- "chrome_sdk_build_chrome": false,
- "grouped": true,
- "name": "wizpig-full",
- "prebuilts": "public",
- "unittests": null,
- "useflags": [
- "-chrome_internal"
- ],
- "vm_tests": []
}
],
- "description": "Full Builds; Group config (boards: orco, heli, wizpig)",
+ "description": "Full Builds; Group config (boards: orco, heli)",
"prebuilts": "public",
"useflags": [
"-chrome_internal"
@@ -15436,21 +16156,9 @@
"name": "heli-release",
"unittests": null,
"vm_tests": []
- },
- {
- "_template": "release",
- "boards": [
- "wizpig"
- ],
- "build_packages_in_background": true,
- "chrome_sdk_build_chrome": false,
- "grouped": true,
- "name": "wizpig-release",
- "unittests": null,
- "vm_tests": []
}
],
- "description": "Release Builds (canary) (internal); Group config (boards: orco, heli, wizpig)",
+ "description": "Release Builds (canary) (internal); Group config (boards: orco, heli)",
"important": true
},
"rambi-firmware": {
@@ -15594,6 +16302,171 @@
"hw_tests_override": [],
"vm_tests": []
},
+ "reks-chrome-pfq": {
+ "_template": "chrome-pfq",
+ "boards": [
+ "reks"
+ ],
+ "important": false,
+ "manifest": "official.xml"
+ },
+ "reks-chromium-pfq": {
+ "_template": "chromium-pfq",
+ "boards": [
+ "reks"
+ ],
+ "description": "Preflight Chromium Uprev & Build (internal)",
+ "doc": "http://www.chromium.org/chromium-os/build/builder-overview#TOC-Chrome-PFQ",
+ "important": false,
+ "internal": true,
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
+ "reks-compile-only-pre-cq": {
+ "_template": "compile-only-pre-cq",
+ "boards": [
+ "reks"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "reks-firmware": {
+ "_template": "firmware",
+ "boards": [
+ "reks"
+ ],
+ "manifest": "official.xml",
+ "useflags": [
+ "chrome_internal",
+ "chromeless_tty"
+ ]
+ },
+ "reks-full": {
+ "_template": "full",
+ "boards": [
+ "reks"
+ ],
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
+ "reks-no-vmtest-pre-cq": {
+ "_template": "no-vmtest-pre-cq",
+ "boards": [
+ "reks"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "reks-paladin": {
+ "_template": "paladin",
+ "boards": [
+ "reks"
+ ],
+ "description": "Commit Queue (internal)",
+ "important": false,
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "private",
+ "useflags": [
+ "chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "reks-payloads": {
+ "_template": "payloads",
+ "boards": [
+ "reks"
+ ]
+ },
+ "reks-pre-cq": {
+ "_template": "pre-cq",
+ "boards": [
+ "reks"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "reks-release": {
+ "_template": "release",
+ "boards": [
+ "reks"
+ ]
+ },
+ "reks-release-afdo": {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "reks"
+ ],
+ "child_configs": [
+ {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "reks"
+ ],
+ "grouped": true,
+ "name": "reks-release-afdo-generate"
+ },
+ {
+ "_template": "release-afdo-use",
+ "boards": [
+ "reks"
+ ],
+ "grouped": true,
+ "name": "reks-release-afdo-use"
+ }
+ ]
+ },
+ "reks-release-afdo-generate": {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "reks"
+ ]
+ },
+ "reks-release-afdo-use": {
+ "_template": "release-afdo-use",
+ "boards": [
+ "reks"
+ ]
+ },
+ "reks-tot-chrome-pfq-informational": {
+ "_template": "chrome-pfq-informational",
+ "boards": [
+ "reks"
+ ],
+ "manifest": "official.xml"
+ },
+ "reks-tot-chromium-pfq-informational": {
+ "_template": "chromium-pfq-informational",
+ "boards": [
+ "reks"
+ ],
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
"rikku-chrome-pfq": {
"_template": "chrome-pfq",
"boards": [
@@ -16918,6 +17791,9 @@
"recovery",
"test"
],
+ "sign_types": [
+ "nv_lp0_firmware"
+ ],
"vm_tests": []
},
"smaug-release-afdo": {
@@ -17387,7 +18263,6 @@
],
"chrome_sdk": false,
"dev_installer_prebuilts": false,
- "hw_tests": [],
"paygen_skip_testing": true,
"signer_tests": false,
"sync_chrome": false,
@@ -17410,7 +18285,6 @@
"chrome_sdk": false,
"dev_installer_prebuilts": false,
"grouped": true,
- "hw_tests": [],
"name": "storm-release",
"paygen_skip_testing": true,
"signer_tests": false,
@@ -17428,7 +18302,6 @@
"chrome_sdk_build_chrome": false,
"dev_installer_prebuilts": false,
"grouped": true,
- "hw_tests": [],
"name": "arkham-release",
"sync_chrome": false,
"unittests": null,
@@ -17444,7 +18317,6 @@
"chrome_sdk": false,
"chrome_sdk_build_chrome": false,
"grouped": true,
- "hw_tests": [],
"name": "whirlwind-release",
"sync_chrome": false,
"unittests": null,
@@ -17454,7 +18326,6 @@
"chrome_sdk": false,
"description": "Release Builds (canary) (internal); Group config (boards: storm, arkham, whirlwind)",
"dev_installer_prebuilts": false,
- "hw_tests": [],
"paygen_skip_testing": true,
"signer_tests": false,
"sync_chrome": false,
@@ -17647,6 +18518,133 @@
"-chrome_internal"
]
},
+ "strago-b-full-group": {
+ "_template": "full",
+ "boards": [
+ "reks"
+ ],
+ "child_configs": [
+ {
+ "_template": "full",
+ "boards": [
+ "reks"
+ ],
+ "grouped": true,
+ "name": "reks-full",
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
+ {
+ "_template": "full",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "build_packages_in_background": true,
+ "chrome_sdk_build_chrome": false,
+ "grouped": true,
+ "name": "cyan-cheets-full",
+ "prebuilts": "public",
+ "unittests": null,
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ {
+ "_template": "full",
+ "boards": [
+ "wizpig"
+ ],
+ "build_packages_in_background": true,
+ "chrome_sdk_build_chrome": false,
+ "grouped": true,
+ "name": "wizpig-full",
+ "prebuilts": "public",
+ "unittests": null,
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ {
+ "_template": "full",
+ "boards": [
+ "terra"
+ ],
+ "build_packages_in_background": true,
+ "chrome_sdk_build_chrome": false,
+ "grouped": true,
+ "name": "terra-full",
+ "prebuilts": "public",
+ "unittests": null,
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
+ }
+ ],
+ "description": "Full Builds; Group config (boards: reks, cyan-cheets, wizpig, terra)",
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
+ "strago-b-release-group": {
+ "_template": "release",
+ "active_waterfall": "chromeos",
+ "boards": [
+ "reks"
+ ],
+ "child_configs": [
+ {
+ "_template": "release",
+ "boards": [
+ "reks"
+ ],
+ "grouped": true,
+ "name": "reks-release"
+ },
+ {
+ "_template": "release",
+ "boards": [
+ "cyan-cheets"
+ ],
+ "build_packages_in_background": true,
+ "chrome_sdk_build_chrome": false,
+ "grouped": true,
+ "name": "cyan-cheets-release",
+ "unittests": null,
+ "vm_tests": []
+ },
+ {
+ "_template": "release",
+ "boards": [
+ "wizpig"
+ ],
+ "build_packages_in_background": true,
+ "chrome_sdk_build_chrome": false,
+ "grouped": true,
+ "name": "wizpig-release",
+ "unittests": null,
+ "vm_tests": []
+ },
+ {
+ "_template": "release",
+ "boards": [
+ "terra"
+ ],
+ "build_packages_in_background": true,
+ "chrome_sdk_build_chrome": false,
+ "grouped": true,
+ "name": "terra-release",
+ "unittests": null,
+ "vm_tests": []
+ }
+ ],
+ "description": "Release Builds (canary) (internal); Group config (boards: reks, cyan-cheets, wizpig, terra)"
+ },
"strago-chrome-pfq": {
"_template": "chrome-pfq",
"boards": [
@@ -18698,6 +19696,171 @@
"builder_class_name": "test_builders.ManifestVersionedSyncBuilder",
"hw_tests_override": []
},
+ "terra-chrome-pfq": {
+ "_template": "chrome-pfq",
+ "boards": [
+ "terra"
+ ],
+ "important": false,
+ "manifest": "official.xml"
+ },
+ "terra-chromium-pfq": {
+ "_template": "chromium-pfq",
+ "boards": [
+ "terra"
+ ],
+ "description": "Preflight Chromium Uprev & Build (internal)",
+ "doc": "http://www.chromium.org/chromium-os/build/builder-overview#TOC-Chrome-PFQ",
+ "important": false,
+ "internal": true,
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
+ "terra-compile-only-pre-cq": {
+ "_template": "compile-only-pre-cq",
+ "boards": [
+ "terra"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "terra-firmware": {
+ "_template": "firmware",
+ "boards": [
+ "terra"
+ ],
+ "manifest": "official.xml",
+ "useflags": [
+ "chrome_internal",
+ "chromeless_tty"
+ ]
+ },
+ "terra-full": {
+ "_template": "full",
+ "boards": [
+ "terra"
+ ],
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
+ "terra-no-vmtest-pre-cq": {
+ "_template": "no-vmtest-pre-cq",
+ "boards": [
+ "terra"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "terra-paladin": {
+ "_template": "paladin",
+ "boards": [
+ "terra"
+ ],
+ "description": "Commit Queue (internal)",
+ "important": false,
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "private",
+ "useflags": [
+ "chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "terra-payloads": {
+ "_template": "payloads",
+ "boards": [
+ "terra"
+ ]
+ },
+ "terra-pre-cq": {
+ "_template": "pre-cq",
+ "boards": [
+ "terra"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "terra-release": {
+ "_template": "release",
+ "boards": [
+ "terra"
+ ]
+ },
+ "terra-release-afdo": {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "terra"
+ ],
+ "child_configs": [
+ {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "terra"
+ ],
+ "grouped": true,
+ "name": "terra-release-afdo-generate"
+ },
+ {
+ "_template": "release-afdo-use",
+ "boards": [
+ "terra"
+ ],
+ "grouped": true,
+ "name": "terra-release-afdo-use"
+ }
+ ]
+ },
+ "terra-release-afdo-generate": {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "terra"
+ ]
+ },
+ "terra-release-afdo-use": {
+ "_template": "release-afdo-use",
+ "boards": [
+ "terra"
+ ]
+ },
+ "terra-tot-chrome-pfq-informational": {
+ "_template": "chrome-pfq-informational",
+ "boards": [
+ "terra"
+ ],
+ "manifest": "official.xml"
+ },
+ "terra-tot-chromium-pfq-informational": {
+ "_template": "chromium-pfq-informational",
+ "boards": [
+ "terra"
+ ],
+ "useflags": [
+ "-chrome_internal"
+ ]
+ },
"test-ap-group": {
"_template": "test-ap",
"boards": [
@@ -18876,32 +20039,6 @@
"-chrome_internal"
]
},
- "toolchain-major": {
- "archive_build_debug": true,
- "boards": [
- "x86-generic",
- "arm-generic",
- "amd64-generic"
- ],
- "build_type": "chroot",
- "builder_class_name": "sdk_builders.ChrootSdkBuilder",
- "chrome_sdk": true,
- "description": "Test next major toolchain revision",
- "doc": "http://www.chromium.org/chromium-os/build/builder-overview#TOC-Continuous",
- "gcc_githash": "svn-mirror/google/main",
- "git_sync": true,
- "hw_tests_override": [],
- "image_test": true,
- "images": [
- "base",
- "recovery",
- "test",
- "factory_install"
- ],
- "latest_toolchain": true,
- "use_sdk": false,
- "usepkg_build_packages": false
- },
"toolchain-minor": {
"archive_build_debug": true,
"boards": [
@@ -19413,28 +20550,12 @@
{
"_template": "full",
"boards": [
- "veyron_thea"
- ],
- "build_packages_in_background": true,
- "chrome_sdk_build_chrome": false,
- "grouped": true,
- "name": "veyron_thea-full",
- "prebuilts": "public",
- "unittests": null,
- "useflags": [
- "-chrome_internal"
- ],
- "vm_tests": []
- },
- {
- "_template": "full",
- "boards": [
- "veyron_shark"
+ "veyron_mickey"
],
"build_packages_in_background": true,
"chrome_sdk_build_chrome": false,
"grouped": true,
- "name": "veyron_shark-full",
+ "name": "veyron_mickey-full",
"prebuilts": "public",
"unittests": null,
"useflags": [
@@ -19443,7 +20564,7 @@
"vm_tests": []
}
],
- "description": "Full Builds; Group config (boards: veyron_brain, veyron_danger, veyron_thea, veyron_shark)",
+ "description": "Full Builds; Group config (boards: veyron_brain, veyron_danger, veyron_mickey)",
"prebuilts": "public",
"useflags": [
"-chrome_internal"
@@ -19481,44 +20602,34 @@
{
"_template": "release",
"boards": [
- "veyron_thea"
- ],
- "build_packages_in_background": true,
- "chrome_sdk_build_chrome": false,
- "grouped": true,
- "name": "veyron_thea-release",
- "unittests": null,
- "vm_tests": []
- },
- {
- "_template": "release",
- "boards": [
- "veyron_shark"
+ "veyron_mickey"
],
"build_packages_in_background": true,
"chrome_sdk_build_chrome": false,
"grouped": true,
- "name": "veyron_shark-release",
+ "hw_tests": [],
+ "name": "veyron_mickey-release",
"unittests": null,
"vm_tests": []
}
],
- "description": "Release Builds (canary) (internal); Group config (boards: veyron_brain, veyron_danger, veyron_thea, veyron_shark)",
+ "description": "Release Builds (canary) (internal); Group config (boards: veyron_brain, veyron_danger, veyron_mickey)",
+ "important": true,
"vm_tests": []
},
"veyron-d-full-group": {
"_template": "full",
"boards": [
- "veyron_mickey"
+ "veyron_shark"
],
"child_configs": [
{
"_template": "full",
"boards": [
- "veyron_mickey"
+ "veyron_shark"
],
"grouped": true,
- "name": "veyron_mickey-full",
+ "name": "veyron_shark-full",
"prebuilts": "public",
"useflags": [
"-chrome_internal"
@@ -19540,9 +20651,25 @@
"-chrome_internal"
],
"vm_tests": []
+ },
+ {
+ "_template": "full",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "build_packages_in_background": true,
+ "chrome_sdk_build_chrome": false,
+ "grouped": true,
+ "name": "veyron_minnie-cheets-full",
+ "prebuilts": "public",
+ "unittests": null,
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
}
],
- "description": "Full Builds; Group config (boards: veyron_mickey, veyron_romy)",
+ "description": "Full Builds; Group config (boards: veyron_shark, veyron_romy, veyron_minnie-cheets)",
"prebuilts": "public",
"useflags": [
"-chrome_internal"
@@ -19553,17 +20680,16 @@
"_template": "release",
"active_waterfall": "chromeos",
"boards": [
- "veyron_mickey"
+ "veyron_shark"
],
"child_configs": [
{
"_template": "release",
"boards": [
- "veyron_mickey"
+ "veyron_shark"
],
"grouped": true,
- "hw_tests": [],
- "name": "veyron_mickey-release",
+ "name": "veyron_shark-release",
"vm_tests": []
},
{
@@ -19578,11 +20704,21 @@
"name": "veyron_romy-release",
"unittests": null,
"vm_tests": []
+ },
+ {
+ "_template": "release",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "build_packages_in_background": true,
+ "chrome_sdk_build_chrome": false,
+ "grouped": true,
+ "name": "veyron_minnie-cheets-release",
+ "unittests": null,
+ "vm_tests": []
}
],
- "description": "Release Builds (canary) (internal); Group config (boards: veyron_mickey, veyron_romy)",
- "hw_tests": [],
- "important": true,
+ "description": "Release Builds (canary) (internal); Group config (boards: veyron_shark, veyron_romy, veyron_minnie-cheets)",
"vm_tests": []
},
"veyron-full-group": {
@@ -20799,10 +21935,14 @@
},
"veyron_mighty-paladin": {
"_template": "paladin",
+ "active_waterfall": "chromeos",
"boards": [
"veyron_mighty"
],
"description": "Commit Queue (internal)",
+ "hw_tests": [
+ "{\n \"async\": false,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 4,\n \"num\": 6,\n \"offload_failures_only\": true,\n \"pool\": \"cq\",\n \"priority\": \"CQ\",\n \"retry\": true,\n \"suite\": \"bvt-inline\",\n \"suite_min_duts\": 0,\n \"timeout\": 7200,\n \"warn_only\": false\n}"
+ ],
"important": false,
"internal": true,
"manifest": "official.xml",
@@ -20901,6 +22041,173 @@
],
"vm_tests": []
},
+ "veyron_minnie-cheets-chrome-pfq": {
+ "_template": "chrome-pfq",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "important": false,
+ "manifest": "official.xml",
+ "vm_tests": []
+ },
+ "veyron_minnie-cheets-chromium-pfq": {
+ "_template": "chromium-pfq",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "description": "Preflight Chromium Uprev & Build (internal)",
+ "doc": "http://www.chromium.org/chromium-os/build/builder-overview#TOC-Chrome-PFQ",
+ "important": false,
+ "internal": true,
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "veyron_minnie-cheets-compile-only-pre-cq": {
+ "_template": "compile-only-pre-cq",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "veyron_minnie-cheets-full": {
+ "_template": "full",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "prebuilts": "public",
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "veyron_minnie-cheets-no-vmtest-pre-cq": {
+ "_template": "no-vmtest-pre-cq",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ]
+ },
+ "veyron_minnie-cheets-paladin": {
+ "_template": "paladin",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "description": "Commit Queue (internal)",
+ "important": false,
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "prebuilts": "private",
+ "useflags": [
+ "chrome_internal"
+ ],
+ "vm_tests": [],
+ "vm_tests_override": null
+ },
+ "veyron_minnie-cheets-payloads": {
+ "_template": "payloads",
+ "boards": [
+ "veyron_minnie-cheets"
+ ]
+ },
+ "veyron_minnie-cheets-pre-cq": {
+ "_template": "pre-cq",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "internal": true,
+ "manifest": "official.xml",
+ "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
+ "overlays": "both",
+ "useflags": [
+ "chrome_internal"
+ ],
+ "vm_tests": []
+ },
+ "veyron_minnie-cheets-release": {
+ "_template": "release",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "vm_tests": []
+ },
+ "veyron_minnie-cheets-release-afdo": {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "child_configs": [
+ {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "grouped": true,
+ "name": "veyron_minnie-cheets-release-afdo-generate",
+ "vm_tests": []
+ },
+ {
+ "_template": "release-afdo-use",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "grouped": true,
+ "name": "veyron_minnie-cheets-release-afdo-use",
+ "vm_tests": []
+ }
+ ],
+ "vm_tests": []
+ },
+ "veyron_minnie-cheets-release-afdo-generate": {
+ "_template": "release-afdo-generate",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "vm_tests": []
+ },
+ "veyron_minnie-cheets-release-afdo-use": {
+ "_template": "release-afdo-use",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "vm_tests": []
+ },
+ "veyron_minnie-cheets-tot-chrome-pfq-informational": {
+ "_template": "chrome-pfq-informational",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "manifest": "official.xml",
+ "vm_tests": []
+ },
+ "veyron_minnie-cheets-tot-chromium-pfq-informational": {
+ "_template": "chromium-pfq-informational",
+ "boards": [
+ "veyron_minnie-cheets"
+ ],
+ "useflags": [
+ "-chrome_internal"
+ ],
+ "vm_tests": []
+ },
"veyron_minnie-chrome-pfq": {
"_template": "chrome-pfq",
"boards": [
@@ -21804,10 +23111,14 @@
},
"veyron_speedy-paladin": {
"_template": "paladin",
+ "active_waterfall": "chromeos",
"boards": [
"veyron_speedy"
],
"description": "Commit Queue (internal)",
+ "hw_tests": [
+ "{\n \"async\": false,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 4,\n \"num\": 6,\n \"offload_failures_only\": true,\n \"pool\": \"cq\",\n \"priority\": \"CQ\",\n \"retry\": true,\n \"suite\": \"bvt-cq\",\n \"suite_min_duts\": 0,\n \"timeout\": 7200,\n \"warn_only\": false\n}"
+ ],
"important": false,
"internal": true,
"manifest": "official.xml",
@@ -21920,173 +23231,6 @@
],
"vm_tests": []
},
- "veyron_thea-chrome-pfq": {
- "_template": "chrome-pfq",
- "boards": [
- "veyron_thea"
- ],
- "important": false,
- "manifest": "official.xml",
- "vm_tests": []
- },
- "veyron_thea-chromium-pfq": {
- "_template": "chromium-pfq",
- "boards": [
- "veyron_thea"
- ],
- "description": "Preflight Chromium Uprev & Build (internal)",
- "doc": "http://www.chromium.org/chromium-os/build/builder-overview#TOC-Chrome-PFQ",
- "important": false,
- "internal": true,
- "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
- "overlays": "both",
- "prebuilts": "public",
- "useflags": [
- "-chrome_internal"
- ],
- "vm_tests": []
- },
- "veyron_thea-compile-only-pre-cq": {
- "_template": "compile-only-pre-cq",
- "boards": [
- "veyron_thea"
- ],
- "internal": true,
- "manifest": "official.xml",
- "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
- "overlays": "both",
- "useflags": [
- "chrome_internal"
- ]
- },
- "veyron_thea-full": {
- "_template": "full",
- "boards": [
- "veyron_thea"
- ],
- "prebuilts": "public",
- "useflags": [
- "-chrome_internal"
- ],
- "vm_tests": []
- },
- "veyron_thea-no-vmtest-pre-cq": {
- "_template": "no-vmtest-pre-cq",
- "boards": [
- "veyron_thea"
- ],
- "internal": true,
- "manifest": "official.xml",
- "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
- "overlays": "both",
- "useflags": [
- "chrome_internal"
- ]
- },
- "veyron_thea-paladin": {
- "_template": "paladin",
- "boards": [
- "veyron_thea"
- ],
- "description": "Commit Queue (internal)",
- "important": false,
- "internal": true,
- "manifest": "official.xml",
- "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
- "overlays": "both",
- "prebuilts": "private",
- "useflags": [
- "chrome_internal"
- ],
- "vm_tests": [],
- "vm_tests_override": null
- },
- "veyron_thea-payloads": {
- "_template": "payloads",
- "boards": [
- "veyron_thea"
- ]
- },
- "veyron_thea-pre-cq": {
- "_template": "pre-cq",
- "boards": [
- "veyron_thea"
- ],
- "internal": true,
- "manifest": "official.xml",
- "manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
- "overlays": "both",
- "useflags": [
- "chrome_internal"
- ],
- "vm_tests": []
- },
- "veyron_thea-release": {
- "_template": "release",
- "boards": [
- "veyron_thea"
- ],
- "vm_tests": []
- },
- "veyron_thea-release-afdo": {
- "_template": "release-afdo-generate",
- "boards": [
- "veyron_thea"
- ],
- "child_configs": [
- {
- "_template": "release-afdo-generate",
- "boards": [
- "veyron_thea"
- ],
- "grouped": true,
- "name": "veyron_thea-release-afdo-generate",
- "vm_tests": []
- },
- {
- "_template": "release-afdo-use",
- "boards": [
- "veyron_thea"
- ],
- "grouped": true,
- "name": "veyron_thea-release-afdo-use",
- "vm_tests": []
- }
- ],
- "vm_tests": []
- },
- "veyron_thea-release-afdo-generate": {
- "_template": "release-afdo-generate",
- "boards": [
- "veyron_thea"
- ],
- "vm_tests": []
- },
- "veyron_thea-release-afdo-use": {
- "_template": "release-afdo-use",
- "boards": [
- "veyron_thea"
- ],
- "vm_tests": []
- },
- "veyron_thea-tot-chrome-pfq-informational": {
- "_template": "chrome-pfq-informational",
- "boards": [
- "veyron_thea"
- ],
- "manifest": "official.xml",
- "vm_tests": []
- },
- "veyron_thea-tot-chromium-pfq-informational": {
- "_template": "chromium-pfq-informational",
- "boards": [
- "veyron_thea"
- ],
- "useflags": [
- "-chrome_internal"
- ],
- "vm_tests": []
- },
"whirlwind-chrome-pfq": {
"_template": "chrome-pfq",
"boards": [
@@ -22160,12 +23304,15 @@
},
"whirlwind-paladin": {
"_template": "paladin",
- "active_waterfall": "chromeos",
"boards": [
"whirlwind"
],
"chrome_sdk": false,
"description": "Commit Queue (internal)",
+ "hw_tests": [
+ "{\n \"async\": false,\n \"blocking\": false,\n \"critical\": false,\n \"file_bugs\": false,\n \"max_retries\": 10,\n \"minimum_duts\": 0,\n \"num\": 6,\n \"offload_failures_only\": false,\n \"pool\": \"cq\",\n \"priority\": \"Build\",\n \"retry\": true,\n \"suite\": \"jetstream_cq\",\n \"suite_min_duts\": 0,\n \"timeout\": 13200,\n \"warn_only\": false\n}"
+ ],
+ "important": false,
"internal": true,
"manifest": "official.xml",
"manifest_repo_url": "https://chrome-internal-review.googlesource.com/chromeos/manifest-internal",
@@ -22206,7 +23353,6 @@
"whirlwind"
],
"chrome_sdk": false,
- "hw_tests": [],
"sync_chrome": false,
"vm_tests": []
},
diff --git a/cbuildbot/config_lib.py b/cbuildbot/config_lib.py
index b9f71b26a..226ad0bf0 100644
--- a/cbuildbot/config_lib.py
+++ b/cbuildbot/config_lib.py
@@ -269,7 +269,6 @@ class HWTestConfig(object):
Some combinations of member settings are invalid:
* A suite config may not specify both blocking and async.
- * A suite config may not specify both retry and async.
* A suite config may not specify both warn_only and critical.
"""
# This timeout is larger than it needs to be because of autotest overhead.
@@ -284,7 +283,7 @@ class HWTestConfig(object):
retry=True, max_retries=10, minimum_duts=0, suite_min_duts=0,
offload_failures_only=False):
"""Constructor -- see members above."""
- assert not async or (not blocking and not retry)
+ assert not async or not blocking
assert not warn_only or not critical
self.suite = suite
self.num = num
@@ -699,6 +698,10 @@ def DefaultSettings():
# manifest versions that we may later want to branch off of.
branch_util_test=False,
+ # If specified, it is passed on to the PushImage script as '--sign-types'
+ # commandline argument. Must be either None or a list of image types.
+ sign_types=None,
+
# TODO(sosa): Collapse to one option.
# ========== Dev installer prebuilts options =======================
@@ -873,7 +876,10 @@ def DefaultSiteParameters():
INTERNAL_MANIFEST_VERSIONS_PATH=None,
# URL of the repo project.
- REPO_URL='https://chromium.googlesource.com/external/repo'
+ REPO_URL='https://chromium.googlesource.com/external/repo',
+
+ # GS URL in which to archive build artifacts.
+ ARCHIVE_URL='gs://chromeos-image-archive',
)
return default_site_params
diff --git a/cbuildbot/constants.py b/cbuildbot/constants.py
index 168f2e785..0a64af3a5 100644
--- a/cbuildbot/constants.py
+++ b/cbuildbot/constants.py
@@ -442,6 +442,7 @@ HWTEST_BVT_SUITE = 'bvt-inline'
HWTEST_COMMIT_SUITE = 'bvt-cq'
HWTEST_CANARY_SUITE = 'bvt-perbuild'
HWTEST_AFDO_SUITE = 'AFDO_record'
+HWTEST_JETSTREAM_COMMIT_SUITE = 'jetstream_cq'
HWTEST_MOBLAB_SUITE = 'moblab'
HWTEST_MOBLAB_QUICK_SUITE = 'moblab_quick'
HWTEST_SANITY_SUITE = 'sanity'
@@ -721,7 +722,6 @@ def ImageBinToGceTar(image_bin):
image_bin)
return '%s_gce.tar.gz' % os.path.splitext(image_bin)[0]
-DEFAULT_ARCHIVE_BUCKET = 'gs://chromeos-image-archive'
RELEASE_BUCKET = 'gs://chromeos-releases'
TRASH_BUCKET = 'gs://chromeos-throw-away-bucket'
CHROME_SYSROOT_TAR = 'sysroot_%s.tar.xz' % _SlashToUnderscore(CHROME_CP)
@@ -760,7 +760,12 @@ IMAGE_TYPE_TEST = 'test'
IMAGE_TYPE_RECOVERY = 'recovery'
IMAGE_TYPE_FACTORY = 'factory'
IMAGE_TYPE_FIRMWARE = 'firmware'
+# NVidia Tegra SoC resume firmware blob.
IMAGE_TYPE_NV_LP0_FIRMWARE = 'nv_lp0_firmware'
+# USB PD accessory microcontroller firmware (e.g. power brick, display dongle).
+IMAGE_TYPE_ACCESSORY_USBPD = 'accessory_usbpd'
+# Standalone accessory microcontroller firmware (e.g. wireless keyboard).
+IMAGE_TYPE_ACCESSORY_RWSIG = 'accessory_rwsig'
IMAGE_TYPE_TO_NAME = {
IMAGE_TYPE_BASE: BASE_IMAGE_BIN,
diff --git a/cbuildbot/lkgm_manager.py b/cbuildbot/lkgm_manager.py
index 05e4eb38e..947b7b25e 100644
--- a/cbuildbot/lkgm_manager.py
+++ b/cbuildbot/lkgm_manager.py
@@ -249,6 +249,11 @@ class LKGMManager(manifest_version.BuildSpecsManager):
self.InitializeManifestVariables(version_info)
self.GenerateBlameListSinceLKGM()
+
+ # Throw away CLs that might not be used this run.
+ if validation_pool:
+ validation_pool.FilterChangesForThrottledTree()
+
new_manifest = self.CreateManifest()
# For Chrome PFQ, add the version of Chrome to use.
@@ -326,6 +331,9 @@ class LKGMManager(manifest_version.BuildSpecsManager):
build_id: Optional integer cidb build id of the build publishing the
manifest.
+ Returns:
+ Path to the manifest version file to use.
+
Raises:
GenerateBuildSpecException in case of failure to check-in the new
manifest because of a git error or the manifest is already checked-in.
diff --git a/cbuildbot/manifest_version.py b/cbuildbot/manifest_version.py
index 92e5de3ce..bc74a8e60 100644
--- a/cbuildbot/manifest_version.py
+++ b/cbuildbot/manifest_version.py
@@ -7,6 +7,7 @@
from __future__ import print_function
import cPickle
+import datetime
import fnmatch
import glob
import os
@@ -418,6 +419,125 @@ class BuilderStatus(object):
dashboard_url=self.dashboard_url))
+class SlaveStatus(object):
+ """A Class to easily interpret data from CIDB regarding slave status.
+
+ This is intended for short lived instances used to determine if the loop on
+ getting the builders statuses should continue or break out. The main function
+ is ShouldWait() with everything else pretty much a helper function for it.
+ """
+
+ BUILDER_START_TIMEOUT = 5
+
+ def __init__(self, status, start_time, builders_array, previous_completed):
+ """Initializes a slave status object.
+
+ Args:
+ status: Dict of the slave status from CIDB.
+ start_time: datetime.datetime object of when the build started.
+ builders_array: List of the expected builders.
+ previous_completed: Set of builders that have finished already.
+ """
+ self.status = status
+ self.start_time = start_time
+ self.builders_array = builders_array
+ self.previous_completed = previous_completed
+ self.completed = []
+
+ def GetMissing(self):
+ """Returns the missing builders.
+
+ Returns:
+ A list of the missing builders
+ """
+ return list(set(self.builders_array) - set(self.status.keys()))
+
+ def GetCompleted(self):
+ """Returns the builders that have completed.
+
+ Returns:
+ A list of the completed builders.
+ """
+ if not self.completed:
+ self.completed = [b for b, s in self.status.iteritems()
+ if s in constants.BUILDER_COMPLETED_STATUSES and
+ b in self.builders_array]
+
+ # Logging of the newly complete builders.
+ for builder in sorted(set(self.completed) - self.previous_completed):
+ logging.info('Build config %s completed with status "%s".',
+ builder, self.status[builder])
+ self.previous_completed.update(set(self.completed))
+ return self.completed
+
+ def Completed(self):
+ """Returns a bool if all builders have completed successfully.
+
+ Returns:
+ A bool of True if all builders successfully completed, False otherwise.
+ """
+ return len(self.GetCompleted()) == len(self.builders_array)
+
+ def ShouldFailForBuilderStartTimeout(self, current_time):
+ """Decides if we should fail if a builder hasn't started within 5 mins.
+
+ If a builder hasn't started within BUILDER_START_TIMEOUT and the rest of the
+ builders have finished, let the caller know that we should fail.
+
+ Args:
+ current_time: A datetime.datetime object letting us know the current time.
+
+ Returns:
+ A bool saying True that we should fail, False otherwise.
+ """
+ # Check that we're at least past the start timeout.
+ builder_start_deadline = datetime.timedelta(
+ minutes=self.BUILDER_START_TIMEOUT)
+ past_deadline = current_time - self.start_time > builder_start_deadline
+
+ # Check that aside from the missing builders the rest have completed.
+ other_builders_completed = (
+ (len(self.GetMissing()) + len(self.GetCompleted())) ==
+ len(self.builders_array))
+
+ # Check that we have missing builders and logging who they are.
+ builders_are_missing = False
+ for builder in self.GetMissing():
+ logging.error('No status found for build config %s.', builder)
+ builders_are_missing = True
+
+ return past_deadline and other_builders_completed and builders_are_missing
+
+ def ShouldWait(self):
+ """Decides if we should continue to wait for the builders to finish.
+
+ This will be the retry function for timeout_util.WaitForSuccess, basically
+ this function will return False if all builders finished or we see a
+ problem with the builders. Otherwise we'll return True to continue polling
+ for the builders statuses.
+
+ Returns:
+ A bool of True if we should continue to wait and False if we should not.
+ """
+ # Check if all builders completed.
+ if self.Completed():
+ return False
+
+ current_time = datetime.datetime.now()
+
+ # Guess there are some builders building, check if there is a problem.
+ if self.ShouldFailForBuilderStartTimeout(current_time):
+ logging.error('Ending build since at least one builder has not started '
+ 'within 5 mins.')
+ return False
+
+ # We got here which means no problems, we should still wait.
+ logging.info('Still waiting for the following builds to complete: %r',
+ sorted(set(self.builders_array).difference(
+ self.GetCompleted())))
+ return True
+
+
class BuildSpecsManager(object):
"""A Class to manage buildspecs and their states."""
@@ -695,41 +815,23 @@ class BuildSpecsManager(object):
A build_config name-> status dictionary of build statuses.
"""
builders_completed = set()
-
- def _GetStatusesFromDB():
- """Helper function that iterates through current statuses."""
- status_dict = self.GetSlaveStatusesFromCIDB(master_build_id)
- for builder in set(builders_array) - set(status_dict.keys()):
- logging.warning('No status found for build config %s.', builder)
-
- latest_completed = set(
- [b for b, s in status_dict.iteritems() if s in
- constants.BUILDER_COMPLETED_STATUSES and b in builders_array])
- for builder in sorted(latest_completed - builders_completed):
- logging.info('Build config %s completed with status "%s".',
- builder, status_dict[builder])
- builders_completed.update(latest_completed)
-
- if len(builders_completed) < len(builders_array):
- logging.info('Still waiting for the following builds to complete: %r',
- sorted(set(builders_array).difference(builders_completed)))
- return None
- else:
- return 'Builds completed.'
+ start_time = datetime.datetime.now()
def _PrintRemainingTime(remaining):
logging.info('%s until timeout...', remaining)
# Check for build completion until all builders report in.
+ builds_timed_out = False
try:
- builds_succeeded = timeout_util.WaitForSuccess(
- lambda x: x is None,
- _GetStatusesFromDB,
+ timeout_util.WaitForSuccess(
+ lambda statuses: statuses.ShouldWait(),
+ lambda: SlaveStatus(self.GetSlaveStatusesFromCIDB(master_build_id),
+ start_time, builders_array, builders_completed),
timeout,
period=self.SLEEP_TIMEOUT,
side_effect_func=_PrintRemainingTime)
except timeout_util.TimeoutError:
- builds_succeeded = None
+ builds_timed_out = True
# Actually fetch the BuildStatus pickles from Google Storage.
builder_statuses = {}
@@ -738,7 +840,7 @@ class BuildSpecsManager(object):
builder_status = self.GetBuildStatus(builder, self.current_version)
builder_statuses[builder] = builder_status
- if not builds_succeeded:
+ if builds_timed_out:
logging.error('Not all builds finished before timeout (%d minutes)'
' reached.', int((timeout / 60) + 0.5))
diff --git a/cbuildbot/manifest_version_unittest.py b/cbuildbot/manifest_version_unittest.py
index 233d8902d..08411a2ef 100644
--- a/cbuildbot/manifest_version_unittest.py
+++ b/cbuildbot/manifest_version_unittest.py
@@ -6,7 +6,7 @@
from __future__ import print_function
-import mox
+import datetime
import os
import tempfile
@@ -59,7 +59,7 @@ class HelperMethodsTest(cros_test_lib.TempDirTestCase):
'Unable to create symlink to %s' % destfile)
-class VersionInfoTest(cros_test_lib.MoxTempDirTestCase):
+class VersionInfoTest(cros_test_lib.MockTempDirTestCase):
"""Test methods testing methods in VersionInfo class."""
@classmethod
@@ -103,25 +103,23 @@ class VersionInfoTest(cros_test_lib.MoxTempDirTestCase):
def CommonTestIncrementVersion(self, incr_type, version, chrome_branch=None):
"""Common test increment. Returns path to new incremented file."""
message = 'Incrementing cuz I sed so'
- self.mox.StubOutWithMock(git, 'CreateBranch')
- self.mox.StubOutWithMock(manifest_version, '_PushGitChanges')
- self.mox.StubOutWithMock(git, 'CleanAndCheckoutUpstream')
-
- git.CreateBranch(self.tempdir, manifest_version.PUSH_BRANCH)
+ create_mock = self.PatchObject(git, 'CreateBranch')
+ push_mock = self.PatchObject(manifest_version, '_PushGitChanges')
+ clean_mock = self.PatchObject(git, 'CleanAndCheckoutUpstream')
version_file = self.CreateFakeVersionFile(
self.tempdir, version=version, chrome_branch=chrome_branch)
-
- manifest_version._PushGitChanges(self.tempdir, message, dry_run=False,
- push_to=None)
-
- git.CleanAndCheckoutUpstream(self.tempdir)
- self.mox.ReplayAll()
info = manifest_version.VersionInfo(version_file=version_file,
incr_type=incr_type)
info.IncrementVersion()
info.UpdateVersionFile(message, dry_run=False)
- self.mox.VerifyAll()
+
+ create_mock.assert_called_once_with(
+ self.tempdir, manifest_version.PUSH_BRANCH)
+ push_mock.assert_called_once_with(
+ self.tempdir, message, dry_run=False, push_to=None)
+ clean_mock.assert_called_once_with(self.tempdir)
+
return version_file
def testIncrementVersionPatch(self):
@@ -154,8 +152,7 @@ class VersionInfoTest(cros_test_lib.MoxTempDirTestCase):
self.assertEqual(new_info.chrome_branch, '30')
-class BuildSpecsManagerTest(cros_test_lib.MoxTempDirTestCase,
- cros_test_lib.MockTestCase):
+class BuildSpecsManagerTest(cros_test_lib.MockTempDirTestCase):
"""Tests for the BuildSpecs manager."""
def setUp(self):
@@ -184,7 +181,7 @@ class BuildSpecsManagerTest(cros_test_lib.MoxTempDirTestCase,
"""Tests that PublishManifest writes a build id."""
expected_message = ('Automatic: Start x86-generic-paladin master 1\n'
'CrOS-Build-Id: %s' % MOCK_BUILD_ID)
- self.mox.StubOutWithMock(self.manager, 'PushSpecChanges')
+ push_mock = self.PatchObject(self.manager, 'PushSpecChanges')
info = manifest_version.VersionInfo(
FAKE_VERSION_STRING, CHROME_BRANCH, incr_type='branch')
@@ -194,16 +191,14 @@ class BuildSpecsManagerTest(cros_test_lib.MoxTempDirTestCase,
osutils.Touch(m)
self.manager.InitializeManifestVariables(info)
- self.manager.PushSpecChanges(expected_message)
-
- self.mox.ReplayAll()
self.manager.PublishManifest(m, '1', build_id=MOCK_BUILD_ID)
- self.mox.VerifyAll()
+
+ push_mock.assert_called_once_with(expected_message)
def testPublishManifestCommitMessageWithNegativeBuildId(self):
"""Tests that PublishManifest doesn't write a negative build_id"""
expected_message = 'Automatic: Start x86-generic-paladin master 1'
- self.mox.StubOutWithMock(self.manager, 'PushSpecChanges')
+ push_mock = self.PatchObject(self.manager, 'PushSpecChanges')
info = manifest_version.VersionInfo(
FAKE_VERSION_STRING, CHROME_BRANCH, incr_type='branch')
@@ -213,16 +208,14 @@ class BuildSpecsManagerTest(cros_test_lib.MoxTempDirTestCase,
osutils.Touch(m)
self.manager.InitializeManifestVariables(info)
- self.manager.PushSpecChanges(expected_message)
-
- self.mox.ReplayAll()
self.manager.PublishManifest(m, '1', build_id=-1)
- self.mox.VerifyAll()
+
+ push_mock.assert_called_once_with(expected_message)
def testPublishManifestCommitMessageWithNoneBuildId(self):
"""Tests that PublishManifest doesn't write a non-existant build_id"""
expected_message = 'Automatic: Start x86-generic-paladin master 1'
- self.mox.StubOutWithMock(self.manager, 'PushSpecChanges')
+ push_mock = self.PatchObject(self.manager, 'PushSpecChanges')
info = manifest_version.VersionInfo(
FAKE_VERSION_STRING, CHROME_BRANCH, incr_type='branch')
@@ -232,11 +225,9 @@ class BuildSpecsManagerTest(cros_test_lib.MoxTempDirTestCase,
osutils.Touch(m)
self.manager.InitializeManifestVariables(info)
- self.manager.PushSpecChanges(expected_message)
-
- self.mox.ReplayAll()
self.manager.PublishManifest(m, '1')
- self.mox.VerifyAll()
+
+ push_mock.assert_called_once_with(expected_message)
def testLoadSpecs(self):
"""Tests whether we can load specs correctly."""
@@ -262,16 +253,14 @@ class BuildSpecsManagerTest(cros_test_lib.MoxTempDirTestCase,
for_build, 'fail', CHROME_BRANCH, os.path.basename(m1)))
manifest_version.CreateSymlink(m1, os.path.join(
for_build, 'pass', CHROME_BRANCH, os.path.basename(m2)))
- self.mox.StubOutWithMock(self.manager, 'GetBuildStatus')
- self.manager.GetBuildStatus(self.build_names[0], '1.2.5').AndReturn(missing)
- self.mox.ReplayAll()
+ m = self.PatchObject(self.manager, 'GetBuildStatus', return_value=missing)
self.manager.InitializeManifestVariables(info)
- self.mox.VerifyAll()
self.assertEqual(self.manager.latest_unprocessed, '1.2.5')
+ m.assert_called_once_with(self.build_names[0], '1.2.5')
def testLatestSpecFromDir(self):
"""Tests whether we can get sorted specs correctly from a directory."""
- self.mox.StubOutWithMock(repository, 'CloneGitRepo')
+ self.PatchObject(repository, 'CloneGitRepo', side_effect=Exception())
info = manifest_version.VersionInfo(
'99.1.2', CHROME_BRANCH, incr_type='branch')
@@ -286,9 +275,7 @@ class BuildSpecsManagerTest(cros_test_lib.MoxTempDirTestCase,
for m in [m1, m2, m3, m4]:
osutils.Touch(m)
- self.mox.ReplayAll()
spec = self.manager._LatestSpecFromDir(info, specs_dir)
- self.mox.VerifyAll()
# Should be the latest on the 99.1 branch.
self.assertEqual(spec, '99.1.10')
@@ -301,26 +288,22 @@ class BuildSpecsManagerTest(cros_test_lib.MoxTempDirTestCase,
FAKE_VERSION_STRING, CHROME_BRANCH, incr_type='branch')
self.manager.latest = None
- self.mox.ReplayAll()
version = self.manager.GetNextVersion(info)
- self.mox.VerifyAll()
self.assertEqual(FAKE_VERSION_STRING, version)
def testGetNextVersionIncrement(self):
"""Tests that we create a new version if a previous one exists."""
- self.mox.StubOutWithMock(manifest_version.VersionInfo, 'UpdateVersionFile')
+ m = self.PatchObject(manifest_version.VersionInfo, 'UpdateVersionFile')
version_file = VersionInfoTest.CreateFakeVersionFile(self.tempdir)
info = manifest_version.VersionInfo(version_file=version_file,
incr_type='branch')
- info.UpdateVersionFile(
- 'Automatic: %s - Updating to a new version number from %s' % (
- self.build_names[0], FAKE_VERSION_STRING), dry_run=True)
self.manager.latest = FAKE_VERSION_STRING
- self.mox.ReplayAll()
version = self.manager.GetNextVersion(info)
- self.mox.VerifyAll()
self.assertEqual(FAKE_VERSION_STRING_NEXT, version)
+ m.assert_called_once_with(
+ 'Automatic: %s - Updating to a new version number from %s' % (
+ self.build_names[0], FAKE_VERSION_STRING), dry_run=True)
def testGetNextBuildSpec(self):
"""End-to-end test of updating the manifest."""
@@ -333,10 +316,8 @@ class BuildSpecsManagerTest(cros_test_lib.MoxTempDirTestCase,
rc = self.StartPatcher(cros_build_lib_unittest.RunCommandMock())
rc.SetDefaultCmdResult()
- self.mox.ReplayAll()
self.manager.GetNextBuildSpec(retries=0)
self.manager.UpdateStatus({self.build_names[0]: True})
- self.mox.VerifyAll()
def testUnpickleBuildStatus(self):
"""Tests that _UnpickleBuildStatus returns the correct values."""
@@ -366,25 +347,19 @@ class BuildSpecsManagerTest(cros_test_lib.MoxTempDirTestCase,
builders: List of builders to get status for.
status_runs: List of dictionaries of expected build and status.
"""
- self.mox.StubOutWithMock(manifest_version.BuildSpecsManager,
- 'GetSlaveStatusesFromCIDB')
- self.mox.StubOutWithMock(manifest_version.BuildSpecsManager,
- 'GetBuildStatus')
- for status_dict in status_runs:
- manifest_version.BuildSpecsManager.GetSlaveStatusesFromCIDB(
- mox.IgnoreArg()).AndReturn(status_dict)
+ self.PatchObject(manifest_version.BuildSpecsManager,
+ 'GetSlaveStatusesFromCIDB', side_effect=status_runs)
final_status_dict = status_runs[-1]
- for builder in builders:
- status = manifest_version.BuilderStatus(
- final_status_dict.get(builder), None)
- manifest_version.BuildSpecsManager.GetBuildStatus(
- builder, mox.IgnoreArg()).AndReturn(status)
+ build_statuses = [
+ manifest_version.BuilderStatus(final_status_dict.get(x), None)
+ for x in builders
+ ]
+ self.PatchObject(manifest_version.BuildSpecsManager,
+ 'GetBuildStatus',
+ side_effect=build_statuses)
- self.mox.ReplayAll()
- statuses = self.manager.GetBuildersStatus(mox.IgnoreArg, builders)
- self.mox.VerifyAll()
- return statuses
+ return self.manager.GetBuildersStatus('builderid', builders)
def testGetBuildersStatusBothFinished(self):
"""Tests GetBuilderStatus where both builds have finished."""
@@ -405,3 +380,139 @@ class BuildSpecsManagerTest(cros_test_lib.MoxTempDirTestCase,
statuses = self._GetBuildersStatus(['build1', 'build2'], status_runs)
self.assertTrue(statuses['build1'].Failed())
self.assertTrue(statuses['build2'].Passed())
+
+
+class SlaveStatusTest(cros_test_lib.TestCase):
+ """Test methods testing methods in SalveStatus class."""
+
+ def testGetMissing(self):
+ """Tests GetMissing returns the missing builders."""
+ status = {'build1': constants.BUILDER_STATUS_FAILED,
+ 'build2': constants.BUILDER_STATUS_INFLIGHT}
+ builders_array = ['build1', 'build2', 'missing_builder']
+ slaveStatus = manifest_version.SlaveStatus(status, datetime.datetime.now(),
+ builders_array, set())
+
+ self.assertEqual(slaveStatus.GetMissing(), ['missing_builder'])
+
+ def testGetMissingNone(self):
+ """Tests GetMissing returns nothing when all builders are accounted for."""
+ status = {'build1': constants.BUILDER_STATUS_FAILED,
+ 'build2': constants.BUILDER_STATUS_INFLIGHT}
+ builders_array = ['build1', 'build2']
+ slaveStatus = manifest_version.SlaveStatus(status, datetime.datetime.now(),
+ builders_array, set())
+
+ self.assertEqual(slaveStatus.GetMissing(), [])
+
+ def testGetCompleted(self):
+ """Tests GetCompleted returns the right builders that have completed."""
+ status = {'passed': constants.BUILDER_STATUS_PASSED,
+ 'failed': constants.BUILDER_STATUS_FAILED,
+ 'aborted': constants.BUILDER_STATUS_ABORTED,
+ 'skipped': constants.BUILDER_STATUS_SKIPPED,
+ 'forgiven': constants.BUILDER_STATUS_FORGIVEN,
+ 'inflight': constants.BUILDER_STATUS_INFLIGHT,
+ 'missing': constants.BUILDER_STATUS_MISSING,
+ 'planned': constants.BUILDER_STATUS_PLANNED}
+ builders_array = ['passed', 'failed', 'aborted', 'skipped', 'forgiven',
+ 'inflight', 'missing', 'planning']
+ previous_completed = set(['passed'])
+ expected_completed = set(['passed', 'failed', 'aborted', 'skipped',
+ 'forgiven'])
+ slaveStatus = manifest_version.SlaveStatus(status,
+ datetime.datetime.now(),
+ builders_array,
+ previous_completed)
+
+ self.assertEqual(sorted(slaveStatus.GetCompleted()),
+ sorted(['passed', 'failed', 'aborted', 'skipped',
+ 'forgiven']))
+ self.assertEqual(slaveStatus.previous_completed, expected_completed)
+
+ def testCompleted(self):
+ """Tests Completed returns proper bool."""
+ statusNotCompleted = {'build1': constants.BUILDER_STATUS_FAILED,
+ 'build2': constants.BUILDER_STATUS_INFLIGHT}
+ statusCompleted = {'build1': constants.BUILDER_STATUS_FAILED,
+ 'build2': constants.BUILDER_STATUS_PASSED}
+ builders_array = ['build1', 'build2']
+ slaveStatusNotCompleted = manifest_version.SlaveStatus(
+ statusNotCompleted, datetime.datetime.now(), builders_array, set())
+ slaveStatusCompleted = manifest_version.SlaveStatus(
+ statusCompleted, datetime.datetime.now(), builders_array, set())
+
+ self.assertFalse(slaveStatusNotCompleted.Completed())
+ self.assertTrue(slaveStatusCompleted.Completed())
+
+ def testShouldFailForBuilderStartTimeoutTrue(self):
+ """Tests that ShouldFailForBuilderStartTimeout says fail when it should."""
+ status = {'build1': constants.BUILDER_STATUS_FAILED}
+ start_time = datetime.datetime.now()
+ builders_array = ['build1', 'build2']
+ slaveStatus = manifest_version.SlaveStatus(status, start_time,
+ builders_array, set())
+ check_time = start_time + datetime.timedelta(
+ minutes=slaveStatus.BUILDER_START_TIMEOUT + 1)
+
+ self.assertTrue(slaveStatus.ShouldFailForBuilderStartTimeout(check_time))
+
+ def testShouldFailForBuilderStartTimeoutFalseTooEarly(self):
+ """Tests that ShouldFailForBuilderStartTimeout doesn't fail.
+
+ Make sure that we don't fail if there are missing builders but we're
+ checking before the timeout and the other builders have completed.
+ """
+ status = {'build1': constants.BUILDER_STATUS_FAILED}
+ start_time = datetime.datetime.now()
+ builders_array = ['build1', 'build2']
+ slaveStatus = manifest_version.SlaveStatus(status, start_time,
+ builders_array, set())
+
+ self.assertFalse(slaveStatus.ShouldFailForBuilderStartTimeout(start_time))
+
+ def testShouldFailForBuilderStartTimeoutFalseNotCompleted(self):
+ """Tests that ShouldFailForBuilderStartTimeout doesn't fail.
+
+ Make sure that we don't fail if there are missing builders and we're
+ checking after the timeout but the other builders haven't completed.
+ """
+ status = {'build1': constants.BUILDER_STATUS_INFLIGHT}
+ start_time = datetime.datetime.now()
+ builders_array = ['build1', 'build2']
+ slaveStatus = manifest_version.SlaveStatus(status, start_time,
+ builders_array, set())
+ check_time = start_time + datetime.timedelta(
+ minutes=slaveStatus.BUILDER_START_TIMEOUT + 1)
+
+ self.assertFalse(slaveStatus.ShouldFailForBuilderStartTimeout(check_time))
+
+ def testShouldWaitAllBuildersCompleted(self):
+ """Tests that ShouldWait says no waiting because all builders finished."""
+ status = {'build1': constants.BUILDER_STATUS_FAILED,
+ 'build2': constants.BUILDER_STATUS_PASSED}
+ builders_array = ['build1', 'build2']
+ slaveStatus = manifest_version.SlaveStatus(status, datetime.datetime.now(),
+ builders_array, set())
+
+ self.assertFalse(slaveStatus.ShouldWait())
+
+ def testShouldWaitMissingBuilder(self):
+ """Tests that ShouldWait says no waiting because a builder is missing."""
+ status = {'build1': constants.BUILDER_STATUS_FAILED}
+ builders_array = ['build1', 'build2']
+ start_time = datetime.datetime.now() - datetime.timedelta(hours=1)
+ slaveStatus = manifest_version.SlaveStatus(status, start_time,
+ builders_array, set())
+
+ self.assertFalse(slaveStatus.ShouldWait())
+
+ def testShouldWaitBuildersStillBuilding(self):
+ """Tests that ShouldWait says to wait because builders still building."""
+ status = {'build1': constants.BUILDER_STATUS_INFLIGHT,
+ 'build2': constants.BUILDER_STATUS_FAILED}
+ builders_array = ['build1', 'build2']
+ slaveStatus = manifest_version.SlaveStatus(status, datetime.datetime.now(),
+ builders_array, set())
+
+ self.assertTrue(slaveStatus.ShouldWait())
diff --git a/bootstrap/brillo b/cbuildbot/results_lib_unittest
index 72196ceea..72196ceea 120000
--- a/bootstrap/brillo
+++ b/cbuildbot/results_lib_unittest
diff --git a/cbuildbot/stages/stage_results_unittest.py b/cbuildbot/results_lib_unittest.py
index e971e4ff6..e971e4ff6 100644
--- a/cbuildbot/stages/stage_results_unittest.py
+++ b/cbuildbot/results_lib_unittest.py
diff --git a/cbuildbot/run_tests.py b/cbuildbot/run_tests.py
index 2c17905e3..0d0ae2811 100644
--- a/cbuildbot/run_tests.py
+++ b/cbuildbot/run_tests.py
@@ -13,6 +13,7 @@ You can add a .testignore file to a dir to disable scanning it.
from __future__ import print_function
import errno
+import json
import multiprocessing
import os
import signal
@@ -28,6 +29,7 @@ from chromite.lib import cros_logging as logging
from chromite.lib import gs
from chromite.lib import namespaces
from chromite.lib import osutils
+from chromite.lib import path_util
from chromite.lib import proctitle
from chromite.lib import timeout_util
@@ -43,6 +45,12 @@ SIGINT_TIMEOUT = 5
CTRL_C_TIMEOUT = SIGINT_TIMEOUT + 5
+# The cache file holds various timing information. This is used later on to
+# optimistically sort tests so the slowest ones run first. That way we don't
+# wait until all of the fast ones finish before we launch the slow ones.
+TIMING_CACHE_FILE = None
+
+
# Test has to run inside the chroot.
INSIDE = 'inside'
@@ -60,11 +68,13 @@ SPECIAL_TESTS = {
'cli/cros/cros_build_unittest': INSIDE,
'cli/cros/cros_chroot_unittest': INSIDE,
'cli/cros/cros_debug_unittest': INSIDE,
+ 'cli/cros/cros_payload_unittest': INSIDE,
'cli/cros/lint_unittest': INSIDE,
'cli/deploy_unittest': INSIDE,
'lib/alerts_unittest': INSIDE,
'lib/chroot_util_unittest': INSIDE,
'lib/filetype_unittest': INSIDE,
+ 'lib/paygen/paygen_payload_lib_unittest': INSIDE,
'lib/upgrade_table_unittest': INSIDE,
'mobmonitor/checkfile/manager_unittest': INSIDE,
'mobmonitor/scripts/mobmonitor_unittest': INSIDE,
@@ -127,6 +137,17 @@ def RunTest(test, cmd, tmpfile, finished, total):
msg = 'Finished'
func('%s [%i/%i] %s (%s)', msg, finished.value, total, test, delta)
+ # Save the timing for this test run for future usage.
+ seconds = delta.total_seconds()
+ try:
+ cache = json.load(open(TIMING_CACHE_FILE))
+ except (IOError, ValueError):
+ cache = {}
+ if test in cache:
+ seconds = (cache[test] + seconds) / 2
+ cache[test] = seconds
+ json.dump(cache, open(TIMING_CACHE_FILE, 'w'))
+
ret = cros_build_lib.TimedCommand(
cros_build_lib.RunCommand, cmd, capture_output=True, error_code_ok=True,
combine_stdout_stderr=True, debug_level=logging.DEBUG,
@@ -140,7 +161,69 @@ def RunTest(test, cmd, tmpfile, finished, total):
return ret.returncode
-def BuildTestSets(tests, chroot_available, network):
+def SortTests(tests, jobs=1, timing_cache_file=None):
+ """Interleave the slowest & fastest
+
+ Hopefully we can pipeline the overall process better by queueing the slowest
+ tests first while also using half the slots for fast tests. We don't need
+ the timing info to be exact, just ballpark.
+
+ Args:
+ tests: The list of tests to sort.
+ jobs: How many jobs will we run in parallel.
+ timing_cache_file: Where to read test timing info.
+
+ Returns:
+ The tests ordered for best execution timing (we hope).
+ """
+ if timing_cache_file is None:
+ timing_cache_file = TIMING_CACHE_FILE
+
+ # Usually |tests| will be a generator -- break it down.
+ tests = list(tests)
+
+ # If we have enough spare cpus to crunch the jobs, just do so.
+ if len(tests) <= jobs:
+ return tests
+
+ # Create a dict mapping tests to their timing information using the cache.
+ try:
+ with cros_build_lib.Open(timing_cache_file) as f:
+ cache = json.load(f)
+ except (IOError, ValueError):
+ cache = {}
+
+ # Sort the cached list of tests from slowest to fastest.
+ sorted_tests = [test for (test, _timing) in
+ sorted(cache.iteritems(), key=lambda x: x[1], reverse=True)]
+ # Then extract the tests from the cache list that we care about -- remember
+ # that the cache could be stale and contain tests that no longer exist, or
+ # the user only wants to run a subset of tests.
+ ret = []
+ for test in sorted_tests:
+ if test in tests:
+ ret.append(test)
+ tests.remove(test)
+ # Any tests not in the cache we just throw on the end. No real way to
+ # predict their speed ahead of time, and we'll get useful data when they
+ # run the test a second time.
+ ret += tests
+
+ # Now interleave the fast & slow tests so every other one mixes.
+ # On systems with fewer cores, this can help out in two ways:
+ # (1) Better utilization of resources when some slow tests are I/O or time
+ # bound, so the other cores can spawn/fork fast tests faster (generally).
+ # (2) If there is common code that is broken, we get quicker feedback if we
+ # churn through the fast tests.
+ # Worse case, this interleaving doesn't slow things down overall.
+ fast = ret[:int(round(len(ret) / 2.0)) - 1:-1]
+ slow = ret[:-len(fast)]
+ ret[::2] = slow
+ ret[1::2] = fast
+ return ret
+
+
+def BuildTestSets(tests, chroot_available, network, jobs=1):
"""Build the tests to execute.
Take care of special test handling like whether it needs to be inside or
@@ -150,12 +233,13 @@ def BuildTestSets(tests, chroot_available, network):
tests: List of tests to execute.
chroot_available: Whether we can execute tests inside the sdk.
network: Whether to execute network tests.
+ jobs: How many jobs will we run in parallel.
Returns:
List of tests to execute and their full command line.
"""
testsets = []
- for test in tests:
+ for test in SortTests(tests, jobs=jobs):
cmd = [test]
# See if this test requires special consideration.
@@ -223,7 +307,7 @@ def RunTests(tests, jobs=1, chroot_available=True, network=False, dryrun=False,
# Launch all the tests!
try:
# Build up the testsets.
- testsets = BuildTestSets(tests, chroot_available, network)
+ testsets = BuildTestSets(tests, chroot_available, network, jobs=jobs)
# Fork each test and add it to the list.
for test, cmd, tmpfile in testsets:
@@ -435,6 +519,10 @@ def main(argv):
if opts.quick:
SPECIAL_TESTS.update(SLOW_TESTS)
+ global TIMING_CACHE_FILE # pylint: disable=global-statement
+ TIMING_CACHE_FILE = os.path.join(
+ path_util.GetCacheDir(), constants.COMMON_CACHE, 'run_tests.cache.json')
+
jobs = opts.jobs or multiprocessing.cpu_count()
with cros_build_lib.ContextManagerStack() as stack:
diff --git a/cbuildbot/run_tests_unittest.py b/cbuildbot/run_tests_unittest.py
index 443299946..1323909ea 100644
--- a/cbuildbot/run_tests_unittest.py
+++ b/cbuildbot/run_tests_unittest.py
@@ -56,6 +56,38 @@ class FindTestsTest(cros_test_lib.TempDirTestCase):
self.assertEqual(list(found), [])
+class SortTest(cros_test_lib.TempDirTestCase):
+ """Tests for the SortTests() func"""
+
+ def SortTests(self, tests, **kwargs):
+ """Helper to set cache file to a local temp one"""
+ kwargs['timing_cache_file'] = os.path.join(self.tempdir, 'cache.json')
+ return run_tests.SortTests(tests, **kwargs)
+
+ def testEmpty(self):
+ """Verify handling of empty test lists"""
+ self.SortTests([])
+ self.SortTests([], jobs=100)
+
+ def testSmallSet(self):
+ """Do nothing when number of tests is lower than number of jobs."""
+ tests = ['small', 'test', 'list', 'is', 'ignored']
+ ret = self.SortTests(tests, jobs=100)
+ self.assertEqual(tests, ret)
+
+ def testOddSet(self):
+ """Verify we can sort odd number of tests."""
+ tests = ['1', '2', '3']
+ ret = self.SortTests(tests, jobs=1)
+ self.assertEqual(set(tests), set(ret))
+
+ def testEvenSet(self):
+ """Verify we can sort even number of tests."""
+ tests = ['1', '2', '3', '4']
+ ret = self.SortTests(tests, jobs=1)
+ self.assertEqual(set(tests), set(ret))
+
+
class MainTest(cros_test_lib.MockOutputTestCase):
"""Tests for the main() func"""
diff --git a/cbuildbot/stages/artifact_stages.py b/cbuildbot/stages/artifact_stages.py
index 0c5803f5e..e3f491535 100644
--- a/cbuildbot/stages/artifact_stages.py
+++ b/cbuildbot/stages/artifact_stages.py
@@ -179,7 +179,6 @@ class ArchiveStage(generic_stages.BoardSpecificBuilderStage,
# \- ArchiveStandaloneArtifact
# \- ArchiveZipFiles
# \- ArchiveHWQual
- # \- ArchiveGceTarballs
# \- PushImage (blocks on BuildAndArchiveAllImages)
# \- ArchiveManifest
# \- ArchiveStrippedPackages
@@ -237,30 +236,6 @@ class ArchiveStage(generic_stages.BoardSpecificBuilderStage,
parallel.RunTasksInProcessPool(ArchiveStandaloneArtifact,
[[x] for x in self.artifacts])
- def ArchiveGceTarballs():
- """Creates .tar.gz files that can be converted to GCE images.
-
- These files will be uploaded to GCS buckets, where they can be
- used as input to the "gcloud compute images create" command.
- This will convert them into images that can be used to create
- GCE VM instances.
- """
- image_bins = []
- if 'base' in config['images']:
- image_bins.append(constants.IMAGE_TYPE_TO_NAME['base'])
- if 'test' in config['images']:
- image_bins.append(constants.IMAGE_TYPE_TO_NAME['test'])
-
- for image_bin in image_bins:
- if not os.path.exists(os.path.join(image_dir, image_bin)):
- logging.warning('Missing image file skipped: %s', image_bin)
- continue
- output_file = commands.BuildGceTarball(
- archive_path, image_dir, image_bin)
- self._release_upload_queue.put([output_file])
-
- self.board_runattrs.SetParallel('gce_tarball_generated', True)
-
def ArchiveZipFiles():
"""Build and archive zip files.
@@ -315,12 +290,16 @@ class ArchiveStage(generic_stages.BoardSpecificBuilderStage,
# For recovery image to be generated correctly, BuildRecoveryImage must
# run before BuildAndArchiveFactoryImages.
if 'recovery' in config.images:
- assert self.IsArchivedFile(constants.BASE_IMAGE_BIN)
+ assert os.path.isfile(os.path.join(image_dir, constants.BASE_IMAGE_BIN))
commands.BuildRecoveryImage(buildroot, board, image_dir, extra_env)
self._recovery_image_status_queue.put(True)
- # Re-generate the artifacts list so we include the newly created
- # recovery image.
- self.LoadArtifactsList(self._current_board, image_dir)
+ recovery_image = constants.RECOVERY_IMAGE_BIN
+ if not self.IsArchivedFile(recovery_image):
+ info = {'paths': [recovery_image],
+ 'input': [recovery_image],
+ 'archive': 'tar',
+ 'compress': 'xz'}
+ self.artifacts.append(info)
else:
self._recovery_image_status_queue.put(False)
@@ -331,8 +310,6 @@ class ArchiveStage(generic_stages.BoardSpecificBuilderStage,
ArchiveStandaloneArtifacts,
ArchiveZipFiles,
]
- if config['upload_gce_images']:
- steps.append(ArchiveGceTarballs)
parallel.RunParallelSteps(steps)
def ArchiveImageScripts():
@@ -358,10 +335,8 @@ class ArchiveStage(generic_stages.BoardSpecificBuilderStage,
# TODO: When we support branches fully, the friendly name of the branch
# needs to be used with PushImages
sign_types = []
- if config['name'].endswith('-%s' % config_lib.CONFIG_TYPE_FIRMWARE):
- sign_types += ['firmware']
- if config['name'].endswith('-%s' % config_lib.CONFIG_TYPE_FACTORY):
- sign_types += ['factory']
+ if config['sign_types']:
+ sign_types = config['sign_types']
urls = commands.PushImages(
board=board,
archive_url=upload_url,
diff --git a/cbuildbot/stages/build_stages.py b/cbuildbot/stages/build_stages.py
index 44b721efa..9533887b5 100644
--- a/cbuildbot/stages/build_stages.py
+++ b/cbuildbot/stages/build_stages.py
@@ -205,17 +205,14 @@ class SetupBoardStage(generic_stages.BoardSpecificBuilderStage, InitSDKStage):
self._build_root, toolchain_boards=[self._current_board],
usepkg=usepkg_toolchain)
- # Only update the board if we need to do so.
- chroot_path = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR)
- board_path = os.path.join(chroot_path, 'build', self._current_board)
- if not os.path.isdir(board_path) or self._run.config.board_replace:
- usepkg = self._run.config.usepkg_build_packages
- commands.SetupBoard(
- self._build_root, board=self._current_board, usepkg=usepkg,
- chrome_binhost_only=self._run.config.chrome_binhost_only,
- force=self._run.config.board_replace,
- extra_env=self._portage_extra_env, chroot_upgrade=False,
- profile=self._run.options.profile or self._run.config.profile)
+ # Always update the board.
+ usepkg = self._run.config.usepkg_build_packages
+ commands.SetupBoard(
+ self._build_root, board=self._current_board, usepkg=usepkg,
+ chrome_binhost_only=self._run.config.chrome_binhost_only,
+ force=self._run.config.board_replace,
+ extra_env=self._portage_extra_env, chroot_upgrade=False,
+ profile=self._run.options.profile or self._run.config.profile)
class BuildPackagesStage(generic_stages.BoardSpecificBuilderStage,
@@ -379,7 +376,8 @@ class BuildImageStage(BuildPackagesStage):
self.board_runattrs.SetParallel('images_generated', True)
parallel.RunParallelSteps(
- [self._BuildVMImage, lambda: self._GenerateAuZip(cbuildbot_image_link)])
+ [self._BuildVMImage, lambda: self._GenerateAuZip(cbuildbot_image_link),
+ self._BuildGceTarballs])
def _BuildVMImage(self):
if self._run.config.vm_tests and not self._afdo_generate_min:
@@ -395,6 +393,28 @@ class BuildImageStage(BuildPackagesStage):
image_dir,
extra_env=self._portage_extra_env)
+ def _BuildGceTarballs(self):
+ """Creates .tar.gz files that can be converted to GCE images.
+
+ These files will be used by VMTestStage for tests on GCE. They will also be
+ be uploaded to GCS buckets, where they can be used as input to the "gcloud
+ compute images create" command. This will convert them into images that can
+ be used to create GCE VM instances.
+ """
+ if self._run.config.upload_gce_images:
+ image_bins = []
+ if 'base' in self._run.config['images']:
+ image_bins.append(constants.IMAGE_TYPE_TO_NAME['base'])
+ if 'test' in self._run.config['images']:
+ image_bins.append(constants.IMAGE_TYPE_TO_NAME['test'])
+
+ image_dir = self.GetImageDirSymlink('latest')
+ for image_bin in image_bins:
+ if os.path.exists(os.path.join(image_dir, image_bin)):
+ commands.BuildGceTarball(image_dir, image_dir, image_bin)
+ else:
+ logging.warning('Missing image file skipped: %s', image_bin)
+
def _HandleStageException(self, exc_info):
"""Tell other stages to not wait on us if we die for some reason."""
self.board_runattrs.SetParallelDefault('images_generated', False)
diff --git a/cbuildbot/stages/build_stages_unittest.py b/cbuildbot/stages/build_stages_unittest.py
index 2f1c4a15f..ff239b294 100644
--- a/cbuildbot/stages/build_stages_unittest.py
+++ b/cbuildbot/stages/build_stages_unittest.py
@@ -95,7 +95,7 @@ class SetupBoardTest(generic_stages_unittest.RunCommandAbstractStageTestCase):
self._Run(dir_exists)
self.assertCommandContains(['./update_chroot'])
cmd = ['./setup_board', '--board=%s' % self._current_board, '--nousepkg']
- self.assertCommandContains(cmd, expected=not dir_exists)
+ self.assertCommandContains(cmd)
cmd = ['./setup_board', '--skip_chroot_upgrade']
self.assertCommandContains(cmd)
@@ -118,30 +118,12 @@ class SetupBoardTest(generic_stages_unittest.RunCommandAbstractStageTestCase):
self._run.options.latest_toolchain)
self.assertCommandContains(['./update_chroot', '--nousepkg'],
expected=update_nousepkg)
- run_setup_board = not dir_exists or self._run.config.board_replace
- self.assertCommandContains(['./setup_board'], expected=run_setup_board)
+ self.assertCommandContains(['./setup_board'])
cmd = ['./setup_board', '--skip_chroot_upgrade']
- self.assertCommandContains(cmd, expected=run_setup_board)
+ self.assertCommandContains(cmd)
cmd = ['./setup_board', '--nousepkg']
self.assertCommandContains(
- cmd,
- expected=run_setup_board and not self._run.config.usepkg_build_packages)
-
- def testBinBuildWithBoard(self):
- """Tests whether we don't create the board when it's there."""
- self._PrepareBin()
- self._RunBin(dir_exists=True)
-
- def testBinBuildWithBoardReplace(self):
- """Tests whether we don't create the board when it's there."""
- self._PrepareBin()
- self._run.config.board_replace = True
- self._RunBin(dir_exists=True)
-
- def testBinBuildWithMissingBoard(self):
- """Tests whether we create the board when it's missing."""
- self._PrepareBin()
- self._RunBin(dir_exists=False)
+ cmd, not self._run.config.usepkg_build_packages)
def testBinBuildWithLatestToolchain(self):
"""Tests whether we use --nousepkg for creating the board."""
diff --git a/cbuildbot/stages/chrome_stages_unittest.py b/cbuildbot/stages/chrome_stages_unittest.py
index b1504a0d4..48bb16f7d 100644
--- a/cbuildbot/stages/chrome_stages_unittest.py
+++ b/cbuildbot/stages/chrome_stages_unittest.py
@@ -112,7 +112,7 @@ class SyncChromeStageTest(generic_stages_unittest.AbstractStageTestCase,
cros_build_lib_unittest.RunCommandTestCase):
"""Tests for SyncChromeStage."""
- # pylint: disable-msg=protected-access
+ # pylint: disable=protected-access
def setUp(self):
self._Prepare()
self.PatchObject(cbuildbot_run._BuilderRunBase, 'DetermineChromeVersion',
@@ -126,4 +126,3 @@ class SyncChromeStageTest(generic_stages_unittest.AbstractStageTestCase,
"""Basic syntax sanity test."""
stage = self.ConstructStage()
stage.PerformStage()
-
diff --git a/cbuildbot/stages/completion_stages.py b/cbuildbot/stages/completion_stages.py
index 2c6b08d34..e1467d3f0 100644
--- a/cbuildbot/stages/completion_stages.py
+++ b/cbuildbot/stages/completion_stages.py
@@ -18,9 +18,7 @@ from chromite.cbuildbot.stages import generic_stages
from chromite.cbuildbot.stages import sync_stages
from chromite.lib import clactions
from chromite.lib import cros_logging as logging
-from chromite.lib import git
from chromite.lib import patch as cros_patch
-from chromite.lib import portage_util
def GetBuilderSuccessMap(builder_run, overall_success):
@@ -182,8 +180,8 @@ class MasterSlaveSyncCompletionStage(ManifestVersionedSyncCompletionStage):
timeout = 3 * 60
manager = self._run.attrs.manifest_manager
- if sync_stages.MasterSlaveLKGMSyncStage.sub_manager:
- manager = sync_stages.MasterSlaveLKGMSyncStage.sub_manager
+ if sync_stages.MasterSlaveLKGMSyncStage.external_manager:
+ manager = sync_stages.MasterSlaveLKGMSyncStage.external_manager
slave_statuses.update(manager.GetBuildersStatus(
self._run.attrs.metadata.GetValue('build_id'),
builder_names,
@@ -223,8 +221,8 @@ class MasterSlaveSyncCompletionStage(ManifestVersionedSyncCompletionStage):
self._run.manifest_branch == 'master' and
self._run.config.build_type != constants.CHROME_PFQ_TYPE):
self._run.attrs.manifest_manager.PromoteCandidate()
- if sync_stages.MasterSlaveLKGMSyncStage.sub_manager:
- sync_stages.MasterSlaveLKGMSyncStage.sub_manager.PromoteCandidate()
+ if sync_stages.MasterSlaveLKGMSyncStage.external_manager:
+ sync_stages.MasterSlaveLKGMSyncStage.external_manager.PromoteCandidate()
def HandleFailure(self, failing, inflight, no_stat):
"""Handle a build failure.
@@ -485,11 +483,6 @@ class CommitQueueCompletionStage(MasterSlaveSyncCompletionStage):
def HandleSuccess(self):
if self._run.config.master:
self.sync_stage.pool.SubmitPool(reason=constants.STRATEGY_CQ_SUCCESS)
- # After submitting the pool, update the commit hashes for uprevved
- # ebuilds.
- manifest = git.ManifestCheckout.Cached(self._build_root)
- portage_util.EBuild.UpdateCommitHashesForChanges(
- self.sync_stage.pool.changes, self._build_root, manifest)
if config_lib.IsPFQType(self._run.config.build_type):
super(CommitQueueCompletionStage, self).HandleSuccess()
@@ -830,10 +823,17 @@ class PublishUprevChangesStage(generic_stages.BuilderStage):
overlays, push_overlays = self._ExtractOverlays()
assert push_overlays, 'push_overlays must be set to run this stage'
- # If the build failed, we don't want to push our local changes, because
- # they might include some CLs that failed. Instead, clean up our local
- # changes and do a fresh uprev.
- if not self.success:
+ # If we're a commit queue, we should clean out our local changes, resync,
+ # and reapply our uprevs. This is necessary so that 1) we are sure to point
+ # at the remote SHA1s, not our local SHA1s; 2) we can avoid doing a
+ # rebase; 3) in the case of failure, we don't submit the changes that were
+ # committed locally.
+ #
+ # If we're not a commit queue and the build succeeded, we can skip the
+ # cleanup here. This is a cheap trick so that the Chrome PFQ pushes its
+ # earlier uprev from the SyncChrome stage (it would be a bit tricky to
+ # replicate the uprev here, so we'll leave it alone).
+ if config_lib.IsCQType(self._run.config.build_type) or not self.success:
# Clean up our root and sync down the latest changes that were
# submitted.
commands.BuildRootGitCleanup(self._build_root)
diff --git a/cbuildbot/stages/report_stages.py b/cbuildbot/stages/report_stages.py
index c89a38fde..badf6c2b2 100644
--- a/cbuildbot/stages/report_stages.py
+++ b/cbuildbot/stages/report_stages.py
@@ -47,7 +47,7 @@ def WriteBasicMetadata(builder_run):
In particular, this method does not write any metadata values that depend
on the builder config, as the config may be modified by patches that are
- applied before the final reexectuion.
+ applied before the final reexectuion. (exception: the config's name itself)
This method is safe to run more than once (for instance, once per cbuildbot
execution) because it will write the same data each time.
@@ -135,6 +135,15 @@ class BuildStartStage(generic_stages.BuilderStage):
self._run.config['doc'])
WriteBasicMetadata(self._run)
+
+ # This is a heuristic value for |important|, since patches that get applied
+ # later in the build might change the config. We write it now anyway,
+ # because in case the build fails before Sync, it is better to have this
+ # heuristic value than None. In BuildReexectuionFinishedStage, we re-write
+ # the definitive value.
+ self._run.attrs.metadata.UpdateWithDict(
+ {'important': self._run.config['important']})
+
d = self._run.attrs.metadata.GetDict()
# BuildStartStage should only run once per build. But just in case it
@@ -165,7 +174,8 @@ class BuildStartStage(generic_stages.BuilderStage):
build_config=d['bot-config'],
bot_hostname=d['bot-hostname'],
master_build_id=d['master_build_id'],
- timeout_seconds=self._GetBuildTimeoutSeconds())
+ timeout_seconds=self._GetBuildTimeoutSeconds(),
+ important=d['important'])
self._run.attrs.metadata.UpdateWithDict({'build_id': build_id,
'db_type': db_type})
logging.info('Inserted build_id %s into cidb database type %s.',
@@ -202,6 +212,51 @@ class BuildStartStage(generic_stages.BuilderStage):
'%s.' % (metadata_dict['db_type'], db_type))
+class SlaveFailureSummaryStage(generic_stages.BuilderStage):
+ """Stage which summarizes and links to the failures of slave builds."""
+
+ @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
+ def PerformStage(self):
+ if not self._run.config.master:
+ logging.info('This stage is only meaningful for master builds. '
+ 'Doing nothing.')
+ return
+
+ build_id, db = self._run.GetCIDBHandle()
+
+ if not db:
+ logging.info('No cidb connection for this build. '
+ 'Doing nothing.')
+ return
+
+ slave_failures = db.GetSlaveFailures(build_id)
+ failures_by_build = cros_build_lib.GroupByKey(slave_failures, 'build_id')
+ for build_id, build_failures in sorted(failures_by_build.items()):
+ failures_by_stage = cros_build_lib.GroupByKey(build_failures,
+ 'build_stage_id')
+ # Surface a link to each slave stage that failed, in stage_id sorted
+ # order.
+ for stage_id in sorted(failures_by_stage):
+ failure = failures_by_stage[stage_id][0]
+ # Ignore failures that did not cause their enclosing stage to fail.
+ # Ignore slave builds that are still inflight, because some stage logs
+ # might not have been printed to buildbot yet.
+ # TODO(akeshet) revisit this approach, if we seem to be suppressing
+ # useful information as a result of it.
+ if (failure['stage_status'] != constants.BUILDER_STATUS_FAILED or
+ failure['build_status'] == constants.BUILDER_STATUS_INFLIGHT):
+ continue
+ waterfall_url = constants.WATERFALL_TO_DASHBOARD[failure['waterfall']]
+ slave_stage_url = tree_status.ConstructDashboardURL(
+ waterfall_url,
+ failure['builder_name'],
+ failure['build_number'],
+ failure['stage_name'])
+ logging.PrintBuildbotLink('%s %s' % (failure['build_config'],
+ failure['stage_name']),
+ slave_stage_url)
+
+
class BuildReexecutionFinishedStage(generic_stages.BuilderStage,
generic_stages.ArchivingStageMixin):
"""The first stage to run after the final cbuildbot reexecution.
@@ -271,6 +326,7 @@ class BuildReexecutionFinishedStage(generic_stages.BuilderStage,
'boards': config['boards'],
'child-configs': child_configs,
'build_type': config['build_type'],
+ 'important': config['important'],
# Data for the toolchain used.
'sdk-version': sdk_verinfo.get('SDK_LATEST_VERSION', '<unknown>'),
@@ -438,17 +494,6 @@ class ReportStage(generic_stages.BuilderStage,
tree_status.SendHealthAlert(self._run, title, '\n\n'.join(body),
extra_fields=extra_fields)
- def _UploadMetadataForRun(self, final_status):
- """Upload metadata.json for this entire run.
-
- Args:
- final_status: Final status string for this run.
- """
- self._run.attrs.metadata.UpdateWithDict(
- self.GetReportMetadata(final_status=final_status,
- completion_instance=self._completion_instance))
- self.UploadMetadata()
-
def _UploadArchiveIndex(self, builder_run):
"""Upload an HTML index for the artifacts at remote archive location.
@@ -559,7 +604,7 @@ class ReportStage(generic_stages.BuilderStage,
# Upload metadata, and update the pass/fail streak counter for the main
# run only. These aren't needed for the child builder runs.
- self._UploadMetadataForRun(final_status)
+ self.UploadMetadata()
self._UpdateRunStreak(self._run, final_status)
# Alert if the Pre-CQ has infra failures.
@@ -615,6 +660,11 @@ class ReportStage(generic_stages.BuilderStage,
# ArchiveResults() depends the existence of this attr.
self._run.attrs.release_tag = None
+ # Set up our report metadata.
+ self._run.attrs.metadata.UpdateWithDict(
+ self.GetReportMetadata(final_status=final_status,
+ completion_instance=self._completion_instance))
+
# Some operations can only be performed if a valid version is available.
try:
self._run.GetVersionInfo()
@@ -626,7 +676,6 @@ class ReportStage(generic_stages.BuilderStage,
archive_urls = ''
metadata_url = ''
-
results_lib.Results.Report(
sys.stdout, archive_urls=archive_urls,
current_version=(self._run.attrs.release_tag or ''))
diff --git a/cbuildbot/stages/report_stages_unittest.py b/cbuildbot/stages/report_stages_unittest.py
index c6a4609b0..afbeda7d7 100644
--- a/cbuildbot/stages/report_stages_unittest.py
+++ b/cbuildbot/stages/report_stages_unittest.py
@@ -23,6 +23,7 @@ from chromite.cbuildbot.stages import report_stages
from chromite.lib import alerts
from chromite.lib import cidb
from chromite.lib import cros_build_lib
+from chromite.lib import cros_logging as logging
from chromite.lib import fake_cidb
from chromite.lib import gs_unittest
from chromite.lib import osutils
@@ -83,6 +84,43 @@ class BuildReexecutionStageTest(generic_stages_unittest.AbstractStageTestCase):
def ConstructStage(self):
return report_stages.BuildReexecutionFinishedStage(self._run)
+
+class SlaveFailureSummaryStageTest(
+ generic_stages_unittest.AbstractStageTestCase):
+ """Tests that SlaveFailureSummaryStage behaves as expected."""
+
+ def setUp(self):
+ self.db = mock.MagicMock()
+ cidb.CIDBConnectionFactory.SetupMockCidb(self.db)
+ self._Prepare(build_id=1)
+
+ def _Prepare(self, **kwargs):
+ """Prepare stage with config['master']=True."""
+ super(SlaveFailureSummaryStageTest, self)._Prepare(**kwargs)
+ self._run.config['master'] = True
+
+ def ConstructStage(self):
+ return report_stages.SlaveFailureSummaryStage(self._run)
+
+ def testPerformStage(self):
+ """Tests that stage runs without syntax errors."""
+ fake_failure = {
+ 'build_id': 10,
+ 'build_stage_id': 11,
+ 'waterfall': constants.WATERFALL_EXTERNAL,
+ 'builder_name': 'builder_name',
+ 'build_number': 12,
+ 'build_config': 'build-config',
+ 'stage_name': 'FailingStage',
+ 'stage_status': constants.BUILDER_STATUS_FAILED,
+ 'build_status': constants.BUILDER_STATUS_FAILED,
+ }
+ self.PatchObject(self.db, 'GetSlaveFailures', return_value=[fake_failure])
+ self.PatchObject(logging, 'PrintBuildbotLink')
+ self.RunStage()
+ self.assertEqual(logging.PrintBuildbotLink.call_count, 1)
+
+
class BuildStartStageTest(generic_stages_unittest.AbstractStageTestCase):
"""Tests that BuildStartStage behaves as expected."""
diff --git a/cbuildbot/stages/sdk_stages_unittest.py b/cbuildbot/stages/sdk_stages_unittest.py
index 96e3d111c..c933b869e 100644
--- a/cbuildbot/stages/sdk_stages_unittest.py
+++ b/cbuildbot/stages/sdk_stages_unittest.py
@@ -8,6 +8,7 @@ from __future__ import print_function
import json
import os
+import unittest
from chromite.cbuildbot import commands
from chromite.cbuildbot import constants
@@ -221,6 +222,8 @@ class SDKPackageToolchainOverlaysStageTest(
def ConstructStage(self):
return sdk_stages.SDKPackageToolchainOverlaysStage(self._run)
+ # TODO(akeshet): determine why this test is flaky
+ @unittest.skip("Skip flaky test.")
def testTarballCreation(self):
"""Tests that tarballs are created for all board toolchains."""
self._Prepare('chromiumos-sdk')
diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py
index dc2df54d3..8eb6ce77f 100644
--- a/cbuildbot/stages/sync_stages.py
+++ b/cbuildbot/stages/sync_stages.py
@@ -733,10 +733,9 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage):
This stage uses an LKGM manifest manager that handles LKGM
candidates and their states.
"""
-
- # TODO(mtennant): Turn this into self._run.attrs.sub_manager or similar.
- # An instance of lkgm_manager.LKGMManager for slave builds.
- sub_manager = None
+ # If we are using an internal manifest, but need to be able to create an
+ # external manifest, we create a second manager for that manifest.
+ external_manager = None
MAX_BUILD_HISTORY_LENGTH = 10
MilestoneVersion = collections.namedtuple(
'MilestoneVersion', ['milestone', 'platform'])
@@ -775,12 +774,13 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage):
self.RegisterManifestManager(self._GetInitializedManager(self.internal))
if self._run.config.master and self._GetSlaveConfigs():
assert self.internal, 'Unified masters must use an internal checkout.'
- MasterSlaveLKGMSyncStage.sub_manager = self._GetInitializedManager(False)
+ MasterSlaveLKGMSyncStage.external_manager = \
+ self._GetInitializedManager(False)
def ForceVersion(self, version):
manifest = super(MasterSlaveLKGMSyncStage, self).ForceVersion(version)
- if MasterSlaveLKGMSyncStage.sub_manager:
- MasterSlaveLKGMSyncStage.sub_manager.BootstrapFromVersion(version)
+ if MasterSlaveLKGMSyncStage.external_manager:
+ MasterSlaveLKGMSyncStage.external_manager.BootstrapFromVersion(version)
return manifest
@@ -806,8 +806,8 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage):
manifest = self.manifest_manager.CreateNewCandidate(
chrome_version=self._chrome_version,
build_id=build_id)
- if MasterSlaveLKGMSyncStage.sub_manager:
- MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(
+ if MasterSlaveLKGMSyncStage.external_manager:
+ MasterSlaveLKGMSyncStage.external_manager.CreateFromManifest(
manifest, build_id=build_id)
return manifest
@@ -917,11 +917,14 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage):
changes_to_test = []
_, db = self._run.GetCIDBHandle()
- actions_for_changes = db.GetActionsForChanges(changes)
- for change in changes:
- status = clactions.GetCLPreCQStatus(change, actions_for_changes)
- if status == constants.CL_STATUS_PASSED:
- changes_to_test.append(change)
+ if db:
+ actions_for_changes = db.GetActionsForChanges(changes)
+ for change in changes:
+ status = clactions.GetCLPreCQStatus(change, actions_for_changes)
+ if status == constants.CL_STATUS_PASSED:
+ changes_to_test.append(change)
+ else:
+ logging.warning("DB not available, unable to filter for PreCQ passed.")
# Allow Commit-Ready=+2 changes to bypass the Pre-CQ, if there are no other
# changes.
@@ -978,7 +981,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage):
if self._run.options.cq_gerrit_override:
query = (self._run.options.cq_gerrit_override, None)
- self.pool = pool = validation_pool.ValidationPool.AcquirePool(
+ self.pool = validation_pool.ValidationPool.AcquirePool(
self._run.config.overlays, self.repo,
self._run.buildnumber, self._run.GetBuilderName(),
query,
@@ -1001,10 +1004,10 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage):
db.ExtendDeadline(build_id, timeout)
logging.info('Creating new candidate manifest.')
- manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool,
- build_id=build_id)
- if MasterSlaveLKGMSyncStage.sub_manager:
- MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(
+ manifest = self.manifest_manager.CreateNewCandidate(
+ validation_pool=self.pool, build_id=build_id)
+ if MasterSlaveLKGMSyncStage.external_manager:
+ MasterSlaveLKGMSyncStage.external_manager.CreateFromManifest(
manifest, build_id=build_id)
return manifest
diff --git a/cbuildbot/stages/test_stages.py b/cbuildbot/stages/test_stages.py
index 0ee4ad874..cf48d01ed 100644
--- a/cbuildbot/stages/test_stages.py
+++ b/cbuildbot/stages/test_stages.py
@@ -158,18 +158,6 @@ class VMTestStage(generic_stages.BoardSpecificBuilderStage,
prefix = ''
self.PrintDownloadLink(filename, prefix)
- def _WaitForGceTarball(self, image_path):
- """Waits until GCE tarball is available."""
- gce_tar_generated = self.GetParallel('gce_tarball_generated')
- if not gce_tar_generated:
- return
- # Still need to check its availability as artifacts are uploaded in the
- # background.
- gs_ctx = gs.GSContext()
- logging.info('Waiting for GCE tarball to be uploaded at %s.' % image_path)
- gs_ctx.WaitForGsPaths([image_path], self.CHECK_GCS_TIMEOUT,
- self.CHECK_GCS_PERIOD)
-
def _RunTest(self, test_type, test_results_dir):
"""Run a VM test.
@@ -184,11 +172,8 @@ class VMTestStage(generic_stages.BoardSpecificBuilderStage,
self._build_root, self._current_board, self.GetImageDirSymlink())
else:
if test_type == constants.GCE_VM_TEST_TYPE:
- # If tests are to run on GCE, use the uploaded tar ball.
- image_path = ('%s/%s' % (self.download_url.rstrip('/'),
- constants.TEST_IMAGE_GCE_TAR))
-
- self._WaitForGceTarball(image_path)
+ image_path = os.path.join(self.GetImageDirSymlink(),
+ constants.TEST_IMAGE_GCE_TAR)
else:
image_path = os.path.join(self.GetImageDirSymlink(),
constants.TEST_IMAGE_BIN)
@@ -203,8 +188,7 @@ class VMTestStage(generic_stages.BoardSpecificBuilderStage,
commands.RunTestSuite(self._build_root,
self._current_board,
image_path,
- os.path.join(test_results_dir,
- 'test_harness'),
+ os.path.join(test_results_dir, 'test_harness'),
test_type=test_type,
whitelist_chrome_crashes=self._chrome_rev is None,
archive_dir=self.bot_archive_root,
@@ -306,6 +290,9 @@ class HWTestStage(generic_stages.BoardSpecificBuilderStage,
current_board_dict = per_board_dict.get(self._current_board)
if current_board_dict:
subsystems = set(current_board_dict.get('subsystems_to_test', []))
+ # 'subsystem:all' indicates to skip the subsystem logic
+ if 'all' in subsystems:
+ subsystems = None
else:
subsystems = None
@@ -332,7 +319,8 @@ class AUTestStage(HWTestStage):
# Wait for UploadHWTestArtifacts to generate the payloads.
if not self.GetParallel('delta_payloads_generated',
pretty_name='delta payloads'):
- logging.PrintBuildbotStepWarnings('missing delta payloads')
+ logging.PrintBuildbotStepText('Missing delta payloads.')
+ logging.PrintBuildbotStepWarnings()
logging.warning('Cannot run HWTest because UploadTestArtifacts failed. '
'See UploadTestArtifacts for details.')
return
diff --git a/cbuildbot/stages/test_stages_unittest.py b/cbuildbot/stages/test_stages_unittest.py
index 02fea21cc..03fd5e9bb 100644
--- a/cbuildbot/stages/test_stages_unittest.py
+++ b/cbuildbot/stages/test_stages_unittest.py
@@ -17,14 +17,12 @@ from chromite.cbuildbot import failures_lib
from chromite.cbuildbot import swarming_lib
from chromite.cbuildbot import topology
from chromite.cbuildbot.stages import artifact_stages
-from chromite.cbuildbot.stages import generic_stages
from chromite.cbuildbot.stages import generic_stages_unittest
from chromite.cbuildbot.stages import test_stages
from chromite.lib import cgroups
from chromite.lib import cros_build_lib_unittest
from chromite.lib import cros_test_lib
from chromite.lib import cros_logging as logging
-from chromite.lib import gs
from chromite.lib import osutils
from chromite.lib import path_util
from chromite.lib import timeout_util
@@ -79,32 +77,19 @@ class VMTestStageTest(generic_stages_unittest.AbstractStageTestCase,
def testGceTests(self):
"""Tests if GCE_VM_TEST_TYPE tests are run on GCE."""
self._run.config['vm_tests'] = [constants.GCE_VM_TEST_TYPE]
- gce_path = constants.TEST_IMAGE_GCE_TAR
- board_runattrs = self._run.GetBoardRunAttrs(self._current_board)
+ gce_tarball = constants.TEST_IMAGE_GCE_TAR
# pylint: disable=unused-argument
def _MockRunTestSuite(buildroot, board, image_path, results_dir, test_type,
*args, **kwargs):
- self.assertEndsWith(image_path, gce_path)
+ self.assertEndsWith(image_path, gce_tarball)
self.assertEqual(test_type, constants.GCE_VM_TEST_TYPE)
# pylint: enable=unused-argument
- def _MockWaitForGsPaths(_, paths, *_args, **_kwargs):
- self.assertEndsWith(paths[0], gce_path)
-
- self.PatchObject(generic_stages.BoardSpecificBuilderStage, 'GetParallel',
- autospec=True)
- self.PatchObject(gs.GSContext, 'WaitForGsPaths',
- side_effect=_MockWaitForGsPaths, autospec=True)
commands.RunTestSuite.side_effect = _MockRunTestSuite
- board_runattrs.SetParallel('gce_tarball_generated', True)
self.RunStage()
- generic_stages.BoardSpecificBuilderStage.GetParallel.assert_any_call(
- mock.ANY, 'gce_tarball_generated')
- self.assertTrue(gs.GSContext.WaitForGsPaths.called and
- gs.GSContext.WaitForGsPaths.call_count == 1)
self.assertTrue(commands.RunTestSuite.called and
commands.RunTestSuite.call_count == 1)
diff --git a/cbuildbot/swarming_lib.py b/cbuildbot/swarming_lib.py
index 435a6a9e7..673b11af6 100644
--- a/cbuildbot/swarming_lib.py
+++ b/cbuildbot/swarming_lib.py
@@ -11,7 +11,9 @@ import json
import os
from chromite.lib import cros_build_lib
+from chromite.lib import cros_logging as logging
from chromite.lib import osutils
+from chromite.lib import retry_util
# Location of swarming_client.py that is used to send swarming requests
_DIR_NAME = os.path.dirname(os.path.abspath(__file__))
@@ -19,6 +21,8 @@ _SWARMING_PROXY_CLIENT = os.path.abspath(os.path.join(
_DIR_NAME, '..', 'third_party', 'swarming.client', 'swarming.py'))
CONNECTION_TYPE_COMMON = 'common'
CONNECTION_TYPE_MOCK = 'mock'
+# Code 80 - bot died.
+RETRIABLE_INTERNAL_FAILURE_STATES = {80}
def RunSwarmingCommand(cmd, swarming_server, task_name=None,
@@ -87,6 +91,55 @@ def RunSwarmingCommand(cmd, swarming_server, task_name=None,
raise cros_build_lib.RunCommandError(e.msg, result, e.exception)
+def SwarmingRetriableErrorCheck(exception):
+ """Check if a swarming error is retriable.
+
+ Args:
+ exception: A cros_build_lib.RunCommandError exception.
+
+ Returns:
+ True if retriable, otherwise False.
+ """
+ if not isinstance(exception, cros_build_lib.RunCommandError):
+ return False
+ result = exception.result
+ if not isinstance(result, SwarmingCommandResult):
+ return False
+ if result.task_summary_json:
+ try:
+ internal_failure = result.task_summary_json[
+ 'shards'][0]['internal_failure']
+ state = result.task_summary_json['shards'][0]['state']
+ if internal_failure and state in RETRIABLE_INTERNAL_FAILURE_STATES:
+ logging.warning(
+ 'Encountered retriable swarming internal failure: %s',
+ json.dumps(result.task_summary_json, indent=2))
+ return True
+ except (IndexError, KeyError) as e:
+ logging.warning(
+ "Could not determine if %s is retriable, error: %s. json: %s",
+ str(exception), str(e),
+ json.dumps(result.task_summary_json, indent=2))
+ return False
+
+
+def RunSwarmingCommandWithRetries(max_retry, *args, **kwargs):
+ """Wrapper for RunSwarmingCommand that will retry a command.
+
+ Args:
+ max_retry: See RetryCommand.
+ *args: See RetryCommand and RunSwarmingCommand.
+ **kwargs: See RetryCommand and RunSwarmingCommand.
+
+ Returns:
+ A SwarmingCommandResult object.
+
+ Raises:
+ RunCommandError: When the command fails.
+ """
+ return retry_util.RetryCommand(RunSwarmingCommand, max_retry, *args, **kwargs)
+
+
class SwarmingCommandResult(cros_build_lib.CommandResult):
"""An object to store result of a command that is run via swarming.
diff --git a/cbuildbot/triage_lib.py b/cbuildbot/triage_lib.py
index 099dc4a84..e6cf6f8e0 100644
--- a/cbuildbot/triage_lib.py
+++ b/cbuildbot/triage_lib.py
@@ -10,6 +10,7 @@ import ConfigParser
import glob
import os
import pprint
+import re
from chromite.cbuildbot import failures_lib
from chromite.cbuildbot import constants
@@ -239,11 +240,9 @@ def GetStagesToIgnoreForChange(build_root, change):
def GetTestSubsystemForChange(build_root, change):
"""Get a list of subsystem that a given |change| affects.
- The list of the subsystem that a change affacts is specified in a config file
- inside the project, named COMMIT-QUEUE.ini. The file would look like this:
-
- [GENERAL]
- subsystem: power graphics
+ If subsystem is specified in the commit message, use that. Otherwise, look in
+ appropriate COMMIT-QUEUE.ini. If subsystem is not specified anywhere,
+ 'subsystem:default' will be used.
Based on the subsystems a given |change| affects, the CQ could tell whether a
failure is potentially caused by this |change|. The CQ could then submit some
@@ -256,8 +255,16 @@ def GetTestSubsystemForChange(build_root, change):
Returns:
A list of subsystem for the given |change|.
"""
- result = GetOptionForChange(build_root, change, 'GENERAL', 'subsystem')
- return result.split() if result else []
+ subsystems = []
+ if change.commit_message:
+ lines = cros_patch.GetOptionLinesFromCommitMessage(
+ change.commit_message, 'subsystem:')
+ if lines:
+ subsystems = [x for x in re.split("[, ]", ' '.join(lines)) if x]
+ if not subsystems:
+ result = GetOptionForChange(build_root, change, 'GENERAL', 'subsystem')
+ subsystems = result.split() if result else []
+ return subsystems if subsystems else ['default']
class CategorizeChanges(object):
"""A collection of methods to help categorize GerritPatch changes.
diff --git a/cbuildbot/triage_lib_unittest.py b/cbuildbot/triage_lib_unittest.py
index ab8716c03..efc5338cb 100644
--- a/cbuildbot/triage_lib_unittest.py
+++ b/cbuildbot/triage_lib_unittest.py
@@ -397,6 +397,34 @@ class GetOptionsTest(patch_unittest.MockPatchBase):
result = triage_lib.GetOptionForChange(build_root, change, 'a', 'b')
self.assertEqual(None, result)
+ def testGetSubsystemFromValidCommitMessage(self):
+ """Test whether we can get subsystem from commit message."""
+ change = sync_stages_unittest.MockPatch(
+ commit_message='First line\nThird line\nsubsystem: network audio\n'
+ 'subsystem: wifi')
+ self.PatchObject(triage_lib, 'GetOptionForChange',
+ return_value='power light')
+ result = triage_lib.GetTestSubsystemForChange('foo/build/root', change)
+ self.assertEqual(['network', 'audio', 'wifi'], result)
+
+ def testGetSubsystemFromInvalidCommitMessage(self):
+ """Test get subsystem from config file when commit message not have it."""
+ change = sync_stages_unittest.MockPatch(
+ commit_message='First line\nThird line\n')
+ self.PatchObject(triage_lib, 'GetOptionForChange',
+ return_value='power light')
+ result = triage_lib.GetTestSubsystemForChange('foo/build/root', change)
+ self.assertEqual(['power', 'light'], result)
+
+ def testGetDefaultSubsystem(self):
+ """Test if we can get default subsystem when subsystem is not specified."""
+ change = sync_stages_unittest.MockPatch(
+ commit_message='First line\nThird line\n')
+ self.PatchObject(triage_lib, 'GetOptionForChange',
+ return_value=None)
+ result = triage_lib.GetTestSubsystemForChange('foo/build/root', change)
+ self.assertEqual(['default'], result)
+
class ConfigFileTest(cros_test_lib.MockTestCase):
"""Tests for functions that read config information for a patch."""
diff --git a/cbuildbot/update_binhost_json.py b/cbuildbot/update_binhost_json.py
index 9d1625ac6..8c7ebf1b8 100644
--- a/cbuildbot/update_binhost_json.py
+++ b/cbuildbot/update_binhost_json.py
@@ -34,7 +34,7 @@ def main(argv):
cros_build_lib.AssertInsideChroot()
opts = _ParseArguments(argv)
- site_config = config_lib.LoadConfigFromFile()
+ site_config = config_lib.GetConfig()
logging.info('Generating board configs. This takes about 2m...')
for key in sorted(binhost.GetChromePrebuiltConfigs(site_config)):
diff --git a/cbuildbot/validation_pool.py b/cbuildbot/validation_pool.py
index b6e96e852..15f18e9d0 100644
--- a/cbuildbot/validation_pool.py
+++ b/cbuildbot/validation_pool.py
@@ -822,7 +822,7 @@ class PatchSeries(object):
@_ManifestDecorator
def Apply(self, changes, frozen=True, honor_ordering=False,
- changes_filter=None, max_change_count=None):
+ changes_filter=None):
"""Applies changes from pool into the build root specified by the manifest.
This method resolves each given change down into a set of transactions-
@@ -855,10 +855,6 @@ class PatchSeries(object):
changes being inspected, and expand the changes if necessary.
Primarily this is of use for cbuildbot patching when dealing w/
uploaded/remote patches.
- max_change_count: If not None, this is a soft integer limit on the number
- of patches to pull in. We stop pulling in patches as soon as we grab
- at least this many patches. Note that this limit may be exceeded by N-1,
- where N is the length of the longest transaction.
Returns:
A tuple of changes-applied, Exceptions for the changes that failed
@@ -885,9 +881,6 @@ class PatchSeries(object):
change, ', '.join(map(str, resolved[-1][-1])))
planned.update(plan)
- if max_change_count is not None and len(planned) >= max_change_count:
- break
-
if not resolved:
# No work to do; either no changes were given to us, or all failed
# to be resolved.
@@ -1129,7 +1122,7 @@ class ValidationPool(object):
def __init__(self, overlays, build_root, build_number, builder_name,
is_master, dryrun, changes=None, non_os_changes=None,
conflicting_changes=None, pre_cq_trybot=False,
- tree_was_open=True, builder_run=None):
+ tree_was_open=True, _applied=None, builder_run=None):
"""Initializes an instance by setting default variables to instance vars.
Generally use AcquirePool as an entry pool to a pool rather than this
@@ -1151,11 +1144,12 @@ class ValidationPool(object):
pre_cq_trybot: If set to True, this is a Pre-CQ trybot. (Note: The Pre-CQ
launcher is NOT considered a Pre-CQ trybot.)
tree_was_open: Whether the tree was open when the pool was created.
+ applied: List of CLs that have been applied to the current repo. Not
+ yet used, but needs to be here for pickling compatibility.
builder_run: BuilderRun instance used to fetch cidb handle and metadata
instance. Please note due to the pickling logic, this MUST be the last
kwarg listed.
"""
-
self.build_root = build_root
# These instances can be instantiated via both older, or newer pickle
@@ -1622,6 +1616,22 @@ class ValidationPool(object):
logging.PrintBuildbotLink(s, change.url)
+ def FilterChangesForThrottledTree(self):
+ """Apply Throttled Tree logic to select patch candidates.
+
+ If the tree is throttled, we only test a random subset of our candidate
+ changes. Call this to select that subset, and throw away unrelated changes.
+
+ If the three was open when this pool was created, it does nothing.
+ """
+ if self.tree_was_open:
+ return
+
+ fail_streak = self._GetFailStreak()
+ test_pool_size = max(1, len(self.changes) / (2**fail_streak))
+ random.shuffle(self.changes)
+ self.changes = self.changes[:test_pool_size]
+
def ApplyPoolIntoRepo(self, manifest=None):
"""Applies changes from pool into the directory specified by the buildroot.
@@ -1640,18 +1650,11 @@ class ValidationPool(object):
failed_inflight = []
patch_series = PatchSeries(self.build_root, helper_pool=self._helper_pool)
- # Only try a subset of the changes if the tree was throttled.
- max_change_count = len(self.changes)
- if not self.tree_was_open:
- random.shuffle(self.changes)
- fail_streak = self._GetFailStreak()
- max_change_count = max(1, len(self.changes) / (2**fail_streak))
-
if self.is_master:
try:
# pylint: disable=E1123
applied, failed_tot, failed_inflight = patch_series.Apply(
- self.changes, manifest=manifest, max_change_count=max_change_count)
+ self.changes, manifest=manifest)
except (KeyboardInterrupt, RuntimeError, SystemExit):
raise
except Exception as e:
diff --git a/cbuildbot/validation_pool_unittest.py b/cbuildbot/validation_pool_unittest.py
index 228c5e4b7..3fab2d100 100644
--- a/cbuildbot/validation_pool_unittest.py
+++ b/cbuildbot/validation_pool_unittest.py
@@ -16,7 +16,6 @@ import mock
import mox
import os
import pickle
-import random
import tempfile
import time
@@ -878,10 +877,7 @@ class TestCoreLogic(MoxBase):
def MakeFailure(self, patch, inflight=True):
return cros_patch.ApplyPatchException(patch, inflight=inflight)
- def GetPool(self, changes, applied=(), tot=(), inflight=(),
- max_change_count=None, **kwargs):
- if not max_change_count:
- max_change_count = len(changes)
+ def GetPool(self, changes, applied=(), tot=(), inflight=(), **kwargs):
pool = self.MakePool(changes=changes, fake_db=self.fake_db, **kwargs)
applied = list(applied)
@@ -889,8 +885,7 @@ class TestCoreLogic(MoxBase):
inflight = [self.MakeFailure(x, inflight=True) for x in inflight]
# pylint: disable=E1120,E1123
validation_pool.PatchSeries.Apply(
- changes, manifest=mox.IgnoreArg(), max_change_count=max_change_count
- ).AndReturn((applied, tot, inflight))
+ changes, manifest=mox.IgnoreArg()).AndReturn((applied, tot, inflight))
for patch in applied:
pool.HandleApplySuccess(patch, mox.IgnoreArg()).AndReturn(None)
@@ -1138,8 +1133,7 @@ class TestCoreLogic(MoxBase):
# pylint: disable=E1120,E1123
validation_pool.PatchSeries.Apply(
- patches, manifest=mox.IgnoreArg(),
- max_change_count=len(patches)).AndRaise(MyException)
+ patches, manifest=mox.IgnoreArg()).AndRaise(MyException)
errors = [mox.Func(functools.partial(VerifyCQError, x)) for x in patches]
pool._HandleApplyFailure(errors).AndReturn(None)
@@ -1366,47 +1360,72 @@ class TestCoreLogic(MoxBase):
self.assertEqual(slave_pool._GetFailStreak(), 0)
- def testApplyWithTreeNotOpen(self):
+ def testFilterChangesForThrottledTree(self):
"""Tests that we can correctly apply exponential fallback."""
patches = self.GetPatches(4)
-
- # We mock out the shuffle so that we can deterministically test.
- self.mox.StubOutWithMock(random, 'shuffle')
self.mox.StubOutWithMock(validation_pool.ValidationPool, '_GetFailStreak')
- slave_pool = self.GetPool(changes=patches, applied=patches[:2],
- max_change_count=2,
- tree_was_open=False, handlers=True)
- random.shuffle(patches) # Mock.
+ #
+ # Test when tree is open.
+ #
+ self.mox.ReplayAll()
+
+ # Perform test.
+ slave_pool = self.MakePool(changes=patches, tree_was_open=True)
+ slave_pool.FilterChangesForThrottledTree()
+
+ # Validate results.
+ self.assertEqual(len(slave_pool.changes), 4)
+ self.mox.VerifyAll()
+ self.mox.ResetAll()
+
+ #
+ # Test when tree is closed with a streak of 1.
+ #
+
# pylint: disable=no-value-for-parameter
validation_pool.ValidationPool._GetFailStreak().AndReturn(1)
-
self.mox.ReplayAll()
- self.runApply(slave_pool, True)
+
+ # Perform test.
+ slave_pool = self.MakePool(changes=patches, tree_was_open=False)
+ slave_pool.FilterChangesForThrottledTree()
+
+ # Validate results.
self.assertEqual(len(slave_pool.changes), 2)
self.mox.VerifyAll()
self.mox.ResetAll()
- slave_pool = self.GetPool(changes=patches, applied=patches[:1],
- max_change_count=1,
- tree_was_open=False, handlers=True)
- random.shuffle(patches) # Mock.
- validation_pool.ValidationPool._GetFailStreak().AndReturn(2)
+ #
+ # Test when tree is closed with a streak of 2.
+ #
+ # pylint: disable=no-value-for-parameter
+ validation_pool.ValidationPool._GetFailStreak().AndReturn(2)
self.mox.ReplayAll()
- self.runApply(slave_pool, True)
+
+ # Perform test.
+ slave_pool = self.MakePool(changes=patches, tree_was_open=False)
+ slave_pool.FilterChangesForThrottledTree()
+
+ # Validate results.
self.assertEqual(len(slave_pool.changes), 1)
self.mox.VerifyAll()
self.mox.ResetAll()
- slave_pool = self.GetPool(changes=patches, applied=patches[:1],
- max_change_count=1,
- tree_was_open=False, handlers=True)
- random.shuffle(patches) # Mock.
- validation_pool.ValidationPool._GetFailStreak().AndReturn(10)
+ #
+ # Test when tree is closed with a streak of many.
+ #
+ # pylint: disable=no-value-for-parameter
+ validation_pool.ValidationPool._GetFailStreak().AndReturn(200)
self.mox.ReplayAll()
- self.runApply(slave_pool, True)
+
+ # Perform test.
+ slave_pool = self.MakePool(changes=patches, tree_was_open=False)
+ slave_pool.FilterChangesForThrottledTree()
+
+ # Validate results.
self.assertEqual(len(slave_pool.changes), 1)
self.mox.VerifyAll()
diff --git a/cidb/developer.readme b/cidb/developer.readme
index 6cc95b314..d8117d30e 100644
--- a/cidb/developer.readme
+++ b/cidb/developer.readme
@@ -12,12 +12,15 @@ $ lib/cidb_integration_test --debug --no-wipe
The test logs the path to the temporary working directory at the end.
You can launch the mysqld server again to play with the database in its final
-state. If `tmpdir` is the temporary directory left behind by the test, Inside
-the chroot, run:
+state. You'll have to fish out the temp directory that it created -- it will
+look something like /tmp/chromite.test_no_cleanup3WqzmO/chromite.testYypd_c/
+
+Set your local tmpdir variable to the path that you found, and run
+(inside the chroot):
$ /usr/sbin/mysqld --no-defaults --datadir ${tmpdir}/mysqld_dir --socket \
${tmpdir}/mysqld_dir/mysqld.socket --port 8440 --pid-file \
- ${tmpdir}/mysqld_dir/mysqld.pid --tmpdir ${tmpdir}/mysqld_dir/tmp
+ ${tmpdir}/mysqld_dir/mysqld.pid --tmpdir ${tmpdir}/mysqld_dir/tmp &
You can connect to this instance using mysql client.
diff --git a/cidb/migrations/00042_create_build_message_table.sql b/cidb/migrations/00042_create_build_message_table.sql
new file mode 100644
index 000000000..6ab047d7d
--- /dev/null
+++ b/cidb/migrations/00042_create_build_message_table.sql
@@ -0,0 +1,15 @@
+CREATE TABLE buildMessageTable (
+ id INT NOT NULL AUTO_INCREMENT,
+ build_id INT NOT NULL,
+ message_type VARCHAR(240),
+ message_subtype VARCHAR(240),
+ message_value VARCHAR(480),
+ timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ PRIMARY KEY(id),
+ FOREIGN KEY (build_id)
+ REFERENCES buildTable(id)
+);
+
+
+INSERT INTO schemaVersionTable (schemaVersion, scriptName) VALUES
+ (42, '00042_create_build_message_table.sql');
diff --git a/cidb/migrations/00043_alter_build_table_add_important.sql b/cidb/migrations/00043_alter_build_table_add_important.sql
new file mode 100644
index 000000000..da4c3d18b
--- /dev/null
+++ b/cidb/migrations/00043_alter_build_table_add_important.sql
@@ -0,0 +1,5 @@
+ALTER TABLE buildTable
+ ADD COLUMN important BOOLEAN DEFAULT NULL;
+
+INSERT INTO schemaVersionTable (schemaVersion, scriptName) VALUES
+ (43, '00043_alter_build_table_add_important.sql');
diff --git a/cidb/migrations/00044_alter_views_add_important.sql b/cidb/migrations/00044_alter_views_add_important.sql
new file mode 100644
index 000000000..eff3284b7
--- /dev/null
+++ b/cidb/migrations/00044_alter_views_add_important.sql
@@ -0,0 +1,32 @@
+-- The failureView consists of:
+-- all failureTable columns, by original name.
+-- all buildStageTable columns except build_id, and its own id (which come
+-- from f.*), with non-colliding names
+-- all buildTable columns, with non-colliding names
+ALTER VIEW failureView AS
+ SELECT f.*,
+ bs.name AS stage_name, bs.board, bs.status AS stage_status,
+ bs.last_updated AS stage_last_updated, bs.start_time AS stage_start_time,
+ bs.finish_time AS stage_finish_time, bs.final AS stage_final,
+ b.id AS build_id, b.last_updated AS build_last_updated, b.master_build_id,
+ b.buildbot_generation, b.builder_name, b.waterfall, b.build_number,
+ b.build_config, b.bot_hostname, b.start_time AS build_start_time,
+ b.finish_time AS build_finish_time, b.status AS build_status, b.build_type,
+ b.chrome_version, b.milestone_version, b.platform_version, b.full_version,
+ b.sdk_version, b.toolchain_url, b.final AS build_final, b.metadata_url,
+ b.summary, b.deadline, b.important
+ FROM failureTable f JOIN buildStageTable bs on f.build_stage_id = bs.id
+ JOIN buildTable b on bs.build_id = b.id;
+
+ALTER VIEW clActionView as
+ SELECT c.*,
+ b.last_updated, b.master_build_id, b.buildbot_generation, b.builder_name,
+ b.waterfall, b.build_number, b.build_config, b.bot_hostname, b.start_time,
+ b.finish_time, b.status, b.build_type, b.chrome_version,
+ b.milestone_version, b.platform_version, b.full_version, b.sdk_version,
+ b.toolchain_url, b.final, b.metadata_url, b.summary, b.deadline,
+ b.important
+ FROM clActionTable c JOIN buildTable b on c.build_id = b.id;
+
+INSERT INTO schemaVersionTable (schemaVersion, scriptName) VALUES
+ (44, '00044_alter_views_add_important.sql')
diff --git a/cidb/migrations/00045_alter_build_message_table_add_board.sql b/cidb/migrations/00045_alter_build_message_table_add_board.sql
new file mode 100644
index 000000000..47385e3a6
--- /dev/null
+++ b/cidb/migrations/00045_alter_build_message_table_add_board.sql
@@ -0,0 +1,6 @@
+ALTER TABLE buildMessageTable
+ ADD COLUMN board VARCHAR(240) DEFAULT NULL;
+
+INSERT INTO schemaVersionTable (schemaVersion, scriptName) VALUES
+ (45, '00045_alter_build_message_table_add_board.sql');
+
diff --git a/cidb/schema.dump b/cidb/schema.dump
index 598bb3eff..7ad64ab25 100644
--- a/cidb/schema.dump
+++ b/cidb/schema.dump
@@ -50,6 +50,24 @@ CREATE TABLE `boardPerBuildTable` (
/*!40101 SET character_set_client = @saved_cs_client */;
+DROP TABLE IF EXISTS `buildMessageTable`;
+/*!40101 SET @saved_cs_client = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `buildMessageTable` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `build_id` int(11) NOT NULL,
+ `message_type` varchar(240) DEFAULT NULL,
+ `message_subtype` varchar(240) DEFAULT NULL,
+ `message_value` varchar(480) DEFAULT NULL,
+ `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ `board` varchar(240) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ KEY `build_id` (`build_id`),
+ CONSTRAINT `buildMessageTable_ibfk_1` FOREIGN KEY (`build_id`) REFERENCES `buildTable` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+
DROP TABLE IF EXISTS `buildStageTable`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
@@ -99,6 +117,7 @@ CREATE TABLE `buildTable` (
`metadata_url` varchar(240) DEFAULT NULL,
`summary` varchar(1024) DEFAULT NULL,
`deadline` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
+ `important` tinyint(1) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `buildbot_generation` (`buildbot_generation`,`builder_name`,`waterfall`,`build_number`),
KEY `master_build_id` (`master_build_id`),
@@ -182,7 +201,8 @@ SET character_set_client = utf8;
`final` tinyint NOT NULL,
`metadata_url` tinyint NOT NULL,
`summary` tinyint NOT NULL,
- `deadline` tinyint NOT NULL
+ `deadline` tinyint NOT NULL,
+ `important` tinyint NOT NULL
) ENGINE=MyISAM */;
SET character_set_client = @saved_cs_client;
@@ -250,7 +270,8 @@ SET character_set_client = utf8;
`build_final` tinyint NOT NULL,
`metadata_url` tinyint NOT NULL,
`summary` tinyint NOT NULL,
- `deadline` tinyint NOT NULL
+ `deadline` tinyint NOT NULL,
+ `important` tinyint NOT NULL
) ENGINE=MyISAM */;
SET character_set_client = @saved_cs_client;
@@ -287,7 +308,8 @@ CREATE TABLE `schemaVersionTable` (
/*!50001 SET character_set_results = utf8 */;
/*!50001 SET collation_connection = utf8_general_ci */;
/*!50001 CREATE ALGORITHM=UNDEFINED */
-/*!50001 VIEW `clActionView` AS select `c`.`id` AS `id`,`c`.`build_id` AS `build_id`,`c`.`change_number` AS `change_number`,`c`.`patch_number` AS `patch_number`,`c`.`change_source` AS `change_source`,`c`.`action` AS `action`,`c`.`reason` AS `reason`,`c`.`timestamp` AS `timestamp`,`b`.`last_updated` AS `last_updated`,`b`.`master_build_id` AS `master_build_id`,`b`.`buildbot_generation` AS `buildbot_generation`,`b`.`builder_name` AS `builder_name`,`b`.`waterfall` AS `waterfall`,`b`.`build_number` AS `build_number`,`b`.`build_config` AS `build_config`,`b`.`bot_hostname` AS `bot_hostname`,`b`.`start_time` AS `start_time`,`b`.`finish_time` AS `finish_time`,`b`.`status` AS `status`,`b`.`build_type` AS `build_type`,`b`.`chrome_version` AS `chrome_version`,`b`.`milestone_version` AS `milestone_version`,`b`.`platform_version` AS `platform_version`,`b`.`full_version` AS `full_version`,`b`.`sdk_version` AS `sdk_version`,`b`.`toolchain_url` AS `toolchain_url`,`b`.`final` AS `final`,`b`.`metadata_url` AS `metadata_url`,`b`.`summary` AS `summary`,`b`.`deadline` AS `deadline` from (`clActionTable` `c` join `buildTable` `b` on((`c`.`build_id` = `b`.`id`))) */;
+/*!50013 DEFINER=`root`@`localhost` SQL SECURITY DEFINER */
+/*!50001 VIEW `clActionView` AS select `c`.`id` AS `id`,`c`.`build_id` AS `build_id`,`c`.`change_number` AS `change_number`,`c`.`patch_number` AS `patch_number`,`c`.`change_source` AS `change_source`,`c`.`action` AS `action`,`c`.`reason` AS `reason`,`c`.`timestamp` AS `timestamp`,`b`.`last_updated` AS `last_updated`,`b`.`master_build_id` AS `master_build_id`,`b`.`buildbot_generation` AS `buildbot_generation`,`b`.`builder_name` AS `builder_name`,`b`.`waterfall` AS `waterfall`,`b`.`build_number` AS `build_number`,`b`.`build_config` AS `build_config`,`b`.`bot_hostname` AS `bot_hostname`,`b`.`start_time` AS `start_time`,`b`.`finish_time` AS `finish_time`,`b`.`status` AS `status`,`b`.`build_type` AS `build_type`,`b`.`chrome_version` AS `chrome_version`,`b`.`milestone_version` AS `milestone_version`,`b`.`platform_version` AS `platform_version`,`b`.`full_version` AS `full_version`,`b`.`sdk_version` AS `sdk_version`,`b`.`toolchain_url` AS `toolchain_url`,`b`.`final` AS `final`,`b`.`metadata_url` AS `metadata_url`,`b`.`summary` AS `summary`,`b`.`deadline` AS `deadline`,`b`.`important` AS `important` from (`clActionTable` `c` join `buildTable` `b` on((`c`.`build_id` = `b`.`id`))) */;
/*!50001 SET character_set_client = @saved_cs_client */;
/*!50001 SET character_set_results = @saved_cs_results */;
/*!50001 SET collation_connection = @saved_col_connection */;
@@ -302,7 +324,8 @@ CREATE TABLE `schemaVersionTable` (
/*!50001 SET character_set_results = utf8 */;
/*!50001 SET collation_connection = utf8_general_ci */;
/*!50001 CREATE ALGORITHM=UNDEFINED */
-/*!50001 VIEW `failureView` AS select `f`.`id` AS `id`,`f`.`build_stage_id` AS `build_stage_id`,`f`.`outer_failure_id` AS `outer_failure_id`,`f`.`exception_type` AS `exception_type`,`f`.`exception_message` AS `exception_message`,`f`.`exception_category` AS `exception_category`,`f`.`extra_info` AS `extra_info`,`f`.`timestamp` AS `timestamp`,`bs`.`name` AS `stage_name`,`bs`.`board` AS `board`,`bs`.`status` AS `stage_status`,`bs`.`last_updated` AS `stage_last_updated`,`bs`.`start_time` AS `stage_start_time`,`bs`.`finish_time` AS `stage_finish_time`,`bs`.`final` AS `stage_final`,`b`.`id` AS `build_id`,`b`.`last_updated` AS `build_last_updated`,`b`.`master_build_id` AS `master_build_id`,`b`.`buildbot_generation` AS `buildbot_generation`,`b`.`builder_name` AS `builder_name`,`b`.`waterfall` AS `waterfall`,`b`.`build_number` AS `build_number`,`b`.`build_config` AS `build_config`,`b`.`bot_hostname` AS `bot_hostname`,`b`.`start_time` AS `build_start_time`,`b`.`finish_time` AS `build_finish_time`,`b`.`status` AS `build_status`,`b`.`build_type` AS `build_type`,`b`.`chrome_version` AS `chrome_version`,`b`.`milestone_version` AS `milestone_version`,`b`.`platform_version` AS `platform_version`,`b`.`full_version` AS `full_version`,`b`.`sdk_version` AS `sdk_version`,`b`.`toolchain_url` AS `toolchain_url`,`b`.`final` AS `build_final`,`b`.`metadata_url` AS `metadata_url`,`b`.`summary` AS `summary`,`b`.`deadline` AS `deadline` from ((`failureTable` `f` join `buildStageTable` `bs` on((`f`.`build_stage_id` = `bs`.`id`))) join `buildTable` `b` on((`bs`.`build_id` = `b`.`id`))) */;
+/*!50013 DEFINER=`root`@`localhost` SQL SECURITY DEFINER */
+/*!50001 VIEW `failureView` AS select `f`.`id` AS `id`,`f`.`build_stage_id` AS `build_stage_id`,`f`.`outer_failure_id` AS `outer_failure_id`,`f`.`exception_type` AS `exception_type`,`f`.`exception_message` AS `exception_message`,`f`.`exception_category` AS `exception_category`,`f`.`extra_info` AS `extra_info`,`f`.`timestamp` AS `timestamp`,`bs`.`name` AS `stage_name`,`bs`.`board` AS `board`,`bs`.`status` AS `stage_status`,`bs`.`last_updated` AS `stage_last_updated`,`bs`.`start_time` AS `stage_start_time`,`bs`.`finish_time` AS `stage_finish_time`,`bs`.`final` AS `stage_final`,`b`.`id` AS `build_id`,`b`.`last_updated` AS `build_last_updated`,`b`.`master_build_id` AS `master_build_id`,`b`.`buildbot_generation` AS `buildbot_generation`,`b`.`builder_name` AS `builder_name`,`b`.`waterfall` AS `waterfall`,`b`.`build_number` AS `build_number`,`b`.`build_config` AS `build_config`,`b`.`bot_hostname` AS `bot_hostname`,`b`.`start_time` AS `build_start_time`,`b`.`finish_time` AS `build_finish_time`,`b`.`status` AS `build_status`,`b`.`build_type` AS `build_type`,`b`.`chrome_version` AS `chrome_version`,`b`.`milestone_version` AS `milestone_version`,`b`.`platform_version` AS `platform_version`,`b`.`full_version` AS `full_version`,`b`.`sdk_version` AS `sdk_version`,`b`.`toolchain_url` AS `toolchain_url`,`b`.`final` AS `build_final`,`b`.`metadata_url` AS `metadata_url`,`b`.`summary` AS `summary`,`b`.`deadline` AS `deadline`,`b`.`important` AS `important` from ((`failureTable` `f` join `buildStageTable` `bs` on((`f`.`build_stage_id` = `bs`.`id`))) join `buildTable` `b` on((`bs`.`build_id` = `b`.`id`))) */;
/*!50001 SET character_set_client = @saved_cs_client */;
/*!50001 SET character_set_results = @saved_cs_results */;
/*!50001 SET collation_connection = @saved_col_connection */;
diff --git a/cidb/schema.dump.readme b/cidb/schema.dump.readme
index d8e2aef0e..d65030012 100644
--- a/cidb/schema.dump.readme
+++ b/cidb/schema.dump.readme
@@ -1,15 +1,15 @@
These instructions assume you have a full chromiumos checkout at
~/chromiumos/
-To generate a schema dump, run
- $ lib/cidb_integration_test.py CIDBMigrationsTest.testMigrations --no-wipe
-to bring the test database instance to the latest schema (as reflected in your
-source tree).
+To generate a schema dump, run this inside the chroot:
+ $ lib/cidb_integration_test CIDBMigrationsTest.testMigrations --no-wipe
+
Then, follow the instructions in the developer.readme to re-launch the mysqld
-daemon from the temporary directory and dump the schema using:
+daemon from the temporary directory and dump the schema using (again, inside the
+chroot):
$ mysqldump -u root -S ${tmpdir}/mysqld_dir/mysqld.socket --no-data \
--single-transaction cidb | grep -v '^--' \
- > ~/chromiumos/chromite/cidb/schema.dump
+ > ~/trunk/chromite/cidb/schema.dump
Remember to cleanup the temporary directory when you're done.
diff --git a/cli/command.py b/cli/command.py
index 360a6ec10..7f4e4b096 100644
--- a/cli/command.py
+++ b/cli/command.py
@@ -19,13 +19,10 @@ import glob
import os
from chromite.cbuildbot import constants
-from chromite.lib import brick_lib
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import cros_import
from chromite.lib import cros_logging as logging
-from chromite.lib import osutils
-from chromite.lib import workspace_lib
# Paths for finding and importing subcommand modules.
@@ -36,26 +33,6 @@ _SUBCOMMAND_MODULE_PREFIX = 'cros_'
_commands = dict()
-def SetupFileLogger(filename='brillo.log', log_level=logging.DEBUG):
- """Store log messages to a file.
-
- In case of an error, this file can be made visible to the user.
- """
- workspace_path = workspace_lib.WorkspacePath()
- if workspace_path is None:
- return
- path = os.path.join(workspace_path, workspace_lib.WORKSPACE_LOGS_DIR,
- filename)
- osutils.Touch(path, makedirs=True)
- logger = logging.getLogger()
- fh = logging.FileHandler(path, mode='w')
- fh.setLevel(log_level)
- fh.setFormatter(
- logging.Formatter(fmt=constants.LOGGER_FMT,
- datefmt=constants.LOGGER_DATE_FMT))
- logger.addHandler(fh)
-
-
def UseProgressBar():
"""Determine whether the progress bar is to be used or not.
@@ -158,8 +135,6 @@ class CliCommand(object):
def __init__(self, options):
self.options = options
- brick = brick_lib.FindBrickInPath()
- self.curr_brick_locator = brick.brick_locator if brick else None
@classmethod
def AddParser(cls, parser):
diff --git a/cli/command_unittest.py b/cli/command_unittest.py
index 33c072567..8d9dea3d2 100644
--- a/cli/command_unittest.py
+++ b/cli/command_unittest.py
@@ -15,10 +15,8 @@ from chromite.cli import command
from chromite.lib import commandline
from chromite.lib import cros_build_lib_unittest
from chromite.lib import cros_import
-from chromite.lib import cros_logging as logging
from chromite.lib import cros_test_lib
from chromite.lib import partial_mock
-from chromite.lib import workspace_lib
# pylint:disable=protected-access
@@ -132,48 +130,3 @@ class CommandTest(cros_test_lib.MockTestCase):
# Pick some commands that are likely to not go away.
self.assertIn('chrome-sdk', cros_commands)
self.assertIn('flash', cros_commands)
-
-
-class FileLoggerSetupTest(cros_test_lib.WorkspaceTestCase):
- """Test that logging to file works correctly."""
-
- def setUp(self):
- self.CreateWorkspace()
-
- def testSetupFileLoggerFilename(self):
- """Test that the filename and path are correct."""
- patch_handler = self.PatchObject(logging, 'FileHandler',
- return_value=logging.StreamHandler())
- command.SetupFileLogger(filename='foo.log')
-
- # Test that the filename is correct.
- patch_handler.assert_called_with(
- os.path.join(self.workspace_path, workspace_lib.WORKSPACE_LOGS_DIR,
- 'foo.log'), mode='w')
-
- def testSetupFileLoggerNoFilename(self):
- """Test that the filename and path are correct with no arguments."""
- patch_handler = self.PatchObject(logging, 'FileHandler',
- return_value=logging.StreamHandler())
- command.SetupFileLogger()
-
- # Test that the filename is correct.
- patch_handler.assert_called_with(
- os.path.join(self.workspace_path, workspace_lib.WORKSPACE_LOGS_DIR,
- 'brillo.log'), mode='w')
-
- def testSetupFileLoggerLogLevels(self):
- """Test that the logger operates at the right level."""
- command.SetupFileLogger('foo.log', log_level=logging.INFO)
- logging.getLogger().setLevel(logging.DEBUG)
- logging.debug('debug')
- logging.info('info')
- logging.notice('notice')
-
- # Test that the logs are correct.
- logs = open(
- os.path.join(self.workspace_path, workspace_lib.WORKSPACE_LOGS_DIR,
- 'foo.log'), 'r').read()
- self.assertNotIn('debug', logs)
- self.assertIn('info', logs)
- self.assertIn('notice', logs)
diff --git a/cli/cros/cros_build.py b/cli/cros/cros_build.py
index b868c8f13..0709288f5 100644
--- a/cli/cros/cros_build.py
+++ b/cli/cros/cros_build.py
@@ -7,15 +7,12 @@
from __future__ import print_function
from chromite.cli import command
-from chromite.lib import blueprint_lib
-from chromite.lib import brick_lib
from chromite.lib import chroot_util
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import operation
from chromite.lib import parallel
-from chromite.lib import toolchain
from chromite.lib import workon_helper
@@ -51,40 +48,23 @@ To just build a single package:
self.host = False
self.board = None
self.brick = None
- self.blueprint = None
if self.options.host:
self.host = True
elif self.options.board:
self.board = self.options.board
- elif self.options.blueprint:
- self.blueprint = blueprint_lib.Blueprint(self.options.blueprint)
-
- if not self.build_pkgs:
- self.build_pkgs = self.blueprint.GetPackages()
- elif self.options.brick or self.curr_brick_locator:
- self.brick = brick_lib.Brick(self.options.brick
- or self.curr_brick_locator)
- self.board = self.brick.FriendlyName()
- if not self.build_pkgs:
- self.build_pkgs = self.brick.MainPackages()
else:
# If nothing is explicitly set, use the default board.
self.board = cros_build_lib.GetDefaultBoard()
# Set sysroot and friendly name. The latter is None if building for host.
- self.sysroot = cros_build_lib.GetSysroot(self.blueprint.FriendlyName()
- if self.blueprint else self.board)
+ self.sysroot = cros_build_lib.GetSysroot(self.board)
@classmethod
def AddParser(cls, parser):
super(cls, BuildCommand).AddParser(parser)
target = parser.add_mutually_exclusive_group()
target.add_argument('--board', help='The board to build packages for.')
- target.add_argument('--brick', type='brick_path',
- help='The brick to build packages for.')
- target.add_argument('--blueprint', type='blueprint_path',
- help='The blueprint to build packages for.')
target.add_argument('--host', help='Build packages for the chroot itself.',
default=False, action='store_true')
parser.add_argument('--no-binary', help="Don't use binary packages.",
@@ -134,7 +114,7 @@ To just build a single package:
Only print the output if this step fails or if we're in debug mode.
"""
- if self.options.deps and not self.host and not self.blueprint:
+ if self.options.deps and not self.host:
cmd = chroot_util.GetEmergeCommand(sysroot=self.sysroot)
cmd += ['-pe', '--backtrack=0'] + self.build_pkgs
try:
@@ -160,7 +140,7 @@ To just build a single package:
self.options.Freeze()
if not self.host:
- if not (self.board or self.brick or self.blueprint):
+ if not (self.board or self.brick):
cros_build_lib.Die('You did not specify a board/brick to build for. '
'You need to be in a brick directory or set '
'--board/--brick/--host')
@@ -169,11 +149,7 @@ To just build a single package:
cros_build_lib.Die('--brick should not be used with board names. Use '
'--board=%s instead.' % self.brick.config['name'])
- if self.blueprint:
- chroot_args = ['--toolchains',
- ','.join(toolchain.GetToolchainsForBrick(
- self.blueprint.GetBSP()).iterkeys())]
- elif self.board:
+ if self.board:
chroot_args = ['--board', self.board]
else:
chroot_args = None
@@ -184,13 +160,7 @@ To just build a single package:
cros_build_lib.Die('No packages found, nothing to build.')
# Set up the sysroots if not building for host.
- if self.blueprint:
- if self.chroot_update:
- chroot_util.UpdateChroot(
- update_host_packages=self.options.host_packages_update,
- brick=brick_lib.Brick(self.blueprint.GetBSP()))
- chroot_util.InitializeSysroots(self.blueprint)
- elif self.brick or self.board:
+ if self.brick or self.board:
chroot_util.SetupBoard(
brick=self.brick, board=self.board,
update_chroot=self.chroot_update,
diff --git a/cli/cros/cros_chrome_sdk.py b/cli/cros/cros_chrome_sdk.py
index 73ac533a3..116a94196 100644
--- a/cli/cros/cros_chrome_sdk.py
+++ b/cli/cros/cros_chrome_sdk.py
@@ -23,6 +23,7 @@ from chromite.lib import gs
from chromite.lib import osutils
from chromite.lib import path_util
from chromite.lib import stats
+from chromite.cbuildbot import archive_lib
from chromite.cbuildbot import config_lib
from chromite.cbuildbot import constants
@@ -91,7 +92,7 @@ class SDKFetcher(object):
force usage of the external configuration if both external and internal
are available.
"""
- site_config = config_lib.LoadConfigFromFile()
+ site_config = config_lib.GetConfig()
self.cache_base = os.path.join(cache_dir, COMMAND_NAME)
if clear_cache:
@@ -104,8 +105,7 @@ class SDKFetcher(object):
self.board = board
self.config = site_config.FindCanonicalConfigForBoard(
board, allow_internal=not use_external_config)
- self.gs_base = '%s/%s' % (constants.DEFAULT_ARCHIVE_BUCKET,
- self.config['name'])
+ self.gs_base = archive_lib.GetBaseUploadURI(self.config)
self.clear_cache = clear_cache
self.chrome_src = chrome_src
self.sdk_path = sdk_path
diff --git a/cli/cros/cros_payload.py b/cli/cros/cros_payload.py
index ebac811ab..9581b597d 100644
--- a/cli/cros/cros_payload.py
+++ b/cli/cros/cros_payload.py
@@ -8,7 +8,6 @@ from __future__ import print_function
import itertools
import os
-import string
import sys
import textwrap
@@ -18,6 +17,8 @@ from chromite.cli import command
# Needed for the dev.host.lib import below.
sys.path.insert(0, os.path.join(constants.SOURCE_ROOT, 'src', 'platform'))
+MAJOR_PAYLOAD_VERSION_CHROMEOS = 1
+MAJOR_PAYLOAD_VERSION_BRILLO = 2
def DisplayValue(key, value):
"""Print out a key, value pair with values left-aligned."""
@@ -35,7 +36,7 @@ def DisplayHexData(data, indent=0):
' '.join('%.2x' % ord(c) for c in chunk) +
' ' * (16 - len(chunk)) +
' | ' +
- ''.join(c if c in string.printable else '.' for c in chunk))
+ ''.join(c if 32 <= ord(c) and ord(c) < 127 else '.' for c in chunk))
@command.CommandDecorator('payload')
@@ -85,30 +86,52 @@ Example:
def _DisplayManifest(self):
"""Show information from the payload manifest."""
manifest = self.payload.manifest
- DisplayValue('Number of operations', len(manifest.install_operations))
- DisplayValue('Number of kernel ops',
- len(manifest.kernel_install_operations))
+ if self.payload.header.version == MAJOR_PAYLOAD_VERSION_BRILLO:
+ DisplayValue('Number of partitions', len(manifest.partitions))
+ for partition in manifest.partitions:
+ DisplayValue(' Number of "%s" ops' % partition.partition_name,
+ len(partition.operations))
+ else:
+ DisplayValue('Number of operations', len(manifest.install_operations))
+ DisplayValue('Number of kernel ops',
+ len(manifest.kernel_install_operations))
DisplayValue('Block size', manifest.block_size)
DisplayValue('Minor version', manifest.minor_version)
def _DisplaySignatures(self):
"""Show information about the signatures from the manifest."""
- manifest = self.payload.manifest
- if not manifest.HasField('signatures_offset'):
- print('No signatures stored in the payload')
- return
-
- signature_msg = 'offset=%d' % manifest.signatures_offset
- if manifest.signatures_size:
- signature_msg += ' (%d bytes)' % manifest.signatures_size
- DisplayValue('Signature blob', signature_msg)
- signatures_blob = self.payload.ReadDataBlob(manifest.signatures_offset,
- manifest.signatures_size)
+ header = self.payload.header
+ if header.metadata_signature_len:
+ offset = header.size + header.manifest_len
+ DisplayValue('Metadata signatures blob',
+ 'file_offset=%d (%d bytes)' %
+ (offset, header.metadata_signature_len))
+ signatures_blob = self.payload.ReadDataBlob(
+ -header.metadata_signature_len,
+ header.metadata_signature_len)
+ self._DisplaySignaturesBlob('Metadata', signatures_blob)
+ else:
+ print('No metadata signatures stored in the payload')
+ manifest = self.payload.manifest
+ if manifest.HasField('signatures_offset'):
+ signature_msg = 'blob_offset=%d' % manifest.signatures_offset
+ if manifest.signatures_size:
+ signature_msg += ' (%d bytes)' % manifest.signatures_size
+ DisplayValue('Payload signatures blob', signature_msg)
+ signatures_blob = self.payload.ReadDataBlob(manifest.signatures_offset,
+ manifest.signatures_size)
+ self._DisplaySignaturesBlob('Payload', signatures_blob)
+ else:
+ print('No payload signatures stored in the payload')
+
+ @staticmethod
+ def _DisplaySignaturesBlob(signature_name, signatures_blob):
from dev.host.lib.update_payload import update_metadata_pb2
signatures = update_metadata_pb2.Signatures()
signatures.ParseFromString(signatures_blob)
- print('Payload signatures: (%d entries)' % len(signatures.signatures))
+ print('%s signatures: (%d entries)' %
+ (signature_name, len(signatures.signatures)))
for signature in signatures.signatures:
print(' version=%s, hex_data: (%d bytes)' %
(signature.version if signature.HasField('version') else None,
@@ -162,8 +185,12 @@ Example:
read_blocks = 0
written_blocks = 0
num_write_seeks = 0
- for operations in (manifest.install_operations,
- manifest.kernel_install_operations):
+ if self.payload.header.version == MAJOR_PAYLOAD_VERSION_BRILLO:
+ partitions_operations = [part.operations for part in manifest.partitions]
+ else:
+ partitions_operations = [manifest.install_operations,
+ manifest.kernel_install_operations]
+ for operations in partitions_operations:
last_ext = None
for curr_op in operations:
read_blocks += sum([ext.num_blocks for ext in curr_op.src_extents])
@@ -207,7 +234,12 @@ Example:
self._DisplayStats(self.payload.manifest)
if self.options.list_ops:
print()
- self._DisplayOps('Install operations',
- self.payload.manifest.install_operations)
- self._DisplayOps('Kernel install operations',
- self.payload.manifest.kernel_install_operations)
+ if self.payload.header.version == MAJOR_PAYLOAD_VERSION_BRILLO:
+ for partition in self.payload.manifest.partitions:
+ self._DisplayOps('%s install operations' % partition.partition_name,
+ partition.operations)
+ else:
+ self._DisplayOps('Install operations',
+ self.payload.manifest.install_operations)
+ self._DisplayOps('Kernel install operations',
+ self.payload.manifest.kernel_install_operations)
diff --git a/cli/cros/cros_payload_unittest.py b/cli/cros/cros_payload_unittest.py
index d99f0e9a1..36f4749bd 100644
--- a/cli/cros/cros_payload_unittest.py
+++ b/cli/cros/cros_payload_unittest.py
@@ -50,10 +50,17 @@ class FakeOp(object):
def HasField(self, field):
return hasattr(self, field)
+class FakePartition(object):
+ """Fake PartitionUpdate field for testing."""
+
+ def __init__(self, partition_name, operations):
+ self.partition_name = partition_name
+ self.operations = operations
+
class FakeManifest(object):
"""Fake manifest for testing."""
- def __init__(self):
+ def __init__(self, major_version):
FakeExtent = collections.namedtuple('FakeExtent',
['start_block', 'num_blocks'])
self.install_operations = [FakeOp([],
@@ -67,6 +74,11 @@ class FakeManifest(object):
[FakeExtent(x, x) for x in xrange(20)],
update_payload.common.OpType.SOURCE_COPY,
src_length=4096)]
+ if major_version == cros_payload.MAJOR_PAYLOAD_VERSION_BRILLO:
+ self.partitions = [FakePartition('rootfs', self.install_operations),
+ FakePartition('kernel',
+ self.kernel_install_operations)]
+ self.install_operations = self.kernel_install_operations = []
self.block_size = 4096
self.minor_version = 4
FakePartInfo = collections.namedtuple('FakePartInfo', ['size'])
@@ -81,19 +93,32 @@ class FakeManifest(object):
"""Fake HasField method based on the python members."""
return hasattr(self, field_name) and getattr(self, field_name) is not None
+class FakeHeader(object):
+ """Fake payload header for testing."""
+
+ def __init__(self, version, manifest_len, metadata_signature_len):
+ self.version = version
+ self.manifest_len = manifest_len
+ self.metadata_signature_len = metadata_signature_len
+
+ @property
+ def size(self):
+ return (20 if self.version == cros_payload.MAJOR_PAYLOAD_VERSION_CHROMEOS
+ else 24)
+
+
class FakePayload(object):
"""Fake payload for testing."""
- def __init__(self):
- FakeHeader = collections.namedtuple('FakeHeader',
- ['version', 'manifest_len'])
- self._header = FakeHeader('111', 222)
+ def __init__(self, major_version):
+ self._header = FakeHeader(major_version, 222, 0)
self.header = None
- self._manifest = FakeManifest()
+ self._manifest = FakeManifest(major_version)
self.manifest = None
self._blobs = {}
- self._signatures = update_metadata_pb2.Signatures()
+ self._payload_signatures = update_metadata_pb2.Signatures()
+ self._metadata_signatures = update_metadata_pb2.Signatures()
def Init(self):
"""Fake Init that sets header and manifest.
@@ -114,15 +139,28 @@ class FakePayload(object):
'actual: %d)' % (len(blob), length))
return blob
- def AddSignature(self, **kwargs):
- new_signature = self._signatures.signatures.add()
+ @staticmethod
+ def _AddSignatureToProto(proto, **kwargs):
+ """Add a new Signature element to the passed proto."""
+ new_signature = proto.signatures.add()
for key, val in kwargs.iteritems():
setattr(new_signature, key, val)
- blob = self._signatures.SerializeToString()
+
+ def AddPayloadSignature(self, **kwargs):
+ self._AddSignatureToProto(self._payload_signatures, **kwargs)
+ blob = self._payload_signatures.SerializeToString()
self._manifest.signatures_offset = 1234
self._manifest.signatures_size = len(blob)
self._blobs[self._manifest.signatures_offset] = blob
+ def AddMetadataSignature(self, **kwargs):
+ self._AddSignatureToProto(self._metadata_signatures, **kwargs)
+ if self._header.metadata_signature_len:
+ del self._blobs[-self._header.metadata_signature_len]
+ blob = self._metadata_signatures.SerializeToString()
+ self._header.metadata_signature_len = len(blob)
+ self._blobs[-len(blob)] = blob
+
class PayloadCommandTest(cros_test_lib.MockOutputTestCase):
"""Test class for our PayloadCommand class."""
@@ -137,13 +175,14 @@ class PayloadCommandTest(cros_test_lib.MockOutputTestCase):
def testRun(self):
"""Verify that Run parses and displays the payload like we expect."""
payload_cmd = cros_payload.PayloadCommand(FakeOption(action='show'))
- self.PatchObject(update_payload, 'Payload', return_value=FakePayload())
+ self.PatchObject(update_payload, 'Payload', return_value=FakePayload(
+ cros_payload.MAJOR_PAYLOAD_VERSION_CHROMEOS))
with self.OutputCapturer() as output:
payload_cmd.Run()
stdout = output.GetStdout()
- expected_out = """Payload version: 111
+ expected_out = """Payload version: 1
Manifest length: 222
Number of operations: 1
Number of kernel ops: 1
@@ -152,17 +191,18 @@ Minor version: 4
"""
self.assertEquals(stdout, expected_out)
- def testListOps(self):
+ def testListOpsOnVersion1(self):
"""Verify that the --list_ops option gives the correct output."""
payload_cmd = cros_payload.PayloadCommand(FakeOption(list_ops=True,
action='show'))
- self.PatchObject(update_payload, 'Payload', return_value=FakePayload())
+ self.PatchObject(update_payload, 'Payload', return_value=FakePayload(
+ cros_payload.MAJOR_PAYLOAD_VERSION_CHROMEOS))
with self.OutputCapturer() as output:
payload_cmd.Run()
stdout = output.GetStdout()
- expected_out = """Payload version: 111
+ expected_out = """Payload version: 1
Manifest length: 222
Number of operations: 1
Number of kernel ops: 1
@@ -185,17 +225,53 @@ Kernel install operations:
"""
self.assertEquals(stdout, expected_out)
- def testStats(self):
+ def testListOpsOnVersion2(self):
+ """Verify that the --list_ops option gives the correct output."""
+ payload_cmd = cros_payload.PayloadCommand(FakeOption(list_ops=True,
+ action='show'))
+ self.PatchObject(update_payload, 'Payload', return_value=FakePayload(
+ cros_payload.MAJOR_PAYLOAD_VERSION_BRILLO))
+
+ with self.OutputCapturer() as output:
+ payload_cmd.Run()
+
+ stdout = output.GetStdout()
+ expected_out = """Payload version: 2
+Manifest length: 222
+Number of partitions: 2
+ Number of "rootfs" ops: 1
+ Number of "kernel" ops: 1
+Block size: 4096
+Minor version: 4
+
+rootfs install operations:
+ 0: REPLACE_BZ
+ Data offset: 1
+ Data length: 1
+ Destination: 2 extents (3 blocks)
+ (1,1) (2,2)
+kernel install operations:
+ 0: SOURCE_COPY
+ Source: 1 extent (1 block)
+ (1,1)
+ Destination: 20 extents (190 blocks)
+ (0,0) (1,1) (2,2) (3,3) (4,4) (5,5) (6,6) (7,7) (8,8) (9,9) (10,10)
+ (11,11) (12,12) (13,13) (14,14) (15,15) (16,16) (17,17) (18,18) (19,19)
+"""
+ self.assertEquals(stdout, expected_out)
+
+ def testStatsOnVersion1(self):
"""Verify that the --stats option works correctly."""
payload_cmd = cros_payload.PayloadCommand(FakeOption(stats=True,
action='show'))
- self.PatchObject(update_payload, 'Payload', return_value=FakePayload())
+ self.PatchObject(update_payload, 'Payload', return_value=FakePayload(
+ cros_payload.MAJOR_PAYLOAD_VERSION_CHROMEOS))
with self.OutputCapturer() as output:
payload_cmd.Run()
stdout = output.GetStdout()
- expected_out = """Payload version: 111
+ expected_out = """Payload version: 1
Manifest length: 222
Number of operations: 1
Number of kernel ops: 1
@@ -207,23 +283,49 @@ Seeks when writing: 18
"""
self.assertEquals(stdout, expected_out)
+ def testStatsOnVersion2(self):
+ """Verify that the --stats option works correctly on version 2."""
+ payload_cmd = cros_payload.PayloadCommand(FakeOption(stats=True,
+ action='show'))
+ self.PatchObject(update_payload, 'Payload', return_value=FakePayload(
+ cros_payload.MAJOR_PAYLOAD_VERSION_BRILLO))
+
+ with self.OutputCapturer() as output:
+ payload_cmd.Run()
+
+ stdout = output.GetStdout()
+ expected_out = """Payload version: 2
+Manifest length: 222
+Number of partitions: 2
+ Number of "rootfs" ops: 1
+ Number of "kernel" ops: 1
+Block size: 4096
+Minor version: 4
+Blocks read: 11
+Blocks written: 193
+Seeks when writing: 18
+"""
+ self.assertEquals(stdout, expected_out)
+
def testEmptySignatures(self):
"""Verify that the --signatures option works with unsigned payloads."""
payload_cmd = cros_payload.PayloadCommand(
FakeOption(action='show', signatures=True))
- self.PatchObject(update_payload, 'Payload', return_value=FakePayload())
+ self.PatchObject(update_payload, 'Payload', return_value=FakePayload(
+ cros_payload.MAJOR_PAYLOAD_VERSION_CHROMEOS))
with self.OutputCapturer() as output:
payload_cmd.Run()
stdout = output.GetStdout()
- expected_out = """Payload version: 111
+ expected_out = """Payload version: 1
Manifest length: 222
Number of operations: 1
Number of kernel ops: 1
Block size: 4096
Minor version: 4
-No signatures stored in the payload
+No metadata signatures stored in the payload
+No payload signatures stored in the payload
"""
self.assertEquals(stdout, expected_out)
@@ -232,22 +334,29 @@ No signatures stored in the payload
"""Verify that the --signatures option shows the present signatures."""
payload_cmd = cros_payload.PayloadCommand(
FakeOption(action='show', signatures=True))
- payload = FakePayload()
- payload.AddSignature(version=1, data='12345678abcdefgh\x00\x01\x02\x03')
- payload.AddSignature(data='I am a signature so access is yes.')
+ payload = FakePayload(cros_payload.MAJOR_PAYLOAD_VERSION_BRILLO)
+ payload.AddPayloadSignature(version=1,
+ data='12345678abcdefgh\x00\x01\x02\x03')
+ payload.AddPayloadSignature(data='I am a signature so access is yes.')
+ payload.AddMetadataSignature(data='\x00\x0a\x0c')
self.PatchObject(update_payload, 'Payload', return_value=payload)
with self.OutputCapturer() as output:
payload_cmd.Run()
stdout = output.GetStdout()
- expected_out = """Payload version: 111
+ expected_out = """Payload version: 2
Manifest length: 222
-Number of operations: 1
-Number of kernel ops: 1
+Number of partitions: 2
+ Number of "rootfs" ops: 1
+ Number of "kernel" ops: 1
Block size: 4096
Minor version: 4
-Signature blob: offset=1234 (64 bytes)
+Metadata signatures blob: file_offset=246 (7 bytes)
+Metadata signatures: (1 entries)
+ version=None, hex_data: (3 bytes)
+ 00 0a 0c | ...
+Payload signatures blob: blob_offset=1234 (64 bytes)
Payload signatures: (2 entries)
version=1, hex_data: (20 bytes)
31 32 33 34 35 36 37 38 61 62 63 64 65 66 67 68 | 12345678abcdefgh
diff --git a/cli/cros/lint.py b/cli/cros/lint.py
index f54c8a6b9..36d07e8e1 100644
--- a/cli/cros/lint.py
+++ b/cli/cros/lint.py
@@ -385,6 +385,8 @@ class SourceChecker(BaseChecker):
class _MessageR9200(object): pass
class _MessageR9201(object): pass
class _MessageR9202(object): pass
+ class _MessageR9203(object): pass
+ class _MessageR9210(object): pass
# pylint: enable=class-missing-docstring,multiple-statements
name = 'source_checker'
@@ -397,6 +399,10 @@ class SourceChecker(BaseChecker):
('missing-shebang'), _MessageR9201),
'R9202': ('Shebang is set, but file is not executable',
('spurious-shebang'), _MessageR9202),
+ 'R9203': ('Unittest not named xxx_unittest.py',
+ ('unittest-misnamed'), _MessageR9203),
+ 'R9210': ('Trailing new lines found at end of file',
+ ('excess-trailing-newlines'), _MessageR9210),
}
options = ()
@@ -405,6 +411,8 @@ class SourceChecker(BaseChecker):
stream = node.file_stream
stream.seek(0)
self._check_shebang(node, stream)
+ self._check_module_name(node)
+ self._check_trailing_lines(node, stream)
def _check_shebang(self, _node, stream):
"""Verify the shebang is version specific"""
@@ -424,6 +432,21 @@ class SourceChecker(BaseChecker):
if parts[0] not in ('#!/usr/bin/python2', '#!/usr/bin/python3'):
self.add_message('R9200')
+ def _check_module_name(self, node):
+ """Make sure the module name is sane"""
+ # Catch various typos.
+ name = node.name.rsplit('.', 2)[-1]
+ if name.rsplit('_', 2)[-1] in ('unittests',):
+ self.add_message('R9203')
+
+ def _check_trailing_lines(self, _node, stream):
+ """Reject trailing lines"""
+ st = os.fstat(stream.fileno())
+ if st.st_size > 1:
+ stream.seek(st.st_size - 2)
+ if not stream.read().strip('\n'):
+ self.add_message('R9210')
+
class ChromiteLoggingChecker(BaseChecker):
"""Make sure we enforce rules on importing logging."""
diff --git a/cli/cros/lint_unittest.py b/cli/cros/lint_unittest.py
index b78c6553e..20b86c78f 100644
--- a/cli/cros/lint_unittest.py
+++ b/cli/cros/lint_unittest.py
@@ -23,7 +23,7 @@ class TestNode(object):
Arg = collections.namedtuple('Arg', ('name',))
def __init__(self, doc='', fromlineno=0, path='foo.py', args=(), vararg='',
- kwarg='', names=None, lineno=0):
+ kwarg='', names=None, lineno=0, name='module'):
if names is None:
names = [('name', None)]
self.doc = doc
@@ -34,6 +34,7 @@ class TestNode(object):
self.args = self.Args(args=[self.Arg(name=x) for x in args],
vararg=vararg, kwarg=kwarg)
self.names = names
+ self.name = name
def argnames(self):
return self.args
@@ -396,3 +397,25 @@ class SourceCheckerTest(CheckerTestCase):
)
with open('/bin/sh') as f:
self._testShebang(shebangs, 0, f.fileno())
+
+ def testGoodUnittestName(self):
+ """Verify _check_module_name accepts good unittest names"""
+ module_names = (
+ 'lint_unittest',
+ )
+ for name in module_names:
+ node = TestNode(name=name)
+ self.results = []
+ self.checker._check_module_name(node)
+ self.assertEqual(len(self.results), 0)
+
+ def testBadUnittestName(self):
+ """Verify _check_module_name accepts good unittest names"""
+ module_names = (
+ 'lint_unittests',
+ )
+ for name in module_names:
+ node = TestNode(name=name)
+ self.results = []
+ self.checker._check_module_name(node)
+ self.assertEqual(len(self.results), 1)
diff --git a/cli/flash_unittest.py b/cli/flash_unittest.py
index 2a7a86866..7a18d4f65 100644
--- a/cli/flash_unittest.py
+++ b/cli/flash_unittest.py
@@ -19,7 +19,6 @@ from chromite.lib import dev_server_wrapper
from chromite.lib import osutils
from chromite.lib import partial_mock
from chromite.lib import remote_access
-from chromite.lib import workspace_lib
class RemoteDeviceUpdaterMock(partial_mock.PartialCmdMock):
@@ -62,7 +61,6 @@ class RemoteDeviceUpdaterTest(cros_test_lib.MockTempDirTestCase):
self.PatchObject(dev_server_wrapper, 'GetUpdatePayloads')
self.PatchObject(remote_access, 'CHECK_INTERVAL', new=0)
self.PatchObject(remote_access, 'ChromiumOSDevice')
- self.PatchObject(workspace_lib, 'WorkspacePath', return_value=None)
def testUpdateAll(self):
"""Tests that update methods are called correctly."""
@@ -141,7 +139,6 @@ class USBImagerTest(cros_test_lib.MockTempDirTestCase):
self.PatchObject(os.path, 'exists', return_value=True)
self.isgpt_mock = self.PatchObject(flash, '_IsFilePathGPTDiskImage',
return_value=True)
- self.PatchObject(workspace_lib, 'WorkspacePath', return_value=None)
def testLocalImagePathCopy(self):
"""Tests that imaging methods are called correctly."""
diff --git a/compute/bot_constants.py b/compute/bot_constants.py
index aa8e1ef51..286bef5b0 100644
--- a/compute/bot_constants.py
+++ b/compute/bot_constants.py
@@ -15,8 +15,10 @@ BUILDBOT_USER = 'chrome-bot'
CHROMITE_URL = 'https://chromium.googlesource.com/chromiumos/chromite'
DEPOT_TOOLS_URL = ('https://chromium.googlesource.com/chromium/tools/'
'depot_tools.git')
-BUILDBOT_GIT_REPO = ('https://chrome-internal.googlesource.com/chrome/tools/'
- 'build/internal.DEPS')
+CHROME_INFRA_SLAVE_REPO = ('https://chrome-internal.googlesource.com/chrome/'
+ 'tools/build/internal.DEPS.git')
+CHROME_INFRA_SLAVE_DEPS_FILE = '.DEPS.git'
+
CHROMIUM_BUILD_URL = 'https://chromium.googlesource.com/chromium/src/build'
GCOMPUTE_TOOLS_URL = 'https://gerrit.googlesource.com/gcompute-tools'
diff --git a/compute/compute_configs.py b/compute/compute_configs.py
index 502db86ff..fece3b521 100644
--- a/compute/compute_configs.py
+++ b/compute/compute_configs.py
@@ -6,7 +6,7 @@
from __future__ import print_function
-from chromite.cbuildbot import constants
+from chromite.cbuildbot import config_lib
# Metadata keys to tag our GCE artifacts with.
@@ -23,7 +23,8 @@ DEFAULT_SCOPES = ('https://www.googleapis.com/auth/devstorage.full_control',
# TODO: We do not archive the official images to Google Storage yet
# because the imaging creating process for this path does not allow
# the rootfs to be larger than 10GB.
-GS_IMAGE_ARCHIVE_BASE_URL = '%s/gce-images' % constants.DEFAULT_ARCHIVE_BUCKET
+GS_IMAGE_ARCHIVE_BASE_URL = ('%s/gce-images' %
+ config_lib.GetConfig().params.ARCHIVE_URL)
IMAGE_SUFFIX = '.tar.gz'
BOOT_DISK = '/dev/sda'
diff --git a/compute/setup_bot.py b/compute/setup_bot.py
index ff4e92376..84d5fe923 100644
--- a/compute/setup_bot.py
+++ b/compute/setup_bot.py
@@ -191,7 +191,8 @@ def SetupBuildbotEnvironment():
# `gclient` relies on depot_tools in $PATH, pass the extra
# envinornment variable.
path_env = '%s:%s' % (os.getenv('PATH'), tmp_depot_tools_path)
- RunCommand(['gclient', 'config', bot_constants.BUILDBOT_GIT_REPO],
+ RunCommand(['gclient', 'config', bot_constants.CHROME_INFRA_SLAVE_REPO,
+ '--deps-file', bot_constants.CHROME_INFRA_SLAVE_DEPS_FILE],
cwd=bot_constants.BUILDBOT_DIR, extra_env={'PATH': path_env})
RunCommand(['gclient', 'sync', '--jobs', '5'],
cwd=bot_constants.BUILDBOT_DIR,
diff --git a/lib/blueprint_lib.py b/lib/blueprint_lib.py
deleted file mode 100644
index befe94c89..000000000
--- a/lib/blueprint_lib.py
+++ /dev/null
@@ -1,159 +0,0 @@
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Utilities to work with blueprints."""
-
-from __future__ import print_function
-
-import os
-
-from chromite.lib import brick_lib
-from chromite.lib import workspace_lib
-
-
-# Field names for specifying initial configuration.
-APP_ID_FIELD = 'buildTargetId'
-BRICKS_FIELD = 'bricks'
-BSP_FIELD = 'bsp'
-
-# Those packages are implicitly built for all blueprints.
-# - target-os is needed to build any image.
-# - target-os-dev and target-os-test are needed to build a developer friendly
-# image. They should not be included in any production images.
-_IMPLICIT_PACKAGES = (
- 'virtual/target-os',
- 'virtual/target-os-dev',
- 'virtual/target-os-test',
-)
-
-
-class BlueprintNotFoundError(Exception):
- """The blueprint does not exist."""
-
-
-class BlueprintCreationError(Exception):
- """Blueprint creation failed."""
-
-
-class Blueprint(object):
- """Encapsulates the interaction with a blueprint."""
-
- def __init__(self, blueprint_loc, initial_config=None):
- """Instantiates a blueprint object.
-
- Args:
- blueprint_loc: blueprint locator. This can be a relative path to CWD, an
- absolute path, or a relative path to the root of the workspace prefixed
- with '//'.
- initial_config: A dictionary of key-value pairs to seed a new blueprint
- with if the specified blueprint doesn't already exist.
-
- Raises:
- BlueprintNotFoundError: No blueprint exists at |blueprint_loc| and no
- |initial_config| was given to create a new one.
- BlueprintCreationError: |initial_config| was specified but a file
- already exists at |blueprint_loc|.
- """
- self._path = (workspace_lib.LocatorToPath(blueprint_loc)
- if workspace_lib.IsLocator(blueprint_loc) else blueprint_loc)
- self._locator = workspace_lib.PathToLocator(self._path)
-
- if initial_config is not None:
- self._CreateBlueprintConfig(initial_config)
-
- try:
- self.config = workspace_lib.ReadConfigFile(self._path)
- except IOError:
- raise BlueprintNotFoundError('Blueprint %s not found.' % self._path)
-
- @property
- def path(self):
- return self._path
-
- @property
- def locator(self):
- return self._locator
-
- def _CreateBlueprintConfig(self, config):
- """Create an initial blueprint config file.
-
- Converts all brick paths in |config| into locators then saves the
- configuration file to |self._path|.
-
- Currently fails if |self._path| already exists, but could be
- generalized to allow re-writing config files if needed.
-
- Args:
- config: configuration dictionary.
-
- Raises:
- BlueprintCreationError: A brick in |config| doesn't exist or an
- error occurred while saving the config file.
- """
- if os.path.exists(self._path):
- raise BlueprintCreationError('File already exists at %s.' % self._path)
-
- try:
- # Turn brick specifications into locators. If bricks or BSPs are
- # unspecified, assign default values so the config file has the proper
- # structure for easy manual editing.
- if config.get(BRICKS_FIELD):
- config[BRICKS_FIELD] = [brick_lib.Brick(b).brick_locator
- for b in config[BRICKS_FIELD]]
- else:
- config[BRICKS_FIELD] = []
- if config.get(BSP_FIELD):
- config[BSP_FIELD] = brick_lib.Brick(config[BSP_FIELD]).brick_locator
- else:
- config[BSP_FIELD] = None
-
- # Create the config file.
- workspace_lib.WriteConfigFile(self._path, config)
- except (brick_lib.BrickNotFound, workspace_lib.ConfigFileError) as e:
- raise BlueprintCreationError('Blueprint creation failed. %s' % e)
-
- def GetBricks(self):
- """Returns the bricks field of a blueprint."""
- return self.config.get(BRICKS_FIELD, [])
-
- def GetBSP(self):
- """Returns the BSP field of a blueprint."""
- return self.config.get(BSP_FIELD)
-
- def GetAppId(self):
- """Returns the APP_ID from a blueprint."""
- app_id = self.config.get(APP_ID_FIELD)
- return app_id
-
- def FriendlyName(self):
- """Returns the friendly name for this blueprint."""
- return workspace_lib.LocatorToFriendlyName(self._locator)
-
- def GetUsedBricks(self):
- """Returns the set of bricks used by this blueprint."""
- brick_map = {}
- for top_brick in self.GetBricks() + [self.GetBSP()]:
- for b in brick_lib.Brick(top_brick).BrickStack():
- brick_map[b.brick_locator] = b
-
- return brick_map.values()
-
- def GetPackages(self, with_implicit=True):
- """Returns the list of packages needed by this blueprint.
-
- This includes the main packages for the bricks and the bsp of this
- blueprint. We don't add the main packages of the bricks dependencies to
- allow inheriting a brick without inheriting its required packages.
-
- Args:
- with_implicit: If True, include packages that are implicitly required by
- the core system.
- """
- packages = []
- for locator in self.GetBricks() + [self.GetBSP()]:
- packages.extend(brick_lib.Brick(locator).MainPackages())
-
- if with_implicit:
- packages.extend(_IMPLICIT_PACKAGES)
- return packages
diff --git a/lib/blueprint_lib_unittest b/lib/blueprint_lib_unittest
deleted file mode 120000
index 72196ceea..000000000
--- a/lib/blueprint_lib_unittest
+++ /dev/null
@@ -1 +0,0 @@
-../scripts/wrapper.py \ No newline at end of file
diff --git a/lib/blueprint_lib_unittest.py b/lib/blueprint_lib_unittest.py
deleted file mode 100644
index d4c238742..000000000
--- a/lib/blueprint_lib_unittest.py
+++ /dev/null
@@ -1,121 +0,0 @@
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Tests for the blueprint library."""
-
-from __future__ import print_function
-
-from chromite.lib import blueprint_lib
-from chromite.lib import brick_lib
-from chromite.lib import cros_test_lib
-from chromite.lib import osutils
-from chromite.lib import workspace_lib
-
-
-class BlueprintLibTest(cros_test_lib.WorkspaceTestCase):
- """Unittest for blueprint_lib.py"""
-
- def setUp(self):
- self.CreateWorkspace()
-
- def testBlueprint(self):
- """Tests getting the basic blueprint getters."""
- bricks = ['//foo', '//bar', '//baz']
- for brick in bricks:
- self.CreateBrick(brick)
- self.CreateBrick('//bsp')
- blueprint = self.CreateBlueprint(bricks=bricks, bsp='//bsp')
- self.assertEqual(blueprint.GetBricks(), bricks)
- self.assertEqual(blueprint.GetBSP(), '//bsp')
-
- def testBlueprintNoBricks(self):
- """Tests that blueprints without bricks return reasonable defaults."""
- self.CreateBrick('//bsp2')
- blueprint = self.CreateBlueprint(bsp='//bsp2')
- self.assertEqual(blueprint.GetBricks(), [])
- self.assertEqual(blueprint.GetBSP(), '//bsp2')
-
- def testEmptyBlueprintFile(self):
- """Tests that empty blueprints create the basic file structure."""
- blueprint = self.CreateBlueprint()
- file_contents = workspace_lib.ReadConfigFile(blueprint.path)
-
- self.assertIn(blueprint_lib.BRICKS_FIELD, file_contents)
- self.assertIn(blueprint_lib.BSP_FIELD, file_contents)
-
- def testGetUsedBricks(self):
- """Tests that we can list all the bricks used."""
- brick_lib.Brick('//a', initial_config={'name':'a'})
- brick_b = brick_lib.Brick('//b', initial_config={'name':'b'})
- brick_c = brick_lib.Brick('//c',
- initial_config={'name':'c',
- 'dependencies': ['//b']})
-
- blueprint = self.CreateBlueprint(name='foo.json',
- bsp='//a', bricks=[brick_c.brick_locator])
- self.assertEqual(3, len(blueprint.GetUsedBricks()))
-
- # We sort out duplicates: c depends on b and b is explicitly listed in
- # bricks too.
- blueprint = self.CreateBlueprint(name='bar.json',
- bsp='//a', bricks=[brick_c.brick_locator,
- brick_b.brick_locator])
- self.assertEqual(3, len(blueprint.GetUsedBricks()))
-
- def testGetPackages(self):
- """Tests that we can get the needed packages for a given blueprint."""
- self.CreateBrick('foo', main_package='app-misc/foopkg')
- self.CreateBrick('bar', main_package='app-misc/barpkg')
- self.CreateBrick('foobar', main_package='app-misc/foobarpkg',
- dependencies=['//foo', '//bar'])
-
- self.CreateBrick('hello', main_package='app-misc/hello')
-
- self.CreateBrick('mybsp', main_package='app-misc/bspbonjour')
-
- blueprint = self.CreateBlueprint(name='//myblueprint',
- bricks=['//hello', '//foobar'],
- bsp='//mybsp')
- packages = blueprint.GetPackages(with_implicit=False)
- self.assertEqual(
- set(('app-misc/foobarpkg', 'app-misc/hello', 'app-misc/bspbonjour')),
- set(packages))
-
- packages = blueprint.GetPackages(with_implicit=True)
- self.assertTrue('virtual/target-os' in packages)
- self.assertTrue('virtual/target-os-dev' in packages)
- self.assertTrue('virtual/target-os-test' in packages)
-
- def testBlueprintAlreadyExists(self):
- """Tests creating a blueprint where one already exists."""
- self.CreateBrick('//foo')
- self.CreateBrick('//bar')
- self.CreateBlueprint(name='//my_blueprint', bricks=['//foo'])
- with self.assertRaises(blueprint_lib.BlueprintCreationError):
- self.CreateBlueprint(name='//my_blueprint', bricks=['//bar'])
- # Make sure the original blueprint is untouched.
- self.assertEqual(['//foo'],
- blueprint_lib.Blueprint('//my_blueprint').GetBricks())
-
- def testBlueprintBrickNotFound(self):
- """Tests creating a blueprint with a non-existent brick fails."""
- with self.assertRaises(blueprint_lib.BlueprintCreationError):
- self.CreateBlueprint(name='//my_blueprint', bricks=['//none'])
-
- def testBlueprintBSPNotFound(self):
- """Tests creating a blueprint with a non-existent BSP fails."""
- with self.assertRaises(blueprint_lib.BlueprintCreationError):
- self.CreateBlueprint(name='//my_blueprint', bsp='//none')
-
- def testBlueprintNotFound(self):
- """Tests loading a non-existent blueprint file."""
- with self.assertRaises(blueprint_lib.BlueprintNotFoundError):
- blueprint_lib.Blueprint('//not/a/blueprint')
-
- def testInvalidBlueprint(self):
- """Tests loading an invalid blueprint file."""
- path = workspace_lib.LocatorToPath('//invalid_file')
- osutils.WriteFile(path, 'invalid contents')
- with self.assertRaises(workspace_lib.ConfigFileError):
- blueprint_lib.Blueprint(path)
diff --git a/lib/bootstrap_lib.py b/lib/bootstrap_lib.py
deleted file mode 100644
index 9734f2ecb..000000000
--- a/lib/bootstrap_lib.py
+++ /dev/null
@@ -1,101 +0,0 @@
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Common utilities used by the chromium/bootstrap scripts."""
-
-from __future__ import print_function
-
-import os
-
-from chromite.lib import cros_build_lib
-from chromite.lib import project_sdk
-from chromite.lib import workspace_lib
-
-
-# This is the subdirectory of the bootstrap, where we store SDKs.
-SDK_CHECKOUTS = 'sdk_checkouts'
-
-
-# This env is used to remember the bootstrap path in child processes.
-BOOTSTRAP_PATH_ENV = 'BRILLO_BOOTSTRAP_PATH'
-
-
-def FindBootstrapPath(save_to_env=False):
- """Find the bootstrap directory.
-
- This is only possible, if the process was initially launched from a bootstrap
- environment, and isn't inside a chroot.
-
- Args:
- save_to_env: If true, preserve the bootstrap path in an ENV for child
- processes. Only intended for the brillo bootstrap wrapper.
-
- Returns:
- Path to root of bootstrap, or None.
- """
- # We never have access to bootstrap if we are inside the chroot.
- if cros_build_lib.IsInsideChroot():
- return None
-
- # See if the path has already been determined, especially in a parent wrapper
- # process.
- env_path = os.environ.get(BOOTSTRAP_PATH_ENV)
- if env_path:
- return env_path
-
- # Base the bootstrap location on our current location, and remember it.
- new_path = os.path.realpath(os.path.join(
- os.path.abspath(__file__), '..', '..'))
-
- # No repo checkout is a valid bootstrap environment, because the bootstrap
- # environment holds repo checkouts inside SDK_CHECKOUTS, and repos cannot
- # exist inside other repos.
- if project_sdk.FindRepoRoot(new_path):
- return None
-
- if save_to_env:
- os.environ[BOOTSTRAP_PATH_ENV] = new_path
-
- return new_path
-
-
-def ComputeSdkPath(bootstrap_path, version):
- """What directory should an SDK be in.
-
- Args:
- bootstrap_path: Bootstrap root directory, or None.
- version: Version of the SDK.
-
- Returns:
- Path in which SDK version should be stored, or None.
- """
- if bootstrap_path is None:
- return None
-
- return os.path.join(bootstrap_path, SDK_CHECKOUTS, version)
-
-
-def GetActiveSdkPath(bootstrap_path, workspace_path):
- """Find the SDK Path associated with a given workspace.
-
- Most code should use constants.SOURCE_ROOT instead.
-
- Args:
- bootstrap_path: Path directory of the bootstrap dir (FindBootstrapPath()).
- workspace_path: Path directory of the workspace (FindWorkspacePath()).
-
- Returns:
- Path to root directory of SDK, if there is an active one, and it exists.
- """
- if bootstrap_path is None:
- return None
-
- version = workspace_lib.GetActiveSdkVersion(workspace_path)
- if version is None:
- return None
-
- sdk_root = ComputeSdkPath(bootstrap_path, version)
-
- # Returns None if there is no active SDK version, or if it's not installed.
- return sdk_root if os.path.isdir(sdk_root) else None
diff --git a/lib/bootstrap_lib_unittest b/lib/bootstrap_lib_unittest
deleted file mode 120000
index 72196ceea..000000000
--- a/lib/bootstrap_lib_unittest
+++ /dev/null
@@ -1 +0,0 @@
-../scripts/wrapper.py \ No newline at end of file
diff --git a/lib/bootstrap_lib_unittest.py b/lib/bootstrap_lib_unittest.py
deleted file mode 100644
index a91353327..000000000
--- a/lib/bootstrap_lib_unittest.py
+++ /dev/null
@@ -1,128 +0,0 @@
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Tests for the project_sdk library."""
-
-from __future__ import print_function
-
-import mock
-import os
-
-from chromite.cbuildbot import constants
-from chromite.lib import cros_build_lib
-from chromite.lib import cros_test_lib
-from chromite.lib import osutils
-from chromite.lib import bootstrap_lib
-from chromite.lib import project_sdk
-from chromite.lib import workspace_lib
-
-# pylint: disable=protected-access
-
-class ProjectSdkTest(cros_test_lib.WorkspaceTestCase):
- """Unittest for bootstrap_lib.py"""
-
- def setUp(self):
- self.version = '1.2.3'
-
- # Don't use "CreateBootstrap" since it mocks out the method we are testing.
- self.bootstrap_path = os.path.join(self.tempdir, 'bootstrap')
- self.CreateWorkspace()
-
-
- @mock.patch.object(project_sdk, 'FindRepoRoot')
- @mock.patch.object(cros_build_lib, 'IsInsideChroot')
- def _RunFindBootstrapPath(self, env, repo, chroot,
- expected_path, expected_env,
- mock_chroot, mock_repo):
-
- orig_env = os.environ.copy()
-
- try:
- # Setup the ENV as requested.
- if env is not None:
- os.environ[bootstrap_lib.BOOTSTRAP_PATH_ENV] = env
- else:
- os.environ.pop(bootstrap_lib.BOOTSTRAP_PATH_ENV, None)
-
- # Setup mocks, as requested.
- mock_repo.return_value = repo
- mock_chroot.return_value = chroot
-
- # Verify that ENV is modified, if save is False.
- self.assertEqual(bootstrap_lib.FindBootstrapPath(), expected_path)
- self.assertEqual(os.environ.get(bootstrap_lib.BOOTSTRAP_PATH_ENV), env)
-
- # The test environment is fully setup, run the test.
- self.assertEqual(bootstrap_lib.FindBootstrapPath(True), expected_path)
- self.assertEqual(os.environ.get(bootstrap_lib.BOOTSTRAP_PATH_ENV),
- expected_env)
-
- finally:
- # Restore the ENV.
- osutils.SetEnvironment(orig_env)
-
-
- def testFindBootstrapPath(self):
- real_result = constants.CHROMITE_DIR
-
- # Test first call in a bootstrap env. Exact results not verified.
- self._RunFindBootstrapPath(None, None, False,
- real_result, real_result)
-
- # Test first call after bootstrap outside an SDK. Not an expected env.
- self._RunFindBootstrapPath('/foo', None, False,
- '/foo', '/foo')
-
- # Test first call after bootstrap inside an SDK.
- self._RunFindBootstrapPath('/foo', '/', False,
- '/foo', '/foo')
-
- # Test first call without bootstrap inside an SDK. Error Case.
- self._RunFindBootstrapPath(None, '/', False,
- None, None)
-
- # Test all InsideChroot Cases.
- self._RunFindBootstrapPath(None, None, True,
- None, None)
- self._RunFindBootstrapPath('/foo', None, True,
- None, '/foo')
- self._RunFindBootstrapPath('/foo', '/', True,
- None, '/foo')
- self._RunFindBootstrapPath(None, '/', True,
- None, None)
-
- def testComputeSdkPath(self):
- # Try to compute path, with no valid bootstrap path.
- self.assertEqual(None, bootstrap_lib.ComputeSdkPath(None, '1.2.3'))
-
- self.assertEqual(
- '/foo/bootstrap/sdk_checkouts/1.2.3',
- bootstrap_lib.ComputeSdkPath('/foo/bootstrap', '1.2.3'))
-
- def testGetActiveSdkPath(self):
- # Try to find SDK Path with no valid bootstrap path.
- sdk_dir = bootstrap_lib.GetActiveSdkPath(None,
- self.workspace_path)
- self.assertEqual(None, sdk_dir)
-
- # Try to find SDK Path of workspace with no active SDK.
- sdk_dir = bootstrap_lib.GetActiveSdkPath(self.bootstrap_path,
- self.workspace_path)
- self.assertEqual(None, sdk_dir)
-
- # Try to find SDK Path of workspace with active SDK, but SDK doesn't exist.
- workspace_lib.SetActiveSdkVersion(self.workspace_path, self.version)
- sdk_dir = bootstrap_lib.GetActiveSdkPath(self.bootstrap_path,
- self.workspace_path)
- self.assertEqual(None, sdk_dir)
-
- # 'Create' the active SDK.
- expected_sdk_root = bootstrap_lib.ComputeSdkPath(self.bootstrap_path,
- self.version)
- osutils.SafeMakedirs(expected_sdk_root)
-
- # Verify that we can Find it now.
- sdk_dir = bootstrap_lib.GetActiveSdkPath(self.bootstrap_path,
- self.workspace_path)
- self.assertEqual(expected_sdk_root, sdk_dir)
diff --git a/lib/brick_lib.py b/lib/brick_lib.py
deleted file mode 100644
index 092568816..000000000
--- a/lib/brick_lib.py
+++ /dev/null
@@ -1,273 +0,0 @@
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Common brick related utilities."""
-
-from __future__ import print_function
-
-import os
-
-from chromite.lib import osutils
-from chromite.lib import workspace_lib
-
-_DEFAULT_LAYOUT_CONF = {'profile_eapi_when_unspecified': '5-progress',
- 'profile-formats': 'portage-2 profile-default-eapi',
- 'thin-manifests': 'true',
- 'use-manifests': 'true'}
-
-_CONFIG_FILE = 'config.json'
-
-_IGNORED_OVERLAYS = ('portage-stable', 'chromiumos', 'eclass-overlay')
-
-
-class BrickCreationFailed(Exception):
- """The brick creation failed."""
-
-
-class BrickNotFound(Exception):
- """The brick does not exist."""
-
-
-class BrickFeatureNotSupported(Exception):
- """Attempted feature not supported for this brick."""
-
-
-class Brick(object):
- """Encapsulates the interaction with a brick."""
-
- def __init__(self, brick_loc, initial_config=None, allow_legacy=True):
- """Instantiates a brick object.
-
- Args:
- brick_loc: brick locator. This can be a relative path to CWD, an absolute
- path, a public board name prefix with 'board:' or a relative path to the
- root of the workspace, prefixed with '//').
- initial_config: The initial configuration as a python dictionary.
- If not None, creates a brick with this configuration.
- allow_legacy: Allow board overlays, simulating a basic read-only config.
- Ignored if |initial_config| is not None.
-
- Raises:
- ValueError: If |brick_loc| is invalid.
- LocatorNotResolved: |brick_loc| is valid but could not be resolved.
- BrickNotFound: If |brick_loc| does not point to a brick and no initial
- config was provided.
- BrickCreationFailed: when the brick could not be created successfully.
- """
- if workspace_lib.IsLocator(brick_loc):
- self.brick_dir = workspace_lib.LocatorToPath(brick_loc)
- self.brick_locator = brick_loc
- else:
- self.brick_dir = brick_loc
- self.brick_locator = workspace_lib.PathToLocator(brick_loc)
-
- self.config = None
- self.legacy = False
- config_json = os.path.join(self.brick_dir, _CONFIG_FILE)
-
- if not os.path.exists(config_json):
- if initial_config:
- if os.path.exists(self.brick_dir):
- raise BrickCreationFailed('directory %s already exists.'
- % self.brick_dir)
- success = False
- try:
- self.UpdateConfig(initial_config)
- osutils.SafeMakedirs(self.OverlayDir())
- osutils.SafeMakedirs(self.SourceDir())
- success = True
- except BrickNotFound as e:
- # If BrickNotFound was raised, the dependencies contain a missing
- # brick.
- raise BrickCreationFailed('dependency not found %s' % e)
- finally:
- if not success:
- # If the brick creation failed for any reason, cleanup the partially
- # created brick.
- osutils.RmDir(self.brick_dir, ignore_missing=True)
-
- elif allow_legacy:
- self.legacy = True
- try:
- masters = self._ReadLayoutConf().get('masters')
- masters_list = masters.split() if masters else []
-
- # Keep general Chromium OS overlays out of this list as they are
- # handled separately by the build system.
- deps = ['board:' + d for d in masters_list
- if d not in _IGNORED_OVERLAYS]
- self.config = {'name': self._ReadLayoutConf()['repo-name'],
- 'dependencies': deps}
- except (IOError, KeyError):
- pass
-
- if self.config is None:
- raise BrickNotFound('Brick not found at %s' % self.brick_dir)
- elif initial_config is None:
- self.config = workspace_lib.ReadConfigFile(config_json)
- else:
- raise BrickCreationFailed('brick %s already exists.' % self.brick_dir)
-
- self.friendly_name = None
- if not self.legacy:
- self.friendly_name = workspace_lib.LocatorToFriendlyName(
- self.brick_locator)
-
- def _LayoutConfPath(self):
- """Returns the path to the layout.conf file."""
- return os.path.join(self.OverlayDir(), 'metadata', 'layout.conf')
-
- def _WriteLayoutConf(self, content):
- """Writes layout.conf.
-
- Sets unset fields to a sensible default and write |content| in layout.conf
- in the right format.
-
- Args:
- content: dictionary containing the set fields in layout.conf.
- """
- for k, v in _DEFAULT_LAYOUT_CONF.iteritems():
- content.setdefault(k, v)
-
- content_str = ''.join(['%s = %s\n' % (k, v)
- for k, v in content.iteritems()])
- osutils.WriteFile(self._LayoutConfPath(), content_str, makedirs=True)
-
- def _ReadLayoutConf(self):
- """Returns the content of layout.conf as a Python dictionary."""
- def ParseConfLine(line):
- k, _, v = line.partition('=')
- return k.strip(), v.strip() or None
-
- content_str = osutils.ReadFile(self._LayoutConfPath())
- return dict(ParseConfLine(line) for line in content_str.splitlines())
-
- def UpdateConfig(self, config, regenerate=True):
- """Updates the brick's configuration.
-
- Writes |config| to the configuration file.
- If |regenerate| is true, regenerate the portage configuration files in
- this brick to match the new configuration.
-
- Args:
- config: brick configuration as a python dict.
- regenerate: if True, regenerate autogenerated brick files.
- """
- if self.legacy:
- raise BrickFeatureNotSupported(
- 'Cannot update configuration of legacy brick %s' % self.brick_dir)
-
- self.config = config
- # All objects must be unambiguously referenced. Normalize all the
- # dependencies according to the workspace.
- self.config['dependencies'] = [d if workspace_lib.IsLocator(d)
- else workspace_lib.PathToLocator(d)
- for d in self.config.get('dependencies', [])]
-
- workspace_lib.WriteConfigFile(os.path.join(self.brick_dir, _CONFIG_FILE),
- config)
-
- if regenerate:
- self.GeneratePortageConfig()
-
- def GeneratePortageConfig(self):
- """Generates all autogenerated brick files."""
- # We don't generate anything in legacy brick so everything is up-to-date.
- if self.legacy:
- return
-
- deps = [b.config['name'] for b in self.Dependencies()]
-
- self._WriteLayoutConf(
- {'masters': ' '.join(
- ['eclass-overlay', 'portage-stable', 'chromiumos'] + deps),
- 'repo-name': self.config['name']})
-
- def Dependencies(self):
- """Returns the dependent bricks."""
- return [Brick(d) for d in self.config.get('dependencies', [])]
-
- def Inherits(self, brick_name):
- """Checks whether this brick contains |brick_name|.
-
- Args:
- brick_name: The name of the brick to check containment.
-
- Returns:
- Whether |brick_name| is contained in this brick.
- """
- return brick_name in [b.config['name'] for b in self.BrickStack()]
-
- def MainPackages(self):
- """Returns the brick's main package(s).
-
- This finds the 'main_package' property. It nevertheless returns a (single
- element) list as it is easier to work with.
-
- Returns:
- A list of main packages; empty if no main package configured.
- """
- main_package = self.config.get('main_package')
- return [main_package] if main_package else []
-
- def OverlayDir(self):
- """Returns the brick's overlay directory."""
- if self.legacy:
- return self.brick_dir
-
- return os.path.join(self.brick_dir, 'packages')
-
- def SourceDir(self):
- """Returns the project's source directory."""
- return os.path.join(self.brick_dir, 'src')
-
- def FriendlyName(self):
- """Return the friendly name for this brick.
-
- This name is used as the board name for legacy commands (--board).
- """
- if self.friendly_name is None:
- raise BrickFeatureNotSupported()
- return self.friendly_name
-
- def BrickStack(self):
- """Returns the brick stack for this brick.
-
- Returns:
- A list of bricks, respecting the partial ordering of bricks as defined by
- dependencies, ordered from the lowest priority to the highest priority.
- """
- seen = set()
- def _stack(brick):
- seen.add(brick.brick_dir)
- l = []
- for dep in brick.Dependencies():
- if dep.brick_dir not in seen:
- l.extend(_stack(dep))
- l.append(brick)
- return l
-
- return _stack(self)
-
-
-def FindBrickInPath(path=None):
- """Returns the root directory of the brick containing a path.
-
- Return the first parent directory of |path| that is the root of a brick.
- This method is used for brick auto-detection and does not consider legacy.
-
- Args:
- path: path to a directory. If |path| is None, |path| will be set to CWD.
-
- Returns:
- The path to the first parent that is a brick directory if one exist.
- Otherwise return None.
- """
- for p in osutils.IteratePathParents(path or os.getcwd()):
- try:
- return Brick(p, allow_legacy=False)
- except BrickNotFound:
- pass
-
- return None
diff --git a/lib/brick_lib_unittest b/lib/brick_lib_unittest
deleted file mode 120000
index 72196ceea..000000000
--- a/lib/brick_lib_unittest
+++ /dev/null
@@ -1 +0,0 @@
-../scripts/wrapper.py \ No newline at end of file
diff --git a/lib/brick_lib_unittest.py b/lib/brick_lib_unittest.py
deleted file mode 100644
index 9eff5b2b3..000000000
--- a/lib/brick_lib_unittest.py
+++ /dev/null
@@ -1,217 +0,0 @@
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Tests for the brick library."""
-
-from __future__ import print_function
-
-import os
-
-from chromite.cbuildbot import constants
-from chromite.lib import brick_lib
-from chromite.lib import cros_test_lib
-from chromite.lib import osutils
-from chromite.lib import workspace_lib
-
-
-class BrickLibTest(cros_test_lib.WorkspaceTestCase):
- """Unittest for brick.py"""
-
- # pylint: disable=protected-access
-
- def setUp(self):
- self.CreateWorkspace()
-
- def SetupLegacyBrick(self, brick_dir=None, brick_name='foo'):
- """Sets up a legacy brick layout."""
- if brick_dir is None:
- brick_dir = self.workspace_path
- layout_conf = 'repo-name = %s\n' % brick_name
- osutils.WriteFile(os.path.join(brick_dir, 'metadata', 'layout.conf'),
- layout_conf, makedirs=True)
-
- def testLayoutFormat(self):
- """Test that layout.conf is correctly formatted."""
- brick = self.CreateBrick()
- content = {'repo-name': 'hello',
- 'bar': 'foo'}
- brick._WriteLayoutConf(content)
-
- path = os.path.join(brick.OverlayDir(), 'metadata', 'layout.conf')
- layout_conf = osutils.ReadFile(path).split('\n')
-
- expected_lines = ['repo-name = hello',
- 'bar = foo',
- 'profile-formats = portage-2 profile-default-eapi']
- for line in expected_lines:
- self.assertTrue(line in layout_conf)
-
- def testConfigurationGenerated(self):
- """Test that portage's files are generated when the config file changes."""
- brick = self.CreateBrick()
- sample_config = {'name': 'hello',
- 'dependencies': []}
-
- brick.UpdateConfig(sample_config)
-
- self.assertExists(brick._LayoutConfPath())
-
- def testFindBrickInPath(self):
- """Test that we can infer the current brick from the current directory."""
- brick = self.CreateBrick()
- os.remove(os.path.join(brick.brick_dir, brick_lib._CONFIG_FILE))
- brick_dir = os.path.join(self.workspace_path, 'foo', 'bar', 'project')
- expected_name = 'hello'
- brick_lib.Brick(brick_dir, initial_config={'name': 'hello'})
-
- with osutils.ChdirContext(self.workspace_path):
- self.assertEqual(None, brick_lib.FindBrickInPath())
-
- with osutils.ChdirContext(brick_dir):
- self.assertEqual(expected_name,
- brick_lib.FindBrickInPath().config['name'])
-
- subdir = os.path.join(brick_dir, 'sub', 'directory')
- osutils.SafeMakedirs(subdir)
- with osutils.ChdirContext(subdir):
- self.assertEqual(expected_name,
- brick_lib.FindBrickInPath().config['name'])
-
- def testBrickCreation(self):
- """Test that brick initialization throws the right errors."""
- brick = self.CreateBrick()
- with self.assertRaises(brick_lib.BrickCreationFailed):
- brick_lib.Brick(brick.brick_dir, initial_config={})
-
- nonexistingbrick = os.path.join(self.workspace_path, 'foo')
- with self.assertRaises(brick_lib.BrickNotFound):
- brick_lib.Brick(nonexistingbrick)
-
- def testLoadNonExistingBrickFails(self):
- """Tests that trying to load a non-existing brick fails."""
- self.assertRaises(brick_lib.BrickNotFound, brick_lib.Brick,
- self.workspace_path)
-
- def testLoadExistingNormalBrickSucceeds(self):
- """Tests that loading an existing brick works."""
- brick = self.CreateBrick(name='my_brick')
- brick = brick_lib.Brick(brick.brick_dir, allow_legacy=False)
- self.assertEquals('my_brick', brick.config.get('name'))
-
- def testLoadExistingLegacyBrickFailsIfNotAllowed(self):
- """Tests that loading a legacy brick fails when not allowed."""
- self.SetupLegacyBrick()
- with self.assertRaises(brick_lib.BrickNotFound):
- brick_lib.Brick(self.workspace_path, allow_legacy=False)
-
- def testLoadExistingLegacyBrickSucceeds(self):
- """Tests that loading a legacy brick fails when not allowed."""
- self.SetupLegacyBrick()
- brick = brick_lib.Brick(self.workspace_path)
- self.assertEquals('foo', brick.config.get('name'))
-
- def testLegacyBrickUpdateConfigFails(self):
- """Tests that a legacy brick config cannot be updated."""
- self.SetupLegacyBrick()
- brick = brick_lib.Brick(self.workspace_path)
- with self.assertRaises(brick_lib.BrickFeatureNotSupported):
- brick.UpdateConfig({'name': 'bar'})
-
- def testInherits(self):
- """Tests the containment checking works as intended."""
- saved_root = constants.SOURCE_ROOT
-
- try:
- # Mock the source root so that we can create fake legacy overlay.
- constants.SOURCE_ROOT = self.workspace_path
- legacy = os.path.join(self.workspace_path, 'src', 'overlays',
- 'overlay-foobar')
- self.SetupLegacyBrick(brick_dir=legacy, brick_name='foobar')
-
- bar_brick = brick_lib.Brick('//bar', initial_config={'name': 'bar'})
- foo_brick = brick_lib.Brick(
- '//foo', initial_config={'name': 'foo',
- 'dependencies': ['//bar', 'board:foobar']})
-
- self.assertTrue(bar_brick.Inherits('bar'))
- self.assertTrue(foo_brick.Inherits('bar'))
- self.assertFalse(bar_brick.Inherits('foo'))
- self.assertTrue(foo_brick.Inherits('foobar'))
- self.assertFalse(foo_brick.Inherits('dontexist'))
-
- finally:
- constants.SOURCE_ROOT = saved_root
-
- def testOverlayDir(self):
- """Tests that overlay directory is returned correctly."""
- self.assertExists(self.CreateBrick().OverlayDir())
-
- def testOpenUsingLocator(self):
- """Tests that we can open a brick given a locator."""
- brick_lib.Brick(os.path.join(self.workspace_path, 'foo'),
- initial_config={'name': 'foo'})
-
- brick_lib.Brick('//foo')
-
- with self.assertRaises(brick_lib.BrickNotFound):
- brick_lib.Brick('//doesnotexist')
-
- def testCreateUsingLocator(self):
- """Tests that we can create a brick using a locator."""
- brick_lib.Brick('//foobar', initial_config={'name': 'foobar'})
- brick_lib.Brick('//bricks/some/path', initial_config={'name': 'path'})
-
- brick_lib.Brick('//foobar')
- brick_lib.Brick('//bricks/some/path')
-
- brick_lib.Brick(os.path.join(self.workspace_path, 'foobar'))
- brick_lib.Brick(os.path.join(self.workspace_path, 'bricks', 'some', 'path'))
-
- def testFriendlyName(self):
- """Tests that the friendly name generation works."""
- first = brick_lib.Brick('//foo/bar/test', initial_config={'name': 'test'})
- self.assertEqual('foo.bar.test', first.FriendlyName())
-
- second = brick_lib.Brick(os.path.join(self.workspace_path, 'test', 'foo'),
- initial_config={'name': 'foo'})
- self.assertEqual('test.foo', second.FriendlyName())
-
- def testMissingDependency(self):
- """Tests that the brick creation fails when a dependency is missing."""
- with self.assertRaises(brick_lib.BrickCreationFailed):
- brick_lib.Brick('//bar',
- initial_config={'name':'bar',
- 'dependencies':['//dont/exist']})
-
- # If the creation failed, the directory is removed cleanly.
- self.assertFalse(os.path.exists(workspace_lib.LocatorToPath('//bar')))
-
- def testNormalizedDependencies(self):
- """Tests that dependencies are normalized during brick creation."""
- brick_lib.Brick('//foo/bar', initial_config={'name': 'bar'})
- with osutils.ChdirContext(os.path.join(self.workspace_path, 'foo')):
- brick_lib.Brick('//baz', initial_config={'name': 'baz',
- 'dependencies': ['bar']})
-
- deps = brick_lib.Brick('//baz').config['dependencies']
- self.assertEqual(1, len(deps))
- self.assertEqual('//foo/bar', deps[0])
-
- def testBrickStack(self):
- """Tests that the brick stacking is correct."""
- def brick_dep(name, deps):
- config = {'name': os.path.basename(name),
- 'dependencies': deps}
- return brick_lib.Brick(name, initial_config=config)
-
- brick_dep('//first', [])
- brick_dep('//second', ['//first'])
- third = brick_dep('//third', ['//first', '//second'])
- fourth = brick_dep('//fourth', ['//second', '//first'])
-
- self.assertEqual(['//first', '//second', '//third'],
- [b.brick_locator for b in third.BrickStack()])
-
- self.assertEqual(['//first', '//second', '//fourth'],
- [b.brick_locator for b in fourth.BrickStack()])
diff --git a/lib/chrome_util.py b/lib/chrome_util.py
index 1772db699..9c7a929a5 100644
--- a/lib/chrome_util.py
+++ b/lib/chrome_util.py
@@ -308,11 +308,6 @@ C = Conditions
_COPY_PATHS_COMMON = (
Path('chrome_sandbox', mode=0o4755, dest=_CHROME_SANDBOX_DEST),
Path('icudtl.dat'),
- # Set as optional for backwards compatibility.
- Path('lib/libpeerconnection.so',
- exe=True,
- cond=C.StagingFlagSet(_CHROME_INTERNAL_FLAG),
- optional=True),
Path('libffmpegsumo.so', exe=True, optional=True),
Path('libosmesa.so', exe=True, optional=True),
Path('libpdf.so', exe=True, optional=True),
diff --git a/lib/chroot_util.py b/lib/chroot_util.py
index 742e5b63f..603cc3df0 100644
--- a/lib/chroot_util.py
+++ b/lib/chroot_util.py
@@ -9,7 +9,6 @@ from __future__ import print_function
import os
from chromite.cbuildbot import constants
-from chromite.lib import brick_lib
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import sysroot_lib
@@ -148,34 +147,6 @@ def SetupBoard(brick=None, board=None, update_chroot=True,
cros_build_lib.RunCommand(cmd)
-def InitializeSysroots(blueprint):
- """Initialize the sysroots needed by |blueprint|.
-
- Args:
- blueprint: a blueprint_lib.Blueprint object.
- """
- bsp = brick_lib.Brick(blueprint.GetBSP())
-
- # Create the brick stack.
- # Removing duplicates while preserving a sane behaviour is hard:
- # brbug.com/1029.
- brick_stack = []
- for brick_locator in blueprint.GetBricks():
- brick_stack.extend(brick_lib.Brick(brick_locator).BrickStack())
-
- # Regenerate the portage configuration for all bricks used by this blueprint.
- for b in blueprint.GetUsedBricks():
- b.GeneratePortageConfig()
-
- sysroot_path = cros_build_lib.GetSysroot(blueprint.FriendlyName())
-
- sysroot = sysroot_lib.Sysroot(sysroot_path)
- sysroot.CreateSkeleton()
- sysroot.WriteConfig(sysroot.GenerateBrickConfig(brick_stack, bsp))
- sysroot.GeneratePortageConfig()
- sysroot.UpdateToolchain()
-
-
def RunUnittests(sysroot, packages, extra_env=None, verbose=False,
retries=None):
"""Runs the unit tests for |packages|.
diff --git a/lib/cidb.py b/lib/cidb.py
index b06457237..0986c70f6 100644
--- a/lib/cidb.py
+++ b/lib/cidb.py
@@ -33,8 +33,9 @@ from chromite.lib import retry_stats
CIDB_MIGRATIONS_DIR = os.path.join(constants.CHROMITE_DIR, 'cidb',
'migrations')
-_RETRYABLE_OPERATIONAL_ERROR_CODES = (
+_RETRYABLE_OPERATIONAL_ERROR_CODES = frozenset([
1053, # 'Server shutdown in progress'
+ 1205, # 'Lock wait timeout exceeded; try restarting transaction'
2003, # 'Can't connect to MySQL server'
2006, # Error code 2006 'MySQL server has gone away' indicates that
# the connection used was closed or dropped
@@ -44,7 +45,7 @@ _RETRYABLE_OPERATIONAL_ERROR_CODES = (
# whether the query completed before or after the connection
# lost.
2026, # 'SSL connection error: unknown error number'
-)
+])
def _IsRetryableException(e):
@@ -64,14 +65,23 @@ def _IsRetryableException(e):
# on the exception name. See crbug.com/483654
if 'OperationalError' in str(type(e)):
# Unwrap the error till we get to the error raised by the DB backend.
+ # Record each error_code that we encounter along the way.
e_orig = e
- while hasattr(e_orig, 'orig'):
- e_orig = e_orig.orig
- error_code = e_orig.args[0] if len(e_orig.args) > 0 else -1
- if error_code in _RETRYABLE_OPERATIONAL_ERROR_CODES:
- if error_code != 2006:
+ encountered_error_codes = set()
+ while e_orig:
+ if len(e_orig.args) and isinstance(e_orig.args[0], int):
+ encountered_error_codes.add(e_orig.args[0])
+ e_orig = getattr(e_orig, 'orig', None)
+
+ if encountered_error_codes & _RETRYABLE_OPERATIONAL_ERROR_CODES:
+ # Suppress logging of error code 2006 retries. They are routine and
+ # expected, and logging them confuses people.
+ if not 2006 in encountered_error_codes:
logging.info('RETRYING cidb query due to %s.', e)
return True
+ else:
+ logging.info('None of error codes encountered %s are-retryable.',
+ encountered_error_codes)
return False
@@ -323,7 +333,8 @@ class SchemaVersionedMySQLConnection(object):
if self._meta is not None:
return
self._meta = MetaData()
- self._meta.reflect(bind=self._GetEngine())
+ fn = lambda: self._meta.reflect(bind=self._GetEngine())
+ self._RunFunctorWithRetries(fn)
def _Insert(self, table, values):
"""Create and execute a one-row INSERT query.
@@ -515,6 +526,10 @@ class SchemaVersionedMySQLConnection(object):
f = lambda: engine.execute(query, *args, **kwargs)
logging.info('Running cidb query on pid %s, repr(query) starts with %s',
os.getpid(), repr(query)[:100])
+ return self._RunFunctorWithRetries(f)
+
+ def _RunFunctorWithRetries(self, functor):
+ """Run the given |functor| with correct retry parameters."""
return retry_stats.RetryWithStats(
retry_stats.CIDB,
handler=_IsRetryableException,
@@ -523,7 +538,7 @@ class SchemaVersionedMySQLConnection(object):
backoff_factor=self.query_retry_args.backoff_factor,
success_functor=_RetrySuccessHandler,
raise_first_exception_on_failure=False,
- functor=f)
+ functor=functor)
def _GetEngine(self):
"""Get the sqlalchemy engine for this process.
@@ -565,6 +580,10 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
'SELECT c.id, b.id, action, c.reason, build_config, '
'change_number, patch_number, change_source, timestamp FROM '
'clActionTable c JOIN buildTable b ON build_id = b.id ')
+ _SQL_FETCH_MESSAGES = (
+ 'SELECT build_id, build_config, waterfall, builder_name, build_number, '
+ 'message_type, message_subtype, message_value, timestamp, board FROM '
+ 'buildMessageTable c JOIN buildTable b ON build_id = b.id ')
_DATE_FORMAT = '%Y-%m-%d'
NUM_RESULTS_NO_LIMIT = -1
@@ -581,10 +600,10 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
"""
return self._Execute('SELECT NOW()').fetchall()[0][0]
- @minimum_schema(32)
+ @minimum_schema(43)
def InsertBuild(self, builder_name, waterfall, build_number,
build_config, bot_hostname, master_build_id=None,
- timeout_seconds=None):
+ timeout_seconds=None, important=None):
"""Insert a build row.
Args:
@@ -596,6 +615,7 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
master_build_id: (Optional) primary key of master build to this build.
timeout_seconds: (Optional) If provided, total time allocated for this
build. A deadline is recorded in cidb for the current build to end.
+ important: (Optional) If provided, the |important| value for this build.
"""
values = {
'builder_name': builder_name,
@@ -605,7 +625,9 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
'build_config': build_config,
'bot_hostname': bot_hostname,
'start_time': sqlalchemy.func.current_timestamp(),
- 'master_build_id': master_build_id}
+ 'master_build_id': master_build_id,
+ 'important': important,
+ }
if timeout_seconds is not None:
now = self.GetTime()
duration = datetime.timedelta(seconds=timeout_seconds)
@@ -730,6 +752,34 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
'extra_info': extra_info}
return self._Insert('failureTable', values)
+ @minimum_schema(42)
+ def InsertBuildMessage(self, build_id, message_type=None,
+ message_subtype=None, message_value=None, board=None):
+ """Insert a build message into database.
+
+ Args:
+ build_id: primary key of build recording this message.
+ message_type: Optional str name of message type.
+ message_subtype: Optional str name of message subtype.
+ message_value: Optional value of message.
+ board: Optional str name of the board.
+ """
+ if message_type:
+ message_type = message_type[:240]
+ if message_subtype:
+ message_subtype = message_subtype[:240]
+ if message_value:
+ message_value = message_value[:480]
+ if board:
+ board = board[:240]
+
+ values = {'build_id': build_id,
+ 'message_type': message_type,
+ 'message_subtype': message_subtype,
+ 'message_value': message_value,
+ 'board': board}
+ return self._Insert('buildMessageTable', values)
+
@minimum_schema(2)
def UpdateMetadata(self, build_id, metadata):
"""Update the given metadata row in database.
@@ -750,7 +800,8 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
'full_version': versions.get('full'),
'sdk_version': d.get('sdk-versions'),
'toolchain_url': d.get('toolchain-url'),
- 'build_type': d.get('build_type')})
+ 'build_type': d.get('build_type'),
+ 'important': d.get('important')})
@minimum_schema(32)
def ExtendDeadline(self, build_id, timeout_seconds):
@@ -872,7 +923,7 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
(status, build_id, child_config))
- @minimum_schema(2)
+ @minimum_schema(43)
def GetBuildStatus(self, build_id):
"""Gets the status of the build.
@@ -886,7 +937,7 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
statuses = self.GetBuildStatuses([build_id])
return statuses[0] if statuses else None
- @minimum_schema(2)
+ @minimum_schema(43)
def GetBuildStatuses(self, build_ids):
"""Gets the statuses of the builds.
@@ -896,17 +947,17 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
Returns:
A list of dictionary with keys (id, build_config, start_time,
finish_time, status, waterfall, build_number, builder_name,
- platform_version, full_version), or None if no build with this
- id was found.
+ platform_version, full_version, important), or None if no build
+ with this id was found.
"""
return self._SelectWhere(
'buildTable',
'id IN (%s)' % ','.join(str(int(x)) for x in build_ids),
['id', 'build_config', 'start_time', 'finish_time', 'status',
'waterfall', 'build_number', 'builder_name', 'platform_version',
- 'full_version'])
+ 'full_version', 'important'])
- @minimum_schema(2)
+ @minimum_schema(43)
def GetSlaveStatuses(self, master_build_id):
"""Gets the statuses of slave builders to given build.
@@ -916,12 +967,12 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
Returns:
A list containing, for each slave build (row) found, a dictionary
- with keys (id, build_config, start_time, finish_time, status).
+ with keys (id, build_config, start_time, finish_time, status, important).
"""
return self._SelectWhere('buildTable',
'master_build_id = %d' % master_build_id,
['id', 'build_config', 'start_time',
- 'finish_time', 'status'])
+ 'finish_time', 'status', 'important'])
@minimum_schema(30)
def GetSlaveStages(self, master_build_id):
@@ -946,6 +997,32 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
columns = bs_table_columns + ['build_config']
return [dict(zip(columns, values)) for values in results]
+ @minimum_schema(44)
+ def GetSlaveFailures(self, master_build_id):
+ """Gets the failure entries for slave builds to given build.
+
+ Args:
+ master_build_id: build id of the master build to fetch failures
+ for.
+
+ Returns:
+ A list containing, for each failure entry, a dictionary with keys
+ (id, build_stage_id, outer_failure_id, exception_type, exception_message,
+ exception_category, extra_info, timestamp, stage_name, board,
+ stage_status, build_id, master_build_id, builder_name, waterfall,
+ build_number, build_config, build_status, important).
+ """
+ columns = ['id', 'build_stage_id', 'outer_failure_id', 'exception_type',
+ 'exception_message', 'exception_category', 'extra_info',
+ 'timestamp', 'stage_name', 'board', 'stage_status', 'build_id',
+ 'master_build_id', 'builder_name', 'waterfall', 'build_number',
+ 'build_config', 'build_status', 'important']
+ columns_string = ', '.join(columns)
+ results = self._Execute('SELECT %s FROM failureView '
+ 'WHERE master_build_id = %s ' %
+ (columns_string, master_build_id)).fetchall()
+ return [dict(zip(columns, values)) for values in results]
+
@minimum_schema(32)
def GetTimeToDeadline(self, build_id):
"""Gets the time remaining till the deadline for given build_id.
@@ -978,7 +1055,7 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
deadline_past = (r[0][0] == 0)
return 0 if deadline_past else abs(time_remaining.total_seconds())
- @minimum_schema(2)
+ @minimum_schema(43)
def GetBuildHistory(self, build_config, num_results,
ignore_build_id=None, start_date=None, end_date=None,
starting_build_number=None):
@@ -1006,11 +1083,12 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
A sorted list of dicts containing up to |number| dictionaries for
build statuses in descending order. Valid keys in the dictionary are
[id, build_config, buildbot_generation, waterfall, build_number,
- start_time, finish_time, platform_version, full_version, status].
+ start_time, finish_time, platform_version, full_version, status,
+ important].
"""
columns = ['id', 'build_config', 'buildbot_generation', 'waterfall',
'build_number', 'start_time', 'finish_time', 'platform_version',
- 'full_version', 'status']
+ 'full_version', 'status', 'important']
where_clauses = ['build_config = "%s"' % build_config]
if start_date is not None:
@@ -1149,6 +1227,43 @@ class CIDBConnection(SchemaVersionedMySQLConnection):
results = self._Execute('SELECT k, v FROM keyvalTable').fetchall()
return dict(results)
+ @minimum_schema(42)
+ def GetBuildMessages(self, build_id):
+ """Gets build messages from buildMessageTable.
+
+ Args:
+ build_id: The build to get messages for.
+
+ Returns:
+ A list of build message dictionaries, where each dictionary contains
+ keys build_id, build_config, builder_name, build_number, message_type,
+ message_subtype, message_value, timestamp, board.
+ """
+ return self._GetBuildMessagesWithClause('build_id = %s' % build_id)
+
+ @minimum_schema(42)
+ def GetSlaveBuildMessages(self, master_build_id):
+ """Gets build messages from buildMessageTable.
+
+ Args:
+ master_build_id: The build to get all slave messages for.
+
+ Returns:
+ A list of build message dictionaries, where each dictionary contains
+ keys build_id, build_config, waterfall, builder_name, build_number,
+ message_type, message_subtype, message_value, timestamp, board.
+ """
+ return self._GetBuildMessagesWithClause(
+ 'master_build_id = %s' % master_build_id)
+
+ def _GetBuildMessagesWithClause(self, clause):
+ """Private helper method for fetching build messages."""
+ columns = ['build_id', 'build_config', 'waterfall', 'builder_name',
+ 'build_number', 'message_type', 'message_subtype',
+ 'message_value', 'timestamp', 'board']
+ results = self._Execute('%s WHERE %s' % (self._SQL_FETCH_MESSAGES,
+ clause)).fetchall()
+ return [dict(zip(columns, values)) for values in results]
def _INV():
raise AssertionError('CIDB connection factory has been invalidated.')
diff --git a/lib/cidb_integration_test.py b/lib/cidb_integration_test.py
index 121a0d825..3a5731662 100644
--- a/lib/cidb_integration_test.py
+++ b/lib/cidb_integration_test.py
@@ -169,6 +169,11 @@ class CIDBMigrationsTest(CIDBIntegrationTest):
def testWaterfallMigration(self):
"""Test that migrating waterfall from enum to varchar preserves value."""
+ self.skipTest('Skipped obsolete waterfall migration test.')
+ # This test no longer runs. It was used only to confirm the correctness of
+ # migration #41. In #43, the InsertBuild API changes in a way that is not
+ # compatible with this test.
+ # The test code remains in place for demonstration purposes only.
db = self._PrepareFreshDatabase(40)
build_id = db.InsertBuild('my builder', 'chromiumos', _random(),
'my config', 'my bot hostname')
@@ -195,6 +200,58 @@ class CIDBAPITest(CIDBIntegrationTest):
current_db_time = db.GetTime()
self.assertEqual(type(current_db_time), datetime.datetime)
+ def testBuildMessages(self):
+ db = self._PrepareFreshDatabase(45)
+ self.assertEqual([], db.GetBuildMessages(1))
+ master_build_id = db.InsertBuild('builder name',
+ constants.WATERFALL_TRYBOT,
+ 1,
+ 'master',
+ 'hostname')
+ slave_build_id = db.InsertBuild('slave builder name',
+ constants.WATERFALL_TRYBOT,
+ 2,
+ 'slave',
+ 'slave hostname',
+ master_build_id=master_build_id)
+ db.InsertBuildMessage(master_build_id)
+ db.InsertBuildMessage(master_build_id, 'message_type', 'message_subtype',
+ 'message_value', 'board')
+ for i in range(10):
+ db.InsertBuildMessage(slave_build_id,
+ 'message_type', 'message_subtype', str(i), 'board')
+
+ master_messages = db.GetBuildMessages(master_build_id)
+ slave_messages = db.GetSlaveBuildMessages(master_build_id)
+
+ self.assertEqual(2, len(master_messages))
+ self.assertEqual(10, len(slave_messages))
+
+ mm2 = master_messages[1]
+ mm2.pop('timestamp')
+ self.assertEqual({'build_id': master_build_id,
+ 'build_config': 'master',
+ 'waterfall': constants.WATERFALL_TRYBOT,
+ 'builder_name': 'builder name',
+ 'build_number': 1L,
+ 'message_type': 'message_type',
+ 'message_subtype': 'message_subtype',
+ 'message_value': 'message_value',
+ 'board': 'board'},
+ mm2)
+ sm10 = slave_messages[9]
+ sm10.pop('timestamp')
+ self.assertEqual({'build_id': slave_build_id,
+ 'build_config': 'slave',
+ 'waterfall': constants.WATERFALL_TRYBOT,
+ 'builder_name': 'slave builder name',
+ 'build_number': 2L,
+ 'message_type': 'message_type',
+ 'message_subtype': 'message_subtype',
+ 'message_value': '9',
+ 'board': 'board'},
+ sm10)
+
def testGetKeyVals(self):
db = self._PrepareFreshDatabase(40)
# In production we would never insert into this table from a bot, but for
@@ -228,9 +285,9 @@ def GetTestDataSeries(test_data_path):
class DataSeries0Test(CIDBIntegrationTest):
"""Simulate a set of 630 master/slave CQ builds."""
- def testCQWithSchema39(self):
- """Run the CQ test with schema version 39."""
- self._PrepareFreshDatabase(39)
+ def testCQWithSchema44(self):
+ """Run the CQ test with schema version 44."""
+ self._PrepareFreshDatabase(44)
self._runCQTest()
def _runCQTest(self):
@@ -416,7 +473,8 @@ class DataSeries0Test(CIDBIntegrationTest):
def simulate_slave(slave_metadata):
build_id = _SimulateBuildStart(db, slave_metadata,
- master_build_id)
+ master_build_id,
+ important=True)
_SimulateCQBuildFinish(db, slave_metadata, build_id)
logging.debug('Simulated slave build %s on pid %s', build_id,
os.getpid())
@@ -488,6 +546,12 @@ class BuildStagesAndFailureTest(CIDBIntegrationTest):
bot_db.InsertFailure(build_stage_id, type(e).__name__, str(e), category)
self.assertTrue(bot_db.HasBuildStageFailed(build_stage_id))
+ failures = bot_db.GetSlaveFailures(master_build_id)
+ self.assertEqual(len(failures),
+ len(constants.EXCEPTION_CATEGORY_ALL_CATEGORIES))
+ for f in failures:
+ self.assertEqual(f['build_id'], build_id)
+
slave_stages = bot_db.GetSlaveStages(master_build_id)
self.assertEqual(len(slave_stages), 1)
self.assertEqual(slave_stages[0]['status'], 'pass')
@@ -568,7 +632,7 @@ class DataSeries1Test(CIDBIntegrationTest):
"""Simulate a single set of canary builds."""
def runTest(self):
- """Simulate a single set of canary builds with database schema v28."""
+ """Simulate a single set of canary builds with database schema v44."""
metadatas = GetTestDataSeries(SERIES_1_TEST_DATA_PATH)
self.assertEqual(len(metadatas), 18, 'Did not load expected amount of '
'test data')
@@ -576,7 +640,7 @@ class DataSeries1Test(CIDBIntegrationTest):
# Migrate db to specified version. As new schema versions are added,
# migrations to later version can be applied after the test builds are
# simulated, to test that db contents are correctly migrated.
- self._PrepareFreshDatabase(39)
+ self._PrepareFreshDatabase(44)
bot_db = self.LocalCIDBConnection(self.CIDB_USER_BOT)
@@ -663,7 +727,7 @@ def _TranslateStatus(status):
return status
-def _SimulateBuildStart(db, metadata, master_build_id=None):
+def _SimulateBuildStart(db, metadata, master_build_id=None, important=None):
"""Returns build_id for the inserted buildTable entry."""
metadata_dict = metadata.GetDict()
# TODO(akeshet): We are pretending that all these builds were on the internal
@@ -677,7 +741,8 @@ def _SimulateBuildStart(db, metadata, master_build_id=None):
metadata_dict['build-number'],
metadata_dict['bot-config'],
metadata_dict['bot-hostname'],
- master_build_id)
+ master_build_id,
+ important=important)
return build_id
diff --git a/lib/cidb_setup_unittest b/lib/cidb_setup_unittest
deleted file mode 120000
index 72196ceea..000000000
--- a/lib/cidb_setup_unittest
+++ /dev/null
@@ -1 +0,0 @@
-../scripts/wrapper.py \ No newline at end of file
diff --git a/lib/cidb_setup_unittest.py b/lib/cidb_setup_unittest.py
deleted file mode 100644
index 680dfbd6f..000000000
--- a/lib/cidb_setup_unittest.py
+++ /dev/null
@@ -1,120 +0,0 @@
-# Copyright 2014 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Unit tests for cidb.py Setup methods."""
-
-from __future__ import print_function
-
-from chromite.cbuildbot import constants
-from chromite.lib import cidb
-from chromite.lib import cros_test_lib
-from chromite.lib import factory
-
-
-class CIDBConnectionFactoryTest(cros_test_lib.MockTestCase):
- """Test that CIDBConnectionFactory behaves as expected."""
-
- def setUp(self):
- # Ensure that we do not create any live connections in this unit test.
- self.connection_mock = self.PatchObject(cidb, 'CIDBConnection')
- # pylint: disable=W0212
- cidb.CIDBConnectionFactory._ClearCIDBSetup()
-
- def tearDown(self):
- # pylint: disable=protected-access
- cidb.CIDBConnectionFactory._ClearCIDBSetup()
-
- def testGetConnectionBeforeSetup(self):
- """Calling GetConnection before Setup should raise exception."""
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder)
-
- def testSetupProd(self):
- """Test that SetupProd behaves as expected."""
- cidb.CIDBConnectionFactory.SetupProdCidb()
- cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()
-
- # Expected constructor call
- self.connection_mock.assert_called_once_with(constants.CIDB_PROD_BOT_CREDS)
- self.assertTrue(cidb.CIDBConnectionFactory.IsCIDBSetup())
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupProdCidb)
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupDebugCidb)
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupMockCidb)
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupNoCidb)
-
- def testSetupDebug(self):
- """Test that SetupDebug behaves as expected."""
- cidb.CIDBConnectionFactory.SetupDebugCidb()
- cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()
-
- # Expected constructor call
- self.connection_mock.assert_called_once_with(constants.CIDB_DEBUG_BOT_CREDS)
- self.assertTrue(cidb.CIDBConnectionFactory.IsCIDBSetup())
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupProdCidb)
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupDebugCidb)
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupMockCidb)
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupNoCidb)
-
- def testInvalidateSetup(self):
- """Test that cidb connection can be invalidated."""
- cidb.CIDBConnectionFactory.SetupProdCidb()
- cidb.CIDBConnectionFactory.InvalidateCIDBSetup()
- self.assertRaises(AssertionError,
- cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder)
-
- def testSetupMock(self):
- """Test that SetupMock behaves as expected."""
- # Set the CIDB to mock mode, but without supplying a mock
- cidb.CIDBConnectionFactory.SetupMockCidb()
-
- # Calls to non-mock Setup methods should fail.
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupProdCidb)
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupDebugCidb)
-
- # Now supply a mock.
- a = object()
- cidb.CIDBConnectionFactory.SetupMockCidb(a)
- self.assertTrue(cidb.CIDBConnectionFactory.IsCIDBSetup())
- self.assertEqual(cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder(),
- a)
-
- # Mock object can be changed by future SetupMockCidb call.
- b = object()
- cidb.CIDBConnectionFactory.SetupMockCidb(b)
- self.assertEqual(cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder(),
- b)
-
- # Mock object can be cleared by future ClearMock call.
- cidb.CIDBConnectionFactory.ClearMock()
-
- # Calls to non-mock Setup methods should still fail.
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupProdCidb)
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupDebugCidb)
-
- def testSetupNo(self):
- """Test that SetupNoCidb behaves as expected."""
- cidb.CIDBConnectionFactory.SetupMockCidb()
- cidb.CIDBConnectionFactory.SetupNoCidb()
- cidb.CIDBConnectionFactory.SetupNoCidb()
- self.assertTrue(cidb.CIDBConnectionFactory.IsCIDBSetup())
- self.assertEqual(cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder(),
- None)
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupProdCidb)
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupDebugCidb)
- self.assertRaises(factory.ObjectFactoryIllegalOperation,
- cidb.CIDBConnectionFactory.SetupMockCidb)
diff --git a/lib/cidb_unittest.py b/lib/cidb_unittest.py
index 9ff8056bf..f94a634ea 100644
--- a/lib/cidb_unittest.py
+++ b/lib/cidb_unittest.py
@@ -9,8 +9,10 @@ from __future__ import print_function
import exceptions
import sqlalchemy
+from chromite.cbuildbot import constants
from chromite.lib import cidb
from chromite.lib import cros_test_lib
+from chromite.lib import factory
class RetryableOperationalError(exceptions.EnvironmentError):
@@ -50,3 +52,111 @@ class HelperFunctionsTest(cros_test_lib.TestCase):
FatalOperationalError())))
self.assertFalse(cidb._IsRetryableException(self._WrapError(
UnknownError())))
+
+
+class CIDBConnectionFactoryTest(cros_test_lib.MockTestCase):
+ """Test that CIDBConnectionFactory behaves as expected."""
+
+ def setUp(self):
+ # Ensure that we do not create any live connections in this unit test.
+ self.connection_mock = self.PatchObject(cidb, 'CIDBConnection')
+ # pylint: disable=W0212
+ cidb.CIDBConnectionFactory._ClearCIDBSetup()
+
+ def tearDown(self):
+ # pylint: disable=protected-access
+ cidb.CIDBConnectionFactory._ClearCIDBSetup()
+
+ def testGetConnectionBeforeSetup(self):
+ """Calling GetConnection before Setup should raise exception."""
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder)
+
+ def testSetupProd(self):
+ """Test that SetupProd behaves as expected."""
+ cidb.CIDBConnectionFactory.SetupProdCidb()
+ cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()
+
+ # Expected constructor call
+ self.connection_mock.assert_called_once_with(constants.CIDB_PROD_BOT_CREDS)
+ self.assertTrue(cidb.CIDBConnectionFactory.IsCIDBSetup())
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupProdCidb)
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupDebugCidb)
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupMockCidb)
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupNoCidb)
+
+ def testSetupDebug(self):
+ """Test that SetupDebug behaves as expected."""
+ cidb.CIDBConnectionFactory.SetupDebugCidb()
+ cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()
+
+ # Expected constructor call
+ self.connection_mock.assert_called_once_with(constants.CIDB_DEBUG_BOT_CREDS)
+ self.assertTrue(cidb.CIDBConnectionFactory.IsCIDBSetup())
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupProdCidb)
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupDebugCidb)
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupMockCidb)
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupNoCidb)
+
+ def testInvalidateSetup(self):
+ """Test that cidb connection can be invalidated."""
+ cidb.CIDBConnectionFactory.SetupProdCidb()
+ cidb.CIDBConnectionFactory.InvalidateCIDBSetup()
+ self.assertRaises(AssertionError,
+ cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder)
+
+ def testSetupMock(self):
+ """Test that SetupMock behaves as expected."""
+ # Set the CIDB to mock mode, but without supplying a mock
+ cidb.CIDBConnectionFactory.SetupMockCidb()
+
+ # Calls to non-mock Setup methods should fail.
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupProdCidb)
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupDebugCidb)
+
+ # Now supply a mock.
+ a = object()
+ cidb.CIDBConnectionFactory.SetupMockCidb(a)
+ self.assertTrue(cidb.CIDBConnectionFactory.IsCIDBSetup())
+ self.assertEqual(cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder(),
+ a)
+
+ # Mock object can be changed by future SetupMockCidb call.
+ b = object()
+ cidb.CIDBConnectionFactory.SetupMockCidb(b)
+ self.assertEqual(cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder(),
+ b)
+
+ # Mock object can be cleared by future ClearMock call.
+ cidb.CIDBConnectionFactory.ClearMock()
+
+ # Calls to non-mock Setup methods should still fail.
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupProdCidb)
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupDebugCidb)
+
+ def testSetupNo(self):
+ """Test that SetupNoCidb behaves as expected."""
+ cidb.CIDBConnectionFactory.SetupMockCidb()
+ cidb.CIDBConnectionFactory.SetupNoCidb()
+ cidb.CIDBConnectionFactory.SetupNoCidb()
+ self.assertTrue(cidb.CIDBConnectionFactory.IsCIDBSetup())
+ self.assertEqual(cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder(),
+ None)
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupProdCidb)
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupDebugCidb)
+ self.assertRaises(factory.ObjectFactoryIllegalOperation,
+ cidb.CIDBConnectionFactory.SetupMockCidb)
diff --git a/lib/cipd.py b/lib/cipd.py
new file mode 100644
index 000000000..427e02755
--- /dev/null
+++ b/lib/cipd.py
@@ -0,0 +1,115 @@
+# Copyright 2015 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Module to download and run the CIPD client.
+
+CIPD is the Chrome Infra Package Deployer, a simple method of resolving
+package/architecture/version tuples into GStorage links and installing them.
+
+"""
+from __future__ import print_function
+
+import hashlib
+import json
+import os
+import pprint
+import urllib
+import urlparse
+
+import third_party.httplib2 as httplib2
+
+import chromite.lib.cros_logging as log
+from chromite.lib import cache
+from chromite.lib import osutils
+from chromite.lib import path_util
+
+
+# The version of CIPD to download.
+# TODO(phobbs) we could make a call to the 'resolveVersion' endpoint
+# to resolve 'latest' into an instance_id for us.
+CIPD_INSTANCE_ID = '03f354ad7a6031c7924d9b69a85f83269cc3c2e0'
+CIPD_PACKAGE = 'infra/tools/cipd/linux-amd64'
+
+CHROME_INFRA_PACKAGES_API_BASE = (
+ 'https://chrome-infra-packages.appspot.com/_ah/api/repo/v1/')
+
+
+def _ChromeInfraRequest(endpoint, request_args=None):
+ """Makes a request to the Chrome Infra Packages API with httplib2.
+
+ Args:
+ endpoint: The endpoint to make a request to.
+ request_args: Keyword arguments to put into the request string.
+
+ Returns:
+ A tuple of (headers, content) returned by the server. The body content is
+ assumed to be JSON.
+ """
+ uri = ''.join([CHROME_INFRA_PACKAGES_API_BASE,
+ endpoint,
+ '?',
+ urllib.urlencode(request_args or {})])
+ result = httplib2.Http().request(uri=uri)
+ try:
+ return result[0], json.loads(result[1])
+ except Exception as e:
+ e.message = 'Encountered exception requesting "%s":\n' + e.message
+ raise
+
+
+def _DownloadCIPD(instance_id):
+ """Finds the CIPD download link and requests the binary.
+
+ The 'client' endpoit of the chrome infra packages API responds with a sha1 and
+ a Google Storage link. After downloading the binary, we validate that the sha1
+ of the response and return it.
+
+ Args:
+ instance_id: The version of CIPD to download.
+
+ Returns:
+ the CIPD binary as a string.
+ """
+ args = {'instance_id': instance_id, 'package_name': CIPD_PACKAGE}
+ _, body = _ChromeInfraRequest('client', request_args=args)
+ if not 'client_binary' in body:
+ log.error(
+ 'Error requesting the link to download CIPD from. Got:\n%s',
+ pprint.pformat(body))
+
+ http = httplib2.Http(cache='.cache')
+ response, binary = http.request(uri=body['client_binary']['fetch_url'])
+ assert response['status'] == '200', (
+ 'Got a %s response from Google Storage.' % response['status'])
+ digest = unicode(hashlib.sha1(binary).hexdigest())
+ assert digest == body['client_binary']['sha1'], (
+ 'The binary downloaded does not match the expected SHA1.')
+ return binary
+
+
+class CipdCache(cache.RemoteCache):
+ """Supports caching of the CIPD download."""
+ def _Fetch(self, key, path):
+ instance_id = urlparse.urlparse(key).netloc
+ binary = _DownloadCIPD(instance_id)
+ log.info('Fetched CIPD package %s:%s', CIPD_PACKAGE, instance_id)
+ osutils.WriteFile(path, binary)
+ os.chmod(path, 0755)
+
+
+def GetCIPDFromCache(instance_id=CIPD_INSTANCE_ID):
+ """Checks the cache, downloading CIPD if it is missing.
+
+ Args:
+ instance_id: The version of CIPD to download. Default CIPD_INSTANCE_ID
+
+ Returns:
+ Path to the CIPD binary.
+ """
+ cache_dir = os.path.join(path_util.GetCacheDir(), 'cipd')
+ bin_cache = CipdCache(cache_dir)
+ key = (instance_id,)
+ ref = bin_cache.Lookup(key)
+ ref.SetDefault('cipd://' + instance_id)
+ return ref.path
diff --git a/lib/commandline.py b/lib/commandline.py
index 6d95514fd..d10824488 100644
--- a/lib/commandline.py
+++ b/lib/commandline.py
@@ -29,7 +29,6 @@ from chromite.lib import gs
from chromite.lib import osutils
from chromite.lib import path_util
from chromite.lib import terminal
-from chromite.lib import workspace_lib
DEVICE_SCHEME_FILE = 'file'
@@ -318,95 +317,6 @@ class DeviceParser(object):
raise ValueError('Unknown device scheme "%s" in "%s"' % (scheme, value))
-def NormalizeWorkspacePath(path, default_dir=None, extension=None):
- """Normalize a workspace path.
-
- Converts |path| into a locator and applies |default_dir| and/or
- |extension| if specified.
-
- Args:
- path: Relative, absolute, or locator path in the CWD workspace.
- default_dir: If |path| does not contain '/', prepend this
- directory to the result.
- extension: If |path| doesn't end in this extension, append this
- extension to the result.
-
- Returns:
- Workspace locator corresponding to the modified |path|.
-
- Raises:
- ValueError: |path| isn't in the workspace.
- """
- if default_dir and '/' not in path:
- path = os.path.join(default_dir, path)
-
- if extension:
- extension = '.' + extension
- if os.path.splitext(path)[1] != extension:
- path += extension
-
- if workspace_lib.IsLocator(path):
- return path
-
- locator = workspace_lib.PathToLocator(path)
- if not locator:
- # argparse ignores exception messages; log it as well so the user sees it.
- error_message = '%s is not in the current workspace.' % path
- logging.error(error_message)
- raise ValueError(error_message)
- return locator
-
-
-def NormalizeBrickPath(path):
- """Normalize a brick path using some common assumptions.
-
- Makes the following changes to |path|:
- 1. Put non-paths in //bricks (e.g. foo -> //bricks/foo).
- 2. Convert to a workspace locator.
-
- Args:
- path: brick path.
-
- Returns:
- Locator to the brick.
- """
- return NormalizeWorkspacePath(path, default_dir='//bricks')
-
-
-def NormalizeBspPath(path):
- """Normalize a BSP path using some common assumptions.
-
- Makes the following changes to |path|:
- 1. Put non-paths in //bsps (e.g. foo -> //bsps/foo).
- 2. Convert to a workspace locator.
-
- Args:
- path: BSP path.
-
- Returns:
- Locator to the BSP.
- """
- return NormalizeWorkspacePath(path, default_dir='//bsps')
-
-
-def NormalizeBlueprintPath(path):
- """Normalize a blueprint path using some common assumptions.
-
- Makes the following changes to |path|:
- 1. Put non-paths in //blueprints (e.g. foo -> //blueprints/foo).
- 2. Add .json if not already present.
- 3. Convert to a workspace locator.
-
- Args:
- path: blueprint path.
-
- Returns:
- Locator to the blueprint.
- """
- return NormalizeWorkspacePath(path, default_dir='//blueprints',
- extension='json')
-
-
VALID_TYPES = {
'bool': ParseBool,
'date': ParseDate,
@@ -414,10 +324,6 @@ VALID_TYPES = {
'gs_path': NormalizeGSPath,
'local_or_gs_path': NormalizeLocalOrGSPath,
'path_or_uri': NormalizeUri,
- 'blueprint_path': NormalizeBlueprintPath,
- 'brick_path': NormalizeBrickPath,
- 'bsp_path': NormalizeBspPath,
- 'workspace_path': NormalizeWorkspacePath,
}
diff --git a/lib/commandline_unittest.py b/lib/commandline_unittest.py
index 533367667..22f13d686 100644
--- a/lib/commandline_unittest.py
+++ b/lib/commandline_unittest.py
@@ -240,81 +240,6 @@ class DeviceParseTest(cros_test_lib.OutputTestCase):
hostname='foo_host')
-class NormalizeWorkspacePathTest(cros_test_lib.WorkspaceTestCase):
- """Tests for NormalizeWorkspacePath() and associated functions."""
-
- def setUp(self):
- self.CreateWorkspace()
- # By default set the CWD to be the workspace directory.
- self.cwd_mock = self.PatchObject(os, 'getcwd')
- self.cwd_mock.return_value = self.workspace_path
-
- def _VerifyNormalized(self, path, expected, **kwargs):
- """Verifies tests on NormalizeWorkspacePath().
-
- Args:
- path: Input path to test.
- expected: Expected output.
- kwargs: Keyword args for NormalizeWorkspacePath().
- """
- self.assertEqual(expected,
- commandline.NormalizeWorkspacePath(path, **kwargs))
-
-
- def testLocatorConversion(self):
- """Tests NormalizeWorkspacePath() conversion to a locator."""
- # Relative paths.
- self._VerifyNormalized('a', '//a')
- self._VerifyNormalized('a/b', '//a/b')
-
- # Absolute paths.
- self._VerifyNormalized(os.path.join(self.workspace_path, 'a'), '//a')
- self._VerifyNormalized(os.path.join(self.workspace_path, 'a', 'b'), '//a/b')
-
- # Locators should be unchanged.
- self._VerifyNormalized('//a', '//a')
- self._VerifyNormalized('//a/b', '//a/b')
-
- # Paths outside the workspace should fail.
- for path in ('/', '..'):
- with self.assertRaises(ValueError):
- commandline.NormalizeWorkspacePath(path)
-
- def testDefaultDir(self):
- """Tests the default_dir parameter."""
- self._VerifyNormalized('a', '//default/a', default_dir='//default')
- self._VerifyNormalized('a/b', '//a/b', default_dir='//default')
- self._VerifyNormalized('./a', '//a', default_dir='//default')
-
- def testExtension(self):
- """Tests the extension parameter."""
- self._VerifyNormalized('a', '//a.txt', extension='txt')
- self._VerifyNormalized('a.bin', '//a.bin.txt', extension='txt')
- self._VerifyNormalized('a.txt', '//a.txt', extension='txt')
-
- def testSpecificPaths(self):
- """Tests normalizing brick/BSP/blueprint paths."""
- self.assertEqual('//bricks/a', commandline.NormalizeBrickPath('a'))
- self.assertEqual('//bsps/a', commandline.NormalizeBspPath('a'))
- self.assertEqual('//blueprints/a.json',
- commandline.NormalizeBlueprintPath('a'))
-
- def testParser(self):
- """Tests adding these types to a parser."""
- parser = commandline.ArgumentParser()
- parser.add_argument('path', type='workspace_path')
- parser.add_argument('brick', type='brick_path')
- parser.add_argument('bsp', type='bsp_path')
- parser.add_argument('blueprint', type='blueprint_path')
-
- options = parser.parse_args(['my_path', 'my_brick', 'my_bsp',
- 'my_blueprint'])
- self.assertEqual('//my_path', options.path)
- self.assertEqual('//bricks/my_brick', options.brick)
- self.assertEqual('//bsps/my_bsp', options.bsp)
- self.assertEqual('//blueprints/my_blueprint.json', options.blueprint)
-
-
class CacheTest(cros_test_lib.MockTempDirTestCase):
"""Test cache dir default / override functionality."""
diff --git a/lib/cros_build_lib.py b/lib/cros_build_lib.py
index 01d965240..eb937fa18 100644
--- a/lib/cros_build_lib.py
+++ b/lib/cros_build_lib.py
@@ -958,6 +958,27 @@ def CreateTarball(target, cwd, sudo=False, compression=COMP_XZ, chroot=None,
return rc_func(cmd, cwd=cwd, **kwargs)
+def GroupByKey(input_iter, key):
+ """Split an iterable of dicts, based on value of a key.
+
+ GroupByKey([{'a': 1}, {'a': 2}, {'a': 1, 'b': 2}], 'a') =>
+ {1: [{'a': 1}, {'a': 1, 'b': 2}], 2: [{'a': 2}]}
+
+ Args:
+ input_iter: An iterable of dicts.
+ key: A string specifying the key name to split by.
+
+ Returns:
+ A dictionary, mapping from each unique value for |key| that
+ was encountered in |input_iter| to a list of entries that had
+ that value.
+ """
+ split_dict = dict()
+ for entry in input_iter:
+ split_dict.setdefault(entry.get(key), []).append(entry)
+ return split_dict
+
+
def GetInput(prompt):
"""Helper function to grab input from a user. Makes testing easier."""
return raw_input(prompt)
diff --git a/lib/cros_build_lib_unittest.py b/lib/cros_build_lib_unittest.py
index 87167ea11..24c750755 100644
--- a/lib/cros_build_lib_unittest.py
+++ b/lib/cros_build_lib_unittest.py
@@ -1330,6 +1330,26 @@ class TestManifestCheckout(cros_test_lib.TempDirTestCase):
self.assertEqual(branches, ['refs/remotes/origin/release-R23-2913.B'])
+class TestGroupByKey(cros_test_lib.TestCase):
+ """Test SplitByKey."""
+
+ def testEmpty(self):
+ self.assertEqual({}, cros_build_lib.GroupByKey([], ''))
+
+ def testGroupByKey(self):
+ input_iter = [{'a': None, 'b': 0},
+ {'a': 1, 'b': 1},
+ {'a': 2, 'b': 2},
+ {'a': 1, 'b': 3}]
+ expected_result = {
+ None: [{'a': None, 'b': 0}],
+ 1: [{'a': 1, 'b': 1},
+ {'a': 1, 'b': 3}],
+ 2: [{'a': 2, 'b': 2}]}
+ self.assertEqual(cros_build_lib.GroupByKey(input_iter, 'a'),
+ expected_result)
+
+
class Test_iflatten_instance(cros_test_lib.TestCase):
"""Test iflatten_instance function."""
diff --git a/lib/cros_test_lib.py b/lib/cros_test_lib.py
index e0a3d1a55..d4b68c2a5 100644
--- a/lib/cros_test_lib.py
+++ b/lib/cros_test_lib.py
@@ -29,9 +29,6 @@ import urllib
from chromite.cbuildbot import config_lib
from chromite.cbuildbot import constants
-from chromite.lib import blueprint_lib
-from chromite.lib import bootstrap_lib
-from chromite.lib import brick_lib
from chromite.lib import cidb
from chromite.lib import commandline
from chromite.lib import cros_build_lib
@@ -46,7 +43,6 @@ from chromite.lib import remote_access
from chromite.lib import retry_util
from chromite.lib import terminal
from chromite.lib import timeout_util
-from chromite.lib import workspace_lib
site_config = config_lib.GetConfig()
@@ -1569,128 +1565,6 @@ class MockLoggingTestCase(MockTestCase, LoggingTestCase):
"""Convenience class mixing Logging and Mock."""
-class WorkspaceTestCase(MockTempDirTestCase):
- """Test case that adds utilities for using workspaces."""
-
- def setUp(self):
- """Define variables populated below, mostly to make lint happy."""
- self.bootstrap_path = None
- self.mock_bootstrap_path = None
-
- self.workspace_path = None
- self.workspace_config = None
- self.mock_workspace_path = None
-
- def CreateBootstrap(self, sdk_version=None):
- """Create a fake bootstrap directory in self.tempdir.
-
- self.bootstrap_path points to new workspace path.
- self.mock_bootstrap_path points to mock of FindBootstrapPath
-
- Args:
- sdk_version: Create a fake SDK version that's present in bootstrap.
- """
- # Create a bootstrap, inside our tempdir.
- self.bootstrap_path = os.path.join(self.tempdir, 'bootstrap')
- osutils.SafeMakedirs(os.path.join(self.bootstrap_path, '.git'))
-
- # If a version is provided, fake it's existence in the bootstrap.
- if sdk_version is not None:
- sdk_path = bootstrap_lib.ComputeSdkPath(self.bootstrap_path, sdk_version)
- osutils.SafeMakedirs(os.path.join(sdk_path, '.repo'))
- osutils.SafeMakedirs(os.path.join(sdk_path, 'chromite', '.git'))
-
- # Fake out bootstrap lookups to find this path.
- self.mock_bootstrap_path = self.PatchObject(
- bootstrap_lib, 'FindBootstrapPath', return_value=self.bootstrap_path)
-
- def CreateWorkspace(self, sdk_version=None):
- """Create a fake workspace directory in self.tempdir.
-
- self.workspace_path points to new workspace path.
- self.workspace_config points to workspace config file.
- self.mock_workspace_path points to mock of WorkspacePath
-
- Args:
- sdk_version: Mark SDK version as active in workspace. Does NOT mean
- it's present in bootstrap.
- """
- # Create a workspace, inside our tempdir.
- self.workspace_path = os.path.join(self.tempdir, 'workspace')
- self.workspace_config = os.path.join(
- self.workspace_path,
- workspace_lib.WORKSPACE_CONFIG)
- osutils.Touch(self.workspace_config, makedirs=True)
-
- # Define an SDK version for it, if needed.
- if sdk_version is not None:
- workspace_lib.SetActiveSdkVersion(self.workspace_path, sdk_version)
-
- # Fake out workspace lookups to find this path.
- self.mock_workspace_path = self.PatchObject(
- workspace_lib, 'WorkspacePath', return_value=self.workspace_path)
-
- def CreateBrick(self, name='thebrickfoo', main_package='category/bar',
- dependencies=None):
- """Creates a new brick.
-
- Args:
- name: Brick name/path relative to the workspace root.
- main_package: Main package to assign.
- dependencies: List of bricks to depend on.
-
- Returns:
- The created Brick object.
- """
- brick_path = os.path.join(self.workspace_path, name)
- config = {'name': name, 'main_package': main_package}
- if dependencies:
- config['dependencies'] = dependencies
-
- return brick_lib.Brick(brick_path, initial_config=config)
-
- def CreateBlueprint(self, name='theblueprintfoo.json', bsp=None, bricks=None,
- buildTargetId=None):
- """Creates a new blueprint.
-
- Args:
- name: Blueprint name/path relative to the workspace root.
- bsp: Path to BSP or None.
- bricks: List of paths to bricks or None.
- buildTargetId: The BuildTargetID to populate the APP_ID with or None.
-
- Returns:
- The created Blueprint object.
- """
- blueprint_path = os.path.join(self.workspace_path, name)
-
- config = {}
- if bricks:
- config[blueprint_lib.BRICKS_FIELD] = bricks
- if bsp:
- config[blueprint_lib.BSP_FIELD] = bsp
- if buildTargetId:
- config[blueprint_lib.APP_ID_FIELD] = buildTargetId
-
- return blueprint_lib.Blueprint(blueprint_path, initial_config=config)
-
- def AssertBlueprintExists(self, name, bsp=None, bricks=None):
- """Verifies a blueprint exists with the specified contents.
-
- Args:
- name: Blueprint name/path relative to the workspace root.
- bsp: Expected blueprint BSP or None.
- bricks: Expected blueprint bricks or None.
- """
- blueprint_path = os.path.join(self.workspace_path, name)
- blueprint = blueprint_lib.Blueprint(blueprint_path)
-
- if bsp is not None:
- self.assertEqual(bsp, blueprint.GetBSP())
- if bricks is not None:
- self.assertListEqual(bricks, blueprint.GetBricks())
-
-
@contextlib.contextmanager
def SetTimeZone(tz):
"""Temporarily set the timezone to the specified value.
diff --git a/lib/cros_test_lib_unittest.py b/lib/cros_test_lib_unittest.py
index 68aad6ca8..5786a2cc1 100644
--- a/lib/cros_test_lib_unittest.py
+++ b/lib/cros_test_lib_unittest.py
@@ -12,14 +12,12 @@ import sys
import time
import unittest
-from chromite.lib import bootstrap_lib
from chromite.lib import cros_test_lib
from chromite.lib import cros_build_lib
from chromite.lib import cros_build_lib_unittest
from chromite.lib import osutils
from chromite.lib import partial_mock
from chromite.lib import timeout_util
-from chromite.lib import workspace_lib
# pylint: disable=W0212,W0233
@@ -274,87 +272,3 @@ class OutputTestCaseTest(cros_test_lib.OutputTestCase,
# Verify that output is actually written to the correct files.
self.assertEqual('foo\n', osutils.ReadFile(stdout_path))
self.assertEqual('bar\n', osutils.ReadFile(stderr_path))
-
-
-class WorkspaceTestCaseTest(cros_test_lib.WorkspaceTestCase):
- """Verification for WorkspaceTestCase."""
-
- def testCreateWorkspace(self):
- """Tests CreateWorkspace()."""
- self.CreateWorkspace()
- self.assertExists(self.workspace_path)
- self.assertEqual(self.workspace_path, workspace_lib.WorkspacePath())
-
- def testCreateWorkspaceSdk(self):
- """Tests CreateWorkspace() with an SDK version."""
- self.CreateWorkspace(sdk_version='1.2.3')
- self.assertEqual('1.2.3',
- workspace_lib.GetActiveSdkVersion(self.workspace_path))
-
- def testCreateBootstrap(self):
- """Tests CreateBootstrap()."""
- self.CreateBootstrap()
- self.assertExists(self.bootstrap_path)
- self.assertEqual(self.bootstrap_path, bootstrap_lib.FindBootstrapPath())
-
- def testCreateBootstrapSdk(self):
- """Tests CreateBootstrap() with an SDK version."""
- self.CreateBootstrap(sdk_version='1.2.3')
- self.assertExists(
- bootstrap_lib.ComputeSdkPath(self.bootstrap_path, '1.2.3'))
-
- def testCreateBrick(self):
- """Tests CreateBrick()."""
- self.CreateWorkspace()
-
- self.CreateBrick(name='bar')
- brick = self.CreateBrick(name='foo', main_package='category/bar',
- dependencies=['//bar'])
- self.assertEqual(os.path.join(self.workspace_path, 'foo'), brick.brick_dir)
- self.assertEqual('foo', brick.FriendlyName())
- self.assertEqual(['category/bar'], brick.MainPackages())
- self.assertEqual(['//bar'], [b.brick_locator for b in brick.Dependencies()])
-
- def testCreateBlueprint(self):
- """Tests CreateBlueprint()."""
- brick_path = '//foo_brick'
- bsp_path = '//foo_bsp'
- blueprint_path = 'foo.json'
-
- self.CreateWorkspace()
- self.CreateBrick(brick_path)
- self.CreateBrick(bsp_path)
-
- blueprint = self.CreateBlueprint(name=blueprint_path, bsp=bsp_path,
- bricks=[brick_path])
- self.assertExists(os.path.join(self.workspace_path, blueprint_path))
- self.assertEqual(bsp_path, blueprint.GetBSP())
- self.assertEqual([brick_path], blueprint.GetBricks())
-
- def testAssertBlueprintExists(self):
- """Tests AssertBlueprintExists()."""
- brick_path = '//foo_brick'
- bsp_path = '//foo_bsp'
- blueprint_path = 'foo.json'
-
- self.CreateWorkspace()
- self.CreateBrick(brick_path)
- self.CreateBrick(bsp_path)
- self.CreateBlueprint(name=blueprint_path, bsp=bsp_path, bricks=[brick_path])
-
- # Test success conditions.
- self.AssertBlueprintExists(blueprint_path)
- self.AssertBlueprintExists(blueprint_path, bsp=bsp_path)
- self.AssertBlueprintExists(blueprint_path, bricks=[brick_path])
- self.AssertBlueprintExists(blueprint_path, bsp=bsp_path,
- bricks=[brick_path])
-
- # Test failure conditions.
- def TestFailure(blueprint_path, bsp=None, bricks=None):
- with self.assertRaises(Exception):
- self.AssertBlueprintExists(blueprint_path, bsp=bsp, bricks=bricks)
-
- TestFailure('//no/blueprint')
- TestFailure(blueprint_path, bsp='//no/bsp')
- TestFailure(blueprint_path, bricks=['//no/brick'])
- TestFailure(blueprint_path, bricks=[brick_path, '//no/brick'])
diff --git a/lib/factory_unittest.py b/lib/factory_unittest.py
index 3e0e77df4..dba22ac8c 100644
--- a/lib/factory_unittest.py
+++ b/lib/factory_unittest.py
@@ -71,5 +71,3 @@ class FactoryTest(cros_test_lib.TestCase):
a = self.of2.GetInstance()
self.of2.Setup('t4', None)
self.assertNotEqual(a, self.of2.GetInstance())
-
-
diff --git a/lib/fake_cidb.py b/lib/fake_cidb.py
index 0efb4e2ea..5cabeb70e 100644
--- a/lib/fake_cidb.py
+++ b/lib/fake_cidb.py
@@ -44,7 +44,8 @@ class FakeCIDBConnection(object):
def InsertBuild(self, builder_name, waterfall, build_number,
build_config, bot_hostname, master_build_id=None,
- timeout_seconds=None, status=constants.BUILDER_STATUS_PASSED):
+ timeout_seconds=None, status=constants.BUILDER_STATUS_PASSED,
+ important=None):
"""Insert a build row.
Note this API slightly differs from cidb as we pass status to avoid having
@@ -66,7 +67,9 @@ class FakeCIDBConnection(object):
'start_time': datetime.datetime.now(),
'master_build_id' : master_build_id,
'deadline': deadline,
- 'status': status}
+ 'status': status,
+ 'finish_time': datetime.datetime.now(),
+ 'important': important}
self.buildTable.append(row)
return build_id
@@ -81,7 +84,8 @@ class FakeCIDBConnection(object):
'full_version': versions.get('full'),
'sdk_version': d.get('sdk-versions'),
'toolchain_url': d.get('toolchain-url'),
- 'build_type': d.get('build_type')})
+ 'build_type': d.get('build_type'),
+ 'important': d.get('important')})
return 1
def InsertCLActions(self, build_id, cl_actions, timestamp=None):
@@ -194,6 +198,11 @@ class FakeCIDBConnection(object):
"""Gets the status of the builds."""
return [self.buildTable[x -1] for x in build_ids]
+ def GetSlaveStatuses(self, master_build_id):
+ """Gets the slaves of given build."""
+ return [b for b in self.buildTable
+ if b['master_build_id'] == master_build_id]
+
def GetBuildHistory(self, build_config, num_results,
ignore_build_id=None, start_date=None, end_date=None,
starting_build_number=None):
@@ -218,6 +227,7 @@ class FakeCIDBConnection(object):
if end_date is not None:
build_configs = [b for b in build_configs
if 'finish_time' in b and
+ b['finish_time'] and
b['finish_time'].date() <= end_date]
if starting_build_number is not None:
build_configs = [b for b in build_configs
diff --git a/lib/gerrit.py b/lib/gerrit.py
index 2a1155d6d..267ba2d7d 100644
--- a/lib/gerrit.py
+++ b/lib/gerrit.py
@@ -397,9 +397,9 @@ class GerritHelper(object):
gob_util.RestoreChange(self.host, self._to_changenum(change))
def DeleteDraft(self, change, dryrun=False):
- """Delete a draft patch set."""
+ """Delete a gerrit change iff all its revisions are drafts."""
if dryrun:
- logging.info('Would have deleted draft patch set %s', change)
+ logging.info('Would have deleted draft change %s', change)
return
gob_util.DeleteDraft(self.host, self._to_changenum(change))
diff --git a/lib/git.py b/lib/git.py
index 694dc804c..b2ed60002 100644
--- a/lib/git.py
+++ b/lib/git.py
@@ -67,6 +67,9 @@ GIT_TRANSIENT_ERRORS = (
# crbug.com/451458, b/19202011
r'repository cannot accept new pushes; contact support',
+
+ # crbug.com/535306
+ r'Service Temporarily Unavailable',
)
GIT_TRANSIENT_ERRORS_RE = re.compile('|'.join(GIT_TRANSIENT_ERRORS),
@@ -146,6 +149,7 @@ def IsGitRepositoryCorrupted(cwd):
"""
cmd = ['fsck', '--no-progress', '--no-dangling']
try:
+ GarbageCollection(cwd)
RunGit(cwd, cmd)
return False
except cros_build_lib.RunCommandError as ex:
diff --git a/lib/gob_util.py b/lib/gob_util.py
index 2fe67534f..d1a6301f8 100644
--- a/lib/gob_util.py
+++ b/lib/gob_util.py
@@ -368,7 +368,7 @@ def RestoreChange(host, change, msg=''):
def DeleteDraft(host, change):
- """Delete a gerrit draft patch set."""
+ """Delete a gerrit draft change."""
path = _GetChangePath(change)
try:
FetchUrl(host, path, reqtype='DELETE', ignore_204=True, ignore_404=False)
diff --git a/lib/graphite_lib/elasticsearch_mock_unittest.py b/lib/graphite_lib/elasticsearch_mock_unittest.py
index 59608fa3e..09edd6037 100644
--- a/lib/graphite_lib/elasticsearch_mock_unittest.py
+++ b/lib/graphite_lib/elasticsearch_mock_unittest.py
@@ -44,4 +44,3 @@ class statsd_mock_test(unittest.TestCase):
raise elasticsearch.ElasticsearchException('error message')
except elasticsearch.ElasticsearchException:
pass
-
diff --git a/lib/gs.py b/lib/gs.py
index 51c1752e1..7634b3865 100644
--- a/lib/gs.py
+++ b/lib/gs.py
@@ -253,7 +253,7 @@ class GSContext(object):
# (1*sleep) the first time, then (2*sleep), continuing via attempt * sleep.
DEFAULT_SLEEP_TIME = 60
- GSUTIL_VERSION = '4.13'
+ GSUTIL_VERSION = '4.15'
GSUTIL_TAR = 'gsutil_%s.tar.gz' % GSUTIL_VERSION
GSUTIL_URL = (PUBLIC_BASE_HTTPS_URL +
'chromeos-mirror/gentoo/distfiles/%s' % GSUTIL_TAR)
@@ -956,15 +956,15 @@ class GSContext(object):
Assorted GSContextException exceptions.
"""
try:
- res = self.DoCommand(['stat', path], redirect_stdout=True, **kwargs)
+ res = self.DoCommand(['stat', '--', path], redirect_stdout=True, **kwargs)
except GSCommandError as e:
# Because the 'gsutil stat' command logs errors itself (instead of
# raising errors internally like other commands), we have to look
- # for errors ourselves. See the bug report here:
+ # for errors ourselves. See the related bug report here:
# https://github.com/GoogleCloudPlatform/gsutil/issues/288
# Example line:
- # INFO 0713 05:58:12.451810 stat.py] No URLs matched gs://bucket/file
- if re.match(r'INFO [ 0-9:.]* stat.py\] No URLs matched', e.result.error):
+ # No URLs matched gs://bucket/file
+ if e.result.error.startswith('No URLs matched'):
raise GSNoSuchKey(path)
# No idea what this is, so just choke.
diff --git a/lib/gs_unittest.py b/lib/gs_unittest.py
index af84d10b9..7d319d5b6 100644
--- a/lib/gs_unittest.py
+++ b/lib/gs_unittest.py
@@ -177,7 +177,7 @@ class GetSizeTest(AbstractGSContextTest):
def testBasic(self):
"""Simple test."""
- self.gs_mock.AddCmdResult(['stat', self.GETSIZE_PATH],
+ self.gs_mock.AddCmdResult(['stat', '--', self.GETSIZE_PATH],
output=StatTest.STAT_OUTPUT)
self.assertEqual(self.GetSize(), 74)
@@ -1009,11 +1009,11 @@ class GSContextTest(AbstractGSContextTest):
def testGetGeneration(self):
"""Test ability to get the generation of a file."""
- self.gs_mock.AddCmdResult(['stat', 'gs://abc/1'],
+ self.gs_mock.AddCmdResult(['stat', '--', 'gs://abc/1'],
output=StatTest.STAT_OUTPUT)
ctx = gs.GSContext()
ctx.GetGeneration('gs://abc/1')
- self.gs_mock.assertCommandContains(['stat', 'gs://abc/1'])
+ self.gs_mock.assertCommandContains(['stat', '--', 'gs://abc/1'])
def testCreateCached(self):
"""Test that the function runs through."""
@@ -1120,16 +1120,15 @@ class StatTest(AbstractGSContextTest):
"""
# When stat throws an error. It's a special snow flake.
- STAT_ERROR_OUTPUT = ('INFO 0713 05:58:12.451810 stat.py] '
- 'No URLs matched gs://abc/1')
+ STAT_ERROR_OUTPUT = 'No URLs matched gs://abc/1'
def testStat(self):
"""Test ability to get the generation of a file."""
- self.gs_mock.AddCmdResult(['stat', 'gs://abc/1'],
+ self.gs_mock.AddCmdResult(['stat', '--', 'gs://abc/1'],
output=self.STAT_OUTPUT)
ctx = gs.GSContext()
result = ctx.Stat('gs://abc/1')
- self.gs_mock.assertCommandContains(['stat', 'gs://abc/1'])
+ self.gs_mock.assertCommandContains(['stat', '--', 'gs://abc/1'])
self.assertEqual(result.creation_time,
datetime.datetime(2014, 8, 23, 6, 53, 20))
@@ -1143,11 +1142,11 @@ class StatTest(AbstractGSContextTest):
def testStatOlderOutput(self):
"""Test ability to get the generation of a file."""
- self.gs_mock.AddCmdResult(['stat', 'gs://abc/1'],
+ self.gs_mock.AddCmdResult(['stat', '--', 'gs://abc/1'],
output=self.STAT_OUTPUT_OLDER)
ctx = gs.GSContext()
result = ctx.Stat('gs://abc/1')
- self.gs_mock.assertCommandContains(['stat', 'gs://abc/1'])
+ self.gs_mock.assertCommandContains(['stat', '--', 'gs://abc/1'])
self.assertEqual(result.creation_time,
datetime.datetime(2014, 8, 23, 6, 53, 20))
@@ -1161,12 +1160,12 @@ class StatTest(AbstractGSContextTest):
def testStatNoExist(self):
"""Test ability to get the generation of a file."""
- self.gs_mock.AddCmdResult(['stat', 'gs://abc/1'],
+ self.gs_mock.AddCmdResult(['stat', '--', 'gs://abc/1'],
error=self.STAT_ERROR_OUTPUT,
returncode=1)
ctx = gs.GSContext()
self.assertRaises(gs.GSNoSuchKey, ctx.Stat, 'gs://abc/1')
- self.gs_mock.assertCommandContains(['stat', 'gs://abc/1'])
+ self.gs_mock.assertCommandContains(['stat', '--', 'gs://abc/1'])
class UnmockedStatTest(cros_test_lib.TempDirTestCase):
diff --git a/lib/launch_control/__init__.py b/lib/launch_control/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/lib/launch_control/__init__.py
diff --git a/cbuildbot/builders/builders_unittest b/lib/launch_control/launch_control
index ef3e37b67..ef3e37b67 120000
--- a/cbuildbot/builders/builders_unittest
+++ b/lib/launch_control/launch_control
diff --git a/lib/launch_control/launch_control.py b/lib/launch_control/launch_control.py
new file mode 100644
index 000000000..da829cd4f
--- /dev/null
+++ b/lib/launch_control/launch_control.py
@@ -0,0 +1,142 @@
+# Copyright 2015 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Select an Android build, and download symbols for it."""
+
+from __future__ import print_function
+
+import json
+import apiclient
+
+from httplib2 import Http
+from apiclient.discovery import build
+from oauth2client.client import SignedJwtAssertionCredentials
+
+from chromite.lib import commandline
+from chromite.lib import cros_logging as logging
+
+
+def OpenBuildApiProxy(json_key_file):
+ """Open an Android Internal Build API Apiary proxy.
+
+ Will NOT error out if authentication fails until the first real request is
+ made.
+
+ Args:
+ json_key_file: A Json key file to authenticate with. Retrieved from the
+ Google Developer Console associated with the account to
+ be used for the requests.
+
+ Returns:
+ Proxy object used to make requests against the API.
+ """
+ # Load the private key associated with the Google service account.
+ with open(json_key_file) as json_file:
+ json_data = json.load(json_file)
+ credentials = SignedJwtAssertionCredentials(
+ json_data['client_email'],
+ json_data['private_key'],
+ 'https://www.googleapis.com/auth/androidbuild.internal')
+
+ # Open an authorized API proxy.
+ # See https://g3doc.corp.google.com/wireless/android/build_tools/
+ # g3doc/public/build_data.md
+ http_auth = credentials.authorize(Http())
+ return build('androidbuildinternal', 'v2beta1', http=http_auth)
+
+
+def FindRecentBuildIds(build_api_proxy, branch, target):
+ """Fetch a list of successful completed build ids for a given branch/target.
+
+ This roughly matches the contents of the first page of build results on the
+ launch control website, except filtered for only successful/completed builds.
+
+ Since builds sometimes complete out of order, new builds can be added to the
+ list out of order.
+
+ Args:
+ build_api_proxy: Result of a previous call to OpenBuildApiProxy.
+ branch: Name of branch to search. Ex. 'git_mnc-dr-ryu-release'
+ target: Build target to search. Ex. 'ryu-userdebug'
+
+ Returns:
+ List of build_ids as integers.
+ """
+ result = build_api_proxy.build().list(
+ buildType='submitted',
+ branch=branch,
+ buildAttemptStatus='complete',
+ successful=True,
+ target=target,
+ ).execute()
+
+ # Extract the build_ids, arrange oldest to newest.
+ return sorted(int(b['buildId']) for b in result['builds'])
+
+
+def FetchBuildArtifact(build_api_proxy, build_id, target, resource_id,
+ output_file):
+ """Fetch debug symbols associated with a given build.
+
+ Args:
+ build_api_proxy: Result of a previous call to OpenBuildApiProxy.
+ build_id: id of the build to fetch symbols for.
+ target: Build to target fetch symbols for. Ex. 'ryu-userdebug'
+ resource_id: Resource id to fetch. Ex. 'ryu-symbols-2282124.zip'
+ output_file: Path to where to write out the downloaded artifact.
+ """
+ # Open the download connection.
+ download_req = build_api_proxy.buildartifact().get_media(
+ buildId=build_id,
+ target=target,
+ attemptId='latest',
+ resourceId=resource_id)
+
+ # Download the symbols file contents.
+ with open(output_file, mode='wb') as fh:
+ downloader = apiclient.http.MediaIoBaseDownload(
+ fh, download_req, chunksize=20 * 1024 * 1024)
+ done = False
+ while not done:
+ _status, done = downloader.next_chunk()
+
+
+def main(argv):
+ """Command line wrapper for integration testing of the above library.
+
+ This library requires the ability to authenticate to an external service that
+ is restricted from the general public. So, allow manual integration testing by
+ users that have the necessary credentials.
+ """
+ parser = commandline.ArgumentParser(description=__doc__)
+
+ parser.add_argument('--json-key-file', type='path', required=True,
+ help='Json key file for authenticating to service.')
+ parser.add_argument('--symbols-file', type='path', default='symbols.zip',
+ help='Where to write symbols file out.')
+ parser.add_argument('--branch', type=str, default='git_mnc-dr-ryu-release',
+ help='Branch to locate build for.')
+ parser.add_argument('--target', type=str, default='ryu-userdebug',
+ help='Target to locate build for.')
+
+ opts = parser.parse_args(argv)
+ opts.Freeze()
+
+ build_proxy = OpenBuildApiProxy(opts.json_key_file)
+
+ build_ids = FindRecentBuildIds(
+ build_proxy,
+ branch=opts.branch,
+ target=opts.target)
+
+ build_id = build_ids[0]
+
+ # 'ryu-userdebug' -> 'ryu'
+ board = opts.target.split('-')[0]
+ # E.g. 'ryu-symbols-2282124.zip'
+ resource_id = '%s-symbols-%s.zip' % (board, build_id)
+
+ logging.info('Selected buildId: %s', build_id)
+ FetchBuildArtifact(build_proxy, build_id, opts.target, resource_id,
+ opts.symbols_file)
diff --git a/lib/launch_control/processed_builds.py b/lib/launch_control/processed_builds.py
new file mode 100644
index 000000000..3e8f614b9
--- /dev/null
+++ b/lib/launch_control/processed_builds.py
@@ -0,0 +1,84 @@
+# Copyright 2015 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Select an Android build, and download symbols for it."""
+
+from __future__ import print_function
+
+import json
+
+from chromite.lib import osutils
+
+
+class ProcessedBuildsStorage(object):
+ """A context manager for storing processed builds.
+
+ This is a context manager that loads recent builds, and allows them to be
+ manipulated, and then saves them on exit. Processed builds are stored per
+ branch/target as a list of integers.
+ """
+ def __init__(self, filename):
+ self.filename = filename
+ self.value = self._read()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self._write(self.value)
+
+ def _read(self):
+ """Load from disk, and default to an empty store on error."""
+ try:
+ return json.loads(osutils.ReadFile(self.filename))
+ except (ValueError, IOError):
+ # If there was no file, or it was corrupt json, return default.
+ return {}
+
+ def _write(self, new_value):
+ """Write the current store to disk."""
+ return osutils.WriteFile(self.filename,
+ json.dumps(new_value, sort_keys=True))
+
+ def GetProcessedBuilds(self, branch, target):
+ """Get a list of builds for a branch/target.
+
+ Args:
+ branch: Name of branch as a string.
+ target: Name of target as a string.
+
+ Returns:
+ List of integers associated with the given branch/target.
+ """
+ self.value.setdefault(branch, {})
+ self.value[branch].setdefault(target, [])
+ return self.value[branch][target]
+
+ def PurgeOldBuilds(self, branch, target, retain_list):
+ """Removes uninteresting builds for a branch/target.
+
+ Any build ids not in the retain list are removed.
+
+ Args:
+ branch: Name of branch as a string.
+ target: Name of target as a string.
+ retain_list: List of build ids that are still relevent.
+ """
+ processed = set(self.GetProcessedBuilds(branch, target))
+ retained_processed = processed.intersection(retain_list)
+ self.value[branch][target] = list(retained_processed)
+
+ def AddProcessedBuild(self, branch, target, build_id):
+ """Adds build_id to list for a branch/target.
+
+ It's safe to add a build_id that is already present.
+
+ Args:
+ branch: Name of branch as a string.
+ target: Name of target as a string.
+ build_id: build_id to add, as an integer.
+ """
+ processed = set(self.GetProcessedBuilds(branch, target))
+ processed.add(build_id)
+ self.value[branch][target] = sorted(processed)
diff --git a/cbuildbot/stages/stage_results_unittest b/lib/launch_control/processed_builds_unittest
index ef3e37b67..ef3e37b67 120000
--- a/cbuildbot/stages/stage_results_unittest
+++ b/lib/launch_control/processed_builds_unittest
diff --git a/lib/launch_control/processed_builds_unittest.py b/lib/launch_control/processed_builds_unittest.py
new file mode 100644
index 000000000..91443b642
--- /dev/null
+++ b/lib/launch_control/processed_builds_unittest.py
@@ -0,0 +1,84 @@
+# Copyright 2015 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Test processed_builds."""
+
+from __future__ import print_function
+
+import os
+
+from chromite.lib.launch_control import processed_builds
+from chromite.lib import cros_test_lib
+
+
+# Unitests often need access to internals of the thing they test.
+# pylint: disable=protected-access
+
+class ProcessedBuildsStorageTest(cros_test_lib.TempDirTestCase):
+ """Test our helper library for storing processed build ids."""
+
+ def setUp(self):
+ self.testfile = os.path.join(self.tempdir, 'testfile.json')
+
+ def testStartStop(self):
+ with processed_builds.ProcessedBuildsStorage(self.testfile):
+ pass
+
+ self.assertFileContents(self.testfile, '{}')
+
+ def testFetchEmpty(self):
+ with processed_builds.ProcessedBuildsStorage(self.testfile) as ps:
+ self.assertEqual(ps.GetProcessedBuilds('branch', 'target'), [])
+
+ self.assertFileContents(self.testfile, '{"branch": {"target": []}}')
+
+ def testPurgeEmpty(self):
+ with processed_builds.ProcessedBuildsStorage(self.testfile) as ps:
+ ps.PurgeOldBuilds('branch', 'target', [1, 2, 3])
+
+ self.assertFileContents(self.testfile, '{"branch": {"target": []}}')
+
+ def testAddEmpty(self):
+ with processed_builds.ProcessedBuildsStorage(self.testfile) as ps:
+ ps.AddProcessedBuild('branch', 'target', 1)
+
+ self.assertFileContents(self.testfile, '{"branch": {"target": [1]}}')
+
+ def testMultipleUses(self):
+ with processed_builds.ProcessedBuildsStorage(self.testfile) as ps:
+ ps.AddProcessedBuild('branch', 'target', 1)
+ ps.AddProcessedBuild('branch', 'target', 2)
+
+ self.assertFileContents(self.testfile, '{"branch": {"target": [1, 2]}}')
+
+ with processed_builds.ProcessedBuildsStorage(self.testfile) as ps:
+ # Try adding twice, should only happen once.
+ ps.AddProcessedBuild('branch', 'target', 3)
+ ps.AddProcessedBuild('branch', 'target', 3)
+
+ self.assertFileContents(self.testfile, '{"branch": {"target": [1, 2, 3]}}')
+
+ with processed_builds.ProcessedBuildsStorage(self.testfile) as ps:
+ ps.PurgeOldBuilds('branch', 'target', [2, 3])
+ ps.AddProcessedBuild('branch', 'target', 4)
+
+ self.assertFileContents(self.testfile, '{"branch": {"target": [2, 3, 4]}}')
+
+ with processed_builds.ProcessedBuildsStorage(self.testfile) as ps:
+ self.assertEqual(ps.GetProcessedBuilds('branch', 'target'), [2, 3, 4])
+
+ def testAddMultipleBranchTargets(self):
+ with processed_builds.ProcessedBuildsStorage(self.testfile) as ps:
+ ps.AddProcessedBuild('branch1', 'target', 1)
+ ps.AddProcessedBuild('branch2', 'target', 1)
+ ps.AddProcessedBuild('branch2', 'target', 2)
+ ps.AddProcessedBuild('branch2', 'target2', 3)
+
+ self.assertEqual(ps.GetProcessedBuilds('branch2', 'target'),
+ [1, 2])
+
+ self.assertFileContents(
+ self.testfile,
+ '{"branch1": {"target": [1]},'
+ ' "branch2": {"target": [1, 2], "target2": [3]}}')
diff --git a/lib/operation.py b/lib/operation.py
index 785278cbf..ca866a798 100644
--- a/lib/operation.py
+++ b/lib/operation.py
@@ -24,7 +24,6 @@ except ImportError:
# pylint: disable=import-error
import queue as Queue
import re
-import shutil
import struct
import sys
import termios
@@ -33,7 +32,6 @@ from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import osutils
from chromite.lib import parallel
-from chromite.lib import workspace_lib
from chromite.lib.terminal import Color
# Define filenames for captured stdout and stderr.
@@ -76,7 +74,6 @@ class ProgressBarOperation(object):
self._stdout_path = None
self._stderr_path = None
self._progress_bar_displayed = False
- self._workspace_path = workspace_lib.WorkspacePath()
self._isatty = os.isatty(sys.stdout.fileno())
def _GetTerminalSize(self, fd=pty.STDOUT_FILENO):
@@ -174,24 +171,12 @@ class ProgressBarOperation(object):
try:
with cros_build_lib.OutputCapturer(
stdout_path=self._stdout_path, stderr_path=self._stderr_path,
- quiet_fail=self._workspace_path is not None):
+ quiet_fail=False):
func(*args, **kwargs)
finally:
self._queue.put(_BackgroundTaskComplete())
logging.getLogger().setLevel(restore_log_level)
- def MoveStdoutStderrFiles(self):
- """On failure, move stdout/stderr files to workspace/WORKSPACE_LOGS_DIR."""
- path = os.path.join(self._workspace_path, workspace_lib.WORKSPACE_LOGS_DIR)
- # TODO(ralphnathan): Not sure if we need this because it should be done when
- # we store the log file for brillo commands.
- osutils.SafeMakedirs(path)
- osutils.SafeUnlink(os.path.join(path, STDOUT_FILE))
- shutil.move(self._stdout_path, path)
- osutils.SafeUnlink(os.path.join(path, STDERR_FILE))
- shutil.move(self._stderr_path, path)
- logging.warning('Please look at %s for more information.', path)
-
# TODO (ralphnathan): Store PID of spawned process.
def Run(self, func, *args, **kwargs):
"""Run func, parse its output, and update the progress bar.
@@ -235,9 +220,6 @@ class ProgressBarOperation(object):
# touching the progress bar.
sys.stdout.write('\n')
logging.error('Oops. Something went wrong.')
- # Move the stdout/stderr files to a location that the user can access.
- if self._workspace_path is not None:
- self.MoveStdoutStderrFiles()
# Raise the exception so it can be caught again.
raise
diff --git a/lib/operation_unittest.py b/lib/operation_unittest.py
index ea9e184af..01e641db7 100644
--- a/lib/operation_unittest.py
+++ b/lib/operation_unittest.py
@@ -13,9 +13,7 @@ import sys
from chromite.lib import cros_logging as logging
from chromite.lib import cros_test_lib
from chromite.lib import operation
-from chromite.lib import osutils
from chromite.lib import parallel
-from chromite.lib import workspace_lib
class TestWrapperProgressBarOperation(operation.ProgressBarOperation):
@@ -40,7 +38,7 @@ class FakeException(Exception):
"""Fake exception used for testing exception handling."""
-class ProgressBarOperationTest(cros_test_lib.WorkspaceTestCase,
+class ProgressBarOperationTest(cros_test_lib.MockTestCase,
cros_test_lib.OutputTestCase,
cros_test_lib.LoggingTestCase):
"""Test the Progress Bar Operation class."""
@@ -53,16 +51,6 @@ class ProgressBarOperationTest(cros_test_lib.WorkspaceTestCase,
return_value=operation._TerminalSize(100, terminal_width))
self.PatchObject(os, 'isatty', return_value=True)
- def _GetStdoutPath(self):
- """Return path to the file where stdout is captured."""
- return os.path.join(self.workspace_path, workspace_lib.WORKSPACE_LOGS_DIR,
- operation.STDOUT_FILE)
-
- def _GetStderrPath(self):
- """Return path to the file where stderr is captured."""
- return os.path.join(self.workspace_path, workspace_lib.WORKSPACE_LOGS_DIR,
- operation.STDERR_FILE)
-
def _VerifyProgressBar(self, width, percent, expected_shaded,
expected_unshaded):
"""Helper to test progress bar with different percentages and lengths."""
@@ -130,8 +118,8 @@ class ProgressBarOperationTest(cros_test_lib.WorkspaceTestCase,
# called once.
self.AssertOutputContainsLine('Calling ParseOutput')
- def testExceptionHandlingNotInWorkspace(self):
- """Test exception handling if not in a workspace."""
+ def testExceptionHandling(self):
+ """Test exception handling."""
def func():
print('foo')
print('bar', file=sys.stderr)
@@ -152,64 +140,6 @@ class ProgressBarOperationTest(cros_test_lib.WorkspaceTestCase,
self.AssertOutputContainsLine('foo')
self.AssertOutputContainsLine('bar', check_stderr=True)
- def testExceptionHandlingInWorkspace(self):
- """Test that stdout/stderr files are moved correctly if in a workspace."""
- def func():
- print('foo')
- print('bar', file=sys.stderr)
- raise FakeException()
-
- self.CreateWorkspace()
- op = TestWrapperProgressBarOperation()
- stdout_file = self._GetStdoutPath()
- stderr_file = self._GetStderrPath()
-
- # Check that the files don't exist before the operation is called.
- self.assertNotExists(stdout_file)
- self.assertNotExists(stderr_file)
-
- try:
- with cros_test_lib.LoggingCapturer() as logs:
- op.Run(func)
- except parallel.BackgroundFailure as e:
- if not e.HasFailureType(FakeException):
- raise e
-
- # Check that the files have been moved to the right location.
- self.assertExists(stdout_file)
- self.assertExists(stderr_file)
-
- # Check that the log message contains the path.
- self.AssertLogsContain(logs, self.workspace_path)
-
- def testExceptionHandlingInWorkspaceFilesAlreadyExist(self):
- """Test that old stdout/stderr files are removed from log directory."""
- def func():
- print('foo')
- print('bar', file=sys.stderr)
- raise FakeException()
-
- self.CreateWorkspace()
- op = TestWrapperProgressBarOperation()
- stdout_file = self._GetStdoutPath()
- stderr_file = self._GetStderrPath()
- osutils.Touch(stdout_file, makedirs=True)
- osutils.Touch(stderr_file, makedirs=True)
-
- # Assert that the files are empty.
- self.assertEqual(osutils.ReadFile(stdout_file), '')
- self.assertEqual(osutils.ReadFile(stderr_file), '')
-
- try:
- op.Run(func)
- except parallel.BackgroundFailure as e:
- if not e.HasFailureType(FakeException):
- raise e
-
- # Check that the files contain the right information.
- self.assertIn('foo', osutils.ReadFile(stdout_file))
- self.assertIn('bar', osutils.ReadFile(stderr_file))
-
def testLogLevel(self):
"""Test that the log level of the function running is set correctly."""
func_log_level = logging.DEBUG
diff --git a/lib/patch.py b/lib/patch.py
index 8271df6f6..09f71327f 100644
--- a/lib/patch.py
+++ b/lib/patch.py
@@ -54,6 +54,7 @@ ATTR_OWNER_EMAIL = 'owner_email'
ATTR_FAIL_COUNT = 'fail_count'
ATTR_PASS_COUNT = 'pass_count'
ATTR_TOTAL_FAIL_COUNT = 'total_fail_count'
+ATTR_COMMIT_MESSAGE = 'commit_message'
ALL_ATTRS = (
ATTR_REMOTE,
@@ -69,6 +70,7 @@ ALL_ATTRS = (
ATTR_FAIL_COUNT,
ATTR_PASS_COUNT,
ATTR_TOTAL_FAIL_COUNT,
+ ATTR_COMMIT_MESSAGE,
)
def ParseSHA1(text, error_ok=True):
@@ -1499,7 +1501,8 @@ class GerritFetchOnlyPatch(GitRepoPatch):
def __init__(self, project_url, project, ref, tracking_branch, remote,
sha1, change_id, gerrit_number, patch_number, owner_email=None,
- fail_count=0, pass_count=0, total_fail_count=0):
+ fail_count=0, pass_count=0, total_fail_count=0,
+ commit_message=None):
"""Initializes a GerritFetchOnlyPatch object."""
super(GerritFetchOnlyPatch, self).__init__(
project_url, project, ref, tracking_branch, remote,
@@ -1520,6 +1523,10 @@ class GerritFetchOnlyPatch(GitRepoPatch):
self.fail_count = fail_count
self.pass_count = pass_count
self.total_fail_count = total_fail_count
+ # commit_message is herited from GitRepoPatch, only override it when passed
+ # in value is not None.
+ if commit_message:
+ self.commit_message = commit_message
@classmethod
def FromAttrDict(cls, attr_dict):
@@ -1541,7 +1548,10 @@ class GerritFetchOnlyPatch(GitRepoPatch):
fail_count=int(attr_dict[ATTR_FAIL_COUNT]),
pass_count=int(attr_dict[ATTR_PASS_COUNT]),
total_fail_count=int(
- attr_dict[ATTR_TOTAL_FAIL_COUNT]))
+ attr_dict[ATTR_TOTAL_FAIL_COUNT]),
+ commit_message=attr_dict.get(
+ ATTR_COMMIT_MESSAGE))
+
def _EnsureId(self, commit_message):
"""Ensure we have a usable Change-Id
@@ -1596,6 +1606,7 @@ class GerritFetchOnlyPatch(GitRepoPatch):
ATTR_FAIL_COUNT: str(self.fail_count),
ATTR_PASS_COUNT: str(self.pass_count),
ATTR_TOTAL_FAIL_COUNT: str(self.total_fail_count),
+ ATTR_COMMIT_MESSAGE: self.commit_message,
}
return attr_dict
diff --git a/lib/patch_unittest.py b/lib/patch_unittest.py
index a3f2c4c46..2a2bcd809 100644
--- a/lib/patch_unittest.py
+++ b/lib/patch_unittest.py
@@ -460,6 +460,92 @@ class TestGitRepoPatch(GitRepoPatchTestCase):
self.assertIn('Change-Id: %s\n' % changeid, patch.commit_message)
+class TestGerritFetchOnlyPatch(cros_test_lib.MockTestCase):
+ """Test of GerritFetchOnlyPatch."""
+
+ def testFromAttrDict(self):
+ """Test whether FromAttrDict can handle with commit message."""
+ attr_dict_without_msg = {
+ cros_patch.ATTR_PROJECT_URL: 'https://host/chromite/tacos',
+ cros_patch.ATTR_PROJECT: 'chromite/tacos',
+ cros_patch.ATTR_REF: 'refs/changes/11/12345/4',
+ cros_patch.ATTR_BRANCH: 'master',
+ cros_patch.ATTR_REMOTE: 'cros-internal',
+ cros_patch.ATTR_COMMIT: '7181e4b5e182b6f7d68461b04253de095bad74f9',
+ cros_patch.ATTR_CHANGE_ID: 'I47ea30385af60ae4cc2acc5d1a283a46423bc6e1',
+ cros_patch.ATTR_GERRIT_NUMBER: '12345',
+ cros_patch.ATTR_PATCH_NUMBER: '4',
+ cros_patch.ATTR_OWNER_EMAIL: 'foo@chromium.org',
+ cros_patch.ATTR_FAIL_COUNT: 1,
+ cros_patch.ATTR_PASS_COUNT: 1,
+ cros_patch.ATTR_TOTAL_FAIL_COUNT: 3}
+
+ attr_dict_with_msg = {
+ cros_patch.ATTR_PROJECT_URL: 'https://host/chromite/tacos',
+ cros_patch.ATTR_PROJECT: 'chromite/tacos',
+ cros_patch.ATTR_REF: 'refs/changes/11/12345/4',
+ cros_patch.ATTR_BRANCH: 'master',
+ cros_patch.ATTR_REMOTE: 'cros-internal',
+ cros_patch.ATTR_COMMIT: '7181e4b5e182b6f7d68461b04253de095bad74f9',
+ cros_patch.ATTR_CHANGE_ID: 'I47ea30385af60ae4cc2acc5d1a283a46423bc6e1',
+ cros_patch.ATTR_GERRIT_NUMBER: '12345',
+ cros_patch.ATTR_PATCH_NUMBER: '4',
+ cros_patch.ATTR_OWNER_EMAIL: 'foo@chromium.org',
+ cros_patch.ATTR_FAIL_COUNT: 1,
+ cros_patch.ATTR_PASS_COUNT: 1,
+ cros_patch.ATTR_TOTAL_FAIL_COUNT: 3,
+ cros_patch.ATTR_COMMIT_MESSAGE: 'commit message'}
+
+ self.PatchObject(cros_patch.GitRepoPatch, '_AddFooters',
+ return_value='commit message')
+
+ result_1 = (cros_patch.GerritFetchOnlyPatch.
+ FromAttrDict(attr_dict_without_msg).commit_message)
+ result_2 = (cros_patch.GerritFetchOnlyPatch.
+ FromAttrDict(attr_dict_with_msg).commit_message)
+ self.assertEqual(None, result_1)
+ self.assertEqual('commit message', result_2)
+
+ def testGetAttributeDict(self):
+ """Test Whether GetAttributeDict can get the commit message properly."""
+ change = cros_patch.GerritFetchOnlyPatch(
+ 'https://host/chromite/tacos',
+ 'chromite/tacos',
+ 'refs/changes/11/12345/4',
+ 'master',
+ 'cros-internal',
+ '7181e4b5e182b6f7d68461b04253de095bad74f9',
+ 'I47ea30385af60ae4cc2acc5d1a283a46423bc6e1',
+ '12345',
+ '4',
+ 'foo@chromium.org',
+ 1,
+ 1,
+ 3)
+
+ expected = {
+ cros_patch.ATTR_PROJECT_URL: 'https://host/chromite/tacos',
+ cros_patch.ATTR_PROJECT: 'chromite/tacos',
+ cros_patch.ATTR_REF: 'refs/changes/11/12345/4',
+ cros_patch.ATTR_BRANCH: 'master',
+ cros_patch.ATTR_REMOTE: 'cros-internal',
+ cros_patch.ATTR_COMMIT: '7181e4b5e182b6f7d68461b04253de095bad74f9',
+ cros_patch.ATTR_CHANGE_ID: 'I47ea30385af60ae4cc2acc5d1a283a46423bc6e1',
+ cros_patch.ATTR_GERRIT_NUMBER: '12345',
+ cros_patch.ATTR_PATCH_NUMBER: '4',
+ cros_patch.ATTR_OWNER_EMAIL: 'foo@chromium.org',
+ cros_patch.ATTR_FAIL_COUNT: '1',
+ cros_patch.ATTR_PASS_COUNT: '1',
+ cros_patch.ATTR_TOTAL_FAIL_COUNT: '3',
+ cros_patch.ATTR_COMMIT_MESSAGE: None}
+ self.assertEqual(change.GetAttributeDict(), expected)
+
+ self.PatchObject(cros_patch.GitRepoPatch, '_AddFooters',
+ return_value='commit message')
+ change.commit_message = 'commit message'
+ expected[cros_patch.ATTR_COMMIT_MESSAGE] = 'commit message'
+ self.assertEqual(change.GetAttributeDict(), expected)
+
class TestGetOptionLinesFromCommitMessage(cros_test_lib.TestCase):
"""Tests of GetOptionFromCommitMessage."""
diff --git a/lib/path_util.py b/lib/path_util.py
index 099023816..bf4e0b66e 100644
--- a/lib/path_util.py
+++ b/lib/path_util.py
@@ -11,7 +11,6 @@ import os
import tempfile
from chromite.cbuildbot import constants
-from chromite.lib import bootstrap_lib
from chromite.lib import cros_build_lib
from chromite.lib import git
from chromite.lib import osutils
@@ -23,7 +22,6 @@ CHROME_CACHE_DIR = '.cros_cache'
CHECKOUT_TYPE_UNKNOWN = 'unknown'
CHECKOUT_TYPE_GCLIENT = 'gclient'
CHECKOUT_TYPE_REPO = 'repo'
-CHECKOUT_TYPE_SDK_BOOTSTRAP = 'bootstrap'
CheckoutInfo = collections.namedtuple(
'CheckoutInfo', ['type', 'root', 'chrome_src_dir'])
@@ -234,28 +232,6 @@ class ChrootPathResolver(object):
return self._ConvertPath(path, self._GetHostPath)
-def _IsSdkBootstrapCheckout(path):
- """Return True if |path| is an SDK bootstrap.
-
- A bootstrap is a lone git checkout of chromite. It cannot be managed by repo.
- Underneath this bootstrap chromite, there are several SDK checkouts, each
- managed by repo.
- """
- submodule_git = os.path.join(path, '.git')
- if not git.IsSubmoduleCheckoutRoot(submodule_git, 'origin',
- constants.CHROMITE_URL):
- # Not a git checkout of chromite.
- return False
-
- # This could be an SDK under sdk_checkouts or the parent bootstrap.
- # It'll be an SDK checkout if it has a parent ".repo".
- if git.FindRepoDir(path):
- # It is managed by repo, therefore it is a child SDK checkout.
- return False
-
- return True
-
-
def DetermineCheckout(cwd):
"""Gather information on the checkout we are in.
@@ -263,11 +239,6 @@ def DetermineCheckout(cwd):
This function determines what checkout type |cwd| is in, for example, if |cwd|
belongs to a `repo` checkout.
- There is a special case when |cwd| is a child SDK checkout of a bootstrap
- chromite (e.g. something under chromite/sdk_checkouts/xxx.yyy.zzz/). This
- case should report that |cwd| belongs to a bootstrap checkout instead of the
- `repo` checkout of the "xxx.yyy.zzz" child SDK.
-
Returns:
A CheckoutInfo object with these attributes:
type: The type of checkout. Valid values are CHECKOUT_TYPE_*.
@@ -278,24 +249,15 @@ def DetermineCheckout(cwd):
checkout_type = CHECKOUT_TYPE_UNKNOWN
root, path = None, None
- # Check for SDK bootstrap first because it goes top to bottom.
- # If we do it bottom to top, we'll hit chromite/sdk_checkouts/*/.repo first
- # and will wrongly conclude that this is a repo checkout. So we go top down
- # to visit chromite/ first.
- for path in osutils.IteratePaths(cwd):
- if _IsSdkBootstrapCheckout(path):
- checkout_type = CHECKOUT_TYPE_SDK_BOOTSTRAP
+ for path in osutils.IteratePathParents(cwd):
+ gclient_file = os.path.join(path, '.gclient')
+ if os.path.exists(gclient_file):
+ checkout_type = CHECKOUT_TYPE_GCLIENT
+ break
+ repo_dir = os.path.join(path, '.repo')
+ if os.path.isdir(repo_dir):
+ checkout_type = CHECKOUT_TYPE_REPO
break
- else:
- for path in osutils.IteratePathParents(cwd):
- gclient_file = os.path.join(path, '.gclient')
- if os.path.exists(gclient_file):
- checkout_type = CHECKOUT_TYPE_GCLIENT
- break
- repo_dir = os.path.join(path, '.repo')
- if os.path.isdir(repo_dir):
- checkout_type = CHECKOUT_TYPE_REPO
- break
if checkout_type != CHECKOUT_TYPE_UNKNOWN:
root = path
@@ -315,9 +277,6 @@ def FindCacheDir():
path = None
if checkout.type == CHECKOUT_TYPE_REPO:
path = os.path.join(checkout.root, GENERAL_CACHE_DIR)
- elif checkout.type == CHECKOUT_TYPE_SDK_BOOTSTRAP:
- path = os.path.join(checkout.root, bootstrap_lib.SDK_CHECKOUTS,
- GENERAL_CACHE_DIR)
elif checkout.type == CHECKOUT_TYPE_GCLIENT:
path = os.path.join(checkout.root, CHROME_CACHE_DIR)
elif checkout.type == CHECKOUT_TYPE_UNKNOWN:
diff --git a/lib/path_util_unittest.py b/lib/path_util_unittest.py
index 3fe93d17e..459997856 100644
--- a/lib/path_util_unittest.py
+++ b/lib/path_util_unittest.py
@@ -12,7 +12,6 @@ import os
import tempfile
from chromite.cbuildbot import constants
-from chromite.lib import bootstrap_lib
from chromite.lib import cros_build_lib_unittest
from chromite.lib import cros_test_lib
from chromite.lib import git
@@ -111,28 +110,8 @@ class DetermineCheckoutTest(cros_test_lib.MockTempDirTestCase):
self.RunTest(['a/.git/'], 'a', None,
path_util.CHECKOUT_TYPE_UNKNOWN, None)
- def testSdkBootstrap(self):
- """Recognizes an SDK bootstrap case."""
- self.rc_mock.AddCmdResult(
- partial_mock.In('config'), output=constants.CHROMITE_URL)
- dir_struct = [
- 'a/.git/',
- 'a/sdk_checkouts/1.0.0/.repo',
- 'a/sdk_checkouts/1.0.0/chromite/.git',
- ]
- self.RunTest(dir_struct, 'a', 'a',
- path_util.CHECKOUT_TYPE_SDK_BOOTSTRAP, None)
- self.RunTest(dir_struct, 'a/b', 'a',
- path_util.CHECKOUT_TYPE_SDK_BOOTSTRAP, None)
- self.RunTest(dir_struct, 'a/sdk_checkouts', 'a',
- path_util.CHECKOUT_TYPE_SDK_BOOTSTRAP, None)
- self.RunTest(dir_struct, 'a/sdk_checkouts/1.0.0', 'a',
- path_util.CHECKOUT_TYPE_SDK_BOOTSTRAP, None)
- self.RunTest(dir_struct, 'a/sdk_checkouts/1.0.0/chromite', 'a',
- path_util.CHECKOUT_TYPE_SDK_BOOTSTRAP, None)
-
-class FindCacheDirTest(cros_test_lib.WorkspaceTestCase):
+class FindCacheDirTest(cros_test_lib.MockTempDirTestCase):
"""Test cache dir specification and finding functionality."""
def setUp(self):
@@ -145,10 +124,6 @@ class FindCacheDirTest(cros_test_lib.WorkspaceTestCase):
self.repo_root = os.path.join(self.tempdir, 'repo')
self.gclient_root = os.path.join(self.tempdir, 'gclient')
self.nocheckout_root = os.path.join(self.tempdir, 'nothing')
- self.CreateBootstrap('1.0.0')
- self.bootstrap_cache = os.path.join(
- self.bootstrap_path, bootstrap_lib.SDK_CHECKOUTS,
- path_util.GENERAL_CACHE_DIR)
self.rc_mock = self.StartPatcher(cros_build_lib_unittest.RunCommandMock())
self.cwd_mock = self.PatchObject(os, 'getcwd')
@@ -174,35 +149,6 @@ class FindCacheDirTest(cros_test_lib.WorkspaceTestCase):
path_util.FindCacheDir(),
os.path.join(tempfile.gettempdir(), ''))
- def testBootstrap(self):
- """Test when running from bootstrap."""
- self.cwd_mock.return_value = self.bootstrap_path
- self.rc_mock.AddCmdResult(
- partial_mock.In('config'), output=constants.CHROMITE_URL)
- self.assertEquals(
- path_util.FindCacheDir(),
- self.bootstrap_cache)
-
- def testSdkCheckoutsInsideBootstrap(self):
- """Test when in the bootstrap SDK checkout location."""
- self.cwd_mock.return_value = os.path.join(
- self.bootstrap_path, bootstrap_lib.SDK_CHECKOUTS)
- self.rc_mock.AddCmdResult(
- partial_mock.In('config'), output=constants.CHROMITE_URL)
- self.assertEquals(
- path_util.FindCacheDir(),
- self.bootstrap_cache)
-
- def testSdkInsideBootstrap(self):
- """Test when in an SDK checkout inside the bootstrap."""
- self.cwd_mock.return_value = os.path.join(
- self.bootstrap_path, bootstrap_lib.SDK_CHECKOUTS, '1.0.0', 'chromite')
- self.rc_mock.AddCmdResult(
- partial_mock.In('config'), output=constants.CHROMITE_URL)
- self.assertEquals(
- path_util.FindCacheDir(),
- self.bootstrap_cache)
-
class TestPathResolver(cros_test_lib.MockTestCase):
"""Tests of ChrootPathResolver class."""
diff --git a/lib/paygen/gslock_unittest.py b/lib/paygen/gslock_unittest.py
index 70a60a638..19347e60f 100644
--- a/lib/paygen/gslock_unittest.py
+++ b/lib/paygen/gslock_unittest.py
@@ -7,8 +7,6 @@
from __future__ import print_function
import multiprocessing
-import os
-import socket
from chromite.lib import cros_build_lib
from chromite.lib import cros_test_lib
@@ -108,42 +106,33 @@ def _InProcessDataUpdate(lock_uri_data_uri):
class GSLockTest(cros_test_lib.MockTestCase):
"""This test suite covers the GSLock file."""
+ # For contention tests, how many parallel workers to spawn. To really
+ # stress test, you can bump it up to 200, but 20 seems to provide good
+ # coverage w/out sucking up too many resources.
+ NUM_THREADS = 20
+
@cros_test_lib.NetworkTest()
def setUp(self):
self.ctx = gs.GSContext()
- # Use the unique id to make sure the tests can be run multiple places.
- unique_id = '%s.%d' % (socket.gethostname(), os.getpid())
-
- self.lock_uri = 'gs://chromeos-releases-test/test-%s-gslock' % unique_id
- self.data_uri = 'gs://chromeos-releases-test/test-%s-data' % unique_id
-
- # Clear out any flags left from previous failure
- self.ctx.Remove(self.lock_uri, ignore_missing=True)
- self.ctx.Remove(self.data_uri, ignore_missing=True)
-
- @cros_test_lib.NetworkTest()
- def tearDown(self):
- self.assertFalse(self.ctx.Exists(self.lock_uri))
- self.assertFalse(self.ctx.Exists(self.data_uri))
-
@cros_test_lib.NetworkTest()
def testLock(self):
"""Test getting a lock."""
# Force a known host name.
self.PatchObject(cros_build_lib, 'MachineDetails', return_value='TestHost')
- lock = gslock.Lock(self.lock_uri)
+ with gs.TemporaryURL('gslock') as lock_uri:
+ lock = gslock.Lock(lock_uri)
- self.assertFalse(self.ctx.Exists(self.lock_uri))
- lock.Acquire()
- self.assertTrue(self.ctx.Exists(self.lock_uri))
+ self.assertFalse(self.ctx.Exists(lock_uri))
+ lock.Acquire()
+ self.assertTrue(self.ctx.Exists(lock_uri))
- contents = self.ctx.Cat(self.lock_uri)
- self.assertEqual(contents, 'TestHost')
+ contents = self.ctx.Cat(lock_uri)
+ self.assertEqual(contents, 'TestHost')
- lock.Release()
- self.assertFalse(self.ctx.Exists(self.lock_uri))
+ lock.Release()
+ self.assertFalse(self.ctx.Exists(lock_uri))
@cros_test_lib.NetworkTest()
def testLockRepetition(self):
@@ -151,117 +140,120 @@ class GSLockTest(cros_test_lib.MockTestCase):
# Force a known host name.
self.PatchObject(cros_build_lib, 'MachineDetails', return_value='TestHost')
- lock = gslock.Lock(self.lock_uri)
+ with gs.TemporaryURL('gslock') as lock_uri:
+ lock = gslock.Lock(lock_uri)
- self.assertFalse(self.ctx.Exists(self.lock_uri))
- lock.Acquire()
- self.assertTrue(self.ctx.Exists(self.lock_uri))
+ self.assertFalse(self.ctx.Exists(lock_uri))
+ lock.Acquire()
+ self.assertTrue(self.ctx.Exists(lock_uri))
- lock.Acquire()
- self.assertTrue(self.ctx.Exists(self.lock_uri))
+ lock.Acquire()
+ self.assertTrue(self.ctx.Exists(lock_uri))
- lock.Release()
- self.assertFalse(self.ctx.Exists(self.lock_uri))
+ lock.Release()
+ self.assertFalse(self.ctx.Exists(lock_uri))
- lock.Acquire()
- self.assertTrue(self.ctx.Exists(self.lock_uri))
+ lock.Acquire()
+ self.assertTrue(self.ctx.Exists(lock_uri))
- lock.Release()
- self.assertFalse(self.ctx.Exists(self.lock_uri))
+ lock.Release()
+ self.assertFalse(self.ctx.Exists(lock_uri))
@cros_test_lib.NetworkTest()
def testLockConflict(self):
"""Test lock conflict."""
+ with gs.TemporaryURL('gslock') as lock_uri:
+ lock1 = gslock.Lock(lock_uri)
+ lock2 = gslock.Lock(lock_uri)
- lock1 = gslock.Lock(self.lock_uri)
- lock2 = gslock.Lock(self.lock_uri)
+ # Manually lock 1, and ensure lock2 can't lock.
+ lock1.Acquire()
+ self.assertRaises(gslock.LockNotAcquired, lock2.Acquire)
+ lock1.Release()
- # Manually lock 1, and ensure lock2 can't lock.
- lock1.Acquire()
- self.assertRaises(gslock.LockNotAcquired, lock2.Acquire)
- lock1.Release()
+ # Use a with clause on 2, and ensure 1 can't lock.
+ with lock2:
+ self.assertRaises(gslock.LockNotAcquired, lock1.Acquire)
- # Use a with clause on 2, and ensure 1 can't lock.
- with lock2:
- self.assertRaises(gslock.LockNotAcquired, lock1.Acquire)
+ # Ensure we can renew a given lock.
+ lock1.Acquire()
+ lock1.Renew()
+ lock1.Release()
- # Ensure we can renew a given lock.
- lock1.Acquire()
- lock1.Renew()
- lock1.Release()
-
- # Ensure we get an error renewing a lock we don't hold.
- self.assertRaises(gslock.LockNotAcquired, lock1.Renew)
+ # Ensure we get an error renewing a lock we don't hold.
+ self.assertRaises(gslock.LockNotAcquired, lock1.Renew)
@cros_test_lib.NetworkTest()
def testLockTimeout(self):
"""Test getting a lock when an old timed out one is present."""
+ with gs.TemporaryURL('gslock') as lock_uri:
+ # Both locks are always timed out.
+ lock1 = gslock.Lock(lock_uri, lock_timeout_mins=-1)
+ lock2 = gslock.Lock(lock_uri, lock_timeout_mins=-1)
- # Both locks are always timed out.
- lock1 = gslock.Lock(self.lock_uri, lock_timeout_mins=-1)
- lock2 = gslock.Lock(self.lock_uri, lock_timeout_mins=-1)
-
- lock1.Acquire()
- lock2.Acquire()
-
- self.ctx.Remove(self.lock_uri)
+ lock1.Acquire()
+ lock2.Acquire()
@cros_test_lib.NetworkTest()
def testRaceToAcquire(self):
"""Have lots of processes race to acquire the same lock."""
- count = 20
+ count = self.NUM_THREADS
pool = multiprocessing.Pool(processes=count)
- results = pool.map(_InProcessAcquire, [self.lock_uri] * count)
+ with gs.TemporaryURL('gslock') as lock_uri:
+ results = pool.map(_InProcessAcquire, [lock_uri] * count)
- # Clean up the lock since the processes explicitly only acquire.
- self.ctx.Remove(self.lock_uri)
+ # Clean up the lock since the processes explicitly only acquire.
+ self.ctx.Remove(lock_uri)
- # Ensure that only one of them got the lock.
- self.assertEqual(results.count(True), 1)
+ # Ensure that only one of them got the lock.
+ self.assertEqual(results.count(True), 1)
@cros_test_lib.NetworkTest()
def testRaceToDoubleAcquire(self):
"""Have lots of processes race to double acquire the same lock."""
- count = 20
+ count = self.NUM_THREADS
pool = multiprocessing.Pool(processes=count)
- results = pool.map(_InProcessDoubleAcquire, [self.lock_uri] * count)
+ with gs.TemporaryURL('gslock') as lock_uri:
+ results = pool.map(_InProcessDoubleAcquire, [lock_uri] * count)
- # Clean up the lock sinc the processes explicitly only acquire.
- self.ctx.Remove(self.lock_uri)
+ # Clean up the lock sinc the processes explicitly only acquire.
+ self.ctx.Remove(lock_uri)
- # Ensure that only one of them got the lock (and got it twice).
- self.assertEqual(results.count(0), count - 1)
- self.assertEqual(results.count(2), 1)
+ # Ensure that only one of them got the lock (and got it twice).
+ self.assertEqual(results.count(0), count - 1)
+ self.assertEqual(results.count(2), 1)
@cros_test_lib.NetworkTest()
def testMultiProcessDataUpdate(self):
"""Have lots of processes update a GS file proctected by a lock."""
- count = 20 # To really stress, bump up to 200.
+ count = self.NUM_THREADS
pool = multiprocessing.Pool(processes=count)
- results = pool.map(_InProcessDataUpdate,
- [(self.lock_uri, self.data_uri)] * count)
-
- self.assertEqual(self.ctx.Cat(self.data_uri), str(count))
+ with gs.TemporaryURL('gslock') as lock_uri:
+ data_uri = lock_uri + '.data'
+ results = pool.map(_InProcessDataUpdate,
+ [(lock_uri, data_uri)] * count)
- # Ensure that all report success
- self.assertEqual(results.count(True), count)
+ self.assertEqual(self.ctx.Cat(data_uri), str(count))
- # Clean up the data file.
- self.ctx.Remove(self.data_uri)
+ # Ensure that all report success
+ self.assertEqual(results.count(True), count)
@cros_test_lib.NetworkTest()
def testDryrunLock(self):
"""Ensure that lcok can be obtained and released in dry-run mode."""
- lock = gslock.Lock(self.lock_uri, dry_run=True)
- self.assertIsNone(lock.Acquire())
- self.assertFalse(self.ctx.Exists(self.lock_uri))
- self.assertIsNone(lock.Release())
+ with gs.TemporaryURL('gslock') as lock_uri:
+ lock = gslock.Lock(lock_uri, dry_run=True)
+ self.assertIsNone(lock.Acquire())
+ self.assertFalse(self.ctx.Exists(lock_uri))
+ self.assertIsNone(lock.Release())
+ @cros_test_lib.NetworkTest()
def testDryrunLockRepetition(self):
"""Test aquiring same lock multiple times in dry-run mode."""
- lock = gslock.Lock(self.lock_uri, dry_run=True)
- self.assertIsNone(lock.Acquire())
- self.assertIsNone(lock.Acquire())
- self.assertIsNone(lock.Release())
- self.assertIsNone(lock.Acquire())
- self.assertIsNone(lock.Release())
+ with gs.TemporaryURL('gslock') as lock_uri:
+ lock = gslock.Lock(lock_uri, dry_run=True)
+ self.assertIsNone(lock.Acquire())
+ self.assertIsNone(lock.Acquire())
+ self.assertIsNone(lock.Release())
+ self.assertIsNone(lock.Acquire())
+ self.assertIsNone(lock.Release())
diff --git a/lib/paygen/gspaths.py b/lib/paygen/gspaths.py
index efcc6d793..87d6232f1 100644
--- a/lib/paygen/gspaths.py
+++ b/lib/paygen/gspaths.py
@@ -138,9 +138,19 @@ class Payload(utils.RestrictedAttrDict):
src_image: A representation of image it updates from. None for
Full updates, or the same type as tgt_image otherwise.
uri: The URI of the payload. This can be any format understood by urilib.
+ labels: A list of strings. Labels are used to catalogue payloads.
+ skip: A boolean. If true, we skip generating this payload.
+ exists: A boolean. If true, artifacts for this build already exist.
"""
_name = 'Payload definition'
- _slots = ('tgt_image', 'src_image', 'uri')
+ _slots = ('tgt_image', 'src_image', 'uri', 'labels', 'skip', 'exists')
+
+ def __init__(self, labels=None, skip=False, exists=False, *args, **kwargs):
+ kwargs.update(labels=labels, skip=skip, exists=exists)
+ super(Payload, self).__init__(*args, **kwargs)
+
+ if self['labels'] is None:
+ self['labels'] = []
def __str__(self):
if self.uri:
diff --git a/lib/paygen/gspaths_unittest.py b/lib/paygen/gspaths_unittest.py
index bc506ec67..4b581dc51 100644
--- a/lib/paygen/gspaths_unittest.py
+++ b/lib/paygen/gspaths_unittest.py
@@ -514,6 +514,9 @@ class GsPathsChromeosReleasesTest(cros_test_lib.TestCase):
key=self.key),
'src_image': None,
'uri': full_uri,
+ 'labels': [],
+ 'skip': False,
+ 'exists': False
})
self.assertDictEqual(
@@ -527,6 +530,9 @@ class GsPathsChromeosReleasesTest(cros_test_lib.TestCase):
version=self.version,
key=self.key),
'uri': delta_uri,
+ 'labels': [],
+ 'skip': False,
+ 'exists': False
})
self.assertDictEqual(
@@ -541,6 +547,9 @@ class GsPathsChromeosReleasesTest(cros_test_lib.TestCase):
image_channel='image-channel'),
'src_image': None,
'uri': max_full_uri,
+ 'labels': [],
+ 'skip': False,
+ 'exists': False
})
self.assertDictEqual(
@@ -558,6 +567,9 @@ class GsPathsChromeosReleasesTest(cros_test_lib.TestCase):
image_version=image_version,
image_channel='image-channel'),
'uri': max_delta_uri,
+ 'labels': [],
+ 'skip': False,
+ 'exists': False
})
diff --git a/lib/paygen/paygen_build_lib.py b/lib/paygen/paygen_build_lib.py
index d86add225..b9d383eca 100644
--- a/lib/paygen/paygen_build_lib.py
+++ b/lib/paygen/paygen_build_lib.py
@@ -13,7 +13,6 @@ and setting flags to show that a build has been processed.
from __future__ import print_function
-import ConfigParser
import json
import operator
import os
@@ -51,17 +50,12 @@ sys.path.insert(0, AUTOTEST_DIR)
# will fail. We quietly ignore the failure, but leave bombs around that will
# explode if people try to really use this library.
try:
- from crostools.config import config
- from crostools.omaha import query
-
# pylint: disable=F0401
from site_utils.autoupdate.lib import test_params
from site_utils.autoupdate.lib import test_control
# pylint: enable=F0401
except ImportError:
- config = None
- query = None
test_params = None
test_control = None
@@ -72,8 +66,10 @@ RUN_SUITE_MIN_MSTONE = 30
# Used to format timestamps on archived paygen.log file names in GS.
PAYGEN_LOG_TIMESTAMP_FORMAT = '%Y%m%d-%H%M%S-UTC'
-# Used to lookup all FSIs for all boards.
+# Board and device information published by goldeneye.
+BOARDS_URI = 'gs://chromeos-build-release-console/boards.json'
FSI_URI = 'gs://chromeos-build-release-console/fsis.json'
+OMAHA_URI = 'gs://chromeos-build-release-console/omaha_status.json'
class Error(Exception):
@@ -118,7 +114,7 @@ class BuildNotReady(EarlyExit):
class BoardNotConfigured(EarlyExit):
- """The board does not exist in the crostools release config."""
+ """The board does not exist in the published goldeneye records."""
RESULT = 26
@@ -278,6 +274,58 @@ def _GenerateSinglePayload(payload, work_dir, sign, au_generator_uri, dry_run):
dry_run=dry_run)
+class PayloadManager(object):
+ """Helper class for classifying discovered payloads."""
+
+ def __init__(self):
+ self.payloads = []
+
+ def Add(self, labels, payloads, skip=False, exists=False):
+ for p in payloads:
+ self.payloads.append(gspaths.Payload(tgt_image=p.tgt_image,
+ src_image=p.src_image,
+ uri=p.uri, labels=labels,
+ skip=skip, exists=exists))
+
+ def Get(self, labels):
+ """Retrieve all payloads that have label sets that contain |labels|.
+
+ Args:
+ labels: A list of strings.
+
+ Returns:
+ A list of gspath.Payload objects that define |labels|.
+
+ Raises:
+ ValueError if |labels| is not a list.
+ """
+ if not isinstance(labels, list):
+ raise ValueError('PayloadManager.Get expects a list of labels.'
+ ' Given %s' % type(labels))
+ labels = set(labels)
+ return [p for p in self.payloads
+ if set(p['labels']).issuperset(labels)]
+
+ def GetOnly(self, labels):
+ """Retrieve all payloads with label sets that are equal to |labels|.
+
+ Args:
+ labels: A list of strings.
+
+ Returns:
+ A list of gspath.Payload objects with label sets equal to |labels|.
+
+ Raises:
+ ValueError if |labels| is not a list.
+ """
+ if not isinstance(labels, list):
+ raise ValueError('PayloadManager.GetOnly expects a list of labels.'
+ ' Given %s' % type(labels))
+
+ labels = set(labels)
+ return [p for p in self.payloads if set(p['labels']) == labels]
+
+
class _PaygenBuild(object):
"""This class is responsible for generating the payloads for a given build.
@@ -366,7 +414,6 @@ class _PaygenBuild(object):
self._skip_nontest_payloads = skip_nontest_payloads
self._control_dir = control_dir
self._output_dir = output_dir
- self._previous_version = None
self._run_parallel = run_parallel
self._run_on_builder = run_on_builder
self._archive_board = None
@@ -374,6 +421,20 @@ class _PaygenBuild(object):
self._archive_build_uri = None
self._au_generator_uri = au_generator_uri
+ # Cached goldeneye data.
+ self.cachedFsisJson = {}
+ self.cachedOmahaJson = {}
+
+ def _GetFsisJson(self):
+ if not self.cachedFsisJson:
+ self.cachedFsisJson = json.loads(gslib.Cat(FSI_URI))
+ return self.cachedFsisJson
+
+ def _GetOmahaJson(self):
+ if not self.cachedOmahaJson:
+ self.cachedOmahaJson = json.loads(gslib.Cat(OMAHA_URI))
+ return self.cachedOmahaJson
+
def _GetFlagURI(self, flag):
"""Find the URI of the lock file associated with this build.
@@ -555,7 +616,7 @@ class _PaygenBuild(object):
return images
- def _DiscoverActiveFsiBuilds(self):
+ def _DiscoverFsiBuildsForDeltas(self):
"""Read fsi_images in release.conf.
fsi_images is a list of chromeos versions. We assume each one is
@@ -570,31 +631,61 @@ class _PaygenBuild(object):
List of gspaths.Build instances for each build so discovered. The list
may be empty.
"""
- # TODO(dgarrett): Switch to JSON mechanism in _DiscoverAllFsiBuilds
- # after it's in production, and after we clear the change with the TPMs.
- # At that time, check and ignore FSIs without the is_delta_supported flag.
- # TODO(pprabhu): Can't switch to _DiscoverAllFsiBuilds till the HACK there
- # is removed.
+ results = []
# FSI versions are only defined for the stable-channel.
if self._build.channel != 'stable-channel':
- return []
+ return results
- try:
- fsi_versions = config.GetListValue(self._build.board, 'fsi_images')
- except ConfigParser.NoOptionError:
- # fsi_images is an optional field.
- return []
+ contents = self._GetFsisJson()
+
+ for fsi in contents.get('fsis', []):
+ fsi_active = fsi['board']['is_active']
+ fsi_board = fsi['board']['public_codename']
+ fsi_version = fsi['chrome_os_version']
+ fsi_support_delta = fsi['is_delta_supported']
+
+ if fsi_active and fsi_support_delta and fsi_board == self._build.board:
+ results.append(gspaths.Build(version=fsi_version,
+ board=fsi_board,
+ channel=self._build.channel,
+ bucket=self._build.bucket))
+
+ return results
+
+ def _DiscoverAllFsiBuildsForDeltaTesting(self):
+ """Pull FSI list from Golden Eye.
+ Finds all FSI builds that are known to support deltas and to be
+ lab stable for testing purposes.
+
+ Returns:
+ A list of gspaths.Build instances for each build so discovered. The list
+ may be empty.
+ """
results = []
- for version in fsi_versions:
- results.append(gspaths.Build(version=version,
- board=self._build.board,
- channel=self._build.channel,
- bucket=self._build.bucket))
+
+ contents = self._GetFsisJson()
+
+ for fsi in contents.get('fsis', []):
+ fsi_active = fsi['board']['is_active']
+ fsi_board = fsi['board']['public_codename']
+ fsi_version = fsi['chrome_os_version']
+ fsi_support_delta = fsi['is_delta_supported']
+ fsi_lab_stable = fsi['is_lab_stable']
+
+ conditions = [fsi_board == self._build.board, fsi_active,
+ fsi_support_delta, fsi_lab_stable]
+
+ if all(conditions):
+ results.append(gspaths.Build(version=fsi_version,
+ board=fsi_board,
+ channel=self._build.channel,
+ bucket=self._build.bucket))
+
return results
- def _DiscoverAllFsiBuilds(self):
+ def _DiscoverAllFsiBuildsForFullTesting(self):
"""Pull FSI list from Golden Eye.
Returns a list of chromeos versions. We assume each one is
@@ -608,20 +699,15 @@ class _PaygenBuild(object):
may be empty.
"""
results = []
- # XXX:HACK -- FSI builds for this board is known to brick the DUTs in the
- # lab. As a workaround, we're dropping test coverage for this board
- # temporarily (crbug.com/460174).
- # TODO(pprabhu) Remove hack once we have a real solution (crbug.com/462320).
- if self._build.board == 'peach-pit':
- return results
- contents = json.loads(gslib.Cat(FSI_URI))
+ contents = self._GetFsisJson()
for fsi in contents.get('fsis', []):
fsi_board = fsi['board']['public_codename']
fsi_version = fsi['chrome_os_version']
+ fsi_lab_stable = fsi['is_lab_stable']
- if fsi_board == self._build.board:
+ if fsi_lab_stable and fsi_board == self._build.board:
results.append(fsi_version)
return results
@@ -638,16 +724,25 @@ class _PaygenBuild(object):
know about the currently published version, this always contain zero or
one entries.
"""
- self._previous_version = query.FindLatestPublished(self._build.channel,
- self._build.board)
+ results = []
- if self._previous_version:
- return [gspaths.Build(gspaths.Build(version=self._previous_version,
- board=self._build.board,
- channel=self._build.channel,
- bucket=self._build.bucket))]
+ # Paygen channel names typically end in '-channel', while Goldeneye
+ # does not maintain the '-channel' ending.
+ channel_name = self._build.channel.replace('-channel', '')
- return []
+ contents = self._GetOmahaJson()
+ for nmo in contents.get('omaha_data', []):
+ nmo_board = nmo['board']['public_codename']
+ nmo_channel = nmo['channel']
+ nmo_version = nmo['chrome_os_version']
+
+ if nmo_board == self._build.board and nmo_channel == channel_name:
+ results.append(gspaths.Build(gspaths.Build(version=nmo_version,
+ board=self._build.board,
+ channel=self._build.channel,
+ bucket=self._build.bucket)))
+
+ return results
def _DiscoverRequiredFullPayloads(self, images):
"""Find the Payload objects for the images from the current build.
@@ -765,9 +860,7 @@ class _PaygenBuild(object):
and may be otherwise detrimental to the release schedule.
Returns:
- A list of tuples of the form (payload, skip), where payload is an
- instance of gspath.Payload and skip is a Boolean that says whether it
- should be skipped (i.e. not generated).
+ A PayloadManager instance.
Raises:
BuildNotReady: If the current build doesn't seem to have all of it's
@@ -776,12 +869,14 @@ class _PaygenBuild(object):
BuildCorrupt: If current or previous builds have unexpected images.
ImageMissing: Raised if expected images are missing for previous builds.
"""
- # Initiate a list that will contain lists of payload subsets, along with a
- # Boolean stating whether or not we need to skip generating them.
- payload_sublists_skip = []
+ images = []
+ previous_images = []
+ fsi_images = []
+
+ payload_manager = PayloadManager()
try:
- # When discovering the images for our current build, they might
+ # When discovering the images for our current build, they might not be
# discoverable right away (GS eventual consistency). So, we retry.
images = retry_util.RetryException(ImageMissing, 3,
self._DiscoverImages, self._build,
@@ -795,110 +890,147 @@ class _PaygenBuild(object):
_LogList('Images found', images)
- # Discover active FSI builds we need deltas from.
- fsi_builds = self._DiscoverActiveFsiBuilds()
+ # Discover and filter active FSI builds.
+ fsi_builds = self._DiscoverFsiBuildsForDeltas()
if fsi_builds:
_LogList('Active FSI builds considered', fsi_builds)
else:
logging.info('No active FSI builds found')
- # Discover other previous builds we need deltas from.
+ for fsi in fsi_builds:
+ fsi_images += self._DiscoverImages(fsi)
+ fsi_images += self._DiscoverTestImageArchives(fsi)
+
+ fsi_images = _FilterForBasic(fsi_images) + _FilterForTest(fsi_images)
+
+ # Discover previous, non-FSI, builds that we also must generate deltas for.
previous_builds = [b for b in self._DiscoverNmoBuild()
if b not in fsi_builds]
if previous_builds:
- _LogList('Other previous builds considered', previous_builds)
+ _LogList('Previous, non-FSI, builds considered', previous_builds)
else:
logging.info('No other previous builds found')
- # Discover the images from those previous builds, and put them into
- # a single list. Raises ImageMissing if no images are found.
- previous_images = []
- for b in previous_builds:
+ # Discover and filter previous images.
+ for p in previous_builds:
try:
- previous_images += self._DiscoverImages(b)
+ previous_images += self._DiscoverImages(p)
except ImageMissing as e:
# Temporarily allow generation of delta payloads to fail because of
# a missing previous build until crbug.com/243916 is addressed.
# TODO(mtennant): Remove this when bug is fixed properly.
logging.warning('Previous build image is missing, skipping: %s', e)
- # We also clear the previous version field so that subsequent code does
- # not attempt to generate a full update test from the N-1 version;
- # since this version has missing images, no payloads were generated for
- # it and test generation is bound to fail.
- # TODO(garnold) This should be reversed together with the rest of this
- # block.
- self._previous_version = None
-
# In this case, we should also skip test image discovery; since no
# signed deltas will be generated from this build, we don't need to
# generate test deltas from it.
continue
+ previous_images += self._DiscoverTestImageArchives(p)
- previous_images += self._DiscoverTestImageArchives(b)
+ previous_images = (
+ _FilterForBasic(previous_images) + _FilterForTest(previous_images))
- for b in fsi_builds:
- previous_images += self._DiscoverImages(b)
- previous_images += self._DiscoverTestImageArchives(b)
+ # Discover and catalogue full, non-test payloads.
+ skip_full = self._skip_full_payloads or self._skip_nontest_payloads
- # Only consider base (signed) and test previous images.
- filtered_previous_images = _FilterForBasic(previous_images)
- filtered_previous_images += _FilterForTest(previous_images)
- previous_images = filtered_previous_images
+ # Full payloads for the current build.
+ payload_manager.Add(
+ ['full'],
+ self._DiscoverRequiredFullPayloads(_FilterForImages(images)),
+ skip=skip_full)
- # Generate full payloads for all non-test images in the current build.
- # Include base, NPO, premp, and mp (if present).
- payload_sublists_skip.append(
- (self._skip_full_payloads or self._skip_nontest_payloads,
- self._DiscoverRequiredFullPayloads(_FilterForImages(images))))
+ # Full payloads for previous builds.
+ payload_manager.Add(
+ ['full', 'previous'],
+ self._DiscoverRequiredFullPayloads(_FilterForImages(previous_images)),
+ skip=skip_full)
+
+ # Discover delta payloads.
+ skip_deltas = self._skip_delta_payloads or self._skip_nontest_payloads
# Deltas for current -> NPO (pre-MP and MP).
- payload_sublists_skip.append(
- (self._skip_delta_payloads or self._skip_nontest_payloads,
- self._DiscoverRequiredNpoDeltas(_FilterForPremp(images))))
- payload_sublists_skip.append(
- (self._skip_delta_payloads or self._skip_nontest_payloads,
- self._DiscoverRequiredNpoDeltas(_FilterForMp(images))))
+ delta_npo_labels = ['delta', 'npo']
+ payload_manager.Add(
+ delta_npo_labels,
+ self._DiscoverRequiredNpoDeltas(_FilterForPremp(images)),
+ skip=skip_deltas)
+ payload_manager.Add(
+ delta_npo_labels,
+ self._DiscoverRequiredNpoDeltas(_FilterForMp(images)),
+ skip=skip_deltas)
# Deltas for previous -> current (pre-MP and MP).
- payload_sublists_skip.append(
- (self._skip_delta_payloads or self._skip_nontest_payloads,
- self._DiscoverRequiredFromPreviousDeltas(
- _FilterForPremp(_FilterForBasic(images)),
- _FilterForPremp(previous_images))))
- payload_sublists_skip.append(
- (self._skip_delta_payloads or self._skip_nontest_payloads,
- self._DiscoverRequiredFromPreviousDeltas(
- _FilterForMp(_FilterForBasic(images)),
- _FilterForMp(previous_images))))
-
- # Only discover test payloads if Autotest is not disabled.
+ delta_previous_labels = ['delta', 'previous']
+ payload_manager.Add(
+ delta_previous_labels,
+ self._DiscoverRequiredFromPreviousDeltas(
+ _FilterForPremp(_FilterForBasic(images)),
+ _FilterForPremp(previous_images)),
+ skip=skip_deltas)
+ payload_manager.Add(
+ delta_previous_labels,
+ self._DiscoverRequiredFromPreviousDeltas(
+ _FilterForMp(_FilterForBasic(images)),
+ _FilterForMp(previous_images)),
+ skip=skip_deltas)
+
+ # Deltats for fsi -> current (pre-MP and MP).
+ delta_fsi_labels = ['delta', 'fsi']
+ payload_manager.Add(
+ delta_fsi_labels,
+ self._DiscoverRequiredFromPreviousDeltas(
+ _FilterForPremp(_FilterForBasic(images)),
+ _FilterForPremp(fsi_images)),
+ skip=skip_deltas)
+ payload_manager.Add(
+ delta_fsi_labels,
+ self._DiscoverRequiredFromPreviousDeltas(
+ _FilterForMp(_FilterForBasic(images)),
+ _FilterForMp(fsi_images)),
+ skip=skip_deltas)
+
+ # Discover test payloads if Autotest is not disabled.
if self._control_dir:
- # Full test payloads.
- payload_sublists_skip.append(
- (self._skip_full_payloads or self._skip_test_payloads,
- self._DiscoverRequiredFullPayloads(_FilterForTest(images))))
+ skip_test_full = self._skip_full_payloads or self._skip_test_payloads
+ skip_test_deltas = self._skip_delta_payloads or self._skip_test_payloads
- # Delta for current -> NPO (test payloads).
- payload_sublists_skip.append(
- (self._skip_delta_payloads or self._skip_test_payloads,
- self._DiscoverRequiredTestNpoDeltas(_FilterForTest(images))))
+ # Full test payloads.
+ payload_manager.Add(
+ ['test', 'full'],
+ self._DiscoverRequiredFullPayloads(_FilterForTest(images)),
+ skip=skip_test_full)
+
+ # Full previous payloads.
+ payload_manager.Add(
+ ['test', 'full', 'previous'],
+ self._DiscoverRequiredFullPayloads(_FilterForTest(previous_images)),
+ skip=skip_test_full)
+
+ # Deltas for current -> NPO (test payloads).
+ payload_manager.Add(
+ ['test', 'delta', 'npo'],
+ self._DiscoverRequiredTestNpoDeltas(_FilterForTest(images)),
+ skip=skip_test_deltas)
# Deltas for previous -> current (test payloads).
- payload_sublists_skip.append(
- (self._skip_delta_payloads or self._skip_test_payloads,
- self._DiscoverRequiredFromPreviousDeltas(
- _FilterForTest(images), _FilterForTest(previous_images))))
+ payload_manager.Add(
+ ['test', 'delta', 'previous'],
+ self._DiscoverRequiredFromPreviousDeltas(
+ _FilterForTest(images), _FilterForTest(previous_images)),
+ skip=skip_test_deltas)
- # Organize everything into a single list of (payload, skip) pairs; also, be
- # sure to fill in a URL for each payload.
- payloads_skip = []
- for (do_skip, payloads) in payload_sublists_skip:
- for payload in payloads:
- paygen_payload_lib.FillInPayloadUri(payload)
- payloads_skip.append((payload, do_skip))
+ # Deltas for fsi -> current (test payloads).
+ payload_manager.Add(
+ ['test', 'delta', 'fsi'],
+ self._DiscoverRequiredFromPreviousDeltas(
+ _FilterForTest(images), _FilterForTest(fsi_images)),
+ skip=skip_test_deltas)
- return payloads_skip
+ # Set the payload URIs.
+ for p in payload_manager.Get([]):
+ paygen_payload_lib.FillInPayloadUri(p)
+
+ return payload_manager
def _GeneratePayloads(self, payloads, lock=None):
"""Generate the payloads called for by a list of payload definitions.
@@ -1012,7 +1144,6 @@ class _PaygenBuild(object):
test = test_params.TestConfig(
self._archive_board,
suite_name, # Name of the test (use the suite name).
- False, # Using test images.
bool(payload.src_image), # Whether this is a delta.
src_version,
payload.tgt_image.version,
@@ -1192,49 +1323,66 @@ class _PaygenBuild(object):
logging.info('No FSIs with artifacts, not scheduling FSI update test.')
return []
- def _CreatePayloadTests(self, payloads):
+ def _CreatePayloadTests(self, payload_manager):
"""Returns a list of test configurations for a given list of payloads.
Args:
- payloads: A list of (already generated) build payloads.
+ payload_manager: A PayloadManager instance.
Returns:
A list of PayloadTest objects defining payload test cases.
"""
payload_tests = []
- for payload in payloads:
- # We are only testing test payloads.
- if payload.tgt_image.get('image_type', 'signed') == 'signed':
- continue
- # Distinguish between delta (source version encoded) and full payloads.
- if payload.src_image is None:
- # Create a full update test from NMO, if we are newer.
- if not self._previous_version:
- logging.warning('No previous build, not testing full update %s from '
- 'NMO', payload)
- elif gspaths.VersionGreater(
- self._previous_version, payload.tgt_image.version):
+ # Pre-fetch lab stable FSIs.
+ lab_stable_fsi_deltas = self._DiscoverAllFsiBuildsForDeltaTesting()
+ lab_stable_fsi_full = self._DiscoverAllFsiBuildsForFullTesting()
+
+ def IsFsiLabStable(fsi_image):
+ for build in lab_stable_fsi_deltas:
+ if all([fsi_image.board == build.board,
+ fsi_image.channel == build.channel,
+ fsi_image.version == build.version,
+ fsi_image.bucket == build.bucket]):
+ return True
+ return False
+
+ # Create full update tests that involve the current build.
+ for p in payload_manager.GetOnly(['test', 'full']):
+
+ # Update tests from previous to current, if we are newer.
+ for p_prev in payload_manager.GetOnly(['test', 'full', 'previous']):
+ if gspaths.VersionGreater(p_prev.tgt_image.version,
+ p.tgt_image.version):
logging.warning(
'NMO (%s) is newer than target (%s), skipping NMO full '
- 'update test.', self._previous_version, payload)
- else:
- payload_tests.append(self.PayloadTest(
- payload, src_channel=self._build.channel,
- src_version=self._previous_version))
+ 'update test.', p_prev, p)
+ continue
- # Create a full update test from the current version to itself.
payload_tests.append(self.PayloadTest(
- payload,
- src_channel=self._build.channel,
- src_version=self._build.version))
+ p,
+ src_channel=p_prev.tgt_image.channel,
+ src_version=p_prev.tgt_image.version))
+
+ # Update test from current version to itself.
+ payload_tests.append(self.PayloadTest(
+ p,
+ src_channel=self._build.channel,
+ src_version=self._build.version))
+
+ # Update test from the oldest viable FSI.
+ payload_tests += self._CreateFsiPayloadTests(p, lab_stable_fsi_full)
+
+ # Create delta payload tests.
+ for p in payload_manager.Get(['test', 'delta']):
+ # FSI deltas are included only if they are known to be lab stable.
+ if 'fsi' in p.labels and not IsFsiLabStable(p.src_image):
+ logging.warning(
+ 'FSI delta payload (%s) is not lab stable, skipping '
+ 'delta update test', p)
+ continue
- # Create a full update test from oldest viable FSI.
- payload_tests += self._CreateFsiPayloadTests(
- payload, self._DiscoverAllFsiBuilds())
- else:
- # Create a delta update test.
- payload_tests.append(self.PayloadTest(payload))
+ payload_tests.append(self.PayloadTest(p))
return payload_tests
@@ -1277,7 +1425,7 @@ class _PaygenBuild(object):
logging.info('Starting: %s', self._build)
- payloads_skip = self._DiscoverRequiredPayloads()
+ payload_manager = self._DiscoverRequiredPayloads()
# Assume we can finish the build until we find a reason we can't.
can_finish = True
@@ -1289,41 +1437,40 @@ class _PaygenBuild(object):
# URI accordingly. In doing so we're creating a list of all payload
# objects and their skip/exist attributes. We're also recording whether
# this run will be skipping any actual work.
- payloads_attrs = []
- for payload, skip in payloads_skip:
+ for p in payload_manager.Get([]):
if self._output_dir:
# output_dir means we are forcing all payloads to be generated
# with a new destination.
result = [os.path.join(self._output_dir,
- os.path.basename(payload.uri))]
+ os.path.basename(p.uri))]
exists = False
else:
- result = paygen_payload_lib.FindExistingPayloads(payload)
+ result = paygen_payload_lib.FindExistingPayloads(p)
exists = bool(result)
if result:
- paygen_payload_lib.SetPayloadUri(payload, result[0])
- elif skip:
+ paygen_payload_lib.SetPayloadUri(p, result[0])
+ elif p['skip']:
can_finish = False
- payloads_attrs.append((payload, skip, exists))
+ p['exists'] = exists
# Display payload generation list, including payload name and whether
# or not it already exists or will be skipped.
log_items = []
- for payload, skip, exists in payloads_attrs:
- desc = str(payload)
- if exists:
+ for p in payload_manager.Get([]):
+ desc = str(p)
+ if p['exists']:
desc += ' (exists)'
- elif skip:
+ elif p['skip']:
desc += ' (skipped)'
log_items.append(desc)
_LogList('All payloads for the build', log_items)
# Generate new payloads.
- new_payloads = [payload for payload, skip, exists in payloads_attrs
- if not (skip or exists)]
+ new_payloads = [p for p in payload_manager.Get([])
+ if not (p['skip'] or p['exists'])]
if new_payloads:
logging.info('Generating %d new payload(s)', len(new_payloads))
self._GeneratePayloads(new_payloads, build_lock)
@@ -1348,8 +1495,7 @@ class _PaygenBuild(object):
# We have a control file directory and all payloads have been
# generated. Lets create the list of tests to conduct.
- payload_tests = self._CreatePayloadTests(
- [payload for payload, _, _ in payloads_attrs])
+ payload_tests = self._CreatePayloadTests(payload_manager)
if payload_tests:
logging.info('Initiating %d payload tests', len(payload_tests))
self._drm(self._AutotestPayloads, payload_tests)
@@ -1413,8 +1559,11 @@ def ValidateBoardConfig(board):
BoardNotConfigured if the board is unknown.
"""
# Right now, we just validate that the board exists.
- if board not in config.GetCompleteBoardSet():
- raise BoardNotConfigured(board)
+ boards = json.loads(gslib.Cat(BOARDS_URI))
+ for b in boards.get('boards', []):
+ if b['public_codename'] == board:
+ return
+ raise BoardNotConfigured(board)
def CreatePayloads(build, work_dir, site_config, dry_run=False,
diff --git a/lib/paygen/paygen_build_lib_unittest.py b/lib/paygen/paygen_build_lib_unittest.py
index 39e7439c8..f828e143d 100644
--- a/lib/paygen/paygen_build_lib_unittest.py
+++ b/lib/paygen/paygen_build_lib_unittest.py
@@ -6,7 +6,6 @@
from __future__ import print_function
-import itertools
import mox
import os
import shutil
@@ -38,6 +37,56 @@ from chromite.lib.paygen import utils
# pylint: disable=protected-access
+class PayloadManagerTest(cros_test_lib.MockTestCase):
+ """Unittests for the PayloadManager."""
+
+ def testAdd(self):
+ """Test adding payloads to the manager."""
+ pm = paygen_build_lib.PayloadManager()
+
+ labels = ['test']
+ payloads = [gspaths.Payload(tgt_image='foo',
+ src_image='bar',
+ uri='baz'),
+ gspaths.Payload(tgt_image='apple',
+ src_image='orange',
+ uri='mango')]
+
+ pm.Add(labels, payloads)
+
+ for payload, p in zip(payloads, pm.payloads):
+ self.assertEquals(labels, p.labels)
+ self.assertEquals(payload.tgt_image, p.tgt_image)
+ self.assertEquals(payload.src_image, p.src_image)
+ self.assertEquals(payload.uri, p.uri)
+
+ def testGet(self):
+ """Test retrieving payloads from the manager."""
+ pm = paygen_build_lib.PayloadManager()
+
+ p1 = gspaths.Payload(tgt_image='foo', labels=['foo', 'test'])
+ p2 = gspaths.Payload(tgt_image='bar', labels=['bar', 'test'])
+
+ pm.payloads = [p1, p2]
+
+ self.assertEquals([p1], pm.Get(['foo']))
+ self.assertEquals([p2], pm.Get(['bar']))
+ self.assertEquals([p1, p2], pm.Get(['test']))
+ self.assertEquals([], pm.Get(['foo', 'bar']))
+
+ def testGetOnly(self):
+ """Test retrieving payloads from the manager."""
+ pm = paygen_build_lib.PayloadManager()
+
+ p1 = gspaths.Payload(tgt_image='bar', labels=['bar', 'test'])
+ p2 = gspaths.Payload(tgt_image='bar', labels=['bar', 'test', 'test2'])
+
+ pm.payloads = [p1, p2]
+
+ self.assertEquals([p1, p2], pm.Get(['bar', 'test']))
+ self.assertEquals([p1], pm.GetOnly(['bar', 'test']))
+
+
class BasePaygenBuildLibTest(cros_test_lib.MoxTempDirTestCase):
"""Base class for testing PaygenBuildLib class."""
@@ -321,17 +370,30 @@ class PaygenBuildLibTest(BasePaygenBuildLibTest):
False,
paygen_build_lib.ImageMissing)
- @unittest.skipIf(not paygen_build_lib.config, 'Internal crostools required.')
- def testDiscoverActiveFsiBuilds(self):
- """Using test release.conf values, test _DiscoverActiveFsiBuilds."""
-
- test_config = """
-[valid-board]
-fsi_images: 2913.331.0,2465.105.0
-
-[no-fsi-board]
-"""
- paygen_build_lib.config.LoadTestConfig(test_config)
+ def testDiscoverFsiBuildsForDeltas(self):
+ """Using test goldeneye values, test _DiscoverFsiBuildsForDeltas."""
+ # Set up mock goldeneye fsi information.
+ mock_return_fsi = paygen_build_lib.json.dumps(
+ {'fsis':
+ [{u'is_delta_supported': True, u'chrome_os_version': u'2465.105.0',
+ u'board':
+ {u'public_codename': u'valid-board', u'is_active': True},
+ u'is_lab_stable': True, u'chrome_version': u'31.0.1650.61'},
+ {u'is_delta_supported': True, u'chrome_os_version': u'2467.109.0',
+ u'board':
+ {u'public_codename': u'valid-board', u'is_active': True},
+ u'is_lab_stable': False, u'chrome_version': u'31.0.1650.61'},
+ {u'is_delta_supported': False, u'chrome_os_version': u'2913.331.0',
+ u'board':
+ {u'public_codename': u'valid-board', u'is_active': True},
+ u'is_lab_stable': True, u'chrome_version': u'31.0.1650.61'}]
+ }
+ )
+
+ self.mox.StubOutWithMock(gslib, 'Cat')
+ gslib.Cat(paygen_build_lib.FSI_URI).AndReturn(mock_return_fsi)
+ gslib.Cat(paygen_build_lib.FSI_URI).AndReturn(mock_return_fsi)
+ self.mox.ReplayAll()
# Test a board with FSI values on stable-channel.
paygen = paygen_build_lib._PaygenBuild(
@@ -341,13 +403,13 @@ fsi_images: 2913.331.0,2465.105.0
config_lib_unittest.MockSiteConfig())
self.assertEqual(
- sorted(paygen._DiscoverActiveFsiBuilds()),
+ sorted(paygen._DiscoverFsiBuildsForDeltas()),
[gspaths.Build(board='valid-board',
channel='stable-channel',
version='2465.105.0'),
gspaths.Build(board='valid-board',
channel='stable-channel',
- version='2913.331.0')])
+ version='2467.109.0')])
# Test a board without FSI values on stable-channel.
paygen = paygen_build_lib._PaygenBuild(
@@ -356,7 +418,7 @@ fsi_images: 2913.331.0,2465.105.0
self.work_dir,
config_lib_unittest.MockSiteConfig())
- self.assertEqual(paygen._DiscoverActiveFsiBuilds(), [])
+ self.assertEqual(paygen._DiscoverFsiBuildsForDeltas(), [])
# Test a board with FSI values on non-stable-channel.
paygen = paygen_build_lib._PaygenBuild(
@@ -365,14 +427,43 @@ fsi_images: 2913.331.0,2465.105.0
self.work_dir,
config_lib_unittest.MockSiteConfig())
- self.assertEqual(paygen._DiscoverActiveFsiBuilds(), [])
+ self.assertEqual(paygen._DiscoverFsiBuildsForDeltas(), [])
+
+ def testDiscoverAllFsiBuildsForDeltaTesting(self):
+ """With goldeneye values, test _DiscoverAllFsiBuildsForDeltaTesting."""
+ mock_return_fsi = paygen_build_lib.json.dumps(
+ {'fsis':
+ [{u'is_delta_supported': True, u'chrome_os_version': u'2465.105.0',
+ u'board':
+ {u'public_codename': u'valid-board', u'is_active': True},
+ u'is_lab_stable': True, u'chrome_version': u'31.0.1650.61'},
+ {u'is_delta_supported': True, u'chrome_os_version': u'2467.109.0',
+ u'board':
+ {u'public_codename': u'valid-board', u'is_active': True},
+ u'is_lab_stable': False, u'chrome_version': u'31.0.1650.61'}]
+ }
+ )
+
+ self.mox.StubOutWithMock(gslib, 'Cat')
+ gslib.Cat(paygen_build_lib.FSI_URI).AndReturn(mock_return_fsi)
+ self.mox.ReplayAll()
- paygen_build_lib.config.LoadGlobalConfig()
+ # Test that only FSIs marked as is_lab_stable are selected.
+ paygen = paygen_build_lib._PaygenBuild(
+ gspaths.Build(channel='stable-channel', board='valid-board',
+ version='1.2.3'),
+ self.work_dir,
+ config_lib_unittest.MockSiteConfig())
+
+ self.assertEqual(
+ paygen._DiscoverAllFsiBuildsForDeltaTesting(),
+ [gspaths.Build(board='valid-board',
+ channel='stable-channel',
+ version='2465.105.0')])
@cros_test_lib.NetworkTest()
- @unittest.skipIf(not paygen_build_lib.config, 'Internal crostools required.')
- def testDiscoverAllFsiBuilds(self):
- """Using test release.conf values, test _DiscoverActiveFsiBuilds."""
+ def testDiscoverAllFsiBuildsForFullTesting(self):
+ """With goldeneye values, test _DiscoverAllFsiBuildsForFullTesting."""
paygen = paygen_build_lib._PaygenBuild(
gspaths.Build(channel='stable-channel', board='x86-alex-he',
version='1.2.3'),
@@ -380,22 +471,59 @@ fsi_images: 2913.331.0,2465.105.0
config_lib_unittest.MockSiteConfig())
# Search for real FSIs for an older/live board.
- self.assertEqual(paygen._DiscoverAllFsiBuilds(),
+ self.assertEqual(paygen._DiscoverAllFsiBuildsForFullTesting(),
['0.12.433.257', '0.14.811.132', '1412.205.0'])
- @unittest.skipIf(not paygen_build_lib.query, 'Internal crostools required.')
def testDiscoverNmoBuild(self):
"""Test _DiscoverNmoBuild (N minus One)."""
paygen = self._GetPaygenBuildInstance()
- self.mox.StubOutWithMock(paygen_build_lib.query, 'FindLatestPublished')
-
- # Set up the test replay script.
- paygen_build_lib.query.FindLatestPublished(
- 'foo-channel', 'foo-board').AndReturn('1.0.0')
-
- paygen_build_lib.query.FindLatestPublished(
- 'foo-channel', 'foo-board').AndReturn(None)
+ # Set up mock goldeneye omaha status information.
+ mock_return_foo = paygen_build_lib.json.dumps(
+ {'omaha_data':
+ [{u'is_mp_keyset': True, u'chrome_version': u'47.0.2514.0',
+ u'keyset': u'foo-mp',
+ u'board':
+ {u'public_codename': u'foo-board', u'is_active': True},
+ u'chrome_os_version': u'7478.0.0', u'channel': u'foo',
+ u'payloads':
+ [{u'max_fraction': False,
+ u'name': u'foo-channel/foo/7478.0.0/payloads/'
+ u'chromeos_7475.0.0-7478.0.0_foo_foo'
+ u'-channel_delta_mp.bin-877f148a914c1cdbe2'
+ u'42aa4247a1d135.signed', u'fraction': 1.0},
+ {u'max_fraction': False,
+ u'name': u'foo-channel/foo/7478.0.0/payloads/'
+ u'chromeos_7478.0.0_foo_foo-channel_'
+ u'full_mp.bin-fddc0ae18c9845325c13704ee00b'
+ u'd0a4.signed', u'fraction': 1.0}]}]
+ }
+ )
+
+ mock_return_not_foo = {
+ 'omaha_data':
+ [{u'is_mp_keyset': True, u'chrome_version': u'47.0.2514.0',
+ u'keyset': u'notfoo-mp',
+ u'board':
+ {u'public_codename': u'notfoo-board', u'is_active': True},
+ u'chrome_os_version': u'7478.0.0', u'channel': u'notfoo-channel',
+ u'payloads':
+ [{u'max_fraction': False,
+ u'name': u'notfoo-channel/notfoo/7478.0.0/payloads/'
+ u'chromeos_7475.0.0-7478.0.0_notfoo_notfoo'
+ u'-channel_delta_mp.bin-877f148a914c1cdbe2'
+ u'42aa4247a1d135.signed', u'fraction': 1.0},
+ {u'max_fraction': False,
+ u'name': u'notfoo-channel/notfoo/7478.0.0/payloads/'
+ u'chromeos_7478.0.0_notfoo_notfoo-channel_'
+ u'full_mp.bin-fddc0ae18c9845325c13704ee00b'
+ u'd0a4.signed', u'fraction': 1.0}]}]
+ }
+
+ # Mock out the actual call to gslib.Cat that occurs before a cached
+ # result exists in the paygen object.
+ self.mox.StubOutWithMock(gslib, 'Cat')
+ gslib.Cat(paygen_build_lib.OMAHA_URI).AndReturn(mock_return_foo)
# Run the test verification.
self.mox.ReplayAll()
@@ -404,8 +532,9 @@ fsi_images: 2913.331.0,2465.105.0
[gspaths.Build(bucket='crt',
channel='foo-channel',
board='foo-board',
- version='1.0.0')])
+ version='7478.0.0')])
+ paygen.cachedOmahaJson = mock_return_not_foo
self.assertEqual(paygen._DiscoverNmoBuild(), [])
def testDiscoverRequiredFullPayloads(self):
@@ -501,7 +630,7 @@ fsi_images: 2913.331.0,2465.105.0
self.mox.StubOutWithMock(paygen, '_DiscoverImages')
self.mox.StubOutWithMock(paygen, '_DiscoverNmoBuild')
- self.mox.StubOutWithMock(paygen, '_DiscoverActiveFsiBuilds')
+ self.mox.StubOutWithMock(paygen, '_DiscoverFsiBuildsForDeltas')
paygen.BUILD_DISCOVER_RETRY_SLEEP = 0
@@ -531,7 +660,7 @@ fsi_images: 2913.331.0,2465.105.0
self.mox.StubOutWithMock(paygen, '_DiscoverImages')
self.mox.StubOutWithMock(paygen, '_DiscoverTestImageArchives')
self.mox.StubOutWithMock(paygen, '_DiscoverNmoBuild')
- self.mox.StubOutWithMock(paygen, '_DiscoverActiveFsiBuilds')
+ self.mox.StubOutWithMock(paygen, '_DiscoverFsiBuildsForDeltas')
self.mox.StubOutWithMock(paygen_payload_lib, 'DefaultPayloadUri')
nmo_build = gspaths.Build(bucket='crt',
@@ -557,14 +686,14 @@ fsi_images: 2913.331.0,2465.105.0
paygen._DiscoverImages(paygen._build).AndReturn(self.images)
paygen._DiscoverTestImageArchives(paygen._build).AndReturn(
[self.test_image])
+ paygen._DiscoverFsiBuildsForDeltas().AndReturn([fsi1_build, fsi2_build])
paygen._DiscoverNmoBuild().AndReturn([nmo_build])
- paygen._DiscoverActiveFsiBuilds().AndReturn([fsi1_build, fsi2_build])
- paygen._DiscoverImages(nmo_build).AndReturn(nmo_images)
- paygen._DiscoverTestImageArchives(nmo_build).AndReturn([nmo_test_image])
paygen._DiscoverImages(fsi1_build).AndReturn(fsi1_images)
paygen._DiscoverTestImageArchives(fsi1_build).AndReturn([fsi1_test_image])
paygen._DiscoverImages(fsi2_build).AndReturn(fsi2_images)
paygen._DiscoverTestImageArchives(fsi2_build).AndReturn([fsi2_test_image])
+ paygen._DiscoverImages(nmo_build).AndReturn(nmo_images)
+ paygen._DiscoverTestImageArchives(nmo_build).AndReturn([nmo_test_image])
# Simplify the output URIs, so it's easy to check them below.
paygen_payload_lib.DefaultPayloadUri(
@@ -573,63 +702,89 @@ fsi_images: 2913.331.0,2465.105.0
# Run the test verification.
self.mox.ReplayAll()
- results = paygen._DiscoverRequiredPayloads()
+ self.maxDiff = None
- expected = [gspaths.Payload(tgt_image=self.basic_image, uri=output_uri),
- gspaths.Payload(tgt_image=self.npo_image, uri=output_uri),
- gspaths.Payload(tgt_image=self.premp_image, uri=output_uri),
- gspaths.Payload(tgt_image=self.premp_npo_image, uri=output_uri),
+ payload_manager = paygen._DiscoverRequiredPayloads()
+
+ expected = [gspaths.Payload(tgt_image=self.basic_image, uri=output_uri,
+ labels=['full']),
+ gspaths.Payload(tgt_image=self.npo_image, uri=output_uri,
+ labels=['full']),
+ gspaths.Payload(tgt_image=self.premp_image, uri=output_uri,
+ labels=['full']),
+ gspaths.Payload(tgt_image=self.premp_npo_image, uri=output_uri,
+ labels=['full']),
+
+ gspaths.Payload(tgt_image=nmo_images[0], uri=output_uri,
+ labels=['full', 'previous']),
+ gspaths.Payload(tgt_image=nmo_images[1], uri=output_uri,
+ labels=['full', 'previous']),
# NPO Deltas
gspaths.Payload(tgt_image=self.npo_image,
src_image=self.basic_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'npo']),
gspaths.Payload(tgt_image=self.premp_npo_image,
src_image=self.premp_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'npo']),
# NMO Delta
gspaths.Payload(tgt_image=self.basic_image,
src_image=nmo_images[0],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'previous']),
gspaths.Payload(tgt_image=self.premp_image,
src_image=nmo_images[1],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'previous']),
# FSI Deltas
gspaths.Payload(tgt_image=self.basic_image,
src_image=fsi1_images[0],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
gspaths.Payload(tgt_image=self.premp_image,
src_image=fsi1_images[1],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
gspaths.Payload(tgt_image=self.basic_image,
src_image=fsi2_images[0],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
gspaths.Payload(tgt_image=self.premp_image,
src_image=fsi2_images[1],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
# Test full payload.
gspaths.Payload(tgt_image=self.test_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['test', 'full']),
+ gspaths.Payload(tgt_image=nmo_test_image,
+ uri=output_uri,
+ labels=['test', 'full', 'previous']),
# Test NPO delta.
gspaths.Payload(tgt_image=self.test_image,
src_image=self.test_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['test', 'delta', 'npo']),
# Test NMO delta.
gspaths.Payload(tgt_image=self.test_image,
src_image=nmo_test_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['test', 'delta', 'previous']),
# Test FSI deltas.
gspaths.Payload(tgt_image=self.test_image,
src_image=fsi1_test_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['test', 'delta', 'fsi']),
gspaths.Payload(tgt_image=self.test_image,
src_image=fsi2_test_image,
- uri=output_uri)]
- expected = zip(expected, itertools.repeat(False))
-
+ uri=output_uri,
+ labels=['test', 'delta', 'fsi'])]
+ results = payload_manager.Get([])
self.assertItemsEqual(sorted(results), sorted(expected))
def testDiscoverRequiredPayloadsPreviousSkipped(self):
@@ -646,7 +801,7 @@ fsi_images: 2913.331.0,2465.105.0
self.mox.StubOutWithMock(paygen, '_DiscoverImages')
self.mox.StubOutWithMock(paygen, '_DiscoverTestImageArchives')
self.mox.StubOutWithMock(paygen, '_DiscoverNmoBuild')
- self.mox.StubOutWithMock(paygen, '_DiscoverActiveFsiBuilds')
+ self.mox.StubOutWithMock(paygen, '_DiscoverFsiBuildsForDeltas')
self.mox.StubOutWithMock(paygen_payload_lib, 'DefaultPayloadUri')
nmo_build = gspaths.Build(bucket='crt',
@@ -670,15 +825,15 @@ fsi_images: 2913.331.0,2465.105.0
paygen._DiscoverImages(paygen._build).AndReturn(self.images)
paygen._DiscoverTestImageArchives(paygen._build).AndReturn(
[self.test_image])
+ paygen._DiscoverFsiBuildsForDeltas().AndReturn([fsi1_build, fsi2_build])
paygen._DiscoverNmoBuild().AndReturn([nmo_build])
- paygen._DiscoverActiveFsiBuilds().AndReturn([fsi1_build, fsi2_build])
- paygen._DiscoverImages(nmo_build).AndRaise(
- paygen_build_lib.ImageMissing('nmo build is missing some image'))
- # _DiscoverTestImageArchives(nmo_build) should NOT be called.
paygen._DiscoverImages(fsi1_build).AndReturn(fsi1_images)
paygen._DiscoverTestImageArchives(fsi1_build).AndReturn([fsi1_test_image])
paygen._DiscoverImages(fsi2_build).AndReturn(fsi2_images)
paygen._DiscoverTestImageArchives(fsi2_build).AndReturn([fsi2_test_image])
+ paygen._DiscoverImages(nmo_build).AndRaise(
+ paygen_build_lib.ImageMissing('nmo build is missing some image'))
+ # _DiscoverTestImageArchives(nmo_build) should NOT be called.
# Simplify the output URIs, so it's easy to check them below.
paygen_payload_lib.DefaultPayloadUri(
@@ -687,53 +842,66 @@ fsi_images: 2913.331.0,2465.105.0
# Run the test verification.
self.mox.ReplayAll()
- results = paygen._DiscoverRequiredPayloads()
+ payload_manager = paygen._DiscoverRequiredPayloads()
# IMPORTANT: we intentionally omit the NMO payload from the expected list
# of payloads as it is a duplicate of one of the FSIs.
- expected = [gspaths.Payload(tgt_image=self.basic_image, uri=output_uri),
- gspaths.Payload(tgt_image=self.npo_image, uri=output_uri),
- gspaths.Payload(tgt_image=self.premp_image, uri=output_uri),
- gspaths.Payload(tgt_image=self.premp_npo_image, uri=output_uri),
+ expected = [gspaths.Payload(tgt_image=self.basic_image, uri=output_uri,
+ labels=['full']),
+ gspaths.Payload(tgt_image=self.npo_image, uri=output_uri,
+ labels=['full']),
+ gspaths.Payload(tgt_image=self.premp_image, uri=output_uri,
+ labels=['full']),
+ gspaths.Payload(tgt_image=self.premp_npo_image, uri=output_uri,
+ labels=['full']),
# NPO Deltas
gspaths.Payload(tgt_image=self.npo_image,
src_image=self.basic_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'npo']),
gspaths.Payload(tgt_image=self.premp_npo_image,
src_image=self.premp_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'npo']),
# FSI Deltas
gspaths.Payload(tgt_image=self.basic_image,
src_image=fsi1_images[0],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
gspaths.Payload(tgt_image=self.premp_image,
src_image=fsi1_images[1],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
gspaths.Payload(tgt_image=self.basic_image,
src_image=fsi2_images[0],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
gspaths.Payload(tgt_image=self.premp_image,
src_image=fsi2_images[1],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
# Test full payload.
gspaths.Payload(tgt_image=self.test_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['test', 'full']),
# Test NPO delta.
gspaths.Payload(tgt_image=self.test_image,
src_image=self.test_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['test', 'delta', 'npo']),
# Test FSI deltas.
gspaths.Payload(tgt_image=self.test_image,
src_image=fsi1_test_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['test', 'delta', 'fsi']),
gspaths.Payload(tgt_image=self.test_image,
src_image=fsi2_test_image,
- uri=output_uri)]
- expected = zip(expected, itertools.repeat(False))
-
+ uri=output_uri,
+ labels=['test', 'delta', 'fsi'])]
+ results = payload_manager.Get([])
self.assertItemsEqual(sorted(results), sorted(expected))
def testDiscoverRequiredPayloadsNmoIsAlsoFsi(self):
@@ -746,7 +914,7 @@ fsi_images: 2913.331.0,2465.105.0
self.mox.StubOutWithMock(paygen, '_DiscoverImages')
self.mox.StubOutWithMock(paygen, '_DiscoverTestImageArchives')
self.mox.StubOutWithMock(paygen, '_DiscoverNmoBuild')
- self.mox.StubOutWithMock(paygen, '_DiscoverActiveFsiBuilds')
+ self.mox.StubOutWithMock(paygen, '_DiscoverFsiBuildsForDeltas')
self.mox.StubOutWithMock(paygen_payload_lib, 'DefaultPayloadUri')
nmo_build = gspaths.Build(bucket='crt',
@@ -770,7 +938,7 @@ fsi_images: 2913.331.0,2465.105.0
paygen._DiscoverImages(paygen._build).AndReturn(self.images)
paygen._DiscoverTestImageArchives(paygen._build).AndReturn(
[self.test_image])
- paygen._DiscoverActiveFsiBuilds().AndReturn([fsi1_build, fsi2_build])
+ paygen._DiscoverFsiBuildsForDeltas().AndReturn([fsi1_build, fsi2_build])
paygen._DiscoverNmoBuild().AndReturn([nmo_build])
paygen._DiscoverImages(fsi1_build).AndReturn(fsi1_images)
paygen._DiscoverImages(fsi2_build).AndReturn(fsi2_images)
@@ -784,52 +952,65 @@ fsi_images: 2913.331.0,2465.105.0
# Run the test verification.
self.mox.ReplayAll()
- results = paygen._DiscoverRequiredPayloads()
+ payload_manager = paygen._DiscoverRequiredPayloads()
- expected = [gspaths.Payload(tgt_image=self.basic_image, uri=output_uri),
- gspaths.Payload(tgt_image=self.npo_image, uri=output_uri),
- gspaths.Payload(tgt_image=self.premp_image, uri=output_uri),
- gspaths.Payload(tgt_image=self.premp_npo_image, uri=output_uri),
+ expected = [gspaths.Payload(tgt_image=self.basic_image, uri=output_uri,
+ labels=['full']),
+ gspaths.Payload(tgt_image=self.npo_image, uri=output_uri,
+ labels=['full']),
+ gspaths.Payload(tgt_image=self.premp_image, uri=output_uri,
+ labels=['full']),
+ gspaths.Payload(tgt_image=self.premp_npo_image, uri=output_uri,
+ labels=['full']),
# NPO Deltas
gspaths.Payload(tgt_image=self.npo_image,
src_image=self.basic_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'npo']),
gspaths.Payload(tgt_image=self.premp_npo_image,
src_image=self.premp_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'npo']),
# FSI Deltas
gspaths.Payload(tgt_image=self.basic_image,
src_image=fsi1_images[0],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
gspaths.Payload(tgt_image=self.premp_image,
src_image=fsi1_images[1],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
gspaths.Payload(tgt_image=self.basic_image,
src_image=fsi2_images[0],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
gspaths.Payload(tgt_image=self.premp_image,
src_image=fsi2_images[1],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
# Test full payload.
gspaths.Payload(tgt_image=self.test_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['test', 'full']),
# Test NPO delta.
gspaths.Payload(tgt_image=self.test_image,
src_image=self.test_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['test', 'delta', 'npo']),
# Test FSI deltas.
gspaths.Payload(tgt_image=self.test_image,
src_image=fsi1_test_image,
- uri=output_uri),
+ uri=output_uri,
+ labels=['test', 'delta', 'fsi']),
gspaths.Payload(tgt_image=self.test_image,
src_image=fsi2_test_image,
- uri=output_uri)]
-
- expected = zip(expected, itertools.repeat(False))
+ uri=output_uri,
+ labels=['test', 'delta', 'fsi'])]
+ results = payload_manager.Get([])
self.assertItemsEqual(sorted(results), sorted(expected))
def testFindFullTestPayloads(self):
@@ -1054,17 +1235,18 @@ fsi_images: 2913.331.0,2465.105.0
finished_uri = paygen._GetFlagURI(gspaths.ChromeosReleases.FINISHED)
lock = self.mox.CreateMockAnything()
- payload = 'foo'
+ payload = gspaths.Payload(tgt_image='foo')
payload_list = [payload]
- payload_skip_list = [(payload, False)]
+ payload_manager = paygen_build_lib.PayloadManager()
+ payload_manager.Add([], payload_list)
+
mock_exception = Exception()
gslock.Lock(lock_uri, dry_run=False).AndReturn(lock)
lock.__enter__().AndReturn(lock)
gslib.Exists(skip_uri).AndReturn(False)
gslib.Exists(finished_uri).AndReturn(False)
- paygen._DiscoverRequiredPayloads(
- ).AndReturn(payload_skip_list)
+ paygen._DiscoverRequiredPayloads().AndReturn(payload_manager)
self.mox.StubOutWithMock(paygen_payload_lib, 'FindExistingPayloads')
paygen_payload_lib.FindExistingPayloads(payload).AndReturn([])
paygen._GeneratePayloads(payload_list, lock).AndRaise(mock_exception)
@@ -1086,22 +1268,23 @@ fsi_images: 2913.331.0,2465.105.0
finished_uri = paygen._GetFlagURI(gspaths.ChromeosReleases.FINISHED)
lock = self.mox.CreateMockAnything()
- payload = 'foo'
+ payload = gspaths.Payload(tgt_image='foo')
payload_list = [payload]
- payload_skip_list = [(payload, False)]
+
+ payload_manager = paygen_build_lib.PayloadManager()
+ payload_manager.Add([], payload_list)
gslock.Lock(lock_uri, dry_run=False).AndReturn(lock)
lock.__enter__().AndReturn(lock)
gslib.Exists(skip_uri).AndReturn(False)
gslib.Exists(finished_uri).AndReturn(False)
- paygen._DiscoverRequiredPayloads(
- ).AndReturn(payload_skip_list)
+ paygen._DiscoverRequiredPayloads().AndReturn(payload_manager)
self.mox.StubOutWithMock(paygen_payload_lib, 'FindExistingPayloads')
paygen_payload_lib.FindExistingPayloads(payload).AndReturn([])
paygen._GeneratePayloads(payload_list, lock)
paygen._MapToArchive('foo-board', '1.2.3').AndReturn(
('archive_board', 'archive_build', 'archive_build_uri'))
- paygen._CreatePayloadTests(['foo']).AndReturn(['Test Payloads'])
+ paygen._CreatePayloadTests(payload_manager).AndReturn(['Test Payloads'])
paygen._AutotestPayloads(['Test Payloads'])
paygen._CleanupBuild()
@@ -1125,16 +1308,17 @@ fsi_images: 2913.331.0,2465.105.0
lock = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(paygen_payload_lib, 'FindExistingPayloads')
self.mox.StubOutWithMock(paygen_payload_lib, 'SetPayloadUri')
- payload_existing = 'foo'
- payload_new = 'bar'
- payload_list = [(payload_existing, False), (payload_new, False)]
+ payload_existing = gspaths.Payload(tgt_image='foo')
+ payload_new = gspaths.Payload(tgt_image='bar')
+
+ payload_manager = paygen_build_lib.PayloadManager()
+ payload_manager.Add([], [payload_existing, payload_new])
gslock.Lock(lock_uri, dry_run=False).AndReturn(lock)
lock.__enter__().AndReturn(lock)
gslib.Exists(skip_uri).AndReturn(False)
gslib.Exists(finished_uri).AndReturn(False)
- paygen._DiscoverRequiredPayloads(
- ).AndReturn(payload_list)
+ paygen._DiscoverRequiredPayloads().AndReturn(payload_manager)
paygen_payload_lib.FindExistingPayloads(payload_existing).AndReturn(
[payload_existing])
paygen_payload_lib.FindExistingPayloads(payload_new).AndReturn([])
@@ -1142,7 +1326,7 @@ fsi_images: 2913.331.0,2465.105.0
paygen._GeneratePayloads([payload_new], lock)
paygen._MapToArchive('foo-board', '1.2.3').AndReturn(
('archive_board', 'archive_build', 'archive_build_uri'))
- paygen._CreatePayloadTests(['foo', 'bar']).AndReturn(['Test Payloads'])
+ paygen._CreatePayloadTests(payload_manager).AndReturn(['Test Payloads'])
paygen._AutotestPayloads(['Test Payloads'])
gslib.CreateWithContents(finished_uri, mox.IgnoreArg())
paygen._CleanupBuild()
@@ -1164,16 +1348,17 @@ fsi_images: 2913.331.0,2465.105.0
finished_uri = paygen._GetFlagURI(gspaths.ChromeosReleases.FINISHED)
lock = self.mox.CreateMockAnything()
- payload = 'foo'
+ payload = gspaths.Payload(tgt_image='foo')
payload_list = [payload]
- payload_skip_list = [(payload, False)]
+
+ payload_manager = paygen_build_lib.PayloadManager()
+ payload_manager.Add([], payload_list)
gslock.Lock(lock_uri, dry_run=False).AndReturn(lock)
lock.__enter__().AndReturn(lock)
gslib.Exists(skip_uri).AndReturn(False)
gslib.Exists(finished_uri).AndReturn(False)
- paygen._DiscoverRequiredPayloads(
- ).AndReturn(payload_skip_list)
+ paygen._DiscoverRequiredPayloads().AndReturn(payload_manager)
self.mox.StubOutWithMock(paygen_payload_lib, 'FindExistingPayloads')
paygen_payload_lib.FindExistingPayloads(payload).AndReturn([])
paygen._GeneratePayloads(payload_list, lock)
@@ -1190,7 +1375,8 @@ fsi_images: 2913.331.0,2465.105.0
def setupCreatePayloadTests(self):
paygen = self._GetPaygenBuildInstance()
- self.mox.StubOutWithMock(paygen, '_DiscoverAllFsiBuilds')
+ self.mox.StubOutWithMock(paygen, '_DiscoverAllFsiBuildsForDeltaTesting')
+ self.mox.StubOutWithMock(paygen, '_DiscoverAllFsiBuildsForFullTesting')
self.mox.StubOutWithMock(paygen, '_FindFullTestPayloads')
return paygen
@@ -1198,12 +1384,18 @@ fsi_images: 2913.331.0,2465.105.0
def testCreatePayloadTestsEmpty(self):
payloads = []
+ payload_manager = paygen_build_lib.PayloadManager()
+ payload_manager.Add([], payloads)
+
paygen = self.setupCreatePayloadTests()
+ paygen._DiscoverAllFsiBuildsForDeltaTesting().AndReturn([])
+ paygen._DiscoverAllFsiBuildsForFullTesting().AndReturn([])
+
# Run the test verification.
self.mox.ReplayAll()
- expected = paygen._CreatePayloadTests(payloads)
+ expected = paygen._CreatePayloadTests(payload_manager)
self.assertEqual(expected, [])
def testCreatePayloadTestsPopulated(self):
@@ -1212,10 +1404,16 @@ fsi_images: 2913.331.0,2465.105.0
gspaths.Payload(tgt_image=self.test_image),
gspaths.Payload(tgt_image=self.prev_image, src_image=self.test_image)
]
+
+ payload_manager = paygen_build_lib.PayloadManager()
+ payload_manager.Add(['test', 'full'], [payloads[0]])
+ payload_manager.Add(['test', 'delta'], [payloads[1]])
+
paygen = self.setupCreatePayloadTests()
# We search for FSIs once for each full payload.
- paygen._DiscoverAllFsiBuilds().AndReturn(['0.9.9', '1.0.0'])
+ paygen._DiscoverAllFsiBuildsForDeltaTesting().AndReturn(['0.9.9', '1.0.0'])
+ paygen._DiscoverAllFsiBuildsForFullTesting().AndReturn(['0.9.9', '1.0.0'])
paygen._FindFullTestPayloads('stable-channel', '0.9.9').AndReturn(False)
paygen._FindFullTestPayloads('stable-channel', '1.0.0').AndReturn(True)
@@ -1224,16 +1422,65 @@ fsi_images: 2913.331.0,2465.105.0
self.maxDiff = None
- expected = paygen._CreatePayloadTests(payloads)
+ labelled_payloads = [
+ gspaths.Payload(tgt_image=self.test_image, labels=['test', 'full']),
+ gspaths.Payload(tgt_image=self.prev_image, src_image=self.test_image,
+ labels=['test', 'delta'])
+ ]
+ expected = paygen._CreatePayloadTests(payload_manager)
self.assertEqual(expected, [
paygen.PayloadTest(
- payloads[0], src_channel='foo-channel', src_version='1.2.3'),
+ labelled_payloads[0], src_channel='foo-channel',
+ src_version='1.2.3'),
paygen.PayloadTest(
- payloads[0], src_channel='stable-channel', src_version='1.0.0'),
+ labelled_payloads[0], src_channel='stable-channel',
+ src_version='1.0.0'),
paygen.PayloadTest(
- payloads[1]),
+ labelled_payloads[1]),
])
+ def testCreatePayloadTestsLabStableFsiDelta(self):
+ payloads = [
+ gspaths.Payload(tgt_image=self.prev_image, src_image=self.test_image)
+ ]
+
+ payload_manager = paygen_build_lib.PayloadManager()
+ payload_manager.Add(['test', 'delta', 'fsi'], payloads)
+
+ paygen = self.setupCreatePayloadTests()
+
+ paygen._DiscoverAllFsiBuildsForDeltaTesting().AndReturn([self.foo_build])
+ paygen._DiscoverAllFsiBuildsForFullTesting().AndReturn([])
+
+ self.mox.ReplayAll()
+
+ labelled_payloads = [
+ gspaths.Payload(tgt_image=self.prev_image, src_image=self.test_image,
+ labels=['test', 'delta', 'fsi'])
+ ]
+ expected = paygen._CreatePayloadTests(payload_manager)
+ self.assertEquals(expected, [paygen.PayloadTest(labelled_payloads[0])])
+
+ def testCreatePayloadTestsLabUnstableFsiDelta(self):
+ payloads = [
+ gspaths.Payload(tgt_image=self.prev_image, src_image=self.test_image)
+ ]
+
+ payload_manager = paygen_build_lib.PayloadManager()
+ payload_manager.Add(['test', 'delta', 'fsi'], payloads)
+
+ paygen = self.setupCreatePayloadTests()
+
+ paygen._DiscoverAllFsiBuildsForDeltaTesting().AndReturn(
+ [gspaths.Build(bucket='crt', channel='not-foo-channel',
+ board='foo-board', version='1.2.3')]
+ )
+ paygen._DiscoverAllFsiBuildsForFullTesting().AndReturn([])
+
+ self.mox.ReplayAll()
+
+ expected = paygen._CreatePayloadTests(payload_manager)
+ self.assertEquals(expected, [])
def testFindControlFileDir(self):
"""Test that we find control files in the proper directory."""
@@ -1250,8 +1497,6 @@ fsi_images: 2913.331.0,2465.105.0
self.assertTrue(result.startswith(
os.path.join(self.tempdir, 'paygen_build-control_files')))
- @unittest.skipIf(not paygen_build_lib.config,
- 'Internal crostools repository needed.')
@unittest.skipIf(not paygen_build_lib.test_control,
'Autotest repository needed.')
def testEmitControlFile(self):
@@ -1308,13 +1553,12 @@ DOC = "Faux doc"
control_contents = osutils.ReadFile(cf)
self.assertEqual(control_contents, '''name = 'paygen_foo'
-image_type = 'test'
update_type = 'delta'
source_release = '1.2.3'
target_release = '1.2.3'
-source_image_uri = 'gs://foo/bar.tar.bz2'
target_payload_uri = 'None'
SUITE = 'paygen_foo'
+source_payload_uri = 'gs://foo/bar.tar.bz2'
source_archive_uri = 'gs://chromeos-releases/foo-channel/foo-board/1.2.3'
AUTHOR = "Chromium OS"
@@ -1526,11 +1770,25 @@ The suite job has another 2:39:39.789250 till timeout.
def testValidateBoardConfig(self):
"""Test ValidateBoardConfig."""
-
- # If we are running on an external builder, we can't see the config.
- # Without the config, we can't validate.
- if not paygen_build_lib.config:
- return
+ mock_return = paygen_build_lib.json.dumps(
+ {'boards':
+ [{u'omaha_config_name': u'autoupdate-ascii-memento.config',
+ u'public_codename': u'x86-mario',
+ u'hwid_match': u'IEC MARIO FISH 2330|IEC MARIO FISH 2330 DEV|'
+ u'IEC MARIO PONY 6101|IEC MARIO PONY DVT 8784|'
+ u'IEC MARIO PONY EVT 3495|IEC MARIO PONY TEST 6101',
+ u'is_active': True,
+ u'app_id': u'{87efface-864d-49a5-9bb3-4b050a7c227a}',
+ u'config_name': u'cr48', u'is_test_blacklist': False,
+ u'omaha_ping_hwid_match': u'IEC MARIO FISH 2330 DEV',
+ u'is_in_canary_release': True}]
+ }
+ )
+
+ self.mox.StubOutWithMock(gslib, 'Cat')
+ gslib.Cat(paygen_build_lib.BOARDS_URI).AndReturn(mock_return)
+ gslib.Cat(paygen_build_lib.BOARDS_URI).AndReturn(mock_return)
+ self.mox.ReplayAll()
# Test a known board works.
paygen_build_lib.ValidateBoardConfig('x86-mario')
@@ -1705,7 +1963,7 @@ class PaygenBuildLibTest_ImageTypes(BasePaygenBuildLibTest):
self.mox.StubOutWithMock(paygen, '_DiscoverImages')
self.mox.StubOutWithMock(paygen, '_DiscoverTestImageArchives')
self.mox.StubOutWithMock(paygen, '_DiscoverNmoBuild')
- self.mox.StubOutWithMock(paygen, '_DiscoverActiveFsiBuilds')
+ self.mox.StubOutWithMock(paygen, '_DiscoverFsiBuildsForDeltas')
self.mox.StubOutWithMock(paygen_payload_lib, 'DefaultPayloadUri')
nmo_build = gspaths.Build(bucket='crt',
@@ -1722,7 +1980,6 @@ class PaygenBuildLibTest_ImageTypes(BasePaygenBuildLibTest):
version='1.1.0')
nmo_images = self._GetBuildImages(nmo_build)
- nmo_test_image = self._GetBuildTestImage(nmo_build)
fsi1_images = self._GetBuildImages(fsi1_build)
fsi1_test_image = self._GetBuildTestImage(fsi1_build)
fsi2_images = self._GetBuildImages(fsi2_build)
@@ -1730,14 +1987,14 @@ class PaygenBuildLibTest_ImageTypes(BasePaygenBuildLibTest):
paygen._DiscoverImages(paygen._build).AndReturn(self.images)
paygen._DiscoverTestImageArchives(paygen._build).AndReturn([])
- paygen._DiscoverNmoBuild().AndReturn([nmo_build])
- paygen._DiscoverActiveFsiBuilds().AndReturn([fsi1_build, fsi2_build])
- paygen._DiscoverImages(nmo_build).AndReturn(nmo_images)
- paygen._DiscoverTestImageArchives(nmo_build).AndReturn([nmo_test_image])
+ paygen._DiscoverFsiBuildsForDeltas().AndReturn([fsi1_build, fsi2_build])
paygen._DiscoverImages(fsi1_build).AndReturn(fsi1_images)
paygen._DiscoverTestImageArchives(fsi1_build).AndReturn([fsi1_test_image])
paygen._DiscoverImages(fsi2_build).AndReturn(fsi2_images)
paygen._DiscoverTestImageArchives(fsi2_build).AndReturn([fsi2_test_image])
+ paygen._DiscoverNmoBuild().AndReturn([nmo_build])
+ paygen._DiscoverImages(nmo_build).AndReturn(nmo_images)
+ paygen._DiscoverTestImageArchives(nmo_build).AndReturn([])
# Simplify the output URIs, so it's easy to check them below.
paygen_payload_lib.DefaultPayloadUri(
@@ -1746,35 +2003,53 @@ class PaygenBuildLibTest_ImageTypes(BasePaygenBuildLibTest):
# Run the test verification.
self.mox.ReplayAll()
- results = paygen._DiscoverRequiredPayloads()
+ self.maxDiff = None
+
+ payload_manager = paygen._DiscoverRequiredPayloads()
expected = [
- gspaths.Payload(tgt_image=self.basic_image, uri=output_uri),
- gspaths.Payload(tgt_image=self.npo_image, uri=output_uri),
- gspaths.Payload(tgt_image=self.premp_image, uri=output_uri),
- gspaths.Payload(tgt_image=self.premp_npo_image, uri=output_uri),
+ gspaths.Payload(tgt_image=self.basic_image, uri=output_uri,
+ labels=['full']),
+ gspaths.Payload(tgt_image=self.npo_image, uri=output_uri,
+ labels=['full']),
+ gspaths.Payload(tgt_image=self.premp_image, uri=output_uri,
+ labels=['full']),
+ gspaths.Payload(tgt_image=self.premp_npo_image, uri=output_uri,
+ labels=['full']),
+
+ gspaths.Payload(tgt_image=nmo_images[0], uri=output_uri,
+ labels=['full', 'previous']),
+ gspaths.Payload(tgt_image=nmo_images[1], uri=output_uri,
+ labels=['full', 'previous']),
# No NPO Deltas because the basic images have different image types.
# NMO deltas.
gspaths.Payload(tgt_image=self.basic_image,
src_image=nmo_images[0],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'previous']),
gspaths.Payload(tgt_image=self.premp_image,
src_image=nmo_images[1],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'previous']),
# FSI Deltas.
gspaths.Payload(tgt_image=self.basic_image,
src_image=fsi1_images[0],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
gspaths.Payload(tgt_image=self.premp_image,
src_image=fsi1_images[1],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
gspaths.Payload(tgt_image=self.basic_image,
src_image=fsi2_images[0],
- uri=output_uri),
+ uri=output_uri,
+ labels=['delta', 'fsi']),
gspaths.Payload(tgt_image=self.premp_image,
src_image=fsi2_images[1],
- uri=output_uri)]
- expected = zip(expected, itertools.repeat(False))
+ uri=output_uri,
+ labels=['delta', 'fsi'])]
+
+ results = payload_manager.Get([])
self.assertItemsEqual(sorted(results), sorted(expected))
diff --git a/lib/paygen/paygen_payload_lib.py b/lib/paygen/paygen_payload_lib.py
index 72731157c..b35ab73b7 100644
--- a/lib/paygen/paygen_payload_lib.py
+++ b/lib/paygen/paygen_payload_lib.py
@@ -306,27 +306,29 @@ class _PaygenPayload(object):
delta_log = self._RunGeneratorCmd(cmd)
self._StoreDeltaLog(delta_log)
- def _GenPayloadHash(self):
- """Generate a hash of payload and metadata.
+ def _GenerateHashes(self):
+ """Generate a payload hash and a metadata hash.
Works from an unsigned update payload.
Returns:
- payload_hash as a string.
+ payload_hash as a string, metadata_hash as a string.
"""
- logging.info('Calculating payload hashes on %s.', self.payload_file)
+ logging.info('Calculating hashes on %s.', self.payload_file)
# How big will the signatures be.
signature_sizes = [str(size) for size in self.PAYLOAD_SIGNATURE_SIZES_BYTES]
- with tempfile.NamedTemporaryFile('rb') as payload_hash_file:
- cmd = ['delta_generator',
- '-in_file=' + self.payload_file,
- '-out_hash_file=' + payload_hash_file.name,
- '-signature_size=' + ':'.join(signature_sizes)]
+ with tempfile.NamedTemporaryFile('rb') as payload_hash_file, \
+ tempfile.NamedTemporaryFile('rb') as metadata_hash_file:
+ cmd = ['brillo_update_payload', 'hash',
+ '--unsigned_payload', self.payload_file,
+ '--payload_hash_file', payload_hash_file.name,
+ '--metadata_hash_file', metadata_hash_file.name,
+ '--signature_size', ':'.join(signature_sizes)]
self._RunGeneratorCmd(cmd)
- return payload_hash_file.read()
+ return payload_hash_file.read(), metadata_hash_file.read()
def _MetadataSize(self, payload_file):
"""Discover the metadata size.
@@ -346,28 +348,6 @@ class _PaygenPayload(object):
payload.Init()
return payload.data_offset
- def _GenMetadataHash(self):
- """Generate a hash of payload and metadata.
-
- Works from an unsigned update payload.
-
- Returns:
- metadata_hash as a string.
- """
- logging.info('Calculating payload hashes on %s.', self.payload_file)
-
- # How big will the signatures be.
- signature_sizes = [str(size) for size in self.PAYLOAD_SIGNATURE_SIZES_BYTES]
-
- with tempfile.NamedTemporaryFile('rb') as metadata_hash_file:
- cmd = ['delta_generator',
- '-in_file=' + self.payload_file,
- '-out_metadata_hash_file=' + metadata_hash_file.name,
- '-signature_size=' + ':'.join(signature_sizes)]
-
- self._RunGeneratorCmd(cmd)
- return metadata_hash_file.read()
-
def _GenerateSignerResultsError(self, format_str, *args):
"""Helper for reporting errors with signer results."""
msg = format_str % args
@@ -538,8 +518,7 @@ class _PaygenPayload(object):
List of payload signatures, List of metadata signatures.
"""
# Create hashes to sign.
- payload_hash = self._GenPayloadHash()
- metadata_hash = self._GenMetadataHash()
+ payload_hash, metadata_hash = self._GenerateHashes()
# Sign them.
# pylint: disable=unpacking-non-sequence
diff --git a/lib/paygen/paygen_payload_lib_unittest.py b/lib/paygen/paygen_payload_lib_unittest.py
index 9526998ed..8c6d18882 100644
--- a/lib/paygen/paygen_payload_lib_unittest.py
+++ b/lib/paygen/paygen_payload_lib_unittest.py
@@ -125,7 +125,7 @@ class PaygenPayloadLibBasicTest(PaygenPayloadLibTest):
if not au_generator_uri_override:
au_generator_uri_override = gspaths.ChromeosReleases.GeneratorUri(
- payload.tgt_image.channel, payload.tgt_image.board, '6351.0.0')
+ payload.tgt_image.channel, payload.tgt_image.board, '7587.0.0')
return paygen_payload_lib._PaygenPayload(
payload=payload,
@@ -403,8 +403,8 @@ class PaygenPayloadLibBasicTest(PaygenPayloadLibTest):
self.mox.ReplayAll()
gen._GenerateUnsignedPayload()
- def testGenPayloadHashes(self):
- """Test _GenPayloadHash via mox."""
+ def testGenerateHashes(self):
+ """Test _GenerateHashes via mox."""
gen = self._GetStdGenerator()
# Stub out the required functions.
@@ -412,34 +412,16 @@ class PaygenPayloadLibBasicTest(PaygenPayloadLibTest):
'_RunGeneratorCmd')
# Record the expected function calls.
- cmd = ['delta_generator',
- '-in_file=' + gen.payload_file,
- mox.IsA(str),
- '-signature_size=256']
+ cmd = ['brillo_update_payload', 'hash',
+ '--unsigned_payload', gen.payload_file,
+ '--payload_hash_file', mox.IsA(str),
+ '--metadata_hash_file', mox.IsA(str),
+ '--signature_size', '256']
gen._RunGeneratorCmd(cmd)
# Run the test.
self.mox.ReplayAll()
- self.assertEqual(gen._GenPayloadHash(), '')
-
- def testGenMetadataHashes(self):
- """Test _GenPayloadHash via mox."""
- gen = self._GetStdGenerator()
-
- # Stub out the required functions.
- self.mox.StubOutWithMock(paygen_payload_lib._PaygenPayload,
- '_RunGeneratorCmd')
-
- # Record the expected function calls.
- cmd = ['delta_generator',
- '-in_file=' + gen.payload_file,
- mox.IsA(str),
- '-signature_size=256']
- gen._RunGeneratorCmd(cmd)
-
- # Run the test.
- self.mox.ReplayAll()
- self.assertEqual(gen._GenMetadataHash(), '')
+ self.assertEqual(gen._GenerateHashes(), ('', ''))
def testSignHashes(self):
"""Test _SignHashes via mox."""
@@ -562,9 +544,7 @@ class PaygenPayloadLibBasicTest(PaygenPayloadLibTest):
# Set up stubs.
self.mox.StubOutWithMock(paygen_payload_lib._PaygenPayload,
- '_GenPayloadHash')
- self.mox.StubOutWithMock(paygen_payload_lib._PaygenPayload,
- '_GenMetadataHash')
+ '_GenerateHashes')
self.mox.StubOutWithMock(paygen_payload_lib._PaygenPayload,
'_SignHashes')
self.mox.StubOutWithMock(paygen_payload_lib._PaygenPayload,
@@ -573,8 +553,7 @@ class PaygenPayloadLibBasicTest(PaygenPayloadLibTest):
'_StoreMetadataSignatures')
# Record expected calls.
- gen._GenPayloadHash().AndReturn(payload_hash)
- gen._GenMetadataHash().AndReturn(metadata_hash)
+ gen._GenerateHashes().AndReturn((payload_hash, metadata_hash))
gen._SignHashes([payload_hash, metadata_hash]).AndReturn(
(payload_sigs, metadata_sigs))
gen._InsertPayloadSignatures(payload_sigs)
@@ -759,7 +738,7 @@ class PaygenPayloadLibEndToEndTest(PaygenPayloadLibTest):
cache=self.cache,
work_dir=self.tempdir,
au_generator_uri=gspaths.ChromeosReleases.GeneratorUri(
- payload.tgt_image.channel, payload.tgt_image.board, '6351.0.0'),
+ payload.tgt_image.channel, payload.tgt_image.board, '7587.0.0'),
sign=sign)
self.assertTrue(os.path.exists(output_uri))
diff --git a/lib/portage_util.py b/lib/portage_util.py
index 41db7642b..c0fe77fd7 100644
--- a/lib/portage_util.py
+++ b/lib/portage_util.py
@@ -857,36 +857,6 @@ class EBuild(object):
return ebuild_projects
- @classmethod
- def UpdateCommitHashesForChanges(cls, changes, buildroot, manifest):
- """Updates the commit hashes for the EBuilds uprevved in changes.
-
- Args:
- changes: Changes from Gerrit that are being pushed.
- buildroot: Path to root of build directory.
- manifest: git.ManifestCheckout object.
- """
- path_sha1s = {}
- overlay_list = FindOverlays(constants.BOTH_OVERLAYS, buildroot=buildroot)
- ebuild_paths = cls._GetEBuildPaths(buildroot, manifest, overlay_list,
- changes)
- for ebuild, paths in ebuild_paths.iteritems():
- # Calculate any SHA1s that are not already in path_sha1s.
- for path in set(paths).difference(path_sha1s):
- path_sha1s[path] = cls._GetSHA1ForPath(manifest, path)
-
- sha1s = [path_sha1s[path] for path in paths]
- logging.info('Updating ebuild for package %s with commit hashes %r',
- ebuild.package, sha1s)
- updates = dict(CROS_WORKON_COMMIT=cls.FormatBashArray(sha1s))
- EBuild.UpdateEBuild(ebuild.ebuild_path, updates)
-
- # Commit any changes to all overlays.
- for overlay in overlay_list:
- if EBuild.GitRepoHasChanges(overlay):
- EBuild.CommitChange('Updating commit hashes in ebuilds '
- 'to match remote repository.', overlay=overlay)
-
class PortageDBException(Exception):
"""Generic PortageDB error."""
diff --git a/lib/portage_util_unittest.py b/lib/portage_util_unittest.py
index 216c7f734..31833032b 100644
--- a/lib/portage_util_unittest.py
+++ b/lib/portage_util_unittest.py
@@ -7,7 +7,6 @@
from __future__ import print_function
import cStringIO
-import mock
import os
from chromite.cbuildbot import constants
@@ -454,33 +453,6 @@ class EBuildRevWorkonTest(cros_test_lib.MockTempDirTestCase):
self.m_ebuild.CommitChange(mock_message, '.')
m.assert_called_once_with('.', ['commit', '-a', '-m', 'Commitme'])
- def testUpdateCommitHashesForChanges(self):
- """Tests that we can update the commit hashes for changes correctly."""
- build_root = 'fakebuildroot'
- overlays = ['public_overlay']
- changes = ['fake change']
- paths = ['fake_path1', 'fake_path2']
- sha1s = ['sha1', 'shaaaaaaaaaaaaaaaa2']
- path_ebuilds = {self.m_ebuild: paths}
-
- self.PatchObject(portage_util, 'FindOverlays', return_value=overlays)
- self.PatchObject(portage_util.EBuild, '_GetEBuildPaths',
- return_value=path_ebuilds)
- self.PatchObject(portage_util.EBuild, '_GetSHA1ForPath',
- side_effect=reversed(sha1s))
- update_mock = self.PatchObject(portage_util.EBuild, 'UpdateEBuild')
- self.PatchObject(portage_util.EBuild, 'GitRepoHasChanges',
- return_value=True)
- commit_mock = self.PatchObject(portage_util.EBuild, 'CommitChange')
-
- portage_util.EBuild.UpdateCommitHashesForChanges(changes, build_root,
- MANIFEST)
-
- update_mock.assert_called_once_with(
- self.m_ebuild.ebuild_path,
- {'CROS_WORKON_COMMIT': '(%s)' % ' '.join('"%s"' % x for x in sha1s)})
- commit_mock.assert_called_once_with(mock.ANY, overlay=overlays[0])
-
def testGitRepoHasChanges(self):
"""Tests that GitRepoHasChanges works correctly."""
git.RunGit(self.tempdir,
diff --git a/lib/project_sdk.py b/lib/project_sdk.py
deleted file mode 100644
index e547a5fa6..000000000
--- a/lib/project_sdk.py
+++ /dev/null
@@ -1,160 +0,0 @@
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Common utilities for working with Project SDK."""
-
-from __future__ import print_function
-
-import os
-import re
-import stat
-
-from chromite.cbuildbot import constants
-from chromite.lib import cros_build_lib
-from chromite.lib import cros_logging as logging
-from chromite.lib import osutils
-from chromite.lib import workspace_lib
-
-
-def FindRepoRoot(sdk_dir=None):
- """Locate the SDK root directly by looking for .repo dir.
-
- This is very similar to constants.SOURCE_ROOT, except that it can operate
- against repo checkouts outside our current code base.
-
- CAUTION! Using SDKs from directories other than the default is likely to
- break assumptions that our tools are built upon. As a rule of thumb, do not
- expose this argument externally unless you know what you're doing.
-
- Args:
- sdk_dir: Path of the SDK, or any dir inside it. None defaults to
- constants.SOURCE_ROOT.
-
- Returns:
- Root dir of SDK, or None.
- """
- if sdk_dir is None:
- return constants.SOURCE_ROOT
-
- # Make sure we're looking at an actual directory.
- if not os.path.isdir(sdk_dir):
- return None
-
- # Find the .repo directory and return the path leading up to it, if found.
- repo_dir = osutils.FindInPathParents('.repo', os.path.abspath(sdk_dir),
- test_func=os.path.isdir)
- return os.path.dirname(repo_dir) if repo_dir else None
-
-
-def VersionFile(sdk_dir):
- return os.path.join(sdk_dir, 'SDK_VERSION')
-
-
-def FindVersion(sdk_dir=None):
- """Find the version of a given SDK.
-
- If the SDK was fetched by any means other than "brillo sdk" then it will
- always appear to be 'non-official', even if an official manifest was used.
-
- Args:
- sdk_dir: path to the SDK, or any of its sub directories.
-
- Returns:
- The version of your SDK as a string. '6500.0.0'
- None if the directory doesn't appear to be an SDK.
- """
- sdk_root = FindRepoRoot(sdk_dir)
- if sdk_root is None:
- return None
-
- v_file = VersionFile(sdk_root)
- return osutils.ReadFile(v_file) if os.path.exists(v_file) else None
-
-
-def _GetExecutableVersion(cmd, version_arg='--version'):
- """Gets an executable version string using |version_flag|.
-
- Args:
- cmd: Executable to check (for example, '/bin/bash').
- version_arg: Argument to get |cmd| to print its version.
-
- Returns:
- Output string or None if the program doesn't exist or gave a
- non-zero exit code.
- """
- try:
- return cros_build_lib.RunCommand(
- [cmd, version_arg], print_cmd=False, capture_output=True).output
- except cros_build_lib.RunCommandError:
- return None
-
-
-def VerifyEnvironment(workspace_path=None):
- """Verify the environment we are installed to.
-
- Disk permissions are only verified if a workspace path is provided.
-
- Args:
- workspace_path: Root directory of the workspace or None.
-
- Returns:
- boolean: True if the environment looks friendly.
- """
- result = True
-
- # Verify Python:
- # We assume the python environment is acceptable, because we got here.
- # However, we can add imports here to check for any required external
- # packages.
-
- # Verify executables that just need to exist.
- for cmd in ('/bin/bash', 'curl'):
- if _GetExecutableVersion(cmd) is None:
- logging.error('%s is required to use the SDK.', cmd)
- result = False
-
- # Verify Git version.
- git_requirement_message = 'git 1.8 or greater is required to use the SDK.'
- git_version = _GetExecutableVersion('git')
- if git_version is None:
- logging.error(git_requirement_message)
- result = False
-
- # Example version string: 'git version 2.2.0.rc0.207.ga3a616c'.
- m = re.match(r'git version (\d+)\.(\d+)', git_version)
- if not m:
- logging.error(git_requirement_message)
- logging.error("git version not recognized from: '%s'.", git_version)
- result = False
- else:
- gv_int_list = [int(d) for d in m.groups()] # Something like [2, 3]
- if gv_int_list < [1, 8]:
- logging.error(git_requirement_message)
- logging.error("Current version: '%s'.", git_version)
- result = False
-
- # If a workspace path is provided, validate chroot requirements.
- if workspace_path:
- chroot_dir = workspace_lib.ChrootPath(workspace_path)
-
- # Create a file with the suid bit set.
- suid_file = os.path.join(chroot_dir, 'suid_test')
- try:
- # Create a file with the SUID set for the owner.
- osutils.Touch(suid_file, makedirs=True, mode=stat.S_ISUID)
-
- # See if the SUID bit will be respected, or ignored.
- st = os.statvfs(suid_file)
-
- # The os.ST_NOSUID constant wasn't added until python-3.2.
- if st.f_flag & 0x2:
- logging.error(
- 'Your current chroot directory (%s) does not support the SUID bit,'
- ' which is required. You can move the chroot to a new location'
- ' using "brillo chroot --move <new_dir>"', chroot_dir)
- result = False
- finally:
- osutils.SafeUnlink(suid_file)
-
- return result
diff --git a/lib/project_sdk_unittest b/lib/project_sdk_unittest
deleted file mode 120000
index 72196ceea..000000000
--- a/lib/project_sdk_unittest
+++ /dev/null
@@ -1 +0,0 @@
-../scripts/wrapper.py \ No newline at end of file
diff --git a/lib/project_sdk_unittest.py b/lib/project_sdk_unittest.py
deleted file mode 100644
index 7859c02f6..000000000
--- a/lib/project_sdk_unittest.py
+++ /dev/null
@@ -1,168 +0,0 @@
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Tests for the project_sdk library."""
-
-from __future__ import print_function
-
-import os
-
-from chromite.cbuildbot import constants
-from chromite.lib import cros_build_lib_unittest
-from chromite.lib import cros_test_lib
-from chromite.lib import osutils
-from chromite.lib import project_sdk
-
-# pylint: disable=protected-access
-
-class ProjectSdkTest(cros_test_lib.TempDirTestCase):
- """Unittest for project_sdk.py"""
-
- def setUp(self):
- self.version = '1.2.3'
-
- # Define assorted paths to test against.
- self.bogus_dir = os.path.join(self.tempdir, 'bogus')
- self.repo_dir = os.path.join(self.tempdir, 'repo')
- self.nested_dir = os.path.join(self.repo_dir, 'foo', 'bar')
-
- # Create fake repo with sub-dirs.
- osutils.SafeMakedirs(os.path.join(self.repo_dir, '.repo'))
- osutils.SafeMakedirs(self.nested_dir)
-
- version_file = project_sdk.VersionFile(self.repo_dir)
- osutils.WriteFile(version_file, self.version)
-
- def testFindRepoRootCurrentRepo(self):
- """Test FindRepoRoot with default of CWD."""
- self.assertEqual(constants.SOURCE_ROOT, project_sdk.FindRepoRoot())
-
- def testFindRepoRootSpecifiedBogus(self):
- """Test FindRepoRoot with non-existent directory outside the repo."""
- self.assertIsNone(project_sdk.FindRepoRoot(self.bogus_dir))
-
- def testFindRepoRootSpecifiedRoot(self):
- """Test FindRepoRoot with top level of repo tree."""
- self.assertEqual(self.repo_dir, project_sdk.FindRepoRoot(self.repo_dir))
-
- def testFindRepoRootSpecifiedNested(self):
- """Test FindRepoRoot with nested inside repo tree."""
- self.assertEqual(self.repo_dir, project_sdk.FindRepoRoot(self.nested_dir))
-
- def testFindRepoRootSpecifiedNonexistent(self):
- """Test FindRepoRoot refuses to scan a nonexistent path."""
- self.assertIsNone(
- project_sdk.FindRepoRoot(os.path.join(self.nested_dir, 'not_there')))
-
- def testVersionFile(self):
- self.assertEqual('/foo/SDK_VERSION', project_sdk.VersionFile('/foo'))
-
- def testFindVersionDefault(self):
- """Test FindVersion with default of CWD."""
- # Expected results are undefined, just ensure we don't crash.
- project_sdk.FindVersion()
-
- def testFindVersionBogus(self):
- """Test FindVersion with non-existent directory outside the repo."""
- self.assertIsNone(project_sdk.FindVersion(self.bogus_dir))
-
- def testFindVersionSpecifiedRoot(self):
- """Test FindVersion with top level of repo tree."""
- self.assertEqual(self.version, project_sdk.FindVersion(self.repo_dir))
-
- def testFindVersionSpecifiedNested(self):
- """Test FindVersion with nested inside repo tree."""
- self.assertEqual(self.version, project_sdk.FindVersion(self.nested_dir))
-
-
-class ProjectSdkVerifyFake(cros_test_lib.MockTempDirTestCase):
- """Test VerifyEnvironment with mocks."""
-
- def setUp(self):
- self.rc_mock = self.StartPatcher(cros_build_lib_unittest.RunCommandMock())
-
- def MockEnvironment(self, bash=True, git='2.2.0', curl=True, suid=True):
- """Mocks an environment for the specified checks.
-
- When adding a new environment check, add it to this function so that
- each test can isolate just the check it wants to test.
-
- Args:
- bash: True to mock a valid bash environment.
- git: If set, git version to mock.
- curl: True to mock a valid curl environment.
- suid: True to mock a valid suid environment.
- """
- if bash:
- self.rc_mock.AddCmdResult(['/bin/bash', '--version'])
- if git:
- self.rc_mock.AddCmdResult(['git', '--version'],
- output='git version %s' % git)
- if curl:
- self.rc_mock.AddCmdResult(['curl', '--version'])
- if suid:
- self.PatchObject(os, 'statvfs', autospec=True).return_value.f_flag = 0x0
-
- def testMockEnvironment(self):
- """Test that MockEnvironment() by itself sets up a valid env."""
- self.MockEnvironment()
-
- self.assertTrue(project_sdk.VerifyEnvironment(self.tempdir))
-
- def testGTrusty(self):
- """Test with mock of 'gTrusty' distribution."""
- self.MockEnvironment(git='2.2.0.rc0.207.ga3a616c')
-
- self.assertTrue(project_sdk.VerifyEnvironment(self.tempdir))
-
- def testUbuntu14(self):
- """Test with mock of 'Ubuntu LTS 14' distribution."""
- self.MockEnvironment(git='2.1.0')
-
- self.assertTrue(project_sdk.VerifyEnvironment(self.tempdir))
-
- def testGitNewEnough(self):
- """Test with mock of git 1.8."""
- self.MockEnvironment(git='1.8.3.1')
-
- self.assertTrue(project_sdk.VerifyEnvironment(self.tempdir))
-
- def testFailNoBash(self):
- """Test with mock of no bash present."""
- self.MockEnvironment(bash=False)
- self.rc_mock.AddCmdResult(['/bin/bash', '--version'], returncode=127)
-
- self.assertFalse(project_sdk.VerifyEnvironment(self.tempdir))
-
- def testFailGitTooOld(self):
- """Test with mock of git too old to use."""
- self.MockEnvironment(git='1.7.10.4')
-
- self.assertFalse(project_sdk.VerifyEnvironment(self.tempdir))
-
- def testFailNoCurl(self):
- """Test with mock of no curl present."""
- self.MockEnvironment(curl=False)
- self.rc_mock.AddCmdResult(['curl', '--version'], returncode=127)
-
- self.assertFalse(project_sdk.VerifyEnvironment(self.tempdir))
-
- def testFailSuid(self):
- """Test with SUID is disabled."""
- self.MockEnvironment(suid=False)
- # The os.ST_NOSUID constant wasn't added until python-3.2.
- self.PatchObject(os, 'statvfs', autospec=True).return_value.f_flag = 0x2
-
- self.assertFalse(project_sdk.VerifyEnvironment(self.tempdir))
-
-class ProjectSdkVerifyReal(cros_test_lib.TempDirTestCase):
- """Test VerifyEnvironment for real."""
-
- def testVerifyEnvironment(self):
- """Test, assuming production environment is valid."""
- self.assertTrue(project_sdk.VerifyEnvironment(self.tempdir))
-
- def testGetExecutableVersionNonExistent(self):
- """Tests _GetExecutableVersion() when the executable doesn't exist."""
- self.assertIsNone(project_sdk._GetExecutableVersion('/not/a/real/program'))
diff --git a/lib/sysroot_lib.py b/lib/sysroot_lib.py
index b04d4bf24..74fe4e94b 100644
--- a/lib/sysroot_lib.py
+++ b/lib/sysroot_lib.py
@@ -316,31 +316,6 @@ class Sysroot(object):
return self._GenerateConfig(toolchains, board_overlays, portdir_overlays,
header, BOARD_USE=board)
- def GenerateBrickConfig(self, bricks, bsp=None):
- """Generates the configuration for a given brick stack and bsp.
-
- Args:
- bricks: The brick stack, expanded, excluding the bsp.
- bsp: BSP to use.
- """
- brick_list = bricks
- if bsp:
- brick_list = bsp.BrickStack() + brick_list
-
- board_overlays = [b.OverlayDir() for b in brick_list]
- portdir_overlays = [_CHROMIUMOS_OVERLAY, _ECLASS_OVERLAY] + board_overlays
-
- # If the bsp is not set use the highest priority brick. This is meant to
- # preserve support for building with --brick.
- # TODO(bsimonnet): remove this once we remove support for --brick
- # (brbug.com/916).
- bsp = bsp or bricks[-1]
- toolchains = toolchain.GetToolchainsForBrick(bsp.brick_locator)
-
- header = '# Autogenerated by chromite.lib.sysroot_lib.'
- return self._GenerateConfig(toolchains, board_overlays, portdir_overlays,
- header)
-
def WriteConfig(self, config):
"""Writes the configuration.
diff --git a/lib/toolchain.py b/lib/toolchain.py
index cedc9a0f4..d20ed4197 100644
--- a/lib/toolchain.py
+++ b/lib/toolchain.py
@@ -9,7 +9,6 @@ from __future__ import print_function
import cStringIO
from chromite.cbuildbot import constants
-from chromite.lib import brick_lib
from chromite.lib import cros_build_lib
from chromite.lib import gs
from chromite.lib import portage_util
@@ -65,20 +64,6 @@ def GetToolchainsForBoard(board, buildroot=constants.SOURCE_ROOT):
return targets
-def GetToolchainsForBrick(brick_locator):
- """Get a dictionary mapping toolchain targets to their options for a brick.
-
- Args:
- brick_locator: locator for the brick.
-
- Returns:
- The list of toolchain tuples for the given brick.
- """
- toolchains = toolchain_list.ToolchainList(
- brick=brick_lib.Brick(brick_locator))
- return toolchains.GetMergedToolchainSettings()
-
-
def FilterToolchains(targets, key, value):
"""Filter out targets based on their attributes.
diff --git a/lib/toolchain_list.py b/lib/toolchain_list.py
index 432436be4..261e36e9f 100644
--- a/lib/toolchain_list.py
+++ b/lib/toolchain_list.py
@@ -22,7 +22,7 @@ _DEFAULT_TOOLCHAIN_KEY = 'default'
class NoDefaultToolchainDefinedError(Exception):
- """Brillo brick stacks are required to define a default toolchain."""
+ """Overlays are required to define a default toolchain."""
class MismatchedToolchainConfigsError(Exception):
@@ -32,28 +32,20 @@ class MismatchedToolchainConfigsError(Exception):
class ToolchainList(object):
"""Represents a list of toolchains."""
- def __init__(self, brick=None, overlays=None):
+ def __init__(self, overlays):
"""Construct an instance.
Args:
- brick: brick_lib.Brick object. We'll add the toolchains used by the brick
- and its dependencies to |self|.
overlays: list of overlay directories to add toolchains from.
"""
- if brick is None and overlays is None:
- raise ValueError('Must specify either brick or overlays.')
- if brick is not None and overlays is not None:
- raise ValueError('Must specify one of brick or overlays.')
+ if overlays is None:
+ raise ValueError('Must specify overlays.')
self._toolchains = []
self._require_explicit_default_toolchain = True
- if brick:
- for each_brick in brick.BrickStack():
- self._AddToolchainsFromBrick(each_brick)
- else:
- self._require_explicit_default_toolchain = False
- for overlay_path in overlays:
- self._AddToolchainsFromOverlayDir(overlay_path)
+ self._require_explicit_default_toolchain = False
+ for overlay_path in overlays:
+ self._AddToolchainsFromOverlayDir(overlay_path)
def _AddToolchainsFromOverlayDir(self, overlay_dir):
"""Add toolchains to |self| from the given overlay.
@@ -79,15 +71,6 @@ class ToolchainList(object):
settings = json.loads(line_pieces[1]) if len(line_pieces) > 1 else {}
self._AddToolchain(target, setting_overrides=settings)
- def _AddToolchainsFromBrick(self, brick):
- """Add toolchains to |self| defined by the given brick.
-
- Args:
- brick: brick_lib.Brick object.
- """
- for target, settings in brick.config.get('toolchains', {}):
- self._AddToolchain(target, setting_overrides=settings)
-
def _AddToolchain(self, target, setting_overrides=None):
"""Add a toolchain to |self|.
diff --git a/lib/toolchain_unittest.py b/lib/toolchain_unittest.py
index 1e4421443..d11c98396 100644
--- a/lib/toolchain_unittest.py
+++ b/lib/toolchain_unittest.py
@@ -9,13 +9,10 @@ from __future__ import print_function
import mock
import os
-from chromite.lib import brick_lib
from chromite.lib import cros_build_lib_unittest
from chromite.lib import cros_test_lib
from chromite.lib import osutils
from chromite.lib import toolchain
-from chromite.lib import toolchain_list
-from chromite.lib import workspace_lib
BASE_TOOLCHAIN_CONF = """# The root of all evil is money, err, this config.
@@ -32,24 +29,6 @@ extra-toolchain # Unlikely to win any performance tests.
bonus-toolchain {"stable": true}
"""
-MODERN_BSP_BRICK_CONFIG = {
- 'name': 'bsp-brick',
- 'toolchains': [('base-target-name', {'default': True}),
- ('bonus-toolchain', {'a setting': 'bonus value'})
- ],
- 'dependencies': ['//custom-firmware-brick'],
-}
-
-MODERN_FIRMWARE_BRICK_CONFIG = {
- 'name': 'custom-firmware-brick',
- 'toolchains': [('bonus-toolchain', {'stable': True}),
- ('extra-toolchain', {})],
-}
-
-TYPICAL_BRICK_WITHOUT_TOOLCHAINS = {
- 'name': 'custom-firmware-brick',
-}
-
EXPECTED_TOOLCHAINS = {
'bonus-toolchain': {
'sdk': True,
@@ -66,13 +45,6 @@ EXPECTED_TOOLCHAINS = {
class ToolchainTest(cros_test_lib.MockTempDirTestCase):
"""Tests for lib.toolchain."""
- def _MakeBrick(self, config):
- return brick_lib.Brick(os.path.join(self.tempdir, config['name']),
- initial_config=config)
-
- def setUp(self):
- self.PatchObject(workspace_lib, 'WorkspacePath', return_value=self.tempdir)
-
def testArchForToolchain(self):
"""Tests that we correctly parse crossdev's output."""
rc_mock = cros_build_lib_unittest.RunCommandMock()
@@ -104,46 +76,3 @@ target=foo
find_overlays_mock.return_value = overlays
actual_targets = toolchain.GetToolchainsForBoard('board_value')
self.assertEqual(EXPECTED_TOOLCHAINS, actual_targets)
-
- def testReadsBrickToolchains(self):
- """Tests that we can read the toolchain for a brick stack."""
- # Creates the brick in a subdirectory of tempdir so that we can create other
- # bricks without interfering with it.
- self._MakeBrick(MODERN_FIRMWARE_BRICK_CONFIG)
- top_brick = self._MakeBrick(MODERN_BSP_BRICK_CONFIG)
- self.assertEqual(EXPECTED_TOOLCHAINS,
- toolchain.GetToolchainsForBrick(top_brick.brick_locator))
-
- def testShouldDetectMissingDefaultsInBricks(self):
- """Tests that we check for a default toolchain in bricks."""
- brick = self._MakeBrick(
- {'name': 'brick-name', 'toolchains': [('base-toolchain', {})]})
- self.assertRaises(toolchain_list.NoDefaultToolchainDefinedError,
- toolchain.GetToolchainsForBrick,
- brick.brick_locator)
-
- def testShouldDetectConflictingOverrides(self):
- """Tests that we disallow toolchains with obvious conflicting settings."""
- conflicting_brick = self._MakeBrick(
- {'name': 'conflicting-brick',
- 'toolchains': [
- ('base-toolchain', {'default': True,
- 'setting': 'conflicting value'}),
- ],
- })
- brick = self._MakeBrick(
- {'name': 'bsp-brick',
- 'toolchains': [
- ('base-toolchain', {'default': True,
- 'setting': 'bsp value'}),
- ],
- 'dependencies': [conflicting_brick.brick_locator],
- })
- self.assertRaises(toolchain_list.MismatchedToolchainConfigsError,
- toolchain.GetToolchainsForBrick,
- brick.brick_locator)
-
- def testToleratesBricksWithoutToolchains(self):
- """Tests that we correctly handle bricks that are toolchain agnostic."""
- simple_brick = self._MakeBrick(TYPICAL_BRICK_WITHOUT_TOOLCHAINS)
- toolchain.GetToolchainsForBrick(simple_brick.brick_locator)
diff --git a/lib/workspace_lib.py b/lib/workspace_lib.py
deleted file mode 100644
index bdcd98cf6..000000000
--- a/lib/workspace_lib.py
+++ /dev/null
@@ -1,329 +0,0 @@
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Utilities for discovering the directories associated with workspaces.
-
-Workspaces have a variety of important concepts:
-
-* The bootstrap repository. BOOTSTRAP/chromite/bootstrap is expected to be in
-the user's path. Most commands run from here redirect to the active SDK.
-
-* The workspace directory. This directory (identified by presence of
-WORKSPACE_CONFIG), contains code, and is associated with exactly one SDK
-instance. It is normally discovered based on CWD.
-
-* The SDK root. This directory contains a specific SDK version, and is stored in
-BOOTSTRAP/sdk_checkouts/<version>.
-
-This library contains helper methods for finding all of the relevant directories
-here.
-"""
-
-from __future__ import print_function
-
-import json
-import os
-
-from chromite.cbuildbot import constants
-from chromite.lib import cros_build_lib
-from chromite.lib import osutils
-
-MAIN_CHROOT_DIR_IN_VM = '/chroots'
-
-# The presence of this file signifies the root of a workspace.
-WORKSPACE_CONFIG = 'workspace-config.json'
-WORKSPACE_LOCAL_CONFIG = '.local.json'
-WORKSPACE_CHROOT_DIR = '.chroot'
-WORKSPACE_IMAGES_DIR = 'build/images'
-WORKSPACE_LOGS_DIR = 'build/logs'
-
-# Prefixes used by locators.
-_BOARD_LOCATOR_PREFIX = 'board:'
-_WORKSPACE_LOCATOR_PREFIX = '//'
-
-
-class LocatorNotResolved(Exception):
- """Given locator could not be resolved."""
-
-
-class ConfigFileError(Exception):
- """Configuration file writing or reading failed."""
-
-
-def WorkspacePath(workspace_reference_dir=None):
- """Returns the path to the current workspace.
-
- This method works both inside and outside the chroot, though results will
- be different.
-
- Args:
- workspace_reference_dir: Any directory inside the workspace. If None,
- will use CWD (outside chroot), or bind mount location (inside chroot).
- You should normally use the default.
-
- Returns:
- Path to root directory of the workspace (if valid), or None.
- """
- if workspace_reference_dir is None:
- if cros_build_lib.IsInsideChroot():
- workspace_reference_dir = constants.CHROOT_WORKSPACE_ROOT
- else:
- workspace_reference_dir = os.getcwd()
-
- workspace_config = osutils.FindInPathParents(
- WORKSPACE_CONFIG,
- os.path.abspath(workspace_reference_dir))
-
- return os.path.dirname(workspace_config) if workspace_config else None
-
-
-def ChrootPath(workspace_path):
- """Returns the path to the chroot associated with the given workspace.
-
- Each workspace has its own associated chroot. This method returns the chroot
- path set in the workspace config if present, or else the default location,
- which varies depending on whether or not we run in a VM.
-
- Args:
- workspace_path: Root directory of the workspace (WorkspacePath()).
-
- Returns:
- Path to where the chroot is, or where it should be created.
- """
- config_value = GetChrootDir(workspace_path)
- if config_value:
- # If the config value is a relative path, we base it in the workspace path.
- # Otherwise, it is an absolute path and will be returned as is.
- return os.path.join(workspace_path, config_value)
-
- # The default for a VM.
- if osutils.IsInsideVm():
- return os.path.join(MAIN_CHROOT_DIR_IN_VM, os.path.basename(workspace_path))
-
- # The default for all other cases.
- return os.path.join(workspace_path, WORKSPACE_CHROOT_DIR)
-
-
-def SetChrootDir(workspace_path, chroot_dir):
- """Set which chroot directory a workspace uses.
-
- This value will overwrite the default value, if set. This is normally only
- used if the user overwrites the default value. This method is NOT atomic.
-
- Args:
- workspace_path: Root directory of the workspace (WorkspacePath()).
- chroot_dir: Directory in which this workspaces chroot should be created.
- """
- # Read the config, update its chroot_dir, and write it.
- config = _ReadLocalConfig(workspace_path)
- config['chroot_dir'] = chroot_dir
- _WriteLocalConfig(workspace_path, config)
-
-
-def GetChrootDir(workspace_path):
- """Get override of chroot directory for a workspace.
-
- You should normally call ChrootPath so that the default value will be
- found if no explicit value has been set.
-
- Args:
- workspace_path: Root directory of the workspace (WorkspacePath()).
-
- Returns:
- version string or None.
- """
- # Config should always return a dictionary.
- config = _ReadLocalConfig(workspace_path)
-
- # If version is present, use it, else return None.
- return config.get('chroot_dir')
-
-
-def GetActiveSdkVersion(workspace_path):
- """Find which SDK version a workspace is associated with.
-
- This SDK may or may not exist in the bootstrap cache. There may be no
- SDK version associated with a workspace.
-
- Args:
- workspace_path: Root directory of the workspace (WorkspacePath()).
-
- Returns:
- version string or None.
- """
- # Config should always return a dictionary.
- config = _ReadLocalConfig(workspace_path)
-
- # If version is present, use it, else return None.
- return config.get('version')
-
-
-def SetActiveSdkVersion(workspace_path, version):
- """Set which SDK version a workspace is associated with.
-
- This method is NOT atomic.
-
- Args:
- workspace_path: Root directory of the workspace (WorkspacePath()).
- version: Version string of the SDK. (Eg. 1.2.3)
- """
- # Read the config, update its version, and write it.
- config = _ReadLocalConfig(workspace_path)
- config['version'] = version
- _WriteLocalConfig(workspace_path, config)
-
-
-def _ReadLocalConfig(workspace_path):
- """Read a local config for a workspace.
-
- Args:
- workspace_path: Root directory of the workspace (WorkspacePath()).
-
- Returns:
- Local workspace config as a Python dictionary.
- """
- try:
- return ReadConfigFile(os.path.join(workspace_path, WORKSPACE_LOCAL_CONFIG))
- except IOError:
- # If the file doesn't exist, it's an empty dictionary.
- return {}
-
-
-def _WriteLocalConfig(workspace_path, config):
- """Save out a new local config for a workspace.
-
- Args:
- workspace_path: Root directory of the workspace (WorkspacePath()).
- config: New local workspace config contents as a Python dictionary.
- """
- WriteConfigFile(os.path.join(workspace_path, WORKSPACE_LOCAL_CONFIG), config)
-
-
-def IsLocator(name):
- """Returns True if name is a specific locator."""
- if not name:
- raise ValueError('Locator is empty')
- return (name.startswith(_WORKSPACE_LOCATOR_PREFIX)
- or name.startswith(_BOARD_LOCATOR_PREFIX))
-
-
-def LocatorToPath(locator):
- """Returns the absolute path for this locator.
-
- Args:
- locator: a locator.
-
- Returns:
- The absolute path defined by this locator.
-
- Raises:
- ValueError: If |locator| is invalid.
- LocatorNotResolved: If |locator| is valid but could not be resolved.
- """
- if locator.startswith(_WORKSPACE_LOCATOR_PREFIX):
- workspace_path = WorkspacePath()
- if workspace_path is None:
- raise LocatorNotResolved(
- 'Workspace not found while trying to resolve %s' % locator)
- return os.path.join(workspace_path,
- locator[len(_WORKSPACE_LOCATOR_PREFIX):])
-
- if locator.startswith(_BOARD_LOCATOR_PREFIX):
- return os.path.join(constants.SOURCE_ROOT, 'src', 'overlays',
- 'overlay-%s' % locator[len(_BOARD_LOCATOR_PREFIX):])
-
- raise ValueError('Invalid locator %s' % locator)
-
-
-def PathToLocator(path):
- """Converts a path to a locator.
-
- This does not raise error if the path does not map to a locator. Some valid
- (legacy) brick path do not map to any locator: chromiumos-overlay,
- private board overlays, etc...
-
- Args:
- path: absolute or relative to CWD path to a workspace object or board
- overlay.
-
- Returns:
- The locator for this path if it exists, None otherwise.
- """
- workspace_path = WorkspacePath()
- path = os.path.abspath(path)
-
- if workspace_path is None:
- return None
-
- # If path is in the current workspace, return the relative path prefixed with
- # the workspace prefix.
- if os.path.commonprefix([path, workspace_path]) == workspace_path:
- return _WORKSPACE_LOCATOR_PREFIX + os.path.relpath(path, workspace_path)
-
- # If path is in the src directory of the checkout, this is a board overlay.
- # Encode it as board locator.
- src_path = os.path.join(constants.SOURCE_ROOT, 'src')
- if os.path.commonprefix([path, src_path]) == src_path:
- parts = os.path.split(os.path.relpath(path, src_path))
- if parts[0] == 'overlays':
- board_name = '-'.join(parts[1].split('-')[1:])
- return _BOARD_LOCATOR_PREFIX + board_name
-
- return None
-
-
-def LocatorToFriendlyName(locator):
- """Returns a friendly name for a given locator.
-
- Args:
- locator: a locator.
- """
- if IsLocator(locator) and locator.startswith(_WORKSPACE_LOCATOR_PREFIX):
- return locator[len(_WORKSPACE_LOCATOR_PREFIX):].replace('/', '.')
-
- raise ValueError('Not a valid workspace locator: %s' % locator)
-
-
-def WriteConfigFile(path, config):
- """Writes |config| to a file at |path|.
-
- Configuration files in a workspace should all use the same format
- whenever possible. Currently it's JSON, but centralizing config
- read/write makes it easier to change when needed.
-
- Args:
- path: path to write.
- config: configuration dictionary to write.
-
- Raises:
- ConfigFileError: |config| cannot be written as JSON.
- """
- # TODO(dpursell): Add support for comments in config files.
- try:
- osutils.WriteFile(
- path,
- json.dumps(config, sort_keys=True, indent=4, separators=(',', ': ')),
- makedirs=True)
- except TypeError as e:
- raise ConfigFileError('Writing config file %s failed: %s', path, e)
-
-
-def ReadConfigFile(path):
- """Reads a configuration file at |path|.
-
- For use with WriteConfigFile().
-
- Args:
- path: file path.
-
- Returns:
- Result of parsing the JSON file.
-
- Raises:
- ConfigFileError: JSON parsing failed.
- """
- try:
- return json.loads(osutils.ReadFile(path))
- except ValueError as e:
- raise ConfigFileError('%s is not in valid JSON format: %s' % (path, e))
diff --git a/lib/workspace_lib_unittest b/lib/workspace_lib_unittest
deleted file mode 120000
index 72196ceea..000000000
--- a/lib/workspace_lib_unittest
+++ /dev/null
@@ -1 +0,0 @@
-../scripts/wrapper.py \ No newline at end of file
diff --git a/lib/workspace_lib_unittest.py b/lib/workspace_lib_unittest.py
deleted file mode 100644
index b5378f930..000000000
--- a/lib/workspace_lib_unittest.py
+++ /dev/null
@@ -1,229 +0,0 @@
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Tests for the workspace_lib library."""
-
-from __future__ import print_function
-
-import mock
-import os
-
-from chromite.cbuildbot import constants
-from chromite.lib import cros_build_lib
-from chromite.lib import cros_test_lib
-from chromite.lib import osutils
-from chromite.lib import workspace_lib
-
-# pylint: disable=protected-access
-
-class WorkspaceLibTest(cros_test_lib.TempDirTestCase):
- """Unittests for workspace_lib.py"""
-
- def setUp(self):
- # Define assorted paths to test against.
- self.bogus_dir = os.path.join(self.tempdir, 'bogus')
-
- self.workspace_dir = os.path.join(self.tempdir, 'workspace')
- self.workspace_config = os.path.join(self.workspace_dir,
- workspace_lib.WORKSPACE_CONFIG)
- self.workspace_nested = os.path.join(self.workspace_dir, 'foo', 'bar')
- # Create workspace directories and files.
- osutils.Touch(self.workspace_config, makedirs=True)
- osutils.SafeMakedirs(self.workspace_nested)
-
- @mock.patch('os.getcwd')
- @mock.patch.object(cros_build_lib, 'IsInsideChroot', return_value=False)
- def testWorkspacePathOutsideChroot(self, _mock_inside, mock_cwd):
- # Set default to a dir outside the workspace.
- mock_cwd.return_value = self.bogus_dir
-
- # Inside the workspace, specified dir.
- self.assertEqual(self.workspace_dir,
- workspace_lib.WorkspacePath(self.workspace_dir))
- self.assertEqual(self.workspace_dir,
- workspace_lib.WorkspacePath(self.workspace_nested))
-
- # Outside the workspace, specified dir.
- self.assertEqual(None, workspace_lib.WorkspacePath(self.tempdir))
- self.assertEqual(None, workspace_lib.WorkspacePath(self.bogus_dir))
-
- # Inside the workspace, default dir.
- mock_cwd.return_value = self.workspace_dir
- self.assertEqual(self.workspace_dir, workspace_lib.WorkspacePath())
-
- mock_cwd.return_value = self.workspace_nested
- self.assertEqual(self.workspace_dir, workspace_lib.WorkspacePath())
-
- # Outside the workspace, default dir.
- mock_cwd.return_value = self.tempdir
- self.assertEqual(None, workspace_lib.WorkspacePath())
-
- mock_cwd.return_value = self.bogus_dir
- self.assertEqual(None, workspace_lib.WorkspacePath())
-
- @mock.patch.object(cros_build_lib, 'IsInsideChroot', return_value=True)
- def testWorkspacePathInsideChroot(self, _mock_inside):
- orig_root = constants.CHROOT_WORKSPACE_ROOT
- try:
- # Set default to a dir outside the workspace.
- constants.CHROOT_WORKSPACE_ROOT = self.bogus_dir
-
- # Inside the workspace, specified dir.
- self.assertEqual(self.workspace_dir,
- workspace_lib.WorkspacePath(self.workspace_dir))
- self.assertEqual(self.workspace_dir,
- workspace_lib.WorkspacePath(self.workspace_nested))
-
- # Outside the workspace, specified dir.
- self.assertEqual(None, workspace_lib.WorkspacePath(self.tempdir))
- self.assertEqual(None, workspace_lib.WorkspacePath(self.bogus_dir))
-
- # Inside the workspace, default dir.
- constants.CHROOT_WORKSPACE_ROOT = self.workspace_dir
- self.assertEqual(self.workspace_dir, workspace_lib.WorkspacePath())
-
- constants.CHROOT_WORKSPACE_ROOT = self.workspace_nested
- self.assertEqual(self.workspace_dir, workspace_lib.WorkspacePath())
-
- # Outside the workspace, default dir.
- constants.CHROOT_WORKSPACE_ROOT = self.tempdir
- self.assertEqual(None, workspace_lib.WorkspacePath())
-
- constants.CHROOT_WORKSPACE_ROOT = self.bogus_dir
- self.assertEqual(None, workspace_lib.WorkspacePath())
-
- finally:
- # Restore our constant to it's real value.
- constants.CHROOT_WORKSPACE_ROOT = orig_root
-
- def testChrootPath(self):
- # Check the default value.
- self.assertEqual(os.path.join(self.workspace_dir, '.chroot'),
- workspace_lib.ChrootPath(self.workspace_dir))
-
- # Set a new absolute value, check that we get it back.
- workspace_lib.SetChrootDir(self.workspace_dir, self.bogus_dir)
- self.assertEqual(self.bogus_dir,
- workspace_lib.ChrootPath(self.workspace_dir))
-
- # Set a new relative path, check that it is properly appended to the
- # workspace path.
- workspace_lib.SetChrootDir(self.workspace_dir, 'some/path')
- self.assertEqual(os.path.join(self.workspace_dir, 'some/path'),
- workspace_lib.ChrootPath(self.workspace_dir))
-
- @mock.patch.object(osutils, 'IsInsideVm', return_value=True)
- def testChrootPathUnderVm(self, _mock_inside_vm):
- """Make sure that inside the VM, chroot dir is under /chroots/..."""
- self.assertEqual(
- os.path.join(workspace_lib.MAIN_CHROOT_DIR_IN_VM,
- os.path.basename(self.workspace_dir)),
- workspace_lib.ChrootPath(self.workspace_dir)
- )
-
- def testReadWriteLocalConfig(self):
- # Non-existent config should read as an empty dictionary.
- config = workspace_lib._ReadLocalConfig(self.workspace_dir)
- self.assertEqual({}, config)
-
- # Write out an empty dict, and make sure we can read it back.
- workspace_lib._WriteLocalConfig(self.workspace_dir, {})
- config = workspace_lib._ReadLocalConfig(self.workspace_dir)
- self.assertEqual({}, config)
-
- # Write out a value, and verify we can read it.
- workspace_lib._WriteLocalConfig(self.workspace_dir, {'version': 'foo'})
- config = workspace_lib._ReadLocalConfig(self.workspace_dir)
- self.assertEqual({'version': 'foo'}, config)
-
- # Overwrite value, and verify we can read it.
- workspace_lib._WriteLocalConfig(self.workspace_dir, {'version': 'bar'})
- config = workspace_lib._ReadLocalConfig(self.workspace_dir)
- self.assertEqual({'version': 'bar'}, config)
-
- def testReadWriteActiveSdkVersion(self):
- # If no version is set, value should be None.
- version = workspace_lib.GetActiveSdkVersion(self.workspace_dir)
- self.assertEqual(None, version)
-
- # Set value, and make sure we can read it.
- workspace_lib.SetActiveSdkVersion(self.workspace_dir, 'foo')
- version = workspace_lib.GetActiveSdkVersion(self.workspace_dir)
- self.assertEqual('foo', version)
-
- # Set different value, and make sure we can read it.
- workspace_lib.SetActiveSdkVersion(self.workspace_dir, 'bar')
- version = workspace_lib.GetActiveSdkVersion(self.workspace_dir)
- self.assertEqual('bar', version)
-
- # Create config with unrelated values, should be same as no config.
- workspace_lib._WriteLocalConfig(self.workspace_dir, {'foo': 'bar'})
- version = workspace_lib.GetActiveSdkVersion(self.workspace_dir)
- self.assertEqual(None, version)
-
- # Set version, and make sure it works.
- workspace_lib.SetActiveSdkVersion(self.workspace_dir, '1.2.3')
- version = workspace_lib.GetActiveSdkVersion(self.workspace_dir)
- self.assertEqual('1.2.3', version)
-
- # Ensure all of config is there afterwords.
- config = workspace_lib._ReadLocalConfig(self.workspace_dir)
- self.assertEqual({'version': '1.2.3', 'foo': 'bar'}, config)
-
- @mock.patch('os.getcwd')
- @mock.patch.object(cros_build_lib, 'IsInsideChroot', return_value=False)
- def testPathToLocator(self, _mock_inside, mock_cwd):
- """Tests the path to locator conversion."""
- ws = self.workspace_dir
- mock_cwd.return_value = ws
-
- foo_path = workspace_lib.PathToLocator(os.path.join(ws, 'foo'))
- baz_path = workspace_lib.PathToLocator(os.path.join(ws, 'bar', 'foo',
- 'baz'))
- daisy_path = workspace_lib.PathToLocator(os.path.join(constants.SOURCE_ROOT,
- 'src', 'overlays',
- 'overlay-daisy'))
- some_path = workspace_lib.PathToLocator(os.path.join(constants.SOURCE_ROOT,
- 'srcs', 'bar'))
-
- self.assertEqual('//foo', foo_path)
- self.assertEqual('//bar/foo/baz', baz_path)
- self.assertEqual('board:daisy', daisy_path)
- self.assertEqual(None, some_path)
-
- def assertReversible(loc):
- path = workspace_lib.LocatorToPath(loc)
- self.assertEqual(loc, workspace_lib.PathToLocator(path))
-
- assertReversible('//foo')
- assertReversible('//foo/bar/baz')
- assertReversible('board:gizmo')
-
-
-class ConfigurationTest(cros_test_lib.TempDirTestCase):
- """Test WriteConfigFile() and ReadConfigFile()."""
-
- def testWriteReadConfigFile(self):
- """Tests WriteConfigFile() then ReadConfigFile()."""
- path = os.path.join(self.tempdir, 'foo.json')
- config = {'foo': 1, 'bar': 2}
-
- workspace_lib.WriteConfigFile(path, config)
- self.assertDictEqual(config, workspace_lib.ReadConfigFile(path))
-
- def testWriteConfigFileInvalid(self):
- """Tests writing an invalid configuration file."""
- path = os.path.join(self.tempdir, 'foo.json')
- config = Exception()
-
- with self.assertRaises(workspace_lib.ConfigFileError):
- workspace_lib.WriteConfigFile(path, config)
-
- def testReadConfigFileInvalid(self):
- """Tests reading an invalid configuration file."""
- path = os.path.join(self.tempdir, 'foo.json')
- osutils.WriteFile(path, 'invalid contents')
-
- with self.assertRaises(workspace_lib.ConfigFileError):
- workspace_lib.ReadConfigFile(path)
diff --git a/mobmonitor/scripts/mobmonitor.py b/mobmonitor/scripts/mobmonitor.py
index 2bc70b874..e3fff6431 100755
--- a/mobmonitor/scripts/mobmonitor.py
+++ b/mobmonitor/scripts/mobmonitor.py
@@ -12,12 +12,14 @@ import json
import os
import sys
+from cherrypy.lib.static import serve_file
from logging import handlers as logging_handlers
from chromite.lib import remote_access
from chromite.lib import commandline
from chromite.lib import cros_logging as logging
from chromite.mobmonitor.checkfile import manager
+from chromite.mobmonitor.util import collect_logs
STATICDIR = '/etc/mobmonitor/static'
@@ -111,6 +113,12 @@ class MobMonitorRoot(object):
args, kwargs)
return json.dumps(manager.MapServiceStatusToDict(status))
+ @cherrypy.expose
+ def CollectLogs(self):
+ tarfile = collect_logs.collect_logs()
+ return serve_file(tarfile, 'application/x-download',
+ 'attachment', os.path.basename(tarfile))
+
def SetupLogging(logdir):
logging.basicConfig(
diff --git a/mobmonitor/static/js/main.js b/mobmonitor/static/js/main.js
index ec1fad689..f09d36c42 100644
--- a/mobmonitor/static/js/main.js
+++ b/mobmonitor/static/js/main.js
@@ -19,7 +19,7 @@ $(document).ready(function() {
// Setup the log collection button.
$(document).on('click', '.collect-logs', function() {
- console.log('Collecting logs!');
+ window.open('/CollectLogs', '_blank');
});
// Setup the repair action buttons
diff --git a/mobmonitor/util/__init__.py b/mobmonitor/util/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/mobmonitor/util/__init__.py
diff --git a/mobmonitor/util/collect_logs.py b/mobmonitor/util/collect_logs.py
new file mode 100644
index 000000000..503fbdf69
--- /dev/null
+++ b/mobmonitor/util/collect_logs.py
@@ -0,0 +1,40 @@
+# Copyright 2015 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Simple log collection script for Mob* Monitor"""
+
+from __future__ import print_function
+
+import os
+import tempfile
+import shutil
+
+from chromite.lib import cros_build_lib
+
+
+TMPDIR_PREFIX = 'moblab_logs_'
+LOG_DIRS = {
+ 'system_logs': '/var/log',
+ 'autotest_logs': '/usr/local/autotest/logs'
+}
+
+
+def collect_logs():
+ tempdir = tempfile.mkdtemp(prefix=TMPDIR_PREFIX)
+ os.chmod(tempdir, 0o777)
+
+ for name, path in LOG_DIRS.iteritems():
+ if not os.path.exists(path):
+ continue
+ shutil.copytree(path, os.path.join(tempdir, name))
+
+ cmd = ['mobmoncli', 'GetStatus']
+ cros_build_lib.RunCommand(
+ cmd,
+ log_stdout_to_file=os.path.join(tempdir, 'mobmonitor_getstatus')
+ )
+
+ tarball = '%s.tgz' % tempdir
+ cros_build_lib.CreateTarball(tarball, tempdir)
+ return tarball
diff --git a/scripts/cidb_admin.py b/scripts/cidb_admin.py
index 7bf6e57c3..d4f592a84 100644
--- a/scripts/cidb_admin.py
+++ b/scripts/cidb_admin.py
@@ -79,5 +79,3 @@ def main(argv):
db = cidb.CIDBConnection(options.cred_dir)
db.DropDatabase()
print('Done.')
-
-
diff --git a/scripts/cros_best_revision.py b/scripts/cros_best_revision.py
index 841672170..ca424ad6d 100644
--- a/scripts/cros_best_revision.py
+++ b/scripts/cros_best_revision.py
@@ -49,7 +49,7 @@ class ChromeCommitter(object):
self._dryrun = dryrun
self._lkgm = None
self._old_lkgm = None
- self.site_config = config_lib.LoadConfigFromFile()
+ self.site_config = config_lib.GetConfig()
def CheckoutChromeLKGM(self):
diff --git a/scripts/cros_brick_utils.py b/scripts/cros_brick_utils.py
deleted file mode 100644
index 102d7a75a..000000000
--- a/scripts/cros_brick_utils.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Collection of tools used in scripts while we migrate to bricks."""
-
-from __future__ import print_function
-
-from chromite.lib import brick_lib
-from chromite.lib import commandline
-from chromite.lib import cros_build_lib
-
-
-def ParseArgs(argv):
- """Parse arguments.
-
- Args:
- argv: array of arguments passed to the script.
- """
- parser = commandline.ArgumentParser(description=__doc__)
- parser.add_argument('brick')
- parser.add_argument(
- '--friendly-name', action='store_true', dest='friendlyname',
- help='Returns the friendly name for a given brick. This name is used in '
- 'the sysroot path and as "board name" in our legacy tools.')
- options = parser.parse_args(argv)
- options.Freeze()
- return options
-
-
-def main(argv):
- opts = ParseArgs(argv)
-
- try:
- brick = brick_lib.Brick(opts.brick, allow_legacy=False)
- except brick_lib.BrickNotFound:
- cros_build_lib.Die('Brick %s not found.' % opts.brick)
-
- if opts.friendlyname:
- print(brick.FriendlyName())
diff --git a/scripts/cros_generate_breakpad_symbols.py b/scripts/cros_generate_breakpad_symbols.py
index 2c7606b30..92a242e14 100644
--- a/scripts/cros_generate_breakpad_symbols.py
+++ b/scripts/cros_generate_breakpad_symbols.py
@@ -63,26 +63,26 @@ def ReadSymsHeader(sym_file):
def GenerateBreakpadSymbol(elf_file, debug_file=None, breakpad_dir=None,
- board=None, strip_cfi=False, num_errors=None):
+ strip_cfi=False, num_errors=None,
+ dump_syms_cmd='dump_syms'):
"""Generate the symbols for |elf_file| using |debug_file|
Args:
elf_file: The file to dump symbols for
debug_file: Split debug file to use for symbol information
breakpad_dir: The dir to store the output symbol file in
- board: If |breakpad_dir| is not specified, use |board| to find it
strip_cfi: Do not generate CFI data
num_errors: An object to update with the error count (needs a .value member)
+ dump_syms_cmd: Command to use for dumping symbols.
Returns:
- The number of errors that were encountered.
+ The name of symbol file written out.
"""
- if breakpad_dir is None:
- breakpad_dir = FindBreakpadDir(board)
+ assert breakpad_dir
if num_errors is None:
num_errors = ctypes.c_int()
- cmd_base = ['dump_syms']
+ cmd_base = [dump_syms_cmd]
if strip_cfi:
cmd_base += ['-c']
# Some files will not be readable by non-root (e.g. set*id /bin/su).
@@ -151,7 +151,7 @@ def GenerateBreakpadSymbol(elf_file, debug_file=None, breakpad_dir=None,
os.chmod(sym_file, 0o644)
temp.delete = False
- return num_errors.value
+ return sym_file
def GenerateBreakpadSymbols(board, breakpad_dir=None, strip_cfi=False,
@@ -264,7 +264,7 @@ def GenerateBreakpadSymbols(board, breakpad_dir=None, strip_cfi=False,
# Now start generating symbols for the discovered elfs.
with parallel.BackgroundTaskRunner(GenerateBreakpadSymbol,
- breakpad_dir=breakpad_dir, board=board,
+ breakpad_dir=breakpad_dir,
strip_cfi=strip_cfi,
num_errors=bg_errors,
processes=num_processes) as queue:
diff --git a/scripts/cros_generate_breakpad_symbols_unittest.py b/scripts/cros_generate_breakpad_symbols_unittest.py
index 9c27c50cb..9290feb3a 100644
--- a/scripts/cros_generate_breakpad_symbols_unittest.py
+++ b/scripts/cros_generate_breakpad_symbols_unittest.py
@@ -243,28 +243,20 @@ class GenerateSymbolTest(cros_test_lib.MockTempDirTestCase):
def testNormal(self):
"""Normal run -- given an ELF and a debug file"""
ret = cros_generate_breakpad_symbols.GenerateBreakpadSymbol(
- self.elf_file, self.debug_file, breakpad_dir=self.breakpad_dir)
- self.assertEqual(ret, 0)
+ self.elf_file, self.debug_file, self.breakpad_dir)
+ self.assertEqual(ret, self.sym_file)
self.assertEqual(self.rc_mock.call_count, 1)
self.assertCommandArgs(0, ['dump_syms', self.elf_file, self.debug_dir])
self.assertExists(self.sym_file)
- def testNormalBoard(self):
- """Normal run w/board info but not breakpad dir"""
- ret = cros_generate_breakpad_symbols.GenerateBreakpadSymbol(
- self.elf_file, board='foo')
- self.assertEqual(ret, 0)
- self.assertCommandArgs(0, ['dump_syms', self.elf_file])
- self.assertEqual(self.rc_mock.call_count, 1)
- self.assertExists(self.sym_file)
-
def testNormalNoCfi(self):
"""Normal run w/out CFI"""
# Make sure the num_errors flag works too.
num_errors = ctypes.c_int()
ret = cros_generate_breakpad_symbols.GenerateBreakpadSymbol(
- self.elf_file, strip_cfi=True, num_errors=num_errors)
- self.assertEqual(ret, 0)
+ self.elf_file, breakpad_dir=self.breakpad_dir,
+ strip_cfi=True, num_errors=num_errors)
+ self.assertEqual(ret, self.sym_file)
self.assertEqual(num_errors.value, 0)
self.assertCommandArgs(0, ['dump_syms', '-c', self.elf_file])
self.assertEqual(self.rc_mock.call_count, 1)
@@ -272,8 +264,9 @@ class GenerateSymbolTest(cros_test_lib.MockTempDirTestCase):
def testNormalElfOnly(self):
"""Normal run -- given just an ELF"""
- ret = cros_generate_breakpad_symbols.GenerateBreakpadSymbol(self.elf_file)
- self.assertEqual(ret, 0)
+ ret = cros_generate_breakpad_symbols.GenerateBreakpadSymbol(
+ self.elf_file, breakpad_dir=self.breakpad_dir)
+ self.assertEqual(ret, self.sym_file)
self.assertCommandArgs(0, ['dump_syms', self.elf_file])
self.assertEqual(self.rc_mock.call_count, 1)
self.assertExists(self.sym_file)
@@ -282,17 +275,18 @@ class GenerateSymbolTest(cros_test_lib.MockTempDirTestCase):
"""Normal run where ELF is readable only by root"""
with mock.patch.object(os, 'access') as mock_access:
mock_access.return_value = False
- ret = cros_generate_breakpad_symbols.GenerateBreakpadSymbol(self.elf_file)
- self.assertEqual(ret, 0)
- self.assertCommandArgs(0, ['sudo', '--', 'dump_syms', self.elf_file])
+ ret = cros_generate_breakpad_symbols.GenerateBreakpadSymbol(
+ self.elf_file, breakpad_dir=self.breakpad_dir)
+ self.assertEqual(ret, self.sym_file)
+ self.assertCommandArgs(0, ['sudo', '--', 'dump_syms', self.elf_file])
def testLargeDebugFail(self):
"""Running w/large .debug failed, but retry worked"""
self.rc_mock.AddCmdResult(['dump_syms', self.elf_file, self.debug_dir],
returncode=1)
ret = cros_generate_breakpad_symbols.GenerateBreakpadSymbol(
- self.elf_file, self.debug_file)
- self.assertEqual(ret, 0)
+ self.elf_file, self.debug_file, self.breakpad_dir)
+ self.assertEqual(ret, self.sym_file)
self.assertEqual(self.rc_mock.call_count, 2)
self.assertCommandArgs(0, ['dump_syms', self.elf_file, self.debug_dir])
self.assertCommandArgs(
@@ -307,8 +301,8 @@ class GenerateSymbolTest(cros_test_lib.MockTempDirTestCase):
self.debug_dir],
returncode=1)
ret = cros_generate_breakpad_symbols.GenerateBreakpadSymbol(
- self.elf_file, self.debug_file)
- self.assertEqual(ret, 0)
+ self.elf_file, self.debug_file, self.breakpad_dir)
+ self.assertEqual(ret, self.sym_file)
self.assertEqual(self.rc_mock.call_count, 3)
self.assertCommandArgs(0, ['dump_syms', self.elf_file, self.debug_dir])
self.assertCommandArgs(
@@ -319,12 +313,13 @@ class GenerateSymbolTest(cros_test_lib.MockTempDirTestCase):
def testCompleteFail(self):
"""Running dump_syms always fails"""
self.rc_mock.SetDefaultCmdResult(returncode=1)
- ret = cros_generate_breakpad_symbols.GenerateBreakpadSymbol(self.elf_file)
+ ret = cros_generate_breakpad_symbols.GenerateBreakpadSymbol(
+ self.elf_file, breakpad_dir=self.breakpad_dir)
self.assertEqual(ret, 1)
# Make sure the num_errors flag works too.
num_errors = ctypes.c_int()
ret = cros_generate_breakpad_symbols.GenerateBreakpadSymbol(
- self.elf_file, num_errors=num_errors)
+ self.elf_file, breakpad_dir=self.breakpad_dir, num_errors=num_errors)
self.assertEqual(ret, 1)
self.assertEqual(num_errors.value, 1)
diff --git a/scripts/cros_list_modified_packages.py b/scripts/cros_list_modified_packages.py
index b5327e927..7b9dfd139 100644
--- a/scripts/cros_list_modified_packages.py
+++ b/scripts/cros_list_modified_packages.py
@@ -34,7 +34,6 @@ except ImportError:
import queue as Queue
from chromite.cbuildbot import constants
-from chromite.lib import brick_lib
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
@@ -142,12 +141,6 @@ def ListWorkonPackagesInfo(sysroot):
vdb_path = os.path.join(sysroot.path, portage.const.VDB_PATH)
for overlay in overlays:
- # Is this a brick overlay? Get its source base directory.
- brick_srcbase = ''
- brick = brick_lib.FindBrickInPath(overlay)
- if brick and brick.OverlayDir() == overlay.rstrip(os.path.sep):
- brick_srcbase = brick.SourceDir()
-
for filename, projects, srcpaths in portage_util.GetWorkonProjectMap(
overlay, packages):
# chromeos-base/power_manager/power_manager-9999
@@ -170,12 +163,9 @@ def ListWorkonPackagesInfo(sysroot):
# Get the modificaton time of the ebuild in the overlay.
src_ebuild_mtime = os.lstat(os.path.join(overlay, filename)).st_mtime
- # Translate relative srcpath values into their absolute counterparts.
- full_srcpaths = [os.path.join(brick_srcbase, s) for s in srcpaths]
-
# Write info into the results dictionary, overwriting any previous
# values. This ensures that overlays override appropriately.
- results[cp] = WorkonPackageInfo(cp, pkg_mtime, projects, full_srcpaths,
+ results[cp] = WorkonPackageInfo(cp, pkg_mtime, projects, srcpaths,
src_ebuild_mtime)
return results.values()
@@ -231,7 +221,6 @@ def _ParseArguments(argv):
target = parser.add_mutually_exclusive_group(required=True)
target.add_argument('--board', help='Board name')
- target.add_argument('--brick', help='Brick locator')
target.add_argument('--host', default=False, action='store_true',
help='Look at host packages instead of board packages')
target.add_argument('--sysroot', help='Sysroot path.')
@@ -245,12 +234,7 @@ def main(argv):
logging.getLogger().setLevel(logging.INFO)
flags = _ParseArguments(argv)
sysroot = None
- if flags.brick:
- try:
- sysroot = cros_build_lib.GetSysroot(brick_lib.Brick(flags.brick))
- except brick_lib.BrickNotFound:
- cros_build_lib.Die('Could not load brick %s.' % flags.brick)
- elif flags.board:
+ if flags.board:
sysroot = cros_build_lib.GetSysroot(flags.board)
elif flags.host:
sysroot = '/'
diff --git a/scripts/cros_list_overlays.py b/scripts/cros_list_overlays.py
index 2cead1b9b..754a041d8 100644
--- a/scripts/cros_list_overlays.py
+++ b/scripts/cros_list_overlays.py
@@ -9,7 +9,6 @@ from __future__ import print_function
import os
from chromite.cbuildbot import constants
-from chromite.lib import brick_lib
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import portage_util
@@ -28,7 +27,6 @@ def _ParseArguments(argv):
'only makes sense when --board is specified.')
parser.add_argument('-a', '--all', default=False, action='store_true',
help='Show all overlays (even common ones).')
- parser.add_argument('--brick', help='Main brick to use')
opts = parser.parse_args(argv)
opts.Freeze()
@@ -36,13 +34,6 @@ def _ParseArguments(argv):
if opts.primary_only and opts.board is None:
parser.error('--board is required when --primary_only is supplied.')
- if opts.brick:
- if opts.board:
- parser.error('--board and --brick are incompatible.')
-
- if opts.all:
- parser.error('Cannot list all overlays with --brick')
-
return opts
@@ -50,21 +41,17 @@ def main(argv):
opts = _ParseArguments(argv)
args = (constants.BOTH_OVERLAYS, opts.board)
- if opts.brick:
- main_brick = brick_lib.Brick(opts.brick)
- overlays = [b.OverlayDir() for b in main_brick.BrickStack()]
+ # Verify that a primary overlay exists.
+ try:
+ primary_overlay = portage_util.FindPrimaryOverlay(*args)
+ except portage_util.MissingOverlayException as ex:
+ cros_build_lib.Die(str(ex))
+
+ # Get the overlays to print.
+ if opts.primary_only:
+ overlays = [primary_overlay]
else:
- # Verify that a primary overlay exists.
- try:
- primary_overlay = portage_util.FindPrimaryOverlay(*args)
- except portage_util.MissingOverlayException as ex:
- cros_build_lib.Die(str(ex))
-
- # Get the overlays to print.
- if opts.primary_only:
- overlays = [primary_overlay]
- else:
- overlays = portage_util.FindOverlays(*args)
+ overlays = portage_util.FindOverlays(*args)
# Exclude any overlays in src/third_party, for backwards compatibility with
# scripts that expected these to not be listed.
diff --git a/scripts/cros_mark_as_stable.py b/scripts/cros_mark_as_stable.py
index 531f652de..4b249538b 100644
--- a/scripts/cros_mark_as_stable.py
+++ b/scripts/cros_mark_as_stable.py
@@ -79,12 +79,8 @@ def CleanStalePackages(srcroot, boards, package_atoms):
# TODO(build): This code needs to be gutted and rebased to cros_build_lib.
def _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd):
"""Returns true if there are local commits."""
- current_branch = git.GetCurrentBranch(cwd)
-
- if current_branch != stable_branch:
- return False
output = git.RunGit(
- cwd, ['rev-parse', 'HEAD', tracking_branch]).output.split()
+ cwd, ['rev-parse', stable_branch, tracking_branch]).output.split()
return output[0] != output[1]
@@ -108,14 +104,23 @@ def PushChange(stable_branch, tracking_branch, dryrun, cwd):
Raises:
OSError: Error occurred while pushing.
"""
+ if not git.DoesCommitExistInRepo(cwd, stable_branch):
+ logging.debug('No branch created for %s. Exiting', cwd)
+ return
+
if not _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd):
- logging.info('No work found to push in %s. Exiting', cwd)
+ logging.debug('No work found to push in %s. Exiting', cwd)
return
# For the commit queue, our local branch may contain commits that were
# just tested and pushed during the CommitQueueCompletion stage. Sync
# and rebase our local branch on top of the remote commits.
- remote_ref = git.GetTrackingBranch(cwd, for_push=True)
+ remote_ref = git.GetTrackingBranch(cwd,
+ branch=stable_branch,
+ for_push=True)
+ # SyncPushBranch rebases HEAD onto the updated remote. We need to checkout
+ # stable_branch here in order to update it.
+ git.RunGit(cwd, ['checkout', stable_branch])
git.SyncPushBranch(cwd, remote_ref.remote, remote_ref.ref)
# Check whether any local changes remain after the sync.
@@ -141,7 +146,8 @@ def PushChange(stable_branch, tracking_branch, dryrun, cwd):
'%s..%s' % (remote_ref.ref, stable_branch)]).output
description = '%s\n\n%s' % (GIT_COMMIT_SUBJECT, description)
logging.info('For %s, using description %s', cwd, description)
- git.CreatePushBranch(constants.MERGE_BRANCH, cwd)
+ git.CreatePushBranch(constants.MERGE_BRANCH, cwd,
+ remote_push_branch=remote_ref)
git.RunGit(cwd, ['merge', '--squash', stable_branch])
git.RunGit(cwd, ['commit', '-m', description])
git.RunGit(cwd, ['config', 'push.default', 'tracking'])
diff --git a/scripts/cros_mark_as_stable_unittest.py b/scripts/cros_mark_as_stable_unittest.py
index 9d87e1b8e..bafb1603a 100644
--- a/scripts/cros_mark_as_stable_unittest.py
+++ b/scripts/cros_mark_as_stable_unittest.py
@@ -41,6 +41,7 @@ class NonClassTests(cros_test_lib.MockTestCase):
git_log = 'Marking test_one as stable\nMarking test_two as stable\n'
fake_description = 'Marking set of ebuilds as stable\n\n%s' % git_log
+ self.PatchObject(git, 'DoesCommitExistInRepo', return_value=True)
self.PatchObject(cros_mark_as_stable, '_DoWeHaveLocalCommits',
return_value=True)
self.PatchObject(cros_mark_as_stable.GitBranch, 'CreateBranch',
@@ -56,6 +57,8 @@ class NonClassTests(cros_test_lib.MockTestCase):
create_mock = self.PatchObject(git, 'CreatePushBranch')
git_mock = self.StartPatcher(RunGitMock())
+ git_mock.AddCmdResult(['checkout', self._branch])
+
cmd = ['log', '--format=short', '--perl-regexp', '--author',
'^(?!chrome-bot)', 'refs/remotes/gerrit/master..%s' % self._branch]
@@ -72,14 +75,13 @@ class NonClassTests(cros_test_lib.MockTestCase):
git_mock.AddCmdResult(['commit', '-m', fake_description])
git_mock.AddCmdResult(['config', 'push.default', 'tracking'])
- try:
- cros_mark_as_stable.PushChange(self._branch, self._target_manifest_branch,
- False, '.')
- finally:
- sync_mock.assert_called_with('.', 'gerrit', 'refs/remotes/gerrit/master')
- if not bad_cls:
- push_mock.assert_called_with('merge_branch', '.', dryrun=False)
- create_mock.assert_called_with('merge_branch', '.')
+ cros_mark_as_stable.PushChange(self._branch, self._target_manifest_branch,
+ False, '.')
+ sync_mock.assert_called_with('.', 'gerrit', 'refs/remotes/gerrit/master')
+ if not bad_cls:
+ push_mock.assert_called_with('merge_branch', '.', dryrun=False)
+ create_mock.assert_called_with('merge_branch', '.',
+ remote_push_branch=mock.ANY)
def testPushChange(self):
"""Verify pushing changes works."""
diff --git a/scripts/cros_setup_toolchains.py b/scripts/cros_setup_toolchains.py
index 6cda17077..ca6d311c8 100644
--- a/scripts/cros_setup_toolchains.py
+++ b/scripts/cros_setup_toolchains.py
@@ -18,7 +18,6 @@ from chromite.lib import cros_logging as logging
from chromite.lib import osutils
from chromite.lib import parallel
from chromite.lib import toolchain
-from chromite.lib import workspace_lib
# Needs to be after chromite imports.
import lddtree
@@ -569,8 +568,8 @@ def ExpandTargets(targets_wanted):
Dictionary of concrete targets and their toolchain tuples.
"""
targets_wanted = set(targets_wanted)
- if targets_wanted in (set(['boards']), set(['bricks'])):
- # Only pull targets from the included boards/bricks.
+ if targets_wanted == set(['boards']):
+ # Only pull targets from the included boards.
return {}
all_targets = toolchain.GetAllTargets()
@@ -589,7 +588,7 @@ def ExpandTargets(targets_wanted):
def UpdateToolchains(usepkg, deleteold, hostonly, reconfig,
- targets_wanted, boards_wanted, bricks_wanted, root='/'):
+ targets_wanted, boards_wanted, root='/'):
"""Performs all steps to create a synchronized toolchain enviroment.
Args:
@@ -599,7 +598,6 @@ def UpdateToolchains(usepkg, deleteold, hostonly, reconfig,
reconfig: Reload crossdev config and reselect toolchains
targets_wanted: All the targets to update
boards_wanted: Load targets from these boards
- bricks_wanted: Load targets from these bricks
root: The root in which to install the toolchains.
"""
targets, crossdev_targets, reconfig_targets = {}, {}, {}
@@ -608,12 +606,10 @@ def UpdateToolchains(usepkg, deleteold, hostonly, reconfig,
# work on bare systems where this is useful.
targets = ExpandTargets(targets_wanted)
- # Now re-add any targets that might be from this board/brick. This is to
+ # Now re-add any targets that might be from this board. This is to
# allow unofficial boards to declare their own toolchains.
for board in boards_wanted:
targets.update(toolchain.GetToolchainsForBoard(board))
- for brick in bricks_wanted:
- targets.update(toolchain.GetToolchainsForBrick(brick))
# First check and initialize all cross targets that need to be.
for target in targets:
@@ -647,12 +643,9 @@ def ShowConfig(name):
"""Show the toolchain tuples used by |name|
Args:
- name: The board name or brick locator to query.
+ name: The board name to query.
"""
- if workspace_lib.IsLocator(name):
- toolchains = toolchain.GetToolchainsForBrick(name)
- else:
- toolchains = toolchain.GetToolchainsForBoard(name)
+ toolchains = toolchain.GetToolchainsForBoard(name)
# Make sure we display the default toolchain first.
print(','.join(
toolchain.FilterToolchains(toolchains, 'default', True).keys() +
@@ -1075,21 +1068,18 @@ def main(argv):
parser.add_argument('-t', '--targets',
dest='targets', default='sdk',
help="Comma separated list of tuples. Special keywords "
- "'host', 'sdk', 'boards', 'bricks' and 'all' are "
+ "'host', 'sdk', 'boards', and 'all' are "
"allowed. Defaults to 'sdk'.")
parser.add_argument('--include-boards', default='', metavar='BOARDS',
help='Comma separated list of boards whose toolchains we '
'will always include. Default: none')
- parser.add_argument('--include-bricks', default='', metavar='BRICKS',
- help='Comma separated list of bricks whose toolchains we '
- 'will always include. Default: none')
parser.add_argument('--hostonly',
dest='hostonly', default=False, action='store_true',
help='Only setup the host toolchain. '
'Useful for bootstrapping chroot')
parser.add_argument('--show-board-cfg', '--show-cfg',
dest='cfg_name', default=None,
- help='Board or brick to list toolchains tuples for')
+ help='Board to list toolchains tuples for')
parser.add_argument('--create-packages',
action='store_true', default=False,
help='Build redistributable packages')
@@ -1110,8 +1100,6 @@ def main(argv):
targets_wanted = set(options.targets.split(','))
boards_wanted = (set(options.include_boards.split(','))
if options.include_boards else set())
- bricks_wanted = (set(options.include_bricks.split(','))
- if options.include_bricks else set())
if options.cfg_name:
ShowConfig(options.cfg_name)
@@ -1129,7 +1117,7 @@ def main(argv):
root = options.sysroot or '/'
UpdateToolchains(options.usepkg, options.deleteold, options.hostonly,
options.reconfig, targets_wanted, boards_wanted,
- bricks_wanted, root=root)
+ root=root)
Crossdev.Save()
return 0
diff --git a/scripts/cros_show_waterfall_layout.py b/scripts/cros_show_waterfall_layout.py
index 0b064a63d..9bad367c6 100644
--- a/scripts/cros_show_waterfall_layout.py
+++ b/scripts/cros_show_waterfall_layout.py
@@ -61,7 +61,7 @@ def _ParseArguments(argv):
def main(argv):
opts = _ParseArguments(argv)
- site_config = config_lib.LoadConfigFromFile()
+ site_config = config_lib.GetConfig()
layout = {}
for config_name, config in site_config.iteritems():
diff --git a/scripts/cros_sysroot_utils.py b/scripts/cros_sysroot_utils.py
index a4d3fbf95..614153e9f 100644
--- a/scripts/cros_sysroot_utils.py
+++ b/scripts/cros_sysroot_utils.py
@@ -10,7 +10,6 @@ from __future__ import print_function
import os
import sys
-from chromite.lib import brick_lib
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import sysroot_lib
@@ -33,7 +32,6 @@ def ParseArgs(argv):
config = subparser.add_parser('generate-config')
target = config.add_mutually_exclusive_group(required=True)
target.add_argument('--board', help='Board to generate the config for.')
- target.add_argument('--brick', help='Brick to generate the config for.')
config.add_argument('--out-file', dest='out_file',
help='File to write into. If not specified, the '
'configuration will be printed to stdout.')
@@ -84,11 +82,7 @@ def main(argv):
if opts.command == 'create-wrappers':
sysroot.CreateAllWrappers(opts.friendlyname)
elif opts.command == 'generate-config':
- if opts.brick:
- config = sysroot.GenerateBrickConfig(
- brick_lib.Brick(opts.brick).BrickStack())
- else:
- config = sysroot.GenerateBoardConfig(opts.board)
+ config = sysroot.GenerateBoardConfig(opts.board)
output.write('\n' + config)
elif opts.command == 'generate-make-conf':
diff --git a/scripts/cros_unittest.py b/scripts/cros_unittest.py
index 7510fcad0..e05ef9056 100644
--- a/scripts/cros_unittest.py
+++ b/scripts/cros_unittest.py
@@ -13,7 +13,7 @@ from chromite.lib import stats_unittest
from chromite.scripts import cros
-class RunScriptTest(cros_test_lib.WorkspaceTestCase):
+class RunScriptTest(cros_test_lib.MockTempDirTestCase):
"""Test the main functionality."""
def setUp(self):
diff --git a/scripts/cros_workon.py b/scripts/cros_workon.py
index 087c0a828..252ebc3be 100644
--- a/scripts/cros_workon.py
+++ b/scripts/cros_workon.py
@@ -12,7 +12,6 @@ source modified and built using the unstable 'live' (9999) ebuild.
from __future__ import print_function
-from chromite.lib import brick_lib
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import terminal
@@ -23,7 +22,6 @@ def main(argv):
shared = commandline.SharedParser()
shared.add_argument('--board', default=cros_build_lib.GetDefaultBoard(),
help='The board to set package keywords for.')
- shared.add_argument('--brick', help='The brick to set package keywords for.')
shared.add_argument('--host', default=False, action='store_true',
help='Uses the host instead of board')
shared.add_argument('--remote', default='',
@@ -81,15 +79,8 @@ def main(argv):
elif options.board:
friendly_name = options.board
sysroot = cros_build_lib.GetSysroot(board=options.board)
- elif options.brick:
- brick = brick_lib.Brick(options.brick)
- friendly_name = brick.FriendlyName()
- # TODO(wiley) This is a hack. It doesn't really make sense to calculate
- # the sysroot from a brick alone, since bricks are installed
- # into sysroots. Revisit this when blueprints are working.
- sysroot = cros_build_lib.GetSysroot(friendly_name)
else:
- cros_build_lib.Die('You must specify either --host, --board or --brick')
+ cros_build_lib.Die('You must specify either --host, --board')
helper = workon_helper.WorkonHelper(sysroot, friendly_name)
try:
diff --git a/scripts/gerrit.py b/scripts/gerrit.py
index deefe4d88..0f66bea9f 100644
--- a/scripts/gerrit.py
+++ b/scripts/gerrit.py
@@ -417,7 +417,7 @@ def UserActTopic(opts, topic, *args):
def UserActDeletedraft(opts, *args):
- """Delete draft patch set <n> [n ...]"""
+ """Delete draft CL <n> [n ...]"""
for arg in args:
helper, cl = GetGerrit(opts, arg)
helper.DeleteDraft(cl, dryrun=opts.dryrun)
diff --git a/scripts/parallel_emerge.py b/scripts/parallel_emerge.py
index 176ef34e0..c2199655a 100644
--- a/scripts/parallel_emerge.py
+++ b/scripts/parallel_emerge.py
@@ -1517,6 +1517,7 @@ class EmergeQueue(object):
def _Status(self):
"""Print status."""
current_time = time.time()
+ current_time_struct = time.localtime(current_time)
no_output = True
# Print interim output every minute if --show-output is used. Otherwise,
@@ -1563,7 +1564,9 @@ class EmergeQueue(object):
if retries:
line += "Retrying %s, " % (retries,)
load = " ".join(str(x) for x in os.getloadavg())
- line += ("[Time %dm%.1fs Load %s]" % (seconds / 60, seconds % 60, load))
+ line += ("[Time %s | Elapsed %dm%.1fs | Load %s]" % (
+ time.strftime('%H:%M:%S', current_time_struct), seconds / 60,
+ seconds % 60, load))
self._Print(line)
def _Finish(self, target):
diff --git a/scripts/pushimage.py b/scripts/pushimage.py
index 58c0ca23c..342295f34 100644
--- a/scripts/pushimage.py
+++ b/scripts/pushimage.py
@@ -49,6 +49,8 @@ _SUPPORTED_IMAGE_TYPES = (
constants.IMAGE_TYPE_FACTORY,
constants.IMAGE_TYPE_FIRMWARE,
constants.IMAGE_TYPE_NV_LP0_FIRMWARE,
+ constants.IMAGE_TYPE_ACCESSORY_USBPD,
+ constants.IMAGE_TYPE_ACCESSORY_RWSIG,
constants.IMAGE_TYPE_BASE,
)
@@ -60,6 +62,10 @@ class PushError(Exception):
class MissingBoardInstructions(Exception):
"""Raised when a board lacks any signer instructions."""
+ def __init__(self, board, image_type, input_insns):
+ Exception.__init__(self, 'Board %s lacks insns for %s image: %s not found' %
+ (board, image_type, input_insns))
+
class InputInsns(object):
"""Object to hold settings for a signable board.
@@ -68,18 +74,38 @@ class InputInsns(object):
reads) is not exactly the same as the instruction file pushimage reads.
"""
- def __init__(self, board):
+ def __init__(self, board, image_type=None):
+ """Initialization.
+
+ Args:
+ board: The board to look up details.
+ image_type: The type of image we will be signing (see --sign-types).
+ """
self.board = board
config = ConfigParser.ConfigParser()
config.readfp(open(self.GetInsnFile('DEFAULT')))
+
# What pushimage internally refers to as 'recovery', are the basic signing
# instructions in practice, and other types are stacked on top.
+ if image_type is None:
+ image_type = constants.IMAGE_TYPE_RECOVERY
+ self.image_type = image_type
input_insns = self.GetInsnFile(constants.IMAGE_TYPE_RECOVERY)
if not os.path.exists(input_insns):
# This board doesn't have any signing instructions.
- raise MissingBoardInstructions(self.board)
+ raise MissingBoardInstructions(self.board, image_type, input_insns)
config.readfp(open(input_insns))
+
+ if image_type is not None:
+ input_insns = self.GetInsnFile(image_type)
+ if not os.path.exists(input_insns):
+ # This type doesn't have any signing instructions.
+ raise MissingBoardInstructions(self.board, image_type, input_insns)
+
+ self.image_type = image_type
+ config.readfp(open(input_insns))
+
self.cfg = config
def GetInsnFile(self, image_type):
@@ -125,24 +151,82 @@ class InputInsns(object):
"""
return self.SplitCfgField(self.cfg.get('insns', 'channel'))
- def GetKeysets(self):
- """Return the list of keysets to sign for this board."""
- return self.SplitCfgField(self.cfg.get('insns', 'keyset'))
+ def GetKeysets(self, insns_merge=None):
+ """Return the list of keysets to sign for this board.
+
+ Args:
+ insns_merge: The additional section to look at over [insns].
+ """
+ # First load the default value from [insns.keyset] if available.
+ sections = ['insns']
+ # Then overlay the [insns.xxx.keyset] if requested.
+ if insns_merge is not None:
+ sections += [insns_merge]
+
+ keyset = ''
+ for section in sections:
+ try:
+ keyset = self.cfg.get(section, 'keyset')
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+ pass
+
+ # We do not perturb the order (e.g. using sorted() or making a set())
+ # because we want the behavior stable, and we want the input insns to
+ # explicitly control the order (since it has an impact on naming).
+ return self.SplitCfgField(keyset)
+
+ def GetAltInsnSets(self):
+ """Return the list of alternative insn sections."""
+ # We do not perturb the order (e.g. using sorted() or making a set())
+ # because we want the behavior stable, and we want the input insns to
+ # explicitly control the order (since it has an impact on naming).
+ ret = [x for x in self.cfg.sections() if x.startswith('insns.')]
+ return ret if ret else [None]
+
+ @staticmethod
+ def CopyConfigParser(config):
+ """Return a copy of a ConfigParser object.
+
+ The python guys broke the ability to use something like deepcopy:
+ https://bugs.python.org/issue16058
+ """
+ # Write the current config to a string io object.
+ data = cStringIO.StringIO()
+ config.write(data)
+ data.seek(0)
+
+ # Create a new ConfigParser from the serialized data.
+ ret = ConfigParser.ConfigParser()
+ ret.readfp(data)
+
+ return ret
- def OutputInsns(self, image_type, output_file, sect_insns, sect_general):
+ def OutputInsns(self, output_file, sect_insns, sect_general,
+ insns_merge=None):
"""Generate the output instruction file for sending to the signer.
+ The override order is (later has precedence):
+ [insns]
+ [insns_merge] (should be named "insns.xxx")
+ sect_insns
+
Note: The format of the instruction file pushimage outputs (and the signer
reads) is not exactly the same as the instruction file pushimage reads.
Args:
- image_type: The type of image we will be signing (see --sign-types).
output_file: The file to write the new instruction file to.
sect_insns: Items to set/override in the [insns] section.
sect_general: Items to set/override in the [general] section.
+ insns_merge: The alternative insns.xxx section to merge.
"""
- config = ConfigParser.ConfigParser()
- config.readfp(open(self.GetInsnFile(image_type)))
+ # Create a copy so we can clobber certain fields.
+ config = self.CopyConfigParser(self.cfg)
+ sect_insns = sect_insns.copy()
+
+ # Merge in the alternative insns section if need be.
+ if insns_merge is not None:
+ for k, v in config.items(insns_merge):
+ sect_insns.setdefault(k, v)
# Clear channel entry in instructions file, ensuring we only get
# one channel for the signer to look at. Then provide all the
@@ -154,11 +238,15 @@ class InputInsns(object):
for k, v in fields.iteritems():
config.set(sect, k, v)
+ # Now prune the alternative sections.
+ for alt in self.GetAltInsnSets():
+ config.remove_section(alt)
+
output = cStringIO.StringIO()
config.write(output)
data = output.getvalue()
osutils.WriteFile(output_file, data)
- logging.debug('generated insns file for %s:\n%s', image_type, data)
+ logging.debug('generated insns file for %s:\n%s', self.image_type, data)
def MarkImageToBeSigned(ctx, tbs_base, insns_path, priority):
@@ -244,14 +332,13 @@ def PushImage(src_path, board, versionrev=None, profile=None, priority=50,
try:
input_insns = InputInsns(board)
except MissingBoardInstructions as e:
- logging.warning('board "%s" is missing base instruction file: %s', board, e)
+ logging.warning('Missing base instruction file: %s', e)
logging.warning('not uploading anything for signing')
return
channels = input_insns.GetChannels()
- # We want force_keysets as a set, and keysets as a list.
+ # We want force_keysets as a set.
force_keysets = set(force_keysets)
- keysets = list(force_keysets) if force_keysets else input_insns.GetKeysets()
if mock:
logging.info('Upload mode: mock; signers will not process anything')
@@ -279,7 +366,6 @@ def PushImage(src_path, board, versionrev=None, profile=None, priority=50,
if dry_run:
logging.info('DRY RUN MODE ACTIVE: NOTHING WILL BE UPLOADED')
logging.info('Signing for channels: %s', ' '.join(channels))
- logging.info('Signing for keysets : %s', ' '.join(keysets))
instruction_urls = {}
@@ -303,6 +389,8 @@ def PushImage(src_path, board, versionrev=None, profile=None, priority=50,
firmware_basename = _ImageNameBase(constants.IMAGE_TYPE_FIRMWARE)
nv_lp0_firmware_basename = _ImageNameBase(
constants.IMAGE_TYPE_NV_LP0_FIRMWARE)
+ acc_usbpd_basename = _ImageNameBase(constants.IMAGE_TYPE_ACCESSORY_USBPD)
+ acc_rwsig_basename = _ImageNameBase(constants.IMAGE_TYPE_ACCESSORY_RWSIG)
test_basename = _ImageNameBase(constants.IMAGE_TYPE_TEST)
base_basename = _ImageNameBase(constants.IMAGE_TYPE_BASE)
hwqual_tarball = 'chromeos-hwqual-%s-%s.tar.bz2' % (board, versionrev)
@@ -335,6 +423,12 @@ def PushImage(src_path, board, versionrev=None, profile=None, priority=50,
('firmware_from_source.tar.bz2', nv_lp0_firmware_basename, 'tar.bz2',
constants.IMAGE_TYPE_NV_LP0_FIRMWARE),
+
+ ('firmware_from_source.tar.bz2', acc_usbpd_basename, 'tar.bz2',
+ constants.IMAGE_TYPE_ACCESSORY_USBPD),
+
+ ('firmware_from_source.tar.bz2', acc_rwsig_basename, 'tar.bz2',
+ constants.IMAGE_TYPE_ACCESSORY_RWSIG),
)
# The following build artifacts are copied and marked for signing, if
@@ -387,66 +481,80 @@ def PushImage(src_path, board, versionrev=None, profile=None, priority=50,
logging.debug('Files to sign: %s', files_to_sign)
# Now go through the subset for signing.
- for keyset in keysets:
- logging.debug('\n\n#### KEYSET: %s ####\n', keyset)
- sect_insns['keyset'] = keyset
- for image_type, dst_name, suffix in files_to_sign:
- dst_archive = '%s.%s' % (dst_name, suffix)
- sect_general['archive'] = dst_archive
- sect_general['type'] = image_type
-
- # In the default/automatic mode, only flag files for signing if the
- # archives were actually uploaded in a previous stage. This additional
- # check can be removed in future once |sign_types| becomes a required
- # argument.
- # TODO: Make |sign_types| a required argument.
- gs_artifact_path = os.path.join(dst_path, dst_archive)
- exists = False
- try:
- exists = ctx.Exists(gs_artifact_path)
- except gs.GSContextException:
- unknown_error[0] = True
- logging.error('Unknown error while checking %s', gs_artifact_path,
- exc_info=True)
- if not exists:
- logging.info('%s does not exist. Nothing to sign.',
- gs_artifact_path)
- continue
-
- input_insn_path = input_insns.GetInsnFile(image_type)
- if not os.path.exists(input_insn_path):
- logging.info('%s does not exist. Nothing to sign.', input_insn_path)
- continue
-
- # Generate the insn file for this artifact that the signer will use,
- # and flag it for signing.
- with tempfile.NamedTemporaryFile(
- bufsize=0, prefix='pushimage.insns.') as insns_path:
- input_insns.OutputInsns(image_type, insns_path.name, sect_insns,
- sect_general)
-
- gs_insns_path = '%s/%s' % (dst_path, dst_name)
- if keyset != keysets[0]:
- gs_insns_path += '-%s' % keyset
- gs_insns_path += '.instructions'
-
- try:
- ctx.Copy(insns_path.name, gs_insns_path)
- except gs.GSContextException:
- unknown_error[0] = True
- logging.error('Unknown error while uploading insns %s',
- gs_insns_path, exc_info=True)
- continue
-
- try:
- MarkImageToBeSigned(ctx, tbs_base, gs_insns_path, priority)
- except gs.GSContextException:
- unknown_error[0] = True
- logging.error('Unknown error while marking for signing %s',
- gs_insns_path, exc_info=True)
- continue
- logging.info('Signing %s image %s', image_type, gs_insns_path)
- instruction_urls.setdefault(channel, []).append(gs_insns_path)
+ for image_type, dst_name, suffix in files_to_sign:
+ try:
+ input_insns = InputInsns(board, image_type=image_type)
+ except MissingBoardInstructions as e:
+ logging.info('Nothing to sign: %s', e)
+ continue
+
+ dst_archive = '%s.%s' % (dst_name, suffix)
+ sect_general['archive'] = dst_archive
+ sect_general['type'] = image_type
+
+ # In the default/automatic mode, only flag files for signing if the
+ # archives were actually uploaded in a previous stage. This additional
+ # check can be removed in future once |sign_types| becomes a required
+ # argument.
+ # TODO: Make |sign_types| a required argument.
+ gs_artifact_path = os.path.join(dst_path, dst_archive)
+ exists = False
+ try:
+ exists = ctx.Exists(gs_artifact_path)
+ except gs.GSContextException:
+ unknown_error[0] = True
+ logging.error('Unknown error while checking %s', gs_artifact_path,
+ exc_info=True)
+ if not exists:
+ logging.info('%s does not exist. Nothing to sign.',
+ gs_artifact_path)
+ continue
+
+ first_image = True
+ for alt_insn_set in input_insns.GetAltInsnSets():
+ # Figure out which keysets have been requested for this type.
+ # We sort the forced set so tests/runtime behavior is stable.
+ keysets = sorted(force_keysets)
+ if not keysets:
+ keysets = input_insns.GetKeysets(insns_merge=alt_insn_set)
+ if not keysets:
+ logging.warning('Skipping %s image signing due to no keysets',
+ image_type)
+
+ for keyset in keysets:
+ sect_insns['keyset'] = keyset
+
+ # Generate the insn file for this artifact that the signer will use,
+ # and flag it for signing.
+ with tempfile.NamedTemporaryFile(
+ bufsize=0, prefix='pushimage.insns.') as insns_path:
+ input_insns.OutputInsns(insns_path.name, sect_insns, sect_general,
+ insns_merge=alt_insn_set)
+
+ gs_insns_path = '%s/%s' % (dst_path, dst_name)
+ if not first_image:
+ gs_insns_path += '-%s' % keyset
+ first_image = False
+ gs_insns_path += '.instructions'
+
+ try:
+ ctx.Copy(insns_path.name, gs_insns_path)
+ except gs.GSContextException:
+ unknown_error[0] = True
+ logging.error('Unknown error while uploading insns %s',
+ gs_insns_path, exc_info=True)
+ continue
+
+ try:
+ MarkImageToBeSigned(ctx, tbs_base, gs_insns_path, priority)
+ except gs.GSContextException:
+ unknown_error[0] = True
+ logging.error('Unknown error while marking for signing %s',
+ gs_insns_path, exc_info=True)
+ continue
+ logging.info('Signing %s image with keyset %s at %s', image_type,
+ keyset, gs_insns_path)
+ instruction_urls.setdefault(channel, []).append(gs_insns_path)
if unknown_error[0]:
raise PushError('hit some unknown error(s)', instruction_urls)
diff --git a/scripts/pushimage_unittest.py b/scripts/pushimage_unittest.py
index ae6f6c861..a423143f8 100644
--- a/scripts/pushimage_unittest.py
+++ b/scripts/pushimage_unittest.py
@@ -62,8 +62,8 @@ class InputInsnsTest(cros_test_lib.MockTestCase):
def testOutputInsnsBasic(self):
"""Verify output instructions are sane"""
exp_content = """[insns]
-keyset = stumpy-mp-v3
channel = dev canary
+keyset = stumpy-mp-v3
chromeos_shell = false
ensure_no_password = true
firmware_update = true
@@ -74,8 +74,9 @@ create_nplusone = true
"""
insns = pushimage.InputInsns('test.board')
+ self.assertEqual(insns.GetAltInsnSets(), [None])
m = self.PatchObject(osutils, 'WriteFile')
- insns.OutputInsns('recovery', '/bogus', {}, {})
+ insns.OutputInsns('/bogus', {}, {})
self.assertTrue(m.called)
content = m.call_args_list[0][0][1]
self.assertEqual(content.rstrip(), exp_content.rstrip())
@@ -83,8 +84,8 @@ create_nplusone = true
def testOutputInsnsReplacements(self):
"""Verify output instructions can be updated"""
exp_content = """[insns]
-keyset = batman
channel = dev
+keyset = batman
chromeos_shell = false
ensure_no_password = true
firmware_update = true
@@ -106,11 +107,62 @@ config_board = test.board
insns = pushimage.InputInsns('test.board')
m = self.PatchObject(osutils, 'WriteFile')
- insns.OutputInsns('recovery', '/a/file', sect_insns, sect_general)
+ insns.OutputInsns('/a/file', sect_insns, sect_general)
self.assertTrue(m.called)
content = m.call_args_list[0][0][1]
self.assertEqual(content.rstrip(), exp_content.rstrip())
+ def testOutputInsnsMergeAlts(self):
+ """Verify handling of alternative insns.xxx sections"""
+ TEMPLATE_CONTENT = """[insns]
+channel = %(channel)s
+chromeos_shell = false
+ensure_no_password = true
+firmware_update = true
+security_checks = true
+create_nplusone = true
+override = sect_insns
+keyset = %(keyset)s
+%(extra)s
+[general]
+board = board
+config_board = test.board
+"""
+
+ exp_alts = ['insns.one', 'insns.two', 'insns.hotsoup']
+ exp_fields = {
+ 'one': {'channel': 'dev canary', 'keyset': 'OneKeyset', 'extra': ''},
+ 'two': {'channel': 'best', 'keyset': 'TwoKeyset', 'extra': ''},
+ 'hotsoup': {
+ 'channel': 'dev canary',
+ 'keyset': 'ColdKeyset',
+ 'extra': 'soup = cheddar\n',
+ },
+ }
+
+ # Make sure this overrides the insn sections.
+ sect_insns = {
+ 'override': 'sect_insns',
+ }
+ sect_insns_copy = sect_insns.copy()
+ sect_general = {
+ 'config_board': 'test.board',
+ 'board': 'board',
+ }
+
+ insns = pushimage.InputInsns('test.multi')
+ self.assertEqual(insns.GetAltInsnSets(), exp_alts)
+ m = self.PatchObject(osutils, 'WriteFile')
+
+ for alt in exp_alts:
+ m.reset_mock()
+ insns.OutputInsns('/a/file', sect_insns, sect_general, insns_merge=alt)
+ self.assertEqual(sect_insns, sect_insns_copy)
+ self.assertTrue(m.called)
+ content = m.call_args_list[0][0][1]
+ exp_content = TEMPLATE_CONTENT % exp_fields[alt[6:]]
+ self.assertEqual(content.rstrip(), exp_content.rstrip())
+
class MarkImageToBeSignedTest(gs_unittest.AbstractGSContextTest):
"""Tests for MarkImageToBeSigned()"""
@@ -233,7 +285,7 @@ class PushImageTests(gs_unittest.AbstractGSContextTest):
with mock.patch.object(gs.GSContext, 'Exists', return_value=True):
urls = pushimage.PushImage('/src', 'test.board', 'R34-5126.0.0',
sign_types=['recovery'])
- self.assertEqual(self.gs_mock.call_count, 22)
+ self.assertEqual(self.gs_mock.call_count, 26)
self.assertTrue(self.mark_mock.called)
self.assertEqual(urls, EXPECTED)
@@ -251,7 +303,7 @@ class PushImageTests(gs_unittest.AbstractGSContextTest):
with mock.patch.object(gs.GSContext, 'Exists', return_value=True):
urls = pushimage.PushImage('/src', 'test.board', 'R34-5126.0.0',
sign_types=['base'])
- self.assertEqual(self.gs_mock.call_count, 24)
+ self.assertEqual(self.gs_mock.call_count, 28)
self.assertTrue(self.mark_mock.called)
self.assertEqual(urls, EXPECTED)
@@ -259,7 +311,7 @@ class PushImageTests(gs_unittest.AbstractGSContextTest):
"""Verify nothing is signed when we request an unavailable type"""
urls = pushimage.PushImage('/src', 'test.board', 'R34-5126.0.0',
sign_types=['nononononono'])
- self.assertEqual(self.gs_mock.call_count, 20)
+ self.assertEqual(self.gs_mock.call_count, 24)
self.assertFalse(self.mark_mock.called)
self.assertEqual(urls, {})
@@ -271,6 +323,55 @@ class PushImageTests(gs_unittest.AbstractGSContextTest):
self.assertRaises(pushimage.PushError, pushimage.PushImage, '/src',
'test.board', 'R34-5126.0.0')
+ def testMultipleKeysets(self):
+ """Verify behavior when processing an insn w/multiple keysets"""
+ EXPECTED = {
+ 'canary': [
+ ('gs://chromeos-releases/canary-channel/test.board/5126.0.0/'
+ 'ChromeOS-recovery-R34-5126.0.0-test.board.instructions'),
+ ('gs://chromeos-releases/canary-channel/test.board/5126.0.0/'
+ 'ChromeOS-recovery-R34-5126.0.0-test.board-key2.instructions'),
+ ('gs://chromeos-releases/canary-channel/test.board/5126.0.0/'
+ 'ChromeOS-recovery-R34-5126.0.0-test.board-key3.instructions'),
+ ],
+ 'dev': [
+ ('gs://chromeos-releases/dev-channel/test.board/5126.0.0/'
+ 'ChromeOS-recovery-R34-5126.0.0-test.board.instructions'),
+ ('gs://chromeos-releases/dev-channel/test.board/5126.0.0/'
+ 'ChromeOS-recovery-R34-5126.0.0-test.board-key2.instructions'),
+ ('gs://chromeos-releases/dev-channel/test.board/5126.0.0/'
+ 'ChromeOS-recovery-R34-5126.0.0-test.board-key3.instructions'),
+ ],
+ }
+ with mock.patch.object(gs.GSContext, 'Exists', return_value=True):
+ urls = pushimage.PushImage('/src', 'test.board', 'R34-5126.0.0',
+ force_keysets=('key1', 'key2', 'key3'))
+ self.assertEqual(urls, EXPECTED)
+
+ def testMultipleAltInsns(self):
+ """Verify behavior when processing an insn w/multiple insn overlays"""
+ EXPECTED = {
+ 'canary': [
+ ('gs://chromeos-releases/canary-channel/test.multi/1.0.0/'
+ 'ChromeOS-recovery-R1-1.0.0-test.multi.instructions'),
+ ('gs://chromeos-releases/canary-channel/test.multi/1.0.0/'
+ 'ChromeOS-recovery-R1-1.0.0-test.multi-TwoKeyset.instructions'),
+ ('gs://chromeos-releases/canary-channel/test.multi/1.0.0/'
+ 'ChromeOS-recovery-R1-1.0.0-test.multi-ColdKeyset.instructions'),
+ ],
+ 'dev': [
+ ('gs://chromeos-releases/dev-channel/test.multi/1.0.0/'
+ 'ChromeOS-recovery-R1-1.0.0-test.multi.instructions'),
+ ('gs://chromeos-releases/dev-channel/test.multi/1.0.0/'
+ 'ChromeOS-recovery-R1-1.0.0-test.multi-TwoKeyset.instructions'),
+ ('gs://chromeos-releases/dev-channel/test.multi/1.0.0/'
+ 'ChromeOS-recovery-R1-1.0.0-test.multi-ColdKeyset.instructions'),
+ ],
+ }
+ with mock.patch.object(gs.GSContext, 'Exists', return_value=True):
+ urls = pushimage.PushImage('/src', 'test.multi', 'R1-1.0.0')
+ self.assertEqual(urls, EXPECTED)
+
class MainTests(cros_test_lib.MockTestCase):
"""Tests for main()"""
@@ -289,4 +390,4 @@ def main(_argv):
signing.INPUT_INSN_DIR = signing.TEST_INPUT_INSN_DIR
# Run the tests.
- cros_test_lib.main(level='info', module=__name__)
+ cros_test_lib.main(level='notice', module=__name__)
diff --git a/scripts/summarize_build_stats.py b/scripts/summarize_build_stats.py
index 404a18d3e..a4a965af4 100644
--- a/scripts/summarize_build_stats.py
+++ b/scripts/summarize_build_stats.py
@@ -7,6 +7,7 @@
from __future__ import print_function
import datetime
+import itertools
import numpy
import re
import sys
@@ -14,6 +15,7 @@ import sys
from chromite.cbuildbot import constants
from chromite.lib import cidb
from chromite.lib import commandline
+from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
@@ -36,6 +38,8 @@ class CLStatsEngine(object):
self.blames = {}
self.summary = {}
self.builds_by_build_id = {}
+ self.slave_builds_by_master_id = {}
+ self.slave_builds_by_config = {}
def GatherBuildAnnotations(self):
"""Gather the failure annotations for builds from cidb."""
@@ -158,6 +162,16 @@ class CLStatsEngine(object):
self.builds_by_build_id.update(
{b['id'] : b for b in self.builds})
+ # Gather slave statuses for each of the master builds. For now this is a
+ # separate query per CQ run, but this could be consolidated to a single
+ # query if necessary (requires adding a cidb.py API method).
+ for bid in self.builds_by_build_id:
+ self.slave_builds_by_master_id[bid] = self.db.GetSlaveStatuses(bid)
+
+ self.slave_builds_by_config = cros_build_lib.GroupByKey(
+ itertools.chain(*self.slave_builds_by_master_id.values()),
+ 'build_config')
+
def _PrintCounts(self, reasons, fmt):
"""Print a sorted list of reasons in descending order of frequency.
@@ -199,7 +213,23 @@ class CLStatsEngine(object):
)
return false_rejection_rate
- def Summarize(self, build_type):
+ def GetBuildRunTimes(self, builds):
+ """Gets the elapsed run times of the completed builds within |builds|.
+
+ Args:
+ builds: Iterable of build statuses as returned by cidb.
+
+ Returns:
+ A list of the elapsed times (in seconds) of the builds that completed.
+ """
+ times = []
+ for b in builds:
+ if b['finish_time']:
+ td = (b['finish_time'] - b['start_time']).total_seconds()
+ times.append(td)
+ return times
+
+ def Summarize(self, build_type, bad_patch_candidates=False):
"""Process, print, and return a summary of statistics.
As a side effect, save summary to self.summary.
@@ -208,11 +238,11 @@ class CLStatsEngine(object):
A dictionary summarizing the statistics.
"""
if build_type == 'cq':
- return self.SummarizeCQ()
+ return self.SummarizeCQ(bad_patch_candidates=bad_patch_candidates)
else:
return self.SummarizePFQ()
- def SummarizeCQ(self):
+ def SummarizeCQ(self, bad_patch_candidates=False):
"""Process, print, and return a summary of cl action statistics.
As a side effect, save summary to self.summary.
@@ -230,6 +260,8 @@ class CLStatsEngine(object):
else:
logging.info('No runs included.')
+ build_times_sec = sorted(self.GetBuildRunTimes(self.builds))
+
build_reason_counts = {}
for reasons in self.reasons.values():
for reason in reasons:
@@ -237,8 +269,11 @@ class CLStatsEngine(object):
build_reason_counts[reason] = build_reason_counts.get(reason, 0) + 1
unique_blames = set()
+ build_blame_counts = {}
for blames in self.blames.itervalues():
unique_blames.update(blames)
+ for blame in blames:
+ build_blame_counts[blame] = build_blame_counts.get(blame, 0) + 1
unique_cl_blames = {blame for blame in unique_blames if
EXTERNAL_CL_BASE_URL in blame}
@@ -290,6 +325,23 @@ class CLStatsEngine(object):
for x in range(max(rejection_counts) + 1):
good_patch_rejection_breakdown.append((x, rejection_counts.count(x)))
+ # For CQ runs that passed, track which slave was the long pole, i.e. the
+ # last to finish.
+ long_pole_slave_counts = {}
+ for bid, master_build in self.builds_by_build_id.items():
+ if master_build['status'] == constants.BUILDER_STATUS_PASSED:
+ if not self.slave_builds_by_master_id[bid]:
+ continue
+ # TODO(akeshet): The set of slaves also includes non-important slaves
+ # (there is no distinction in cidb between important and non-important).
+ # To protect max(...) from hitting any None values we need the if check
+ # below. Revisit this once we can filter out non-important slaves.
+ _, long_config = max((slave['finish_time'], slave['build_config'])
+ for slave in self.slave_builds_by_master_id[bid]
+ if slave['finish_time'])
+ long_pole_slave_counts[long_config] = (
+ long_pole_slave_counts.get(long_config, 0) + 1)
+
summary = {
'total_cl_actions': len(self.claction_history),
'unique_cls': len(self.claction_history.affected_cls),
@@ -306,6 +358,7 @@ class CLStatsEngine(object):
'patch_handling_time': patch_handle_times,
'bad_cl_candidates': bad_cl_candidates,
'unique_blames_change_count': len(unique_cl_blames),
+ 'long_pole_slave_counts': long_pole_slave_counts,
}
logging.info('CQ committed %s changes', summary['submitted_patches'])
@@ -388,11 +441,19 @@ class CLStatsEngine(object):
numpy.percentile(cq_handle_times, 90) / 3600.0)
logging.info('')
+ # Log some statistics about cq-master run-time.
+ logging.info('CQ-master run time:')
+ logging.info(' 50th percentile: %.2f hours',
+ numpy.percentile(build_times_sec, 50) / 3600.0)
+ logging.info(' 90th percenfile: %.2f hours',
+ numpy.percentile(build_times_sec, 90) / 3600.0)
+
for bot_type, patches in summary['bad_cl_candidates'].items():
logging.info('%d bad patch candidates were rejected by the %s',
len(patches), bot_type)
- for k in patches:
- logging.info('Bad patch candidate in: %s', k)
+ if bad_patch_candidates:
+ for k in patches:
+ logging.info('Bad patch candidate in: %s', k)
fmt_fai = ' %(cnt)d failures in %(reason)s'
fmt_rej = ' %(cnt)d rejections due to %(reason)s'
@@ -406,6 +467,21 @@ class CLStatsEngine(object):
logging.info('Reasons why builds failed:')
self._PrintCounts(build_reason_counts, fmt_fai)
+ logging.info('Bugs or CLs responsible for build failures:')
+ self._PrintCounts(build_blame_counts, fmt_fai)
+
+ total_counts = sum(long_pole_slave_counts.values())
+ logging.info('Slowest CQ slaves out of %s passing runs:', total_counts)
+ for (count, config) in sorted(
+ (v, k) for (k, v) in long_pole_slave_counts.items()):
+ if count < (total_counts / 20.0):
+ continue
+ build_times = self.GetBuildRunTimes(self.slave_builds_by_config[config])
+ logging.info('%s times the slowest slave was %s', count, config)
+ logging.info(' 50th percentile: %.2f hours, 90th percentile: %.2f hours',
+ numpy.percentile(build_times, 50) / 3600.0,
+ numpy.percentile(build_times, 90) / 3600.0)
+
return summary
# TODO(akeshet): some of this logic is copied directly from SummarizeCQ.
@@ -490,6 +566,10 @@ def GetParser():
parser.add_argument('--build-type', choices=['cq', 'chrome-pfq'],
default='cq',
help='Build type to summarize. Default: cq.')
+ parser.add_argument('--bad-patch-candidates', action='store_true',
+ default=False,
+ help='In CQ mode, whether to print bad patch '
+ 'candidates.')
return parser
@@ -527,4 +607,5 @@ def main(argv):
cl_stats_engine = CLStatsEngine(db)
cl_stats_engine.Gather(start_date, end_date, master_config,
starting_build_number=options.starting_build)
- cl_stats_engine.Summarize(options.build_type)
+ cl_stats_engine.Summarize(options.build_type,
+ options.bad_patch_candidates)
diff --git a/scripts/summarize_build_stats_unittest.py b/scripts/summarize_build_stats_unittest.py
index c6a417be1..f6442af1c 100644
--- a/scripts/summarize_build_stats_unittest.py
+++ b/scripts/summarize_build_stats_unittest.py
@@ -177,6 +177,7 @@ class TestCLActionLogic(cros_test_lib.TestCase):
'good_patch_rejection_count': {CQ: 1, PRE_CQ: 1},
'good_patch_rejections': 2,
'false_rejection_rate': {CQ: 20., PRE_CQ: 20., 'combined': 100. / 3},
+ 'long_pole_slave_counts': {},
'submitted_patches': 4,
'submit_fails': 0,
'unique_cls': 4,
diff --git a/scripts/upload_symbols.py b/scripts/upload_symbols.py
index 2a912a2f1..8e94e8064 100644
--- a/scripts/upload_symbols.py
+++ b/scripts/upload_symbols.py
@@ -91,8 +91,8 @@ DEDUPE_NOTIFY_TIMEOUT = 20
# The unique namespace in the dedupe server that only we use. Helps avoid
# collisions with all the hashed values and unrelated content.
-OFFICIAL_DEDUPE_NAMESPACE = 'chromium-os-upload-symbols'
-STAGING_DEDUPE_NAMESPACE = '%s-staging' % OFFICIAL_DEDUPE_NAMESPACE
+OFFICIAL_DEDUPE_NAMESPACE_TMPL = '%s-upload-symbols'
+STAGING_DEDUPE_NAMESPACE_TMPL = '%s-staging' % OFFICIAL_DEDUPE_NAMESPACE_TMPL
# The minimum average rate (in bytes per second) that we expect to maintain
@@ -167,7 +167,7 @@ def GetUploadTimeout(path):
return max(os.path.getsize(path) / UPLOAD_MIN_RATE, UPLOAD_MIN_TIMEOUT)
-def SymUpload(upload_url, sym_item):
+def SymUpload(upload_url, sym_item, product_name):
"""Upload a symbol file to a HTTP server
The upload is a multipart/form-data POST with the following parameters:
@@ -186,6 +186,7 @@ def SymUpload(upload_url, sym_item):
Args:
upload_url: The crash URL to POST the |sym_file| to
sym_item: A SymbolItem containing the path to the breakpad symbol to upload
+ product_name: A string for stats purposes. Usually 'ChromeOS' or 'Android'.
"""
sym_header = sym_item.sym_header
sym_file = sym_item.sym_file
@@ -202,7 +203,7 @@ def SymUpload(upload_url, sym_item):
# Not sure what to set for the version. Maybe the git sha1 of this file.
# Note: the server restricts this to 30 chars.
#('version', None),
- ('product', 'ChromeOS'),
+ ('product', product_name),
('os', sym_header.os),
('cpu', sym_header.cpu),
poster.encode.MultipartParam.from_file('symbol_file', sym_file),
@@ -214,7 +215,7 @@ def SymUpload(upload_url, sym_item):
urllib2.urlopen(request, timeout=GetUploadTimeout(sym_file))
-def TestingSymUpload(upload_url, sym_item):
+def TestingSymUpload(upload_url, sym_item, _product_name):
"""A stub version of SymUpload for --testing usage"""
cmd = ['sym_upload', sym_item.sym_file, upload_url]
# Randomly fail 80% of the time (the retry logic makes this 80%/3 per file).
@@ -277,9 +278,9 @@ def _UpdateCounter(counter, adj):
_Update()
-def UploadSymbol(upload_url, symbol_element, file_limit=DEFAULT_FILE_LIMIT,
- sleep=0, num_errors=None, watermark_errors=None,
- failed_queue=None, passed_queue=None):
+def UploadSymbol(upload_url, symbol_element, product_name,
+ file_limit=DEFAULT_FILE_LIMIT, sleep=0, num_errors=None,
+ watermark_errors=None, failed_queue=None, passed_queue=None):
"""Upload |sym_element.symbol_item| to |upload_url|
Args:
@@ -289,6 +290,7 @@ def UploadSymbol(upload_url, symbol_element, file_limit=DEFAULT_FILE_LIMIT,
to upload. symbol_element.opaque_push_state is an object of
_IsolateServerPushState or None if the item doesn't have
a push state.
+ product_name: A string for stats purposes. Usually 'ChromeOS' or 'Android'.
file_limit: The max file size of a symbol file before we try to strip it
sleep: Number of seconds to sleep before running
num_errors: An object to update with the error count (needs a .value member)
@@ -349,7 +351,7 @@ def UploadSymbol(upload_url, symbol_element, file_limit=DEFAULT_FILE_LIMIT,
cros_build_lib.TimedCommand(
retry_util.RetryException,
(urllib2.HTTPError, urllib2.URLError), MAX_RETRIES, SymUpload,
- upload_url, upload_item, sleep=INITIAL_RETRY_DELAY,
+ upload_url, upload_item, product_name, sleep=INITIAL_RETRY_DELAY,
timed_log_msg=('upload of %10i bytes took %%(delta)s: %s' %
(file_size, os.path.basename(sym_file))))
success = True
@@ -573,7 +575,8 @@ def WriteQueueToFile(listing, queue, relpath=None):
def UploadSymbols(board=None, official=False, server=None, breakpad_dir=None,
file_limit=DEFAULT_FILE_LIMIT, sleep=DEFAULT_SLEEP_DELAY,
upload_limit=None, sym_paths=None, failed_list=None,
- root=None, retry=True, dedupe_namespace=None):
+ root=None, retry=True, dedupe_namespace=None,
+ product_name='ChromeOS'):
"""Upload all the generated symbols for |board| to the crash server
You can use in a few ways:
@@ -596,6 +599,7 @@ def UploadSymbols(board=None, official=False, server=None, breakpad_dir=None,
root: The tree to prefix to |breakpad_dir| (if |breakpad_dir| is not set)
retry: Whether we should retry failures.
dedupe_namespace: The isolateserver namespace to dedupe uploaded symbols.
+ product_name: A string for stats purposes. Usually 'ChromeOS' or 'Android'.
Returns:
The number of errors that were encountered.
@@ -650,9 +654,10 @@ def UploadSymbols(board=None, official=False, server=None, breakpad_dir=None,
watermark_errors = multiprocessing.Value('f')
failed_queue = multiprocessing.Queue()
uploader = functools.partial(
- UploadSymbol, upload_url, file_limit=file_limit, sleep=sleep,
- num_errors=bg_errors, watermark_errors=watermark_errors,
- failed_queue=failed_queue, passed_queue=dedupe_queue)
+ UploadSymbol, upload_url, product_name=product_name,
+ file_limit=file_limit, sleep=sleep, num_errors=bg_errors,
+ watermark_errors=watermark_errors, failed_queue=failed_queue,
+ passed_queue=dedupe_queue)
start_time = datetime.datetime.now()
Counters = cros_build_lib.Collection(
@@ -804,7 +809,7 @@ def main(argv):
parser.add_argument('sym_paths', type='path_or_uri', nargs='*', default=None,
help='symbol file or directory or URL or tarball')
parser.add_argument('--board', default=None,
- help='board to build packages for')
+ help='Used to find default breakpad_root.')
parser.add_argument('--breakpad_root', type='path', default=None,
help='full path to the breakpad symbol directory')
parser.add_argument('--root', type='path', default=None,
@@ -828,20 +833,20 @@ def main(argv):
help='run in testing mode')
parser.add_argument('--yes', action='store_true', default=False,
help='answer yes to all prompts')
+ parser.add_argument('--product_name', type=str, default='ChromeOS',
+ help='Produce Name for breakpad stats.')
opts = parser.parse_args(argv)
opts.Freeze()
- if opts.sym_paths:
+ if opts.sym_paths or opts.breakpad_root:
if opts.regenerate:
- cros_build_lib.Die('--regenerate may not be used with specific files')
+ cros_build_lib.Die('--regenerate may not be used with specific files, '
+ 'or breakpad_root')
else:
if opts.board is None:
cros_build_lib.Die('--board is required')
- if opts.breakpad_root and opts.regenerate:
- cros_build_lib.Die('--regenerate may not be used with --breakpad_root')
-
if opts.testing:
# TODO(build): Kill off --testing mode once unittests are up-to-snuff.
logging.info('running in testing mode')
@@ -853,9 +858,9 @@ def main(argv):
dedupe_namespace = None
if opts.dedupe:
if opts.official_build and not opts.testing:
- dedupe_namespace = OFFICIAL_DEDUPE_NAMESPACE
+ dedupe_namespace = OFFICIAL_DEDUPE_NAMESPACE_TMPL % opts.product_name
else:
- dedupe_namespace = STAGING_DEDUPE_NAMESPACE
+ dedupe_namespace = STAGING_DEDUPE_NAMESPACE_TMPL % opts.product_name
if not opts.yes:
prolog = '\n'.join(textwrap.wrap(textwrap.dedent("""
@@ -880,7 +885,8 @@ def main(argv):
file_limit=opts.strip_cfi, sleep=DEFAULT_SLEEP_DELAY,
upload_limit=opts.upload_limit, sym_paths=opts.sym_paths,
failed_list=opts.failed_list, root=opts.root,
- dedupe_namespace=dedupe_namespace)
+ dedupe_namespace=dedupe_namespace,
+ product_name=opts.product_name)
if ret:
logging.error('encountered %i problem(s)', ret)
# Since exit(status) gets masked, clamp it to 1 so we don't inadvertently
diff --git a/scripts/upload_symbols_unittest.py b/scripts/upload_symbols_unittest.py
index c8f3eb428..62dd0d9ba 100644
--- a/scripts/upload_symbols_unittest.py
+++ b/scripts/upload_symbols_unittest.py
@@ -380,9 +380,9 @@ class UploadSymbolTest(cros_test_lib.MockTempDirTestCase):
"""Verify we try to upload on a normal file"""
osutils.Touch(self.sym_file)
sym_element = upload_symbols.SymbolElement(self.sym_item, None)
- ret = upload_symbols.UploadSymbol(self.url, sym_element)
+ ret = upload_symbols.UploadSymbol(self.url, sym_element, 'TestProduct')
self.assertEqual(ret, 0)
- self.upload_mock.assert_called_with(self.url, self.sym_item)
+ self.upload_mock.assert_called_with(self.url, self.sym_item, 'TestProduct')
self.assertEqual(self.upload_mock.call_count, 1)
def testUploadSymbolErrorCountExceeded(self):
@@ -390,8 +390,8 @@ class UploadSymbolTest(cros_test_lib.MockTempDirTestCase):
errors = ctypes.c_int(10000)
# Pass in garbage values so that we crash if num_errors isn't handled.
ret = upload_symbols.UploadSymbol(
- None, upload_symbols.SymbolElement(self.sym_item, None), sleep=None,
- num_errors=errors)
+ None, upload_symbols.SymbolElement(self.sym_item, None), 'TestProduct',
+ sleep=None, num_errors=errors)
self.assertEqual(ret, 0)
def testUploadRetryErrors(self, side_effect=None):
@@ -402,9 +402,10 @@ class UploadSymbolTest(cros_test_lib.MockTempDirTestCase):
errors = ctypes.c_int()
item = upload_symbols.FakeItem(sym_file='/dev/null')
element = upload_symbols.SymbolElement(item, None)
- ret = upload_symbols.UploadSymbol(self.url, element, num_errors=errors)
+ ret = upload_symbols.UploadSymbol(self.url, element, 'TestProduct',
+ num_errors=errors)
self.assertEqual(ret, 1)
- self.upload_mock.assert_called_with(self.url, item)
+ self.upload_mock.assert_called_with(self.url, item, 'TestProduct')
self.assertTrue(self.upload_mock.call_count >= upload_symbols.MAX_RETRIES)
def testConnectRetryErrors(self):
@@ -426,7 +427,7 @@ class UploadSymbolTest(cros_test_lib.MockTempDirTestCase):
osutils.WriteFile(self.sym_file, content)
ret = upload_symbols.UploadSymbol(
self.url, upload_symbols.SymbolElement(self.sym_item, None),
- file_limit=1)
+ 'TestProduct', file_limit=1)
self.assertEqual(ret, 0)
# Make sure the item passed to the upload has a temp file and not the
# original -- only the temp one has been stripped down.
@@ -443,7 +444,8 @@ class UploadSymbolTest(cros_test_lib.MockTempDirTestCase):
f.write('STACK CFI 1234\n\n')
ret = upload_symbols.UploadSymbol(
self.url,
- upload_symbols.SymbolElement(self.sym_item, None))
+ upload_symbols.SymbolElement(self.sym_item, None),
+ 'TestProduct')
self.assertEqual(ret, 0)
# Make sure the item passed to the upload has a temp file and not the
# original -- only the temp one has been truncated.
@@ -468,7 +470,7 @@ PUBLIC 1471 0 main"""
def testPostUpload(self):
"""Verify HTTP POST has all the fields we need"""
m = self.PatchObject(urllib2, 'urlopen', autospec=True)
- upload_symbols.SymUpload(self.SYM_URL, self.sym_item)
+ upload_symbols.SymUpload(self.SYM_URL, self.sym_item, 'TestProduct')
self.assertEquals(m.call_count, 1)
req = m.call_args[0][0]
self.assertEquals(req.get_full_url(), self.SYM_URL)
@@ -503,7 +505,7 @@ PUBLIC 1471 0 main"""
(50 * 1024 * 1024, 257),
)
for size.return_value, timeout in tests:
- upload_symbols.SymUpload(self.SYM_URL, self.sym_item)
+ upload_symbols.SymUpload(self.SYM_URL, self.sym_item, 'TestProduct')
self.assertEqual(m.call_args[1]['timeout'], timeout)
diff --git a/scripts/wrapper.py b/scripts/wrapper.py
index 2715ad8ff..8153a8578 100755
--- a/scripts/wrapper.py
+++ b/scripts/wrapper.py
@@ -12,6 +12,7 @@ lots of places.
from __future__ import print_function
+import imp
import os
import sys
@@ -96,6 +97,8 @@ def FindTarget(target):
$ ln -s $PWD/cbuildbot/cbuildbot ~/bin/; cbuildbot --help
# No $PATH needed, but a relative symlink to a symlink to the chromite dir.
$ cd ~; ln -s bin/cbuildbot ./; ./cbuildbot --help
+ # External chromite module
+ $ ln -s ../chromite/scripts/wrapper.py foo; ./foo
Args:
target: Path to the script we're trying to run.
@@ -103,6 +106,8 @@ def FindTarget(target):
Returns:
The module main functor.
"""
+ # We assume/require the script we're wrapping ends in a .py.
+ full_path = target + '.py'
while True:
# Walk back one symlink at a time until we get into the chromite dir.
parent, base = os.path.split(target)
@@ -111,21 +116,38 @@ def FindTarget(target):
target = base
break
target = os.path.join(os.path.dirname(target), os.readlink(target))
- assert parent.startswith(CHROMITE_PATH), (
- 'could not figure out leading path\n'
- '\tparent: %s\n'
- '\tCHROMITE_PATH: %s' % (parent, CHROMITE_PATH))
- parent = parent[len(CHROMITE_PATH):].split(os.sep)
- target = ['chromite'] + parent + [target]
-
- if target[-2] == 'bin':
- # Convert <path>/bin/foo -> <path>/scripts/foo.
- target[-2] = 'scripts'
- elif target[1] == 'bootstrap' and len(target) == 3:
- # Convert <git_repo>/bootstrap/foo -> <git_repo>/bootstrap/scripts/foo.
- target.insert(2, 'scripts')
-
- module = cros_import.ImportModule(target)
+
+ # If we walked all the way back to wrapper.py, it means we're trying to run
+ # an external module. So we have to import it by filepath and not via the
+ # chromite.xxx.yyy namespace.
+ if target != 'wrapper.py':
+ assert parent.startswith(CHROMITE_PATH), (
+ 'could not figure out leading path\n'
+ '\tparent: %s\n'
+ '\tCHROMITE_PATH: %s' % (parent, CHROMITE_PATH))
+ parent = parent[len(CHROMITE_PATH):].split(os.sep)
+ target = ['chromite'] + parent + [target]
+
+ if target[-2] == 'bin':
+ # Convert <path>/bin/foo -> <path>/scripts/foo.
+ target[-2] = 'scripts'
+ elif target[1] == 'bootstrap' and len(target) == 3:
+ # Convert <git_repo>/bootstrap/foo -> <git_repo>/bootstrap/scripts/foo.
+ target.insert(2, 'scripts')
+
+ try:
+ module = cros_import.ImportModule(target)
+ except ImportError as e:
+ print('%s: could not import chromite module: %s: %s'
+ % (sys.argv[0], full_path, e), file=sys.stderr)
+ sys.exit(1)
+ else:
+ try:
+ module = imp.load_source('main', full_path)
+ except IOError as e:
+ print('%s: could not import external module: %s: %s'
+ % (sys.argv[0], full_path, e), file=sys.stderr)
+ sys.exit(1)
# Run the module's main func if it has one.
main = getattr(module, 'main', None)
diff --git a/signing/signer_instructions/README b/signing/signer_instructions/README
new file mode 100644
index 000000000..e7f90f124
--- /dev/null
+++ b/signing/signer_instructions/README
@@ -0,0 +1,76 @@
+=== PREFACE ===
+NOTE: The files in chromite/ are currently only used for testing. The actual
+files used by releases live in crostools/signer_instructions/. The program
+managers would prefer to keep them internal for now.
+
+=== OVERVIEW ===
+This directory holds instruction files that are used when uploading files for
+signing with official keys. The pushimage script will process them to create
+output instruction files which are then posted to a Google Storage bucket that
+the signing processes watch. The input files tell pushimage how to operate,
+and output files tell the signer how to operate.
+
+This file covers things that pushimage itself cares about. It does not get into
+the fields that the signer utilizes. See REFERENCES below for that.
+
+=== FILES ===
+DEFAULT.instructions - default values for all boards/artifacts; loaded first
+DEFAULT.$TYPE.instructions - default values for all boards for a specific type
+$BOARD.instructions - default values for all artifacts for $BOARD, and used for
+ recovery images
+$BOARD.$TYPE.instructions - values specific to a board and artifact type; see
+ the --sign-types argument to pushimage
+
+=== FORMAT ===
+There are a few main sections that pushimage cares about:
+[insns]
+[insns.XXX] (Where XXX can be anything)
+[general]
+
+Other sections are passed through to the signer untouched, and many fields in
+the above sections are also unmodified.
+
+The keys that pushimage looks at are:
+[insns]
+channels = comma/space delimited list of the channels to flag for signing
+keysets = comma/space delimited list of the keysets to use when signing
+
+A bunch of fields will also be clobbered in the [general] section as pushimage
+writes out metadata based on the command line flags/artifacts.
+
+=== MULTI CHANNEL/KEYSET ===
+When you want to sign a single board/artifact type for multiple channels or
+keysets, simply list them in insns.channels and insn.keysets. The pushimage
+script will take care of posting to the right subdirs and creating unique
+filenames based on those.
+
+=== MULTI INPUTS ===
+When you want to sign multiple artifacts for a single board (and all the same
+artifact type), you need to use the multiple input form instead. When you
+create multiple sections that start with "insns.", pushimage will overlay that
+on top of the insns section, and then produce multiple ouput requests.
+
+So if you wrote a file like:
+ [insns]
+ channel = dev
+ [insns.one]
+ keyset = Zinger
+ input_files = zinger/ec.bin
+ [insns.two]
+ keyset = Hoho
+ input_files = hoho/ec.bin
+
+Pushimage will produce two requests for the signer:
+ [insns]
+ channel = dev
+ keyset = Zinger
+ input_files = zinger/ec.bin
+And:
+ [insns]
+ channel = dev
+ keyset = Hoho
+ input_files = hoho/ec.bin
+
+=== REFERENCES ===
+For details on the fields that the signer uses:
+https://sites.google.com/a/google.com/chromeos/resources/engineering/releng/signer-documentation
diff --git a/signing/signer_instructions/test.multi.instructions b/signing/signer_instructions/test.multi.instructions
new file mode 100644
index 000000000..2ac94a1c0
--- /dev/null
+++ b/signing/signer_instructions/test.multi.instructions
@@ -0,0 +1,20 @@
+[insns]
+channel = dev canary
+chromeos_shell = false
+ensure_no_password = true
+firmware_update = true
+security_checks = true
+create_nplusone = true
+override = base
+
+[insns.one]
+keyset = OneKeyset
+override = alt
+
+[insns.two]
+keyset = TwoKeyset
+channel = best
+
+[insns.hotsoup]
+keyset = ColdKeyset
+soup = cheddar