diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2024-02-26 04:32:29 +0000 |
---|---|---|
committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2024-02-26 04:32:29 +0000 |
commit | 4d6da8e0b8b5555e544c7898e7e68e6736db482d (patch) | |
tree | 08d8f4c3a1bd8d18488991e5cbcae075c7559479 | |
parent | f29e2417ac1123b2eeaf8a684150a05a12334b02 (diff) | |
parent | 6ac52c2f183afff8648f41be4ffc2644c2e87178 (diff) | |
download | build-4d6da8e0b8b5555e544c7898e7e68e6736db482d.tar.gz |
Snap for 11484144 from 6ac52c2f183afff8648f41be4ffc2644c2e87178 to common-android14-6.1-2023-05-exp-release
Change-Id: I8b3a62a1bf06b2ee98df4bbedc0ff8a403b86d4d
Signed-off-by: Coastguard Worker <android-build-coastguard-worker@google.com>
23 files changed, 488 insertions, 82 deletions
diff --git a/build_utils.sh b/build_utils.sh index 10e8077c..6f023615 100644 --- a/build_utils.sh +++ b/build_utils.sh @@ -112,8 +112,8 @@ function create_modules_order_lists() { declare -A module_lists_arr module_lists_arr["modules.order"]=${modules_list_file} - module_lists_arr["modules.order.recovery"]=${modules_recoverylist_file} - module_lists_arr["modules.order.charger"]=${modules_chargerlist_file} + module_lists_arr["modules.order.recovery"]=${modules_recovery_list_file} + module_lists_arr["modules.order.charger"]=${modules_charger_list_file} for mod_order_file in ${!module_lists_arr[@]}; do local mod_list_file=${module_lists_arr[${mod_order_file}]} @@ -180,8 +180,8 @@ function create_modules_staging() { local dest_dir=$3/lib/modules/${version} local dest_stage=$3 local modules_blocklist_file=$4 - local modules_recoverylist_file=$5 - local modules_chargerlist_file=$6 + local modules_recovery_list_file=$5 + local modules_charger_list_file=$6 local depmod_flags=$7 rm -rf ${dest_dir} @@ -193,7 +193,11 @@ function create_modules_staging() { cp ${src_dir}/modules.builtin ${dest_dir}/modules.builtin cp ${src_dir}/modules.builtin.modinfo ${dest_dir}/modules.builtin.modinfo - if [[ -n "${EXT_MODULES}" ]] || [[ -n "${EXT_MODULES_MAKEFILE}" ]]; then + if [[ -n "${KLEAF_MODULES_ORDER}" ]] && [[ -d "${src_dir}/extra" ]]; then + mkdir -p ${dest_dir}/extra/ + cp -r ${src_dir}/extra/* ${dest_dir}/extra/ + cat ${KLEAF_MODULES_ORDER} >> ${dest_dir}/modules.order + elif [[ -n "${EXT_MODULES}" ]] || [[ -n "${EXT_MODULES_MAKEFILE}" ]]; then mkdir -p ${dest_dir}/extra/ cp -r ${src_dir}/extra/* ${dest_dir}/extra/ @@ -283,17 +287,21 @@ function create_modules_staging() { # Re-run depmod to detect any dependencies between in-kernel and external # modules, as well as recovery and charger modules. Then, create the # modules.order files based on all the modules compiled. - declare -A module_load_lists_arr - module_load_lists_arr["modules.order"]="modules.load" - module_load_lists_arr["modules.order.recovery"]="modules.load.recovery" - module_load_lists_arr["modules.order.charger"]="modules.load.charger" - - for mod_order_file in ${!module_load_lists_arr[@]}; do - local mod_order_filepath=${dest_dir}/${mod_order_file} - local mod_load_filepath=${dest_dir}/${module_load_lists_arr[${mod_order_file}]} + # + # It is important that "modules.order" is last, as that will force depmod + # to run on all the modules in the directory, instead of just a list of them. + # It is desirable for depmod to run with all the modules last so that the + # dependency information is available for all modules, not just the recovery + # or charger sets. + modules_order_files=("modules.order.recovery" "modules.order.charger" "modules.order") + modules_load_files=("modules.load.recovery" "modules.load.charger" "modules.load") + + for i in ${!modules_order_files[@]}; do + local mod_order_filepath=${dest_dir}/${modules_order_files[$i]} + local mod_load_filepath=${dest_dir}/${modules_load_files[$i]} if [[ -f ${mod_order_filepath} ]]; then - if [[ "${mod_order_file}" == "modules.order" ]]; then + if [[ "${modules_order_files[$i]}" == "modules.order" ]]; then run_depmod ${dest_stage} "${depmod_flags}" "${version}" else run_depmod ${dest_stage} "${depmod_flags}" "${version}" "${mod_order_filepath}" @@ -308,9 +316,12 @@ function build_system_dlkm() { echo " Creating system_dlkm image" rm -rf ${SYSTEM_DLKM_STAGING_DIR} + # MODULES_[RECOVERY_LIST|CHARGER]_LIST should not influence system_dlkm, as + # GKI modules are not loaded when booting into either recovery or charger + # modes, so do not consider them, and pass empty strings instead. create_modules_staging "${SYSTEM_DLKM_MODULES_LIST:-${MODULES_LIST}}" "${MODULES_STAGING_DIR}" \ ${SYSTEM_DLKM_STAGING_DIR} "${SYSTEM_DLKM_MODULES_BLOCKLIST:-${MODULES_BLOCKLIST}}" \ - "${MODULES_RECOVERY_LIST:-""}" "${MODULES_CHARGER_LIST:-""}" "-e" + "" "" "-e" local system_dlkm_root_dir=$(echo ${SYSTEM_DLKM_STAGING_DIR}/lib/modules/*) cp ${system_dlkm_root_dir}/modules.load ${DIST_DIR}/system_dlkm.modules.load diff --git a/gki/download_from_ci b/gki/download_from_ci index 07ef9743..7d10ddb2 100755 --- a/gki/download_from_ci +++ b/gki/download_from_ci @@ -319,6 +319,7 @@ def clean_download_dir(download_dir): os.makedirs(download_dir, exist_ok=True) subprocess.run(["git", "checkout", "HEAD", "Android.bp"], cwd=download_dir) subprocess.run(["git", "checkout", "HEAD", "16k"], cwd=download_dir) + subprocess.run(["git", "checkout", "HEAD", "microdroid-gki"], cwd=download_dir) def update_android_4_1x_pq(output_dir, version): output_dir = os.path.join(output_dir, "device", "google", "cuttlefish_kernel") diff --git a/kleaf/artifact_tests/initramfs_modules_lists_test.py b/kleaf/artifact_tests/initramfs_modules_lists_test.py index ac31d0c8..99770339 100644 --- a/kleaf/artifact_tests/initramfs_modules_lists_test.py +++ b/kleaf/artifact_tests/initramfs_modules_lists_test.py @@ -13,11 +13,12 @@ # limitations under the License. import argparse -import os import subprocess import sys import tempfile import unittest +import re +import pathlib from absl.testing import absltest @@ -27,6 +28,8 @@ def load_arguments(): parser.add_argument("--expected_modules_list") parser.add_argument("--expected_modules_recovery_list") parser.add_argument("--expected_modules_charger_list") + parser.add_argument("--build_vendor_boot") + parser.add_argument("--build_vendor_kernel_boot") parser.add_argument("files", nargs="*", default=[]) return parser.parse_known_args() @@ -35,6 +38,36 @@ arguments = None class InitramfsModulesLists(unittest.TestCase): + def _detect_decompression_cmd(self, initramfs): + """Determines what commands to use for decompressing initramfs.img + + Args: + initramfs: The path to the initramfs.img to decompress + + Returns: + The command that should be used to decompress the image. + """ + magic_to_decompression_command = { + # GZIP + b'\x1f\x8b\x08': ["gzip", "-c", "-d"], + # LZ4 + # The kernel build uses legacy LZ4 compression (i.e. lz4 -l ...), + # so the legacy LZ4 magic must be used in little-endian format. + b'\x02\x21\x4c\x18': ["lz4", "-c", "-d", "-l"], + } + max_magic_len = max(len(magic) for magic in magic_to_decompression_command) + + with open(initramfs, "rb") as initramfs_file: + hdr = initramfs_file.read(max_magic_len) + + self.assertIsNotNone(hdr) + + for magic, command in magic_to_decompression_command.items(): + if hdr.startswith(magic): + return command + + self.fail("No suitable compression method found") + def _decompress_initramfs(self, initramfs, temp_dir): """Decompress initramfs into temp_dir. @@ -42,13 +75,14 @@ class InitramfsModulesLists(unittest.TestCase): initramfs: path to initramfs.img gzip file to be decompressed temp_dir: directory in which to decompress initramfs.img into """ + decompression_cmd = self._detect_decompression_cmd(initramfs) with open(initramfs) as initramfs_file: with subprocess.Popen(["cpio", "-i"], cwd=temp_dir, stdin=subprocess.PIPE, stdout=subprocess.PIPE) as cpio_sp: - with subprocess.Popen(["gzip", "-c", "-d"], stdin=initramfs_file, - stdout=cpio_sp.stdin) as gzip_sp: - gzip_sp.communicate() - self.assertEqual(0, gzip_sp.returncode) + with subprocess.Popen(decompression_cmd, stdin=initramfs_file, + stdout=cpio_sp.stdin) as decompress_sp: + decompress_sp.communicate() + self.assertEqual(0, decompress_sp.returncode) def _diff_modules_lists(self, modules_lists_map, modules_dir): """Compares generated modules lists against expected modules lists for equality. @@ -61,18 +95,20 @@ class InitramfsModulesLists(unittest.TestCase): modules_dir: directory that contains the modules.load* files """ for modules_load, expected_modules_list_path in modules_lists_map.items(): - modules_load_path = os.path.join(modules_dir, modules_load) - self.assertTrue(os.path.isfile(modules_load_path), f"Can't find {modules_load_path}") + modules_load_path = pathlib.Path(modules_dir, modules_load) + self.assertTrue(modules_load_path.is_file(), f"Can't find {modules_load_path}") with open(modules_load_path) as modules_load_file, \ open(expected_modules_list_path) as expected_modules_list_file: - modules_load_lines = [os.path.basename(f) for f in modules_load_file.readlines()] - expected_modules_list_lines = expected_modules_list_file.readlines() - self.assertEqual(modules_load_lines.sort(), expected_modules_list_lines.sort()) + modules_load_lines = [pathlib.Path(line.strip()).name for line in modules_load_file] + expected_modules_list_lines = [line.strip() for line in expected_modules_list_file] + self.assertCountEqual(modules_load_lines, expected_modules_list_lines) def test_diff(self): - initramfs_list = [f for f in arguments.files if os.path.basename(f) == "initramfs.img"] - self.assertEqual(1, len(initramfs_list)) + initramfs_list = [file + for file in arguments.files + if pathlib.Path(file).name == "initramfs.img"] + self.assertEqual(len(initramfs_list), 1) initramfs = initramfs_list[0] modules_lists_map = {} @@ -88,14 +124,114 @@ class InitramfsModulesLists(unittest.TestCase): with tempfile.TemporaryDirectory() as temp_dir: self._decompress_initramfs(initramfs, temp_dir) - lib_modules = os.path.join(temp_dir, "lib/modules") - self.assertTrue(os.path.isdir(lib_modules)) + lib_modules = pathlib.Path(temp_dir, "lib/modules") + self.assertTrue(lib_modules.is_dir()) - kernel_versions = os.listdir(lib_modules) - for v in kernel_versions: - modules_dir = os.path.join(lib_modules, v) + for kernel_version in lib_modules.iterdir(): + modules_dir = pathlib.Path(lib_modules, kernel_version) self._diff_modules_lists(modules_lists_map, modules_dir) + def _verify_modules_load_lists(self, modules_list_name, vendor_boot_name, image_files): + """Given a modules.load* name, ensures that no extraneous such files exist. + + This tests to ensure that if a modules.load* file exists, then no other + modules.load* file must exist, except for vendor_boot.modules.load* or + vendor_kernel_boot.modules.load. + + Args: + modules_list_name: The name of the modules.load list. + vendor_boot_name: Either vendor_boot or vendor_kernel_boot. + image_files: The files associated with the kernel_images() target. + """ + modules_load_lists = [modules_list_name] + if vendor_boot_name: + modules_load_lists.append(f"{vendor_boot_name}.{modules_list_name}") + + modules_load_list_re = re.compile(f".*{modules_list_name}$") + modules_load_list_matches = [pathlib.Path(file).name + for file in image_files + if modules_load_list_re.fullmatch(pathlib.Path(file).name)] + + self.assertCountEqual(modules_load_lists, modules_load_list_matches) + + def test_modules_lists_existence(self): + vendor_boot_name = None + + if arguments.build_vendor_boot: + vendor_boot_name = "vendor_boot" + elif arguments.build_vendor_kernel_boot: + vendor_boot_name = "vendor_kernel_boot" + + if arguments.expected_modules_list: + self._verify_modules_load_lists("modules.load", vendor_boot_name, arguments.files) + + if arguments.expected_modules_recovery_list: + self._verify_modules_load_lists("modules.load.recovery", vendor_boot_name, arguments.files) + + if arguments.expected_modules_charger_list: + self._verify_modules_load_lists("modules.load.charger", vendor_boot_name, arguments.files) + + def _verify_modules_dep_contains_modules_lists(self, modules_lists, modules_dir): + """Ensures that modules.dep contains all entries needed for modules.load*. + + Given a list of modules lists, this function ensures that modules.dep + contains an entry for each module in all of the modules lists. + + Args: + modules_lists: The list of modules.load* that need to be tested. + modules_dir: The directory in which the modules reside in. + """ + modules_dep_path = pathlib.Path(modules_dir, "modules.dep") + modules_dep_set = set() + + self.assertTrue(modules_dep_path.is_file(), f"Can't find {modules_dep_path}") + + with open(modules_dep_path) as modules_dep_file: + for line in modules_dep_file: + # depmod entries have the form: + # mod_path: dep_path_1 dep_path_2 + mod_name = line.split(":")[0].strip() + modules_dep_set.add(pathlib.Path(mod_name).name) + + for mod_list in modules_lists: + mod_list_path = pathlib.Path(modules_dir, mod_list) + self.assertTrue(mod_list_path.is_file(), f"Can't find {mod_list_path}") + mod_list_modules = set() + + with open(mod_list_path) as mod_list_file: + for line in mod_list_file: + mod_list_modules.add(pathlib.Path(line.strip()).name) + + self.assertTrue(mod_list_modules.issubset(modules_dep_set), + "modules.dep does not contain an entry for each module to be loaded") + + def test_modules_dep_contains_all_modules_lists(self): + initramfs_list = [file + for file in arguments.files + if pathlib.Path(file).name == "initramfs.img"] + self.assertEqual(len(initramfs_list), 1) + initramfs = initramfs_list[0] + modules_lists = [] + + if arguments.expected_modules_list: + modules_lists.append("modules.load") + + if arguments.expected_modules_recovery_list: + modules_lists.append("modules.load.recovery") + + if arguments.expected_modules_charger_list: + modules_lists.append("modules.load.charger") + + with tempfile.TemporaryDirectory() as temp_dir: + self._decompress_initramfs(initramfs, temp_dir) + + lib_modules = pathlib.Path(temp_dir, "lib/modules") + self.assertTrue(lib_modules.is_dir()) + + for kernel_version in lib_modules.iterdir(): + modules_dir = pathlib.Path(lib_modules, kernel_version) + self._verify_modules_dep_contains_modules_lists(modules_lists, modules_dir) + if __name__ == '__main__': arguments, unknown = load_arguments() sys.argv[1:] = unknown diff --git a/kleaf/artifact_tests/kernel_test.bzl b/kleaf/artifact_tests/kernel_test.bzl index 2631ab61..fe919986 100644 --- a/kleaf/artifact_tests/kernel_test.bzl +++ b/kleaf/artifact_tests/kernel_test.bzl @@ -124,6 +124,8 @@ def initramfs_modules_lists_test( expected_modules_list = None, expected_modules_recovery_list = None, expected_modules_charger_list = None, + build_vendor_boot = None, + build_vendor_kernel_boot = None, **kwargs): """Tests that the initramfs has modules.load* files with the given content. @@ -133,6 +135,8 @@ def initramfs_modules_lists_test( expected_modules_list: file with the expected content for `modules.load` expected_modules_recovery_list: file with the expected content for `modules.load.recovery` expected_modules_charger_list: file with the expected content for `modules.load.charger` + build_vendor_boot: If the `kernel_images` target builds vendor_boot.img + build_vendor_kernel_boot: If the `kernel_images` target builds vendor_kernel_boot.img **kwargs: Additional attributes to the internal rule, e.g. [`visibility`](https://docs.bazel.build/versions/main/visibility.html). See complete list @@ -159,6 +163,11 @@ def initramfs_modules_lists_test( "$(rootpath {})".format(expected_modules_charger_list), ] + if build_vendor_boot: + args.append("--build_vendor_boot") + elif build_vendor_kernel_boot: + args.append("--build_vendor_kernel_boot") + args.append("$(rootpaths {})".format(kernel_images)) hermetic_exec_test( diff --git a/kleaf/common_kernels.bzl b/kleaf/common_kernels.bzl index 375a9446..f51eb8b7 100644 --- a/kleaf/common_kernels.bzl +++ b/kleaf/common_kernels.bzl @@ -536,6 +536,18 @@ def define_common_kernels( "BUILD.bazel", "**/*.bzl", ".git/**", + + # ctag files + "tags", + "TAGS", + + # temporary ctag files + "tags.temp", + "tags.lock", + + # cscope files + "cscope.*", + "ncscope.*", ], ), ) diff --git a/kleaf/docs/build_configs.md b/kleaf/docs/build_configs.md index 41255748..440d6994 100644 --- a/kleaf/docs/build_configs.md +++ b/kleaf/docs/build_configs.md @@ -164,9 +164,11 @@ See [documentation for all rules]. ## EXT\_MODULES ```python -kernel_module() +ddk_module() ``` +NOTE: Prefer `ddk_module` over the legacy `kernel_module`. + See [documentation for all rules]. ## EXT\_MODULES\_MAKEFILE diff --git a/kleaf/impl/cache_dir.bzl b/kleaf/impl/cache_dir.bzl index d6b41aa5..2ceffddf 100644 --- a/kleaf/impl/cache_dir.bzl +++ b/kleaf/impl/cache_dir.bzl @@ -21,9 +21,7 @@ _FLOCK_FD = 0x41F # KLF def _get_flock_cmd(ctx): if ctx.attr._debug_cache_dir_conflict[BuildSettingInfo].value == "none": - pre_cmd = """ - unset KLEAF_CACHED_COMMON_OUT_DIR - """ + pre_cmd = "" post_cmd = "" return struct( pre_cmd = pre_cmd, @@ -43,9 +41,9 @@ def _get_flock_cmd(ctx): pre_cmd = """ ( - echo "DEBUG: [$(date -In)] {label}: Locking ${{KLEAF_CACHED_COMMON_OUT_DIR}}/kleaf_config_tags.json before using" >&2 + echo "DEBUG: [$(date -In)] {label}: Locking ${{COMMON_OUT_DIR}}/kleaf_config_tags.json before using" >&2 if ! flock -x {lock_args} {flock_fd}; then - echo "ERROR: [$(date -In)] {label}: Unable to lock ${{KLEAF_CACHED_COMMON_OUT_DIR}}/kleaf_config_tags.json." >&2 + echo "ERROR: [$(date -In)] {label}: Unable to lock ${{COMMON_OUT_DIR}}/kleaf_config_tags.json." >&2 echo " Please file a bug! See build/kernel/kleaf/docs/errors.md" >&2 exit 1 fi @@ -55,8 +53,8 @@ def _get_flock_cmd(ctx): flock_fd = _FLOCK_FD, ) post_cmd = """ - ) {flock_fd}<"${{KLEAF_CACHED_COMMON_OUT_DIR}}/kleaf_config_tags.json" - echo "DEBUG: [$(date -In)] {label}: Unlocked ${{KLEAF_CACHED_COMMON_OUT_DIR}}/kleaf_config_tags.json after using" >&2 + ) {flock_fd}<"${{COMMON_OUT_DIR}}/kleaf_config_tags.json" + echo "DEBUG: [$(date -In)] {label}: Unlocked ${{COMMON_OUT_DIR}}/kleaf_config_tags.json after using" >&2 """.format( label = ctx.label, flock_fd = _FLOCK_FD, @@ -115,15 +113,15 @@ def _get_step(ctx, common_config_tags, symlink_name): export OUT_DIR_SUFFIX=$(cat ${{KLEAF_CONFIG_TAGS_TMP}} | sha1sum -b | cut -c-8) - KLEAF_CACHED_COMMON_OUT_DIR={cache_dir}/${{OUT_DIR_SUFFIX}} - export OUT_DIR=${{KLEAF_CACHED_COMMON_OUT_DIR}}/${{KERNEL_DIR}} + export COMMON_OUT_DIR={cache_dir}/${{OUT_DIR_SUFFIX}} + export OUT_DIR=${{COMMON_OUT_DIR}}/${{KERNEL_DIR}} mkdir -p "${{OUT_DIR}}" # Reconcile differences between expected file and target file, if any, # to prevent hash collision. {cache_dir_config_tags} \\ --base "${{KLEAF_CONFIG_TAGS_TMP}}" \\ - --dest "${{KLEAF_CACHED_COMMON_OUT_DIR}}/kleaf_config_tags.json" + --dest "${{COMMON_OUT_DIR}}/kleaf_config_tags.json" rm -f "${{KLEAF_CONFIG_TAGS_TMP}}" unset KLEAF_CONFIG_TAGS_TMP diff --git a/kleaf/impl/common_providers.bzl b/kleaf/impl/common_providers.bzl index fc5173f7..994eca5d 100644 --- a/kleaf/impl/common_providers.bzl +++ b/kleaf/impl/common_providers.bzl @@ -289,6 +289,10 @@ KernelModuleInfo = provider( names of all external modules in an unspecified order. This corresponds to `EXT_MODULES` in `build.sh`.""", "label": "Label to the `kernel_module` target.", + "modules_order": """A [depset](https://bazel.build/extending/depsets) of `modules.order` + files from ddk_module's, kernel_module, etc. + It uses [`postorder`](https://bazel.build/rules/lib/builtins/depset) ordering (dependencies + first).""", }, ) @@ -318,8 +322,7 @@ ModuleSymversInfo = provider( doc = "A provider that provides `Module.symvers` for `modpost`.", fields = { "restore_paths": """A [depset](https://bazel.build/extending/depsets) of - paths relative to <the root of the output directory> (e.g. - `<sandbox_root>/out/<branch>`) where the `Module.symvers` files will be + paths relative to `COMMON_OUT_DIR` where the `Module.symvers` files will be restored to by `KernelModuleSetupInfo`.""", }, ) diff --git a/kleaf/impl/image/image_utils.bzl b/kleaf/impl/image/image_utils.bzl index c577e9d9..3357fa5a 100644 --- a/kleaf/impl/image/image_utils.bzl +++ b/kleaf/impl/image/image_utils.bzl @@ -164,6 +164,18 @@ def _build_modules_image_impl_common( options = "-al --chmod=F+w --include=source --include=build --exclude='*'", ) + modules_order_cmd = "" + if ctx.attr.create_modules_order: + modules_order_depset = ctx.attr.kernel_modules_install[KernelModuleInfo].modules_order + modules_order_depset_list = modules_order_depset.to_list() + inputs += modules_order_depset_list + modules_order_cmd = """ + cat {modules_order} > kleaf_modules.order + KLEAF_MODULES_ORDER=kleaf_modules.order + """.format( + modules_order = " ".join([modules_order.path for modules_order in modules_order_depset_list]), + ) + if set_ext_modules and ctx.attr._set_ext_modules[BuildSettingInfo].value: ext_modules = ctx.attr.kernel_modules_install[KernelModuleInfo].packages.to_list() command += """EXT_MODULES={quoted_ext_modules}""".format( @@ -181,9 +193,11 @@ file a bug.""") mkdir -p ${{DIST_DIR}} cp {system_map} ${{DIST_DIR}}/System.map + {modules_order_cmd} {build_command} """.format( system_map = system_map.path, + modules_order_cmd = modules_order_cmd, build_command = build_command, ) @@ -214,6 +228,12 @@ def _build_modules_image_attrs_common(additional = None): "_set_ext_modules": attr.label( default = "//build/kernel/kleaf:set_ext_modules", ), + "create_modules_order": attr.bool( + default = True, + doc = """Whether to create and keep a modules.order file generated + by a postorder traversal of the `kernel_modules_install` sources. + It defaults to `True`.""", + ), } if additional != None: ret.update(additional) diff --git a/kleaf/impl/image/initramfs.bzl b/kleaf/impl/image/initramfs.bzl index 783f942a..85023d96 100644 --- a/kleaf/impl/image/initramfs.bzl +++ b/kleaf/impl/image/initramfs.bzl @@ -31,26 +31,12 @@ def _initramfs_impl(ctx): initramfs_img = ctx.actions.declare_file("{}/initramfs.img".format(ctx.label.name)) modules_load = ctx.actions.declare_file("{}/modules.load".format(ctx.label.name)) vendor_boot_modules_load = ctx.outputs.vendor_boot_modules_load - vendor_boot_modules_load_recovery = ctx.outputs.vendor_boot_modules_load_recovery - vendor_boot_modules_load_charger = ctx.outputs.vendor_boot_modules_load_charger initramfs_staging_archive = ctx.actions.declare_file("{}/initramfs_staging_archive.tar.gz".format(ctx.label.name)) outputs = [ initramfs_img, modules_load, ] - if vendor_boot_modules_load: - outputs.append(vendor_boot_modules_load) - - if vendor_boot_modules_load_recovery: - outputs.append(vendor_boot_modules_load_recovery) - - if vendor_boot_modules_load_charger: - outputs.append(vendor_boot_modules_load_charger) - - modules_staging_dir = initramfs_img.dirname + "/staging" - initramfs_staging_dir = modules_staging_dir + "/initramfs_staging" - cp_vendor_boot_modules_load_cmd = "" if vendor_boot_modules_load: cp_vendor_boot_modules_load_cmd = """ @@ -58,30 +44,50 @@ def _initramfs_impl(ctx): """.format( vendor_boot_modules_load = vendor_boot_modules_load.path, ) + outputs.append(vendor_boot_modules_load) - cp_vendor_boot_modules_load_recovery_cmd = "" - if vendor_boot_modules_load_recovery: + cp_modules_load_recovery_cmd = "" + if ctx.attr.modules_recovery_list: modules_load_recovery = ctx.actions.declare_file("{}/modules.load.recovery".format(ctx.label.name)) + cp_modules_load_recovery_cmd = """ + cp ${{modules_root_dir}}/modules.load.recovery {modules_load_recovery} + """.format( + modules_load_recovery = modules_load_recovery.path, + ) outputs.append(modules_load_recovery) + + cp_vendor_boot_modules_load_recovery_cmd = "" + vendor_boot_modules_load_recovery = ctx.outputs.vendor_boot_modules_load_recovery + if vendor_boot_modules_load_recovery: cp_vendor_boot_modules_load_recovery_cmd = """ - cp ${{modules_root_dir}}/modules.load.recovery {modules_load_recovery} cp ${{modules_root_dir}}/modules.load.recovery {vendor_boot_modules_load_recovery} """.format( - modules_load_recovery = modules_load_recovery.path, vendor_boot_modules_load_recovery = vendor_boot_modules_load_recovery.path, ) + outputs.append(vendor_boot_modules_load_recovery) - cp_vendor_boot_modules_load_charger_cmd = "" - if vendor_boot_modules_load_charger: + cp_modules_load_charger_cmd = "" + if ctx.attr.modules_charger_list: modules_load_charger = ctx.actions.declare_file("{}/modules.load.charger".format(ctx.label.name)) + cp_modules_load_charger_cmd = """ + cp ${{modules_root_dir}}/modules.load.charger {modules_load_charger} + """.format( + modules_load_charger = modules_load_charger.path, + ) outputs.append(modules_load_charger) + + cp_vendor_boot_modules_load_charger_cmd = "" + vendor_boot_modules_load_charger = ctx.outputs.vendor_boot_modules_load_charger + if vendor_boot_modules_load_charger: cp_vendor_boot_modules_load_charger_cmd = """ - cp ${{modules_root_dir}}/modules.load.charger {modules_load_charger} cp ${{modules_root_dir}}/modules.load.charger {vendor_boot_modules_load_charger} """.format( - modules_load_charger = modules_load_charger.path, vendor_boot_modules_load_charger = vendor_boot_modules_load_charger.path, ) + outputs.append(vendor_boot_modules_load_charger) + + modules_staging_dir = initramfs_img.dirname + "/staging" + initramfs_staging_dir = modules_staging_dir + "/initramfs_staging" additional_inputs = [] if ctx.file.modules_options: @@ -112,7 +118,9 @@ def _initramfs_impl(ctx): modules_root_dir=$(readlink -e {initramfs_staging_dir}/lib/modules/*) || exit 1 cp ${{modules_root_dir}}/modules.load {modules_load} {cp_vendor_boot_modules_load_cmd} + {cp_modules_load_recovery_cmd} {cp_vendor_boot_modules_load_recovery_cmd} + {cp_modules_load_charger_cmd} {cp_vendor_boot_modules_load_charger_cmd} {cp_modules_options_cmd} mkbootfs "{initramfs_staging_dir}" >"{modules_staging_dir}/initramfs.cpio" @@ -129,7 +137,9 @@ def _initramfs_impl(ctx): initramfs_img = initramfs_img.path, initramfs_staging_archive = initramfs_staging_archive.path, cp_vendor_boot_modules_load_cmd = cp_vendor_boot_modules_load_cmd, + cp_modules_load_recovery_cmd = cp_modules_load_recovery_cmd, cp_vendor_boot_modules_load_recovery_cmd = cp_vendor_boot_modules_load_recovery_cmd, + cp_modules_load_charger_cmd = cp_modules_load_charger_cmd, cp_vendor_boot_modules_load_charger_cmd = cp_vendor_boot_modules_load_charger_cmd, cp_modules_options_cmd = cp_modules_options_cmd, ) diff --git a/kleaf/impl/image/kernel_images.bzl b/kleaf/impl/image/kernel_images.bzl index 1d0b4ce3..7e9dcba6 100644 --- a/kleaf/impl/image/kernel_images.bzl +++ b/kleaf/impl/image/kernel_images.bzl @@ -66,6 +66,7 @@ def kernel_images( avb_boot_algorithm = None, avb_boot_partition_name = None, dedup_dlkm_modules = None, + create_modules_order = None, **kwargs): """Build multiple kernel images. @@ -302,7 +303,9 @@ def kernel_images( If set, **additional changes in the userspace is required** so that `system_dlkm` modules are loaded before `vendor_dlkm` modules. - + create_modules_order: Whether to create and keep a modules.order file + generated by a postorder traversal of the `kernel_modules_install` sources. + It applies to building `initramfs` and `vendor_dlkm`. **kwargs: Additional attributes to the internal rule, e.g. [`visibility`](https://docs.bazel.build/versions/main/visibility.html). See complete list @@ -357,13 +360,14 @@ def kernel_images( vendor_boot_modules_load_recovery = None vendor_boot_modules_load_charger = None if build_initramfs: - vendor_boot_modules_load = "{}_initramfs/{}.modules.load".format(name, vendor_boot_name) + if vendor_boot_name: + vendor_boot_modules_load = "{}_initramfs/{}.modules.load".format(name, vendor_boot_name) - if modules_recovery_list: - vendor_boot_modules_load_recovery = "{}_initramfs/{}.modules.load.recovery".format(name, vendor_boot_name) + if modules_recovery_list: + vendor_boot_modules_load_recovery = "{}_initramfs/{}.modules.load.recovery".format(name, vendor_boot_name) - if modules_charger_list: - vendor_boot_modules_load_charger = "{}_initramfs/{}.modules.load.charger".format(name, vendor_boot_name) + if modules_charger_list: + vendor_boot_modules_load_charger = "{}_initramfs/{}.modules.load.charger".format(name, vendor_boot_name) if ramdisk_compression_args and ramdisk_compression != "lz4": fail( @@ -386,6 +390,7 @@ def kernel_images( modules_options = modules_options, ramdisk_compression = ramdisk_compression, ramdisk_compression_args = ramdisk_compression_args, + create_modules_order = create_modules_order, **kwargs ) all_rules.append(":{}_initramfs".format(name)) @@ -406,6 +411,7 @@ def kernel_images( system_dlkm_modules_list = system_dlkm_modules_list, system_dlkm_modules_blocklist = system_dlkm_modules_blocklist, system_dlkm_props = system_dlkm_props, + create_modules_order = False, **kwargs ) all_rules.append(":{}_system_dlkm_image".format(name)) @@ -428,6 +434,7 @@ def kernel_images( dedup_dlkm_modules = dedup_dlkm_modules, system_dlkm_image = "{}_system_dlkm_image".format(name) if build_system_dlkm else None, base_kernel_images = base_kernel_images, + create_modules_order = create_modules_order, **kwargs ) all_rules.append(":{}_vendor_dlkm_image".format(name)) diff --git a/kleaf/impl/kernel_build.bzl b/kleaf/impl/kernel_build.bzl index 41b73443..b8874fa8 100644 --- a/kleaf/impl/kernel_build.bzl +++ b/kleaf/impl/kernel_build.bzl @@ -778,7 +778,7 @@ def _create_kbuild_mixed_tree(ctx): rm -rf ${{KBUILD_MIXED_TREE}} mkdir -p ${{KBUILD_MIXED_TREE}} for base_kernel_file in {base_kernel_files}; do - ln -s $(readlink -m ${{base_kernel_file}}) ${{KBUILD_MIXED_TREE}} + cp -a -t ${{KBUILD_MIXED_TREE}} $(readlink -m ${{base_kernel_file}}) done """.format( base_kernel_files = " ".join([file.path for file in base_kernel_files.to_list()]), diff --git a/kleaf/impl/kernel_module.bzl b/kleaf/impl/kernel_module.bzl index 7d1975ce..d24210b5 100644 --- a/kleaf/impl/kernel_module.bzl +++ b/kleaf/impl/kernel_module.bzl @@ -417,6 +417,18 @@ def _kernel_module_impl(ctx): command += modpost_warn.cmd command_outputs += modpost_warn.outputs + # Keep a record of the modules.order generated by `make`. + modules_order = ctx.actions.declare_file("{}/modules.order".format(ctx.attr.name)) + command_outputs.append(modules_order) + grab_modules_order_cmd = """ + # Backup modules.order files before optionally dropping them. + cp -L -p {modules_staging_dir}/lib/modules/*/extra/{ext_mod}/modules.order.* {modules_order} + """.format( + ext_mod = ext_mod, + modules_staging_dir = modules_staging_dws.directory.path, + modules_order = modules_order.path, + ) + make_filter = "" if not ctx.attr.generate_btf: # Filter out warnings if there is no need for BTF generation @@ -459,7 +471,8 @@ def _kernel_module_impl(ctx): {grab_cmd_cmd} # Move Module.symvers rsync -aL ${{OUT_DIR}}/${{ext_mod_rel}}/Module.symvers {module_symvers} - + # Grab and then drop modules.order + {grab_modules_order_cmd} {drop_modules_order_cmd} """.format( label = ctx.label, @@ -476,6 +489,7 @@ def _kernel_module_impl(ctx): all_module_names_file = all_module_names_file.path, grab_unstripped_cmd = grab_unstripped_cmd, check_no_remaining = check_no_remaining.path, + grab_modules_order_cmd = grab_modules_order_cmd, drop_modules_order_cmd = drop_modules_order_cmd, grab_cmd_cmd = grab_cmd_step.cmd, ) @@ -550,6 +564,7 @@ def _kernel_module_impl(ctx): progress_message = "Copying outputs {}".format(ctx.label), ) + module_symvers_restore_path = paths.join(ext_mod, ctx.attr.internal_module_symvers_name) setup = """ # Use a new shell to avoid polluting variables ( @@ -559,14 +574,15 @@ def _kernel_module_impl(ctx): mkdir -p ${{ROOT_DIR}}/{ext_mod} ext_mod_rel=$(realpath ${{ROOT_DIR}}/{ext_mod} --relative-to ${{KERNEL_DIR}}) # Restore Modules.symvers - mkdir -p $(dirname ${{OUT_DIR}}/${{ext_mod_rel}}/{internal_module_symvers_name}) - rsync -aL {module_symvers} ${{OUT_DIR}}/${{ext_mod_rel}}/{internal_module_symvers_name} + mkdir -p $(dirname ${{COMMON_OUT_DIR}}/{module_symvers_restore_path}) + rsync -aL {module_symvers} ${{COMMON_OUT_DIR}}/{module_symvers_restore_path} # New shell ends ) """.format( ext_mod = ext_mod, module_symvers = module_symvers.path, internal_module_symvers_name = ctx.attr.internal_module_symvers_name, + module_symvers_restore_path = module_symvers_restore_path, ) if ctx.attr.internal_ddk_makefiles_dir: @@ -606,6 +622,7 @@ def _kernel_module_impl(ctx): files = depset(output_files), packages = depset([ext_mod]), label = ctx.label, + modules_order = depset([modules_order]), ), KernelUnstrippedModulesInfo( directories = depset([unstripped_dir], order = "postorder"), @@ -615,7 +632,7 @@ def _kernel_module_impl(ctx): # path/to/package/target_name/target_name_Module.symvers -> path/to/package/target_name_Module.symvers; # This is similar to ${{OUT_DIR}}/${{ext_mod_rel}} # It is needed to remove the `target_name` because we declare_file({name}/{internal_module_symvers_name}) above. - restore_paths = depset([paths.join(ext_mod, ctx.attr.internal_module_symvers_name)]), + restore_paths = depset([module_symvers_restore_path]), ), ddk_headers_info, ddk_config_info, diff --git a/kleaf/impl/kernel_module_group.bzl b/kleaf/impl/kernel_module_group.bzl index 28409f72..831ad36d 100644 --- a/kleaf/impl/kernel_module_group.bzl +++ b/kleaf/impl/kernel_module_group.bzl @@ -65,6 +65,10 @@ def _kernel_module_group_impl(ctx): ]), packages = depset(transitive = [target[KernelModuleInfo].packages for target in targets]), label = ctx.label, + modules_order = depset( + transitive = [target[KernelModuleInfo].modules_order for target in targets], + order = "postorder", + ), ) unstripped_modules_info = KernelUnstrippedModulesInfo( diff --git a/kleaf/impl/kernel_modules_install.bzl b/kleaf/impl/kernel_modules_install.bzl index cb381067..c8ab5341 100644 --- a/kleaf/impl/kernel_modules_install.bzl +++ b/kleaf/impl/kernel_modules_install.bzl @@ -194,6 +194,10 @@ def _kernel_modules_install_impl(ctx): for target in ctx.attr.kernel_modules ]), label = ctx.label, + modules_order = depset(transitive = [ + target[KernelModuleInfo].modules_order + for target in ctx.attr.kernel_modules + ], order = "postorder"), ), cmds_info, ] diff --git a/kleaf/impl/utils.bzl b/kleaf/impl/utils.bzl index 545afbc7..95316534 100644 --- a/kleaf/impl/utils.bzl +++ b/kleaf/impl/utils.bzl @@ -166,7 +166,7 @@ def _get_check_sandbox_cmd(): Note: This is not always accurate.""" return """ - if [[ $PWD != */sandbox/* ]]; then + if [[ ! $PWD =~ /(sandbox|bazel-working-directory)/ ]]; then echo "FATAL: this action must be executed in a sandbox!" >&2 exit 1 fi diff --git a/kleaf/tests/ddk_test/BUILD.bazel b/kleaf/tests/ddk_test/BUILD.bazel index 4d6b7bbf..cd2ada24 100644 --- a/kleaf/tests/ddk_test/BUILD.bazel +++ b/kleaf/tests/ddk_test/BUILD.bazel @@ -16,12 +16,15 @@ load("//build/kernel/kleaf/impl:ddk/ddk_module.bzl", "ddk_module") load("//build/kernel/kleaf/impl:kernel_build.bzl", "kernel_build") load("//build/kernel/kleaf/tests:failure_test.bzl", "failure_test") load(":ddk_headers_test.bzl", "ddk_headers_test_suite") +load(":ddk_images_test.bzl", "ddk_images_test_suite") load(":ddk_module_test.bzl", "ddk_module_test_suite") load(":ddk_submodule_test.bzl", "ddk_submodule_test") load(":makefiles_test.bzl", "makefiles_test_suite") ddk_headers_test_suite(name = "ddk_headers_test_suite") +ddk_images_test_suite(name = "ddk_images_test_suite") + ddk_module_test_suite(name = "ddk_module_test_suite") ddk_submodule_test(name = "ddk_submodule_test") diff --git a/kleaf/tests/ddk_test/ddk_images_test.bzl b/kleaf/tests/ddk_test/ddk_images_test.bzl new file mode 100644 index 00000000..7a05d7a2 --- /dev/null +++ b/kleaf/tests/ddk_test/ddk_images_test.bzl @@ -0,0 +1,166 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for `kernel_images` with `ddk_module`'s.""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_skylib//rules:write_file.bzl", "write_file") +load( + "//build/kernel/kleaf:kernel.bzl", + "ddk_module", + "kernel_images", + "kernel_module_group", + "kernel_modules_install", +) +load("//build/kernel/kleaf/impl:hermetic_toolchain.bzl", "hermetic_toolchain") +load("//build/kernel/kleaf/tests/utils:contain_lines_test.bzl", "contain_lines_test") + +def _get_top_level_file_impl(ctx): + out = ctx.actions.declare_file(paths.join(ctx.attr.name, ctx.attr.filename)) + src = None + for file in ctx.files.target: + if file.basename == ctx.attr.filename: + src = file + break + + hermetic_tools = hermetic_toolchain.get(ctx) + command = hermetic_tools.setup + """ + if [[ -f {src} ]]; then + cp -pL {src} {out} + else + : > {out} + fi + """.format( + src = src.path, + out = out.path, + ) + ctx.actions.run_shell( + outputs = [out], + inputs = ctx.files.target, + tools = hermetic_tools.deps, + command = command, + ) + return DefaultInfo( + files = depset([out]), + runfiles = ctx.runfiles(files = [out]), + ) + +get_top_level_file = rule( + implementation = _get_top_level_file_impl, + doc = "Gets the top level file from a rule.", + attrs = { + "target": attr.label(allow_files = True), + "filename": attr.string(), + }, + toolchains = [hermetic_toolchain.type], +) + +def ddk_images_test_suite(name): + """Defines analysis test for `kernel_images` with `ddk_module`'s. + + Args: + name: Name for this test suite. + """ + + # Setup BEGIN + # TODO: Find a way to remove this dependency on ACK. + kernel_build_name = "//common:kernel_aarch64" + ddk_deps = ["//common:all_headers_aarch64"] + + ddk_module( + name = name + "_base", + out = name + "_base.ko", + srcs = ["license.c"], + kernel_build = kernel_build_name, + deps = ddk_deps, + tags = ["manual"], + ) + + ddk_module( + name = name + "_self", + out = name + "_self.ko", + srcs = ["license.c"], + kernel_build = kernel_build_name, + deps = ddk_deps, + tags = ["manual"], + ) + + ddk_module( + name = name + "_dep", + out = name + "_dep.ko", + srcs = ["license.c"], + kernel_build = kernel_build_name, + deps = ddk_deps, + tags = ["manual"], + ) + + kernel_module_group( + name = name + "_group", + srcs = [ + name + "_self", + name + "_base", + ], + tags = ["manual"], + ) + + kernel_modules_install( + name = name + "_modules_install", + kernel_build = kernel_build_name, + kernel_modules = [ + name + "_group", + name + "_dep", + ], + tags = ["manual"], + ) + + kernel_images( + name = name + "_image", + kernel_modules_install = name + "_modules_install", + build_initramfs = True, + tags = ["manual"], + ) + # Setup END + + tests = [] + module_prefix = "extra/build/kernel/kleaf/tests/ddk_test/" + write_file( + name = name + "_expected", + out = name + "_expected/modules.load", + content = [ + module_prefix + name + "_self.ko", + module_prefix + name + "_base.ko", + module_prefix + name + "_dep.ko", + ], + tags = ["manual"], + ) + + get_top_level_file( + name = name + "_modules_load", + filename = "modules.load", + target = name + "_image_initramfs", + tags = ["manual"], + ) + + contain_lines_test( + name = name + "_modules_load_test", + expected = name + "_expected", + actual = name + "_modules_load", + order = True, + ) + tests.append(name + "_modules_load_test") + + native.test_suite( + name = name, + tests = tests, + ) diff --git a/kleaf/tests/ddk_test/license.c b/kleaf/tests/ddk_test/license.c new file mode 100644 index 00000000..5b221984 --- /dev/null +++ b/kleaf/tests/ddk_test/license.c @@ -0,0 +1,2 @@ +#include <linux/module.h> +MODULE_LICENSE("GPL v2");
\ No newline at end of file diff --git a/kleaf/tests/ddk_test/makefiles_test.bzl b/kleaf/tests/ddk_test/makefiles_test.bzl index 4d8960dc..555d1241 100644 --- a/kleaf/tests/ddk_test/makefiles_test.bzl +++ b/kleaf/tests/ddk_test/makefiles_test.bzl @@ -151,7 +151,7 @@ def _get_top_level_file_impl(ctx): hermetic_tools = hermetic_toolchain.get(ctx) command = hermetic_tools.setup + """ if [[ -f {src} ]]; then - cp -pl {src} {out} + cp -pL {src} {out} else : > {out} fi diff --git a/kleaf/tests/kernel_config_test/kernel_config_option_test.bzl b/kleaf/tests/kernel_config_test/kernel_config_option_test.bzl index 2898e132..bf2e0683 100644 --- a/kleaf/tests/kernel_config_test/kernel_config_option_test.bzl +++ b/kleaf/tests/kernel_config_test/kernel_config_option_test.bzl @@ -56,7 +56,7 @@ def _get_config_file(ctx, kernel_build, filename): hermetic_tools = hermetic_toolchain.get(ctx) command = hermetic_tools.setup + """ - cp -pl {out_dir}/.config {out} + cp -pL {out_dir}/.config {out} """.format( out_dir = out_dir.path, out = out.path, diff --git a/kleaf/tests/kernel_config_test/kernel_defconfig_fragments_test.bzl b/kleaf/tests/kernel_config_test/kernel_defconfig_fragments_test.bzl index 8da1c3d7..2a84672e 100644 --- a/kleaf/tests/kernel_config_test/kernel_defconfig_fragments_test.bzl +++ b/kleaf/tests/kernel_config_test/kernel_defconfig_fragments_test.bzl @@ -68,7 +68,7 @@ def _get_config_impl(ctx): out = ctx.actions.declare_file("{}/.config".format(ctx.label.name)) hermetic_tools = hermetic_toolchain.get(ctx) command = hermetic_tools.setup + """ - cp -pl {out_dir}/.config {out} + cp -pL {out_dir}/.config {out} """.format( out_dir = out_dir.path, out = out.path, diff --git a/static_analysis/checkpatch_ignorelist b/static_analysis/checkpatch_ignorelist index 40d3389c..2283a773 100644 --- a/static_analysis/checkpatch_ignorelist +++ b/static_analysis/checkpatch_ignorelist @@ -11,6 +11,7 @@ ARRAY_SIZE AVOID_BUG AVOID_EXTERNS BAD_AUTHOR +BAD_REPORTED_BY_LINK BAD_SIGN_OFF BLOCK_COMMENT_STYLE BRACES |