diff options
Diffstat (limited to 'pkg/private/pkg_files.bzl')
| -rw-r--r-- | pkg/private/pkg_files.bzl | 397 |
1 files changed, 238 insertions, 159 deletions
diff --git a/pkg/private/pkg_files.bzl b/pkg/private/pkg_files.bzl index f0fe60b..6fc0971 100644 --- a/pkg/private/pkg_files.bzl +++ b/pkg/private/pkg_files.bzl @@ -34,7 +34,6 @@ Concepts and terms: load("//pkg:path.bzl", "compute_data_path", "dest_path") load( "//pkg:providers.bzl", - "PackageArtifactInfo", "PackageDirsInfo", "PackageFilegroupInfo", "PackageFilesInfo", @@ -44,9 +43,10 @@ load( ENTRY_IS_FILE = "file" # Entry is a file: take content from <src> ENTRY_IS_LINK = "symlink" # Entry is a symlink: dest -> <src> ENTRY_IS_DIR = "dir" # Entry is an empty dir -ENTRY_IS_TREE = "tree" # Entry is a tree artifact: take tree from <src> +ENTRY_IS_TREE = "tree" # Entry is a tree artifact: take tree from <src> ENTRY_IS_EMPTY_FILE = "empty-file" # Entry is a an empty file +# buildifier: disable=name-conventions _DestFile = provider( doc = """Information about each destination in the final package.""", fields = { @@ -57,10 +57,79 @@ _DestFile = provider( "link_to": "path to link to. src must not be set", "entry_type": "string. See ENTRY_IS_* values above.", "origin": "target which added this", + "uid": "uid, or empty", + "gid": "gid, or empty", }, ) -def _check_dest(content_map, dest, src, origin): +# buildifier: disable=name-conventions +_MappingContext = provider( + doc = """Fields passed to process_* methods.""", + fields = { + "content_map": "in/out The content_map we are building up", + "file_deps": "in/out list of file Depsets represented in the map", + "label": "ctx.label", + + # Behaviors + "allow_duplicates_with_different_content": "bool: don't fail when you double mapped files", + "include_runfiles": "bool: include runfiles", + "strip_prefix": "strip_prefix", + + "path_mapper": "function to map destination paths", + + # Defaults + "default_mode": "Default mode to apply to file without a mode setting", + "default_user": "Default user name to apply to file without a user", + "default_group": "Default group name to apply to file without a group", + "default_uid": "Default numeric uid to apply to file without a uid", + "default_gid": "Default numeric gid to apply to file without a gid", + }, +) + +# buildifier: disable=function-docstring-args +def create_mapping_context_from_ctx( + ctx, + label, + allow_duplicates_with_different_content = None, + strip_prefix = None, + include_runfiles = None, + default_mode = None, + path_mapper = None + ): + """Construct a MappingContext. + + Args: See the provider definition. + + Returns: + _MappingContext + """ + if allow_duplicates_with_different_content == None: + allow_duplicates_with_different_content = ctx.attr.allow_duplicates_with_different_content if hasattr(ctx.attr, "allow_duplicates_with_different_content") else False + if strip_prefix == None: + strip_prefix = ctx.attr.strip_prefix if hasattr(ctx.attr, "strip_prefix") else "" + if include_runfiles == None: + include_runfiles = ctx.attr.include_runfiles if hasattr(ctx.attr, "include_runfiles") else False + if default_mode == None: + default_mode = ctx.attr.mode if hasattr(ctx.attr, "default_mode") else "" + + return _MappingContext( + content_map = dict(), + file_deps = list(), + label = label, + allow_duplicates_with_different_content = allow_duplicates_with_different_content, + strip_prefix = strip_prefix, + include_runfiles = include_runfiles, + default_mode = default_mode, + path_mapper = path_mapper or (lambda x: x), + # TODO(aiuto): allow these to be passed in as needed. But, before doing + # that, explore defauilt_uid/gid as 0 rather than None + default_user = "", + default_group = "", + default_uid = None, + default_gid = None, + ) + +def _check_dest(content_map, dest, src, origin, allow_duplicates_with_different_content = False): old_entry = content_map.get(dest) if not old_entry: return @@ -72,96 +141,116 @@ def _check_dest(content_map, dest, src, origin): # people specify the owner in one place, but another overly broad glob # brings in the file with a different owner. if old_entry.src.path != src.path: - # buildifier: disable=print - print( - "Duplicate output path: <%s>, declared in %s and %s" % ( - dest, - origin, - content_map[dest].origin, - ), - "\n SRC:", + msg = "Duplicate output path: <%s>, declared in %s and %s\n SRC: %s" % ( + dest, + origin, + content_map[dest].origin, src, ) + if allow_duplicates_with_different_content: + # buildifier: disable=print + print("WARNING:", msg) + else: + # When we default to this behaviour, we should consider telling + # users the attribute to set to deal with this. + # For now though, let's not, since they've explicitly opted in. + fail(msg) -def _merge_attributes(info, mode, user, group): +def _merge_attributes(info, mode, user, group, uid, gid): if hasattr(info, "attributes"): attrs = info.attributes mode = attrs.get("mode") or mode user = attrs.get("user") or user group = attrs.get("group") or group - return (mode, user, group) -def _process_pkg_dirs(content_map, pkg_dirs_info, origin, default_mode, default_user, default_group): - attrs = _merge_attributes(pkg_dirs_info, default_mode, default_user, default_group) + new_uid = attrs.get("uid") + if new_uid != None: + uid = new_uid + new_gid = attrs.get("gid") + if new_gid != None: + gid = new_gid + return (mode, user, group, uid, gid) + +def _merge_context_attributes(info, mapping_context): + """Merge defaults from mapping context with those in the source provider. + + Args: + info: provider from a pkt_* target + mapping_context: MappingContext with the defaults. + """ + default_mode = mapping_context.default_mode if hasattr(mapping_context, "default_mode") else "" + default_user = mapping_context.default_user if hasattr(mapping_context, "default_user") else "" + default_group = mapping_context.default_group if hasattr(mapping_context, "default_group") else "" + default_uid = mapping_context.default_uid if hasattr(mapping_context, "default_uid") else "" + default_gid = mapping_context.default_gid if hasattr(mapping_context, "default_gid") else "" + return _merge_attributes(info, default_mode, default_user, default_group, default_uid, default_gid) + +def _process_pkg_dirs(mapping_context, pkg_dirs_info, origin): + attrs = _merge_context_attributes(pkg_dirs_info, mapping_context) for dir in pkg_dirs_info.dirs: dest = dir.strip("/") - _check_dest(content_map, dest, None, origin) - content_map[dest] = _DestFile( + _check_dest(mapping_context.content_map, dest, None, origin, mapping_context.allow_duplicates_with_different_content) + mapping_context.content_map[dest] = _DestFile( src = None, entry_type = ENTRY_IS_DIR, mode = attrs[0], user = attrs[1], group = attrs[2], + uid = attrs[3], + gid = attrs[4], origin = origin, ) -def _process_pkg_files(content_map, pkg_files_info, origin, default_mode, default_user, default_group): - attrs = _merge_attributes(pkg_files_info, default_mode, default_user, default_group) +def _process_pkg_files(mapping_context, pkg_files_info, origin): + attrs = _merge_context_attributes(pkg_files_info, mapping_context) for filename, src in pkg_files_info.dest_src_map.items(): dest = filename.strip("/") - _check_dest(content_map, dest, src, origin) - content_map[dest] = _DestFile( + _check_dest(mapping_context.content_map, dest, src, origin, mapping_context.allow_duplicates_with_different_content) + mapping_context.content_map[dest] = _DestFile( src = src, entry_type = ENTRY_IS_TREE if src.is_directory else ENTRY_IS_FILE, mode = attrs[0], user = attrs[1], group = attrs[2], + uid = attrs[3], + gid = attrs[4], origin = origin, ) -def _process_pkg_symlink(content_map, pkg_symlink_info, origin, default_mode, default_user, default_group): +def _process_pkg_symlink(mapping_context, pkg_symlink_info, origin): dest = pkg_symlink_info.destination.strip("/") - attrs = _merge_attributes(pkg_symlink_info, default_mode, default_user, default_group) - _check_dest(content_map, dest, None, origin) - content_map[dest] = _DestFile( + attrs = _merge_context_attributes(pkg_symlink_info, mapping_context) + _check_dest(mapping_context.content_map, dest, None, origin, mapping_context.allow_duplicates_with_different_content) + mapping_context.content_map[dest] = _DestFile( src = None, entry_type = ENTRY_IS_LINK, mode = attrs[0], user = attrs[1], group = attrs[2], + uid = attrs[3], + gid = attrs[4], origin = origin, link_to = pkg_symlink_info.target, ) -def _process_pkg_filegroup(content_map, pkg_filegroup_info, origin, default_mode, default_user, default_group): +def _process_pkg_filegroup(mapping_context, pkg_filegroup_info): if hasattr(pkg_filegroup_info, "pkg_dirs"): for d in pkg_filegroup_info.pkg_dirs: - _process_pkg_dirs(content_map, d[0], d[1], default_mode, default_user, default_group) + _process_pkg_dirs(mapping_context, d[0], d[1]) if hasattr(pkg_filegroup_info, "pkg_files"): for pf in pkg_filegroup_info.pkg_files: - _process_pkg_files(content_map, pf[0], pf[1], default_mode, default_user, default_group) + _process_pkg_files(mapping_context, pf[0], pf[1]) if hasattr(pkg_filegroup_info, "pkg_symlinks"): for psl in pkg_filegroup_info.pkg_symlinks: - _process_pkg_symlink(content_map, psl[0], psl[1], default_mode, default_user, default_group) + _process_pkg_symlink(mapping_context, psl[0], psl[1]) -def process_src( - content_map, - files, - src, - origin, - default_mode, - default_user, - default_group): +def process_src(mapping_context, src, origin): """Add an entry to the content map. Args: - content_map: in/out The content map - files: in/out list of file Depsets represented in the map + mapping_context: (r/w) a MappingContext src: Source Package*Info object origin: The rule instance adding this entry - default_mode: fallback mode to use for Package*Info elements without mode - default_user: fallback user to use for Package*Info elements without user - default_group: fallback mode to use for Package*Info elements without group Returns: True if src was a Package*Info and added to content_map. @@ -170,168 +259,133 @@ def process_src( # Gather the files for every srcs entry here, even if it is not from # a pkg_* rule. if DefaultInfo in src: - files.append(src[DefaultInfo].files) + mapping_context.file_deps.append(src[DefaultInfo].files) found_info = False if PackageFilesInfo in src: _process_pkg_files( - content_map, + mapping_context, src[PackageFilesInfo], origin, - default_mode = default_mode, - default_user = default_user, - default_group = default_group, ) found_info = True if PackageFilegroupInfo in src: _process_pkg_filegroup( - content_map, + mapping_context, src[PackageFilegroupInfo], - origin, - default_mode = default_mode, - default_user = default_user, - default_group = default_group, ) found_info = True if PackageSymlinkInfo in src: _process_pkg_symlink( - content_map, + mapping_context, src[PackageSymlinkInfo], origin, - default_mode = default_mode, - default_user = default_user, - default_group = default_group, ) found_info = True if PackageDirsInfo in src: _process_pkg_dirs( - content_map, + mapping_context, src[PackageDirsInfo], origin, - default_mode = "0555", - default_user = default_user, - default_group = default_group, ) found_info = True return found_info -def add_directory(content_map, dir_path, origin, mode = None, user = None, group = None): +def add_directory(mapping_context, dir_path, origin, mode = None, user = None, group = None, uid = None, gid = None): """Add an empty directory to the content map. Args: - content_map: The content map + mapping_context: (r/w) a MappingContext dir_path: Where to place the file in the package. origin: The rule instance adding this entry mode: fallback mode to use for Package*Info elements without mode user: fallback user to use for Package*Info elements without user group: fallback mode to use for Package*Info elements without group + uid: numeric uid + gid: numeric gid """ - content_map[dir_path.strip("/")] = _DestFile( + mapping_context.content_map[dir_path.strip("/")] = _DestFile( src = None, entry_type = ENTRY_IS_DIR, origin = origin, mode = mode, - user = user, - group = group, + user = user or mapping_context.default_user, + group = group or mapping_context.default_group, + uid = uid or mapping_context.default_uid, + gid = gid or mapping_context.default_gid, ) -def add_empty_file(content_map, dest_path, origin, mode = None, user = None, group = None): +def add_empty_file(mapping_context, dest_path, origin, mode = None, user = None, group = None, uid = None, gid = None): """Add a single file to the content map. Args: - content_map: The content map + mapping_context: (r/w) a MappingContext dest_path: Where to place the file in the package. origin: The rule instance adding this entry mode: fallback mode to use for Package*Info elements without mode user: fallback user to use for Package*Info elements without user group: fallback mode to use for Package*Info elements without group + uid: numeric uid + gid: numeric gid """ dest = dest_path.strip("/") - _check_dest(content_map, dest, None, origin) - content_map[dest] = _DestFile( + _check_dest(mapping_context.content_map, dest, None, origin) + mapping_context.content_map[dest] = _DestFile( src = None, entry_type = ENTRY_IS_EMPTY_FILE, origin = origin, mode = mode, - user = user, - group = group, + user = user or mapping_context.default_user, + group = group or mapping_context.default_group, + uid = uid or mapping_context.default_uid, + gid = gid or mapping_context.default_gid, ) -def add_label_list( - ctx, - content_map, - file_deps, - srcs, - default_mode = None, - default_user = None, - default_group = None): +def add_label_list(mapping_context, srcs): """Helper method to add a list of labels (typically 'srcs') to a content_map. Args: - ctx: rule context. - content_map: (r/w) The content map to update. - file_deps: (r/w) The list of file Depsets that srcs depend on. - srcs: List of source objects. - default_mode: fallback mode to use for Package*Info elements without mode - default_user: fallback user to use for Package*Info elements without user - default_group: fallback mode to use for Package*Info elements without group + mapping_context: (r/w) a MappingContext + srcs: List of source objects """ - if hasattr(ctx.attr, "include_runfiles"): - include_runfiles = ctx.attr.include_runfiles - else: - include_runfiles = False - # Compute the relative path data_path = compute_data_path( - ctx, - ctx.attr.strip_prefix if hasattr(ctx.attr, "strip_prefix") else "", + mapping_context.label, + mapping_context.strip_prefix, + ) + data_path_without_prefix = compute_data_path( + mapping_context.label, + ".", ) - data_path_without_prefix = compute_data_path(ctx, ".") for src in srcs: if not process_src( - content_map, - file_deps, + mapping_context, src = src, origin = src.label, - default_mode = default_mode, - default_user = default_user, - default_group = default_group, ): # Add in the files of srcs which are not pkg_* types add_from_default_info( - content_map, - file_deps, + mapping_context, src, data_path, data_path_without_prefix, - default_mode = default_mode, - default_user = default_user, - default_group = default_group, - include_runfiles = include_runfiles, + include_runfiles = mapping_context.include_runfiles, ) def add_from_default_info( - content_map, - file_deps, + mapping_context, src, data_path, data_path_without_prefix, - default_mode = None, - default_user = None, - default_group = None, include_runfiles = False): """Helper method to add the DefaultInfo of a target to a content_map. Args: - content_map: (r/w) The content map to update. - file_deps: (r/w) The list of file Depsets that srcs depend on. + mapping_context: (r/w) a MappingContext src: A source object. data_path: path to package data_path_without_prefix: path to the package after prefix stripping - default_mode: fallback mode to use for Package*Info elements without mode - default_user: fallback user to use for Package*Info elements without user - default_group: fallback mode to use for Package*Info elements without group include_runfiles: Include runfiles """ if not DefaultInfo in src: @@ -341,43 +395,54 @@ def add_from_default_info( the_executable = get_my_executable(src) all_files = src[DefaultInfo].files.to_list() for f in all_files: - d_path = dest_path(f, data_path, data_path_without_prefix) + d_path = mapping_context.path_mapper( + dest_path(f, data_path, data_path_without_prefix)) if f.is_directory: add_tree_artifact( - content_map, - d_path, - f, + mapping_context.content_map, + dest_path = d_path, + src = f, origin = src.label, - mode = default_mode, - user = default_user, - group = default_group, + mode = mapping_context.default_mode, + user = mapping_context.default_user, + group = mapping_context.default_group, ) else: - fmode = "0755" if f == the_executable else default_mode + fmode = "0755" if f == the_executable else mapping_context.default_mode add_single_file( - content_map, + mapping_context, dest_path = d_path, src = f, origin = src.label, mode = fmode, - user = default_user, - group = default_group, + user = mapping_context.default_user, + group = mapping_context.default_group, ) if include_runfiles: runfiles = src[DefaultInfo].default_runfiles if runfiles: - base_path = d_path + ".runfiles" + mapping_context.file_deps.append(runfiles.files) + + # Computing the runfiles root is subtle. It should be based off of + # the executable, but that is not always obvious. When in doubt, + # the first file of DefaultInfo.files should be the right target. + base_file = the_executable or all_files[0] + base_file_path = dest_path(base_file, data_path, data_path_without_prefix) + base_path = base_file_path + ".runfiles" + for rf in runfiles.files.to_list(): - d_path = base_path + "/" + rf.short_path - fmode = "0755" if rf == the_executable else default_mode - _check_dest(content_map, d_path, rf, src.label) - content_map[d_path] = _DestFile( + d_path = mapping_context.path_mapper(base_path + "/" + rf.short_path) + fmode = "0755" if rf == the_executable else mapping_context.default_mode + _check_dest(mapping_context.content_map, d_path, rf, src.label, mapping_context.allow_duplicates_with_different_content) + mapping_context.content_map[d_path] = _DestFile( src = rf, entry_type = ENTRY_IS_FILE, origin = src.label, mode = fmode, - user = default_user, - group = default_group, + user = mapping_context.default_user, + group = mapping_context.default_group, + uid = mapping_context.default_uid, + gid = mapping_context.default_gid, ) def get_my_executable(src): @@ -409,54 +474,61 @@ def get_my_executable(src): return ftr.executable return None -def add_single_file(content_map, dest_path, src, origin, mode = None, user = None, group = None): +def add_single_file(mapping_context, dest_path, src, origin, mode = None, user = None, group = None, uid = None, gid = None): """Add an single file to the content map. Args: - content_map: The content map + mapping_context: the MappingContext dest_path: Where to place the file in the package. src: Source object. Must have len(src[DefaultInfo].files) == 1 origin: The rule instance adding this entry mode: fallback mode to use for Package*Info elements without mode user: fallback user to use for Package*Info elements without user group: fallback mode to use for Package*Info elements without group + uid: numeric uid + gid: numeric gid """ dest = dest_path.strip("/") - _check_dest(content_map, dest, src, origin) - content_map[dest] = _DestFile( + _check_dest(mapping_context.content_map, dest, src, origin, mapping_context.allow_duplicates_with_different_content) + mapping_context.content_map[dest] = _DestFile( src = src, entry_type = ENTRY_IS_FILE, origin = origin, mode = mode, - user = user, - group = group, + user = user or mapping_context.default_user, + group = group or mapping_context.default_group, + uid = uid or mapping_context.default_uid, + gid = gid or mapping_context.default_gid, ) -def add_symlink(content_map, dest_path, src, origin, mode = None, user = None, group = None): +def add_symlink(mapping_context, dest_path, src, origin): """Add a symlink to the content map. + TODO(aiuto): This is a vestage left from the pkg_tar use. We could + converge code by having pkg_tar be a macro that expands symlinks to + pkg_symlink targets and srcs them in. + Args: - content_map: The content map + mapping_context: the MappingContext dest_path: Where to place the file in the package. src: Path to link to. origin: The rule instance adding this entry - mode: fallback mode to use for Package*Info elements without mode - user: fallback user to use for Package*Info elements without user - group: fallback mode to use for Package*Info elements without group """ dest = dest_path.strip("/") - _check_dest(content_map, dest, None, origin) - content_map[dest] = _DestFile( + _check_dest(mapping_context.content_map, dest, None, origin) + mapping_context.content_map[dest] = _DestFile( src = None, link_to = src, entry_type = ENTRY_IS_LINK, origin = origin, - mode = mode, - user = user, - group = group, + mode = mapping_context.default_mode, + user = mapping_context.default_user, + group = mapping_context.default_group, + uid = mapping_context.default_uid, + gid = mapping_context.default_gid, ) -def add_tree_artifact(content_map, dest_path, src, origin, mode = None, user = None, group = None): +def add_tree_artifact(content_map, dest_path, src, origin, mode = None, user = None, group = None, uid = None, gid = None): """Add an tree artifact (directory output) to the content map. Args: @@ -465,8 +537,10 @@ def add_tree_artifact(content_map, dest_path, src, origin, mode = None, user = N src: Source object. Must have len(src[DefaultInfo].files) == 1 origin: The rule instance adding this entry mode: fallback mode to use for Package*Info elements without mode - user: fallback user to use for Package*Info elements without user - group: fallback mode to use for Package*Info elements without group + user: User name for the entry (probably unused) + group: group name for the entry (probably unused) + uid: User id for the entry (probably unused) + gid: Group id for the entry (probably unused) """ content_map[dest_path] = _DestFile( src = src, @@ -475,9 +549,11 @@ def add_tree_artifact(content_map, dest_path, src, origin, mode = None, user = N mode = mode, user = user, group = group, + uid = uid, + gid = gid, ) -def write_manifest(ctx, manifest_file, content_map, use_short_path=False, pretty_print=False): +def write_manifest(ctx, manifest_file, content_map, use_short_path = False, pretty_print = False): """Write a content map to a manifest file. The format of this file is currently undocumented, as it is a private @@ -491,6 +567,7 @@ def write_manifest(ctx, manifest_file, content_map, use_short_path=False, pretty manifest_file: File object used as the output destination content_map: content_map (see concepts at top of file) use_short_path: write out the manifest file destinations in terms of "short" paths, suitable for `bazel run`. + pretty_print: indent the output nicely. Takes more space so it is off by default. """ ctx.actions.write( manifest_file, @@ -498,11 +575,11 @@ def write_manifest(ctx, manifest_file, content_map, use_short_path=False, pretty [ _encode_manifest_entry(dst, content_map[dst], use_short_path, pretty_print) for dst in sorted(content_map.keys()) - ] - ) + "\n]\n" + ], + ) + "\n]\n", ) -def _encode_manifest_entry(dest, df, use_short_path, pretty_print=False): +def _encode_manifest_entry(dest, df, use_short_path, pretty_print = False): entry_type = df.entry_type if hasattr(df, "entry_type") else ENTRY_IS_FILE if df.src: src = df.src.short_path if use_short_path else df.src.path @@ -522,16 +599,18 @@ def _encode_manifest_entry(dest, df, use_short_path, pretty_print=False): # Since this causes all sorts of chaos with our tests, be consistent across # all Bazel versions. origin_str = str(df.origin) - if not origin_str.startswith('@'): - origin_str = '@' + origin_str + if not origin_str.startswith("@"): + origin_str = "@" + origin_str data = { - "type": df.entry_type, + "type": entry_type, "src": src, "dest": dest.strip("/"), "mode": df.mode or "", "user": df.user or None, "group": df.group or None, + "uid": df.uid, + "gid": df.gid, "origin": origin_str, } |