diff options
Diffstat (limited to 'python/pip_install/pip_repository.bzl')
| -rw-r--r-- | python/pip_install/pip_repository.bzl | 622 |
1 files changed, 411 insertions, 211 deletions
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 99d1fb0..3e4878b 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -14,19 +14,29 @@ "" -load("//python:repositories.bzl", "get_interpreter_dirname", "is_standalone_interpreter") +load("@bazel_skylib//lib:sets.bzl", "sets") +load("//python:repositories.bzl", "is_standalone_interpreter") load("//python:versions.bzl", "WINDOWS_NAME") load("//python/pip_install:repositories.bzl", "all_requirements") load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") +load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") +load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:parse_whl_name.bzl", "parse_whl_name") +load("//python/private:patch_whl.bzl", "patch_whl") +load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases") load("//python/private:toolchains_repo.bzl", "get_host_os_arch") +load("//python/private:which.bzl", "which_with_fail") +load("//python/private:whl_target_platforms.bzl", "whl_target_platforms") CPPFLAGS = "CPPFLAGS" COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" +_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point" + def _construct_pypath(rctx): """Helper function to construct a PYTHONPATH. @@ -35,18 +45,15 @@ def _construct_pypath(rctx): Args: rctx: Handle to the repository_context. + Returns: String of the PYTHONPATH. """ - # Get the root directory of these rules - rules_root = rctx.path(Label("//:BUILD.bazel")).dirname - thirdparty_roots = [ - # Includes all the external dependencies from repositories.bzl - rctx.path(Label("@" + repo + "//:BUILD.bazel")).dirname - for repo in all_requirements - ] separator = ":" if not "windows" in rctx.os.name.lower() else ";" - pypath = separator.join([str(p) for p in [rules_root] + thirdparty_roots]) + pypath = separator.join([ + str(rctx.path(entry).dirname) + for entry in rctx.attr._python_path_entries + ]) return pypath def _get_python_interpreter_attr(rctx): @@ -71,7 +78,9 @@ def _resolve_python_interpreter(rctx): Args: rctx: Handle to the rule repository context. - Returns: Python interpreter path. + + Returns: + `path` object, for the resolved path to the Python interpreter. """ python_interpreter = _get_python_interpreter_attr(rctx) @@ -86,10 +95,13 @@ def _resolve_python_interpreter(rctx): if os == WINDOWS_NAME: python_interpreter = python_interpreter.realpath elif "/" not in python_interpreter: + # It's a plain command, e.g. "python3", to look up in the environment. found_python_interpreter = rctx.which(python_interpreter) if not found_python_interpreter: fail("python interpreter `{}` not found in PATH".format(python_interpreter)) python_interpreter = found_python_interpreter + else: + python_interpreter = rctx.path(python_interpreter) return python_interpreter def _get_xcode_location_cflags(rctx): @@ -104,10 +116,7 @@ def _get_xcode_location_cflags(rctx): if not rctx.os.name.lower().startswith("mac os"): return [] - # Locate xcode-select - xcode_select = rctx.which("xcode-select") - - xcode_sdk_location = rctx.execute([xcode_select, "--print-path"]) + xcode_sdk_location = rctx.execute([which_with_fail("xcode-select", rctx), "--print-path"]) if xcode_sdk_location.return_code != 0: return [] @@ -121,7 +130,7 @@ def _get_xcode_location_cflags(rctx): "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root), ] -def _get_toolchain_unix_cflags(rctx): +def _get_toolchain_unix_cflags(rctx, python_interpreter): """Gather cflags from a standalone toolchain for unix systems. Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg @@ -133,11 +142,11 @@ def _get_toolchain_unix_cflags(rctx): return [] # Only update the location when using a standalone toolchain. - if not is_standalone_interpreter(rctx, rctx.attr.python_interpreter_target): + if not is_standalone_interpreter(rctx, python_interpreter): return [] er = rctx.execute([ - rctx.path(rctx.attr.python_interpreter_target).realpath, + python_interpreter, "-c", "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')", ]) @@ -145,7 +154,7 @@ def _get_toolchain_unix_cflags(rctx): fail("could not get python version from interpreter (status {}): {}".format(er.return_code, er.stderr)) _python_version = er.stdout include_path = "{}/include/python{}".format( - get_interpreter_dirname(rctx, rctx.attr.python_interpreter_target), + python_interpreter.dirname, _python_version, ) @@ -216,11 +225,12 @@ def _parse_optional_attrs(rctx, args): return args -def _create_repository_execution_environment(rctx): +def _create_repository_execution_environment(rctx, python_interpreter): """Create a environment dictionary for processes we spawn with rctx.execute. Args: - rctx: The repository context. + rctx (repository_ctx): The repository context. + python_interpreter (path): The resolved python interpreter. Returns: Dictionary of environment variable suitable to pass to rctx.execute. """ @@ -228,7 +238,7 @@ def _create_repository_execution_environment(rctx): # Gather any available CPPFLAGS values cppflags = [] cppflags.extend(_get_xcode_location_cflags(rctx)) - cppflags.extend(_get_toolchain_unix_cflags(rctx)) + cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter)) env = { "PYTHONPATH": _construct_pypath(rctx), @@ -268,163 +278,49 @@ A requirements_lock attribute must be specified, or a platform-specific lockfile """) return requirements_txt -def _pkg_aliases(rctx, repo_name, bzl_packages): - """Create alias declarations for each python dependency. - - The aliases should be appended to the pip_repository BUILD.bazel file. These aliases - allow users to use requirement() without needed a corresponding `use_repo()` for each dep - when using bzlmod. - - Args: - rctx: the repository context. - repo_name: the repository name of the parent that is visible to the users. - bzl_packages: the list of packages to setup. - """ - for name in bzl_packages: - build_content = """package(default_visibility = ["//visibility:public"]) - -alias( - name = "{name}", - actual = "@{repo_name}_{dep}//:pkg", -) - -alias( - name = "pkg", - actual = "@{repo_name}_{dep}//:pkg", -) - -alias( - name = "whl", - actual = "@{repo_name}_{dep}//:whl", -) - -alias( - name = "data", - actual = "@{repo_name}_{dep}//:data", -) - -alias( - name = "dist_info", - actual = "@{repo_name}_{dep}//:dist_info", -) -""".format( - name = name, - repo_name = repo_name, - dep = name, - ) - rctx.file("{}/BUILD.bazel".format(name), build_content) - -def _create_pip_repository_bzlmod(rctx, bzl_packages, requirements): - repo_name = rctx.attr.repo_name - build_contents = _BUILD_FILE_CONTENTS - _pkg_aliases(rctx, repo_name, bzl_packages) - - # NOTE: we are using the canonical name with the double '@' in order to - # always uniquely identify a repository, as the labels are being passed as - # a string and the resolution of the label happens at the call-site of the - # `requirement`, et al. macros. - macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) - - rctx.file("BUILD.bazel", build_contents) - rctx.template("requirements.bzl", rctx.attr._template, substitutions = { - "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "data") - for p in bzl_packages - ]), - "%%ALL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, p) - for p in bzl_packages - ]), - "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "whl") - for p in bzl_packages - ]), - "%%MACRO_TMPL%%": macro_tmpl, - "%%NAME%%": rctx.attr.name, - "%%REQUIREMENTS_LOCK%%": requirements, - }) - -def _pip_hub_repository_bzlmod_impl(rctx): - bzl_packages = rctx.attr.whl_library_alias_names - _create_pip_repository_bzlmod(rctx, bzl_packages, "") - -pip_hub_repository_bzlmod_attrs = { - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", - ), - "whl_library_alias_names": attr.string_list( - mandatory = True, - doc = "The list of whl alias that we use to build aliases and the whl names", - ), - "_template": attr.label( - default = ":pip_hub_repository_requirements_bzlmod.bzl.tmpl", - ), -} - -pip_hub_repository_bzlmod = repository_rule( - attrs = pip_hub_repository_bzlmod_attrs, - doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""", - implementation = _pip_hub_repository_bzlmod_impl, -) - -def _pip_repository_bzlmod_impl(rctx): +def _pip_repository_impl(rctx): requirements_txt = locked_requirements_label(rctx, rctx.attr) content = rctx.read(requirements_txt) parsed_requirements_txt = parse_requirements(content) packages = [(normalize_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements] - bzl_packages = sorted([name for name, _ in packages]) - _create_pip_repository_bzlmod(rctx, bzl_packages, str(requirements_txt)) - -pip_repository_bzlmod_attrs = { - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name", - ), - "requirements_darwin": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Mac OS", - ), - "requirements_linux": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Linux", - ), - "requirements_lock": attr.label( - allow_single_file = True, - doc = """ -A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead -of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that -wheels are fetched/built only for the targets specified by 'build/run/test'. -""", - ), - "requirements_windows": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Windows", - ), - "_template": attr.label( - default = ":pip_repository_requirements_bzlmod.bzl.tmpl", - ), -} - -pip_repository_bzlmod = repository_rule( - attrs = pip_repository_bzlmod_attrs, - doc = """A rule for bzlmod pip_repository creation. Intended for private use only.""", - implementation = _pip_repository_bzlmod_impl, -) - -def _pip_repository_impl(rctx): - requirements_txt = locked_requirements_label(rctx, rctx.attr) - content = rctx.read(requirements_txt) - parsed_requirements_txt = parse_requirements(content) + bzl_packages = sorted([normalize_name(name) for name, _ in parsed_requirements_txt.requirements]) - packages = [(normalize_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements] + # Normalize cycles first + requirement_cycles = { + name: sorted(sets.to_list(sets.make(deps))) + for name, deps in rctx.attr.experimental_requirement_cycles.items() + } - bzl_packages = sorted([name for name, _ in packages]) + # Check for conflicts between cycles _before_ we normalize package names so + # that reported errors use the names the user specified + for i in range(len(requirement_cycles)): + left_group = requirement_cycles.keys()[i] + left_deps = requirement_cycles.values()[i] + for j in range(len(requirement_cycles) - (i + 1)): + right_deps = requirement_cycles.values()[1 + i + j] + right_group = requirement_cycles.keys()[1 + i + j] + for d in left_deps: + if d in right_deps: + fail("Error: Requirement %s cannot be repeated between cycles %s and %s; please merge the cycles." % (d, left_group, right_group)) + + # And normalize the names as used in the cycle specs + # + # NOTE: We must check that a listed dependency is actually in the actual + # requirements set for the current platform so that we can support cycles in + # platform-conditional requirements. Otherwise we'll blindly generate a + # label referencing a package which may not be installed on the current + # platform. + requirement_cycles = { + normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages]) + for name, group in requirement_cycles.items() + } imports = [ - 'load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")', + # NOTE: Maintain the order consistent with `buildifier` + 'load("@rules_python//python:pip.bzl", "pip_utils")', + 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")', ] annotations = {} @@ -456,28 +352,37 @@ def _pip_repository_impl(rctx): if rctx.attr.python_interpreter_target: config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target) + if rctx.attr.experimental_target_platforms: + config["experimental_target_platforms"] = rctx.attr.experimental_target_platforms if rctx.attr.incompatible_generate_aliases: - _pkg_aliases(rctx, rctx.attr.name, bzl_packages) + macro_tmpl = "@%s//{}:{}" % rctx.attr.name + aliases = render_pkg_aliases(repo_name = rctx.attr.name, bzl_packages = bzl_packages) + for path, contents in aliases.items(): + rctx.file(path, contents) + else: + macro_tmpl = "@%s_{}//:{}" % rctx.attr.name rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([ - "@{}//{}:data".format(rctx.attr.name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:data".format(rctx.attr.name, p) + macro_tmpl.format(p, "data") for p in bzl_packages ]), "%%ALL_REQUIREMENTS%%": _format_repr_list([ - "@{}//{}".format(rctx.attr.name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:pkg".format(rctx.attr.name, p) + macro_tmpl.format(p, "pkg") for p in bzl_packages ]), - "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ - "@{}//{}:whl".format(rctx.attr.name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:whl".format(rctx.attr.name, p) + "%%ALL_REQUIREMENT_GROUPS%%": _format_dict(_repr_dict(requirement_cycles)), + "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": _format_dict(_repr_dict({ + p: macro_tmpl.format(p, "whl") for p in bzl_packages - ]), + })), "%%ANNOTATIONS%%": _format_dict(_repr_dict(annotations)), "%%CONFIG%%": _format_dict(_repr_dict(config)), "%%EXTRA_PIP_ARGS%%": json.encode(options), - "%%IMPORTS%%": "\n".join(sorted(imports)), + "%%IMPORTS%%": "\n".join(imports), + "%%MACRO_TMPL%%": macro_tmpl, "%%NAME%%": rctx.attr.name, "%%PACKAGES%%": _format_repr_list( [ @@ -522,6 +427,86 @@ can be passed. """, default = {}, ), + "experimental_requirement_cycles": attr.string_list_dict( + default = {}, + doc = """\ +A mapping of dependency cycle names to a list of requirements which form that cycle. + +Requirements which form cycles will be installed together and taken as +dependencies together in order to ensure that the cycle is always satisified. + +Example: + `sphinx` depends on `sphinxcontrib-serializinghtml` + When listing both as requirements, ala + + ``` + py_binary( + name = "doctool", + ... + deps = [ + "@pypi//sphinx:pkg", + "@pypi//sphinxcontrib_serializinghtml", + ] + ) + ``` + + Will produce a Bazel error such as + + ``` + ERROR: .../external/pypi_sphinxcontrib_serializinghtml/BUILD.bazel:44:6: in alias rule @pypi_sphinxcontrib_serializinghtml//:pkg: cycle in dependency graph: + //:doctool (...) + @pypi//sphinxcontrib_serializinghtml:pkg (...) + .-> @pypi_sphinxcontrib_serializinghtml//:pkg (...) + | @pypi_sphinxcontrib_serializinghtml//:_pkg (...) + | @pypi_sphinx//:pkg (...) + | @pypi_sphinx//:_pkg (...) + `-- @pypi_sphinxcontrib_serializinghtml//:pkg (...) + ``` + + Which we can resolve by configuring these two requirements to be installed together as a cycle + + ``` + pip_parse( + ... + experimental_requirement_cycles = { + "sphinx": [ + "sphinx", + "sphinxcontrib-serializinghtml", + ] + }, + ) + ``` + +Warning: + If a dependency participates in multiple cycles, all of those cycles must be + collapsed down to one. For instance `a <-> b` and `a <-> c` cannot be listed + as two separate cycles. +""", + ), + "experimental_target_platforms": attr.string_list( + default = [], + doc = """\ +A list of platforms that we will generate the conditional dependency graph for +cross platform wheels by parsing the wheel metadata. This will generate the +correct dependencies for packages like `sphinx` or `pylint`, which include +`colorama` when installed and used on Windows platforms. + +An empty list means falling back to the legacy behaviour where the host +platform is the target platform. + +WARNING: It may not work as expected in cases where the python interpreter +implementation that is being used at runtime is different between different platforms. +This has been tested for CPython only. + +Special values: `all` (for generating deps for all platforms), `host` (for +generating deps for the host platform only). `linux_*` and other `<os>_*` values. +In the future we plan to set `all` as the default to this attribute. + +For specific target platforms use values of the form `<os>_<arch>` where `<os>` +is one of `linux`, `osx`, `windows` and arch is one of `x86_64`, `x86_32`, +`aarch64`, `s390x` and `ppc64le`. +""", + ), "extra_pip_args": attr.string_list( doc = "Extra arguments to pass on to pip. Must not contain spaces.", ), @@ -580,8 +565,21 @@ pip_repository_attrs = { doc = "Optional annotations to apply to packages", ), "incompatible_generate_aliases": attr.bool( - default = False, - doc = "Allow generating aliases '@pip//<pkg>' -> '@pip_<pkg>//:pkg'.", + default = True, + doc = """\ +If true, extra aliases will be created in the main `hub` repo - i.e. the repo +where the `requirements.bzl` is located. This means that for a Python package +`PyYAML` initialized within a `pip` `hub_repo` there will be the following +aliases generated: +- `@pip//pyyaml` will point to `@pip_pyyaml//:pkg` +- `@pip//pyyaml:data` will point to `@pip_pyyaml//:data` +- `@pip//pyyaml:dist_info` will point to `@pip_pyyaml//:dist_info` +- `@pip//pyyaml:pkg` will point to `@pip_pyyaml//:pkg` +- `@pip//pyyaml:whl` will point to `@pip_pyyaml//:whl` + +This is to keep the dependencies coming from PyPI to have more ergonomic label +names and support smooth transition to `bzlmod`. +""", ), "requirements_darwin": attr.label( allow_single_file = True, @@ -593,10 +591,14 @@ pip_repository_attrs = { ), "requirements_lock": attr.label( allow_single_file = True, - doc = """ -A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead -of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that -wheels are fetched/built only for the targets specified by 'build/run/test'. + doc = """\ +A fully resolved 'requirements.txt' pip requirement file containing the +transitive set of your dependencies. If this file is passed instead of +'requirements' no resolve will take place and pip_repository will create +individual repositories for each of your dependencies so that wheels are +fetched/built only for the targets specified by 'build/run/test'. Note that if +your lockfile is platform-dependent, you can use the `requirements_[platform]` +attributes. """, ), "requirements_windows": attr.label( @@ -612,22 +614,31 @@ pip_repository_attrs.update(**common_attrs) pip_repository = repository_rule( attrs = pip_repository_attrs, - doc = """A rule for importing `requirements.txt` dependencies into Bazel. + doc = """Accepts a locked/compiled requirements file and installs the dependencies listed within. + +Those dependencies become available in a generated `requirements.bzl` file. +You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below. + +In your WORKSPACE file: -This rule imports a `requirements.txt` file and generates a new -`requirements.bzl` file. This is used via the `WORKSPACE` pattern: +```starlark +load("@rules_python//python:pip.bzl", "pip_parse") -```python -pip_repository( - name = "foo", - requirements = ":requirements.txt", +pip_parse( + name = "pip_deps", + requirements_lock = ":requirements.txt", ) + +load("@pip_deps//:requirements.bzl", "install_deps") + +install_deps() ``` -You can then reference imported dependencies from your `BUILD` file with: +You can then reference installed dependencies from a `BUILD` file with: + +```starlark +load("@pip_deps//:requirements.bzl", "requirement") -```python -load("@foo//:requirements.bzl", "requirement") py_library( name = "bar", ... @@ -639,17 +650,52 @@ py_library( ) ``` -Or alternatively: -```python -load("@foo//:requirements.bzl", "all_requirements") -py_binary( - name = "baz", - ... - deps = [ - ":foo", - ] + all_requirements, +In addition to the `requirement` macro, which is used to access the generated `py_library` +target generated from a package's wheel, The generated `requirements.bzl` file contains +functionality for exposing [entry points][whl_ep] as `py_binary` targets as well. + +[whl_ep]: https://packaging.python.org/specifications/entry-points/ + +```starlark +load("@pip_deps//:requirements.bzl", "entry_point") + +alias( + name = "pip-compile", + actual = entry_point( + pkg = "pip-tools", + script = "pip-compile", + ), ) ``` + +Note that for packages whose name and script are the same, only the name of the package +is needed when calling the `entry_point` macro. + +```starlark +load("@pip_deps//:requirements.bzl", "entry_point") + +alias( + name = "flake8", + actual = entry_point("flake8"), +) +``` + +### Vendoring the requirements.bzl file + +In some cases you may not want to generate the requirements.bzl file as a repository rule +while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module +such as a ruleset, you may want to include the requirements.bzl file rather than make your users +install the WORKSPACE setup to generate it. +See https://github.com/bazelbuild/rules_python/issues/608 + +This is the same workflow as Gazelle, which creates `go_repository` rules with +[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) + +To do this, use the "write to source file" pattern documented in +https://blog.aspect.dev/bazel-can-write-to-the-source-folder +to put a copy of the generated requirements.bzl into your project. +Then load the requirements.bzl file directly rather than from the generated repository. +See the example in rules_python/examples/pip_parse_vendored. """, implementation = _pip_repository_impl, environ = common_env, @@ -663,23 +709,59 @@ def _whl_library_impl(rctx): "python.pip_install.tools.wheel_installer.wheel_installer", "--requirement", rctx.attr.requirement, - "--repo", - rctx.attr.repo, - "--repo-prefix", - rctx.attr.repo_prefix, ] - if rctx.attr.annotation: - args.extend([ - "--annotation", - rctx.path(rctx.attr.annotation), - ]) args = _parse_optional_attrs(rctx, args) + # Manually construct the PYTHONPATH since we cannot use the toolchain here + environment = _create_repository_execution_environment(rctx, python_interpreter) + result = rctx.execute( args, - # Manually construct the PYTHONPATH since we cannot use the toolchain here - environment = _create_repository_execution_environment(rctx), + environment = environment, + quiet = rctx.attr.quiet, + timeout = rctx.attr.timeout, + ) + if result.return_code: + fail("whl_library %s failed: %s (%s) error code: '%s'" % (rctx.attr.name, result.stdout, result.stderr, result.return_code)) + + whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"]) + if not rctx.delete("whl_file.json"): + fail("failed to delete the whl_file.json file") + + if rctx.attr.whl_patches: + patches = {} + for patch_file, json_args in rctx.attr.whl_patches.items(): + patch_dst = struct(**json.decode(json_args)) + if whl_path.basename in patch_dst.whls: + patches[patch_file] = patch_dst.patch_strip + + whl_path = patch_whl( + rctx, + python_interpreter = python_interpreter, + whl_path = whl_path, + patches = patches, + quiet = rctx.attr.quiet, + timeout = rctx.attr.timeout, + ) + + target_platforms = rctx.attr.experimental_target_platforms + if target_platforms: + parsed_whl = parse_whl_name(whl_path.basename) + if parsed_whl.platform_tag != "any": + # NOTE @aignas 2023-12-04: if the wheel is a platform specific + # wheel, we only include deps for that target platform + target_platforms = [ + "{}_{}".format(p.os, p.cpu) + for p in whl_target_platforms(parsed_whl.platform_tag) + ] + + result = rctx.execute( + args + [ + "--whl-file", + whl_path, + ] + ["--platform={}".format(p) for p in target_platforms], + environment = environment, quiet = rctx.attr.quiet, timeout = rctx.attr.timeout, ) @@ -687,8 +769,76 @@ def _whl_library_impl(rctx): if result.return_code: fail("whl_library %s failed: %s (%s) error code: '%s'" % (rctx.attr.name, result.stdout, result.stderr, result.return_code)) + metadata = json.decode(rctx.read("metadata.json")) + rctx.delete("metadata.json") + + entry_points = {} + for item in metadata["entry_points"]: + name = item["name"] + module = item["module"] + attribute = item["attribute"] + + # There is an extreme edge-case with entry_points that end with `.py` + # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 + entry_point_without_py = name[:-3] + "_py" if name.endswith(".py") else name + entry_point_target_name = ( + _WHEEL_ENTRY_POINT_PREFIX + "_" + entry_point_without_py + ) + entry_point_script_name = entry_point_target_name + ".py" + + rctx.file( + entry_point_script_name, + _generate_entry_point_contents(module, attribute), + ) + entry_points[entry_point_without_py] = entry_point_script_name + + build_file_contents = generate_whl_library_build_bazel( + repo_prefix = rctx.attr.repo_prefix, + whl_name = whl_path.basename, + dependencies = metadata["deps"], + dependencies_by_platform = metadata["deps_by_platform"], + group_name = rctx.attr.group_name, + group_deps = rctx.attr.group_deps, + data_exclude = rctx.attr.pip_data_exclude, + tags = [ + "pypi_name=" + metadata["name"], + "pypi_version=" + metadata["version"], + ], + entry_points = entry_points, + annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))), + ) + rctx.file("BUILD.bazel", build_file_contents) + return +def _generate_entry_point_contents( + module, + attribute, + shebang = "#!/usr/bin/env python3"): + """Generate the contents of an entry point script. + + Args: + module (str): The name of the module to use. + attribute (str): The name of the attribute to call. + shebang (str, optional): The shebang to use for the entry point python + file. + + Returns: + str: A string of python code. + """ + contents = """\ +{shebang} +import sys +from {module} import {attribute} +if __name__ == "__main__": + sys.exit({attribute}()) +""".format( + shebang = shebang, + module = module, + attribute = attribute, + ) + return contents + whl_library_attrs = { "annotation": attr.label( doc = ( @@ -697,6 +847,13 @@ whl_library_attrs = { ), allow_files = True, ), + "group_deps": attr.string_list( + doc = "List of dependencies to skip in order to break the cycles within a dependency group.", + default = [], + ), + "group_name": attr.string( + doc = "Name of the group, if any.", + ), "repo": attr.string( mandatory = True, doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.", @@ -705,6 +862,26 @@ whl_library_attrs = { mandatory = True, doc = "Python requirement string describing the package to make available", ), + "whl_patches": attr.label_keyed_string_dict( + doc = """a label-keyed-string dict that has + json.encode(struct([whl_file], patch_strip]) as values. This + is to maintain flexibility and correct bzlmod extension interface + until we have a better way to define whl_library and move whl + patching to a separate place. INTERNAL USE ONLY.""", + ), + "_python_path_entries": attr.label_list( + # Get the root directory of these rules and keep them as a default attribute + # in order to avoid unnecessary repository fetching restarts. + # + # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478 + default = [ + Label("//:BUILD.bazel"), + ] + [ + # Includes all the external dependencies from repositories.bzl + Label("@" + repo + "//:BUILD.bazel") + for repo in all_requirements + ], + ), } whl_library_attrs.update(**common_attrs) @@ -752,6 +929,29 @@ def package_annotation( srcs_exclude_glob = srcs_exclude_glob, )) +def _group_library_impl(rctx): + build_file_contents = generate_group_library_build_bazel( + repo_prefix = rctx.attr.repo_prefix, + groups = rctx.attr.groups, + ) + rctx.file("BUILD.bazel", build_file_contents) + +group_library = repository_rule( + attrs = { + "groups": attr.string_list_dict( + doc = "A mapping of group names to requirements within that group.", + ), + "repo_prefix": attr.string( + doc = "Prefix used for the whl_library created components of each group", + ), + }, + implementation = _group_library_impl, + doc = """ +Create a package containing only wrapper py_library and whl_library rules for implementing dependency groups. +This is an implementation detail of dependency groups and should not be used alone. + """, +) + # pip_repository implementation def _format_list(items): |