diff options
86 files changed, 2195 insertions, 1118 deletions
@@ -3,8 +3,8 @@ # This lets us glob() up all the files inside the examples to make them inputs to tests # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points test --test_output=errors diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0d305b8..66903df 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,7 @@ PR Instructions/requirements * Title uses `type: description` format. See CONTRIBUTING.md for types. -* Common types are: build, docs, feat, fix, refactor, revert, test + * Common types are: build, docs, feat, fix, refactor, revert, test + * Update `CHANGELOG.md` as applicable * Breaking changes include "!" after the type and a "BREAKING CHANGES:" section at the bottom. * Body text describes: diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..502545a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,80 @@ +# rules_python Changelog + +This is a human-friendly changelog in a keepachangelog.com style format. +Because this changelog is for end-user consumption of meaningful changes,only +a summary of a release's changes is described. This means every commit is not +necessarily mentioned, and internal refactors or code cleanups are omitted +unless they're particularly notable. + +A brief description of the categories of changes: + +* `Changed`: Some behavior changed. If the change is expected to break a + public API or supported behavior, it will be marked as **BREAKING**. Note that + beta APIs will not have breaking API changes called out. +* `Fixed`: A bug, or otherwise incorrect behavior, was fixed. +* `Added`: A new feature, API, or behavior was added in a backwards compatible + manner. +* Particular sub-systems are identified using parentheses, e.g. `(bzlmod)` or + `(docs)`. + +## [0.25.0] - 2023-08-22 + +### Changed + +* (bzlmod) `pip.parse` can no longer automatically use the default + Python version; this was an unreliable and unsafe behavior. The + `python_version` arg must always be explicitly specified. + +### Fixed + +* (docs) Update docs to use correct bzlmod APIs and clarify how and when to use + various APIs. +* (multi-version) The `main` arg is now correctly computed and usually optional. +* (bzlmod) `pip.parse` no longer requires a call for whatever the configured + default Python version is. + +### Added + +* Created a changelog. +* (gazelle) Stop generating unnecessary imports. +* (toolchains) s390x supported for Python 3.9.17, 3.10.12, and 3.11.4. + +[0.25.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.25.0 + +## [0.24.0] - 2023-07-11 + +### Changed + +* **BREAKING** (gazelle) Gazelle 0.30.0 or higher is required +* (bzlmod) `@python_aliases` renamed to `@python_versions +* (bzlmod) `pip.parse` arg `name` renamed to `hub_name` +* (bzlmod) `pip.parse` arg `incompatible_generate_aliases` removed and always + true. + +### Fixed + +* (bzlmod) Fixing Windows Python Interpreter symlink issues +* (py_wheel) Allow twine tags and args +* (toolchain, bzlmod) Restrict coverage tool visibility under bzlmod +* (pip) Ignore temporary pyc.NNN files in wheels +* (pip) Add format() calls to glob_exclude templates +* plugin_output in py_proto_library rule + +### Added + +* Using Gazelle's lifecycle manager to manage external processes +* (bzlmod) `pip.parse` can be called multiple times with different Python + versions +* (bzlmod) Allow bzlmod `pip.parse` to reference the default python toolchain and interpreter +* (bzlmod) Implementing wheel annotations via `whl_mods` +* (gazelle) support multiple requirements files in manifest generation +* (py_wheel) Support for specifying `Description-Content-Type` and `Summary` in METADATA +* (py_wheel) Support for specifying `Project-URL` +* (compile_pip_requirements) Added `generate_hashes` arg (default True) to + control generating hashes +* (pip) Create all_data_requirements alias +* Expose Python C headers through the toolchain. + +[0.24.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.24.0 + + diff --git a/DEVELOPING.md b/DEVELOPING.md index 2972d96..3c9e89d 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -1,5 +1,16 @@ # For Developers +## Updating internal dependencies + +1. Modify the `./python/pip_install/tools/requirements.txt` file and run: + ``` + bazel run //tools/private/update_deps:update_pip_deps + ``` +1. Bump the coverage dependencies using the script using: + ``` + bazel run //tools/private/update_deps:update_coverage_deps <VERSION> + ``` + ## Releasing Start from a clean checkout at `main`. @@ -1,15 +1,15 @@ name: "bazelbuild-rules_python" -description: - "A repository of Starlark implementation of Python rules in Bazel" - +description: "A repository of Starlark implementation of Python rules in Bazel" third_party { - url { - type: GIT + license_type: NOTICE + last_upgrade_date { + year: 2024 + month: 1 + day: 17 + } + identifier { + type: "Git" value: "https://github.com/bazelbuild/rules_python" + version: "0.25.0" } - version: "4082693e23ec9615f3e9b2ed9fae542e2b3bed12" - last_upgrade_date { year: 2023 month: 7 day: 24 } - license_type: NOTICE } - - diff --git a/MODULE.bazel b/MODULE.bazel index b7a0411..aaa5c86 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -15,6 +15,7 @@ internal_deps = use_extension("@rules_python//python/extensions/private:internal internal_deps.install() use_repo( internal_deps, + # START: maintained by 'bazel run //tools/private:update_pip_deps' "pypi__build", "pypi__click", "pypi__colorama", @@ -29,6 +30,7 @@ use_repo( "pypi__tomli", "pypi__wheel", "pypi__zipp", + # END: maintained by 'bazel run //tools/private:update_pip_deps' ) # We need to do another use_extension call to expose the "pythons_hub" @@ -1,37 +1,31 @@ # Python Rules for Bazel -* Postsubmit [](https://buildkite.com/bazel/python-rules-python-postsubmit) -* Postsubmit + Current Bazel Incompatible Flags [](https://buildkite.com/bazel/rules-python-plus-bazelisk-migrate) +[](https://buildkite.com/bazel/python-rules-python-postsubmit) ## Overview This repository is the home of the core Python rules -- `py_library`, `py_binary`, `py_test`, `py_proto_library`, and related symbols that provide the basis for Python -support in Bazel. It also contains package installation rules for integrating with PyPI and other package indices. Documentation lives in the +support in Bazel. It also contains package installation rules for integrating with PyPI and other indices. + +Documentation for rules_python lives in the [`docs/`](https://github.com/bazelbuild/rules_python/tree/main/docs) directory and in the [Bazel Build Encyclopedia](https://docs.bazel.build/versions/master/be/python.html). -Currently the core rules are bundled with Bazel itself, and the symbols in this -repository are simple aliases. However, in the future the rules will be -migrated to Starlark and debundled from Bazel. Therefore, the future-proof way -to depend on Python rules is via this repository. See[`Migrating from the Bundled Rules`](#Migrating-from-the-bundled-rules) below. +Examples live in the [examples](examples) directory. + +Currently, the core rules build into the Bazel binary, and the symbols in this +repository are simple aliases. However, we are migrating the rules to Starlark and removing them from the Bazel binary. Therefore, the future-proof way to depend on Python rules is via this repository. See[`Migrating from the Bundled Rules`](#Migrating-from-the-bundled-rules) below. The core rules are stable. Their implementation in Bazel is subject to Bazel's [backward compatibility policy](https://docs.bazel.build/versions/master/backward-compatibility.html). -Once they are fully migrated to rules_python, they may evolve at a different -rate, but this repository will still follow -[semantic versioning](https://semver.org). +Once migrated to rules_python, they may evolve at a different +rate, but this repository will still follow [semantic versioning](https://semver.org). -The package installation rules (`pip_install`, `pip_parse` etc.) are less stable. We may make breaking -changes as they evolve. +The Bazel community maintains this repository. Neither Google nor the Bazel team provides support for the code. However, this repository is part of the test suite used to vet new Bazel releases. See [How to contribute](CONTRIBUTING.md) page for information on our development workflow. -This repository is maintained by the Bazel community. Neither Google, nor the -Bazel team, provides support for the code. However, this repository is part of -the test suite used to vet new Bazel releases. See the [How to -contribute](CONTRIBUTING.md) page for information on our development workflow. - -## `bzlmod` support +## Bzlmod support - Status: Beta - Full Feature Parity: No @@ -40,71 +34,104 @@ See [Bzlmod support](BZLMOD_SUPPORT.md) for more details. ## Getting started -The next two sections cover using `rules_python` with bzlmod and +The following two sections cover using `rules_python` with bzlmod and the older way of configuring bazel with a `WORKSPACE` file. ### Using bzlmod -NOTE: bzlmod support is still experimental; APIs subject to change. +**IMPORTANT: bzlmod support is still in Beta; APIs are subject to change.** -To import rules_python in your project, you first need to add it to your -`MODULE.bazel` file, using the snippet provided in the -[release you choose](https://github.com/bazelbuild/rules_python/releases). +The first step to using rules_python with bzlmod is to add the dependency to +your MODULE.bazel file: -Once the dependency is added, a Python toolchain will be automatically -registered and you'll be able to create runnable programs and tests. +```starlark +# Update the version "0.0.0" to the release found here: +# https://github.com/bazelbuild/rules_python/releases. +bazel_dep(name = "rules_python", version = "0.0.0") +``` +Once added, you can load the rules and use them: -#### Toolchain registration with bzlmod +```starlark +load("@rules_python//python:py_binary.bzl", "py_binary") + +py_binary(...) +``` -NOTE: bzlmod support is still experimental; APIs subject to change. +Depending on what you're doing, you likely want to do some additional +configuration to control what Python version is used; read the following +sections for how to do that. -A default toolchain is automatically configured for by depending on +#### Toolchain registration with bzlmod + +A default toolchain is automatically configured depending on `rules_python`. Note, however, the version used tracks the most recent Python release and will change often. -If you want to register specific Python versions, then use -`python.toolchain()` for each version you need: +If you want to use a specific Python version for your programs, then how +to do so depends on if you're configuring the root module or not. The root +module is special because it can set the *default* Python version, which +is used by the version-unaware rules (e.g. `//python:py_binary.bzl` et al). For +submodules, it's recommended to use the version-aware rules to pin your programs +to a specific Python version so they don't accidentally run with a different +version configured by the root module. + +##### Configuring and using the default Python version + +To specify what the default Python version is, set `is_default = True` when +calling `python.toolchain()`. This can only be done by the root module; it is +silently ignored if a submodule does it. Similarly, using the version-unaware +rules (which always use the default Python version) should only be done by the +root module. If submodules use them, then they may run with a different Python +version than they expect. ```starlark -python = use_extension("@rules_python//python:extensions.bzl", "python") +python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( - python_version = "3.9", + python_version = "3.11", + is_default = True, ) ``` -### Using pip with bzlmod +Then use the base rules from e.g. `//python:py_binary.bzl`. + +##### Pinning to a Python version + +Pinning to a version allows targets to force that a specific Python version is +used, even if the root module configures a different version as a default. This +is most useful for two cases: -NOTE: bzlmod support is still experimental; APIs subject to change. +1. For submodules to ensure they run with the appropriate Python version +2. To allow incremental, per-target, upgrading to newer Python versions, + typically in a mono-repo situation. -To use dependencies from PyPI, the `pip.parse()` extension is used to -convert a requirements file into Bazel dependencies. +To configure a submodule with the version-aware rules, request the particular +version you need, then use the `@python_versions` repo to use the rules that +force specific versions: ```starlark python = use_extension("@rules_python//python/extensions:python.bzl", "python") + python.toolchain( - python_version = "3.9", + python_version = "3.11", ) +use_repo(python, "python_versions") +``` -interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter") -interpreter.install( - name = "interpreter", - python_name = "python_3_9", -) -use_repo(interpreter, "interpreter") +Then use e.g. `load("@python_versions//3.11:defs.bzl", "py_binary")` to use +the rules that force that particular version. Multiple versions can be specified +and use within a single build. -pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") -pip.parse( - hub_name = "pip", - python_interpreter_target = "@interpreter//:python", - requirements_lock = "//:requirements_lock.txt", - requirements_windows = "//:requirements_windows.txt", -) -use_repo(pip, "pip") -``` +For more documentation, see the bzlmod examples under the [examples](examples) folder. Look for the examples that contain a `MODULE.bazel` file. + +##### Other toolchain details -For more documentation see the bzlmod examples under the [examples](examples) folder. +The `python.toolchain()` call makes its contents available under a repo named +`python_X_Y`, where X and Y are the major and minor versions. For example, +`python.toolchain(python_version="3.11")` creates the repo `@python_3_11`. +Remember to call `use_repo()` to make repos visible to your module: +`use_repo(python, "python_3_11")` ### Using a WORKSPACE file @@ -112,37 +139,46 @@ To import rules_python in your project, you first need to add it to your `WORKSPACE` file, using the snippet provided in the [release you choose](https://github.com/bazelbuild/rules_python/releases) -To depend on a particular unreleased version, you can do: +To depend on a particular unreleased version, you can do the following: -```python +```starlark load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -rules_python_version = "740825b7f74930c62f44af95c9a4c1bd428d2c53" # Latest @ 2021-06-23 + +# Update the SHA and VERSION to the lastest version available here: +# https://github.com/bazelbuild/rules_python/releases. + +SHA="84aec9e21cc56fbc7f1335035a71c850d1b9b5cc6ff497306f84cced9a769841" + +VERSION="0.23.1" http_archive( name = "rules_python", - # Bazel will print the proper value to add here during the first build. - # sha256 = "FIXME", - strip_prefix = "rules_python-{}".format(rules_python_version), - url = "https://github.com/bazelbuild/rules_python/archive/{}.zip".format(rules_python_version), + sha256 = SHA, + strip_prefix = "rules_python-{}".format(VERSION), + url = "https://github.com/bazelbuild/rules_python/releases/download/{}/rules_python-{}.tar.gz".format(VERSION,VERSION), ) + +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() ``` #### Toolchain registration To register a hermetic Python toolchain rather than rely on a system-installed interpreter for runtime execution, you can add to the `WORKSPACE` file: -```python +```starlark load("@rules_python//python:repositories.bzl", "python_register_toolchains") python_register_toolchains( - name = "python3_9", + name = "python_3_11", # Available versions are listed in @rules_python//python:versions.bzl. # We recommend using the same version your team is already standardized on. - python_version = "3.9", + python_version = "3.11", ) -load("@python3_9//:defs.bzl", "interpreter") +load("@python_3_11//:defs.bzl", "interpreter") load("@rules_python//python:pip.bzl", "pip_parse") @@ -155,19 +191,18 @@ pip_parse( After registration, your Python targets will use the toolchain's interpreter during execution, but a system-installed interpreter is still used to 'bootstrap' Python targets (see https://github.com/bazelbuild/rules_python/issues/691). -You may also find some quirks while using this toolchain. Please refer to [python-build-standalone documentation's _Quirks_ section](https://python-build-standalone.readthedocs.io/en/latest/quirks.html) for details. +You may also find some quirks while using this toolchain. Please refer to [python-build-standalone documentation's _Quirks_ section](https://python-build-standalone.readthedocs.io/en/latest/quirks.html). ### Toolchain usage in other rules -Python toolchains can be utilised in other bazel rules, such as `genrule()`, by adding the `toolchains=["@rules_python//python:current_py_toolchain"]` attribute. The path to the python interpreter can be obtained by using the `$(PYTHON2)` and `$(PYTHON3)` ["Make" Variables](https://bazel.build/reference/be/make-variables). See the [`test_current_py_toolchain`](tests/load_from_macro/BUILD.bazel) target for an example. - +Python toolchains can be utilized in other bazel rules, such as `genrule()`, by adding the `toolchains=["@rules_python//python:current_py_toolchain"]` attribute. You can obtain the path to the Python interpreter using the `$(PYTHON2)` and `$(PYTHON3)` ["Make" Variables](https://bazel.build/reference/be/make-variables). See the [`test_current_py_toolchain`](tests/load_from_macro/BUILD.bazel) target for an example. ### "Hello World" Once you've imported the rule set into your `WORKSPACE` using any of these -methods, you can then load the core rules in your `BUILD` files with: +methods, you can then load the core rules in your `BUILD` files with the following: -``` python +```starlark load("@rules_python//python:defs.bzl", "py_binary") py_binary( @@ -176,44 +211,35 @@ py_binary( ) ``` -## Using the package installation rules +## Using dependencies from PyPI -Usage of the packaging rules involves two main steps. +Using PyPI packages (aka "pip install") involves two main steps. 1. [Installing third_party packages](#installing-third_party-packages) -2. [Using third_party packages as dependencies](#using-third_party-packages-as-dependencies) - -The package installation rules create two kinds of repositories: A central external repo that holds -downloaded wheel files, and individual external repos for each wheel's extracted -contents. Users only need to interact with the central external repo; the wheel repos -are essentially an implementation detail. The central external repo provides a -`WORKSPACE` macro to create the wheel repos, as well as a function, `requirement()`, for use in -`BUILD` files that translates a pip package name into the label of a `py_library` -target in the appropriate wheel repo. +2. [Using third_party packages as dependencies](#using-third_party-packages-as-dependencies ### Installing third_party packages #### Using bzlmod -To add pip dependencies to your `MODULE.bazel` file, use the `pip.parse` extension, and call it to create the -central external repo and individual wheel external repos. +To add pip dependencies to your `MODULE.bazel` file, use the `pip.parse` extension, and call it to create the central external repo and individual wheel external repos. Include in the `MODULE.bazel` the toolchain extension as shown in the first bzlmod example above. -```python +```starlark +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( hub_name = "my_deps", - requirements_lock = "//:requirements_lock.txt", + python_version = "3.11", + requirements_lock = "//:requirements_lock_3_11.txt", ) - use_repo(pip, "my_deps") ``` +For more documentation, including how the rules can update/create a requirements file, see the bzlmod examples under the [examples](examples) folder. #### Using a WORKSPACE file -To add pip dependencies to your `WORKSPACE`, load the `pip_parse` function, and call it to create the -central external repo and individual wheel external repos. - +To add pip dependencies to your `WORKSPACE`, load the `pip_parse` function and call it to create the central external repo and individual wheel external repos. -```python +```starlark load("@rules_python//python:pip.bzl", "pip_parse") # Create a central repo that knows about the dependencies needed from @@ -222,7 +248,7 @@ pip_parse( name = "my_deps", requirements_lock = "//path/to:requirements_lock.txt", ) -# Load the starlark macro which will define your dependencies. +# Load the starlark macro, which will define your dependencies. load("@my_deps//:requirements.bzl", "install_deps") # Call it to define repos for your requirements. install_deps() @@ -233,31 +259,31 @@ install_deps() Note that since `pip_parse` is a repository rule and therefore executes pip at WORKSPACE-evaluation time, Bazel has no information about the Python toolchain and cannot enforce that the interpreter used to invoke pip matches the interpreter used to run `py_binary` targets. By -default, `pip_parse` uses the system command `"python3"`. This can be overridden by passing the +default, `pip_parse` uses the system command `"python3"`. To override this, pass in the `python_interpreter` attribute or `python_interpreter_target` attribute to `pip_parse`. -You can have multiple `pip_parse`s in the same workspace. This will create multiple external repos that have no relation to one another, and may result in downloading the same wheels multiple times. +You can have multiple `pip_parse`s in the same workspace. Or use the pip extension multiple times when using bzlmod. +This configuration will create multiple external repos that have no relation to one another +and may result in downloading the same wheels numerous times. As with any repository rule, if you would like to ensure that `pip_parse` is -re-executed in order to pick up a non-hermetic change to your environment (e.g., +re-executed to pick up a non-hermetic change to your environment (e.g., updating your system `python` interpreter), you can force it to re-execute by running `bazel sync --only [pip_parse name]`. -Note: The `pip_install` rule is deprecated. `pip_parse` offers identical functionality and both `pip_install` -and `pip_parse` now have the same implementation. The name `pip_install` may be removed in a future version of the rules. -The maintainers have taken all reasonable efforts to faciliate a smooth transition, but some users of `pip_install` will -need to replace their existing `requirements.txt` with a fully resolved set of dependencies using a tool such as -`pip-tools` or the `compile_pip_requirements` repository rule. +Note: The `pip_install` rule is deprecated. `pip_parse` offers identical functionality, and both `pip_install` and `pip_parse` now have the same implementation. The name `pip_install` may be removed in a future version of the rules. + +The maintainers have made all reasonable efforts to facilitate a smooth transition. Still, some users of `pip_install` will need to replace their existing `requirements.txt` with a fully resolved set of dependencies using a tool such as `pip-tools` or the `compile_pip_requirements` repository rule. ### Using third_party packages as dependencies Each extracted wheel repo contains a `py_library` target representing the wheel's contents. There are two ways to access this library. The -first is using the `requirement()` function defined in the central +first uses the `requirement()` function defined in the central repo's `//:requirements.bzl` file. This function maps a pip package name to a label: -```python +```starlark load("@my_deps//:requirements.bzl", "requirement") py_library( @@ -273,15 +299,15 @@ py_library( The reason `requirement()` exists is that the pattern for the labels, while not expected to change frequently, is not guaranteed to be -stable. Using `requirement()` ensures that you do not have to refactor +stable. Using `requirement()` ensures you do not have to refactor your `BUILD` files if the pattern changes. On the other hand, using `requirement()` has several drawbacks; see [this issue][requirements-drawbacks] for an enumeration. If you don't -want to use `requirement()` then you can instead use the library -labels directly. For `pip_parse` the labels are of the form +want to use `requirement()`, you can use the library +labels directly instead. For `pip_parse`, the labels are of the following form: -``` +```starlark @{name}_{package}//:pkg ``` @@ -291,13 +317,13 @@ Bazel label names (e.g. `-`, `.`) replaced with `_`. If you need to update `name` from "old" to "new", then you can run the following buildozer command: -``` +```shell buildozer 'substitute deps @old_([^/]+)//:pkg @new_${1}//:pkg' //...:* ``` -For `pip_install` the labels are instead of the form +For `pip_install`, the labels are instead of the form: -``` +```starlark @{name}//pypi__{package} ``` @@ -305,15 +331,14 @@ For `pip_install` the labels are instead of the form #### 'Extras' dependencies -Any 'extras' specified in the requirements lock-file will be automatically added as transitive dependencies of the -package. In the example above, you'd just put `requirement("useful_dep")`. +Any 'extras' specified in the requirements lock file will be automatically added as transitive dependencies of the package. In the example above, you'd just put `requirement("useful_dep")`. ### Consuming Wheel Dists Directly -If you need to depend on the wheel dists themselves, for instance to pass them +If you need to depend on the wheel dists themselves, for instance, to pass them to some other packaging tool, you can get a handle to them with the `whl_requirement` macro. For example: -```python +```starlark filegroup( name = "whl_files", data = [ @@ -321,6 +346,14 @@ filegroup( ] ) ``` +# Python Gazelle plugin + +[Gazelle](https://github.com/bazelbuild/bazel-gazelle) +is a build file generator for Bazel projects. It can create new `BUILD.bazel` files for a project that follows language conventions and update existing build files to include new sources, dependencies, and options. + +Bazel may run Gazelle using the Gazelle rule, or it may be installed and run as a command line tool. + +See the documentation for Gazelle with rules_python [here](gazelle). ## Migrating from the bundled rules @@ -338,9 +371,10 @@ appropriate `load()` statements and rewrite uses of `native.py_*`. buildifier --lint=fix --warnings=native-py <files> ``` -Currently the `WORKSPACE` file needs to be updated manually as per [Getting +Currently, the `WORKSPACE` file needs to be updated manually as per [Getting started](#Getting-started) above. Note that Starlark-defined bundled symbols underneath `@bazel_tools//tools/python` are also deprecated. These are not yet rewritten by buildifier. + @@ -72,7 +72,7 @@ _py_gazelle_deps() # Install twine for our own runfiles wheel publishing. # Eventually we might want to install twine automatically for users too, see: # https://github.com/bazelbuild/rules_python/issues/1016. -load("@python//3.11.1:defs.bzl", "interpreter") +load("@python//3.11.4:defs.bzl", "interpreter") load("@rules_python//python:pip.bzl", "pip_parse") pip_parse( diff --git a/docs/pip.md b/docs/pip.md index 6b96607..b3ad331 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -18,7 +18,7 @@ whl_library_alias(<a href="#whl_library_alias-name">name</a>, <a href="#whl_libr | Name | Description | Type | Mandatory | Default | | :------------- | :------------- | :------------- | :------------- | :------------- | | <a id="whl_library_alias-name"></a>name | A unique name for this repository. | <a href="https://bazel.build/concepts/labels#target-names">Name</a> | required | | -| <a id="whl_library_alias-default_version"></a>default_version | - | String | required | | +| <a id="whl_library_alias-default_version"></a>default_version | Optional Python version in major.minor format, e.g. '3.10'.The Python version of the wheel to use when the versions from <code>version_map</code> don't match. This allows the default (version unaware) rules to match and select a wheel. If not specified, then the default rules won't be able to resolve a wheel and an error will occur. | String | optional | <code>""</code> | | <a id="whl_library_alias-repo_mapping"></a>repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<p>For example, an entry <code>"@foo": "@bar"</code> declares that, for any time this repository depends on <code>@foo</code> (such as a dependency on <code>@foo//some:target</code>, it should actually resolve that dependency within globally-declared <code>@bar</code> (<code>@bar//some:target</code>). | <a href="https://bazel.build/rules/lib/dict">Dictionary: String -> String</a> | required | | | <a id="whl_library_alias-version_map"></a>version_map | - | <a href="https://bazel.build/rules/lib/dict">Dictionary: String -> String</a> | required | | | <a id="whl_library_alias-wheel_name"></a>wheel_name | - | String | required | | diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index df88ae8..be9466d 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -11,10 +11,6 @@ local_path_override( path = "../..", ) -# Setting python.toolchain is optional as rules_python -# sets a toolchain for you, using the latest supported version -# of Python. We do recomend that you set a version here. - # We next initialize the python toolchain using the extension. # You can set different Python versions in this block. python = use_extension("@rules_python//python/extensions:python.bzl", "python") @@ -91,10 +87,9 @@ use_repo(pip, "whl_mods_hub") # call. # Alternatively, `python_interpreter_target` can be used to directly specify # the Python interpreter to run to resolve dependencies. -# Because we do not have a python_version defined here -# pip.parse uses the python toolchain that is set as default. pip.parse( hub_name = "pip", + python_version = "3.9", requirements_lock = "//:requirements_lock_3_9.txt", requirements_windows = "//:requirements_windows_3_9.txt", # These modifications were created above and we diff --git a/examples/bzlmod/other_module/BUILD.bazel b/examples/bzlmod/other_module/BUILD.bazel new file mode 100644 index 0000000..d50a3a0 --- /dev/null +++ b/examples/bzlmod/other_module/BUILD.bazel @@ -0,0 +1,9 @@ +load("@python_versions//3.11:defs.bzl", compile_pip_requirements_311 = "compile_pip_requirements") + +# NOTE: To update the requirements, you need to uncomment the rules_python +# override in the MODULE.bazel. +compile_pip_requirements_311( + name = "requirements", + requirements_in = "requirements.in", + requirements_txt = "requirements_lock_3_11.txt", +) diff --git a/examples/bzlmod/other_module/MODULE.bazel b/examples/bzlmod/other_module/MODULE.bazel index cc23a51..959501a 100644 --- a/examples/bzlmod/other_module/MODULE.bazel +++ b/examples/bzlmod/other_module/MODULE.bazel @@ -6,10 +6,20 @@ module( # that the parent module uses. bazel_dep(name = "rules_python", version = "") -# It is not best practice to use a python.toolchian in -# a submodule. This code only exists to test that -# we support doing this. This code is only for rules_python -# testing purposes. +# The story behind this commented out override: +# This override is necessary to generate/update the requirements file +# for this module. This is because running it via the outer +# module doesn't work -- the `requirements.update` target can't find +# the correct file to update. +# Running in the submodule itself works, but submodules using overrides +# is considered an error until Bazel 6.3, which prevents the outer module +# from depending on this module. +# So until 6.3 and higher is the minimum, we leave this commented out. +# local_path_override( +# module_name = "rules_python", +# path = "../../..", +# ) + PYTHON_NAME_39 = "python_3_9" PYTHON_NAME_311 = "python_3_11" @@ -29,6 +39,20 @@ python.toolchain( # created by the above python.toolchain calls. use_repo( python, + "python_versions", PYTHON_NAME_39, PYTHON_NAME_311, ) + +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "other_module_pip", + # NOTE: This version must be different than the root module's + # default python version. + # This is testing that a sub-module can use pip.parse() and only specify + # Python versions that DON'T include whatever the root-module's default + # Python version is. + python_version = "3.11", + requirements_lock = ":requirements_lock_3_11.txt", +) +use_repo(pip, "other_module_pip") diff --git a/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel b/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel index 6e37df8..021c969 100644 --- a/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel +++ b/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel @@ -1,4 +1,7 @@ -load("@python_3_11//:defs.bzl", py_binary_311 = "py_binary") +load( + "@python_3_11//:defs.bzl", + py_binary_311 = "py_binary", +) load("@rules_python//python:defs.bzl", "py_library") py_library( @@ -13,11 +16,16 @@ py_library( # used only when you need to support multiple versions of Python # in the same project. py_binary_311( - name = "lib_311", - srcs = ["lib.py"], + name = "bin", + srcs = ["bin.py"], data = ["data/data.txt"], + main = "bin.py", visibility = ["//visibility:public"], - deps = ["@rules_python//python/runfiles"], + deps = [ + ":lib", + "@other_module_pip//absl_py", + "@rules_python//python/runfiles", + ], ) exports_files(["data/data.txt"]) diff --git a/examples/bzlmod/other_module/other_module/pkg/bin.py b/examples/bzlmod/other_module/other_module/pkg/bin.py new file mode 100644 index 0000000..3e28ca2 --- /dev/null +++ b/examples/bzlmod/other_module/other_module/pkg/bin.py @@ -0,0 +1,6 @@ +import sys + +import absl + +print("Python version:", sys.version) +print("Module 'absl':", absl) diff --git a/examples/bzlmod/other_module/requirements.in b/examples/bzlmod/other_module/requirements.in new file mode 100644 index 0000000..b998a06 --- /dev/null +++ b/examples/bzlmod/other_module/requirements.in @@ -0,0 +1 @@ +absl-py diff --git a/examples/bzlmod/other_module/requirements_lock_3_11.txt b/examples/bzlmod/other_module/requirements_lock_3_11.txt new file mode 100644 index 0000000..7e350f2 --- /dev/null +++ b/examples/bzlmod/other_module/requirements_lock_3_11.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# bazel run //other_module/pkg:requirements.update +# +absl-py==1.4.0 \ + --hash=sha256:0d3fe606adfa4f7db64792dd4c7aee4ee0c38ab75dfd353b7a83ed3e957fcb47 \ + --hash=sha256:d2c244d01048ba476e7c080bd2c6df5e141d211de80223460d5b3b8a2a58433d + # via -r other_module/pkg/requirements.in diff --git a/examples/bzlmod/tests/other_module/BUILD.bazel b/examples/bzlmod/tests/other_module/BUILD.bazel new file mode 100644 index 0000000..1bd8a90 --- /dev/null +++ b/examples/bzlmod/tests/other_module/BUILD.bazel @@ -0,0 +1,14 @@ +# Tests to verify the root module can interact with the "other_module" +# submodule. +# +# Note that other_module is seen as "our_other_module" due to repo-remapping +# in the root module. + +load("@bazel_skylib//rules:build_test.bzl", "build_test") + +build_test( + name = "other_module_bin_build_test", + targets = [ + "@our_other_module//other_module/pkg:bin", + ], +) diff --git a/gazelle/python/target.go b/gazelle/python/target.go index fdc99fc..e310405 100644 --- a/gazelle/python/target.go +++ b/gazelle/python/target.go @@ -122,6 +122,11 @@ func (t *targetBuilder) setTestonly() *targetBuilder { // case, the value we add is on Bazel sub-packages to be able to perform imports // relative to the root project package. func (t *targetBuilder) generateImportsAttribute() *targetBuilder { + if t.pythonProjectRoot == "" { + // When gazelle:python_root is not set or is at the root of the repo, we don't need + // to set imports, because that's the Bazel's default. + return t + } p, _ := filepath.Rel(t.bzlPackage, t.pythonProjectRoot) p = filepath.Clean(p) if p == "." { diff --git a/gazelle/python/testdata/dependency_resolution_order/bar/BUILD.out b/gazelle/python/testdata/dependency_resolution_order/bar/BUILD.out index da9915d..5291471 100644 --- a/gazelle/python/testdata/dependency_resolution_order/bar/BUILD.out +++ b/gazelle/python/testdata/dependency_resolution_order/bar/BUILD.out @@ -3,6 +3,5 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "bar", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], ) diff --git a/gazelle/python/testdata/dependency_resolution_order/baz/BUILD.out b/gazelle/python/testdata/dependency_resolution_order/baz/BUILD.out index 749fd3d..fadf5c1 100644 --- a/gazelle/python/testdata/dependency_resolution_order/baz/BUILD.out +++ b/gazelle/python/testdata/dependency_resolution_order/baz/BUILD.out @@ -3,6 +3,5 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "baz", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], ) diff --git a/gazelle/python/testdata/dependency_resolution_order/foo/BUILD.out b/gazelle/python/testdata/dependency_resolution_order/foo/BUILD.out index 4404d30..58498ee 100644 --- a/gazelle/python/testdata/dependency_resolution_order/foo/BUILD.out +++ b/gazelle/python/testdata/dependency_resolution_order/foo/BUILD.out @@ -3,6 +3,5 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "foo", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], ) diff --git a/gazelle/python/testdata/dependency_resolution_order/somewhere/bar/BUILD.out b/gazelle/python/testdata/dependency_resolution_order/somewhere/bar/BUILD.out index a0d421b..5291471 100644 --- a/gazelle/python/testdata/dependency_resolution_order/somewhere/bar/BUILD.out +++ b/gazelle/python/testdata/dependency_resolution_order/somewhere/bar/BUILD.out @@ -3,6 +3,5 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "bar", srcs = ["__init__.py"], - imports = ["../.."], visibility = ["//:__subpackages__"], ) diff --git a/gazelle/python/testdata/first_party_file_and_directory_modules/foo/BUILD.out b/gazelle/python/testdata/first_party_file_and_directory_modules/foo/BUILD.out index 3decd90..8c54e3c 100644 --- a/gazelle/python/testdata/first_party_file_and_directory_modules/foo/BUILD.out +++ b/gazelle/python/testdata/first_party_file_and_directory_modules/foo/BUILD.out @@ -6,7 +6,6 @@ py_library( "__init__.py", "bar.py", ], - imports = [".."], visibility = ["//:__subpackages__"], deps = ["//one"], ) diff --git a/gazelle/python/testdata/first_party_file_and_directory_modules/one/BUILD.out b/gazelle/python/testdata/first_party_file_and_directory_modules/one/BUILD.out index 7063141..3ae64b6 100644 --- a/gazelle/python/testdata/first_party_file_and_directory_modules/one/BUILD.out +++ b/gazelle/python/testdata/first_party_file_and_directory_modules/one/BUILD.out @@ -6,6 +6,5 @@ py_library( "__init__.py", "two.py", ], - imports = [".."], visibility = ["//:__subpackages__"], ) diff --git a/gazelle/python/testdata/from_imports/foo/BUILD.out b/gazelle/python/testdata/from_imports/foo/BUILD.out index 4404d30..58498ee 100644 --- a/gazelle/python/testdata/from_imports/foo/BUILD.out +++ b/gazelle/python/testdata/from_imports/foo/BUILD.out @@ -3,6 +3,5 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "foo", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], ) diff --git a/gazelle/python/testdata/from_imports/import_from_init_py/BUILD.out b/gazelle/python/testdata/from_imports/import_from_init_py/BUILD.out index 99b4861..8098aa7 100644 --- a/gazelle/python/testdata/from_imports/import_from_init_py/BUILD.out +++ b/gazelle/python/testdata/from_imports/import_from_init_py/BUILD.out @@ -3,7 +3,6 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "import_from_init_py", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], deps = ["//foo/bar"], )
\ No newline at end of file diff --git a/gazelle/python/testdata/from_imports/import_from_multiple/BUILD.out b/gazelle/python/testdata/from_imports/import_from_multiple/BUILD.out index d8219bb..f5e113b 100644 --- a/gazelle/python/testdata/from_imports/import_from_multiple/BUILD.out +++ b/gazelle/python/testdata/from_imports/import_from_multiple/BUILD.out @@ -3,7 +3,6 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "import_from_multiple", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], deps = [ "//foo/bar", diff --git a/gazelle/python/testdata/from_imports/import_nested_file/BUILD.out b/gazelle/python/testdata/from_imports/import_nested_file/BUILD.out index 662da9c..930216b 100644 --- a/gazelle/python/testdata/from_imports/import_nested_file/BUILD.out +++ b/gazelle/python/testdata/from_imports/import_nested_file/BUILD.out @@ -3,7 +3,6 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "import_nested_file", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], deps = ["//foo/bar:baz"], )
\ No newline at end of file diff --git a/gazelle/python/testdata/from_imports/import_nested_module/BUILD.out b/gazelle/python/testdata/from_imports/import_nested_module/BUILD.out index ec6da50..51d3b8c 100644 --- a/gazelle/python/testdata/from_imports/import_nested_module/BUILD.out +++ b/gazelle/python/testdata/from_imports/import_nested_module/BUILD.out @@ -3,7 +3,6 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "import_nested_module", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], deps = ["//foo/bar"], )
\ No newline at end of file diff --git a/gazelle/python/testdata/from_imports/import_nested_var/BUILD.out b/gazelle/python/testdata/from_imports/import_nested_var/BUILD.out index 8ee527e..2129c32 100644 --- a/gazelle/python/testdata/from_imports/import_nested_var/BUILD.out +++ b/gazelle/python/testdata/from_imports/import_nested_var/BUILD.out @@ -3,7 +3,6 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "import_nested_var", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], deps = ["//foo/bar:baz"], )
\ No newline at end of file diff --git a/gazelle/python/testdata/from_imports/import_top_level_var/BUILD.out b/gazelle/python/testdata/from_imports/import_top_level_var/BUILD.out index 6b584d7..c8ef6f4 100644 --- a/gazelle/python/testdata/from_imports/import_top_level_var/BUILD.out +++ b/gazelle/python/testdata/from_imports/import_top_level_var/BUILD.out @@ -3,7 +3,6 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "import_top_level_var", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], deps = ["//foo"], )
\ No newline at end of file diff --git a/gazelle/python/testdata/from_imports/std_module/BUILD.out b/gazelle/python/testdata/from_imports/std_module/BUILD.out index 4903999..b3597a9 100644 --- a/gazelle/python/testdata/from_imports/std_module/BUILD.out +++ b/gazelle/python/testdata/from_imports/std_module/BUILD.out @@ -3,6 +3,5 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "std_module", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], )
\ No newline at end of file diff --git a/gazelle/python/testdata/naming_convention/dont_rename/BUILD.out b/gazelle/python/testdata/naming_convention/dont_rename/BUILD.out index 4d4ead8..8d418be 100644 --- a/gazelle/python/testdata/naming_convention/dont_rename/BUILD.out +++ b/gazelle/python/testdata/naming_convention/dont_rename/BUILD.out @@ -3,14 +3,12 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") py_library( name = "dont_rename", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], ) py_binary( name = "my_dont_rename_binary", srcs = ["__main__.py"], - imports = [".."], main = "__main__.py", visibility = ["//:__subpackages__"], deps = [":dont_rename"], @@ -19,7 +17,6 @@ py_binary( py_test( name = "my_dont_rename_test", srcs = ["__test__.py"], - imports = [".."], main = "__test__.py", deps = [":dont_rename"], ) diff --git a/gazelle/python/testdata/naming_convention/resolve_conflict/BUILD.out b/gazelle/python/testdata/naming_convention/resolve_conflict/BUILD.out index 3fa5de2..e155fa6 100644 --- a/gazelle/python/testdata/naming_convention/resolve_conflict/BUILD.out +++ b/gazelle/python/testdata/naming_convention/resolve_conflict/BUILD.out @@ -9,14 +9,12 @@ go_test(name = "resolve_conflict_test") py_library( name = "my_resolve_conflict_library", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], ) py_binary( name = "my_resolve_conflict_binary", srcs = ["__main__.py"], - imports = [".."], main = "__main__.py", visibility = ["//:__subpackages__"], deps = [":my_resolve_conflict_library"], @@ -25,7 +23,6 @@ py_binary( py_test( name = "my_resolve_conflict_test", srcs = ["__test__.py"], - imports = [".."], main = "__test__.py", deps = [":my_resolve_conflict_library"], ) diff --git a/gazelle/python/testdata/python_ignore_files_directive/bar/BUILD.out b/gazelle/python/testdata/python_ignore_files_directive/bar/BUILD.out index af3c398..94259f9 100644 --- a/gazelle/python/testdata/python_ignore_files_directive/bar/BUILD.out +++ b/gazelle/python/testdata/python_ignore_files_directive/bar/BUILD.out @@ -3,6 +3,5 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "bar", srcs = ["baz.py"], - imports = [".."], visibility = ["//:__subpackages__"], ) diff --git a/gazelle/python/testdata/relative_imports/package2/BUILD.out b/gazelle/python/testdata/relative_imports/package2/BUILD.out index bbbc9f8..cf61691 100644 --- a/gazelle/python/testdata/relative_imports/package2/BUILD.out +++ b/gazelle/python/testdata/relative_imports/package2/BUILD.out @@ -8,6 +8,5 @@ py_library( "module4.py", "subpackage1/module5.py", ], - imports = [".."], visibility = ["//:__subpackages__"], ) diff --git a/gazelle/python/testdata/sibling_imports/pkg/BUILD.out b/gazelle/python/testdata/sibling_imports/pkg/BUILD.out index edb40a8..cae6c3f 100644 --- a/gazelle/python/testdata/sibling_imports/pkg/BUILD.out +++ b/gazelle/python/testdata/sibling_imports/pkg/BUILD.out @@ -7,20 +7,17 @@ py_library( "a.py", "b.py", ], - imports = [".."], visibility = ["//:__subpackages__"], ) py_test( name = "test_util", srcs = ["test_util.py"], - imports = [".."], ) py_test( name = "unit_test", srcs = ["unit_test.py"], - imports = [".."], deps = [ ":pkg", ":test_util", diff --git a/gazelle/python/testdata/simple_library_without_init/foo/BUILD.out b/gazelle/python/testdata/simple_library_without_init/foo/BUILD.out index 2faa046..8e50095 100644 --- a/gazelle/python/testdata/simple_library_without_init/foo/BUILD.out +++ b/gazelle/python/testdata/simple_library_without_init/foo/BUILD.out @@ -3,6 +3,5 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "foo", srcs = ["foo.py"], - imports = [".."], visibility = ["//:__subpackages__"], ) diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/BUILD.out b/gazelle/python/testdata/simple_test_with_conftest/bar/BUILD.out index e42c499..4a1204e 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/bar/BUILD.out +++ b/gazelle/python/testdata/simple_test_with_conftest/bar/BUILD.out @@ -6,7 +6,6 @@ py_library( "__init__.py", "bar.py", ], - imports = [".."], visibility = ["//:__subpackages__"], ) @@ -14,14 +13,12 @@ py_library( name = "conftest", testonly = True, srcs = ["conftest.py"], - imports = [".."], visibility = ["//:__subpackages__"], ) py_test( name = "bar_test", srcs = ["__test__.py"], - imports = [".."], main = "__test__.py", deps = [ ":bar", diff --git a/gazelle/python/testdata/subdir_sources/foo/BUILD.out b/gazelle/python/testdata/subdir_sources/foo/BUILD.out index f99857d..9107d2d 100644 --- a/gazelle/python/testdata/subdir_sources/foo/BUILD.out +++ b/gazelle/python/testdata/subdir_sources/foo/BUILD.out @@ -8,6 +8,5 @@ py_library( "baz/baz.py", "foo.py", ], - imports = [".."], visibility = ["//:__subpackages__"], ) diff --git a/gazelle/python/testdata/subdir_sources/foo/has_build/BUILD.out b/gazelle/python/testdata/subdir_sources/foo/has_build/BUILD.out index 0ef0cc1..d5196e5 100644 --- a/gazelle/python/testdata/subdir_sources/foo/has_build/BUILD.out +++ b/gazelle/python/testdata/subdir_sources/foo/has_build/BUILD.out @@ -3,6 +3,5 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "has_build", srcs = ["python/my_module.py"], - imports = ["../.."], visibility = ["//:__subpackages__"], ) diff --git a/gazelle/python/testdata/subdir_sources/foo/has_init/BUILD.out b/gazelle/python/testdata/subdir_sources/foo/has_init/BUILD.out index ce59ee2..de61008 100644 --- a/gazelle/python/testdata/subdir_sources/foo/has_init/BUILD.out +++ b/gazelle/python/testdata/subdir_sources/foo/has_init/BUILD.out @@ -6,6 +6,5 @@ py_library( "__init__.py", "python/my_module.py", ], - imports = ["../.."], visibility = ["//:__subpackages__"], ) diff --git a/gazelle/python/testdata/subdir_sources/foo/has_main/BUILD.out b/gazelle/python/testdata/subdir_sources/foo/has_main/BUILD.out index 265c08b..1c56f72 100644 --- a/gazelle/python/testdata/subdir_sources/foo/has_main/BUILD.out +++ b/gazelle/python/testdata/subdir_sources/foo/has_main/BUILD.out @@ -3,14 +3,12 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library") py_library( name = "has_main", srcs = ["python/my_module.py"], - imports = ["../.."], visibility = ["//:__subpackages__"], ) py_binary( name = "has_main_bin", srcs = ["__main__.py"], - imports = ["../.."], main = "__main__.py", visibility = ["//:__subpackages__"], deps = [":has_main"], diff --git a/gazelle/python/testdata/subdir_sources/foo/has_test/BUILD.out b/gazelle/python/testdata/subdir_sources/foo/has_test/BUILD.out index 80739d9..a99278e 100644 --- a/gazelle/python/testdata/subdir_sources/foo/has_test/BUILD.out +++ b/gazelle/python/testdata/subdir_sources/foo/has_test/BUILD.out @@ -3,14 +3,12 @@ load("@rules_python//python:defs.bzl", "py_library", "py_test") py_library( name = "has_test", srcs = ["python/my_module.py"], - imports = ["../.."], visibility = ["//:__subpackages__"], ) py_test( name = "has_test_test", srcs = ["__test__.py"], - imports = ["../.."], main = "__test__.py", deps = [":has_test"], ) diff --git a/gazelle/python/testdata/subdir_sources/one/BUILD.out b/gazelle/python/testdata/subdir_sources/one/BUILD.out index f2e5745..b78b650 100644 --- a/gazelle/python/testdata/subdir_sources/one/BUILD.out +++ b/gazelle/python/testdata/subdir_sources/one/BUILD.out @@ -3,6 +3,5 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "one", srcs = ["__init__.py"], - imports = [".."], visibility = ["//:__subpackages__"], ) diff --git a/gazelle/python/testdata/subdir_sources/one/two/BUILD.out b/gazelle/python/testdata/subdir_sources/one/two/BUILD.out index f632eed..8f0ac17 100644 --- a/gazelle/python/testdata/subdir_sources/one/two/BUILD.out +++ b/gazelle/python/testdata/subdir_sources/one/two/BUILD.out @@ -6,7 +6,6 @@ py_library( "__init__.py", "three.py", ], - imports = ["../.."], visibility = ["//:__subpackages__"], deps = ["//foo"], ) diff --git a/python/extensions/pip.bzl b/python/extensions/pip.bzl index add69a4..3ba0d3e 100644 --- a/python/extensions/pip.bzl +++ b/python/extensions/pip.bzl @@ -15,9 +15,9 @@ "pip module extension for use with bzlmod" load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") -load("@rules_python//python:pip.bzl", "whl_library_alias") +load("//python:pip.bzl", "whl_library_alias") load( - "@rules_python//python/pip_install:pip_repository.bzl", + "//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_hub_repository_bzlmod", "pip_repository_attrs", @@ -25,7 +25,7 @@ load( "use_isolated", "whl_library", ) -load("@rules_python//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") +load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:version_label.bzl", "version_label") @@ -296,15 +296,10 @@ def _pip_impl(module_ctx): for hub_name, whl_map in hub_whl_map.items(): for whl_name, version_map in whl_map.items(): - if DEFAULT_PYTHON_VERSION not in version_map: - fail(( - "Default python version '{version}' missing in pip " + - "hub '{hub}': update your pip.parse() calls so that " + - 'includes `python_version = "{version}"`' - ).format( - version = DEFAULT_PYTHON_VERSION, - hub = hub_name, - )) + if DEFAULT_PYTHON_VERSION in version_map: + whl_default_version = DEFAULT_PYTHON_VERSION + else: + whl_default_version = None # Create the alias repositories which contains different select # statements These select statements point to the different pip @@ -312,7 +307,7 @@ def _pip_impl(module_ctx): whl_library_alias( name = hub_name + "_" + whl_name, wheel_name = whl_name, - default_version = DEFAULT_PYTHON_VERSION, + default_version = whl_default_version, version_map = version_map, ) @@ -348,23 +343,22 @@ Targets from different hubs should not be used together. """, ), "python_version": attr.string( - default = DEFAULT_PYTHON_VERSION, + mandatory = True, doc = """ -The Python version to use for resolving the pip dependencies. If not specified, -then the default Python version (as set by the root module or rules_python) -will be used. +The Python version to use for resolving the pip dependencies, in Major.Minor +format (e.g. "3.11"). Patch level granularity (e.g. "3.11.1") is not supported. +If not specified, then the default Python version (as set by the root module or +rules_python) will be used. The version specified here must have a corresponding `python.toolchain()` -configured. This attribute defaults to the version of the toolchain -that is set as the default Python version. Or if only one toolchain -is used, this attribute defaults to that version of Python. +configured. """, ), "whl_modifications": attr.label_keyed_string_dict( mandatory = False, doc = """\ A dict of labels to wheel names that is typically generated by the whl_modifications. -The labels are JSON config files describing the modifications. +The labels are JSON config files describing the modifications. """, ), }, **pip_repository_attrs) @@ -397,7 +391,7 @@ executable.""", ), "copy_files": attr.string_dict( doc = """\ -(dict, optional): A mapping of `src` and `out` files for +(dict, optional): A mapping of `src` and `out` files for [@bazel_skylib//rules:copy_file.bzl][cf]""", ), "data": attr.string_list( @@ -458,10 +452,10 @@ the BUILD files for wheels. attrs = _pip_parse_ext_attrs(), doc = """\ This tag class is used to create a pip hub and all of the spokes that are part of that hub. -This tag class reuses most of the pip attributes that are found in +This tag class reuses most of the pip attributes that are found in @rules_python//python/pip_install:pip_repository.bzl. -The exceptions are it does not use the args 'repo_prefix', -and 'incompatible_generate_aliases'. We set the repository prefix +The exceptions are it does not use the args 'repo_prefix', +and 'incompatible_generate_aliases'. We set the repository prefix for the user and the alias arg is always True in bzlmod. """, ), @@ -485,7 +479,7 @@ def _whl_mods_repo_impl(rctx): _whl_mods_repo = repository_rule( doc = """\ -This rule creates json files based on the whl_mods attribute. +This rule creates json files based on the whl_mods attribute. """, implementation = _whl_mods_repo_impl, attrs = { diff --git a/python/extensions/private/internal_deps.bzl b/python/extensions/private/internal_deps.bzl index 27e290c..8a98b82 100644 --- a/python/extensions/private/internal_deps.bzl +++ b/python/extensions/private/internal_deps.bzl @@ -8,7 +8,7 @@ "Python toolchain module extension for internal rule use" -load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") +load("//python/pip_install:repositories.bzl", "pip_install_dependencies") # buildifier: disable=unused-variable def _internal_deps_impl(module_ctx): diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index 2d4032a..2d00726 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -250,7 +250,9 @@ A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. ), "python_version": attr.string( mandatory = True, - doc = "The Python version, in `major.minor` format, e.g '3.12', to create a toolchain for.", + doc = "The Python version, in `major.minor` format, e.g " + + "'3.12', to create a toolchain for. Patch level " + + "granularity (e.g. '3.12.1') is not supported.", ), }, ), diff --git a/python/pip.bzl b/python/pip.bzl index cae1591..0c6e90f 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -17,6 +17,7 @@ load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annot load("//python/pip_install:repositories.bzl", "pip_install_dependencies") load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") +load("//python/private:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE") load(":versions.bzl", "MINOR_MAPPING") compile_pip_requirements = _compile_pip_requirements @@ -260,13 +261,10 @@ _multi_pip_parse = repository_rule( def _whl_library_alias_impl(rctx): rules_python = rctx.attr._rules_python_workspace.workspace_name - if rctx.attr.default_version not in rctx.attr.version_map: - fail( - """ -Unable to find '{}' in your version map, you may need to update your requirement files. - """.format(rctx.attr.version_map), - ) - default_repo_prefix = rctx.attr.version_map[rctx.attr.default_version] + if rctx.attr.default_version: + default_repo_prefix = rctx.attr.version_map[rctx.attr.default_version] + else: + default_repo_prefix = None version_map = rctx.attr.version_map.items() build_content = ["# Generated by python/pip.bzl"] for alias_name in ["pkg", "whl", "data", "dist_info"]: @@ -289,6 +287,7 @@ def _whl_library_render_alias_target( # is canonical, so we have to add a second @. if BZLMOD_ENABLED: rules_python = "@" + rules_python + alias = ["""\ alias( name = "{alias_name}", @@ -304,23 +303,42 @@ alias( ), rules_python = rules_python, )) - alias.append("""\ - "//conditions:default": "{default_actual}", - }}), - visibility = ["//visibility:public"], -)""".format( + if default_repo_prefix: default_actual = "@{repo_prefix}{wheel_name}//:{alias_name}".format( repo_prefix = default_repo_prefix, wheel_name = wheel_name, alias_name = alias_name, - ), - )) + ) + alias.append(' "//conditions:default": "{default_actual}",'.format( + default_actual = default_actual, + )) + + alias.append(" },") # Close select expression condition dict + if not default_repo_prefix: + supported_versions = sorted([python_version for python_version, _ in version_map]) + alias.append(' no_match_error="""{}""",'.format( + NO_MATCH_ERROR_MESSAGE_TEMPLATE.format( + supported_versions = ", ".join(supported_versions), + rules_python = rules_python, + ), + )) + alias.append(" ),") # Close the select expression + alias.append(' visibility = ["//visibility:public"],') + alias.append(")") # Close the alias() expression return "\n".join(alias) whl_library_alias = repository_rule( _whl_library_alias_impl, attrs = { - "default_version": attr.string(mandatory = True), + "default_version": attr.string( + mandatory = False, + doc = "Optional Python version in major.minor format, e.g. '3.10'." + + "The Python version of the wheel to use when the versions " + + "from `version_map` don't match. This allows the default " + + "(version unaware) rules to match and select a wheel. If " + + "not specified, then the default rules won't be able to " + + "resolve a wheel and an error will occur.", + ), "version_map": attr.string_dict(mandatory = True), "wheel_name": attr.string(mandatory = True), "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index e8e8633..4e4fbb4 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -4,13 +4,24 @@ filegroup( "BUILD.bazel", "//python/pip_install/private:distribution", "//python/pip_install/tools/dependency_resolver:distribution", - "//python/pip_install/tools/lib:distribution", "//python/pip_install/tools/wheel_installer:distribution", ], visibility = ["//:__pkg__"], ) filegroup( + name = "repositories", + srcs = ["repositories.bzl"], + visibility = ["//tools/private/update_deps:__pkg__"], +) + +filegroup( + name = "requirements_txt", + srcs = ["tools/requirements.txt"], + visibility = ["//tools/private/update_deps:__pkg__"], +) + +filegroup( name = "bzl", srcs = glob(["*.bzl"]) + [ "//python/pip_install/private:bzl_srcs", @@ -22,7 +33,6 @@ filegroup( name = "py_srcs", srcs = [ "//python/pip_install/tools/dependency_resolver:py_srcs", - "//python/pip_install/tools/lib:py_srcs", "//python/pip_install/tools/wheel_installer:py_srcs", ], visibility = ["//python/pip_install/private:__pkg__"], diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 99d1fb0..87c7f6b 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -18,15 +18,20 @@ load("//python:repositories.bzl", "get_interpreter_dirname", "is_standalone_inte load("//python:versions.bzl", "WINDOWS_NAME") load("//python/pip_install:repositories.bzl", "all_requirements") load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") +load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases") load("//python/private:toolchains_repo.bzl", "get_host_os_arch") +load("//python/private:which.bzl", "which_with_fail") CPPFLAGS = "CPPFLAGS" COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" +_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point" + def _construct_pypath(rctx): """Helper function to construct a PYTHONPATH. @@ -104,10 +109,7 @@ def _get_xcode_location_cflags(rctx): if not rctx.os.name.lower().startswith("mac os"): return [] - # Locate xcode-select - xcode_select = rctx.which("xcode-select") - - xcode_sdk_location = rctx.execute([xcode_select, "--print-path"]) + xcode_sdk_location = rctx.execute([which_with_fail("xcode-select", rctx), "--print-path"]) if xcode_sdk_location.return_code != 0: return [] @@ -268,56 +270,12 @@ A requirements_lock attribute must be specified, or a platform-specific lockfile """) return requirements_txt -def _pkg_aliases(rctx, repo_name, bzl_packages): - """Create alias declarations for each python dependency. - - The aliases should be appended to the pip_repository BUILD.bazel file. These aliases - allow users to use requirement() without needed a corresponding `use_repo()` for each dep - when using bzlmod. - - Args: - rctx: the repository context. - repo_name: the repository name of the parent that is visible to the users. - bzl_packages: the list of packages to setup. - """ - for name in bzl_packages: - build_content = """package(default_visibility = ["//visibility:public"]) - -alias( - name = "{name}", - actual = "@{repo_name}_{dep}//:pkg", -) - -alias( - name = "pkg", - actual = "@{repo_name}_{dep}//:pkg", -) - -alias( - name = "whl", - actual = "@{repo_name}_{dep}//:whl", -) - -alias( - name = "data", - actual = "@{repo_name}_{dep}//:data", -) - -alias( - name = "dist_info", - actual = "@{repo_name}_{dep}//:dist_info", -) -""".format( - name = name, - repo_name = repo_name, - dep = name, - ) - rctx.file("{}/BUILD.bazel".format(name), build_content) - def _create_pip_repository_bzlmod(rctx, bzl_packages, requirements): repo_name = rctx.attr.repo_name build_contents = _BUILD_FILE_CONTENTS - _pkg_aliases(rctx, repo_name, bzl_packages) + aliases = render_pkg_aliases(repo_name = repo_name, bzl_packages = bzl_packages) + for path, contents in aliases.items(): + rctx.file(path, contents) # NOTE: we are using the canonical name with the double '@' in order to # always uniquely identify a repository, as the labels are being passed as @@ -458,7 +416,9 @@ def _pip_repository_impl(rctx): config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target) if rctx.attr.incompatible_generate_aliases: - _pkg_aliases(rctx, rctx.attr.name, bzl_packages) + aliases = render_pkg_aliases(repo_name = rctx.attr.name, bzl_packages = bzl_packages) + for path, contents in aliases.items(): + rctx.file(path, contents) rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { @@ -663,16 +623,7 @@ def _whl_library_impl(rctx): "python.pip_install.tools.wheel_installer.wheel_installer", "--requirement", rctx.attr.requirement, - "--repo", - rctx.attr.repo, - "--repo-prefix", - rctx.attr.repo_prefix, ] - if rctx.attr.annotation: - args.extend([ - "--annotation", - rctx.path(rctx.attr.annotation), - ]) args = _parse_optional_attrs(rctx, args) @@ -687,8 +638,72 @@ def _whl_library_impl(rctx): if result.return_code: fail("whl_library %s failed: %s (%s) error code: '%s'" % (rctx.attr.name, result.stdout, result.stderr, result.return_code)) + metadata = json.decode(rctx.read("metadata.json")) + rctx.delete("metadata.json") + + entry_points = {} + for item in metadata["entry_points"]: + name = item["name"] + module = item["module"] + attribute = item["attribute"] + + # There is an extreme edge-case with entry_points that end with `.py` + # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 + entry_point_without_py = name[:-3] + "_py" if name.endswith(".py") else name + entry_point_target_name = ( + _WHEEL_ENTRY_POINT_PREFIX + "_" + entry_point_without_py + ) + entry_point_script_name = entry_point_target_name + ".py" + + rctx.file( + entry_point_script_name, + _generate_entry_point_contents(module, attribute), + ) + entry_points[entry_point_without_py] = entry_point_script_name + + build_file_contents = generate_whl_library_build_bazel( + repo_prefix = rctx.attr.repo_prefix, + dependencies = metadata["deps"], + data_exclude = rctx.attr.pip_data_exclude, + tags = [ + "pypi_name=" + metadata["name"], + "pypi_version=" + metadata["version"], + ], + entry_points = entry_points, + annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))), + ) + rctx.file("BUILD.bazel", build_file_contents) + return +def _generate_entry_point_contents( + module, + attribute, + shebang = "#!/usr/bin/env python3"): + """Generate the contents of an entry point script. + + Args: + module (str): The name of the module to use. + attribute (str): The name of the attribute to call. + shebang (str, optional): The shebang to use for the entry point python + file. + + Returns: + str: A string of python code. + """ + contents = """\ +{shebang} +import sys +from {module} import {attribute} +if __name__ == "__main__": + sys.exit({attribute}()) +""".format( + shebang = shebang, + module = module, + attribute = attribute, + ) + return contents + whl_library_attrs = { "annotation": attr.label( doc = ( diff --git a/python/pip_install/private/generate_whl_library_build_bazel.bzl b/python/pip_install/private/generate_whl_library_build_bazel.bzl new file mode 100644 index 0000000..229a917 --- /dev/null +++ b/python/pip_install/private/generate_whl_library_build_bazel.bzl @@ -0,0 +1,224 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate the BUILD.bazel contents for a repo defined by a whl_library.""" + +load("//python/private:normalize_name.bzl", "normalize_name") + +_WHEEL_FILE_LABEL = "whl" +_PY_LIBRARY_LABEL = "pkg" +_DATA_LABEL = "data" +_DIST_INFO_LABEL = "dist_info" +_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point" + +_COPY_FILE_TEMPLATE = """\ +copy_file( + name = "{dest}.copy", + src = "{src}", + out = "{dest}", + is_executable = {is_executable}, +) +""" + +_ENTRY_POINT_RULE_TEMPLATE = """\ +py_binary( + name = "{name}", + srcs = ["{src}"], + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["."], + deps = ["{pkg}"], +) +""" + +_BUILD_TEMPLATE = """\ +load("@rules_python//python:defs.bzl", "py_library", "py_binary") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "{dist_info_label}", + srcs = glob(["site-packages/*.dist-info/**"], allow_empty = True), +) + +filegroup( + name = "{data_label}", + srcs = glob(["data/**"], allow_empty = True), +) + +filegroup( + name = "{whl_file_label}", + srcs = glob(["*.whl"], allow_empty = True), + data = {whl_file_deps}, +) + +py_library( + name = "{name}", + srcs = glob( + ["site-packages/**/*.py"], + exclude={srcs_exclude}, + # Empty sources are allowed to support wheels that don't have any + # pure-Python code, e.g. pymssql, which is written in Cython. + allow_empty = True, + ), + data = {data} + glob( + ["site-packages/**/*"], + exclude={data_exclude}, + ), + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["site-packages"], + deps = {dependencies}, + tags = {tags}, +) +""" + +def generate_whl_library_build_bazel( + repo_prefix, + dependencies, + data_exclude, + tags, + entry_points, + annotation = None): + """Generate a BUILD file for an unzipped Wheel + + Args: + repo_prefix: the repo prefix that should be used for dependency lists. + dependencies: a list of PyPI packages that are dependencies to the py_library. + data_exclude: more patterns to exclude from the data attribute of generated py_library rules. + tags: list of tags to apply to generated py_library rules. + entry_points: A dict of entry points to add py_binary rules for. + annotation: The annotation for the build file. + + Returns: + A complete BUILD file as a string + """ + + additional_content = [] + data = [] + srcs_exclude = [] + data_exclude = [] + data_exclude + dependencies = sorted(dependencies) + tags = sorted(tags) + + for entry_point, entry_point_script_name in entry_points.items(): + additional_content.append( + _generate_entry_point_rule( + name = "{}_{}".format(_WHEEL_ENTRY_POINT_PREFIX, entry_point), + script = entry_point_script_name, + pkg = ":" + _PY_LIBRARY_LABEL, + ), + ) + + if annotation: + for src, dest in annotation.copy_files.items(): + data.append(dest) + additional_content.append(_generate_copy_commands(src, dest)) + for src, dest in annotation.copy_executables.items(): + data.append(dest) + additional_content.append( + _generate_copy_commands(src, dest, is_executable = True), + ) + data.extend(annotation.data) + data_exclude.extend(annotation.data_exclude_glob) + srcs_exclude.extend(annotation.srcs_exclude_glob) + if annotation.additive_build_content: + additional_content.append(annotation.additive_build_content) + + _data_exclude = [ + "**/* *", + "**/*.py", + "**/*.pyc", + "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNNN are created + # RECORD is known to contain sha256 checksums of files which might include the checksums + # of generated files produced when wheels are installed. The file is ignored to avoid + # Bazel caching issues. + "**/*.dist-info/RECORD", + ] + for item in data_exclude: + if item not in _data_exclude: + _data_exclude.append(item) + + lib_dependencies = [ + "@" + repo_prefix + normalize_name(d) + "//:" + _PY_LIBRARY_LABEL + for d in dependencies + ] + whl_file_deps = [ + "@" + repo_prefix + normalize_name(d) + "//:" + _WHEEL_FILE_LABEL + for d in dependencies + ] + + contents = "\n".join( + [ + _BUILD_TEMPLATE.format( + name = _PY_LIBRARY_LABEL, + dependencies = repr(lib_dependencies), + data_exclude = repr(_data_exclude), + whl_file_label = _WHEEL_FILE_LABEL, + whl_file_deps = repr(whl_file_deps), + tags = repr(tags), + data_label = _DATA_LABEL, + dist_info_label = _DIST_INFO_LABEL, + entry_point_prefix = _WHEEL_ENTRY_POINT_PREFIX, + srcs_exclude = repr(srcs_exclude), + data = repr(data), + ), + ] + additional_content, + ) + + # NOTE: Ensure that we terminate with a new line + return contents.rstrip() + "\n" + +def _generate_copy_commands(src, dest, is_executable = False): + """Generate a [@bazel_skylib//rules:copy_file.bzl%copy_file][cf] target + + [cf]: https://github.com/bazelbuild/bazel-skylib/blob/1.1.1/docs/copy_file_doc.md + + Args: + src (str): The label for the `src` attribute of [copy_file][cf] + dest (str): The label for the `out` attribute of [copy_file][cf] + is_executable (bool, optional): Whether or not the file being copied is executable. + sets `is_executable` for [copy_file][cf] + + Returns: + str: A `copy_file` instantiation. + """ + return _COPY_FILE_TEMPLATE.format( + src = src, + dest = dest, + is_executable = is_executable, + ) + +def _generate_entry_point_rule(*, name, script, pkg): + """Generate a Bazel `py_binary` rule for an entry point script. + + Note that the script is used to determine the name of the target. The name of + entry point targets should be uniuqe to avoid conflicts with existing sources or + directories within a wheel. + + Args: + name (str): The name of the generated py_binary. + script (str): The path to the entry point's python file. + pkg (str): The package owning the entry point. This is expected to + match up with the `py_library` defined for each repository. + + Returns: + str: A `py_binary` instantiation. + """ + return _ENTRY_POINT_RULE_TEMPLATE.format( + name = name, + src = script.replace("\\", "/"), + pkg = pkg, + ) diff --git a/python/pip_install/private/srcs.bzl b/python/pip_install/private/srcs.bzl index f3064a3..e342d90 100644 --- a/python/pip_install/private/srcs.bzl +++ b/python/pip_install/private/srcs.bzl @@ -9,10 +9,7 @@ This file is auto-generated from the `@rules_python//python/pip_install/private: PIP_INSTALL_PY_SRCS = [ "@rules_python//python/pip_install/tools/dependency_resolver:__init__.py", "@rules_python//python/pip_install/tools/dependency_resolver:dependency_resolver.py", - "@rules_python//python/pip_install/tools/lib:__init__.py", - "@rules_python//python/pip_install/tools/lib:annotation.py", - "@rules_python//python/pip_install/tools/lib:arguments.py", - "@rules_python//python/pip_install/tools/lib:bazel.py", + "@rules_python//python/pip_install/tools/wheel_installer:arguments.py", "@rules_python//python/pip_install/tools/wheel_installer:namespace_pkgs.py", "@rules_python//python/pip_install/tools/wheel_installer:wheel.py", "@rules_python//python/pip_install/tools/wheel_installer:wheel_installer.py", diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index efe3bc7..4b209b3 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -20,6 +20,7 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load("//:version.bzl", "MINIMUM_BAZEL_VERSION") _RULE_DEPS = [ + # START: maintained by 'bazel run //tools/private:update_pip_deps' ( "pypi__build", "https://files.pythonhosted.org/packages/03/97/f58c723ff036a8d8b4d3115377c0a37ed05c1f68dd9a0d66dab5e82c5c1c/build-0.9.0-py3-none-any.whl", @@ -36,11 +37,21 @@ _RULE_DEPS = [ "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", ), ( + "pypi__importlib_metadata", + "https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl", + "bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", + ), + ( "pypi__installer", "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", ), ( + "pypi__more_itertools", + "https://files.pythonhosted.org/packages/bd/3f/c4b3dbd315e248f84c388bd4a72b131a29f123ecacc37ffb2b3834546e42/more_itertools-8.13.0-py3-none-any.whl", + "c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb", + ), + ( "pypi__packaging", "https://files.pythonhosted.org/packages/8f/7b/42582927d281d7cb035609cd3a543ffac89b74f3f4ee8e1c50914bcb57eb/packaging-22.0-py3-none-any.whl", "957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3", @@ -76,20 +87,11 @@ _RULE_DEPS = [ "b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8", ), ( - "pypi__importlib_metadata", - "https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl", - "bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", - ), - ( "pypi__zipp", "https://files.pythonhosted.org/packages/f4/50/cc72c5bcd48f6e98219fc4a88a5227e9e28b81637a99c49feba1d51f4d50/zipp-1.0.0-py2.py3-none-any.whl", "8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656", ), - ( - "pypi__more_itertools", - "https://files.pythonhosted.org/packages/bd/3f/c4b3dbd315e248f84c388bd4a72b131a29f123ecacc37ffb2b3834546e42/more_itertools-8.13.0-py3-none-any.whl", - "c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb", - ), + # END: maintained by 'bazel run //tools/private:update_pip_deps' ] _GENERIC_WHEEL = """\ diff --git a/python/pip_install/tools/lib/BUILD.bazel b/python/pip_install/tools/lib/BUILD.bazel deleted file mode 100644 index 37a8b09..0000000 --- a/python/pip_install/tools/lib/BUILD.bazel +++ /dev/null @@ -1,82 +0,0 @@ -load("//python:defs.bzl", "py_library", "py_test") -load(":annotations_test_helpers.bzl", "package_annotation", "package_annotations_file") - -py_library( - name = "lib", - srcs = [ - "annotation.py", - "arguments.py", - "bazel.py", - ], - visibility = ["//python/pip_install:__subpackages__"], -) - -package_annotations_file( - name = "mock_annotations", - annotations = { - "pkg_a": package_annotation(), - "pkg_b": package_annotation( - data_exclude_glob = [ - "*.foo", - "*.bar", - ], - ), - "pkg_c": package_annotation( - # The `join` and `strip` here accounts for potential differences - # in new lines between unix and windows hosts. - additive_build_content = "\n".join([line.strip() for line in """\ -cc_library( - name = "my_target", - hdrs = glob(["**/*.h"]), - srcs = glob(["**/*.cc"]), -) -""".splitlines()]), - data = [":my_target"], - ), - "pkg_d": package_annotation( - srcs_exclude_glob = ["pkg_d/tests/**"], - ), - }, - tags = ["manual"], -) - -py_test( - name = "annotations_test", - size = "small", - srcs = ["annotations_test.py"], - data = [":mock_annotations"], - env = {"MOCK_ANNOTATIONS": "$(rootpath :mock_annotations)"}, - deps = [ - ":lib", - "//python/runfiles", - ], -) - -py_test( - name = "arguments_test", - size = "small", - srcs = [ - "arguments_test.py", - ], - deps = [ - ":lib", - ], -) - -filegroup( - name = "distribution", - srcs = glob( - ["*"], - exclude = ["*_test.py"], - ), - visibility = ["//python/pip_install:__subpackages__"], -) - -filegroup( - name = "py_srcs", - srcs = glob( - include = ["**/*.py"], - exclude = ["**/*_test.py"], - ), - visibility = ["//python/pip_install:__subpackages__"], -) diff --git a/python/pip_install/tools/lib/annotation.py b/python/pip_install/tools/lib/annotation.py deleted file mode 100644 index c980080..0000000 --- a/python/pip_install/tools/lib/annotation.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import logging -from collections import OrderedDict -from pathlib import Path -from typing import Any, Dict, List - - -class Annotation(OrderedDict): - """A python representation of `@rules_python//python:pip.bzl%package_annotation`""" - - def __init__(self, content: Dict[str, Any]) -> None: - - missing = [] - ordered_content = OrderedDict() - for field in ( - "additive_build_content", - "copy_executables", - "copy_files", - "data", - "data_exclude_glob", - "srcs_exclude_glob", - ): - if field not in content: - missing.append(field) - continue - ordered_content.update({field: content.pop(field)}) - - if missing: - raise ValueError("Data missing from initial annotation: {}".format(missing)) - - if content: - raise ValueError( - "Unexpected data passed to annotations: {}".format( - sorted(list(content.keys())) - ) - ) - - return OrderedDict.__init__(self, ordered_content) - - @property - def additive_build_content(self) -> str: - return self["additive_build_content"] - - @property - def copy_executables(self) -> Dict[str, str]: - return self["copy_executables"] - - @property - def copy_files(self) -> Dict[str, str]: - return self["copy_files"] - - @property - def data(self) -> List[str]: - return self["data"] - - @property - def data_exclude_glob(self) -> List[str]: - return self["data_exclude_glob"] - - @property - def srcs_exclude_glob(self) -> List[str]: - return self["srcs_exclude_glob"] - - -class AnnotationsMap: - """A mapping of python package names to [Annotation]""" - - def __init__(self, json_file: Path): - content = json.loads(json_file.read_text()) - - self._annotations = {pkg: Annotation(data) for (pkg, data) in content.items()} - - @property - def annotations(self) -> Dict[str, Annotation]: - return self._annotations - - def collect(self, requirements: List[str]) -> Dict[str, Annotation]: - unused = self.annotations - collection = {} - for pkg in requirements: - if pkg in unused: - collection.update({pkg: unused.pop(pkg)}) - - if unused: - logging.warning( - "Unused annotations: {}".format(sorted(list(unused.keys()))) - ) - - return collection - - -def annotation_from_str_path(path: str) -> Annotation: - """Load an annotation from a json encoded file - - Args: - path (str): The path to a json encoded file - - Returns: - Annotation: The deserialized annotations - """ - json_file = Path(path) - content = json.loads(json_file.read_text()) - return Annotation(content) - - -def annotations_map_from_str_path(path: str) -> AnnotationsMap: - """Load an annotations map from a json encoded file - - Args: - path (str): The path to a json encoded file - - Returns: - AnnotationsMap: The deserialized annotations map - """ - return AnnotationsMap(Path(path)) diff --git a/python/pip_install/tools/lib/annotations_test.py b/python/pip_install/tools/lib/annotations_test.py deleted file mode 100644 index f7c360f..0000000 --- a/python/pip_install/tools/lib/annotations_test.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import os -import textwrap -import unittest -from pathlib import Path - -from python.pip_install.tools.lib.annotation import Annotation, AnnotationsMap -from python.runfiles import runfiles - - -class AnnotationsTestCase(unittest.TestCase): - - maxDiff = None - - def test_annotations_constructor(self) -> None: - annotations_env = os.environ.get("MOCK_ANNOTATIONS") - self.assertIsNotNone(annotations_env) - - r = runfiles.Create() - - annotations_path = Path(r.Rlocation("rules_python/{}".format(annotations_env))) - self.assertTrue(annotations_path.exists()) - - annotations_map = AnnotationsMap(annotations_path) - self.assertListEqual( - list(annotations_map.annotations.keys()), - ["pkg_a", "pkg_b", "pkg_c", "pkg_d"], - ) - - collection = annotations_map.collect(["pkg_a", "pkg_b", "pkg_c", "pkg_d"]) - - self.assertEqual( - collection["pkg_a"], - Annotation( - { - "additive_build_content": None, - "copy_executables": {}, - "copy_files": {}, - "data": [], - "data_exclude_glob": [], - "srcs_exclude_glob": [], - } - ), - ) - - self.assertEqual( - collection["pkg_b"], - Annotation( - { - "additive_build_content": None, - "copy_executables": {}, - "copy_files": {}, - "data": [], - "data_exclude_glob": ["*.foo", "*.bar"], - "srcs_exclude_glob": [], - } - ), - ) - - self.assertEqual( - collection["pkg_c"], - Annotation( - { - # The `join` and `strip` here accounts for potential - # differences in new lines between unix and windows - # hosts. - "additive_build_content": "\n".join( - [ - line.strip() - for line in textwrap.dedent( - """\ - cc_library( - name = "my_target", - hdrs = glob(["**/*.h"]), - srcs = glob(["**/*.cc"]), - ) - """ - ).splitlines() - ] - ), - "copy_executables": {}, - "copy_files": {}, - "data": [":my_target"], - "data_exclude_glob": [], - "srcs_exclude_glob": [], - } - ), - ) - - self.assertEqual( - collection["pkg_d"], - Annotation( - { - "additive_build_content": None, - "copy_executables": {}, - "copy_files": {}, - "data": [], - "data_exclude_glob": [], - "srcs_exclude_glob": ["pkg_d/tests/**"], - } - ), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/python/pip_install/tools/lib/annotations_test_helpers.bzl b/python/pip_install/tools/lib/annotations_test_helpers.bzl deleted file mode 100644 index 4f56bb7..0000000 --- a/python/pip_install/tools/lib/annotations_test_helpers.bzl +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper macros and rules for testing the `annotations` module of `tools`""" - -load("//python:pip.bzl", _package_annotation = "package_annotation") - -package_annotation = _package_annotation - -def _package_annotations_file_impl(ctx): - output = ctx.actions.declare_file(ctx.label.name + ".annotations.json") - - annotations = {package: json.decode(data) for (package, data) in ctx.attr.annotations.items()} - ctx.actions.write( - output = output, - content = json.encode_indent(annotations, indent = " " * 4), - ) - - return DefaultInfo( - files = depset([output]), - runfiles = ctx.runfiles(files = [output]), - ) - -package_annotations_file = rule( - implementation = _package_annotations_file_impl, - doc = ( - "Consumes `package_annotation` definitions in the same way " + - "`pip_repository` rules do to produce an annotations file." - ), - attrs = { - "annotations": attr.string_dict( - doc = "See `@rules_python//python:pip.bzl%package_annotation", - mandatory = True, - ), - }, -) diff --git a/python/pip_install/tools/lib/bazel.py b/python/pip_install/tools/lib/bazel.py deleted file mode 100644 index 81119e9..0000000 --- a/python/pip_install/tools/lib/bazel.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -WHEEL_FILE_LABEL = "whl" -PY_LIBRARY_LABEL = "pkg" -DATA_LABEL = "data" -DIST_INFO_LABEL = "dist_info" -WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point" - - -def sanitise_name(name: str, prefix: str) -> str: - """Sanitises the name to be compatible with Bazel labels. - - See the doc in ../../../private/normalize_name.bzl. - """ - return prefix + re.sub(r"[-_.]+", "_", name).lower() - - -def _whl_name_to_repo_root(whl_name: str, repo_prefix: str) -> str: - return "@{}//".format(sanitise_name(whl_name, prefix=repo_prefix)) - - -def sanitised_repo_library_label(whl_name: str, repo_prefix: str) -> str: - return '"{}:{}"'.format( - _whl_name_to_repo_root(whl_name, repo_prefix), PY_LIBRARY_LABEL - ) - - -def sanitised_repo_file_label(whl_name: str, repo_prefix: str) -> str: - return '"{}:{}"'.format( - _whl_name_to_repo_root(whl_name, repo_prefix), WHEEL_FILE_LABEL - ) diff --git a/python/pip_install/tools/requirements.txt b/python/pip_install/tools/requirements.txt new file mode 100755 index 0000000..e8de112 --- /dev/null +++ b/python/pip_install/tools/requirements.txt @@ -0,0 +1,14 @@ +build==0.9 +click==8.0.1 +colorama +importlib_metadata==1.4.0 +installer +more_itertools==8.13.0 +packaging==22.0 +pep517 +pip==22.3.1 +pip_tools==6.12.1 +setuptools==60.10 +tomli +wheel==0.38.4 +zipp==1.0.0 diff --git a/python/pip_install/tools/wheel_installer/BUILD.bazel b/python/pip_install/tools/wheel_installer/BUILD.bazel index 54bbc46..6360ca5 100644 --- a/python/pip_install/tools/wheel_installer/BUILD.bazel +++ b/python/pip_install/tools/wheel_installer/BUILD.bazel @@ -4,12 +4,12 @@ load("//python/pip_install:repositories.bzl", "requirement") py_library( name = "lib", srcs = [ + "arguments.py", "namespace_pkgs.py", "wheel.py", "wheel_installer.py", ], deps = [ - "//python/pip_install/tools/lib", requirement("installer"), requirement("pip"), requirement("setuptools"), @@ -25,6 +25,17 @@ py_binary( ) py_test( + name = "arguments_test", + size = "small", + srcs = [ + "arguments_test.py", + ], + deps = [ + ":lib", + ], +) + +py_test( name = "namespace_pkgs_test", size = "small", srcs = [ diff --git a/python/pip_install/tools/lib/arguments.py b/python/pip_install/tools/wheel_installer/arguments.py index 974f03c..aac3c01 100644 --- a/python/pip_install/tools/lib/arguments.py +++ b/python/pip_install/tools/wheel_installer/arguments.py @@ -12,16 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse import json -from argparse import ArgumentParser +from typing import Any -def parse_common_args(parser: ArgumentParser) -> ArgumentParser: +def parser(**kwargs: Any) -> argparse.ArgumentParser: + """Create a parser for the wheel_installer tool.""" + parser = argparse.ArgumentParser( + **kwargs, + ) parser.add_argument( - "--repo", + "--requirement", action="store", required=True, - help="The external repo name to install dependencies. In the format '@{REPO_NAME}'", + help="A single PEP508 requirement specifier string.", ) parser.add_argument( "--isolated", @@ -49,11 +54,6 @@ def parse_common_args(parser: ArgumentParser) -> ArgumentParser: help="Extra environment variables to set on the pip environment.", ) parser.add_argument( - "--repo-prefix", - required=True, - help="Prefix to prepend to packages", - ) - parser.add_argument( "--download_only", action="store_true", help="Use 'pip download' instead of 'pip wheel'. Disables building wheels from source, but allows use of " diff --git a/python/pip_install/tools/lib/arguments_test.py b/python/pip_install/tools/wheel_installer/arguments_test.py index dfa96a8..7193f4a 100644 --- a/python/pip_install/tools/lib/arguments_test.py +++ b/python/pip_install/tools/wheel_installer/arguments_test.py @@ -16,35 +16,30 @@ import argparse import json import unittest -from python.pip_install.tools.lib import arguments +from python.pip_install.tools.wheel_installer import arguments class ArgumentsTestCase(unittest.TestCase): def test_arguments(self) -> None: - parser = argparse.ArgumentParser() - parser = arguments.parse_common_args(parser) + parser = arguments.parser() repo_name = "foo" repo_prefix = "pypi_" index_url = "--index_url=pypi.org/simple" extra_pip_args = [index_url] + requirement = "foo==1.0.0 --hash=sha256:deadbeef" args_dict = vars( parser.parse_args( args=[ - "--repo", - repo_name, + f'--requirement="{requirement}"', f"--extra_pip_args={json.dumps({'arg': extra_pip_args})}", - "--repo-prefix", - repo_prefix, ] ) ) args_dict = arguments.deserialize_structured_args(args_dict) - self.assertIn("repo", args_dict) + self.assertIn("requirement", args_dict) self.assertIn("extra_pip_args", args_dict) self.assertEqual(args_dict["pip_data_exclude"], []) self.assertEqual(args_dict["enable_implicit_namespace_pkgs"], False) - self.assertEqual(args_dict["repo"], repo_name) - self.assertEqual(args_dict["repo_prefix"], repo_prefix) self.assertEqual(args_dict["extra_pip_args"], extra_pip_args) def test_deserialize_structured_args(self) -> None: diff --git a/python/pip_install/tools/wheel_installer/wheel_installer.py b/python/pip_install/tools/wheel_installer/wheel_installer.py index 9b363c3..c6c2961 100644 --- a/python/pip_install/tools/wheel_installer/wheel_installer.py +++ b/python/pip_install/tools/wheel_installer/wheel_installer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Build and/or fetch a single wheel based on the requirement passed in""" + import argparse import errno import glob @@ -28,8 +30,7 @@ from typing import Dict, Iterable, List, Optional, Set, Tuple from pip._vendor.packaging.utils import canonicalize_name -from python.pip_install.tools.lib import annotation, arguments, bazel -from python.pip_install.tools.wheel_installer import namespace_pkgs, wheel +from python.pip_install.tools.wheel_installer import arguments, namespace_pkgs, wheel def _configure_reproducible_wheels() -> None: @@ -103,201 +104,11 @@ def _setup_namespace_pkg_compatibility(wheel_dir: str) -> None: namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir) -def _generate_entry_point_contents( - module: str, attribute: str, shebang: str = "#!/usr/bin/env python3" -) -> str: - """Generate the contents of an entry point script. - - Args: - module (str): The name of the module to use. - attribute (str): The name of the attribute to call. - shebang (str, optional): The shebang to use for the entry point python - file. - - Returns: - str: A string of python code. - """ - return textwrap.dedent( - """\ - {shebang} - import sys - from {module} import {attribute} - if __name__ == "__main__": - sys.exit({attribute}()) - """.format( - shebang=shebang, module=module, attribute=attribute - ) - ) - - -def _generate_entry_point_rule(name: str, script: str, pkg: str) -> str: - """Generate a Bazel `py_binary` rule for an entry point script. - - Note that the script is used to determine the name of the target. The name of - entry point targets should be uniuqe to avoid conflicts with existing sources or - directories within a wheel. - - Args: - name (str): The name of the generated py_binary. - script (str): The path to the entry point's python file. - pkg (str): The package owning the entry point. This is expected to - match up with the `py_library` defined for each repository. - - - Returns: - str: A `py_binary` instantiation. - """ - return textwrap.dedent( - """\ - py_binary( - name = "{name}", - srcs = ["{src}"], - # This makes this directory a top-level in the python import - # search path for anything that depends on this. - imports = ["."], - deps = ["{pkg}"], - ) - """.format( - name=name, src=str(script).replace("\\", "/"), pkg=pkg - ) - ) - - -def _generate_copy_commands(src, dest, is_executable=False) -> str: - """Generate a [@bazel_skylib//rules:copy_file.bzl%copy_file][cf] target - - [cf]: https://github.com/bazelbuild/bazel-skylib/blob/1.1.1/docs/copy_file_doc.md - - Args: - src (str): The label for the `src` attribute of [copy_file][cf] - dest (str): The label for the `out` attribute of [copy_file][cf] - is_executable (bool, optional): Whether or not the file being copied is executable. - sets `is_executable` for [copy_file][cf] - - Returns: - str: A `copy_file` instantiation. - """ - return textwrap.dedent( - """\ - copy_file( - name = "{dest}.copy", - src = "{src}", - out = "{dest}", - is_executable = {is_executable}, - ) - """.format( - src=src, - dest=dest, - is_executable=is_executable, - ) - ) - - -def _generate_build_file_contents( - name: str, - dependencies: List[str], - whl_file_deps: List[str], - data_exclude: List[str], - tags: List[str], - srcs_exclude: List[str] = [], - data: List[str] = [], - additional_content: List[str] = [], -) -> str: - """Generate a BUILD file for an unzipped Wheel - - Args: - name: the target name of the py_library - dependencies: a list of Bazel labels pointing to dependencies of the library - whl_file_deps: a list of Bazel labels pointing to wheel file dependencies of this wheel. - data_exclude: more patterns to exclude from the data attribute of generated py_library rules. - tags: list of tags to apply to generated py_library rules. - additional_content: A list of additional content to append to the BUILD file. - - Returns: - A complete BUILD file as a string - - We allow for empty Python sources as for Wheels containing only compiled C code - there may be no Python sources whatsoever (e.g. packages written in Cython: like `pymssql`). - """ - - data_exclude = list( - set( - [ - "**/* *", - "**/*.py", - "**/*.pyc", - "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNNN are created - # RECORD is known to contain sha256 checksums of files which might include the checksums - # of generated files produced when wheels are installed. The file is ignored to avoid - # Bazel caching issues. - "**/*.dist-info/RECORD", - ] - + data_exclude - ) - ) - - return "\n".join( - [ - textwrap.dedent( - """\ - load("@rules_python//python:defs.bzl", "py_library", "py_binary") - load("@bazel_skylib//rules:copy_file.bzl", "copy_file") - - package(default_visibility = ["//visibility:public"]) - - filegroup( - name = "{dist_info_label}", - srcs = glob(["site-packages/*.dist-info/**"], allow_empty = True), - ) - - filegroup( - name = "{data_label}", - srcs = glob(["data/**"], allow_empty = True), - ) - - filegroup( - name = "{whl_file_label}", - srcs = glob(["*.whl"], allow_empty = True), - data = [{whl_file_deps}], - ) - - py_library( - name = "{name}", - srcs = glob(["site-packages/**/*.py"], exclude={srcs_exclude}, allow_empty = True), - data = {data} + glob(["site-packages/**/*"], exclude={data_exclude}), - # This makes this directory a top-level in the python import - # search path for anything that depends on this. - imports = ["site-packages"], - deps = [{dependencies}], - tags = [{tags}], - ) - """.format( - name=name, - dependencies=",".join(sorted(dependencies)), - data_exclude=json.dumps(sorted(data_exclude)), - whl_file_label=bazel.WHEEL_FILE_LABEL, - whl_file_deps=",".join(sorted(whl_file_deps)), - tags=",".join(sorted(['"%s"' % t for t in tags])), - data_label=bazel.DATA_LABEL, - dist_info_label=bazel.DIST_INFO_LABEL, - entry_point_prefix=bazel.WHEEL_ENTRY_POINT_PREFIX, - srcs_exclude=json.dumps(sorted(srcs_exclude)), - data=json.dumps(sorted(data)), - ) - ) - ] - + additional_content - ) - - def _extract_wheel( wheel_file: str, extras: Dict[str, Set[str]], - pip_data_exclude: List[str], enable_implicit_namespace_pkgs: bool, - repo_prefix: str, installation_dir: Path = Path("."), - annotation: Optional[annotation.Annotation] = None, ) -> None: """Extracts wheel into given directory and creates py_library and filegroup targets. @@ -305,9 +116,7 @@ def _extract_wheel( wheel_file: the filepath of the .whl installation_dir: the destination directory for installation of the wheel. extras: a list of extras to add as dependencies for the installed wheel - pip_data_exclude: list of file patterns to exclude from the generated data section of the py_library enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is - annotation: An optional set of annotations to apply to the BUILD contents of the wheel. """ whl = wheel.Wheel(wheel_file) @@ -322,83 +131,25 @@ def _extract_wheel( self_edge_dep = set([whl.name]) whl_deps = sorted(whl.dependencies(extras_requested) - self_edge_dep) - sanitised_dependencies = [ - bazel.sanitised_repo_library_label(d, repo_prefix=repo_prefix) for d in whl_deps - ] - sanitised_wheel_file_dependencies = [ - bazel.sanitised_repo_file_label(d, repo_prefix=repo_prefix) for d in whl_deps - ] - - entry_points = [] - for name, (module, attribute) in sorted(whl.entry_points().items()): - # There is an extreme edge-case with entry_points that end with `.py` - # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 - entry_point_without_py = f"{name[:-3]}_py" if name.endswith(".py") else name - entry_point_target_name = ( - f"{bazel.WHEEL_ENTRY_POINT_PREFIX}_{entry_point_without_py}" - ) - entry_point_script_name = f"{entry_point_target_name}.py" - (installation_dir / entry_point_script_name).write_text( - _generate_entry_point_contents(module, attribute) - ) - entry_points.append( - _generate_entry_point_rule( - entry_point_target_name, - entry_point_script_name, - bazel.PY_LIBRARY_LABEL, - ) - ) - - with open(os.path.join(installation_dir, "BUILD.bazel"), "w") as build_file: - additional_content = entry_points - data = [] - data_exclude = pip_data_exclude - srcs_exclude = [] - if annotation: - for src, dest in annotation.copy_files.items(): - data.append(dest) - additional_content.append(_generate_copy_commands(src, dest)) - for src, dest in annotation.copy_executables.items(): - data.append(dest) - additional_content.append( - _generate_copy_commands(src, dest, is_executable=True) - ) - data.extend(annotation.data) - data_exclude.extend(annotation.data_exclude_glob) - srcs_exclude.extend(annotation.srcs_exclude_glob) - if annotation.additive_build_content: - additional_content.append(annotation.additive_build_content) - - contents = _generate_build_file_contents( - name=bazel.PY_LIBRARY_LABEL, - dependencies=sanitised_dependencies, - whl_file_deps=sanitised_wheel_file_dependencies, - data_exclude=data_exclude, - data=data, - srcs_exclude=srcs_exclude, - tags=["pypi_name=" + whl.name, "pypi_version=" + whl.version], - additional_content=additional_content, - ) - build_file.write(contents) + with open(os.path.join(installation_dir, "metadata.json"), "w") as f: + metadata = { + "name": whl.name, + "version": whl.version, + "deps": whl_deps, + "entry_points": [ + { + "name": name, + "module": module, + "attribute": attribute, + } + for name, (module, attribute) in sorted(whl.entry_points().items()) + ], + } + json.dump(metadata, f) def main() -> None: - parser = argparse.ArgumentParser( - description="Build and/or fetch a single wheel based on the requirement passed in" - ) - parser.add_argument( - "--requirement", - action="store", - required=True, - help="A single PEP508 requirement specifier string.", - ) - parser.add_argument( - "--annotation", - type=annotation.annotation_from_str_path, - help="A json encoded file containing annotations for rendered packages.", - ) - arguments.parse_common_args(parser) - args = parser.parse_args() + args = arguments.parser(description=__doc__).parse_args() deserialized_args = dict(vars(args)) arguments.deserialize_structured_args(deserialized_args) @@ -441,10 +192,7 @@ def main() -> None: _extract_wheel( wheel_file=whl, extras=extras, - pip_data_exclude=deserialized_args["pip_data_exclude"], enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs, - repo_prefix=args.repo_prefix, - annotation=args.annotation, ) diff --git a/python/pip_install/tools/wheel_installer/wheel_installer_test.py b/python/pip_install/tools/wheel_installer/wheel_installer_test.py index 8758b67..b24e500 100644 --- a/python/pip_install/tools/wheel_installer/wheel_installer_test.py +++ b/python/pip_install/tools/wheel_installer/wheel_installer_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import shutil import tempfile @@ -54,28 +55,29 @@ class TestRequirementExtrasParsing(unittest.TestCase): ) -class BazelTestCase(unittest.TestCase): - def test_generate_entry_point_contents(self): - got = wheel_installer._generate_entry_point_contents("sphinx.cmd.build", "main") - want = """#!/usr/bin/env python3 -import sys -from sphinx.cmd.build import main -if __name__ == "__main__": - sys.exit(main()) -""" - self.assertEqual(got, want) - - def test_generate_entry_point_contents_with_shebang(self): - got = wheel_installer._generate_entry_point_contents( - "sphinx.cmd.build", "main", shebang="#!/usr/bin/python" - ) - want = """#!/usr/bin/python -import sys -from sphinx.cmd.build import main -if __name__ == "__main__": - sys.exit(main()) -""" - self.assertEqual(got, want) +# TODO @aignas 2023-07-21: migrate to starlark +# class BazelTestCase(unittest.TestCase): +# def test_generate_entry_point_contents(self): +# got = wheel_installer._generate_entry_point_contents("sphinx.cmd.build", "main") +# want = """#!/usr/bin/env python3 +# import sys +# from sphinx.cmd.build import main +# if __name__ == "__main__": +# sys.exit(main()) +# """ +# self.assertEqual(got, want) +# +# def test_generate_entry_point_contents_with_shebang(self): +# got = wheel_installer._generate_entry_point_contents( +# "sphinx.cmd.build", "main", shebang="#!/usr/bin/python" +# ) +# want = """#!/usr/bin/python +# import sys +# from sphinx.cmd.build import main +# if __name__ == "__main__": +# sys.exit(main()) +# """ +# self.assertEqual(got, want) class TestWhlFilegroup(unittest.TestCase): @@ -93,15 +95,33 @@ class TestWhlFilegroup(unittest.TestCase): self.wheel_path, installation_dir=Path(self.wheel_dir), extras={}, - pip_data_exclude=[], enable_implicit_namespace_pkgs=False, - repo_prefix="prefix_", ) - self.assertIn(self.wheel_name, os.listdir(self.wheel_dir)) - with open("{}/BUILD.bazel".format(self.wheel_dir)) as build_file: - build_file_content = build_file.read() - self.assertIn("filegroup", build_file_content) + want_files = [ + "metadata.json", + "site-packages", + self.wheel_name, + ] + self.assertEqual( + sorted(want_files), + sorted( + [ + str(p.relative_to(self.wheel_dir)) + for p in Path(self.wheel_dir).glob("*") + ] + ), + ) + with open("{}/metadata.json".format(self.wheel_dir)) as metadata_file: + metadata_file_content = json.load(metadata_file) + + want = dict( + version="0.0.1", + name="example-minimal-package", + deps=[], + entry_points=[], + ) + self.assertEqual(want, metadata_file_content) if __name__ == "__main__": diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 10af17e..29b5a6c 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -24,6 +24,12 @@ filegroup( visibility = ["//python:__pkg__"], ) +filegroup( + name = "coverage_deps", + srcs = ["coverage_deps.bzl"], + visibility = ["//tools/private/update_deps:__pkg__"], +) + # Filegroup of bzl files that can be used by downstream rules for documentation generation filegroup( name = "bzl", @@ -52,6 +58,15 @@ bzl_library( ) bzl_library( + name = "which_bzl", + srcs = ["which.bzl"], + visibility = [ + "//docs:__subpackages__", + "//python:__subpackages__", + ], +) + +bzl_library( name = "py_cc_toolchain_bzl", srcs = [ "py_cc_toolchain_macro.bzl", diff --git a/python/private/coverage_deps.bzl b/python/private/coverage_deps.bzl index 93938e9..863d496 100644 --- a/python/private/coverage_deps.bzl +++ b/python/private/coverage_deps.bzl @@ -19,8 +19,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load("//python/private:version_label.bzl", "version_label") -# Update with './tools/update_coverage_deps.py <version>' -#START: managed by update_coverage_deps.py script +# START: maintained by 'bazel run //tools/private:update_coverage_deps' _coverage_deps = { "cp310": { "aarch64-apple-darwin": ( @@ -95,7 +94,7 @@ _coverage_deps = { ), }, } -#END: managed by update_coverage_deps.py script +# END: maintained by 'bazel run //tools/private:update_coverage_deps' _coverage_patch = Label("//python/private:coverage.patch") diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl new file mode 100644 index 0000000..bcbfc8c --- /dev/null +++ b/python/private/render_pkg_aliases.bzl @@ -0,0 +1,182 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""render_pkg_aliases is a function to generate BUILD.bazel contents used to create user-friendly aliases. + +This is used in bzlmod and non-bzlmod setups.""" + +load("//python/private:normalize_name.bzl", "normalize_name") +load(":text_util.bzl", "render") +load(":version_label.bzl", "version_label") + +NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ +No matching wheel for current configuration's Python version. + +The current build configuration's Python version doesn't match any of the Python +versions available for this wheel. This wheel supports the following Python versions: + {supported_versions} + +As matched by the `@{rules_python}//python/config_settings:is_python_<version>` +configuration settings. + +To determine the current configuration's Python version, run: + `bazel config <config id>` (shown further below) +and look for + {rules_python}//python/config_settings:python_version + +If the value is missing, then the "default" Python version is being used, +which has a "null" version value and will not match version constraints. +""" + +def _render_whl_library_alias( + *, + name, + repo_name, + dep, + target, + default_version, + versions, + rules_python): + """Render an alias for common targets + + If the versions is passed, then the `rules_python` must be passed as well and + an alias with a select statement based on the python version is going to be + generated. + """ + if versions == None: + return render.alias( + name = name, + actual = repr("@{repo_name}_{dep}//:{target}".format( + repo_name = repo_name, + dep = dep, + target = target, + )), + ) + + # Create the alias repositories which contains different select + # statements These select statements point to the different pip + # whls that are based on a specific version of Python. + selects = {} + for full_version in versions: + condition = "@@{rules_python}//python/config_settings:is_python_{full_python_version}".format( + rules_python = rules_python, + full_python_version = full_version, + ) + actual = "@{repo_name}_{version}_{dep}//:{target}".format( + repo_name = repo_name, + version = version_label(full_version), + dep = dep, + target = target, + ) + selects[condition] = actual + + if default_version: + no_match_error = None + default_actual = "@{repo_name}_{version}_{dep}//:{target}".format( + repo_name = repo_name, + version = version_label(default_version), + dep = dep, + target = target, + ) + selects["//conditions:default"] = default_actual + else: + no_match_error = "_NO_MATCH_ERROR" + + return render.alias( + name = name, + actual = render.select( + selects, + no_match_error = no_match_error, + ), + ) + +def _render_common_aliases(repo_name, name, versions = None, default_version = None, rules_python = None): + lines = [ + """package(default_visibility = ["//visibility:public"])""", + ] + + if versions: + versions = sorted(versions) + + if versions and not default_version: + error_msg = NO_MATCH_ERROR_MESSAGE_TEMPLATE.format( + supported_versions = ", ".join(versions), + rules_python = rules_python, + ) + + lines.append("_NO_MATCH_ERROR = \"\"\"\\\n{error_msg}\"\"\"".format( + error_msg = error_msg, + )) + + lines.append( + render.alias( + name = name, + actual = repr(":pkg"), + ), + ) + lines.extend( + [ + _render_whl_library_alias( + name = target, + repo_name = repo_name, + dep = name, + target = target, + versions = versions, + default_version = default_version, + rules_python = rules_python, + ) + for target in ["pkg", "whl", "data", "dist_info"] + ], + ) + + return "\n\n".join(lines) + +def render_pkg_aliases(*, repo_name, bzl_packages = None, whl_map = None, rules_python = None, default_version = None): + """Create alias declarations for each PyPI package. + + The aliases should be appended to the pip_repository BUILD.bazel file. These aliases + allow users to use requirement() without needed a corresponding `use_repo()` for each dep + when using bzlmod. + + Args: + repo_name: the repository name of the hub repository that is visible to the users that is + also used as the prefix for the spoke repo names (e.g. "pip", "pypi"). + bzl_packages: the list of packages to setup, if not specified, whl_map.keys() will be used instead. + whl_map: the whl_map for generating Python version aware aliases. + default_version: the default version to be used for the aliases. + rules_python: the name of the rules_python workspace. + + Returns: + A dict of file paths and their contents. + """ + if not bzl_packages and whl_map: + bzl_packages = list(whl_map.keys()) + + contents = {} + for name in bzl_packages: + versions = None + if whl_map != None: + versions = whl_map[name] + name = normalize_name(name) + + filename = "{}/BUILD.bazel".format(name) + contents[filename] = _render_common_aliases( + repo_name = repo_name, + name = name, + versions = versions, + rules_python = rules_python, + default_version = default_version, + ).strip() + + return contents diff --git a/python/private/text_util.bzl b/python/private/text_util.bzl new file mode 100644 index 0000000..3d72b8d --- /dev/null +++ b/python/private/text_util.bzl @@ -0,0 +1,65 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Text manipulation utilities useful for repository rule writing.""" + +def _indent(text, indent = " " * 4): + if "\n" not in text: + return indent + text + + return "\n".join([indent + line for line in text.splitlines()]) + +def _render_alias(name, actual): + return "\n".join([ + "alias(", + _indent("name = \"{}\",".format(name)), + _indent("actual = {},".format(actual)), + ")", + ]) + +def _render_dict(d): + return "\n".join([ + "{", + _indent("\n".join([ + "{}: {},".format(repr(k), repr(v)) + for k, v in d.items() + ])), + "}", + ]) + +def _render_select(selects, *, no_match_error = None): + dict_str = _render_dict(selects) + "," + + if no_match_error: + args = "\n".join([ + "", + _indent(dict_str), + _indent("no_match_error = {},".format(no_match_error)), + "", + ]) + else: + args = "\n".join([ + "", + _indent(dict_str), + "", + ]) + + return "select({})".format(args) + +render = struct( + indent = _indent, + alias = _render_alias, + dict = _render_dict, + select = _render_select, +) diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index 5923787..b2919c1 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -30,6 +30,7 @@ load( "PLATFORMS", "WINDOWS_NAME", ) +load(":which.bzl", "which_with_fail") def get_repository_name(repository_workspace): dummy_label = "//:_" @@ -325,7 +326,7 @@ def get_host_os_arch(rctx): os_name = WINDOWS_NAME else: # This is not ideal, but bazel doesn't directly expose arch. - arch = rctx.execute(["uname", "-m"]).stdout.strip() + arch = rctx.execute([which_with_fail("uname", rctx), "-m"]).stdout.strip() # Normalize the os_name. if "mac" in os_name.lower(): diff --git a/python/pip_install/tools/lib/__init__.py b/python/private/which.bzl index bbdfb4c..b0cbddb 100644 --- a/python/pip_install/tools/lib/__init__.py +++ b/python/private/which.bzl @@ -12,3 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Wrapper for repository which call""" + +_binary_not_found_msg = "Unable to find the binary '{binary_name}'. Please update your PATH to include '{binary_name}'." + +def which_with_fail(binary_name, rctx): + """Tests to see if a binary exists, and otherwise fails with a message. + + Args: + binary_name: name of the binary to find. + rctx: repository context. + + Returns: + rctx.Path for the binary. + """ + binary = rctx.which(binary_name) + if binary == None: + fail(_binary_not_found_msg.format(binary_name = binary_name)) + return binary diff --git a/python/repositories.bzl b/python/repositories.bzl index 62d9421..bd06f0b 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -27,6 +27,7 @@ load( "toolchain_aliases", "toolchains_repo", ) +load("//python/private:which.bzl", "which_with_fail") load( ":versions.bzl", "DEFAULT_RELEASE_BASE_URL", @@ -123,8 +124,9 @@ def _python_repository_impl(rctx): sha256 = rctx.attr.zstd_sha256, ) working_directory = "zstd-{version}".format(version = rctx.attr.zstd_version) + make_result = rctx.execute( - ["make", "--jobs=4"], + [which_with_fail("make", rctx), "--jobs=4"], timeout = 600, quiet = True, working_directory = working_directory, @@ -140,7 +142,7 @@ def _python_repository_impl(rctx): rctx.symlink(zstd, unzstd) exec_result = rctx.execute([ - "tar", + which_with_fail("tar", rctx), "--extract", "--strip-components=2", "--use-compress-program={unzstd}".format(unzstd = unzstd), @@ -179,15 +181,16 @@ def _python_repository_impl(rctx): if not rctx.attr.ignore_root_user_error: if "windows" not in rctx.os.name: lib_dir = "lib" if "windows" not in platform else "Lib" - exec_result = rctx.execute(["chmod", "-R", "ugo-w", lib_dir]) + + exec_result = rctx.execute([which_with_fail("chmod", rctx), "-R", "ugo-w", lib_dir]) if exec_result.return_code != 0: fail_msg = "Failed to make interpreter installation read-only. 'chmod' error msg: {}".format( exec_result.stderr, ) fail(fail_msg) - exec_result = rctx.execute(["touch", "{}/.test".format(lib_dir)]) + exec_result = rctx.execute([which_with_fail("touch", rctx), "{}/.test".format(lib_dir)]) if exec_result.return_code == 0: - exec_result = rctx.execute(["id", "-u"]) + exec_result = rctx.execute([which_with_fail("id", rctx), "-u"]) if exec_result.return_code != 0: fail("Could not determine current user ID. 'id -u' error msg: {}".format( exec_result.stderr, diff --git a/python/versions.bzl b/python/versions.bzl index a88c982..1ef3172 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -153,6 +153,19 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.9.17": { + "url": "20230726/cpython-{python_version}+20230726-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "73dbe2d702210b566221da9265acc274ba15275c5d0d1fa327f44ad86cde9aa1", + "aarch64-unknown-linux-gnu": "b77012ddaf7e0673e4aa4b1c5085275a06eee2d66f33442b5c54a12b62b96cbe", + "ppc64le-unknown-linux-gnu": "c591a28d943dce5cf9833e916125fdfbeb3120270c4866ee214493ccb5b83c3c", + "s390x-unknown-linux-gnu": "01454d7cc7c9c2fccde42ba868c4f372eaaafa48049d49dd94c9cf2875f497e6", + "x86_64-apple-darwin": "dfe1bea92c94b9cb779288b0b06e39157c5ff7e465cdd24032ac147c2af485c0", + "x86_64-pc-windows-msvc": "9b9a1e21eff29dcf043cea38180cf8ca3604b90117d00062a7b31605d4157714", + "x86_64-unknown-linux-gnu": "26c4a712b4b8e11ed5c027db5654eb12927c02da4857b777afb98f7a930ce637", + }, + "strip_prefix": "python", + }, "3.10.2": { "url": "20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz", "sha256": { @@ -220,6 +233,19 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.10.12": { + "url": "20230726/cpython-{python_version}+20230726-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "bc66c706ea8c5fc891635fda8f9da971a1a901d41342f6798c20ad0b2a25d1d6", + "aarch64-unknown-linux-gnu": "fee80e221663eca5174bd794cb5047e40d3910dbeadcdf1f09d405a4c1c15fe4", + "ppc64le-unknown-linux-gnu": "bb5e8cb0d2e44241725fa9b342238245503e7849917660006b0246a9c97b1d6c", + "s390x-unknown-linux-gnu": "8d33d435ae6fb93ded7fc26798cc0a1a4f546a4e527012a1e2909cc314b332df", + "x86_64-apple-darwin": "8a6e3ed973a671de468d9c691ed9cb2c3a4858c5defffcf0b08969fba9c1dd04", + "x86_64-pc-windows-msvc": "c1a31c353ca44de7d1b1a3b6c55a823e9c1eed0423d4f9f66e617bdb1b608685", + "x86_64-unknown-linux-gnu": "a476dbca9184df9fc69fe6309cda5ebaf031d27ca9e529852437c94ec1bc43d3", + }, + "strip_prefix": "python", + }, "3.11.1": { "url": "20230116/cpython-{python_version}+20230116-{platform}-{build}.tar.gz", "sha256": { @@ -243,14 +269,27 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.11.4": { + "url": "20230726/cpython-{python_version}+20230726-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "cb6d2948384a857321f2aa40fa67744cd9676a330f08b6dad7070bda0b6120a4", + "aarch64-unknown-linux-gnu": "2e84fc53f4e90e11963281c5c871f593abcb24fc796a50337fa516be99af02fb", + "ppc64le-unknown-linux-gnu": "df7b92ed9cec96b3bb658fb586be947722ecd8e420fb23cee13d2e90abcfcf25", + "s390x-unknown-linux-gnu": "e477f0749161f9aa7887964f089d9460a539f6b4a8fdab5166f898210e1a87a4", + "x86_64-apple-darwin": "47e1557d93a42585972772e82661047ca5f608293158acb2778dccf120eabb00", + "x86_64-pc-windows-msvc": "878614c03ea38538ae2f758e36c85d2c0eb1eaaca86cd400ff8c76693ee0b3e1", + "x86_64-unknown-linux-gnu": "e26247302bc8e9083a43ce9e8dd94905b40d464745b1603041f7bc9a93c65d05", + }, + "strip_prefix": "python", + }, } # buildifier: disable=unsorted-dict-items MINOR_MAPPING = { "3.8": "3.8.15", - "3.9": "3.9.16", - "3.10": "3.10.9", - "3.11": "3.11.1", + "3.9": "3.9.17", + "3.10": "3.10.12", + "3.11": "3.11.4", } PLATFORMS = { @@ -286,6 +325,17 @@ PLATFORMS = { # repository_ctx.execute(["uname", "-m"]).stdout.strip() arch = "ppc64le", ), + "s390x-unknown-linux-gnu": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:s390x", + ], + os_name = LINUX_NAME, + # Note: this string differs between OSX and Linux + # Matches the value returned from: + # repository_ctx.execute(["uname", "-m"]).stdout.strip() + arch = "s390x", + ), "x86_64-apple-darwin": struct( compatible_with = [ "@platforms//os:macos", diff --git a/tests/cc/BUILD.bazel b/tests/cc/BUILD.bazel index 876d163..3f7925d 100644 --- a/tests/cc/BUILD.bazel +++ b/tests/cc/BUILD.bazel @@ -19,6 +19,8 @@ load(":fake_cc_toolchain_config.bzl", "fake_cc_toolchain_config") package(default_visibility = ["//:__subpackages__"]) +exports_files(["fake_header.h"]) + toolchain( name = "fake_py_cc_toolchain", tags = PREVENT_IMPLICIT_BUILDING_TAGS, diff --git a/tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel b/tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel new file mode 100644 index 0000000..f2e0126 --- /dev/null +++ b/tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel @@ -0,0 +1,3 @@ +load(":render_pkg_aliases_test.bzl", "render_pkg_aliases_test_suite") + +render_pkg_aliases_test_suite(name = "render_pkg_aliases_tests") diff --git a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl new file mode 100644 index 0000000..28d95ff --- /dev/null +++ b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -0,0 +1,251 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""render_pkg_aliases tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_legacy_aliases(env): + actual = render_pkg_aliases( + bzl_packages = ["foo"], + repo_name = "pypi", + ) + + want = { + "foo/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +alias( + name = "foo", + actual = ":pkg", +) + +alias( + name = "pkg", + actual = "@pypi_foo//:pkg", +) + +alias( + name = "whl", + actual = "@pypi_foo//:whl", +) + +alias( + name = "data", + actual = "@pypi_foo//:data", +) + +alias( + name = "dist_info", + actual = "@pypi_foo//:dist_info", +)""", + } + + env.expect.that_dict(actual).contains_exactly(want) + +_tests.append(_test_legacy_aliases) + +def _test_all_legacy_aliases_are_created(env): + actual = render_pkg_aliases( + bzl_packages = ["foo", "bar"], + repo_name = "pypi", + ) + + want_files = ["bar/BUILD.bazel", "foo/BUILD.bazel"] + + env.expect.that_dict(actual).keys().contains_exactly(want_files) + +_tests.append(_test_all_legacy_aliases_are_created) + +def _test_bzlmod_aliases(env): + actual = render_pkg_aliases( + default_version = "3.2.3", + repo_name = "pypi", + rules_python = "rules_python", + whl_map = { + "bar-baz": ["3.2.3"], + }, + ) + + want = { + "bar_baz/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +alias( + name = "bar_baz", + actual = ":pkg", +) + +alias( + name = "pkg", + actual = select( + { + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:pkg", + "//conditions:default": "@pypi_32_bar_baz//:pkg", + }, + ), +) + +alias( + name = "whl", + actual = select( + { + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:whl", + "//conditions:default": "@pypi_32_bar_baz//:whl", + }, + ), +) + +alias( + name = "data", + actual = select( + { + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:data", + "//conditions:default": "@pypi_32_bar_baz//:data", + }, + ), +) + +alias( + name = "dist_info", + actual = select( + { + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:dist_info", + "//conditions:default": "@pypi_32_bar_baz//:dist_info", + }, + ), +)""", + } + + env.expect.that_dict(actual).contains_exactly(want) + +_tests.append(_test_bzlmod_aliases) + +def _test_bzlmod_aliases_with_no_default_version(env): + actual = render_pkg_aliases( + default_version = None, + repo_name = "pypi", + rules_python = "rules_python", + whl_map = { + "bar-baz": ["3.2.3", "3.1.3"], + }, + ) + + want_key = "bar_baz/BUILD.bazel" + want_content = """\ +package(default_visibility = ["//visibility:public"]) + +_NO_MATCH_ERROR = \"\"\"\\ +No matching wheel for current configuration's Python version. + +The current build configuration's Python version doesn't match any of the Python +versions available for this wheel. This wheel supports the following Python versions: + 3.1.3, 3.2.3 + +As matched by the `@rules_python//python/config_settings:is_python_<version>` +configuration settings. + +To determine the current configuration's Python version, run: + `bazel config <config id>` (shown further below) +and look for + rules_python//python/config_settings:python_version + +If the value is missing, then the "default" Python version is being used, +which has a "null" version value and will not match version constraints. +\"\"\" + +alias( + name = "bar_baz", + actual = ":pkg", +) + +alias( + name = "pkg", + actual = select( + { + "@@rules_python//python/config_settings:is_python_3.1.3": "@pypi_31_bar_baz//:pkg", + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:pkg", + }, + no_match_error = _NO_MATCH_ERROR, + ), +) + +alias( + name = "whl", + actual = select( + { + "@@rules_python//python/config_settings:is_python_3.1.3": "@pypi_31_bar_baz//:whl", + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:whl", + }, + no_match_error = _NO_MATCH_ERROR, + ), +) + +alias( + name = "data", + actual = select( + { + "@@rules_python//python/config_settings:is_python_3.1.3": "@pypi_31_bar_baz//:data", + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:data", + }, + no_match_error = _NO_MATCH_ERROR, + ), +) + +alias( + name = "dist_info", + actual = select( + { + "@@rules_python//python/config_settings:is_python_3.1.3": "@pypi_31_bar_baz//:dist_info", + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:dist_info", + }, + no_match_error = _NO_MATCH_ERROR, + ), +)""" + + env.expect.that_collection(actual.keys()).contains_exactly([want_key]) + env.expect.that_str(actual[want_key]).equals(want_content) + +_tests.append(_test_bzlmod_aliases_with_no_default_version) + +def _test_bzlmod_aliases_are_created_for_all_wheels(env): + actual = render_pkg_aliases( + default_version = "3.2.3", + repo_name = "pypi", + rules_python = "rules_python", + whl_map = { + "bar": ["3.1.2", "3.2.3"], + "foo": ["3.1.2", "3.2.3"], + }, + ) + + want_files = [ + "bar/BUILD.bazel", + "foo/BUILD.bazel", + ] + + env.expect.that_dict(actual).keys().contains_exactly(want_files) + +_tests.append(_test_bzlmod_aliases_are_created_for_all_wheels) + +def render_pkg_aliases_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/pip_install/BUILD.bazel b/tests/pip_install/BUILD.bazel new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tests/pip_install/BUILD.bazel diff --git a/tests/pip_install/whl_library/BUILD.bazel b/tests/pip_install/whl_library/BUILD.bazel new file mode 100644 index 0000000..5a27e11 --- /dev/null +++ b/tests/pip_install/whl_library/BUILD.bazel @@ -0,0 +1,3 @@ +load(":generate_build_bazel_tests.bzl", "generate_build_bazel_test_suite") + +generate_build_bazel_test_suite(name = "generate_build_bazel_tests") diff --git a/tests/pip_install/whl_library/generate_build_bazel_tests.bzl b/tests/pip_install/whl_library/generate_build_bazel_tests.bzl new file mode 100644 index 0000000..365233d --- /dev/null +++ b/tests/pip_install/whl_library/generate_build_bazel_tests.bzl @@ -0,0 +1,225 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_simple(env): + want = """\ +load("@rules_python//python:defs.bzl", "py_library", "py_binary") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "dist_info", + srcs = glob(["site-packages/*.dist-info/**"], allow_empty = True), +) + +filegroup( + name = "data", + srcs = glob(["data/**"], allow_empty = True), +) + +filegroup( + name = "whl", + srcs = glob(["*.whl"], allow_empty = True), + data = ["@pypi_bar_baz//:whl", "@pypi_foo//:whl"], +) + +py_library( + name = "pkg", + srcs = glob( + ["site-packages/**/*.py"], + exclude=[], + # Empty sources are allowed to support wheels that don't have any + # pure-Python code, e.g. pymssql, which is written in Cython. + allow_empty = True, + ), + data = [] + glob( + ["site-packages/**/*"], + exclude=["**/* *", "**/*.py", "**/*.pyc", "**/*.pyc.*", "**/*.dist-info/RECORD"], + ), + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["site-packages"], + deps = ["@pypi_bar_baz//:pkg", "@pypi_foo//:pkg"], + tags = ["tag1", "tag2"], +) +""" + actual = generate_whl_library_build_bazel( + repo_prefix = "pypi_", + dependencies = ["foo", "bar-baz"], + data_exclude = [], + tags = ["tag1", "tag2"], + entry_points = {}, + annotation = None, + ) + env.expect.that_str(actual).equals(want) + +_tests.append(_test_simple) + +def _test_with_annotation(env): + want = """\ +load("@rules_python//python:defs.bzl", "py_library", "py_binary") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "dist_info", + srcs = glob(["site-packages/*.dist-info/**"], allow_empty = True), +) + +filegroup( + name = "data", + srcs = glob(["data/**"], allow_empty = True), +) + +filegroup( + name = "whl", + srcs = glob(["*.whl"], allow_empty = True), + data = ["@pypi_bar_baz//:whl", "@pypi_foo//:whl"], +) + +py_library( + name = "pkg", + srcs = glob( + ["site-packages/**/*.py"], + exclude=["srcs_exclude_all"], + # Empty sources are allowed to support wheels that don't have any + # pure-Python code, e.g. pymssql, which is written in Cython. + allow_empty = True, + ), + data = ["file_dest", "exec_dest"] + glob( + ["site-packages/**/*"], + exclude=["**/* *", "**/*.py", "**/*.pyc", "**/*.pyc.*", "**/*.dist-info/RECORD", "data_exclude_all"], + ), + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["site-packages"], + deps = ["@pypi_bar_baz//:pkg", "@pypi_foo//:pkg"], + tags = ["tag1", "tag2"], +) + +copy_file( + name = "file_dest.copy", + src = "file_src", + out = "file_dest", + is_executable = False, +) + +copy_file( + name = "exec_dest.copy", + src = "exec_src", + out = "exec_dest", + is_executable = True, +) + +# SOMETHING SPECIAL AT THE END +""" + actual = generate_whl_library_build_bazel( + repo_prefix = "pypi_", + dependencies = ["foo", "bar-baz"], + data_exclude = [], + tags = ["tag1", "tag2"], + entry_points = {}, + annotation = struct( + copy_files = {"file_src": "file_dest"}, + copy_executables = {"exec_src": "exec_dest"}, + data = [], + data_exclude_glob = ["data_exclude_all"], + srcs_exclude_glob = ["srcs_exclude_all"], + additive_build_content = """# SOMETHING SPECIAL AT THE END""", + ), + ) + env.expect.that_str(actual).equals(want) + +_tests.append(_test_with_annotation) + +def _test_with_entry_points(env): + want = """\ +load("@rules_python//python:defs.bzl", "py_library", "py_binary") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "dist_info", + srcs = glob(["site-packages/*.dist-info/**"], allow_empty = True), +) + +filegroup( + name = "data", + srcs = glob(["data/**"], allow_empty = True), +) + +filegroup( + name = "whl", + srcs = glob(["*.whl"], allow_empty = True), + data = ["@pypi_bar_baz//:whl", "@pypi_foo//:whl"], +) + +py_library( + name = "pkg", + srcs = glob( + ["site-packages/**/*.py"], + exclude=[], + # Empty sources are allowed to support wheels that don't have any + # pure-Python code, e.g. pymssql, which is written in Cython. + allow_empty = True, + ), + data = [] + glob( + ["site-packages/**/*"], + exclude=["**/* *", "**/*.py", "**/*.pyc", "**/*.pyc.*", "**/*.dist-info/RECORD"], + ), + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["site-packages"], + deps = ["@pypi_bar_baz//:pkg", "@pypi_foo//:pkg"], + tags = ["tag1", "tag2"], +) + +py_binary( + name = "rules_python_wheel_entry_point_fizz", + srcs = ["buzz.py"], + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["."], + deps = [":pkg"], +) +""" + actual = generate_whl_library_build_bazel( + repo_prefix = "pypi_", + dependencies = ["foo", "bar-baz"], + data_exclude = [], + tags = ["tag1", "tag2"], + entry_points = {"fizz": "buzz.py"}, + annotation = None, + ) + env.expect.that_str(actual).equals(want) + +_tests.append(_test_with_entry_points) + +def generate_build_bazel_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tools/private/update_deps/BUILD.bazel b/tools/private/update_deps/BUILD.bazel new file mode 100644 index 0000000..2ab7cc7 --- /dev/null +++ b/tools/private/update_deps/BUILD.bazel @@ -0,0 +1,76 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +load("//python:py_binary.bzl", "py_binary") +load("//python:py_library.bzl", "py_library") +load("//python:py_test.bzl", "py_test") + +licenses(["notice"]) + +py_library( + name = "args", + srcs = ["args.py"], + imports = ["../../.."], + deps = ["//python/runfiles"], +) + +py_library( + name = "update_file", + srcs = ["update_file.py"], + imports = ["../../.."], +) + +py_binary( + name = "update_coverage_deps", + srcs = ["update_coverage_deps.py"], + data = [ + "//python/private:coverage_deps", + ], + env = { + "UPDATE_FILE": "$(rlocationpath //python/private:coverage_deps)", + }, + imports = ["../../.."], + deps = [ + ":args", + ":update_file", + ], +) + +py_binary( + name = "update_pip_deps", + srcs = ["update_pip_deps.py"], + data = [ + "//:MODULE.bazel", + "//python/pip_install:repositories", + "//python/pip_install:requirements_txt", + ], + env = { + "MODULE_BAZEL": "$(rlocationpath //:MODULE.bazel)", + "REPOSITORIES_BZL": "$(rlocationpath //python/pip_install:repositories)", + "REQUIREMENTS_TXT": "$(rlocationpath //python/pip_install:requirements_txt)", + }, + imports = ["../../.."], + deps = [ + ":args", + ":update_file", + ], +) + +py_test( + name = "update_file_test", + srcs = ["update_file_test.py"], + imports = ["../../.."], + deps = [ + ":update_file", + ], +) diff --git a/tools/private/update_deps/args.py b/tools/private/update_deps/args.py new file mode 100644 index 0000000..293294c --- /dev/null +++ b/tools/private/update_deps/args.py @@ -0,0 +1,35 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A small library for common arguments when updating files.""" + +import pathlib + +from python.runfiles import runfiles + + +def path_from_runfiles(input: str) -> pathlib.Path: + """A helper to create a path from runfiles. + + Args: + input: the string input to construct a path. + + Returns: + the pathlib.Path path to a file which is verified to exist. + """ + path = pathlib.Path(runfiles.Create().Rlocation(input)) + if not path.exists(): + raise ValueError(f"Path '{path}' does not exist") + + return path diff --git a/tools/update_coverage_deps.py b/tools/private/update_deps/update_coverage_deps.py index 57b7850..72baa44 100755 --- a/tools/update_coverage_deps.py +++ b/tools/private/update_deps/update_coverage_deps.py @@ -22,6 +22,7 @@ We are not running this with 'bazel run' to keep the dependencies minimal import argparse import difflib import json +import os import pathlib import sys import textwrap @@ -30,6 +31,9 @@ from dataclasses import dataclass from typing import Any from urllib import request +from tools.private.update_deps.args import path_from_runfiles +from tools.private.update_deps.update_file import update_file + # This should be kept in sync with //python:versions.bzl _supported_platforms = { # Windows is unsupported right now @@ -110,64 +114,6 @@ def _map( ) -def _writelines(path: pathlib.Path, lines: list[str]): - with open(path, "w") as f: - f.writelines(lines) - - -def _difflines(path: pathlib.Path, lines: list[str]): - with open(path) as f: - input = f.readlines() - - rules_python = pathlib.Path(__file__).parent.parent - p = path.relative_to(rules_python) - - print(f"Diff of the changes that would be made to '{p}':") - for line in difflib.unified_diff( - input, - lines, - fromfile=f"a/{p}", - tofile=f"b/{p}", - ): - print(line, end="") - - # Add an empty line at the end of the diff - print() - - -def _update_file( - path: pathlib.Path, - snippet: str, - start_marker: str, - end_marker: str, - dry_run: bool = True, -): - with open(path) as f: - input = f.readlines() - - out = [] - skip = False - for line in input: - if skip: - if not line.startswith(end_marker): - continue - - skip = False - - out.append(line) - - if not line.startswith(start_marker): - continue - - skip = True - out.extend([f"{line}\n" for line in snippet.splitlines()]) - - if dry_run: - _difflines(path, out) - else: - _writelines(path, out) - - def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(__doc__) parser.add_argument( @@ -193,6 +139,12 @@ def _parse_args() -> argparse.Namespace: action="store_true", help="Wether to write to files", ) + parser.add_argument( + "--update-file", + type=path_from_runfiles, + default=os.environ.get("UPDATE_FILE"), + help="The path for the file to be updated, defaults to the value taken from UPDATE_FILE", + ) return parser.parse_args() @@ -230,14 +182,12 @@ def main(): urls.sort(key=lambda x: f"{x.python}_{x.platform}") - rules_python = pathlib.Path(__file__).parent.parent - # Update the coverage_deps, which are used to register deps - _update_file( - path=rules_python / "python" / "private" / "coverage_deps.bzl", + update_file( + path=args.update_file, snippet=f"_coverage_deps = {repr(Deps(urls))}\n", - start_marker="#START: managed by update_coverage_deps.py script", - end_marker="#END: managed by update_coverage_deps.py script", + start_marker="# START: maintained by 'bazel run //tools/private:update_coverage_deps'", + end_marker="# END: maintained by 'bazel run //tools/private:update_coverage_deps'", dry_run=args.dry_run, ) diff --git a/tools/private/update_deps/update_file.py b/tools/private/update_deps/update_file.py new file mode 100644 index 0000000..ab3e8a8 --- /dev/null +++ b/tools/private/update_deps/update_file.py @@ -0,0 +1,114 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A small library to update bazel files within the repo. + +This is reused in other files updating coverage deps and pip deps. +""" + +import argparse +import difflib +import pathlib +import sys + + +def _writelines(path: pathlib.Path, out: str): + with open(path, "w") as f: + f.write(out) + + +def unified_diff(name: str, a: str, b: str) -> str: + return "".join( + difflib.unified_diff( + a.splitlines(keepends=True), + b.splitlines(keepends=True), + fromfile=f"a/{name}", + tofile=f"b/{name}", + ) + ).strip() + + +def replace_snippet( + current: str, + snippet: str, + start_marker: str, + end_marker: str, +) -> str: + """Update a file on disk to replace text in a file between two markers. + + Args: + path: pathlib.Path, the path to the file to be modified. + snippet: str, the snippet of code to insert between the markers. + start_marker: str, the text that marks the start of the region to be replaced. + end_markr: str, the text that marks the end of the region to be replaced. + dry_run: bool, if set to True, then the file will not be written and instead we are going to print a diff to + stdout. + """ + lines = [] + skip = False + found_match = False + for line in current.splitlines(keepends=True): + if line.lstrip().startswith(start_marker.lstrip()): + found_match = True + lines.append(line) + lines.append(snippet.rstrip() + "\n") + skip = True + elif skip and line.lstrip().startswith(end_marker): + skip = False + lines.append(line) + continue + elif not skip: + lines.append(line) + + if not found_match: + raise RuntimeError(f"Start marker '{start_marker}' was not found") + if skip: + raise RuntimeError(f"End marker '{end_marker}' was not found") + + return "".join(lines) + + +def update_file( + path: pathlib.Path, + snippet: str, + start_marker: str, + end_marker: str, + dry_run: bool = True, +): + """update a file on disk to replace text in a file between two markers. + + Args: + path: pathlib.Path, the path to the file to be modified. + snippet: str, the snippet of code to insert between the markers. + start_marker: str, the text that marks the start of the region to be replaced. + end_markr: str, the text that marks the end of the region to be replaced. + dry_run: bool, if set to True, then the file will not be written and instead we are going to print a diff to + stdout. + """ + current = path.read_text() + out = replace_snippet(current, snippet, start_marker, end_marker) + + if not dry_run: + _writelines(path, out) + return + + relative = path.relative_to( + pathlib.Path(__file__).resolve().parent.parent.parent.parent + ) + name = f"{relative}" + diff = unified_diff(name, current, out) + if diff: + print(f"Diff of the changes that would be made to '{name}':\n{diff}") + else: + print(f"'{name}' is up to date") diff --git a/tools/private/update_deps/update_file_test.py b/tools/private/update_deps/update_file_test.py new file mode 100644 index 0000000..01c6ec7 --- /dev/null +++ b/tools/private/update_deps/update_file_test.py @@ -0,0 +1,128 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from tools.private.update_deps.update_file import replace_snippet, unified_diff + + +class TestReplaceSnippet(unittest.TestCase): + def test_replace_simple(self): + current = """\ +Before the snippet + +# Start marker +To be replaced +It may have the '# Start marker' or '# End marker' in the middle, +But it has to be in the beginning of the line to mark the end of a region. +# End marker + +After the snippet +""" + snippet = "Replaced" + got = replace_snippet( + current=current, + snippet="Replaced", + start_marker="# Start marker", + end_marker="# End marker", + ) + + want = """\ +Before the snippet + +# Start marker +Replaced +# End marker + +After the snippet +""" + self.assertEqual(want, got) + + def test_replace_indented(self): + current = """\ +Before the snippet + + # Start marker + To be replaced + # End marker + +After the snippet +""" + got = replace_snippet( + current=current, + snippet=" Replaced", + start_marker="# Start marker", + end_marker="# End marker", + ) + + want = """\ +Before the snippet + + # Start marker + Replaced + # End marker + +After the snippet +""" + self.assertEqual(want, got) + + def test_raises_if_start_is_not_found(self): + with self.assertRaises(RuntimeError) as exc: + replace_snippet( + current="foo", + snippet="", + start_marker="start", + end_marker="end", + ) + + self.assertEqual(exc.exception.args[0], "Start marker 'start' was not found") + + def test_raises_if_end_is_not_found(self): + with self.assertRaises(RuntimeError) as exc: + replace_snippet( + current="start", + snippet="", + start_marker="start", + end_marker="end", + ) + + self.assertEqual(exc.exception.args[0], "End marker 'end' was not found") + + +class TestUnifiedDiff(unittest.TestCase): + def test_diff(self): + give_a = """\ +First line +second line +Third line +""" + give_b = """\ +First line +Second line +Third line +""" + got = unified_diff("filename", give_a, give_b) + want = """\ +--- a/filename ++++ b/filename +@@ -1,3 +1,3 @@ + First line +-second line ++Second line + Third line""" + self.assertEqual(want, got) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/private/update_deps/update_pip_deps.py b/tools/private/update_deps/update_pip_deps.py new file mode 100755 index 0000000..8a2dd5f --- /dev/null +++ b/tools/private/update_deps/update_pip_deps.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A script to manage internal pip dependencies.""" + +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import re +import sys +import tempfile +import textwrap +from dataclasses import dataclass + +from pip._internal.cli.main import main as pip_main + +from tools.private.update_deps.args import path_from_runfiles +from tools.private.update_deps.update_file import update_file + + +@dataclass +class Dep: + name: str + url: str + sha256: str + + +def _dep_snippet(deps: list[Dep]) -> str: + lines = [] + for dep in deps: + lines.extend( + [ + "(\n", + f' "{dep.name}",\n', + f' "{dep.url}",\n', + f' "{dep.sha256}",\n', + "),\n", + ] + ) + + return textwrap.indent("".join(lines), " " * 4) + + +def _module_snippet(deps: list[Dep]) -> str: + lines = [] + for dep in deps: + lines.append(f'"{dep.name}",\n') + + return textwrap.indent("".join(lines), " " * 4) + + +def _generate_report(requirements_txt: pathlib.Path) -> dict: + with tempfile.NamedTemporaryFile() as tmp: + tmp_path = pathlib.Path(tmp.name) + sys.argv = [ + "pip", + "install", + "--dry-run", + "--ignore-installed", + "--report", + f"{tmp_path}", + "-r", + f"{requirements_txt}", + ] + pip_main() + with open(tmp_path) as f: + return json.load(f) + + +def _get_deps(report: dict) -> list[Dep]: + deps = [] + for dep in report["install"]: + try: + dep = Dep( + name="pypi__" + + re.sub( + "[._-]+", + "_", + dep["metadata"]["name"], + ), + url=dep["download_info"]["url"], + sha256=dep["download_info"]["archive_info"]["hash"][len("sha256=") :], + ) + except: + debug_dep = textwrap.indent(json.dumps(dep, indent=4), " " * 4) + print(f"Could not parse the response from 'pip':\n{debug_dep}") + raise + + deps.append(dep) + + return sorted(deps, key=lambda dep: dep.name) + + +def main(): + parser = argparse.ArgumentParser(__doc__) + parser.add_argument( + "--start", + type=str, + default="# START: maintained by 'bazel run //tools/private:update_pip_deps'", + help="The text to match in a file when updating them.", + ) + parser.add_argument( + "--end", + type=str, + default="# END: maintained by 'bazel run //tools/private:update_pip_deps'", + help="The text to match in a file when updating them.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Wether to write to files", + ) + parser.add_argument( + "--requirements-txt", + type=path_from_runfiles, + default=os.environ.get("REQUIREMENTS_TXT"), + help="The requirements.txt path for the pip_install tools, defaults to the value taken from REQUIREMENTS_TXT", + ) + parser.add_argument( + "--module-bazel", + type=path_from_runfiles, + default=os.environ.get("MODULE_BAZEL"), + help="The path for the file to be updated, defaults to the value taken from MODULE_BAZEL", + ) + parser.add_argument( + "--repositories-bzl", + type=path_from_runfiles, + default=os.environ.get("REPOSITORIES_BZL"), + help="The path for the file to be updated, defaults to the value taken from REPOSITORIES_BZL", + ) + args = parser.parse_args() + + report = _generate_report(args.requirements_txt) + deps = _get_deps(report) + + update_file( + path=args.repositories_bzl, + snippet=_dep_snippet(deps), + start_marker=args.start, + end_marker=args.end, + dry_run=args.dry_run, + ) + + update_file( + path=args.module_bazel, + snippet=_module_snippet(deps), + start_marker=args.start, + end_marker=args.end, + dry_run=args.dry_run, + ) + + +if __name__ == "__main__": + main() |