diff options
Diffstat (limited to 'pw_build/python_venv.gni')
| -rw-r--r-- | pw_build/python_venv.gni | 274 |
1 files changed, 245 insertions, 29 deletions
diff --git a/pw_build/python_venv.gni b/pw_build/python_venv.gni index c36b7f90f..6a5ae770c 100644 --- a/pw_build/python_venv.gni +++ b/pw_build/python_venv.gni @@ -46,6 +46,9 @@ import("$dir_pw_build/python_action.gni") # requirements: A list of requirements files to install into this virtualenv # on creation. By default this is set to pw_build_PIP_REQUIREMENTS # +# pip_generate_hashes: (Default: false) Use --generate-hashes When +# running pip-compile to compute the final requirements.txt +# # source_packages: A list of in-tree pw_python_package targets that will be # checked for external third_party pip dependencies to install into this # virtualenv. Note this list of targets isn't actually installed into the @@ -54,6 +57,8 @@ import("$dir_pw_build/python_action.gni") # this page for a setup.cfg example: # https://setuptools.pypa.io/en/latest/userguide/declarative_config.html # +# output_logs: (Default: true) Commands will output logs. +# template("pw_python_venv") { assert(defined(invoker.path), "pw_python_venv requires a 'path'") @@ -62,6 +67,9 @@ template("pw_python_venv") { _generated_requirements_file = "$target_gen_dir/$target_name/generated_requirements.txt" + _compiled_requirements_file = + "$target_gen_dir/$target_name/compiled_requirements.txt" + _source_packages = [] if (defined(invoker.source_packages)) { _source_packages += invoker.source_packages @@ -71,6 +79,14 @@ template("pw_python_venv") { "_generated_requirements_file", ]) } + _output_logs = true + if (defined(invoker.output_logs)) { + _output_logs = invoker.output_logs + } + if (!defined(invoker.output_logs) || + current_toolchain != pw_build_PYTHON_TOOLCHAIN) { + not_needed([ "_output_logs" ]) + } _source_package_labels = [] foreach(pkg, _source_packages) { @@ -101,6 +117,8 @@ template("pw_python_venv") { path = rebase_path(_path, root_build_dir) generated_requirements = rebase_path(_generated_requirements_file, root_build_dir) + compiled_requirements = + rebase_path(_compiled_requirements_file, root_build_dir) requirements = rebase_path(_requirements, root_build_dir) constraints = rebase_path(_constraints, root_build_dir) interpreter = rebase_path(_python_interpreter, root_build_dir) @@ -115,10 +133,17 @@ template("pw_python_venv") { # outputs and must stamp instead. stamp = true + # The virtualenv should depend on the version of Python currently in use. + stampfile = "$target_gen_dir/$target_name.pw_pystamp" + depfile = "$target_gen_dir/$target_name.d" script = "$dir_pw_build/py/pw_build/create_gn_venv.py" args = [ + "--depfile", + rebase_path(depfile, root_build_dir), "--destination-dir", rebase_path(_path, root_build_dir), + "--stampfile", + rebase_path(stampfile, root_build_dir), ] } @@ -145,9 +170,19 @@ template("pw_python_venv") { python_metadata_deps += _pkg_gn_labels args = [ - "--requirement", + "--gn-root-build-dir", + rebase_path(root_build_dir, root_build_dir), + "--output-requirement-file", rebase_path(_generated_requirements_file, root_build_dir), ] + + if (_constraints != []) { + args += [ "--constraint-files" ] + } + foreach(_constraints_file, _constraints) { + args += [ rebase_path(_constraints_file, root_build_dir) ] + } + args += [ "--gn-packages", string_join(",", _pkg_gn_labels), @@ -161,27 +196,131 @@ template("pw_python_venv") { } } + _pip_generate_hashes = false + if (defined(invoker.pip_generate_hashes)) { + _pip_generate_hashes = invoker.pip_generate_hashes + } else { + not_needed([ "_pip_generate_hashes" ]) + } + if (defined(invoker.source_packages) || defined(invoker.requirements)) { if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) { - # This target will run 'pip install wheel' in the venv. This is purposely - # run before further pip installs so packages that run bdist_wheel as part - # of their install process will succeed. Packages that run native compiles - # typically do this. + # Compile requirements with hashes + pw_python_action("${target_name}._compile_requirements") { + module = "piptools" + + # Set the venv to run this pip install in. + _pw_internal_run_in_venv = true + _skip_installing_external_python_deps = true + venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") + + _pip_args = [] + if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { + _pip_args += [ "--no-index" ] + } + if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { + _pip_args += [ "--no-cache-dir" ] + } + if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { + foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { + _pip_args += + [ "--find-links=" + rebase_path(link_dir, root_build_dir) ] + } + } + + args = [ "compile" ] + + if (_pip_generate_hashes) { + args += [ + "--generate-hashes", + "--reuse-hashes", + ] + } + + args += [ + "--resolver=backtracking", + + # --allow-unsafe will force pinning pip and setuptools. + "--allow-unsafe", + "--output-file", + rebase_path(_compiled_requirements_file, root_build_dir), + + # Input requirements file: + rebase_path(_generated_requirements_file, root_build_dir), + ] + + # Pass offline related pip args through the pip-compile command. + if (_pip_args != []) { + args += [ + "--pip-args", + string_join(" ", _pip_args), + ] + } + + # Extra requirements files + foreach(_requirements_file, _requirements) { + args += [ rebase_path(_requirements_file, root_build_dir) ] + } + + inputs = [] + + # NOTE: constraint files are included in the content of the + # _generated_requirements_file. This occurs in the + # ._generate_3p_requirements target. + inputs += _constraints + inputs += _requirements + inputs += [ _generated_requirements_file ] + + deps = [ + ":${invoker.target_name}._generate_3p_requirements($pw_build_PYTHON_TOOLCHAIN)", + ":${invoker.target_name}._install_base_3p_deps($pw_build_PYTHON_TOOLCHAIN)", + ] + outputs = [ _compiled_requirements_file ] + } + + # This target will run 'pip install' in the venv to provide base + # dependencies needed to run piptools commands. That is required for the + # _compile_requirements sub target. pw_python_action("${target_name}._install_base_3p_deps") { module = "pip" + + # Set the venv to run this pip install in. _pw_internal_run_in_venv = true _skip_installing_external_python_deps = true + venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") + + _base_requirement_file = "$dir_pw_env_setup/py/pw_env_setup/virtualenv_setup/python_base_requirements.txt" + args = [ "install", - "wheel", + "--requirement", + rebase_path(_base_requirement_file, root_build_dir), ] - inputs = _constraints - - foreach(_constraints_file, _constraints) { + if (_output_logs) { + _pip_install_log_file = + "$target_gen_dir/$target_name/pip_install_log.txt" args += [ - "--constraint", - rebase_path(_constraints_file, root_build_dir), + "--log", + rebase_path(_pip_install_log_file, root_build_dir), ] + outputs = [ _pip_install_log_file ] + } + + # NOTE: Constraints should be ignored for this step. + + if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { + args += [ "--no-index" ] + } + if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { + args += [ "--no-cache-dir" ] + } + if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { + foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { + args += [ + "--find-links", + rebase_path(link_dir, root_build_dir), + ] + } } deps = [ ":${invoker.target_name}._create_virtualenv($pw_build_PYTHON_TOOLCHAIN)" ] @@ -192,55 +331,132 @@ template("pw_python_venv") { # Install all 3rd party Python dependencies. pw_python_action("${target_name}._install_3p_deps") { module = "pip" + + # Set the venv to run this pip install in. _pw_internal_run_in_venv = true _skip_installing_external_python_deps = true - args = [ "install" ] + venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") - # Note: --no-build-isolation should be avoided for installing 3rd party - # Python packages that use C/C++ extension modules. - # https://setuptools.pypa.io/en/latest/userguide/ext_modules.html - inputs = _constraints + _requirements + args = pw_build_PYTHON_PIP_DEFAULT_OPTIONS + args += [ + "install", + "--upgrade", + ] - # Constraints - foreach(_constraints_file, _constraints) { + if (_output_logs) { + _pip_install_log_file = + "$target_gen_dir/$target_name/pip_install_log.txt" args += [ - "--constraint", - rebase_path(_constraints_file, root_build_dir), + "--log", + rebase_path(_pip_install_log_file, root_build_dir), ] } - # Extra requirements files - foreach(_requirements_file, _requirements) { - args += [ - "--requirement", - rebase_path(_requirements_file, root_build_dir), - ] + if (_pip_generate_hashes) { + args += [ "--require-hashes" ] + } + + if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { + args += [ "--no-index" ] + } + if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { + args += [ "--no-cache-dir" ] + } + if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { + foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { + args += [ + "--find-links", + rebase_path(link_dir, root_build_dir), + ] + } } - # Generated Python requirements file. + # Note: --no-build-isolation should be avoided for installing 3rd party + # Python packages that use C/C++ extension modules. + # https://setuptools.pypa.io/en/latest/userguide/ext_modules.html + inputs = _constraints + _requirements + [ _compiled_requirements_file ] + + # Use the pip-tools compiled requiremets file. This contains the fully + # expanded list of deps with constraints applied. if (defined(invoker.source_packages)) { - inputs += [ _generated_requirements_file ] + inputs += [ _compiled_requirements_file ] args += [ "--requirement", - rebase_path(_generated_requirements_file, root_build_dir), + rebase_path(_compiled_requirements_file, root_build_dir), ] } deps = [ + ":${invoker.target_name}._compile_requirements($pw_build_PYTHON_TOOLCHAIN)", ":${invoker.target_name}._generate_3p_requirements($pw_build_PYTHON_TOOLCHAIN)", ":${invoker.target_name}._install_base_3p_deps($pw_build_PYTHON_TOOLCHAIN)", ] stamp = true pool = "$dir_pw_build/pool:pip($default_toolchain)" } + + # Target to create a Python package cache for this pw_python_venv. + pw_python_action("${target_name}.vendor_wheels") { + _wheel_output_dir = "$target_gen_dir/$target_name/wheels" + _pip_download_logfile = + "$target_gen_dir/$target_name/pip_download_log.txt" + _pip_wheel_logfile = "$target_gen_dir/$target_name/pip_wheel_log.txt" + metadata = { + pw_python_package_wheels = [ _wheel_output_dir ] + } + + script = "$dir_pw_build/py/pw_build/generate_python_wheel_cache.py" + + # Set the venv to run this pip install in. + _pw_internal_run_in_venv = true + _skip_installing_external_python_deps = true + venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") + + args = [ + "--pip-download-log", + rebase_path(_pip_download_logfile, root_build_dir), + "--wheel-dir", + rebase_path(_wheel_output_dir, root_build_dir), + "-r", + rebase_path(_compiled_requirements_file, root_build_dir), + ] + + if (pw_build_PYTHON_PIP_DOWNLOAD_ALL_PLATFORMS) { + args += [ "--download-all-platforms" ] + } + + deps = [ + ":${invoker.target_name}._compile_requirements($pw_build_PYTHON_TOOLCHAIN)", + ":${invoker.target_name}._generate_3p_requirements($pw_build_PYTHON_TOOLCHAIN)", + ] + + outputs = [ + _wheel_output_dir, + _pip_wheel_logfile, + _pip_download_logfile, + ] + pool = "$dir_pw_build/pool:pip($default_toolchain)" + } + + # End pw_build_PYTHON_TOOLCHAIN check } else { + group("${target_name}._compile_requirements") { + public_deps = [ ":${target_name}($pw_build_PYTHON_TOOLCHAIN)" ] + } group("${target_name}._install_3p_deps") { public_deps = [ ":${target_name}($pw_build_PYTHON_TOOLCHAIN)" ] } + group("${target_name}.vendor_wheels") { + public_deps = [ ":${target_name}($pw_build_PYTHON_TOOLCHAIN)" ] + } } } else { + group("${target_name}._compile_requirements") { + } group("${target_name}._install_3p_deps") { } + group("${target_name}.vendor_wheels") { + } } # Have this target directly depend on _install_3p_deps |