From ee09c40a26c46eac7c72d07016de358c4ab1fe72 Mon Sep 17 00:00:00 2001 From: madisetti Date: Mon, 5 Apr 2021 12:32:51 -0700 Subject: [PATCH 1/6] Allow hook for setting library --- WORKSPACE | 1 + python_configure.bzl | 115 ++++++++++++++++++++++++++++--------------- 2 files changed, 77 insertions(+), 39 deletions(-) create mode 100644 WORKSPACE diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000..9bfe9f1 --- /dev/null +++ b/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "pybind11_bazel") diff --git a/python_configure.bzl b/python_configure.bzl index 1f5bffa..6aa7015 100644 --- a/python_configure.bzl +++ b/python_configure.bzl @@ -2,8 +2,11 @@ `python_configure` depends on the following environment variables: - * `PYTHON_BIN_PATH`: location of python binary. + * `PYTHON_BIN_PATH`: Location of python binary. * `PYTHON_LIB_PATH`: Location of python libraries. + +These can be directly set with the `python_interpreter_target` and +`python_library_target` respectively. """ _BAZEL_SH = "BAZEL_SH" @@ -11,6 +14,45 @@ _PYTHON_BIN_PATH = "PYTHON_BIN_PATH" _PYTHON_CONFIG_BIN_PATH = "PYTHON_CONFIG_BIN_PATH" _PYTHON_LIB_PATH = "PYTHON_LIB_PATH" +# TODO: Move scripts to respective files and expose to rule. +_INCLUDE_SCRIPT = """ +from __future__ import print_function +from distutils import sysconfig +import shutil + +python_inc_dir = sysconfig.get_python_inc() +config_h_path = sysconfig.get_config_h_filename() +shutil.copyfile(config_h_path, python_inc_dir + "/pyconfig.h") +print(python_inc_dir)""" + +_LIBRARY_SCRIPT = """ +from __future__ import print_function +import site +import os + +try: + input = raw_input +except NameError: + pass + +python_paths = [] +if os.getenv('PYTHONPATH') is not None: + python_paths = os.getenv('PYTHONPATH').split(':') +try: + library_paths = site.getsitepackages() +except AttributeError: + from distutils.sysconfig import get_python_lib + library_paths = [get_python_lib()] + +all_paths = set(python_paths + library_paths) +paths = [] +for path in all_paths: + if os.path.isdir(path): + paths.append(path) +if len(paths) >=1: + print(paths[0]) +""" + def _tpl(repository_ctx, tpl, substitutions = {}, out = None): if not out: out = tpl @@ -96,7 +138,8 @@ def _genrule(src_dir, genrule_name, command, outs, local): "genrule(\n" + ' name = "' + genrule_name + '",\n' + ( - " local = 1,\n" if local else "") + + " local = 1,\n" if local else "" + ) + " outs = [\n" + outs + "\n ],\n" + @@ -194,37 +237,26 @@ def _get_bash_bin(repository_ctx): def _get_python_lib(repository_ctx, python_bin): """Gets the python lib path.""" - python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) - if python_lib != None: - return python_lib - print_lib = ("<=1:\n" + - " print(paths[0])\n" + - "END") - cmd = "%s - %s" % (python_bin, print_lib) - result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + if repository_ctx.attr.python_library_target != None: + return str(repository_ctx.path(repository_ctx.attr.python_library_target)) + + # If an interpreter is explicitly passed down, we should should not regard + # the environmental variable. + if repository_ctx.attr.python_interpreter_target == None: + python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) + if python_lib: + return python_lib + + result = _execute( + repository_ctx, + [ + python_bin, + "-c", + _LIBRARY_SCRIPT, + ], + error_msg = "Unable to detect library path.", + error_details = ("Is Python installed correctly?"), + ) return result.stdout.strip("\n") def _check_python_lib(repository_ctx, python_lib): @@ -251,9 +283,7 @@ def _get_python_include(repository_ctx, python_bin): [ python_bin, "-c", - "from __future__ import print_function;" + - "from distutils import sysconfig;" + - "print(sysconfig.get_python_inc())", + _INCLUDE_SCRIPT, ], error_msg = "Problem getting python include path.", error_details = ("Is the Python binary path set up right? " + @@ -314,6 +344,7 @@ def _get_embed_flags(repository_ctx, python_config): # See https://github.com/python/cpython/pull/13500 link_cmd = python_config + " --ldflags --embed" + # TODO: Resolve ctx.execute vs _execute. err = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", err_cmd]).stdout.strip("\n") compiler_flags = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", comp_cmd]).stdout.strip("\n") linker_flags = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", link_cmd]).stdout.strip("\n") @@ -395,8 +426,9 @@ python_configure = repository_rule( _PYTHON_LIB_PATH, ], attrs = { - "python_version": attr.string(default=""), + "python_version": attr.string(default = ""), "python_interpreter_target": attr.label(), + "python_library_target": attr.string(), }, ) """Detects and configures the local Python. @@ -416,6 +448,11 @@ Args: By default, will build for whatever Python version is returned by `which python`. python_interpreter_target: If set to a target providing a Python binary, - will configure for that Python version instead of the installed - binary. This configuration takes precedence over python_version. + this will configure for that Python version instead of the installed + binary. This configuration takes precedence over python_version. In + addition, setting this label will ignore `PYTHON_LIB_PATH`- if needed + use `python_library_target` to explicitly set the desired library. + python_library_target: If this string is set, this will override the + environmental variable `PYTHON_LIB_PATH`. Otherwise implicit discovery, + is used from the availible binary. """ From 992381ced716ae12122360b0fbadbc3dda436dbf Mon Sep 17 00:00:00 2001 From: yuvalk Date: Sat, 18 Sep 2021 00:40:58 +0300 Subject: [PATCH 2/6] Add support for hermetic Python (#29) --- README.md | 12 ++++++++++++ python_configure.bzl | 13 +++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9ee536a..ad1a86c 100644 --- a/README.md +++ b/README.md @@ -57,3 +57,15 @@ Then, in your `BUILD` file: ```starlark load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") ``` + +## Hermetic Python + +To configure `pybind11_bazel` for hermetic Python, `python_configure` can take +the target providing the Python runtime as an argument: + +```starlark +python_configure( + name = "local_config_python", + python_interpreter_target = "@python_interpreter//:python_bin", +) +``` diff --git a/python_configure.bzl b/python_configure.bzl index ade190a..1f5bffa 100644 --- a/python_configure.bzl +++ b/python_configure.bzl @@ -87,7 +87,7 @@ def _read_dir(repository_ctx, src_dir): result = find_result.stdout return result -def _genrule(src_dir, genrule_name, command, outs): +def _genrule(src_dir, genrule_name, command, outs, local): """Returns a string with a genrule. Genrule executes the given command and produces the given outputs. @@ -95,7 +95,8 @@ def _genrule(src_dir, genrule_name, command, outs): return ( "genrule(\n" + ' name = "' + - genrule_name + '",\n' + + genrule_name + '",\n' + ( + " local = 1,\n" if local else "") + " outs = [\n" + outs + "\n ],\n" + @@ -149,11 +150,15 @@ def _symlink_genrule_for_dir( genrule_name, " && ".join(command), "\n".join(outs), + local = True, ) return genrule def _get_python_bin(repository_ctx): """Gets the python bin path.""" + if repository_ctx.attr.python_interpreter_target != None: + return str(repository_ctx.path(repository_ctx.attr.python_interpreter_target)) + python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) if python_bin != None: return python_bin @@ -391,6 +396,7 @@ python_configure = repository_rule( ], attrs = { "python_version": attr.string(default=""), + "python_interpreter_target": attr.label(), }, ) """Detects and configures the local Python. @@ -409,4 +415,7 @@ Args: If set to "2", will build for Python 2 (`which python2`). By default, will build for whatever Python version is returned by `which python`. + python_interpreter_target: If set to a target providing a Python binary, + will configure for that Python version instead of the installed + binary. This configuration takes precedence over python_version. """ From 72cbbf1fbc830e487e3012862b7b720001b70672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Virgile=20H=C3=B6gman?= Date: Sat, 4 Dec 2021 02:09:09 +0100 Subject: [PATCH 3/6] Remove 'local' tag from pybind_extension (#35) * Remove 'local' tag from pybind_extension See #34 for more details. Remote cache in CI works as a charm. --- build_defs.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_defs.bzl b/build_defs.bzl index 4b99f83..1e55548 100644 --- a/build_defs.bzl +++ b/build_defs.bzl @@ -45,7 +45,7 @@ def pybind_extension( "//conditions:default": ["-Wl,-Bsymbolic"], }), linkshared = 1, - tags = tags + ["local"], + tags = tags, deps = deps + PYBIND_DEPS, **kwargs ) From 829d3b8475bca1162765f22f8fe37fd77e7b04fb Mon Sep 17 00:00:00 2001 From: dylan madisetti Date: Tue, 12 Apr 2022 22:35:24 -0400 Subject: [PATCH 4/6] shutil causes breakage on some systems --- python_configure.bzl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python_configure.bzl b/python_configure.bzl index 6aa7015..af79e24 100644 --- a/python_configure.bzl +++ b/python_configure.bzl @@ -18,12 +18,8 @@ _PYTHON_LIB_PATH = "PYTHON_LIB_PATH" _INCLUDE_SCRIPT = """ from __future__ import print_function from distutils import sysconfig -import shutil -python_inc_dir = sysconfig.get_python_inc() -config_h_path = sysconfig.get_config_h_filename() -shutil.copyfile(config_h_path, python_inc_dir + "/pyconfig.h") -print(python_inc_dir)""" +print(sysconfig.get_python_inc())""" _LIBRARY_SCRIPT = """ from __future__ import print_function From f53e30a7b4677ed9fad90a427f3fab14482cf491 Mon Sep 17 00:00:00 2001 From: dylan madisetti Date: Wed, 4 May 2022 03:03:17 +0000 Subject: [PATCH 5/6] Library context can be None or empty string/ Fix accordingly --- python_configure.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_configure.bzl b/python_configure.bzl index af79e24..33a9008 100644 --- a/python_configure.bzl +++ b/python_configure.bzl @@ -233,7 +233,7 @@ def _get_bash_bin(repository_ctx): def _get_python_lib(repository_ctx, python_bin): """Gets the python lib path.""" - if repository_ctx.attr.python_library_target != None: + if repository_ctx.attr.python_library_target: return str(repository_ctx.path(repository_ctx.attr.python_library_target)) # If an interpreter is explicitly passed down, we should should not regard From 5bef7af844ef372ddaf02eaeba497db86e44976b Mon Sep 17 00:00:00 2001 From: dylan madisetti Date: Tue, 2 Aug 2022 14:42:32 -0400 Subject: [PATCH 6/6] Address pybind/pybind11_bazel/37 --- python_configure.bzl | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/python_configure.bzl b/python_configure.bzl index 33a9008..46cb168 100644 --- a/python_configure.bzl +++ b/python_configure.bzl @@ -17,9 +17,8 @@ _PYTHON_LIB_PATH = "PYTHON_LIB_PATH" # TODO: Move scripts to respective files and expose to rule. _INCLUDE_SCRIPT = """ from __future__ import print_function -from distutils import sysconfig - -print(sysconfig.get_python_inc())""" +from sysconfig import get_path +print(get_path("include"))""" _LIBRARY_SCRIPT = """ from __future__ import print_function @@ -37,8 +36,8 @@ if os.getenv('PYTHONPATH') is not None: try: library_paths = site.getsitepackages() except AttributeError: - from distutils.sysconfig import get_python_lib - library_paths = [get_python_lib()] + from sysconfig import get_path + library_paths = [get_path("purelib")] all_paths = set(python_paths + library_paths) paths = [] @@ -283,8 +282,7 @@ def _get_python_include(repository_ctx, python_bin): ], error_msg = "Problem getting python include path.", error_details = ("Is the Python binary path set up right? " + - "(See ./configure or " + _PYTHON_BIN_PATH + ".) " + - "Is distutils installed?"), + "(See ./configure or " + _PYTHON_BIN_PATH + ".)"), ) return result.stdout.splitlines()[0]