fix: improve Python executable detection and error handling in CMake

.github/workflows/build-wheel-310.yml

@@ -16,6 +16,7 @@ jobs:
         options: --privileged
     env:
       CIBW_BUILD: cp310-manylinux_x86_64
+      CIBW_BUILD_FRONTEND: "pip; args: --no-build-isolation"
       CIBW_BEFORE_ALL: >
         yum install -y openblas-devel gcc-toolset-12 &&
         source /opt/rh/gcc-toolset-12/enable

.github/workflows/build-wheel-311.yml

@@ -16,10 +16,23 @@ jobs:
         options: --privileged
     env:
       CIBW_BUILD: cp311-manylinux_x86_64
+      CIBW_BUILD_FRONTEND: "pip; args: --no-build-isolation"
       CIBW_BEFORE_ALL: >
         yum install -y openblas-devel
-      CIBW_BEFORE_BUILD: >
-        pip install -e .
+      CIBW_BEFORE_BUILD: |
+        pip install \
+          "cmake>=4.2.1" \
+          "scikit-build>=0.18.1" \
+          scikit-learn \
+          "pybind11>=3.0.1" \
+          "torch>=2.9.1" \
+          torchvision \
+          scipy \
+          "requests>=2.32,<3" \
+          numpy \
+          protobuf \
+          tqdm \
+          mypy
       CIBW_MANYLINUX_X86_64_IMAGE: "quay.io/pypa/manylinux_2_28_x86_64"
       CIBW_REPAIR_WHEEL_COMMAND: >
         auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so

.github/workflows/build-wheel-312.yml

@@ -16,10 +16,23 @@ jobs:
         options: --privileged
     env:
       CIBW_BUILD: cp312-manylinux_x86_64
+      CIBW_BUILD_FRONTEND: "pip; args: --no-build-isolation"
       CIBW_BEFORE_ALL: >
         yum install -y openblas-devel
-      CIBW_BEFORE_BUILD: >
-        pip install -e .
+      CIBW_BEFORE_BUILD: |
+        pip install \
+          "cmake>=4.2.1" \
+          "scikit-build>=0.18.1" \
+          scikit-learn \
+          "pybind11>=3.0.1" \
+          "torch>=2.9.1" \
+          torchvision \
+          scipy \
+          "requests>=2.32,<3" \
+          numpy \
+          protobuf \
+          tqdm \
+          mypy
       CIBW_MANYLINUX_X86_64_IMAGE: "quay.io/pypa/manylinux_2_28_x86_64"
       CIBW_REPAIR_WHEEL_COMMAND: >
         auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so

cmake/dependencies.cmake

@@ -81,12 +81,26 @@ find_package(Python3 COMPONENTS Interpreter Development.Module)
 include_directories(${Python3_INCLUDE_DIRS})  # order matters (before pybind)
 
 set(ignoreMe "${Python3_EXECUTABLE}${Python3_FIND_REGISTRY}${Python3_INCLUDE_DIR}${Python3_NumPy_INCLUDE_DIRS}${Python3_ROOT_DIR}${Python_INCLUDE_DIR}${Python_NumPy_INCLUDE_DIRS}")
+set(RPU_PYTHON_EXECUTABLE "${Python3_EXECUTABLE}")
+
+if(NOT RPU_PYTHON_EXECUTABLE AND DEFINED PYTHON_EXECUTABLE AND NOT "${PYTHON_EXECUTABLE}" STREQUAL "")
+  # Keep compatibility with legacy FindPython variables if provided by callers.
+  set(RPU_PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}")
+endif()
+
+if(NOT RPU_PYTHON_EXECUTABLE)
+  message(FATAL_ERROR "Could not determine a Python executable for CMake helper commands.")
+endif()
 
 # Find pybind11Config.cmake
-execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import pybind11; print(pybind11.get_cmake_dir())"
+execute_process(COMMAND "${RPU_PYTHON_EXECUTABLE}" -c "import pybind11; print(pybind11.get_cmake_dir())"
+    RESULT_VARIABLE PYBIND11_CMAKE_DIR_RESULT
     OUTPUT_VARIABLE CUSTOM_PYTHON_PYBIND11_PATH
     OUTPUT_STRIP_TRAILING_WHITESPACE
-    ERROR_QUIET)
+    ERROR_VARIABLE PYBIND11_CMAKE_DIR_ERROR)
+if(NOT PYBIND11_CMAKE_DIR_RESULT EQUAL 0 OR CUSTOM_PYTHON_PYBIND11_PATH STREQUAL "")
+  message(FATAL_ERROR "Failed to query pybind11 CMake dir with ${RPU_PYTHON_EXECUTABLE}: ${PYBIND11_CMAKE_DIR_ERROR}")
+endif()
 set(pybind11_DIR ${CUSTOM_PYTHON_PYBIND11_PATH})
 
 find_package(pybind11 CONFIG REQUIRED)
@@ -98,13 +112,33 @@ include_directories(${TORCH_INCLUDE_DIRS})
 link_directories(${TORCH_LIB_DIR})
 
 if (CMAKE_COMPILER_IS_GNUCXX)
-  # check for pytorch's ABI
-  execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import torch; print('1' if torch._C._GLIBCXX_USE_CXX11_ABI else '0')"
-    OUTPUT_VARIABLE OUTPUT_GNU_ABI
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-    ERROR_QUIET)
-  add_compile_definitions("_GLIBCXX_USE_CXX11_ABI=${OUTPUT_GNU_ABI}")
-  message(STATUS "Set _GLIBCXX_USE_CXX11_ABI=${OUTPUT_GNU_ABI}")
+  # Prefer ABI from Torch CMake config (works even when Python build isolation
+  # makes `import torch` unavailable in helper subprocesses).
+  set(OUTPUT_GNU_ABI "")
+  if(DEFINED TORCH_CXX_FLAGS AND NOT "${TORCH_CXX_FLAGS}" STREQUAL "")
+    string(REGEX MATCH "-D_GLIBCXX_USE_CXX11_ABI=[01]" TORCH_ABI_DEFINE "${TORCH_CXX_FLAGS}")
+    if(NOT TORCH_ABI_DEFINE STREQUAL "")
+      string(REGEX REPLACE ".*=([01]).*" "\\1" OUTPUT_GNU_ABI "${TORCH_ABI_DEFINE}")
+    endif()
+  endif()
+
+  if(NOT OUTPUT_GNU_ABI MATCHES "^[01]$")
+    # Fallback: query torch Python runtime if ABI is not present in TORCH_CXX_FLAGS.
+    execute_process(COMMAND "${RPU_PYTHON_EXECUTABLE}" -c "import torch; print('1' if torch._C._GLIBCXX_USE_CXX11_ABI else '0')"
+      RESULT_VARIABLE TORCH_GNU_ABI_RESULT
+      OUTPUT_VARIABLE OUTPUT_GNU_ABI
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      ERROR_VARIABLE TORCH_GNU_ABI_ERROR)
+    if(NOT TORCH_GNU_ABI_RESULT EQUAL 0)
+      message(FATAL_ERROR "Failed to determine torch CXX11 ABI. TORCH_CXX_FLAGS='${TORCH_CXX_FLAGS}'. Python probe error with ${RPU_PYTHON_EXECUTABLE}: ${TORCH_GNU_ABI_ERROR}")
+    endif()
+  endif()
+
+  if(NOT OUTPUT_GNU_ABI MATCHES "^[01]$")
+    message(FATAL_ERROR "Invalid torch CXX11 ABI value: '${OUTPUT_GNU_ABI}'. Expected 0 or 1.")
+  endif()
+  add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=${OUTPUT_GNU_ABI})
+  message(STATUS "Set _GLIBCXX_USE_CXX11_ABI=${OUTPUT_GNU_ABI} (TORCH_CXX_FLAGS='${TORCH_CXX_FLAGS}')")
 endif()
 
 # Set compile definitions

pyproject.toml

@@ -1,5 +1,11 @@
 [build-system]
-requires = ["setuptools", "wheel", "scikit-build >= 0.18.1", "pybind11 >= 3.0.1", "ninja"]
+requires = [
+  "setuptools",
+  "wheel",
+  "scikit-build >= 0.18.1",
+  "pybind11 >= 3.0.1",
+  "ninja"
+]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -10,7 +16,7 @@ dependencies = [
   "scikit-build >= 0.18.1",
   "scikit-learn",
   "pybind11 >= 3.0.1",
-  "torch == 2.9.1",
+  "torch >= 2.9.1",
   "torchvision",
   "scipy",
   "requests >= 2.32, <3",