diff --git a/docker/install-mamba.sh b/.github/deprecated/install-mamba.sh similarity index 70% rename from docker/install-mamba.sh rename to .github/deprecated/install-mamba.sh index fc67b5d..e278529 100755 --- a/docker/install-mamba.sh +++ b/.github/deprecated/install-mamba.sh @@ -6,7 +6,11 @@ sudo apt-get update && sudo apt-get install -y curl # Install mamba for different python versions # Python - instead of a system python we install mamba -curl -L https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Mambaforge-24.3.0-0-Linux-x86_64.sh > mambaforge.sh +if [[ "$(uname)" == "Darwin" ]]; then + curl -L https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Darwin-x86_64.sh > mambaforge.sh +else + curl -L https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Mambaforge-24.3.0-0-Linux-x86_64.sh > mambaforge.sh +fi # Does not build 3.8 # curl -L https://github.com/conda-forge/miniforge/releases/download/24.11.3-0/Miniforge3-24.11.3-0-Linux-x86_64.sh > mambaforge.sh @@ -14,7 +18,7 @@ chmod +x ./mambaforge.sh sudo bash mambaforge.sh -b -p /opt/conda rm mambaforge.sh sudo /opt/conda/bin/mamba create --name build --yes python=${PYTHON_VERSION} -sudo /opt/conda/bin/mamba activate build +/opt/conda/bin/mamba init # This is intended to run in the container echo "Building Python version ${version}" diff --git a/.github/deprecated/test-build.yaml b/.github/deprecated/test-build.yaml new file mode 100644 index 0000000..899320d --- /dev/null +++ b/.github/deprecated/test-build.yaml @@ -0,0 +1,67 @@ +name: test-build flux-python + +on: + pull_request: [] + schedule: + - cron: "5 4 * * *" + +jobs: + build-ubuntu: + name: Build on Ubuntu + runs-on: ubuntu-latest + # The container provides a pre-built environment, specific to Linux. + container: + image: fluxrm/testenv:focal + steps: + - uses: actions/checkout@v4 + + - name: Build Flux Core + env: + FLUX_RELEASE_VERSION: "0.78.0" + run: /bin/bash .github/scripts/setup.sh + + - name: Build Python Bindings + run: | + mv /code/src/bindings/python/flux ./flux + python3 setup.py sdist + + - name: Build Python Wheels + run: | + /bin/bash ./docker/install-mamba.sh + /bin/bash ./docker/build-wheels.sh + + build-macos: + name: Build on macOS + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Micromamba + # macOS runners do not use containers, so we must set up the + # Mamba environment from scratch using a standard action. + uses: mamba-org/setup-micromamba@v1 + with: + environment-name: build-env + create-args: >- + python=3.9 + + - name: Build Flux Core + env: + FLUX_RELEASE_VERSION: "0.50.0" + + # Using a login shell to ensure micromamba is activated + shell: bash -l {0} + run: /bin/bash .github/scripts/setup.sh + + - name: Build Python Bindings + shell: bash -l {0} + run: | + mv /code/src/bindings/python/flux ./flux + python3 setup.py sdist + + - name: Build Python Wheels + shell: bash -l {0} + run: | + # The setup-micromamba action handles the installation, + # so we can skip the install-mamba.sh script here. + /bin/bash ./docker/build-wheels.sh diff --git a/.github/scripts/build-bindings.sh b/.github/scripts/build-bindings.sh new file mode 100755 index 0000000..1fcf533 --- /dev/null +++ b/.github/scripts/build-bindings.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +here=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +FLUX_REPO=${1:-0} +FLUX_BRANCH=${2:-3.11} + +echo "Flux Repo: ${FLUX_REPO}" +echo "Flux Branch: ${FLUX_BRANCH}" + +# Use default values if inputs are empty +FLUX_REPO_URL=${FLUX_REPO:-"https://github.com/flux-framework/flux-core"} +FLUX_BRANCH_NAME=${FLUX_BRANCH:-"master"} + +git clone -b ${FLUX_BRANCH_NAME} ${FLUX_REPO_URL} /tmp/flux-core +mv /tmp/flux-core/src/bindings/python/flux ./flux +python setup.py sdist + +# I think ldconfig is Linux-specific +sudo ldconfig || echo "Not running on Linux" diff --git a/.github/scripts/setup.sh b/.github/scripts/setup.sh index 2b778c8..4a55cb5 100755 --- a/.github/scripts/setup.sh +++ b/.github/scripts/setup.sh @@ -1,90 +1,167 @@ #!/bin/bash +# Exit on error, unset variable, or pipe failure set -euo pipefail -# This will be empty for nightly test, and we will clone master branch -FLUX_RELEASE_VERSION=${FLUX_RELEASE_VERSION:-0.68.0} -FLUX_VERSION=${FLUX_VERSION:-0.68.0} +# This must be set +PYTHON_VERSION=${1:-py311} -export DEBIAN_FRONTEND=noninteractive -export TZ=UTC -sudo ln -snf /usr/share/zoneinfo/$TZ /etc/localtime -echo $TZ > /tmp/timezone -sudo mv /tmp/timezone /etc/timezone +# These will be empty for nightly test, and we will clone master branch +FLUX_RELEASE_VERSION=${2:-0.78.0} +FLUX_VERSION=${3:-0.78.0} -# Prepare the version file +if [[ "$(uname)" == "Darwin" ]]; then + brew install zeromq libsodium epoll-shim libev +else + export DEBIAN_FRONTEND=noninteractive + export TZ=UTC + sudo ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ | sudo tee /etc/timezone + + sudo apt-get update + sudo apt-get install -y \ + libjson-glib-dev \ + wget \ + curl \ + automake \ + libsodium-dev \ + libzmq3-dev \ + libczmq-dev \ + libjansson-dev \ + libncursesw5-dev \ + lua5.3 \ + liblua5.3-dev \ + liblz4-dev \ + libsqlite3-dev \ + uuid-dev \ + libhwloc-dev \ + libmpich-dev \ + libs3-dev \ + libevent-dev \ + libarchive-dev \ + libtool \ + git \ + build-essential \ + libpam-dev + + sudo rm -rf /var/lib/apt/lists/* + sudo ldconfig + + # Set the dynamic library path variable for Linux. + export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib + echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}" +fi + +echo "Found Python $(which python3)" +echo "PATH: ${PATH}" + +# Here we can start common setup (we hope) echo "Flux Version for pypi is ${FLUX_VERSION}" -sed -i "s/version = \"0.0.0\"/version = \"$FLUX_VERSION\"/" pyproject.toml +if [[ "$(uname)" == "Darwin" ]]; then + PIXI_ROOT=/Users/runner/work/flux-python/flux-python/.pixi + PIXI_ENV=${PIXI_ROOT}/envs/${PYTHON_VERSION} + sed -i '' "s/version = \"0.0.0\"/version = \"$FLUX_VERSION\"/" pyproject.toml + echo $(find /Users/runner/ -name python3) || echo $(find /Users/runner/ -name python) + + # Tell where to look for lua.h + CPPFLAGS="-I${PIXI_ENV}/include" + + # This makes me want to kill homebrew and MacOS with fire. + # I know most of these aren't required. I can't remove them because of the ❤️‍🔥 + export PYTHON_SITE_PACKAGES=$(find ${PIXI_ENV} -name site-packages) + export PYTHON_EXEC_PREFIX=${PYTHON_SITE_PACKAGES} + export PYTHON_SITE_PKG=${PYTHON_SITE_PACKAGES} + export PYTHON_PLATFORM_SITE_PKG=${PYTHON_SITE_PACKAGES} + export PYTHON_SITELIB=${PYTHON_SITE_PACKAGES} + export PYTHON_PREFIX=${PYTHON_SITE_PACKAGES} + export PKG_CONFIG_PATH=${PIXI_ENV}/lib/pkgconfig + export PYTHONDIR=${PYTHON_SITE_PACKAGES} + + # Set these to empty. + export PYTHON_EXTRA_LDFLAGS=" " + export PYTHON_EXTRA_LIBS=" " + + echo "PKG_CONFIG_PATH: ${PKG_CONFIG_PATH}" + echo "PYTHON_PREFIX: ${PYTHON_PREFIX}" + echo "PYTHON_SITE_PACKAGES: ${PYTHON_SITE_PACKAGES}" + echo "PYTHON_EXEC_PREFIX: ${PYTHON_EXEC_PREFIX}" +else + PIXI_ROOT=/home/runner/work/flux-python/flux-python/.pixi + export PIXI_ENV=${PIXI_ROOT}/envs/${PYTHON_VERSION} + sed -i "s/version = \"0.0.0\"/version = \"$FLUX_VERSION\"/" pyproject.toml + CPPFLAGS="-I${PIXI_ENV}/include" +fi + +# Can we remove need for setuptools? +export PYTHON=${PIXI_ENV}/bin/python3 +export PYTHON_EXECUTABLE=${PIXI_ENV}/bin/python3 +export PYTHON_NOVERSIONCHECK=yes +export LUA_CFLAGS="-I${PIXI_ENV}/include" +export LDFLAGS="-L${PIXI_ENV}/lib -L${PIXI_ENV}/lib64" +echo "Found lua.h in:\n$(find ${PIXI_ROOT} -name lua.h)" || echo "Did not find lua.h" +# Store current directory here=$(pwd) -sudo apt-get update -sudo apt-get install -y \ - libjson-glib-dev \ - wget \ - curl \ - automake \ - libsodium-dev \ - libzmq3-dev \ - libczmq-dev \ - libjansson-dev \ - libmunge-dev \ - libncursesw5-dev \ - lua5.3 \ - liblua5.3-dev \ - liblz4-dev \ - libsqlite3-dev \ - uuid-dev \ - libhwloc-dev \ - libmpich-dev \ - libs3-dev \ - libevent-dev \ - libarchive-dev \ - python3 \ - python3-dev \ - python3-pip \ - python3-sphinx \ - libtool \ - git \ - build-essential \ - libpam-dev - -sudo ldconfig -sudo rm -rf /var/lib/apt/lists/* - -# noble and later requires --break-system-packages -PIP_INSTALL="sudo /opt/conda/envs/build/bin/python3 -m pip install" -${PIP_INSTALL} IPython || ${PIP_INSTALL} IPython --break-system-packages -${PIP_INSTALL} -r .github/scripts/requirements-dev.txt || ${PIP_INSTALL} -r .github/scripts/requirements-dev.txt --break-system-packages -export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/opt/conda/envs/build/lib - -git clone https://github.com/flux-framework/flux-security ~/security -cd ~/security -./autogen.sh -PYTHON=/opt/conda/envs/build/bin/python3 ./configure --prefix=/usr/local -make -sudo make install -sudo ldconfig - -# This is a build from a release +luarocks install luaposix + +# Flux Security -- +# Does not have a variant for Mac +if [[ "$(uname)" != "Darwin" ]]; then + + # Install munge and libmunge + MUNGE_VERSION="0.5.16" + curl -L "https://github.com/dun/munge/releases/download/munge-${MUNGE_VERSION}/munge-${MUNGE_VERSION}.tar.xz" -o munge.tar.xz + tar -xf munge.tar.xz + cd "munge-${MUNGE_VERSION}" + ./configure --prefix=/usr + make + sudo make install + cd .. + + git clone https://github.com/flux-framework/flux-security ~/security + cd ~/security + ./autogen.sh + ./configure --prefix=/usr/local + make + sudo make install + sudo ldconfig +fi + +# Flux Core -- +cd "${here}" echo "Flux Release Version is ${FLUX_RELEASE_VERSION}" -echo "Flux version requested to build is ${FLUX_RELEASE_VERSION}" wget https://github.com/flux-framework/flux-core/releases/download/v${FLUX_RELEASE_VERSION}/flux-core-${FLUX_RELEASE_VERSION}.tar.gz tar -xzvf flux-core-${FLUX_RELEASE_VERSION}.tar.gz -sudo mv flux-core-${FLUX_RELEASE_VERSION} /code - -sudo chown -R $(id -u) /code -cd /code +cd flux-core-${FLUX_RELEASE_VERSION} chmod +x etc/gen-cmdhelp.py -# This is only needed for non-releases ./autogen.sh || echo "No autogen here" -PYTHON=/opt/conda/envs/build/bin/python3 ./configure --prefix=/usr/local -# We don't really care about the version here -just building python bindings -make VERBOSE=1 +if [[ "$(uname)" == "Darwin" ]]; then + + # Mac you are a JERK + ZMQ_PREFIX=$(brew --prefix zeromq) + SODIUM_PREFIX=$(brew --prefix libsodium) + LIBEV_PREFIX=$(brew --prefix libev) + EPOLL_SHIM_PREFIX=$(brew --prefix epoll-shim) + + CPPFLAGS="-I'"$EPOLL_SHIM_PREFIX"'/include -I'"$LIBEV_PREFIX"'/include -I'"$ZMQ_PREFIX"'/include -I'"$SODIUM_PREFIX"'/include -I$PIXI_ENV/include" + CPPFLAGS="-I$(brew --prefix libev)/include ${CPPFLAGS}" + export CPPFLAGS="-I$(brew --prefix epoll-shim)/include/libepoll-shim ${CPPFLAGS}" + export LDFLAGS="$LDFLAGS -L'"$EPOLL_SHIM_PREFIX"'/lib -L'"$LIBEV_PREFIX"'/lib -L'"$SODIUM_PREFIX"'/lib -L'"$ZMQ_PREFIX"'/lib -Wl,-rpath,'"$SODIUM_PREFIX"'/lib -Wl,-rpath,$PIXI_ENV/lib" + CPPFLAGS="$CPPFLAGS" CFLAGS="${CPPFLAGS}" LDFLAGS="${LDFLAGS}" PKG_CONFIG_PATH=$PKG_CONFIG_PATH ./configure --prefix=/usr/local --with-external-libev || (cp ./config.log ../ && exit 1) + + # Surgery. This is the biggest hairball ever - you can't build libsodium / zeromq from source because it doesn't detect curve+libsodium. + # You also can't use easily from Homebrew without this hack. I can't even believe I figured this out! + find . -name "Makefile" -exec sed -i.bak "s/^\(LIBS = .*\)$/\1 -lepoll-shim/" {} + + make -j || (cp ./config.log ../ && exit 1) + sudo make install || (cp ./config.log ../ && exit 1) +else + ./configure CPPFLAGS="$CPPFLAGS" CFLAGS="${CPPFLAGS}" LDFLAGS="${LDFLAGS}" --prefix=/usr/local + cp ./config.log ../ + make VERBOSE=1 + sudo make install || true + sudo make install + sudo ldconfig +fi -# This sometimes fails the first time but then works ok? Weird -sudo make install || true -sudo make install -sudo ldconfig -cd ${here} +cd "${here}" diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index b6d2d2d..dcf50d3 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -1,4 +1,4 @@ -name: build flux-python +name: flux-python on: pull_request: {} @@ -8,7 +8,7 @@ on: description: 'Pypi version' release_version: description: 'Version of flux to build' - default: "0.68.0" + default: "0.78.0" rc: description: 'Release candidate to build (for wheel)' default: "0" @@ -20,90 +20,73 @@ on: default: "https://github.com/flux-framework/flux-core" jobs: - build-manual: - runs-on: ubuntu-latest + build: + name: ${{ matrix.os }} Python ${{ matrix.python }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python: ["3.8", "3.9", "3.10", "3.11"] - container: - image: fluxrm/testenv:jammy + # The matrix now includes both os and python + os: [ubuntu-latest, macos-latest] + python: ["py38", "py39", "py310", "py311", "py312"] + steps: - uses: actions/checkout@v4 - - name: Setup Python - env: - python_version: ${{ matrix.python }} - # Installs to /opt/conda/envs/build/bin/python3 - run: /bin/bash ./docker/install-mamba.sh ${python_version} + - uses: prefix-dev/setup-pixi@v0.9.0 + with: + environments: ${{ matrix.python }} - name: Build Flux Core Branch - env: - FLUX_RELEASE_VERSION: ${{ inputs.release_version }} - FLUX_VERSION: ${{ inputs.version }} - run: /bin/bash .github/scripts/setup.sh + run: pixi run -e ${{ matrix.python }} bash .github/scripts/setup.sh ${{ matrix.python }} ${{ inputs.release_version }} ${{ inputs.version }} + + - name: Upload logs on failure + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: config-log-${{ matrix.python }}-${{ matrix.os }} + path: ./config.log - name: Build Python Bindings - env: - FLUX_BRANCH: ${{ inputs.branch }} - FLUX_REPO: ${{ inputs.repo }} - shell: bash - run: | - echo "Flux Repo: ${FLUX_REPO}" - echo "Flux Branch: ${FLUX_BRANCH}" - if [[ "${FLUX_REPO}" == "" ]]; then - FLUX_REPO="https://github.com/flux-framework/flux-core" - fi - if [[ "${FLUX_BRANCH}" == "" ]]; then - FLUX_BRANCH="master" - fi - git clone -b ${FLUX_BRANCH} ${FLUX_REPO} /tmp/flux-core - mv /tmp/flux-core/src/bindings/python/flux ./flux - /opt/conda/envs/build/bin/python3 setup.py sdist - sudo ldconfig + run: pixi run -e ${{ matrix.python }} bash .github/scripts/build-bindings.sh ${{ inputs.branch }} ${{ inputs.repo }} - name: Build Python Wheels - env: - build_number: ${{ inputs.rc }} - python_version: ${{ matrix.python }} - run: | - /bin/bash ./docker/build-wheels.sh ${build_number} ${python_version} - ls ./dist + run: pixi run -e ${{ matrix.python }} bash ./docker/build-wheels.sh ${{ env.build_number }} ${{ env.python_version }} - name: Upload distributions - if: success() uses: actions/upload-artifact@v4 with: - name: ${{ matrix.python }} + # The artifact name must be unique for each job in the matrix + name: dist-${{ matrix.python }}-${{ matrix.os }} path: ./dist upload: runs-on: ubuntu-latest - needs: [build-manual] + # This job now needs the entire build matrix to complete successfully + needs: [build] steps: - name: Download Artifacts uses: actions/download-artifact@v4 - # If name unspecified, all artifacts download for run + with: + # Download all artifacts created by the build jobs + path: artifacts - - name: Show Files + - name: Show and Organize Files run: | - ls */ - # We just need one of these - first_dir=$(ls . | head -n 1) + ls -R artifacts mkdir -p ./dist - mv */*.whl ./dist/ - cp ${first_dir}/*.tar.gz ./dist/ + # This command finds all wheel and tar.gz files in all subdirectories + # and moves them into a single ./dist directory for upload. + find artifacts -type f \( -name "*.whl" -o -name "*.tar.gz" \) -exec mv {} ./dist/ \; echo echo "Files to distribute:" ls ./dist - name: Build and publish + # Ensure this only runs on pushes to a main branch, not PRs if: (github.event_name != 'pull_request') env: TWINE_USERNAME: ${{ secrets.PYPI_USER }} TWINE_PASSWORD: ${{ secrets.PYPI_PASS }} run: | - ls dist/ - python3 -m pip install setuptools wheel - python3 -m pip install twine==6.0.1 - python3 -m pip install --upgrade pkginfo - twine upload --skip-existing dist/flux_python* + python3 -m pip install --upgrade twine + twine upload --skip-existing dist/* diff --git a/.github/workflows/test-build.yaml b/.github/workflows/test-build.yaml deleted file mode 100644 index 19fc480..0000000 --- a/.github/workflows/test-build.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: test-build flux-python - -on: - pull_request: [] - schedule: - - cron: "5 4 * * *" - -jobs: - build: - runs-on: ubuntu-latest - container: - image: fluxrm/testenv:focal - steps: - - uses: actions/checkout@v3 - - - name: Build Flux Core - env: - FLUX_RELEASE_VERSION: "0.50.0" - run: /bin/bash .github/scripts/setup.sh - - - name: Build Python Bindings - run: | - mv /code/src/bindings/python/flux ./flux - python3 setup.py sdist - - - name: Build Python Wheels - run: | - /bin/bash ./docker/install-mamba.sh - /bin/bash ./docker/build-wheels.sh \ No newline at end of file diff --git a/README.md b/README.md index 17f1874..bc3ec5f 100644 --- a/README.md +++ b/README.md @@ -121,12 +121,9 @@ $ python3 setup.py sdist $ python3 setup.py sdist bdist_wheel ``` -You can build versions of the Python wheels across 3.6 to 3.10 like: +Given an environment with dependencies, you can build versions of the Python wheels across 3.6 to 3.12 like: ```bash -# only need to run this once in the container -/bin/bash ./docker/install-mamba.sh - # And this to install the current Flux + version in setup.py as a wheel # The number is the build number /bin/bash ./docker/build-wheels.sh 2 diff --git a/docker/build-wheels.sh b/docker/build-wheels.sh index 6de1336..7d504c8 100755 --- a/docker/build-wheels.sh +++ b/docker/build-wheels.sh @@ -7,12 +7,7 @@ version=${2:-3.11} # This is intended to run in the container echo "Building Python version ${version}" -export PATH=/opt/conda/envs/build/bin:$PATH -export PYTHONPATH=/opt/conda/envs/build/lib/python${version}/site-packages - -/opt/conda/bin/mamba activate build || true - # Build the bindings for this python version! -/opt/conda/envs/build/bin/python3 setup.py sdist -/opt/conda/envs/build/bin/python3 setup.py bdist_wheel --plat-name=any --build-number=${build_number} -unset PYTHONPATH +python3 setup.py sdist +python3 setup.py bdist_wheel --plat-name=any --build-number=${build_number} +ls ./dist diff --git a/pixi.toml b/pixi.toml new file mode 100644 index 0000000..ae3e21e --- /dev/null +++ b/pixi.toml @@ -0,0 +1,60 @@ +[workspace] +name = "flux-python" +channels = ["conda-forge"] +platforms = ["linux-64", "osx-arm64"] + +[dependencies] +python = ">=3.8" +pip = "*" + +[feature.py38.dependencies] +python = "3.8.*" +[feature.py39.dependencies] +python = "3.9.*" +[feature.py310.dependencies] +python = "3.10.*" +[feature.py311.dependencies] +python = "3.11.*" +[feature.py312.dependencies] +python = "3.12.*" + +[feature.common.dependencies] +cffi = "*" +pyyaml = "*" +jsonschema = "*" +docutils = "*" +ply = "*" +black = "*" +ipython = "*" +wheel = "*" +setuptools = "*" +luarocks = "*" +lua = "*" +lua-lpeg = "*" +libev = "*" +sphinx = "*" +libarchive = "*" +autoconf = "*" +automake = "*" +libtool = "*" +make = "*" +pkg-config = "*" +zeromq = "*" +jansson = "*" +libhwloc = "*" +libcxx = "*" +libsodium = "*" +argp-standalone = "*" +libuuid = "*" +libxml2 = "*" +libxml2-devel = "*" + +[pypi-dependencies] +setuptools = "*" + +[environments] +py38 = { features = ["py38", "common"] } +py39 = { features = ["py39", "common"] } +py310 = { features = ["py310", "common"] } +py311 = { features = ["py311", "common"] } +py312 = { features = ["py312", "common"] }